codexLB-up 1.25.0b10__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- app/__init__.py +10 -0
- app/admin_cli.py +494 -0
- app/cli.py +339 -0
- app/codex_sessions_retag.py +487 -0
- app/core/__init__.py +0 -0
- app/core/audit/__init__.py +0 -0
- app/core/audit/service.py +216 -0
- app/core/audit/types.py +94 -0
- app/core/auth/__init__.py +236 -0
- app/core/auth/api_key_cache.py +65 -0
- app/core/auth/dashboard_access.py +429 -0
- app/core/auth/dashboard_mode.py +171 -0
- app/core/auth/dashboard_session_ttl.py +74 -0
- app/core/auth/dashboard_users_cache.py +120 -0
- app/core/auth/dependencies.py +710 -0
- app/core/auth/external_identity.py +78 -0
- app/core/auth/guardian.py +374 -0
- app/core/auth/models.py +49 -0
- app/core/auth/providers/__init__.py +129 -0
- app/core/auth/providers/oidc.py +838 -0
- app/core/auth/providers/registry.py +118 -0
- app/core/auth/refresh.py +394 -0
- app/core/auth/step_up.py +117 -0
- app/core/auth/totp.py +81 -0
- app/core/balancer/__init__.py +71 -0
- app/core/balancer/logic.py +1612 -0
- app/core/balancer/rendezvous_hash.py +21 -0
- app/core/balancer/types.py +21 -0
- app/core/bootstrap.py +172 -0
- app/core/cache/invalidation.py +426 -0
- app/core/clients/__init__.py +0 -0
- app/core/clients/codex.py +996 -0
- app/core/clients/codex_version.py +187 -0
- app/core/clients/codex_version_snapshot.py +2 -0
- app/core/clients/files.py +534 -0
- app/core/clients/headers.py +29 -0
- app/core/clients/http.py +511 -0
- app/core/clients/model_fetcher.py +276 -0
- app/core/clients/native_egress.py +1304 -0
- app/core/clients/oauth.py +477 -0
- app/core/clients/proxy.py +6089 -0
- app/core/clients/proxy_websocket.py +1411 -0
- app/core/clients/rate_limit_reset_credits.py +412 -0
- app/core/clients/stream_errors.py +12 -0
- app/core/clients/thread_cache_identity.py +347 -0
- app/core/clients/usage.py +467 -0
- app/core/clock.py +170 -0
- app/core/config/__init__.py +0 -0
- app/core/config/background_jobs.py +73 -0
- app/core/config/context_window_overrides.py +166 -0
- app/core/config/dashboard_overrides.py +145 -0
- app/core/config/inheritable.py +47 -0
- app/core/config/key_fingerprint.py +115 -0
- app/core/config/settings.py +872 -0
- app/core/config/settings_cache.py +100 -0
- app/core/config/spool_retention.py +217 -0
- app/core/config/tiers.py +174 -0
- app/core/conversation_archive.py +568 -0
- app/core/crypto.py +37 -0
- app/core/draining.py +11 -0
- app/core/errors.py +365 -0
- app/core/exceptions.py +155 -0
- app/core/handlers/__init__.py +3 -0
- app/core/handlers/exceptions.py +478 -0
- app/core/http_protocol.py +176 -0
- app/core/http_protocol_httptools.py +204 -0
- app/core/ingress_limits.py +17 -0
- app/core/metadata_store.py +18 -0
- app/core/metrics/__init__.py +0 -0
- app/core/metrics/middleware.py +74 -0
- app/core/metrics/prometheus.py +604 -0
- app/core/middleware/__init__.py +25 -0
- app/core/middleware/api_firewall.py +111 -0
- app/core/middleware/app_version.py +37 -0
- app/core/middleware/dashboard_auth_proxy.py +55 -0
- app/core/middleware/dashboard_csrf.py +177 -0
- app/core/middleware/dashboard_gzip.py +41 -0
- app/core/middleware/dashboard_overrides.py +59 -0
- app/core/middleware/firewall_cache.py +91 -0
- app/core/middleware/inflight.py +126 -0
- app/core/middleware/multipart_content_encoding.py +129 -0
- app/core/middleware/path_rewrite.py +148 -0
- app/core/middleware/request_body_limit.py +196 -0
- app/core/middleware/request_decompression.py +209 -0
- app/core/middleware/request_id.py +72 -0
- app/core/middleware/required_capability_http.py +116 -0
- app/core/middleware/trusted_proxy_headers.py +36 -0
- app/core/multipart.py +237 -0
- app/core/multipart_fields.py +91 -0
- app/core/openai/__init__.py +0 -0
- app/core/openai/chat_requests.py +493 -0
- app/core/openai/chat_responses.py +931 -0
- app/core/openai/contracts.py +46 -0
- app/core/openai/exceptions.py +24 -0
- app/core/openai/host_models.py +14 -0
- app/core/openai/images.py +413 -0
- app/core/openai/message_coercion.py +376 -0
- app/core/openai/model_refresh_scheduler.py +423 -0
- app/core/openai/model_registry.py +997 -0
- app/core/openai/model_registry_store.py +548 -0
- app/core/openai/models.py +172 -0
- app/core/openai/parsing.py +97 -0
- app/core/openai/requests.py +1980 -0
- app/core/openai/strict_schema.py +199 -0
- app/core/openai/tool_call_safety.py +99 -0
- app/core/openai/v1_requests.py +187 -0
- app/core/plan_types.py +90 -0
- app/core/prestop.py +202 -0
- app/core/rate_limiter/__init__.py +0 -0
- app/core/rate_limiter/db_rate_limiter.py +215 -0
- app/core/request_locality.py +473 -0
- app/core/resilience/__init__.py +0 -0
- app/core/resilience/backpressure.py +57 -0
- app/core/resilience/bulkhead.py +202 -0
- app/core/resilience/circuit_breaker.py +150 -0
- app/core/resilience/degradation.py +55 -0
- app/core/resilience/loop_lag_monitor.py +60 -0
- app/core/resilience/memory_monitor.py +137 -0
- app/core/resilience/network_recovery.py +256 -0
- app/core/resilience/overload.py +98 -0
- app/core/resilience/toggles.py +114 -0
- app/core/retention/__init__.py +0 -0
- app/core/retention/job.py +240 -0
- app/core/retention/scheduler.py +69 -0
- app/core/runtime_logging.py +601 -0
- app/core/scheduling/__init__.py +0 -0
- app/core/scheduling/leader_election.py +993 -0
- app/core/scheduling/leader_election_handle.py +25 -0
- app/core/server.py +140 -0
- app/core/shutdown.py +326 -0
- app/core/socket_peer.py +22 -0
- app/core/startup.py +41 -0
- app/core/timeout_invariants.py +347 -0
- app/core/tracing/__init__.py +0 -0
- app/core/tracing/otel.py +207 -0
- app/core/types.py +6 -0
- app/core/upstream_proxy/__init__.py +13 -0
- app/core/upstream_proxy/cache.py +130 -0
- app/core/upstream_proxy/resolver.py +189 -0
- app/core/upstream_proxy/types.py +104 -0
- app/core/usage/__init__.py +407 -0
- app/core/usage/depletion.py +109 -0
- app/core/usage/live_hub.py +41 -0
- app/core/usage/live_snapshots.py +174 -0
- app/core/usage/logs.py +212 -0
- app/core/usage/metadata_scheduler.py +118 -0
- app/core/usage/models.py +54 -0
- app/core/usage/pricing.py +576 -0
- app/core/usage/pricing_catalog.py +241 -0
- app/core/usage/pricing_snapshot.json +821 -0
- app/core/usage/quota.py +133 -0
- app/core/usage/refresh_policy.py +21 -0
- app/core/usage/refresh_scheduler.py +713 -0
- app/core/usage/reset_credits_refresh_scheduler.py +435 -0
- app/core/usage/types.py +168 -0
- app/core/utils/__init__.py +0 -0
- app/core/utils/json_guards.py +18 -0
- app/core/utils/locks.py +33 -0
- app/core/utils/masking.py +20 -0
- app/core/utils/proxy_env.py +104 -0
- app/core/utils/request_id.py +65 -0
- app/core/utils/retry.py +77 -0
- app/core/utils/shared_future.py +172 -0
- app/core/utils/sse.py +286 -0
- app/core/utils/time.py +24 -0
- app/db/__init__.py +0 -0
- app/db/account_identity_lock.py +56 -0
- app/db/alembic/__init__.py +1 -0
- app/db/alembic/env.py +70 -0
- app/db/alembic/revision_ids.py +43 -0
- app/db/alembic/versions/20260213_000000_base_schema.py +248 -0
- app/db/alembic/versions/20260213_000100_normalize_account_plan_types.py +45 -0
- app/db/alembic/versions/20260213_000200_add_request_logs_reasoning_effort.py +45 -0
- app/db/alembic/versions/20260213_000300_add_accounts_reset_at.py +45 -0
- app/db/alembic/versions/20260213_000400_add_accounts_chatgpt_account_id.py +45 -0
- app/db/alembic/versions/20260213_000500_add_dashboard_settings.py +70 -0
- app/db/alembic/versions/20260213_000600_add_dashboard_settings_totp.py +64 -0
- app/db/alembic/versions/20260213_000700_add_dashboard_settings_password.py +59 -0
- app/db/alembic/versions/20260213_000800_add_api_keys.py +87 -0
- app/db/alembic/versions/20260214_000000_add_api_key_limits.py +143 -0
- app/db/alembic/versions/20260215_000000_add_idx_logs_requested_at.py +38 -0
- app/db/alembic/versions/20260218_000000_add_api_key_usage_reservations.py +117 -0
- app/db/alembic/versions/20260218_000100_add_import_without_overwrite_and_drop_accounts_email_unique.py +198 -0
- app/db/alembic/versions/20260225_000000_add_dashboard_settings_routing_strategy.py +53 -0
- app/db/alembic/versions/20260228_020000_align_api_key_limit_enum_types.py +104 -0
- app/db/alembic/versions/20260228_030000_add_api_firewall_allowlist.py +42 -0
- app/db/alembic/versions/20260306_000000_add_request_logs_service_tier.py +45 -0
- app/db/alembic/versions/20260307_000000_add_api_key_enforcement_fields.py +56 -0
- app/db/alembic/versions/20260308_000000_add_sqlite_performance_indexes.py +41 -0
- app/db/alembic/versions/20260309_000000_add_additional_usage_history.py +71 -0
- app/db/alembic/versions/20260309_000000_request_logs_nullable_account_id.py +28 -0
- app/db/alembic/versions/20260310_000000_add_request_logs_transport.py +45 -0
- app/db/alembic/versions/20260310_000000_fix_postgresql_enum_value_casing.py +79 -0
- app/db/alembic/versions/20260310_120000_add_sticky_session_kinds_and_affinity_ttl.py +110 -0
- app/db/alembic/versions/20260312_000000_add_additional_usage_quota_key.py +193 -0
- app/db/alembic/versions/20260312_000000_split_sticky_sessions_primary_key_by_kind.py +96 -0
- app/db/alembic/versions/20260312_010000_merge_additional_usage_and_sticky_session_heads.py +25 -0
- app/db/alembic/versions/20260312_020000_merge_request_logs_transport_and_feature_heads.py +25 -0
- app/db/alembic/versions/20260312_020000_merge_request_logs_transport_and_sticky_heads.py +25 -0
- app/db/alembic/versions/20260312_120000_add_dashboard_upstream_stream_transport.py +62 -0
- app/db/alembic/versions/20260313_024500_merge_request_log_transport_heads.py +27 -0
- app/db/alembic/versions/20260319_100937_increase_prompt_cache_ttl_to_1800s.py +73 -0
- app/db/alembic/versions/20260319_111500_merge_prompt_cache_ttl_and_upstream_transport_heads.py +25 -0
- app/db/alembic/versions/20260319_183000_normalize_sqlite_account_status_casing.py +48 -0
- app/db/alembic/versions/20260320_000000_add_request_log_requested_actual_tiers.py +51 -0
- app/db/alembic/versions/20260321_190000_tighten_dashboard_db_indexes.py +64 -0
- app/db/alembic/versions/20260321_210000_merge_request_log_tiers_and_dashboard_index_heads.py +25 -0
- app/db/alembic/versions/20260325_000000_add_request_log_cost.py +193 -0
- app/db/alembic/versions/20260328_000000_add_rate_limit_attempts.py +48 -0
- app/db/alembic/versions/20260328_120000_add_request_log_ttft.py +37 -0
- app/db/alembic/versions/20260328_130000_add_audit_logs_table.py +45 -0
- app/db/alembic/versions/20260328_140000_add_scheduler_leader_table.py +39 -0
- app/db/alembic/versions/20260330_000000_add_cache_locality_settings.py +69 -0
- app/db/alembic/versions/20260330_010000_merge_scheduler_leader_and_cache_locality_heads.py +25 -0
- app/db/alembic/versions/20260330_020000_add_bridge_ring_members.py +46 -0
- app/db/alembic/versions/20260401_000000_add_api_key_enforced_service_tier.py +52 -0
- app/db/alembic/versions/20260401_000000_add_cache_invalidation.py +41 -0
- app/db/alembic/versions/20260401_020000_merge_cache_invalidation_and_api_key_service_tier_heads.py +22 -0
- app/db/alembic/versions/20260402_000000_switch_dashboard_routing_default_to_capacity_weighted.py +66 -0
- app/db/alembic/versions/20260403_000000_add_credit_api_key_limit_values.py +44 -0
- app/db/alembic/versions/20260406_000000_add_api_key_account_assignments.py +57 -0
- app/db/alembic/versions/20260406_010000_add_api_key_assignment_scope_flag.py +75 -0
- app/db/alembic/versions/20260407_000000_add_bridge_gateway_safe_mode.py +51 -0
- app/db/alembic/versions/20260407_010000_merge_api_key_assignment_and_bridge_gateway_heads.py +25 -0
- app/db/alembic/versions/20260408_000000_switch_import_without_overwrite_default_to_true.py +73 -0
- app/db/alembic/versions/20260408_010000_merge_import_without_overwrite_and_assignment_heads.py +25 -0
- app/db/alembic/versions/20260409_000000_add_http_bridge_sessions.py +130 -0
- app/db/alembic/versions/20260409_000000_switch_sticky_threads_and_prefer_earlier_reset_defaults_to_true.py +89 -0
- app/db/alembic/versions/20260409_010000_add_dashboard_settings_bootstrap_token.py +57 -0
- app/db/alembic/versions/20260409_010000_merge_http_bridge_and_import_without_overwrite_heads.py +25 -0
- app/db/alembic/versions/20260409_020000_fix_http_bridge_last_seen_index.py +46 -0
- app/db/alembic/versions/20260410_000000_backfill_pristine_dashboard_settings_defaults.py +77 -0
- app/db/alembic/versions/20260410_030000_restore_import_without_overwrite_default_true.py +73 -0
- app/db/alembic/versions/20260410_040000_merge_dashboard_defaults_and_import_default_heads.py +25 -0
- app/db/alembic/versions/20260413_000000_add_accounts_blocked_at.py +44 -0
- app/db/alembic/versions/20260415_160000_add_automations_tables.py +189 -0
- app/db/alembic/versions/20260415_160000_add_request_logs_response_lookup_index.py +67 -0
- app/db/alembic/versions/20260417_000000_add_request_log_plan_type.py +44 -0
- app/db/alembic/versions/20260418_190000_add_automation_runs_cycle_key.py +87 -0
- app/db/alembic/versions/20260419_000000_add_automation_run_cycle_metadata.py +56 -0
- app/db/alembic/versions/20260419_020000_add_automation_run_cycles_snapshot_tables.py +342 -0
- app/db/alembic/versions/20260421_000000_add_dashboard_session_ttl_setting.py +51 -0
- app/db/alembic/versions/20260421_120000_merge_request_log_lookup_and_plan_type_heads.py +25 -0
- app/db/alembic/versions/20260421_130000_merge_automation_and_request_log_heads.py +25 -0
- app/db/alembic/versions/20260422_103000_normalize_legacy_automation_run_cycle_keys.py +648 -0
- app/db/alembic/versions/20260423_120000_add_api_key_limit_reset_at_index.py +43 -0
- app/db/alembic/versions/20260424_000000_merge_dashboard_session_ttl_and_request_log_heads.py +25 -0
- app/db/alembic/versions/20260426_000000_add_dashboard_relative_availability_settings.py +68 -0
- app/db/alembic/versions/20260427_130000_add_warmup_model_and_request_kind.py +75 -0
- app/db/alembic/versions/20260506_000000_add_account_routing_policy.py +37 -0
- app/db/alembic/versions/20260509_000000_add_api_key_traffic_class.py +37 -0
- app/db/alembic/versions/20260509_010000_add_additional_quota_routing_policies.py +37 -0
- app/db/alembic/versions/20260513_000000_add_accounts_alias.py +44 -0
- app/db/alembic/versions/20260513_000000_add_api_key_apply_to_codex_model.py +59 -0
- app/db/alembic/versions/20260514_000000_add_request_logs_api_key_time_index.py +43 -0
- app/db/alembic/versions/20260515_000000_add_prefer_earlier_reset_window.py +47 -0
- app/db/alembic/versions/20260515_000000_soft_delete_request_logs_on_account_delete.py +83 -0
- app/db/alembic/versions/20260515_020000_add_split_sticky_budget_thresholds.py +62 -0
- app/db/alembic/versions/20260516_000000_add_sqlite_hot_path_indexes.py +75 -0
- app/db/alembic/versions/20260517_000000_merge_sqlite_hot_path_and_request_log_delete_heads.py +24 -0
- app/db/alembic/versions/20260518_000000_add_http_bridge_durable_input_prefix.py +44 -0
- app/db/alembic/versions/20260518_010000_merge_http_bridge_and_sqlite_recovery_heads.py +25 -0
- app/db/alembic/versions/20260518_120000_add_dashboard_guest_access.py +57 -0
- app/db/alembic/versions/20260520_000000_merge_api_key_and_http_bridge_heads.py +26 -0
- app/db/alembic/versions/20260520_010000_add_request_logs_api_key_account_index.py +45 -0
- app/db/alembic/versions/20260520_010000_merge_automation_and_api_key_heads.py +26 -0
- app/db/alembic/versions/20260520_030000_add_quota_planner.py +194 -0
- app/db/alembic/versions/20260520_040000_merge_quota_planner_and_request_log_index_heads.py +24 -0
- app/db/alembic/versions/20260521_000000_add_account_security_work_authorized.py +46 -0
- app/db/alembic/versions/20260521_000000_merge_dashboard_guest_and_api_key_heads.py +26 -0
- app/db/alembic/versions/20260521_000000_merge_prefer_reset_window_head.py +26 -0
- app/db/alembic/versions/20260521_000000_merge_warmup_and_request_log_index_heads.py +25 -0
- app/db/alembic/versions/20260522_000000_add_limit_warmup_trigger.py +195 -0
- app/db/alembic/versions/20260524_000000_merge_accounts_alias_and_reset_window_heads.py +24 -0
- app/db/alembic/versions/20260525_000000_add_usage_raw_window_latest_index.py +31 -0
- app/db/alembic/versions/20260526_000000_add_request_log_failure_metadata.py +100 -0
- app/db/alembic/versions/20260531_000000_add_account_workspace_identity.py +39 -0
- app/db/alembic/versions/20260531_000000_add_single_account_routing.py +42 -0
- app/db/alembic/versions/20260601_000000_merge_relative_availability_and_usage_raw_heads.py +24 -0
- app/db/alembic/versions/20260601_010000_add_api_key_usage_sections.py +58 -0
- app/db/alembic/versions/20260601_010000_add_routing_policy_persistence.py +67 -0
- app/db/alembic/versions/20260601_010000_merge_security_work_and_relative_availability_heads.py +25 -0
- app/db/alembic/versions/20260601_010000_merge_split_sticky_and_relative_usage_heads.py +25 -0
- app/db/alembic/versions/20260601_010000_merge_warmup_and_relative_availability_heads.py +25 -0
- app/db/alembic/versions/20260601_020000_merge_additional_quota_routing_and_relative_availability_heads.py +25 -0
- app/db/alembic/versions/20260601_020000_merge_quota_planner_and_relative_availability_heads.py +25 -0
- app/db/alembic/versions/20260601_020000_merge_reset_window_and_relative_availability_heads.py +25 -0
- app/db/alembic/versions/20260601_020000_merge_routing_policy_and_split_threshold_heads.py +24 -0
- app/db/alembic/versions/20260601_020000_merge_warmup_and_request_log_failure_heads.py +25 -0
- app/db/alembic/versions/20260601_030000_merge_single_account_and_routing_quota_heads.py +29 -0
- app/db/alembic/versions/20260601_040000_merge_security_work_and_routing_consolidation_heads.py +25 -0
- app/db/alembic/versions/20260601_050000_merge_quota_planner_and_routing_consolidation_heads.py +25 -0
- app/db/alembic/versions/20260602_050000_add_upstream_proxy_routing.py +164 -0
- app/db/alembic/versions/20260602_060000_merge_account_workspace_and_failure_heads.py +25 -0
- app/db/alembic/versions/20260602_070000_add_request_log_useragent_fields.py +86 -0
- app/db/alembic/versions/20260602_070000_merge_quota_planner_and_account_workspace_heads.py +25 -0
- app/db/alembic/versions/20260602_080000_merge_upstream_proxy_and_quota_planner_heads.py +25 -0
- app/db/alembic/versions/20260602_080000_merge_useragent_and_upstream_proxy_heads.py +24 -0
- app/db/alembic/versions/20260603_000000_add_weekly_pace_working_days.py +52 -0
- app/db/alembic/versions/20260603_000000_free_account_monthly_window.py +60 -0
- app/db/alembic/versions/20260604_000000_add_reauth_required_account_status.py +159 -0
- app/db/alembic/versions/20260604_010000_merge_automations_and_reauth_status_heads.py +25 -0
- app/db/alembic/versions/20260604_010000_merge_dashboard_guest_and_reauth_status_heads.py +25 -0
- app/db/alembic/versions/20260604_010000_merge_useragent_and_reauth_heads.py +24 -0
- app/db/alembic/versions/20260607_000000_merge_weekly_monthly_useragent_heads.py +28 -0
- app/db/alembic/versions/20260608_000000_add_hide_upstream_quota_from_api_keys.py +50 -0
- app/db/alembic/versions/20260611_000000_add_request_log_archive_request_id.py +47 -0
- app/db/alembic/versions/20260611_000000_merge_automations_and_weekly_monthly_useragent_heads.py +26 -0
- app/db/alembic/versions/20260611_000000_merge_dashboard_guest_and_weekly_useragent_heads.py +26 -0
- app/db/alembic/versions/20260611_010000_add_request_log_client_ip.py +57 -0
- app/db/alembic/versions/20260611_020000_merge_request_log_archive_and_client_ip_heads.py +26 -0
- app/db/alembic/versions/20260611_030000_merge_dashboard_guest_and_request_log_client_ip_heads.py +26 -0
- app/db/alembic/versions/20260613_000000_add_accounts_codex_installation_id.py +57 -0
- app/db/alembic/versions/20260613_010000_merge_installation_id_and_dashboard_weekly_heads.py +26 -0
- app/db/alembic/versions/20260615_000000_merge_automations_and_dashboard_guest_heads.py +26 -0
- app/db/alembic/versions/20260623_000000_add_limit_warmup_exhausted_threshold.py +51 -0
- app/db/alembic/versions/20260626_000000_add_http_downstream_transport_policy.py +46 -0
- app/db/alembic/versions/20260626_010000_add_request_logs_upstream_transport.py +31 -0
- app/db/alembic/versions/20260629_000000_add_dashboard_query_hot_path_indexes.py +53 -0
- app/db/alembic/versions/20260630_000000_add_limit_warmup_staggered_idle.py +47 -0
- app/db/alembic/versions/20260630_000000_merge_request_log_client_ip_and_dashboard_hot_path_heads.py +26 -0
- app/db/alembic/versions/20260630_010000_merge_warmup_and_request_log_dashboard_heads.py +26 -0
- app/db/alembic/versions/20260630_020000_merge_automations_and_main_heads.py +26 -0
- app/db/alembic/versions/20260630_020000_merge_warmup_threshold_and_main_heads.py +26 -0
- app/db/alembic/versions/20260630_030000_add_automation_run_model_snapshot.py +59 -0
- app/db/alembic/versions/20260630_040000_merge_automation_snapshot_and_warmup_threshold_heads.py +26 -0
- app/db/alembic/versions/20260630_050000_add_automation_run_prompt_snapshot.py +52 -0
- app/db/alembic/versions/20260701_000000_add_weekly_pace_smoothing_minutes.py +51 -0
- app/db/alembic/versions/20260702_000000_add_model_sources.py +176 -0
- app/db/alembic/versions/20260705_000000_add_model_source_audio_transcriptions.py +50 -0
- app/db/alembic/versions/20260705_000000_harden_dashboard_session_ttl.py +62 -0
- app/db/alembic/versions/20260706_000000_add_model_source_audio_per_minute.py +43 -0
- app/db/alembic/versions/20260707_000000_merge_model_sources_and_session_ttl.py +24 -0
- app/db/alembic/versions/20260707_010000_merge_automations_and_model_sources_heads.py +24 -0
- app/db/alembic/versions/20260707_020000_add_automation_job_account_scope.py +125 -0
- app/db/alembic/versions/20260709_000000_add_ttft_phase_observability.py +66 -0
- app/db/alembic/versions/20260711_000000_add_dashboard_prohibit_fast_mode.py +42 -0
- app/db/alembic/versions/20260711_010000_add_account_chatgpt_user_id.py +40 -0
- app/db/alembic/versions/20260711_020000_add_dashboard_account_concurrency_caps.py +71 -0
- app/db/alembic/versions/20260711_030000_add_limit_warmup_idle_threshold.py +51 -0
- app/db/alembic/versions/20260712_010000_add_account_usage_rollups.py +56 -0
- app/db/alembic/versions/20260712_020000_add_api_key_usage_rollups.py +53 -0
- app/db/alembic/versions/20260713_020000_add_model_registry_snapshot.py +42 -0
- app/db/alembic/versions/20260713_040000_add_account_refresh_claims.py +42 -0
- app/db/alembic/versions/20260713_040000_add_replica_guardrails.py +64 -0
- app/db/alembic/versions/20260713_070000_add_reset_credit_redeem_tables.py +58 -0
- app/db/alembic/versions/20260714_000000_add_oauth_flow_states.py +63 -0
- app/db/alembic/versions/20260715_000000_add_request_log_queue_latency.py +44 -0
- app/db/alembic/versions/20260716_000000_add_oauth_device_flow_slots.py +41 -0
- app/db/alembic/versions/20260716_010000_add_dashboard_retention_settings.py +48 -0
- app/db/alembic/versions/20260716_010000_add_reset_credit_display_settings.py +73 -0
- app/db/alembic/versions/20260717_000000_add_http_bridge_retry_circuits.py +47 -0
- app/db/alembic/versions/20260717_000000_merge_retention_and_reset_credit_display_heads.py +26 -0
- app/db/alembic/versions/20260717_000000_optimize_dashboard_hot_path_indexes.py +162 -0
- app/db/alembic/versions/20260717_000001_merge_retry_circuits_and_dashboard_indexes.py +22 -0
- app/db/alembic/versions/20260720_000000_add_request_log_conversation_id.py +57 -0
- app/db/alembic/versions/20260722_000000_backfill_request_log_useragent_families.py +40 -0
- app/db/alembic/versions/20260724_000000_add_request_usage_time_rollups.py +176 -0
- app/db/alembic/versions/20260724_000000_merge_request_log_schema_heads.py +33 -0
- app/db/alembic/versions/20260725_000000_add_http_bridge_pending_tool_calls.py +43 -0
- app/db/alembic/versions/20260726_000000_add_account_plan_downgrade_observations.py +44 -0
- app/db/alembic/versions/20260726_000000_repair_request_usage_rollups_after_merge.py +37 -0
- app/db/alembic/versions/20260727_000000_add_sticky_session_continuity_abandoned_at.py +45 -0
- app/db/alembic/versions/20260728_000000_merge_pending_tool_calls_and_rollup_repair_heads.py +31 -0
- app/db/alembic/versions/20260730_000000_add_api_key_fair_share_threshold.py +44 -0
- app/db/alembic/versions/20260730_000000_add_http_bridge_recovery_attempts.py +60 -0
- app/db/alembic/versions/20260731_000000_add_capability_lineage_markers.py +38 -0
- app/db/alembic/versions/20260803_000000_merge_http_bridge_recovery_and_capability_lineage_heads.py +30 -0
- app/db/alembic/versions/20260804_000000_add_http_bridge_operations.py +63 -0
- app/db/alembic/versions/20260804_000001_add_global_http_bridge_operation_fingerprint.py +53 -0
- app/db/alembic/versions/20260804_230000_add_request_log_connection_request_kind.py +43 -0
- app/db/alembic/versions/20260805_000000_add_http_bridge_operation_spool.py +75 -0
- app/db/alembic/versions/20260805_000001_finalize_http_bridge_operation_spool.py +65 -0
- app/db/alembic/versions/20260806_000000_add_additional_usage_alias_probe_indexes.py +84 -0
- app/db/alembic/versions/20260806_000000_add_anonymous_telemetry.py +49 -0
- app/db/alembic/versions/20260806_010000_add_conversation_presence_rollup.py +72 -0
- app/db/alembic/versions/20260806_020000_add_usage_history_bulk_covering_indexes.py +99 -0
- app/db/alembic/versions/20260806_030000_add_api_key_allowed_reasoning_efforts.py +65 -0
- app/db/alembic/versions/20260806_120000_add_http_bridge_owner_process_epoch.py +57 -0
- app/db/alembic/versions/20260807_000000_merge_http_bridge_operation_spool_and_latest_main.py +24 -0
- app/db/alembic/versions/20260808_000000_tune_usage_history_autovacuum.py +54 -0
- app/db/alembic/versions/20260810_000000_add_http_bridge_recovery_dispatch_count.py +44 -0
- app/db/alembic/versions/20260811_000000_add_hourly_rollup_cancelled_count.py +92 -0
- app/db/alembic/versions/20260812_000000_merge_recovery_dispatch_and_hourly_cancelled_heads.py +25 -0
- app/db/alembic/versions/20260812_120000_add_sticky_abandonment_scope.py +43 -0
- app/db/alembic/versions/20260813_000000_add_file_account_pins.py +40 -0
- app/db/alembic/versions/20260816_000000_add_account_pending_deletion.py +81 -0
- app/db/alembic/versions/20260816_000000_add_model_source_embeddings.py +50 -0
- app/db/alembic/versions/20260821_000000_add_retry_circuit_admission_generation.py +49 -0
- app/db/alembic/versions/20260826_000000_add_http_bridge_event_chunks.py +107 -0
- app/db/alembic/versions/20260828_000000_add_accounts_chatgpt_identity_index.py +55 -0
- app/db/alembic/versions/20260830_000000_add_quota_warmup_claim_expiry.py +83 -0
- app/db/alembic/versions/20260908_000000_add_guest_session_generation.py +37 -0
- app/db/alembic/versions/20260908_000000_add_subscription_overflow.py +140 -0
- app/db/alembic/versions/20260908_000000_replace_upstream_stream_transport_default_sentinel.py +50 -0
- app/db/alembic/versions/20260908_020000_merge_overflow_transport_heads.py +27 -0
- app/db/alembic/versions/20260909_000000_add_dashboard_roles.py +72 -0
- app/db/alembic/versions/20260909_010000_add_dashboard_users.py +183 -0
- app/db/alembic/versions/20260909_020000_reproject_compat_admin_credentials.py +105 -0
- app/db/alembic/versions/20260909_030000_add_audit_actor_columns.py +103 -0
- app/db/alembic/versions/20260909_030000_dashboard_resilience_toggle_settings.py +63 -0
- app/db/alembic/versions/20260909_040000_add_dashboard_user_invites.py +54 -0
- app/db/alembic/versions/20260909_040000_dashboard_timeout_settings.py +68 -0
- app/db/alembic/versions/20260909_050000_dashboard_routing_overload_settings.py +65 -0
- app/db/alembic/versions/20260909_060000_add_report_rollup.py +74 -0
- app/db/alembic/versions/20260909_070000_automation_run_claim_budget.py +50 -0
- app/db/alembic/versions/20260909_080000_dashboard_stream_bridge_budgets.py +62 -0
- app/db/alembic/versions/20260909_090000_dashboard_background_job_toggles.py +63 -0
- app/db/alembic/versions/20260909_100000_dashboard_codex_prewarm.py +49 -0
- app/db/alembic/versions/20260909_110000_model_context_window_overrides.py +45 -0
- app/db/alembic/versions/20260909_120000_dashboard_conversation_archive.py +48 -0
- app/db/alembic/versions/20260909_130000_add_request_logs_live_facet_indexes.py +96 -0
- app/db/alembic/versions/20260910_000000_add_totp_required_for_admin_role.py +44 -0
- app/db/alembic/versions/20260910_000000_request_logs_missing_cost_index.py +52 -0
- app/db/alembic/versions/20260910_010000_add_dashboard_auth_providers.py +134 -0
- app/db/alembic/versions/20260910_010000_dashboard_spool_retention.py +52 -0
- app/db/alembic/versions/20260910_020000_add_dashboard_role_mappings.py +74 -0
- app/db/alembic/versions/20260910_143000_request_log_affinity.py +28 -0
- app/db/alembic/versions/20260910_180000_merge_affinity_guest_heads.py +17 -0
- app/db/alembic/versions/20260910_200000_merge_affinity_identity_heads.py +17 -0
- app/db/alembic/versions/20260910_220000_merge_affinity_invite_heads.py +17 -0
- app/db/alembic/versions/20260911_000000_model_source_pins_kind_expires_index.py +43 -0
- app/db/alembic/versions/20260911_010000_merge_pin_index_and_affinity_heads.py +17 -0
- app/db/alembic/versions/20260911_020000_add_http_bridge_terminal_append_phase.py +54 -0
- app/db/alembic/versions/20260911_030000_add_local_login_policy.py +48 -0
- app/db/alembic/versions/20260911_040000_add_thread_cache_identity_mode.py +48 -0
- app/db/alembic/versions/20260911_060000_add_bridge_session_continuity_abandonment.py +40 -0
- app/db/alembic/versions/20260912_000000_merge_thread_cache_and_bridge_retirement_heads.py +22 -0
- app/db/alembic/versions/20260912_010000_drop_legacy_dashboard_credentials.py +140 -0
- app/db/alembic/versions/20260913_000000_add_oidc_provider_flow.py +134 -0
- app/db/alembic/versions/20260914_000000_add_scim_tokens.py +78 -0
- app/db/alembic/versions/20260914_000000_drop_subscription_overflow_schema.py +132 -0
- app/db/alembic/versions/20260918_000000_merge_scim_and_overflow_heads.py +17 -0
- app/db/alembic/versions/__init__.py +1 -0
- app/db/backup.py +64 -0
- app/db/migrate.py +1249 -0
- app/db/migration_lock.py +160 -0
- app/db/migration_url.py +18 -0
- app/db/models.py +3100 -0
- app/db/recover.py +244 -0
- app/db/session.py +1295 -0
- app/db/snapshot.py +46 -0
- app/db/sqlite_lock_retry.py +190 -0
- app/db/sqlite_utils.py +549 -0
- app/dependencies.py +462 -0
- app/main.py +1240 -0
- app/modules/__init__.py +0 -0
- app/modules/accounts/__init__.py +0 -0
- app/modules/accounts/api.py +426 -0
- app/modules/accounts/auth_manager.py +1439 -0
- app/modules/accounts/background_repository.py +160 -0
- app/modules/accounts/deletion.py +540 -0
- app/modules/accounts/mappers.py +654 -0
- app/modules/accounts/refresh_claims.py +352 -0
- app/modules/accounts/repository.py +1679 -0
- app/modules/accounts/schemas.py +308 -0
- app/modules/accounts/service.py +809 -0
- app/modules/accounts/usage_rollup.py +486 -0
- app/modules/accounts/usage_rollup_scheduler.py +75 -0
- app/modules/accounts/usage_time_rollup.py +1037 -0
- app/modules/accounts/usage_time_rollup_read.py +522 -0
- app/modules/api_keys/__init__.py +1 -0
- app/modules/api_keys/api.py +344 -0
- app/modules/api_keys/last_used_coalescer.py +216 -0
- app/modules/api_keys/limit_windows.py +41 -0
- app/modules/api_keys/repository.py +1200 -0
- app/modules/api_keys/reset_scheduler.py +85 -0
- app/modules/api_keys/schemas.py +135 -0
- app/modules/api_keys/service.py +2123 -0
- app/modules/audit/__init__.py +0 -0
- app/modules/audit/api.py +88 -0
- app/modules/audit/repository.py +74 -0
- app/modules/audit/schemas.py +33 -0
- app/modules/audit/service.py +71 -0
- app/modules/auth_providers/__init__.py +0 -0
- app/modules/auth_providers/api.py +127 -0
- app/modules/auth_providers/config.py +343 -0
- app/modules/auth_providers/repository.py +101 -0
- app/modules/auth_providers/schemas.py +88 -0
- app/modules/auth_providers/seed.py +105 -0
- app/modules/auth_providers/service.py +289 -0
- app/modules/automations/__init__.py +1 -0
- app/modules/automations/api.py +369 -0
- app/modules/automations/repository.py +1896 -0
- app/modules/automations/scheduler.py +107 -0
- app/modules/automations/schemas.py +158 -0
- app/modules/automations/service.py +2548 -0
- app/modules/cache_isolation_probe/__init__.py +0 -0
- app/modules/cache_isolation_probe/api.py +188 -0
- app/modules/cache_isolation_probe/prefix.py +144 -0
- app/modules/cache_isolation_probe/schemas.py +94 -0
- app/modules/cache_isolation_probe/sender.py +229 -0
- app/modules/cache_isolation_probe/service.py +525 -0
- app/modules/conversation_archive/__init__.py +1 -0
- app/modules/conversation_archive/api.py +101 -0
- app/modules/conversation_archive/schemas.py +28 -0
- app/modules/conversation_archive/service.py +165 -0
- app/modules/dashboard/__init__.py +1 -0
- app/modules/dashboard/api.py +106 -0
- app/modules/dashboard/builders.py +107 -0
- app/modules/dashboard/repository.py +304 -0
- app/modules/dashboard/schemas.py +148 -0
- app/modules/dashboard/service.py +534 -0
- app/modules/dashboard/timeframes.py +19 -0
- app/modules/dashboard/weekly_pace.py +617 -0
- app/modules/dashboard_auth/__init__.py +1 -0
- app/modules/dashboard_auth/api.py +1344 -0
- app/modules/dashboard_auth/oidc_api.py +753 -0
- app/modules/dashboard_auth/oidc_flows.py +330 -0
- app/modules/dashboard_auth/repository.py +452 -0
- app/modules/dashboard_auth/schemas.py +232 -0
- app/modules/dashboard_auth/service.py +1384 -0
- app/modules/dashboard_roles/__init__.py +0 -0
- app/modules/dashboard_roles/api.py +64 -0
- app/modules/dashboard_roles/repository.py +54 -0
- app/modules/dashboard_roles/schemas.py +29 -0
- app/modules/dashboard_roles/seed.py +80 -0
- app/modules/dashboard_roles/service.py +142 -0
- app/modules/dashboard_users/__init__.py +0 -0
- app/modules/dashboard_users/api.py +278 -0
- app/modules/dashboard_users/break_glass.py +169 -0
- app/modules/dashboard_users/credentials.py +18 -0
- app/modules/dashboard_users/identity_resolver.py +665 -0
- app/modules/dashboard_users/repository.py +725 -0
- app/modules/dashboard_users/schemas.py +117 -0
- app/modules/dashboard_users/service.py +1054 -0
- app/modules/firewall/__init__.py +1 -0
- app/modules/firewall/api.py +78 -0
- app/modules/firewall/repository.py +51 -0
- app/modules/firewall/schemas.py +28 -0
- app/modules/firewall/service.py +93 -0
- app/modules/fleet/__init__.py +1 -0
- app/modules/fleet/api.py +177 -0
- app/modules/fleet/mappers.py +56 -0
- app/modules/fleet/observability.py +372 -0
- app/modules/fleet/schemas.py +119 -0
- app/modules/health/__init__.py +0 -0
- app/modules/health/api.py +246 -0
- app/modules/health/schemas.py +27 -0
- app/modules/limit_warmup/__init__.py +1 -0
- app/modules/limit_warmup/repository.py +159 -0
- app/modules/limit_warmup/service.py +1001 -0
- app/modules/model_sources/__init__.py +1 -0
- app/modules/model_sources/api.py +99 -0
- app/modules/model_sources/catalog.py +307 -0
- app/modules/model_sources/forwarding.py +1435 -0
- app/modules/model_sources/projection.py +55 -0
- app/modules/model_sources/repository.py +201 -0
- app/modules/model_sources/schemas.py +79 -0
- app/modules/model_sources/selection.py +166 -0
- app/modules/model_sources/service.py +236 -0
- app/modules/oauth/__init__.py +0 -0
- app/modules/oauth/api.py +101 -0
- app/modules/oauth/repository.py +238 -0
- app/modules/oauth/schemas.py +49 -0
- app/modules/oauth/service.py +1196 -0
- app/modules/oauth/templates/oauth_success.html +122 -0
- app/modules/proxy/__init__.py +0 -0
- app/modules/proxy/_load_balancer/__init__.py +1 -0
- app/modules/proxy/_load_balancer/error_rate.py +116 -0
- app/modules/proxy/_load_balancer/latency_cohort.py +154 -0
- app/modules/proxy/_load_balancer/model_eligibility.py +250 -0
- app/modules/proxy/_load_balancer/opportunistic_admission.py +350 -0
- app/modules/proxy/_load_balancer/overload_backoff.py +394 -0
- app/modules/proxy/_load_balancer/sticky_selection.py +2417 -0
- app/modules/proxy/_load_balancer/throughput_cohort.py +233 -0
- app/modules/proxy/_load_balancer/ttft_cohort.py +214 -0
- app/modules/proxy/_load_balancer/tunables.py +97 -0
- app/modules/proxy/_load_balancer/types.py +96 -0
- app/modules/proxy/_load_balancer/unbound_selection.py +534 -0
- app/modules/proxy/_service/__init__.py +6 -0
- app/modules/proxy/_service/api_key_usage.py +745 -0
- app/modules/proxy/_service/clock_budget.py +38 -0
- app/modules/proxy/_service/codex_control.py +611 -0
- app/modules/proxy/_service/compact.py +2211 -0
- app/modules/proxy/_service/file_ops.py +749 -0
- app/modules/proxy/_service/http_bridge/__init__.py +3 -0
- app/modules/proxy/_service/http_bridge/accepted_replay.py +331 -0
- app/modules/proxy/_service/http_bridge/account_sessions.py +57 -0
- app/modules/proxy/_service/http_bridge/activity.py +178 -0
- app/modules/proxy/_service/http_bridge/helpers.py +3959 -0
- app/modules/proxy/_service/http_bridge/mixin.py +2435 -0
- app/modules/proxy/_service/http_bridge/owner_forwarding.py +609 -0
- app/modules/proxy/_service/http_bridge/protocol.py +167 -0
- app/modules/proxy/_service/http_bridge/proxy_failover.py +58 -0
- app/modules/proxy/_service/http_bridge/quarantine.py +443 -0
- app/modules/proxy/_service/http_bridge/request_submit.py +4582 -0
- app/modules/proxy/_service/http_bridge/retry_circuit.py +2442 -0
- app/modules/proxy/_service/http_bridge/service_stubs.py +492 -0
- app/modules/proxy/_service/http_bridge/session_registry.py +778 -0
- app/modules/proxy/_service/http_bridge/streaming.py +5328 -0
- app/modules/proxy/_service/http_bridge/upstream_events.py +4538 -0
- app/modules/proxy/_service/observability.py +410 -0
- app/modules/proxy/_service/rate_limit.py +361 -0
- app/modules/proxy/_service/realtime_live.py +655 -0
- app/modules/proxy/_service/refresh.py +154 -0
- app/modules/proxy/_service/request_log.py +585 -0
- app/modules/proxy/_service/response_create.py +930 -0
- app/modules/proxy/_service/streaming/__init__.py +3 -0
- app/modules/proxy/_service/streaming/helpers.py +1317 -0
- app/modules/proxy/_service/streaming/mixin.py +1100 -0
- app/modules/proxy/_service/streaming/protocol.py +32 -0
- app/modules/proxy/_service/streaming/retry.py +3762 -0
- app/modules/proxy/_service/support.py +2122 -0
- app/modules/proxy/_service/transcribe.py +434 -0
- app/modules/proxy/_service/warmup.py +483 -0
- app/modules/proxy/_service/websocket/__init__.py +3 -0
- app/modules/proxy/_service/websocket/helpers.py +2360 -0
- app/modules/proxy/_service/websocket/mixin.py +7285 -0
- app/modules/proxy/_service/websocket/protocol.py +74 -0
- app/modules/proxy/account_cache.py +229 -0
- app/modules/proxy/account_eligibility.py +87 -0
- app/modules/proxy/additional_model_limits.py +40 -0
- app/modules/proxy/affinity.py +830 -0
- app/modules/proxy/affinity_observation.py +101 -0
- app/modules/proxy/api.py +10254 -0
- app/modules/proxy/api_key_usage.py +113 -0
- app/modules/proxy/cap_partitioning.py +280 -0
- app/modules/proxy/capability_lineage.py +65 -0
- app/modules/proxy/capability_lineage_repository.py +82 -0
- app/modules/proxy/capability_routing.py +253 -0
- app/modules/proxy/continuity.py +112 -0
- app/modules/proxy/downstream_delivery.py +277 -0
- app/modules/proxy/durable_bridge_coordinator.py +1217 -0
- app/modules/proxy/durable_bridge_repository.py +4516 -0
- app/modules/proxy/durable_bridge_runtime.py +9 -0
- app/modules/proxy/durable_bridge_transcript_codec.py +122 -0
- app/modules/proxy/fair_share.py +131 -0
- app/modules/proxy/file_pin_repository.py +210 -0
- app/modules/proxy/helpers.py +439 -0
- app/modules/proxy/http_bridge_event_batcher.py +613 -0
- app/modules/proxy/http_bridge_forwarding.py +797 -0
- app/modules/proxy/http_continuation.py +76 -0
- app/modules/proxy/images_observability.py +62 -0
- app/modules/proxy/images_service.py +1054 -0
- app/modules/proxy/load_balancer.py +3016 -0
- app/modules/proxy/rate_limit_cache.py +46 -0
- app/modules/proxy/replay_safety.py +1063 -0
- app/modules/proxy/repo_bundle.py +31 -0
- app/modules/proxy/request_policy.py +1054 -0
- app/modules/proxy/ring_membership.py +251 -0
- app/modules/proxy/schemas.py +345 -0
- app/modules/proxy/selection_errors.py +41 -0
- app/modules/proxy/service.py +2600 -0
- app/modules/proxy/source_admission.py +125 -0
- app/modules/proxy/source_dispatch.py +999 -0
- app/modules/proxy/sticky_repository.py +831 -0
- app/modules/proxy/thread_anchors.py +491 -0
- app/modules/proxy/tool_call_dedupe.py +802 -0
- app/modules/proxy/types.py +54 -0
- app/modules/proxy/work_admission.py +121 -0
- app/modules/quota_planner/__init__.py +1 -0
- app/modules/quota_planner/api.py +276 -0
- app/modules/quota_planner/logic.py +743 -0
- app/modules/quota_planner/repository.py +722 -0
- app/modules/quota_planner/scheduler.py +212 -0
- app/modules/quota_planner/schemas.py +102 -0
- app/modules/quota_planner/warmup.py +693 -0
- app/modules/rate_limit_reset_credits/__init__.py +0 -0
- app/modules/rate_limit_reset_credits/api.py +601 -0
- app/modules/rate_limit_reset_credits/redeem_coordination.py +274 -0
- app/modules/rate_limit_reset_credits/store.py +114 -0
- app/modules/reports/__init__.py +1 -0
- app/modules/reports/api.py +134 -0
- app/modules/reports/cache.py +74 -0
- app/modules/reports/filters.py +27 -0
- app/modules/reports/repository.py +525 -0
- app/modules/reports/rollup.py +150 -0
- app/modules/reports/rollup_read.py +148 -0
- app/modules/reports/schemas.py +123 -0
- app/modules/reports/service.py +287 -0
- app/modules/reports/thread_identity.py +256 -0
- app/modules/request_logs/__init__.py +0 -0
- app/modules/request_logs/api.py +224 -0
- app/modules/request_logs/cost_backfill.py +154 -0
- app/modules/request_logs/mappers.py +97 -0
- app/modules/request_logs/repository.py +1765 -0
- app/modules/request_logs/schemas.py +145 -0
- app/modules/request_logs/service.py +348 -0
- app/modules/role_mappings/__init__.py +0 -0
- app/modules/role_mappings/api.py +206 -0
- app/modules/role_mappings/matching.py +55 -0
- app/modules/role_mappings/repository.py +106 -0
- app/modules/role_mappings/schemas.py +67 -0
- app/modules/role_mappings/service.py +334 -0
- app/modules/runtime/__init__.py +1 -0
- app/modules/runtime/api.py +18 -0
- app/modules/runtime/schemas.py +14 -0
- app/modules/runtime/service.py +218 -0
- app/modules/scim/__init__.py +1 -0
- app/modules/scim/api.py +194 -0
- app/modules/scim/dependencies.py +112 -0
- app/modules/scim/errors.py +77 -0
- app/modules/scim/management_api.py +156 -0
- app/modules/scim/repository.py +212 -0
- app/modules/scim/schemas.py +145 -0
- app/modules/scim/service.py +786 -0
- app/modules/scim/tokens.py +43 -0
- app/modules/settings/__init__.py +1 -0
- app/modules/settings/api.py +1677 -0
- app/modules/settings/repository.py +606 -0
- app/modules/settings/schemas.py +496 -0
- app/modules/settings/service.py +769 -0
- app/modules/shared/__init__.py +0 -0
- app/modules/shared/schemas.py +22 -0
- app/modules/sticky_sessions/__init__.py +1 -0
- app/modules/sticky_sessions/api.py +118 -0
- app/modules/sticky_sessions/cleanup_scheduler.py +487 -0
- app/modules/sticky_sessions/schemas.py +78 -0
- app/modules/sticky_sessions/service.py +196 -0
- app/modules/telemetry/__init__.py +5 -0
- app/modules/telemetry/api.py +115 -0
- app/modules/telemetry/clients.py +62 -0
- app/modules/telemetry/consent.py +115 -0
- app/modules/telemetry/scheduler.py +85 -0
- app/modules/telemetry/schemas.py +163 -0
- app/modules/telemetry/sender.py +221 -0
- app/modules/telemetry/snapshot.py +490 -0
- app/modules/usage/__init__.py +0 -0
- app/modules/usage/additional_quota_keys.py +379 -0
- app/modules/usage/api.py +45 -0
- app/modules/usage/background_repository.py +147 -0
- app/modules/usage/builders.py +559 -0
- app/modules/usage/depletion_service.py +277 -0
- app/modules/usage/live_ingest.py +356 -0
- app/modules/usage/mappers.py +29 -0
- app/modules/usage/plan_downgrade_observations.py +488 -0
- app/modules/usage/repository.py +1890 -0
- app/modules/usage/schemas.py +68 -0
- app/modules/usage/service.py +102 -0
- app/modules/usage/updater.py +1488 -0
- app/static/assets/Trans-Cj-J9mVA.js +1 -0
- app/static/assets/access-page-DHzZiXZi.js +1 -0
- app/static/assets/account-cost-donut-DGhfA4U8.js +1 -0
- app/static/assets/account-identifiers-CxbdLshZ.js +1 -0
- app/static/assets/account-status-DIsU5sUS.js +1 -0
- app/static/assets/account-trend-chart-BptlfwwQ.js +1 -0
- app/static/assets/accounts-page-Bbai8ift.js +3 -0
- app/static/assets/api-Dwa7XxPe.js +1 -0
- app/static/assets/api-client-dugX1Pti.js +55 -0
- app/static/assets/api-key-create-dialog-DnRm0oYN.js +1 -0
- app/static/assets/api-key-created-dialog-B6jkDG9f.js +1 -0
- app/static/assets/api-key-edit-dialog-Ck3Bnc3V.js +1 -0
- app/static/assets/api-trend-chart-BchCw7Hj.js +1 -0
- app/static/assets/apis-page-BvXz9c6i.js +2 -0
- app/static/assets/arrow-up-Dqh_KzJo.js +1 -0
- app/static/assets/arrow-up-down-LBRgVDGY.js +1 -0
- app/static/assets/automations-page-DQgNAJOR.js +3 -0
- app/static/assets/badge-DqMb9es0.js +1 -0
- app/static/assets/blur-email-HEvKagG2.js +1 -0
- app/static/assets/button-BjVmmiGW.js +1 -0
- app/static/assets/checkbox-apsl9vzk.js +1 -0
- app/static/assets/chevron-left-1d_pHVa5.js +1 -0
- app/static/assets/circle-alert-BS0xd_o1.js +1 -0
- app/static/assets/clsx-DB0hHKMi.js +1 -0
- app/static/assets/colors-BBy8GEtr.js +1 -0
- app/static/assets/copy-button-D7nAvOKR.js +1 -0
- app/static/assets/cost-per-day-chart-YdqlkZVM.js +1 -0
- app/static/assets/createLucideIcon-DBLzLNh2.js +1 -0
- app/static/assets/dashboard-page-CSdhBy0f.js +2 -0
- app/static/assets/distribution-metric-format-yZTy1ZEu.js +1 -0
- app/static/assets/donut-chart-Bxc69cra.js +1 -0
- app/static/assets/download-kQteIcsB.js +1 -0
- app/static/assets/ellipsis-B_5OgIre.js +1 -0
- app/static/assets/empty-state-JcA7tc8m.js +1 -0
- app/static/assets/es2015-bMH8YKZT.js +41 -0
- app/static/assets/es6-B264ni6W.js +69 -0
- app/static/assets/external-link-UVAOgMQ0.js +1 -0
- app/static/assets/i18n-BWqZ9Ua2.js +1 -0
- app/static/assets/inbox-B_RM2s1J.js +1 -0
- app/static/assets/index-DIpd264Z.js +23 -0
- app/static/assets/index-DMeLKRJx.css +2 -0
- app/static/assets/inherit-badge-QhTYVaU6.js +1 -0
- app/static/assets/invite-dialog-0CYh0GiQ.js +1 -0
- app/static/assets/iso-Bpe5Ihbb.js +1 -0
- app/static/assets/label-CGaLmLNf.js +1 -0
- app/static/assets/lazy-recharts-Pv8p9ArM.js +2 -0
- app/static/assets/loading-overlay-CzKQfHLI.js +1 -0
- app/static/assets/mini-quota-bar-cGT_t8Ae.js +1 -0
- app/static/assets/model-distribution-donut-BKjAfT48.js +1 -0
- app/static/assets/model-source-multi-select-D2d06Vc7.js +1 -0
- app/static/assets/multi-select-filter-KIA5qU6Z.js +1 -0
- app/static/assets/oauth-dialog-XeaGGDjS.js +1 -0
- app/static/assets/pagination-controls-D1Ypd0RU.js +1 -0
- app/static/assets/payload-CIYnGZz9.js +1 -0
- app/static/assets/queue-wait-chart-VoV7Yg9L.js +1 -0
- app/static/assets/report-chart-card-BXDliwcc.js +1 -0
- app/static/assets/reports-page-CGCJeMHG.js +3 -0
- app/static/assets/reset-credit-confirm-dialog-QOifTnVB.js +1 -0
- app/static/assets/rolldown-runtime-hePW80VL.js +1 -0
- app/static/assets/search-HfhqwERt.js +1 -0
- app/static/assets/select-Cl33bz-g.js +1 -0
- app/static/assets/settings-page-mZj01HYl.js +2 -0
- app/static/assets/shim-CZwyM4Q7.js +1 -0
- app/static/assets/skeleton-C_EUc4Rp.js +1 -0
- app/static/assets/sparkline-chart-DdNNsddM.js +1 -0
- app/static/assets/status-badge-CHHA5X6c.js +1 -0
- app/static/assets/switch-CzxgokcM.js +1 -0
- app/static/assets/table-DXaNSQ0G.js +1 -0
- app/static/assets/time-to-first-token-chart--68L0u_v.js +1 -0
- app/static/assets/tokens-per-day-chart-Cs82BFuE.js +1 -0
- app/static/assets/tokens-per-second-chart-AD2dfOn6.js +1 -0
- app/static/assets/totp-settings-CRHtxQ7j.js +1 -0
- app/static/assets/trash-D3KyWr1h.js +1 -0
- app/static/assets/use-account-quota-display-BPmrTBTC.js +1 -0
- app/static/assets/use-accounts-PWlVGw3z.js +1 -0
- app/static/assets/use-auth-CuDlW3iC.js +1 -0
- app/static/assets/use-chart-colors-DIu6tClI.js +1 -0
- app/static/assets/use-model-sources-Bul24JBu.js +1 -0
- app/static/assets/use-models-DUCPUuZg.js +1 -0
- app/static/assets/use-reduced-motion-hAm5r6_d.js +1 -0
- app/static/assets/useTranslation--56nHhlH.js +1 -0
- app/static/assets/useragent-distribution-donut-Dup6sf_n.js +1 -0
- app/static/assets/users-uIC3TIlE.js +1 -0
- app/static/assets/vendor-query-_bHZuxEx.js +1 -0
- app/static/assets/vendor-react-BVL0YZPW.js +9 -0
- app/static/favicon.svg +10 -0
- app/static/fonts/GeistSans-Variable.woff2 +0 -0
- app/static/fonts/jetbrains-mono-latin-ext.woff2 +0 -0
- app/static/fonts/jetbrains-mono-latin.woff2 +0 -0
- app/static/index.html +29 -0
- codexlb_up-1.25.0b10.dist-info/METADATA +124 -0
- codexlb_up-1.25.0b10.dist-info/RECORD +827 -0
- codexlb_up-1.25.0b10.dist-info/WHEEL +4 -0
- codexlb_up-1.25.0b10.dist-info/entry_points.txt +3 -0
- codexlb_up-1.25.0b10.dist-info/licenses/LICENSE +21 -0
- config/additional_quota_registry.json +11 -0
|
@@ -0,0 +1,3959 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
import asyncio
|
|
4
|
+
import inspect
|
|
5
|
+
import json
|
|
6
|
+
import logging
|
|
7
|
+
import math
|
|
8
|
+
import sys
|
|
9
|
+
import time
|
|
10
|
+
from collections.abc import Callable, Coroutine, Iterable, Sequence
|
|
11
|
+
from dataclasses import dataclass, replace
|
|
12
|
+
from hashlib import sha256
|
|
13
|
+
from ipaddress import ip_address
|
|
14
|
+
from typing import Any, Final, Literal, Mapping, TypeVar, cast
|
|
15
|
+
from urllib.parse import urlparse
|
|
16
|
+
|
|
17
|
+
from app.core import shutdown as shutdown_state
|
|
18
|
+
from app.core.balancer.rendezvous_hash import select_node
|
|
19
|
+
from app.core.clients.files import create_file as core_create_file # noqa: F401
|
|
20
|
+
from app.core.clients.files import finalize_file as core_finalize_file # noqa: F401
|
|
21
|
+
from app.core.clients.proxy import CodexControlResponse as CodexControlResponse
|
|
22
|
+
from app.core.clients.proxy import ( # noqa: F401 # noqa: F401
|
|
23
|
+
ImageFetchSession,
|
|
24
|
+
ProxyResponseError,
|
|
25
|
+
UpstreamProxyRouteTrace,
|
|
26
|
+
_as_image_fetch_session,
|
|
27
|
+
_inline_content_images,
|
|
28
|
+
_inline_input_image_urls,
|
|
29
|
+
_ws_transport_payload_budget_bytes,
|
|
30
|
+
filter_inbound_headers,
|
|
31
|
+
pop_compact_timeout_overrides,
|
|
32
|
+
pop_stream_timeout_overrides,
|
|
33
|
+
pop_transcribe_timeout_overrides,
|
|
34
|
+
push_compact_timeout_overrides,
|
|
35
|
+
push_stream_timeout_overrides,
|
|
36
|
+
push_transcribe_timeout_overrides,
|
|
37
|
+
)
|
|
38
|
+
from app.core.clients.proxy import codex_control_request as core_codex_control_request # noqa: F401
|
|
39
|
+
from app.core.clients.proxy import compact_responses as core_compact_responses # noqa: F401
|
|
40
|
+
from app.core.clients.proxy import transcribe_audio as core_transcribe_audio # noqa: F401
|
|
41
|
+
from app.core.clock import REAL_SCHEDULER, Scheduler, clock_for, scheduler_for
|
|
42
|
+
from app.core.config.settings import Settings, get_settings
|
|
43
|
+
from app.core.config.settings_cache import get_settings_cache
|
|
44
|
+
from app.core.errors import (
|
|
45
|
+
HTTP_BRIDGE_EVENTLESS_TIMEOUT_CODE,
|
|
46
|
+
HTTP_BRIDGE_LOCAL_RESET_MESSAGE,
|
|
47
|
+
OpenAIErrorDetail,
|
|
48
|
+
OpenAIErrorEnvelope,
|
|
49
|
+
OpenAIErrorParam,
|
|
50
|
+
coerce_error_param,
|
|
51
|
+
openai_error,
|
|
52
|
+
previous_response_stream_incomplete_error,
|
|
53
|
+
response_failed_event,
|
|
54
|
+
)
|
|
55
|
+
from app.core.metrics.prometheus import (
|
|
56
|
+
PROMETHEUS_AVAILABLE,
|
|
57
|
+
bridge_drain_recovery_allowed_total,
|
|
58
|
+
bridge_first_turn_timeout_total,
|
|
59
|
+
bridge_handoff_compatibility_rejection_total,
|
|
60
|
+
bridge_instance_mismatch_total,
|
|
61
|
+
bridge_reattach_total,
|
|
62
|
+
bridge_unanchored_handoff_recovery_total,
|
|
63
|
+
http_bridge_connections_total,
|
|
64
|
+
http_bridge_prewarm_total,
|
|
65
|
+
http_bridge_stuck_retire_total,
|
|
66
|
+
)
|
|
67
|
+
from app.core.openai.models import OpenAIEvent
|
|
68
|
+
from app.core.openai.parsing import parse_sse_event
|
|
69
|
+
from app.core.openai.requests import (
|
|
70
|
+
ResponsesRequest,
|
|
71
|
+
)
|
|
72
|
+
from app.core.resilience.overload import local_overload_error
|
|
73
|
+
from app.core.types import JsonValue
|
|
74
|
+
from app.core.utils.request_id import get_request_id
|
|
75
|
+
from app.core.utils.shared_future import _await_task_deferring_cancellation, wait_on_shared_future
|
|
76
|
+
from app.core.utils.sse import format_sse_event, parse_sse_data_json
|
|
77
|
+
from app.core.utils.time import to_utc_naive, utcnow
|
|
78
|
+
from app.db.models import (
|
|
79
|
+
AccountStatus,
|
|
80
|
+
DashboardSettings,
|
|
81
|
+
StickySessionKind,
|
|
82
|
+
)
|
|
83
|
+
from app.modules.api_keys.service import (
|
|
84
|
+
ApiKeyData,
|
|
85
|
+
)
|
|
86
|
+
from app.modules.proxy._service.api_key_usage import (
|
|
87
|
+
_API_KEY_RESERVATION_HEARTBEAT_SECONDS as _API_KEY_RESERVATION_HEARTBEAT_SECONDS,
|
|
88
|
+
)
|
|
89
|
+
from app.modules.proxy._service.compact import (
|
|
90
|
+
_service_tier_from_compact_payload as _service_tier_from_compact_payload,
|
|
91
|
+
)
|
|
92
|
+
from app.modules.proxy._service.compact import (
|
|
93
|
+
_sticky_key_for_compact_request as _sticky_key_for_compact_request,
|
|
94
|
+
)
|
|
95
|
+
from app.modules.proxy._service.compact import (
|
|
96
|
+
_sticky_key_from_compact_payload as _sticky_key_from_compact_payload,
|
|
97
|
+
)
|
|
98
|
+
from app.modules.proxy._service.http_bridge.protocol import _HTTPBridgeServiceProtocol
|
|
99
|
+
from app.modules.proxy._service.observability import (
|
|
100
|
+
_hash_identifier as _hash_identifier,
|
|
101
|
+
)
|
|
102
|
+
from app.modules.proxy._service.observability import (
|
|
103
|
+
_hash_identifier_or_none as _hash_identifier_or_none,
|
|
104
|
+
)
|
|
105
|
+
from app.modules.proxy._service.observability import (
|
|
106
|
+
_interesting_header_keys as _interesting_header_keys,
|
|
107
|
+
)
|
|
108
|
+
from app.modules.proxy._service.observability import (
|
|
109
|
+
_maybe_log_proxy_request_payload as _maybe_log_proxy_request_payload,
|
|
110
|
+
)
|
|
111
|
+
from app.modules.proxy._service.observability import (
|
|
112
|
+
_maybe_log_proxy_request_shape as _maybe_log_proxy_request_shape,
|
|
113
|
+
)
|
|
114
|
+
from app.modules.proxy._service.observability import (
|
|
115
|
+
_maybe_log_proxy_service_tier_trace as _maybe_log_proxy_service_tier_trace,
|
|
116
|
+
)
|
|
117
|
+
from app.modules.proxy._service.observability import (
|
|
118
|
+
_record_continuity_fail_closed as _record_continuity_fail_closed,
|
|
119
|
+
)
|
|
120
|
+
from app.modules.proxy._service.observability import (
|
|
121
|
+
_record_continuity_owner_resolution as _record_continuity_owner_resolution,
|
|
122
|
+
)
|
|
123
|
+
from app.modules.proxy._service.observability import (
|
|
124
|
+
_record_continuity_replay_rejected as _record_continuity_replay_rejected,
|
|
125
|
+
)
|
|
126
|
+
from app.modules.proxy._service.observability import (
|
|
127
|
+
_summarize_input as _summarize_input,
|
|
128
|
+
)
|
|
129
|
+
from app.modules.proxy._service.observability import (
|
|
130
|
+
_tools_hash as _tools_hash,
|
|
131
|
+
)
|
|
132
|
+
from app.modules.proxy._service.observability import (
|
|
133
|
+
_truncate_identifier as _truncate_identifier,
|
|
134
|
+
)
|
|
135
|
+
from app.modules.proxy._service.support import (
|
|
136
|
+
_HARD_HTTP_BRIDGE_AFFINITY_KINDS, # noqa: F401
|
|
137
|
+
_REQUEST_TRANSPORT_HTTP,
|
|
138
|
+
_WEBSOCKET_FULL_REPLAY_WAIT_POLL_SECONDS, # noqa: F401
|
|
139
|
+
_http_bridge_session_supports_service_tier,
|
|
140
|
+
_HTTPBridgeResponseCreateAttempt,
|
|
141
|
+
_HTTPBridgeRetryCircuitAttemptSelection,
|
|
142
|
+
_HTTPBridgeSession,
|
|
143
|
+
_HTTPBridgeSessionKey,
|
|
144
|
+
_WebSocketRequestState,
|
|
145
|
+
)
|
|
146
|
+
from app.modules.proxy._service.support import (
|
|
147
|
+
_websocket_route_log_kwargs as _websocket_route_log_kwargs,
|
|
148
|
+
)
|
|
149
|
+
from app.modules.proxy._service.warmup import (
|
|
150
|
+
WarmupExecutionData as WarmupExecutionData,
|
|
151
|
+
)
|
|
152
|
+
from app.modules.proxy._service.warmup import (
|
|
153
|
+
WarmupFailedAccountData as WarmupFailedAccountData,
|
|
154
|
+
)
|
|
155
|
+
from app.modules.proxy._service.warmup import (
|
|
156
|
+
WarmupSkippedAccountData as WarmupSkippedAccountData,
|
|
157
|
+
)
|
|
158
|
+
from app.modules.proxy._service.warmup import (
|
|
159
|
+
WarmupSubmittedAccountData as WarmupSubmittedAccountData,
|
|
160
|
+
)
|
|
161
|
+
from app.modules.proxy._service.warmup import (
|
|
162
|
+
_is_warmup_usage_eligible as _is_warmup_usage_eligible,
|
|
163
|
+
)
|
|
164
|
+
from app.modules.proxy._service.warmup import (
|
|
165
|
+
_materialize_warmup_account as _materialize_warmup_account,
|
|
166
|
+
)
|
|
167
|
+
from app.modules.proxy._service.warmup import (
|
|
168
|
+
_snapshot_warmup_account as _snapshot_warmup_account,
|
|
169
|
+
)
|
|
170
|
+
from app.modules.proxy._service.warmup import (
|
|
171
|
+
_WarmupAccountSnapshot as _WarmupAccountSnapshot,
|
|
172
|
+
)
|
|
173
|
+
from app.modules.proxy._service.warmup import (
|
|
174
|
+
_WarmupSubmitResult as _WarmupSubmitResult,
|
|
175
|
+
)
|
|
176
|
+
from app.modules.proxy._service.warmup import (
|
|
177
|
+
_WarmupUsageSnapshot as _WarmupUsageSnapshot,
|
|
178
|
+
)
|
|
179
|
+
from app.modules.proxy.account_cache import is_account_routing_unavailable
|
|
180
|
+
from app.modules.proxy.account_eligibility import reauth_access_token_is_expired
|
|
181
|
+
from app.modules.proxy.affinity import (
|
|
182
|
+
_AffinityPolicy,
|
|
183
|
+
_codex_backend_identity,
|
|
184
|
+
_extract_model_class,
|
|
185
|
+
_sticky_key_from_session_header,
|
|
186
|
+
_sticky_key_from_turn_state_header,
|
|
187
|
+
)
|
|
188
|
+
from app.modules.proxy.continuity import (
|
|
189
|
+
is_http_bridge_account_neutral_replay,
|
|
190
|
+
make_http_bridge_account_neutral_replay_key,
|
|
191
|
+
)
|
|
192
|
+
from app.modules.proxy.durable_bridge_coordinator import (
|
|
193
|
+
DurableBridgeLookup,
|
|
194
|
+
)
|
|
195
|
+
from app.modules.proxy.durable_bridge_repository import (
|
|
196
|
+
DurableBridgeAliasRegistration,
|
|
197
|
+
DurableBridgeAliasRegistrationReceipt,
|
|
198
|
+
)
|
|
199
|
+
from app.modules.proxy.durable_bridge_runtime import http_bridge_owner_process_epoch
|
|
200
|
+
from app.modules.proxy.helpers import (
|
|
201
|
+
_normalize_error_code,
|
|
202
|
+
_parse_openai_error,
|
|
203
|
+
)
|
|
204
|
+
from app.modules.proxy.http_continuation import inferred_http_bridge_key
|
|
205
|
+
from app.modules.proxy.ring_membership import (
|
|
206
|
+
RING_STALE_THRESHOLD_SECONDS,
|
|
207
|
+
RingMembershipService,
|
|
208
|
+
)
|
|
209
|
+
from app.modules.proxy.selection_errors import selection_failure_response
|
|
210
|
+
|
|
211
|
+
logger = logging.getLogger("app.modules.proxy.service")
|
|
212
|
+
_TASK_CANCEL_TIMEOUT_SECONDS = 1.0
|
|
213
|
+
_TaskResultT = TypeVar("_TaskResultT")
|
|
214
|
+
_HTTP_BRIDGE_PENDING_COUNT_WARNING_INTERVAL_SECONDS = 60.0
|
|
215
|
+
_http_bridge_pending_count_warning_last_logged: dict[tuple[str, str, str], float] = {}
|
|
216
|
+
_HTTP_BRIDGE_BACKGROUND_CLOSE_TIMEOUT_SECONDS = 5.0
|
|
217
|
+
# A healthy upstream acknowledges response.create promptly. Keep the
|
|
218
|
+
# Keep the owner-side watchdog within the client-safe contract while honoring
|
|
219
|
+
# the fixed stuck-gate threshold when it is shorter.
|
|
220
|
+
_HTTP_BRIDGE_EVENTLESS_RESPONSE_CREATED_MAX_SECONDS = 60.0
|
|
221
|
+
# Fixed bridge session lifecycle values (constantize-session-bridge-tunables).
|
|
222
|
+
# These were never tuned in any deployment; tests monkeypatch the module
|
|
223
|
+
# attribute, and every consumer reads it through this module at call time.
|
|
224
|
+
HTTP_BRIDGE_IDLE_TTL_SECONDS: Final = 120.0
|
|
225
|
+
HTTP_BRIDGE_CODEX_IDLE_TTL_SECONDS: Final = 900.0
|
|
226
|
+
# Owner-side stuck handoff gate; anchored to the 300s wait Codex Desktop uses.
|
|
227
|
+
HTTP_BRIDGE_STUCK_GATE_RETIRE_AFTER_SECONDS: Final = 300.0
|
|
228
|
+
_HTTP_BRIDGE_MISSING_RESPONSE_CREATED_TIMEOUT_DETAIL = "missing_response_created_timeout"
|
|
229
|
+
# Keep process-local *uncaptured* entries bounded. A denied entry is retained
|
|
230
|
+
# until the matching durable anchor is confirmed cleared; evicting it would let
|
|
231
|
+
# a stale durable row regain generation zero after enough unrelated denials.
|
|
232
|
+
# This is intentionally a correctness bound, not an unconditional memory cap.
|
|
233
|
+
_HTTP_BRIDGE_DENIED_ANCHOR_FENCE_MAX_IDS = 512
|
|
234
|
+
|
|
235
|
+
|
|
236
|
+
@dataclass
|
|
237
|
+
class _HTTPBridgeDeniedAnchorFence:
|
|
238
|
+
generation: int
|
|
239
|
+
active_request_ids: set[str]
|
|
240
|
+
owner_key: str | None = None
|
|
241
|
+
owner_epoch: int | None = None
|
|
242
|
+
superseded: bool = False
|
|
243
|
+
# A stale predecessor may not own the current durable slot, so its
|
|
244
|
+
# positive denial tombstone must survive request-pin release until a
|
|
245
|
+
# current owner confirms that durable anchor is gone.
|
|
246
|
+
retain_until_durable_clear: bool = False
|
|
247
|
+
|
|
248
|
+
|
|
249
|
+
def _http_bridge_denied_anchor_fence_entry(
|
|
250
|
+
service: Any,
|
|
251
|
+
response_id: str,
|
|
252
|
+
*,
|
|
253
|
+
create: bool = False,
|
|
254
|
+
) -> _HTTPBridgeDeniedAnchorFence | None:
|
|
255
|
+
"""Read process-local denial state, tolerating older service doubles."""
|
|
256
|
+
fences = getattr(service, "_http_bridge_denied_anchor_fences", None)
|
|
257
|
+
if not isinstance(fences, dict):
|
|
258
|
+
if not create:
|
|
259
|
+
return None
|
|
260
|
+
fences = {}
|
|
261
|
+
setattr(service, "_http_bridge_denied_anchor_fences", fences)
|
|
262
|
+
entry = fences.get(response_id)
|
|
263
|
+
if isinstance(entry, _HTTPBridgeDeniedAnchorFence):
|
|
264
|
+
return entry
|
|
265
|
+
if isinstance(entry, int):
|
|
266
|
+
upgraded = _HTTPBridgeDeniedAnchorFence(entry, set())
|
|
267
|
+
fences[response_id] = upgraded
|
|
268
|
+
return upgraded
|
|
269
|
+
if not create:
|
|
270
|
+
return None
|
|
271
|
+
entry = _HTTPBridgeDeniedAnchorFence(0, set())
|
|
272
|
+
fences[response_id] = entry
|
|
273
|
+
return entry
|
|
274
|
+
|
|
275
|
+
|
|
276
|
+
def _http_bridge_denied_anchor_fence_current_map(service: Any) -> dict[str, str]:
|
|
277
|
+
"""Read/create the owner-to-current-denial map used to bound fence state."""
|
|
278
|
+
current = getattr(service, "_http_bridge_denied_anchor_fence_current", None)
|
|
279
|
+
if not isinstance(current, dict):
|
|
280
|
+
current = {}
|
|
281
|
+
setattr(service, "_http_bridge_denied_anchor_fence_current", current)
|
|
282
|
+
return current
|
|
283
|
+
|
|
284
|
+
|
|
285
|
+
def _prune_http_bridge_denied_anchor_fences(service: Any) -> None:
|
|
286
|
+
"""Drop idle, non-denied entries without evicting a live denial fence."""
|
|
287
|
+
fences = getattr(service, "_http_bridge_denied_anchor_fences", None)
|
|
288
|
+
if not isinstance(fences, dict):
|
|
289
|
+
return
|
|
290
|
+
while len(fences) > _HTTP_BRIDGE_DENIED_ANCHOR_FENCE_MAX_IDS:
|
|
291
|
+
idle_entries = [
|
|
292
|
+
(response_id, entry)
|
|
293
|
+
for response_id, entry in fences.items()
|
|
294
|
+
if (
|
|
295
|
+
isinstance(entry, _HTTPBridgeDeniedAnchorFence)
|
|
296
|
+
and entry.generation == 0
|
|
297
|
+
and not entry.active_request_ids
|
|
298
|
+
)
|
|
299
|
+
]
|
|
300
|
+
if not idle_entries:
|
|
301
|
+
return
|
|
302
|
+
oldest_response_id, _oldest_entry = min(idle_entries, key=lambda item: item[1].generation)
|
|
303
|
+
fences.pop(oldest_response_id, None)
|
|
304
|
+
|
|
305
|
+
|
|
306
|
+
def _drop_older_unpinned_http_bridge_denial_predecessors(
|
|
307
|
+
service: Any,
|
|
308
|
+
owner_key: str,
|
|
309
|
+
*,
|
|
310
|
+
response_id: str,
|
|
311
|
+
) -> None:
|
|
312
|
+
"""Keep only the newest unpinned stale predecessor for one owner.
|
|
313
|
+
|
|
314
|
+
A detached predecessor can publish after a successor has taken the same
|
|
315
|
+
durable owner. Its denial still needs a positive fence while a prepared
|
|
316
|
+
request pins that response id, but retaining every late predecessor would
|
|
317
|
+
make the process-local ledger grow once per owner epoch. Older entries
|
|
318
|
+
without request pins are superseded by the newest predecessor and can be
|
|
319
|
+
dropped. Pinned entries are marked superseded and remain only until their
|
|
320
|
+
request releases the pin.
|
|
321
|
+
"""
|
|
322
|
+
fences = getattr(service, "_http_bridge_denied_anchor_fences", None)
|
|
323
|
+
if not isinstance(fences, dict):
|
|
324
|
+
return
|
|
325
|
+
for predecessor_response_id, entry in tuple(fences.items()):
|
|
326
|
+
if predecessor_response_id == response_id:
|
|
327
|
+
continue
|
|
328
|
+
if not isinstance(entry, _HTTPBridgeDeniedAnchorFence):
|
|
329
|
+
continue
|
|
330
|
+
if entry.owner_key != owner_key or not entry.retain_until_durable_clear:
|
|
331
|
+
continue
|
|
332
|
+
if entry.active_request_ids:
|
|
333
|
+
entry.superseded = True
|
|
334
|
+
entry.retain_until_durable_clear = False
|
|
335
|
+
else:
|
|
336
|
+
fences.pop(predecessor_response_id, None)
|
|
337
|
+
|
|
338
|
+
|
|
339
|
+
def _retire_http_bridge_denied_anchor_predecessors_after_durable_clear(
|
|
340
|
+
service: Any,
|
|
341
|
+
owner_key: str,
|
|
342
|
+
*,
|
|
343
|
+
owner_epoch: int | None,
|
|
344
|
+
preserve_response_ids: Sequence[str] = (),
|
|
345
|
+
) -> None:
|
|
346
|
+
"""Retire stale predecessor fences after a newer owner clears its anchor.
|
|
347
|
+
|
|
348
|
+
A successful clear under the current owner proves that older response ids
|
|
349
|
+
are no longer the durable latest anchor. Drop unpinned predecessor slots
|
|
350
|
+
and mark pinned slots superseded so their final release removes them. The
|
|
351
|
+
explicit preserve set is used by session close while an unresolved cleanup
|
|
352
|
+
still needs its fence.
|
|
353
|
+
"""
|
|
354
|
+
if owner_epoch is None:
|
|
355
|
+
return
|
|
356
|
+
fences = getattr(service, "_http_bridge_denied_anchor_fences", None)
|
|
357
|
+
if not isinstance(fences, dict):
|
|
358
|
+
return
|
|
359
|
+
preserved = set(preserve_response_ids)
|
|
360
|
+
for response_id, entry in tuple(fences.items()):
|
|
361
|
+
if response_id in preserved or not isinstance(entry, _HTTPBridgeDeniedAnchorFence):
|
|
362
|
+
continue
|
|
363
|
+
if (
|
|
364
|
+
entry.owner_key != owner_key
|
|
365
|
+
or not entry.retain_until_durable_clear
|
|
366
|
+
or entry.owner_epoch is None
|
|
367
|
+
or entry.owner_epoch >= owner_epoch
|
|
368
|
+
):
|
|
369
|
+
continue
|
|
370
|
+
if entry.active_request_ids:
|
|
371
|
+
entry.superseded = True
|
|
372
|
+
entry.retain_until_durable_clear = False
|
|
373
|
+
else:
|
|
374
|
+
fences.pop(response_id, None)
|
|
375
|
+
_prune_http_bridge_denied_anchor_fences(service)
|
|
376
|
+
|
|
377
|
+
|
|
378
|
+
def _schedule_http_bridge_background_cleanup(
|
|
379
|
+
service: Any,
|
|
380
|
+
awaitable: Coroutine[Any, Any, Any],
|
|
381
|
+
*,
|
|
382
|
+
name: str,
|
|
383
|
+
error_message: str,
|
|
384
|
+
attribute: tuple[str, Any] | None = None,
|
|
385
|
+
) -> asyncio.Task[Any] | None:
|
|
386
|
+
"""Track one best-effort cleanup task until it settles.
|
|
387
|
+
|
|
388
|
+
Denied-anchor retries and recovery-settlement retries have the same
|
|
389
|
+
ownership contract: callers must retain the task in the service registry,
|
|
390
|
+
remove it exactly once on completion, and consume failures so a transient
|
|
391
|
+
bookkeeping error cannot become an unhandled task exception.
|
|
392
|
+
"""
|
|
393
|
+
cleanup_tasks = getattr(service, "_background_cleanup_tasks", None)
|
|
394
|
+
if cleanup_tasks is None:
|
|
395
|
+
if inspect.iscoroutine(awaitable):
|
|
396
|
+
awaitable.close()
|
|
397
|
+
return None
|
|
398
|
+
task = scheduler_for(service).create_task(awaitable, name=name)
|
|
399
|
+
if attribute is not None:
|
|
400
|
+
setattr(task, attribute[0], attribute[1])
|
|
401
|
+
cleanup_tasks.add(task)
|
|
402
|
+
|
|
403
|
+
def _discard(done_task: asyncio.Task[Any]) -> None:
|
|
404
|
+
cleanup_tasks.discard(done_task)
|
|
405
|
+
try:
|
|
406
|
+
done_task.result()
|
|
407
|
+
except asyncio.CancelledError:
|
|
408
|
+
pass
|
|
409
|
+
except Exception:
|
|
410
|
+
logger.error(error_message, exc_info=True)
|
|
411
|
+
|
|
412
|
+
task.add_done_callback(_discard)
|
|
413
|
+
return task
|
|
414
|
+
|
|
415
|
+
|
|
416
|
+
def _http_bridge_denied_anchor_fence_generation(service: Any, response_id: str) -> int:
|
|
417
|
+
entry = _http_bridge_denied_anchor_fence_entry(service, response_id)
|
|
418
|
+
return entry.generation if entry is not None else 0
|
|
419
|
+
|
|
420
|
+
|
|
421
|
+
def _http_bridge_denied_anchor_fence_was_recorded(service: Any, response_id: str) -> bool:
|
|
422
|
+
"""Return whether this process has already recorded a denial for an id."""
|
|
423
|
+
entry = _http_bridge_denied_anchor_fence_entry(service, response_id)
|
|
424
|
+
return bool(entry is not None and entry.generation > 0)
|
|
425
|
+
|
|
426
|
+
|
|
427
|
+
def _retain_http_bridge_denied_anchor_fence(
|
|
428
|
+
service: Any,
|
|
429
|
+
response_id: str,
|
|
430
|
+
request_id: str,
|
|
431
|
+
) -> int:
|
|
432
|
+
"""Pin a response-id fence for the lifetime of one bridge request."""
|
|
433
|
+
entry = _http_bridge_denied_anchor_fence_entry(service, response_id, create=True)
|
|
434
|
+
assert entry is not None
|
|
435
|
+
entry.active_request_ids.add(request_id)
|
|
436
|
+
_prune_http_bridge_denied_anchor_fences(service)
|
|
437
|
+
return entry.generation
|
|
438
|
+
|
|
439
|
+
|
|
440
|
+
def _capture_http_bridge_denied_anchor_fence(
|
|
441
|
+
service: Any,
|
|
442
|
+
response_id: str,
|
|
443
|
+
request_id: str,
|
|
444
|
+
) -> tuple[int, bool]:
|
|
445
|
+
"""Capture one anchor's generation and whether its denial is already known."""
|
|
446
|
+
generation = _retain_http_bridge_denied_anchor_fence(service, response_id, request_id)
|
|
447
|
+
return generation, _http_bridge_denied_anchor_fence_was_recorded(service, response_id)
|
|
448
|
+
|
|
449
|
+
|
|
450
|
+
def _bind_http_bridge_proxy_injected_anchor(
|
|
451
|
+
service: Any,
|
|
452
|
+
request_state: _WebSocketRequestState,
|
|
453
|
+
*,
|
|
454
|
+
response_id: str | None,
|
|
455
|
+
proxy_injected: bool = True,
|
|
456
|
+
fence_request_id: str | None = None,
|
|
457
|
+
) -> None:
|
|
458
|
+
"""Bind denial provenance to the response id currently carried by a request."""
|
|
459
|
+
previous_fence_request_id = request_state.denied_proxy_injected_anchor_fence_request_id
|
|
460
|
+
_release_http_bridge_denied_anchor_fences(service, request_state.request_id)
|
|
461
|
+
if previous_fence_request_id is not None and previous_fence_request_id != request_state.request_id:
|
|
462
|
+
_release_http_bridge_denied_anchor_fences(service, previous_fence_request_id)
|
|
463
|
+
request_state.previous_response_id = response_id
|
|
464
|
+
request_state.proxy_injected_previous_response_id = proxy_injected and response_id is not None
|
|
465
|
+
request_state.denied_proxy_injected_anchor_fence_response_id = (
|
|
466
|
+
response_id if request_state.proxy_injected_previous_response_id else None
|
|
467
|
+
)
|
|
468
|
+
effective_fence_request_id = fence_request_id or previous_fence_request_id or request_state.request_id
|
|
469
|
+
request_state.denied_proxy_injected_anchor_fence_request_id = (
|
|
470
|
+
effective_fence_request_id if request_state.proxy_injected_previous_response_id else None
|
|
471
|
+
)
|
|
472
|
+
request_state.denied_proxy_injected_anchor_fence_generation_at_prepare = None
|
|
473
|
+
request_state.denied_proxy_injected_anchor_fence_was_already_denied = False
|
|
474
|
+
if request_state.proxy_injected_previous_response_id:
|
|
475
|
+
assert response_id is not None
|
|
476
|
+
(
|
|
477
|
+
request_state.denied_proxy_injected_anchor_fence_generation_at_prepare,
|
|
478
|
+
request_state.denied_proxy_injected_anchor_fence_was_already_denied,
|
|
479
|
+
) = _capture_http_bridge_denied_anchor_fence(
|
|
480
|
+
service,
|
|
481
|
+
response_id,
|
|
482
|
+
effective_fence_request_id,
|
|
483
|
+
)
|
|
484
|
+
|
|
485
|
+
|
|
486
|
+
def _forget_http_bridge_denied_anchor_fence(
|
|
487
|
+
service: Any,
|
|
488
|
+
response_id: str,
|
|
489
|
+
*,
|
|
490
|
+
owner_key: str | None = None,
|
|
491
|
+
owner_epoch: int | None = None,
|
|
492
|
+
) -> bool:
|
|
493
|
+
"""Release a denial tombstone once durable cleanup has been confirmed."""
|
|
494
|
+
fences = getattr(service, "_http_bridge_denied_anchor_fences", None)
|
|
495
|
+
if not isinstance(fences, dict):
|
|
496
|
+
return False
|
|
497
|
+
entry = fences.get(response_id)
|
|
498
|
+
if not isinstance(entry, _HTTPBridgeDeniedAnchorFence):
|
|
499
|
+
return False
|
|
500
|
+
if owner_key is not None and entry.owner_key != owner_key:
|
|
501
|
+
return False
|
|
502
|
+
if owner_epoch is not None and entry.owner_epoch != owner_epoch:
|
|
503
|
+
return False
|
|
504
|
+
current = _http_bridge_denied_anchor_fence_current_map(service)
|
|
505
|
+
if entry.owner_key is not None and current.get(entry.owner_key) == response_id:
|
|
506
|
+
current.pop(entry.owner_key, None)
|
|
507
|
+
# Durable cleanup is the confirmation that makes a positive tombstone
|
|
508
|
+
# dispensable. Clear stale-predecessor retention before waiting for a
|
|
509
|
+
# pinned request's final release.
|
|
510
|
+
entry.retain_until_durable_clear = False
|
|
511
|
+
if entry.active_request_ids:
|
|
512
|
+
# A request prepared before the denial must remain fenced until its
|
|
513
|
+
# final pin is released, even after durable cleanup succeeds.
|
|
514
|
+
entry.superseded = True
|
|
515
|
+
else:
|
|
516
|
+
fences.pop(response_id, None)
|
|
517
|
+
if entry.owner_key is not None:
|
|
518
|
+
_retire_http_bridge_denied_anchor_predecessors_after_durable_clear(
|
|
519
|
+
service,
|
|
520
|
+
entry.owner_key,
|
|
521
|
+
owner_epoch=entry.owner_epoch,
|
|
522
|
+
)
|
|
523
|
+
_prune_http_bridge_denied_anchor_fences(service)
|
|
524
|
+
return True
|
|
525
|
+
|
|
526
|
+
|
|
527
|
+
def _forget_http_bridge_denied_anchor_fence_owner(
|
|
528
|
+
service: Any,
|
|
529
|
+
owner_key: str,
|
|
530
|
+
*,
|
|
531
|
+
owner_epoch: int | None = None,
|
|
532
|
+
preserve_response_ids: Sequence[str] = (),
|
|
533
|
+
) -> None:
|
|
534
|
+
"""Drop unpinned fence slots for a closed or superseded owner epoch."""
|
|
535
|
+
fences = getattr(service, "_http_bridge_denied_anchor_fences", None)
|
|
536
|
+
if not isinstance(fences, dict):
|
|
537
|
+
return
|
|
538
|
+
for response_id, entry in tuple(fences.items()):
|
|
539
|
+
if not isinstance(entry, _HTTPBridgeDeniedAnchorFence) or entry.owner_key != owner_key:
|
|
540
|
+
continue
|
|
541
|
+
if response_id in preserve_response_ids:
|
|
542
|
+
continue
|
|
543
|
+
if owner_epoch is not None and entry.owner_epoch not in (None, owner_epoch):
|
|
544
|
+
continue
|
|
545
|
+
_forget_http_bridge_denied_anchor_fence(
|
|
546
|
+
service,
|
|
547
|
+
response_id,
|
|
548
|
+
owner_key=owner_key,
|
|
549
|
+
owner_epoch=entry.owner_epoch,
|
|
550
|
+
)
|
|
551
|
+
|
|
552
|
+
|
|
553
|
+
def _release_http_bridge_denied_anchor_fences(service: Any, request_id: str) -> None:
|
|
554
|
+
"""Release all response-id fences captured by a completed bridge request."""
|
|
555
|
+
fences = getattr(service, "_http_bridge_denied_anchor_fences", None)
|
|
556
|
+
if not isinstance(fences, dict):
|
|
557
|
+
return
|
|
558
|
+
for response_id, entry in list(fences.items()):
|
|
559
|
+
if not isinstance(entry, _HTTPBridgeDeniedAnchorFence):
|
|
560
|
+
continue
|
|
561
|
+
entry.active_request_ids.discard(request_id)
|
|
562
|
+
if (
|
|
563
|
+
entry.generation == 0 or (entry.superseded and not entry.retain_until_durable_clear)
|
|
564
|
+
) and not entry.active_request_ids:
|
|
565
|
+
fences.pop(response_id, None)
|
|
566
|
+
_prune_http_bridge_denied_anchor_fences(service)
|
|
567
|
+
|
|
568
|
+
|
|
569
|
+
def _record_http_bridge_denied_anchor_fence(
|
|
570
|
+
service: Any,
|
|
571
|
+
response_id: str,
|
|
572
|
+
*,
|
|
573
|
+
owner_key: str | None = None,
|
|
574
|
+
owner_epoch: int | None = None,
|
|
575
|
+
) -> int:
|
|
576
|
+
"""Record a denial after a request may have captured the old generation."""
|
|
577
|
+
current: dict[str, str] | None = None
|
|
578
|
+
if owner_key is not None:
|
|
579
|
+
current = _http_bridge_denied_anchor_fence_current_map(service)
|
|
580
|
+
fences = getattr(service, "_http_bridge_denied_anchor_fences", None)
|
|
581
|
+
prior_response_id = current.get(owner_key) if owner_key is not None and current is not None else None
|
|
582
|
+
existing_entry = fences.get(response_id) if isinstance(fences, dict) else None
|
|
583
|
+
stale_owner_publication = False
|
|
584
|
+
if (
|
|
585
|
+
isinstance(existing_entry, _HTTPBridgeDeniedAnchorFence)
|
|
586
|
+
and existing_entry.owner_key is not None
|
|
587
|
+
and not existing_entry.owner_key.startswith("local:")
|
|
588
|
+
and existing_entry.owner_epoch is not None
|
|
589
|
+
and (owner_key is None or owner_key.startswith("local:"))
|
|
590
|
+
and owner_epoch is None
|
|
591
|
+
# A local/ownerless predecessor cannot replace a durable successor's
|
|
592
|
+
# denial for the same anchor. Keep the successor's owner mapping and
|
|
593
|
+
# generation authoritative; the late predecessor has no durable lease
|
|
594
|
+
# that could justify rewriting it.
|
|
595
|
+
):
|
|
596
|
+
return existing_entry.generation
|
|
597
|
+
if (
|
|
598
|
+
isinstance(existing_entry, _HTTPBridgeDeniedAnchorFence)
|
|
599
|
+
and existing_entry.owner_key == owner_key
|
|
600
|
+
and owner_epoch is not None
|
|
601
|
+
and existing_entry.owner_epoch is not None
|
|
602
|
+
and owner_epoch < existing_entry.owner_epoch
|
|
603
|
+
):
|
|
604
|
+
# A detached predecessor can publish after a successor has already
|
|
605
|
+
# claimed the same durable owner at a newer epoch. The stale
|
|
606
|
+
# publication for this response was already recorded by the newer
|
|
607
|
+
# epoch; return its generation without rolling that entry back.
|
|
608
|
+
return existing_entry.generation
|
|
609
|
+
if prior_response_id is not None and prior_response_id != response_id and isinstance(fences, dict):
|
|
610
|
+
prior_entry = fences.get(prior_response_id)
|
|
611
|
+
if (
|
|
612
|
+
isinstance(prior_entry, _HTTPBridgeDeniedAnchorFence)
|
|
613
|
+
and owner_epoch is not None
|
|
614
|
+
and prior_entry.owner_epoch is not None
|
|
615
|
+
and owner_epoch < prior_entry.owner_epoch
|
|
616
|
+
):
|
|
617
|
+
# The current owner slot belongs to a newer epoch. Ignore this
|
|
618
|
+
# stale publication rather than superseding the successor fence.
|
|
619
|
+
# A pinned predecessor capture still needs a positive generation,
|
|
620
|
+
# though, so a prepared request cannot resend its denied anchor.
|
|
621
|
+
# Keep the positive tombstone even when no request is pinned: a
|
|
622
|
+
# later recapture of a stale durable row must observe this denial.
|
|
623
|
+
stale_owner_publication = True
|
|
624
|
+
if stale_owner_publication and owner_key is not None and owner_epoch is not None:
|
|
625
|
+
_drop_older_unpinned_http_bridge_denial_predecessors(
|
|
626
|
+
service,
|
|
627
|
+
owner_key,
|
|
628
|
+
response_id=response_id,
|
|
629
|
+
)
|
|
630
|
+
if (
|
|
631
|
+
not stale_owner_publication
|
|
632
|
+
and isinstance(fences, dict)
|
|
633
|
+
and prior_response_id is not None
|
|
634
|
+
and prior_response_id != response_id
|
|
635
|
+
):
|
|
636
|
+
prior_entry = fences.get(prior_response_id)
|
|
637
|
+
if isinstance(prior_entry, _HTTPBridgeDeniedAnchorFence):
|
|
638
|
+
prior_entry.superseded = True
|
|
639
|
+
if not prior_entry.active_request_ids:
|
|
640
|
+
fences.pop(prior_response_id, None)
|
|
641
|
+
if owner_key is not None and current is not None:
|
|
642
|
+
# A session can first deny an anchor while it has only a process-local
|
|
643
|
+
# owner and later rebind that same denial to its durable owner. The
|
|
644
|
+
# fence entry has one authoritative owner, so retire any stale alias
|
|
645
|
+
# for the same response before publishing the new owner mapping.
|
|
646
|
+
for prior_owner_key, prior_response_id_for_owner in tuple(current.items()):
|
|
647
|
+
if prior_owner_key != owner_key and prior_response_id_for_owner == response_id:
|
|
648
|
+
current.pop(prior_owner_key, None)
|
|
649
|
+
if owner_key is not None and current is not None and not stale_owner_publication:
|
|
650
|
+
current[owner_key] = response_id
|
|
651
|
+
entry = _http_bridge_denied_anchor_fence_entry(service, response_id, create=True)
|
|
652
|
+
assert entry is not None
|
|
653
|
+
generation = getattr(service, "_http_bridge_denied_anchor_fence_generation", 0)
|
|
654
|
+
if not isinstance(generation, int):
|
|
655
|
+
generation = 0
|
|
656
|
+
generation += 1
|
|
657
|
+
setattr(service, "_http_bridge_denied_anchor_fence_generation", generation)
|
|
658
|
+
entry.generation = generation
|
|
659
|
+
entry.owner_key = owner_key
|
|
660
|
+
entry.owner_epoch = owner_epoch
|
|
661
|
+
# Keep a stale predecessor generation until a current owner confirms the
|
|
662
|
+
# durable anchor is gone; finalization may release its request pin without
|
|
663
|
+
# dropping this non-current owner entry prematurely.
|
|
664
|
+
entry.superseded = stale_owner_publication
|
|
665
|
+
entry.retain_until_durable_clear = stale_owner_publication
|
|
666
|
+
_prune_http_bridge_denied_anchor_fences(service)
|
|
667
|
+
return generation
|
|
668
|
+
|
|
669
|
+
|
|
670
|
+
def _http_bridge_denied_anchor_fence_advanced(
|
|
671
|
+
service: Any,
|
|
672
|
+
request_state: _WebSocketRequestState,
|
|
673
|
+
) -> bool:
|
|
674
|
+
response_id = request_state.previous_response_id
|
|
675
|
+
captured_generation = request_state.denied_proxy_injected_anchor_fence_generation_at_prepare
|
|
676
|
+
return bool(
|
|
677
|
+
request_state.proxy_injected_previous_response_id
|
|
678
|
+
and response_id is not None
|
|
679
|
+
and request_state.denied_proxy_injected_anchor_fence_response_id == response_id
|
|
680
|
+
and (
|
|
681
|
+
request_state.denied_proxy_injected_anchor_fence_was_already_denied
|
|
682
|
+
or (
|
|
683
|
+
captured_generation is not None
|
|
684
|
+
and captured_generation < _http_bridge_denied_anchor_fence_generation(service, response_id)
|
|
685
|
+
)
|
|
686
|
+
)
|
|
687
|
+
)
|
|
688
|
+
|
|
689
|
+
|
|
690
|
+
# Silence before ``response.created`` is a different failure class than a
|
|
691
|
+
# stream that started and then went quiet. When no unmatched upstream liveness
|
|
692
|
+
# was observed, nothing exists upstream yet, so the request is safe to retry and
|
|
693
|
+
# the upstream is not proven at fault. Keep it out of ``stream_idle_timeout``,
|
|
694
|
+
# whose budget is the post-start ``stream_idle_timeout_seconds``.
|
|
695
|
+
_HTTP_BRIDGE_EVENTLESS_TIMEOUT_DETAIL = HTTP_BRIDGE_EVENTLESS_TIMEOUT_CODE
|
|
696
|
+
_HTTP_BRIDGE_EVENTLESS_TIMEOUT_MESSAGE = (
|
|
697
|
+
"HTTP responses session bridge saw no response events before its pre-response budget expired; "
|
|
698
|
+
"no response was created upstream, so the request is safe to retry"
|
|
699
|
+
)
|
|
700
|
+
_HTTP_BRIDGE_EVENTLESS_TIMEOUT_UNMATCHED_LIVENESS_MESSAGE = (
|
|
701
|
+
"HTTP responses session bridge saw upstream liveness before its pre-response budget expired, "
|
|
702
|
+
"but no response events were matched; retry may duplicate upstream work"
|
|
703
|
+
)
|
|
704
|
+
_HTTP_BRIDGE_EVENTLESS_COOLDOWN_MESSAGE = (
|
|
705
|
+
"HTTP responses session bridge is cooling down after repeated upstream timeouts; retry shortly."
|
|
706
|
+
)
|
|
707
|
+
# Local bridge recovery closes and rebuilds our own upstream session. Reserve
|
|
708
|
+
# upstream-close wording for frames the upstream actually sent.
|
|
709
|
+
_HTTP_BRIDGE_LOCAL_RESET_MESSAGE = HTTP_BRIDGE_LOCAL_RESET_MESSAGE
|
|
710
|
+
T = TypeVar("T")
|
|
711
|
+
|
|
712
|
+
_HTTP_BRIDGE_INFLIGHT_STARTED_AT_ATTR = "_codex_lb_started_at"
|
|
713
|
+
# Provenance marker for bridge failures raised strictly before the current
|
|
714
|
+
# request dispatched upstream. Only exceptions carrying this attribute are
|
|
715
|
+
# safe for the streaming wrapper's raw-HTTP replay.
|
|
716
|
+
_HTTP_BRIDGE_PRE_SUBMIT_FAILURE_ATTR = "http_bridge_pre_submit_failure"
|
|
717
|
+
# Provenance for a bridge failure whose prepared payload carried a continuity
|
|
718
|
+
# anchor the incoming payload does not, making a raw-HTTP replay of the
|
|
719
|
+
# incoming payload continuity-incomplete.
|
|
720
|
+
_HTTP_BRIDGE_PREPARED_ANCHOR_ATTR = "http_bridge_prepared_continuity_anchor"
|
|
721
|
+
# Provenance for a bridge retry-circuit cooldown suppression proved replay-safe
|
|
722
|
+
# at the pre-dispatch submission gate. It cannot be inferred from the error
|
|
723
|
+
# code: ``_raise_proxy_budget_exhausted`` emits the same
|
|
724
|
+
# ``upstream_request_timeout`` for an ordinary pre-submit budget exhaustion,
|
|
725
|
+
# which is overload or host-network evidence and must not be replayed.
|
|
726
|
+
_HTTP_BRIDGE_COOLDOWN_SUPPRESSION_ATTR = "http_bridge_cooldown_suppression"
|
|
727
|
+
_HTTP_BRIDGE_STALE_INFLIGHT_MIN_SECONDS = 120.0
|
|
728
|
+
# Upper bound for the per-request fail-safe sweep's wait on a detached
|
|
729
|
+
# session's ``pending_lock``. The sweep runs on every bridge request, so a
|
|
730
|
+
# single detached generation whose lock never frees (2026-09-07: an anyio 4.13
|
|
731
|
+
# Lock lost-wakeup left ~100 live request tasks queued behind one detached
|
|
732
|
+
# session) must not park every request; a skipped pass is revisited by the
|
|
733
|
+
# next request's sweep and by the session's own close/drain paths.
|
|
734
|
+
_HTTP_BRIDGE_DETACHED_RETIRE_LOCK_WAIT_SECONDS = 5.0
|
|
735
|
+
_HTTP_BRIDGE_STALE_INFLIGHT_TIMEOUT_MULTIPLIER = 6.0
|
|
736
|
+
|
|
737
|
+
|
|
738
|
+
def _http_bridge_eventless_timeout_message(unmatched_upstream_liveness_count: int) -> str:
|
|
739
|
+
if unmatched_upstream_liveness_count > 0:
|
|
740
|
+
return _HTTP_BRIDGE_EVENTLESS_TIMEOUT_UNMATCHED_LIVENESS_MESSAGE
|
|
741
|
+
return _HTTP_BRIDGE_EVENTLESS_TIMEOUT_MESSAGE
|
|
742
|
+
|
|
743
|
+
|
|
744
|
+
@dataclass(frozen=True, slots=True)
|
|
745
|
+
class _HTTPBridgeRuntimeConfig:
|
|
746
|
+
enabled: bool
|
|
747
|
+
idle_ttl_seconds: float
|
|
748
|
+
codex_idle_ttl_seconds: float
|
|
749
|
+
max_sessions: int
|
|
750
|
+
queue_limit: int
|
|
751
|
+
prompt_cache_idle_ttl_seconds: float
|
|
752
|
+
gateway_safe_mode: bool
|
|
753
|
+
|
|
754
|
+
|
|
755
|
+
def _service_module() -> Any:
|
|
756
|
+
service_module = sys.modules.get("app.modules.proxy.service")
|
|
757
|
+
if service_module is None:
|
|
758
|
+
raise RuntimeError("app.modules.proxy.service is not loaded")
|
|
759
|
+
return service_module
|
|
760
|
+
|
|
761
|
+
|
|
762
|
+
def _service_global(name: str) -> Any:
|
|
763
|
+
return getattr(_service_module(), name)
|
|
764
|
+
|
|
765
|
+
|
|
766
|
+
def _service_global_or(name: str, fallback: T) -> T:
|
|
767
|
+
service_module = sys.modules.get("app.modules.proxy.service")
|
|
768
|
+
if service_module is None:
|
|
769
|
+
return fallback
|
|
770
|
+
return cast(T, getattr(service_module, name, fallback))
|
|
771
|
+
|
|
772
|
+
|
|
773
|
+
def _service_get_settings() -> Any:
|
|
774
|
+
return _service_global_or("get_settings", get_settings)()
|
|
775
|
+
|
|
776
|
+
|
|
777
|
+
def _service_get_settings_cache() -> Any:
|
|
778
|
+
return _service_global_or("get_settings_cache", get_settings_cache)()
|
|
779
|
+
|
|
780
|
+
|
|
781
|
+
def _http_bridge_client_full_history_recovery_error() -> OpenAIErrorEnvelope:
|
|
782
|
+
"""Ask the client to drop the anchor and resend its full local history.
|
|
783
|
+
|
|
784
|
+
Only used where the durable ledger has *proved* the anchored turn was
|
|
785
|
+
abandoned, so the resend cannot duplicate an accepted upstream response.
|
|
786
|
+
"""
|
|
787
|
+
payload = openai_error(
|
|
788
|
+
"previous_response_not_found",
|
|
789
|
+
"Previous response was not found; retry without previous_response_id.",
|
|
790
|
+
error_type="invalid_request_error",
|
|
791
|
+
)
|
|
792
|
+
payload["error"]["param"] = "previous_response_id"
|
|
793
|
+
return payload
|
|
794
|
+
|
|
795
|
+
|
|
796
|
+
def _proxy_admission_wait_timeout_seconds() -> float:
|
|
797
|
+
return cast(Callable[[], float], _service_global("_proxy_admission_wait_timeout_seconds"))()
|
|
798
|
+
|
|
799
|
+
|
|
800
|
+
def _http_bridge_stale_inflight_seconds() -> float:
|
|
801
|
+
try:
|
|
802
|
+
admission_timeout = _proxy_admission_wait_timeout_seconds()
|
|
803
|
+
except Exception:
|
|
804
|
+
admission_timeout = 10.0
|
|
805
|
+
return max(
|
|
806
|
+
_HTTP_BRIDGE_STALE_INFLIGHT_MIN_SECONDS,
|
|
807
|
+
admission_timeout * _HTTP_BRIDGE_STALE_INFLIGHT_TIMEOUT_MULTIPLIER,
|
|
808
|
+
)
|
|
809
|
+
|
|
810
|
+
|
|
811
|
+
def _normalize_responses_request_payload_for_bridge(payload: ResponsesRequest) -> ResponsesRequest:
|
|
812
|
+
return cast(
|
|
813
|
+
Callable[[ResponsesRequest], ResponsesRequest],
|
|
814
|
+
_service_global("_normalize_responses_request_payload_for_bridge"),
|
|
815
|
+
)(payload)
|
|
816
|
+
|
|
817
|
+
|
|
818
|
+
def _websocket_top_level_error_payload(*args: Any, **kwargs: Any) -> Any:
|
|
819
|
+
return _service_global("_websocket_top_level_error_payload")(*args, **kwargs)
|
|
820
|
+
|
|
821
|
+
|
|
822
|
+
def _header_value_case_insensitive(*args: Any, **kwargs: Any) -> Any:
|
|
823
|
+
return _service_global("_header_value_case_insensitive")(*args, **kwargs)
|
|
824
|
+
|
|
825
|
+
|
|
826
|
+
def _is_previous_response_not_found_error(*args: Any, **kwargs: Any) -> Any:
|
|
827
|
+
return _service_global("_is_previous_response_not_found_error")(*args, **kwargs)
|
|
828
|
+
|
|
829
|
+
|
|
830
|
+
def _http_bridge_startup_wait_timeout_error(
|
|
831
|
+
stage: str,
|
|
832
|
+
*,
|
|
833
|
+
code: str = "global_admission_timeout",
|
|
834
|
+
) -> ProxyResponseError:
|
|
835
|
+
message = f"codex-lb is temporarily overloaded during {stage}"
|
|
836
|
+
return ProxyResponseError(429, local_overload_error(message, code=code))
|
|
837
|
+
|
|
838
|
+
|
|
839
|
+
def _http_bridge_pending_count_nowait(
|
|
840
|
+
session: "_HTTPBridgeSession",
|
|
841
|
+
*,
|
|
842
|
+
context: str,
|
|
843
|
+
) -> int | None:
|
|
844
|
+
try:
|
|
845
|
+
session.pending_lock.acquire_nowait()
|
|
846
|
+
except Exception as exc:
|
|
847
|
+
if type(exc).__name__ not in {"WouldBlock", "RuntimeError"}:
|
|
848
|
+
raise
|
|
849
|
+
warning_key = (
|
|
850
|
+
context,
|
|
851
|
+
session.key.affinity_kind,
|
|
852
|
+
_hash_identifier(session.key.affinity_key),
|
|
853
|
+
)
|
|
854
|
+
now = time.monotonic()
|
|
855
|
+
last_logged = _http_bridge_pending_count_warning_last_logged.get(warning_key)
|
|
856
|
+
if last_logged is None or now - last_logged >= _HTTP_BRIDGE_PENDING_COUNT_WARNING_INTERVAL_SECONDS:
|
|
857
|
+
_http_bridge_pending_count_warning_last_logged[warning_key] = now
|
|
858
|
+
if len(_http_bridge_pending_count_warning_last_logged) > 2048:
|
|
859
|
+
cutoff = now - _HTTP_BRIDGE_PENDING_COUNT_WARNING_INTERVAL_SECONDS * 2
|
|
860
|
+
for key, timestamp in tuple(_http_bridge_pending_count_warning_last_logged.items()):
|
|
861
|
+
if timestamp < cutoff:
|
|
862
|
+
_http_bridge_pending_count_warning_last_logged.pop(key, None)
|
|
863
|
+
logger.warning(
|
|
864
|
+
"http_bridge_pending_count_unavailable context=%s bridge_kind=%s bridge_key=%s account_id=%s model=%s",
|
|
865
|
+
context,
|
|
866
|
+
session.key.affinity_kind,
|
|
867
|
+
warning_key[2],
|
|
868
|
+
session.account.id,
|
|
869
|
+
session.request_model,
|
|
870
|
+
)
|
|
871
|
+
return None
|
|
872
|
+
try:
|
|
873
|
+
request_counts_against_queue = _service_global("_http_bridge_request_counts_against_queue")
|
|
874
|
+
visible_pending_count = sum(
|
|
875
|
+
1 for request_state in session.pending_requests if request_counts_against_queue(request_state)
|
|
876
|
+
)
|
|
877
|
+
# A registered admission waiter is live work not yet counted into the
|
|
878
|
+
# queue — it may be suspended on the pre-lock fair-share resolve with
|
|
879
|
+
# pending_lock free (issue #1971). Counting it keeps capacity LRU
|
|
880
|
+
# eviction and shutdown drain from treating the session as idle and
|
|
881
|
+
# closing it under the admitting turn.
|
|
882
|
+
return max(visible_pending_count, session.queued_request_count, session.admission_waiter_count)
|
|
883
|
+
finally:
|
|
884
|
+
session.pending_lock.release()
|
|
885
|
+
|
|
886
|
+
|
|
887
|
+
def _cleanup_http_bridge_inflight_sessions_nowait(service: Any) -> dict[str, int]:
|
|
888
|
+
now = clock_for(service).monotonic()
|
|
889
|
+
stale_after_seconds = _http_bridge_stale_inflight_seconds()
|
|
890
|
+
cleaned = 0
|
|
891
|
+
stale = 0
|
|
892
|
+
oldest_age_seconds = 0
|
|
893
|
+
try:
|
|
894
|
+
service._http_bridge_lock.acquire_nowait()
|
|
895
|
+
except Exception as exc:
|
|
896
|
+
if type(exc).__name__ not in {"WouldBlock", "RuntimeError"}:
|
|
897
|
+
raise
|
|
898
|
+
for future in service._http_bridge_inflight_sessions.values():
|
|
899
|
+
started_at = getattr(future, _HTTP_BRIDGE_INFLIGHT_STARTED_AT_ATTR, None)
|
|
900
|
+
age_seconds = max(0.0, now - started_at) if isinstance(started_at, (int, float)) else 0.0
|
|
901
|
+
oldest_age_seconds = max(oldest_age_seconds, int(age_seconds))
|
|
902
|
+
if isinstance(started_at, (int, float)) and age_seconds >= stale_after_seconds:
|
|
903
|
+
stale += 1
|
|
904
|
+
return {
|
|
905
|
+
"cleaned": 0,
|
|
906
|
+
"stale": stale,
|
|
907
|
+
"oldest_age_seconds": oldest_age_seconds,
|
|
908
|
+
}
|
|
909
|
+
try:
|
|
910
|
+
for key, future in list(service._http_bridge_inflight_sessions.items()):
|
|
911
|
+
current_future = service._http_bridge_inflight_sessions.get(key)
|
|
912
|
+
if current_future is not future:
|
|
913
|
+
continue
|
|
914
|
+
started_at = getattr(future, _HTTP_BRIDGE_INFLIGHT_STARTED_AT_ATTR, None)
|
|
915
|
+
age_seconds = max(0.0, now - started_at) if isinstance(started_at, (int, float)) else 0.0
|
|
916
|
+
oldest_age_seconds = max(oldest_age_seconds, int(age_seconds))
|
|
917
|
+
cleanup_reason: str | None = None
|
|
918
|
+
is_stale = isinstance(started_at, (int, float)) and age_seconds >= stale_after_seconds
|
|
919
|
+
if is_stale:
|
|
920
|
+
stale += 1
|
|
921
|
+
if future.done():
|
|
922
|
+
cleanup_reason = "done"
|
|
923
|
+
if cleanup_reason is None:
|
|
924
|
+
continue
|
|
925
|
+
service._http_bridge_inflight_sessions.pop(key, None)
|
|
926
|
+
cleaned += 1
|
|
927
|
+
if future.done() and not future.cancelled():
|
|
928
|
+
try:
|
|
929
|
+
future.exception()
|
|
930
|
+
except Exception:
|
|
931
|
+
pass
|
|
932
|
+
logger.warning(
|
|
933
|
+
"http_bridge_inflight_session_create_cleanup reason=%s bridge_kind=%s bridge_key=%s"
|
|
934
|
+
" age_seconds=%d stale_after_seconds=%d done=%s cancelled=%s",
|
|
935
|
+
cleanup_reason,
|
|
936
|
+
key.affinity_kind,
|
|
937
|
+
_hash_identifier(key.affinity_key),
|
|
938
|
+
int(age_seconds),
|
|
939
|
+
int(stale_after_seconds),
|
|
940
|
+
future.done(),
|
|
941
|
+
future.cancelled(),
|
|
942
|
+
)
|
|
943
|
+
finally:
|
|
944
|
+
service._http_bridge_lock.release()
|
|
945
|
+
return {
|
|
946
|
+
"cleaned": cleaned,
|
|
947
|
+
"stale": stale,
|
|
948
|
+
"oldest_age_seconds": oldest_age_seconds,
|
|
949
|
+
}
|
|
950
|
+
|
|
951
|
+
|
|
952
|
+
def _http_bridge_inflight_creation_count(service: Any) -> int:
|
|
953
|
+
return sum(
|
|
954
|
+
1
|
|
955
|
+
for future in service._http_bridge_inflight_sessions.values()
|
|
956
|
+
if not getattr(future, "_http_bridge_handoff", False)
|
|
957
|
+
)
|
|
958
|
+
|
|
959
|
+
|
|
960
|
+
def _http_bridge_session_generation_count(service: Any) -> int:
|
|
961
|
+
return len(service._http_bridge_sessions) + len(service._http_bridge_detached_sessions)
|
|
962
|
+
|
|
963
|
+
|
|
964
|
+
def _http_bridge_capacity_generation_count(service: Any) -> int:
|
|
965
|
+
return _http_bridge_session_generation_count(service) + _http_bridge_inflight_creation_count(service)
|
|
966
|
+
|
|
967
|
+
|
|
968
|
+
def _http_bridge_capacity_after_planned_closes(
|
|
969
|
+
service: Any,
|
|
970
|
+
sessions_to_close_before_create: Sequence["_HTTPBridgeSession"],
|
|
971
|
+
) -> int:
|
|
972
|
+
return _http_bridge_capacity_generation_count(service) - len(sessions_to_close_before_create)
|
|
973
|
+
|
|
974
|
+
|
|
975
|
+
def _plan_http_bridge_lru_capacity_closes(
|
|
976
|
+
service: Any,
|
|
977
|
+
*,
|
|
978
|
+
max_sessions: int,
|
|
979
|
+
model_transition_parent_key: "_HTTPBridgeSessionKey | None",
|
|
980
|
+
sessions_to_close_before_create: list["_HTTPBridgeSession"],
|
|
981
|
+
) -> None:
|
|
982
|
+
while (
|
|
983
|
+
_http_bridge_capacity_after_planned_closes(service, sessions_to_close_before_create) >= max_sessions
|
|
984
|
+
and service._http_bridge_sessions
|
|
985
|
+
):
|
|
986
|
+
evictable_sessions: list[tuple[_HTTPBridgeSessionKey, _HTTPBridgeSession]] = []
|
|
987
|
+
for candidate_key, candidate_session in service._http_bridge_sessions.items():
|
|
988
|
+
if candidate_key == model_transition_parent_key:
|
|
989
|
+
continue
|
|
990
|
+
if getattr(candidate_session, "unanchored_reservation_id", None) is not None:
|
|
991
|
+
continue
|
|
992
|
+
pending_count = service._http_bridge_pending_count_nowait(
|
|
993
|
+
candidate_session,
|
|
994
|
+
context="capacity_evict_scan",
|
|
995
|
+
)
|
|
996
|
+
if pending_count is None or pending_count:
|
|
997
|
+
continue
|
|
998
|
+
evictable_sessions.append((candidate_key, candidate_session))
|
|
999
|
+
if not evictable_sessions:
|
|
1000
|
+
break
|
|
1001
|
+
lru_key, lru_session = min(evictable_sessions, key=lambda item: _http_bridge_eviction_priority(item[1]))
|
|
1002
|
+
_log_http_bridge_event(
|
|
1003
|
+
"evict_lru",
|
|
1004
|
+
lru_key,
|
|
1005
|
+
account_id=lru_session.account.id,
|
|
1006
|
+
model=lru_session.request_model,
|
|
1007
|
+
cache_key_family=lru_key.affinity_kind,
|
|
1008
|
+
model_class=_extract_model_class(lru_session.request_model) if lru_session.request_model else None,
|
|
1009
|
+
)
|
|
1010
|
+
detached = service._detach_http_bridge_session_locked(lru_key, expected_session=lru_session)
|
|
1011
|
+
if detached is not None:
|
|
1012
|
+
sessions_to_close_before_create.append(detached)
|
|
1013
|
+
|
|
1014
|
+
|
|
1015
|
+
def http_bridge_activity_snapshot_nowait(service: Any) -> dict[str, int | bool]:
|
|
1016
|
+
inflight_cleanup = _cleanup_http_bridge_inflight_sessions_nowait(service)
|
|
1017
|
+
live_sessions = 0
|
|
1018
|
+
pending_or_queued_requests = 0
|
|
1019
|
+
pending_unknown_sessions = 0
|
|
1020
|
+
|
|
1021
|
+
# A canonical key names only the newest generation. Detached predecessors
|
|
1022
|
+
# remain live work and must still block restart until their requests settle.
|
|
1023
|
+
for session in [
|
|
1024
|
+
*service._http_bridge_sessions.values(),
|
|
1025
|
+
*service._http_bridge_detached_sessions.values(),
|
|
1026
|
+
]:
|
|
1027
|
+
if not session.closed:
|
|
1028
|
+
live_sessions += 1
|
|
1029
|
+
# `closed` fences new admission; it does not prove that a detached
|
|
1030
|
+
# request has finished queue/pending settlement. Drain must observe
|
|
1031
|
+
# that work even after the generation stops counting as a live socket.
|
|
1032
|
+
pending_count = _http_bridge_pending_count_nowait(session, context="drain_status")
|
|
1033
|
+
if pending_count is None:
|
|
1034
|
+
pending_unknown_sessions += 1
|
|
1035
|
+
else:
|
|
1036
|
+
pending_or_queued_requests += max(0, pending_count)
|
|
1037
|
+
|
|
1038
|
+
inflight_session_creates = len(service._http_bridge_inflight_sessions)
|
|
1039
|
+
active_cleanup_tasks = sum(
|
|
1040
|
+
1
|
|
1041
|
+
for task in service._background_cleanup_tasks
|
|
1042
|
+
if not task.done()
|
|
1043
|
+
and (
|
|
1044
|
+
task.get_name().startswith("proxy-http_bridge_session_close-")
|
|
1045
|
+
or task.get_name().startswith("http-bridge-close-")
|
|
1046
|
+
)
|
|
1047
|
+
)
|
|
1048
|
+
bridge_active = (
|
|
1049
|
+
live_sessions > 0
|
|
1050
|
+
or pending_or_queued_requests > 0
|
|
1051
|
+
or pending_unknown_sessions > 0
|
|
1052
|
+
or inflight_session_creates > 0
|
|
1053
|
+
)
|
|
1054
|
+
restart_blocking = pending_or_queued_requests > 0 or pending_unknown_sessions > 0 or inflight_session_creates > 0
|
|
1055
|
+
return {
|
|
1056
|
+
"http_bridge_live_sessions": live_sessions,
|
|
1057
|
+
"http_bridge_pending_or_queued_requests": pending_or_queued_requests,
|
|
1058
|
+
"http_bridge_pending_unknown_sessions": pending_unknown_sessions,
|
|
1059
|
+
"http_bridge_inflight_session_creates": inflight_session_creates,
|
|
1060
|
+
"http_bridge_inflight_session_create_oldest_age_seconds": inflight_cleanup["oldest_age_seconds"],
|
|
1061
|
+
"http_bridge_stale_inflight_session_creates": inflight_cleanup["stale"],
|
|
1062
|
+
"http_bridge_cleaned_inflight_session_creates": inflight_cleanup["cleaned"],
|
|
1063
|
+
"http_bridge_background_cleanup_tasks": active_cleanup_tasks,
|
|
1064
|
+
"http_bridge_active": bridge_active,
|
|
1065
|
+
"http_bridge_restart_blocking": restart_blocking,
|
|
1066
|
+
}
|
|
1067
|
+
|
|
1068
|
+
|
|
1069
|
+
def _log_http_bridge_startup_wait_timeout(
|
|
1070
|
+
*,
|
|
1071
|
+
stage: str,
|
|
1072
|
+
timeout_seconds: float,
|
|
1073
|
+
key: "_HTTPBridgeSessionKey | None" = None,
|
|
1074
|
+
request_id: str | None = None,
|
|
1075
|
+
request_model: str | None = None,
|
|
1076
|
+
pending_count: int | None = None,
|
|
1077
|
+
inflight_count: int | None = None,
|
|
1078
|
+
queued_count: int | None = None,
|
|
1079
|
+
available: int | None = None,
|
|
1080
|
+
pending_request_ids: Sequence[str] | None = None,
|
|
1081
|
+
pending_request_ages_seconds: Sequence[float] | None = None,
|
|
1082
|
+
) -> None:
|
|
1083
|
+
logger.warning(
|
|
1084
|
+
"http_bridge_startup_wait_timeout request_id=%s stage=%s wait_timeout_seconds=%.1f "
|
|
1085
|
+
"affinity_kind=%s bridge_key=%s model_class=%s pending_count=%s queued_count=%s "
|
|
1086
|
+
"inflight_count=%s available=%s pending_request_ids=%s pending_request_ages_seconds=%s",
|
|
1087
|
+
request_id or get_request_id() or "unknown",
|
|
1088
|
+
stage,
|
|
1089
|
+
timeout_seconds,
|
|
1090
|
+
key.affinity_kind if key is not None else None,
|
|
1091
|
+
_hash_identifier(key.affinity_key) if key is not None else None,
|
|
1092
|
+
_extract_model_class(request_model) if request_model else None,
|
|
1093
|
+
pending_count,
|
|
1094
|
+
queued_count,
|
|
1095
|
+
inflight_count,
|
|
1096
|
+
available,
|
|
1097
|
+
",".join(pending_request_ids) if pending_request_ids else None,
|
|
1098
|
+
",".join(f"{age:.1f}" for age in pending_request_ages_seconds) if pending_request_ages_seconds else None,
|
|
1099
|
+
)
|
|
1100
|
+
|
|
1101
|
+
|
|
1102
|
+
def _http_bridge_precreated_retry_failure_error(
|
|
1103
|
+
exc: BaseException,
|
|
1104
|
+
) -> tuple[int, str, str, str, OpenAIErrorParam | None]:
|
|
1105
|
+
if isinstance(exc, ProxyResponseError):
|
|
1106
|
+
parsed = _parse_openai_error(exc.payload)
|
|
1107
|
+
code = _normalize_error_code(parsed.code if parsed else None, parsed.type if parsed else None)
|
|
1108
|
+
message = parsed.message if parsed and parsed.message else "HTTP bridge pre-created retry failed"
|
|
1109
|
+
error_type = parsed.type if parsed and parsed.type else "server_error"
|
|
1110
|
+
error_param = parsed.param_state if parsed else None
|
|
1111
|
+
return exc.status_code, code, message, error_type, error_param
|
|
1112
|
+
if isinstance(exc, TimeoutError):
|
|
1113
|
+
return (
|
|
1114
|
+
502,
|
|
1115
|
+
"upstream_unavailable",
|
|
1116
|
+
"HTTP bridge pre-created retry failed: upstream websocket reconnect timed out",
|
|
1117
|
+
"server_error",
|
|
1118
|
+
None,
|
|
1119
|
+
)
|
|
1120
|
+
message = str(exc).strip() or "HTTP bridge pre-created retry failed"
|
|
1121
|
+
return 502, "upstream_unavailable", message, "server_error", None
|
|
1122
|
+
|
|
1123
|
+
|
|
1124
|
+
def _trim_http_bridge_previous_response_input_items(input_items: list[JsonValue]) -> list[JsonValue]:
|
|
1125
|
+
first_output_index = next(
|
|
1126
|
+
(
|
|
1127
|
+
index
|
|
1128
|
+
for index, item in enumerate(input_items)
|
|
1129
|
+
if _http_bridge_input_item_type(item)
|
|
1130
|
+
in {"function_call_output", "custom_tool_call_output", "apply_patch_call_output"}
|
|
1131
|
+
),
|
|
1132
|
+
None,
|
|
1133
|
+
)
|
|
1134
|
+
if first_output_index is None or first_output_index == 0:
|
|
1135
|
+
return input_items
|
|
1136
|
+
prefix = input_items[:first_output_index]
|
|
1137
|
+
if not all(_is_http_bridge_previous_response_output_item(item) for item in prefix):
|
|
1138
|
+
return input_items
|
|
1139
|
+
return input_items[first_output_index:]
|
|
1140
|
+
|
|
1141
|
+
|
|
1142
|
+
def _is_http_bridge_previous_response_output_item(item: JsonValue) -> bool:
|
|
1143
|
+
item_type = _http_bridge_input_item_type(item)
|
|
1144
|
+
if item_type in {"reasoning", "function_call", "custom_tool_call", "apply_patch_call"}:
|
|
1145
|
+
return _has_http_bridge_response_output_marker(item)
|
|
1146
|
+
if item_type != "message" or not isinstance(item, dict):
|
|
1147
|
+
return False
|
|
1148
|
+
role = item.get("role")
|
|
1149
|
+
return role == "assistant" and _has_http_bridge_response_output_marker(item)
|
|
1150
|
+
|
|
1151
|
+
|
|
1152
|
+
def _has_http_bridge_response_output_marker(item: JsonValue) -> bool:
|
|
1153
|
+
if not isinstance(item, dict):
|
|
1154
|
+
return False
|
|
1155
|
+
item_id = item.get("id")
|
|
1156
|
+
if isinstance(item_id, str) and item_id.strip():
|
|
1157
|
+
return True
|
|
1158
|
+
status = item.get("status")
|
|
1159
|
+
return status in {"completed", "in_progress"}
|
|
1160
|
+
|
|
1161
|
+
|
|
1162
|
+
def _http_bridge_input_item_type(item: JsonValue) -> str | None:
|
|
1163
|
+
if not isinstance(item, dict):
|
|
1164
|
+
return None
|
|
1165
|
+
item_type = item.get("type")
|
|
1166
|
+
return item_type if isinstance(item_type, str) else None
|
|
1167
|
+
|
|
1168
|
+
|
|
1169
|
+
def _normalize_http_bridge_error_event(
|
|
1170
|
+
*,
|
|
1171
|
+
event: OpenAIEvent | None,
|
|
1172
|
+
payload: dict[str, JsonValue] | None,
|
|
1173
|
+
request_state: _WebSocketRequestState | None,
|
|
1174
|
+
) -> tuple[str, dict[str, JsonValue] | None, OpenAIEvent | None, str]:
|
|
1175
|
+
error_code_value: str | None = None
|
|
1176
|
+
error_type_value: str | None = None
|
|
1177
|
+
error_message_value: str | None = None
|
|
1178
|
+
error_param_value: OpenAIErrorParam | None = None
|
|
1179
|
+
explicit_error_code = False
|
|
1180
|
+
rate_limit_metadata: OpenAIErrorDetail = {}
|
|
1181
|
+
|
|
1182
|
+
if event is not None and event.error is not None:
|
|
1183
|
+
error_code_value = event.error.code
|
|
1184
|
+
error_type_value = event.error.type
|
|
1185
|
+
error_message_value = event.error.message
|
|
1186
|
+
error_param_value = event.error.param_state
|
|
1187
|
+
if isinstance(error_code_value, str) and error_code_value.strip():
|
|
1188
|
+
explicit_error_code = True
|
|
1189
|
+
elif isinstance(payload, dict):
|
|
1190
|
+
payload_error = payload.get("error")
|
|
1191
|
+
if not isinstance(payload_error, dict):
|
|
1192
|
+
payload_error = _websocket_top_level_error_payload(payload)
|
|
1193
|
+
if isinstance(payload_error, dict):
|
|
1194
|
+
code_value = payload_error.get("code")
|
|
1195
|
+
if isinstance(code_value, str):
|
|
1196
|
+
stripped = code_value.strip()
|
|
1197
|
+
if stripped:
|
|
1198
|
+
error_code_value = stripped
|
|
1199
|
+
explicit_error_code = True
|
|
1200
|
+
type_value = payload_error.get("type")
|
|
1201
|
+
if isinstance(type_value, str):
|
|
1202
|
+
stripped = type_value.strip()
|
|
1203
|
+
if stripped:
|
|
1204
|
+
error_type_value = stripped
|
|
1205
|
+
message_value = payload_error.get("message")
|
|
1206
|
+
if isinstance(message_value, str):
|
|
1207
|
+
stripped = message_value.strip()
|
|
1208
|
+
if stripped:
|
|
1209
|
+
error_message_value = stripped
|
|
1210
|
+
if isinstance(payload, dict):
|
|
1211
|
+
raw_error = payload.get("error")
|
|
1212
|
+
if not isinstance(raw_error, dict):
|
|
1213
|
+
raw_error = _websocket_top_level_error_payload(payload)
|
|
1214
|
+
if isinstance(raw_error, dict):
|
|
1215
|
+
if error_param_value is None and "param" in raw_error:
|
|
1216
|
+
error_param_value = OpenAIErrorParam.from_mapping(cast(Mapping[str, JsonValue], raw_error))
|
|
1217
|
+
plan_type = raw_error.get("plan_type")
|
|
1218
|
+
if isinstance(plan_type, str):
|
|
1219
|
+
rate_limit_metadata["plan_type"] = plan_type
|
|
1220
|
+
resets_at = raw_error.get("resets_at")
|
|
1221
|
+
if isinstance(resets_at, int | float):
|
|
1222
|
+
rate_limit_metadata["resets_at"] = resets_at
|
|
1223
|
+
resets_in = raw_error.get("resets_in_seconds")
|
|
1224
|
+
if isinstance(resets_in, int | float):
|
|
1225
|
+
rate_limit_metadata["resets_in_seconds"] = resets_in
|
|
1226
|
+
|
|
1227
|
+
if request_state is not None:
|
|
1228
|
+
if request_state.error_code_override is not None:
|
|
1229
|
+
error_code_value = request_state.error_code_override
|
|
1230
|
+
explicit_error_code = True
|
|
1231
|
+
if request_state.error_type_override is not None:
|
|
1232
|
+
error_type_value = request_state.error_type_override
|
|
1233
|
+
if request_state.error_message_override is not None:
|
|
1234
|
+
error_message_value = request_state.error_message_override
|
|
1235
|
+
if request_state.error_param_override is not None:
|
|
1236
|
+
error_param_value = coerce_error_param(request_state.error_param_override)
|
|
1237
|
+
|
|
1238
|
+
normalized_error_code = _normalize_error_code(error_code_value, error_type_value) or "upstream_error"
|
|
1239
|
+
if not explicit_error_code and normalized_error_code == "error":
|
|
1240
|
+
normalized_error_code = "upstream_error"
|
|
1241
|
+
normalized_error_type = error_type_value or "server_error"
|
|
1242
|
+
normalized_error_message = error_message_value or "Upstream error"
|
|
1243
|
+
|
|
1244
|
+
normalized_response_id = None
|
|
1245
|
+
if request_state is not None:
|
|
1246
|
+
normalized_response_id = request_state.response_id or request_state.request_id
|
|
1247
|
+
|
|
1248
|
+
normalized_event = response_failed_event(
|
|
1249
|
+
normalized_error_code,
|
|
1250
|
+
normalized_error_message,
|
|
1251
|
+
error_type=normalized_error_type,
|
|
1252
|
+
response_id=normalized_response_id,
|
|
1253
|
+
error_param=error_param_value,
|
|
1254
|
+
)
|
|
1255
|
+
if rate_limit_metadata:
|
|
1256
|
+
normalized_event["response"]["error"].update(rate_limit_metadata)
|
|
1257
|
+
normalized_event_block = format_sse_event(normalized_event)
|
|
1258
|
+
normalized_payload = parse_sse_data_json(normalized_event_block)
|
|
1259
|
+
parsed_event = parse_sse_event(normalized_event_block)
|
|
1260
|
+
return normalized_event_block, normalized_payload, parsed_event, "response.failed"
|
|
1261
|
+
|
|
1262
|
+
|
|
1263
|
+
def _http_bridge_pending_state_is_stale(
|
|
1264
|
+
request_state: _WebSocketRequestState,
|
|
1265
|
+
*,
|
|
1266
|
+
now: float,
|
|
1267
|
+
threshold_seconds: float,
|
|
1268
|
+
session_closed: bool = False,
|
|
1269
|
+
) -> bool:
|
|
1270
|
+
"""Identify pre-created bridge requests that never received upstream activity."""
|
|
1271
|
+
if request_state.transport != _REQUEST_TRANSPORT_HTTP or request_state.skip_request_log:
|
|
1272
|
+
return False
|
|
1273
|
+
# Previous-response and turn-state identifiers preserve hard continuity
|
|
1274
|
+
# while the socket is live. Once it closes, that continuity is already lost.
|
|
1275
|
+
# A silent-but-open upstream must not preserve continuity forever: with
|
|
1276
|
+
# zero upstream activity the anchored request wedges the create gate and
|
|
1277
|
+
# every follow-up on the session, so cap the anchored wait at 2x the
|
|
1278
|
+
# stuck-gate threshold before allowing retirement.
|
|
1279
|
+
if not session_closed and (request_state.previous_response_id is not None or request_state.hard_continuity_anchor):
|
|
1280
|
+
anchored_wait_started_at = (
|
|
1281
|
+
request_state.response_create_gate_wait_started_at
|
|
1282
|
+
if request_state.response_create_gate_wait_started_at is not None
|
|
1283
|
+
else request_state.started_at
|
|
1284
|
+
)
|
|
1285
|
+
if max(0.0, now - anchored_wait_started_at) < threshold_seconds * 2.0:
|
|
1286
|
+
return False
|
|
1287
|
+
# Do not require the gate/awaiting flags: retry and reader re-enqueue
|
|
1288
|
+
# paths re-register states whose admission flags were already cleared,
|
|
1289
|
+
# and leaning on them starves the watchdog entirely (observed prod
|
|
1290
|
+
# wedges 2026-07-20). A reattached stream can deliver events whose
|
|
1291
|
+
# response.created was lost, so use the most recent upstream event as
|
|
1292
|
+
# the silence clock instead of aging every active stream from its
|
|
1293
|
+
# original gate wait.
|
|
1294
|
+
if request_state.response_id is not None:
|
|
1295
|
+
return False
|
|
1296
|
+
if request_state.latency_response_created_ms is not None:
|
|
1297
|
+
return False
|
|
1298
|
+
wait_started_at = (
|
|
1299
|
+
request_state.response_create_gate_wait_started_at
|
|
1300
|
+
if request_state.response_create_gate_wait_started_at is not None
|
|
1301
|
+
else request_state.started_at
|
|
1302
|
+
)
|
|
1303
|
+
has_response_lifecycle_activity = request_state.response_event_count > 0 or request_state.upstream_model_output_seen
|
|
1304
|
+
if has_response_lifecycle_activity and request_state.last_upstream_activity_at is not None:
|
|
1305
|
+
wait_started_at = request_state.last_upstream_activity_at
|
|
1306
|
+
return max(0.0, now - wait_started_at) >= threshold_seconds
|
|
1307
|
+
|
|
1308
|
+
|
|
1309
|
+
def _http_bridge_request_counts_against_queue(request_state: _WebSocketRequestState) -> bool:
|
|
1310
|
+
return not request_state.draining_until_terminal
|
|
1311
|
+
|
|
1312
|
+
|
|
1313
|
+
def _http_bridge_eventless_precreated_deadline(
|
|
1314
|
+
request_state: _WebSocketRequestState,
|
|
1315
|
+
*,
|
|
1316
|
+
stuck_gate_retire_after_seconds: float,
|
|
1317
|
+
) -> float | None:
|
|
1318
|
+
sent_at = request_state.response_create_sent_at
|
|
1319
|
+
if (
|
|
1320
|
+
request_state.transport != "http"
|
|
1321
|
+
or request_state.skip_request_log
|
|
1322
|
+
or not request_state.response_create_gate_acquired
|
|
1323
|
+
or request_state.response_create_gate is None
|
|
1324
|
+
or not request_state.awaiting_response_created
|
|
1325
|
+
or sent_at is None
|
|
1326
|
+
or request_state.response_id is not None
|
|
1327
|
+
or request_state.latency_response_created_ms is not None
|
|
1328
|
+
or request_state.downstream_visible
|
|
1329
|
+
or request_state.last_downstream_sequence_number is not None
|
|
1330
|
+
):
|
|
1331
|
+
return None
|
|
1332
|
+
# Non-response telemetry (for example ``codex.rate_limits``) may update
|
|
1333
|
+
# the generic activity marker, but it must not extend the response.create
|
|
1334
|
+
# acknowledgement deadline. Keep the send-time anchor until a matched
|
|
1335
|
+
# response-lifecycle event exists; then use the latest lifecycle activity
|
|
1336
|
+
# so deferred reasoning is not retired from the original send time.
|
|
1337
|
+
deadline_anchor = sent_at
|
|
1338
|
+
has_response_lifecycle_activity = request_state.response_event_count > 0 or request_state.upstream_model_output_seen
|
|
1339
|
+
if has_response_lifecycle_activity and request_state.last_upstream_activity_at is not None:
|
|
1340
|
+
deadline_anchor = request_state.last_upstream_activity_at
|
|
1341
|
+
return deadline_anchor + min(
|
|
1342
|
+
float(stuck_gate_retire_after_seconds),
|
|
1343
|
+
_HTTP_BRIDGE_EVENTLESS_RESPONSE_CREATED_MAX_SECONDS,
|
|
1344
|
+
)
|
|
1345
|
+
|
|
1346
|
+
|
|
1347
|
+
def _http_bridge_retry_circuit_attempt_selection_for_pending_requests(
|
|
1348
|
+
request_states: Sequence[_WebSocketRequestState],
|
|
1349
|
+
) -> _HTTPBridgeRetryCircuitAttemptSelection:
|
|
1350
|
+
eligible_attempts: list[_HTTPBridgeResponseCreateAttempt] = []
|
|
1351
|
+
recorded_attempts: list[_HTTPBridgeResponseCreateAttempt] = []
|
|
1352
|
+
settled_attempts: list[_HTTPBridgeResponseCreateAttempt] = []
|
|
1353
|
+
attempt_seen = False
|
|
1354
|
+
for request_state in request_states:
|
|
1355
|
+
attempt = getattr(request_state, "response_create_attempt", None)
|
|
1356
|
+
if attempt is None:
|
|
1357
|
+
continue
|
|
1358
|
+
attempt_seen = True
|
|
1359
|
+
# A request still holding a verified safe replay must not be charged
|
|
1360
|
+
# anywhere: in a mixed batch its attempt would otherwise be the sole
|
|
1361
|
+
# selectable one (charging the recoverable request) or make the
|
|
1362
|
+
# stranded request's failure look ambiguous. It still counts as a
|
|
1363
|
+
# seen attempt, so a safe-only batch classifies as ineligible rather
|
|
1364
|
+
# than absent — absent is what authorizes an unscoped strike.
|
|
1365
|
+
if _http_bridge_request_state_holds_safe_replay(request_state):
|
|
1366
|
+
continue
|
|
1367
|
+
if attempt.retry_circuit_failure_recorded:
|
|
1368
|
+
recorded_attempts.append(attempt)
|
|
1369
|
+
continue
|
|
1370
|
+
if attempt.disarmed or attempt.response_observed:
|
|
1371
|
+
settled_attempts.append(attempt)
|
|
1372
|
+
continue
|
|
1373
|
+
if (
|
|
1374
|
+
request_state.transport != _REQUEST_TRANSPORT_HTTP
|
|
1375
|
+
or request_state.skip_request_log
|
|
1376
|
+
or request_state.response_id is not None
|
|
1377
|
+
or request_state.latency_response_created_ms is not None
|
|
1378
|
+
or request_state.response_event_count != 0
|
|
1379
|
+
or request_state.downstream_visible
|
|
1380
|
+
):
|
|
1381
|
+
continue
|
|
1382
|
+
eligible_attempts.append(attempt)
|
|
1383
|
+
|
|
1384
|
+
for kind, attempts in (
|
|
1385
|
+
("eligible", eligible_attempts),
|
|
1386
|
+
("recorded", recorded_attempts),
|
|
1387
|
+
("settled", settled_attempts),
|
|
1388
|
+
):
|
|
1389
|
+
unique_attempts: list[_HTTPBridgeResponseCreateAttempt] = []
|
|
1390
|
+
for attempt in attempts:
|
|
1391
|
+
if not any(candidate is attempt for candidate in unique_attempts):
|
|
1392
|
+
unique_attempts.append(attempt)
|
|
1393
|
+
if unique_attempts:
|
|
1394
|
+
return _HTTPBridgeRetryCircuitAttemptSelection(
|
|
1395
|
+
kind=kind,
|
|
1396
|
+
attempts=tuple(unique_attempts),
|
|
1397
|
+
)
|
|
1398
|
+
return _HTTPBridgeRetryCircuitAttemptSelection(kind="ineligible" if attempt_seen else "absent")
|
|
1399
|
+
|
|
1400
|
+
|
|
1401
|
+
def _http_bridge_continuity_bound_without_safe_replay(request_state: _WebSocketRequestState) -> bool:
|
|
1402
|
+
"""Return whether retrying would require replaying an unsafe continuation."""
|
|
1403
|
+
if request_state.previous_response_id is not None:
|
|
1404
|
+
return not _http_bridge_request_state_holds_safe_replay(request_state)
|
|
1405
|
+
return request_state.hard_continuity_anchor and not _http_bridge_request_state_holds_safe_replay(request_state)
|
|
1406
|
+
|
|
1407
|
+
|
|
1408
|
+
def _http_bridge_request_state_holds_safe_replay(request_state: Any) -> bool:
|
|
1409
|
+
"""Whether the request still holds a verified safe replay to protect.
|
|
1410
|
+
|
|
1411
|
+
Holding one means the proof fields are set AND the one permitted replay is
|
|
1412
|
+
still available. ``_retry_http_bridge_request_on_fresh_upstream`` leaves
|
|
1413
|
+
the proof fields in place after consuming the replay, so a request whose
|
|
1414
|
+
permitted replay already failed is stranded like any other: it must strike
|
|
1415
|
+
the circuit and must not keep an abandonment from settling it. The same
|
|
1416
|
+
retry path also refuses a request that has observed a response event, so a
|
|
1417
|
+
started response holds no replay either — counting it would leave the
|
|
1418
|
+
circuit cooling for its full backoff after a successful abandonment, for a
|
|
1419
|
+
replay that can never dispatch.
|
|
1420
|
+
"""
|
|
1421
|
+
return bool(
|
|
1422
|
+
getattr(request_state, "fresh_upstream_request_is_retry_safe", False)
|
|
1423
|
+
and getattr(request_state, "fresh_upstream_request_text", None)
|
|
1424
|
+
and getattr(request_state, "replay_count", 0) == 0
|
|
1425
|
+
and getattr(request_state, "response_event_count", 0) == 0
|
|
1426
|
+
# A deferred-reasoning prelude marks the response started without
|
|
1427
|
+
# counting a response event, and a started response cannot replay.
|
|
1428
|
+
and not getattr(request_state, "upstream_model_output_seen", False)
|
|
1429
|
+
)
|
|
1430
|
+
|
|
1431
|
+
|
|
1432
|
+
def _http_bridge_abandonment_may_settle_circuit(request_states: Iterable[Any]) -> bool:
|
|
1433
|
+
"""Return whether this abandonment may settle the retry circuit with it.
|
|
1434
|
+
|
|
1435
|
+
The circuit must survive exactly one thing: a request that still holds a
|
|
1436
|
+
safe replay. That replay claims the circuit's generation at dispatch
|
|
1437
|
+
(#1863), so clearing the circuit under it removes the fence it depends on.
|
|
1438
|
+
Every other case is a cooldown backing off a cause this abandonment just
|
|
1439
|
+
removed — anchored or not. Requiring every state to also be
|
|
1440
|
+
continuity-bound let an unanchored full-resend request, which has no
|
|
1441
|
+
replay to protect, block the settle and keep the key cooling for its full
|
|
1442
|
+
backoff after the anchor was already gone.
|
|
1443
|
+
|
|
1444
|
+
An empty set therefore settles. Terminal notification drains
|
|
1445
|
+
``pending_requests`` before retirement, so the funnels routinely reach here
|
|
1446
|
+
with a pre-drain count and no states at all; nothing is holding the
|
|
1447
|
+
generation.
|
|
1448
|
+
"""
|
|
1449
|
+
return not any(_http_bridge_request_state_holds_safe_replay(state) for state in request_states if state is not None)
|
|
1450
|
+
|
|
1451
|
+
|
|
1452
|
+
def _http_bridge_session_has_admission_waiter(session: object | None) -> bool:
|
|
1453
|
+
"""Keep a closed bridge registered while an unsent request owns its handoff."""
|
|
1454
|
+
return session is not None and bool(getattr(session, "admission_waiter_count", 0))
|
|
1455
|
+
|
|
1456
|
+
|
|
1457
|
+
def _http_bridge_session_has_visible_requests(session: "_HTTPBridgeSession") -> bool:
|
|
1458
|
+
return session.queued_request_count > 0 or any(
|
|
1459
|
+
_http_bridge_request_counts_against_queue(request_state) for request_state in session.pending_requests
|
|
1460
|
+
)
|
|
1461
|
+
|
|
1462
|
+
|
|
1463
|
+
async def _raise_if_http_bridge_creation_superseded(
|
|
1464
|
+
service: Any,
|
|
1465
|
+
key: "_HTTPBridgeSessionKey",
|
|
1466
|
+
*,
|
|
1467
|
+
inflight_future: Any,
|
|
1468
|
+
) -> None:
|
|
1469
|
+
"""Abort a creation whose inflight slot another creator already took.
|
|
1470
|
+
|
|
1471
|
+
An evicted creator has lost the registry slot, so claiming would only
|
|
1472
|
+
advance the durable epoch past the session that won — fencing that
|
|
1473
|
+
winner's own renewals out of a row it legitimately owns (issue #1695).
|
|
1474
|
+
Failing here leaves the row untouched; the caller's failure path then
|
|
1475
|
+
closes this session without releasing the winner's row.
|
|
1476
|
+
"""
|
|
1477
|
+
async with service._http_bridge_lock:
|
|
1478
|
+
superseded = service._http_bridge_inflight_sessions.get(key) is not inflight_future
|
|
1479
|
+
if superseded:
|
|
1480
|
+
raise _http_bridge_startup_wait_timeout_error(
|
|
1481
|
+
"http_bridge_session_registration",
|
|
1482
|
+
code="capacity_exhausted_active_sessions",
|
|
1483
|
+
)
|
|
1484
|
+
|
|
1485
|
+
|
|
1486
|
+
async def _settle_failed_http_bridge_creation(
|
|
1487
|
+
service: Any,
|
|
1488
|
+
key: "_HTTPBridgeSessionKey",
|
|
1489
|
+
*,
|
|
1490
|
+
inflight_future: Any,
|
|
1491
|
+
created_session: "_HTTPBridgeSession | None",
|
|
1492
|
+
exc: BaseException,
|
|
1493
|
+
) -> bool:
|
|
1494
|
+
"""Retire a failed creation and report whether another session replaced it.
|
|
1495
|
+
|
|
1496
|
+
A rejected creator claimed the durable row last, so its epoch is the
|
|
1497
|
+
current one and its fenced release WOULD succeed — closing the row out
|
|
1498
|
+
from under the session that actually won the registry slot (issue #1695).
|
|
1499
|
+
The caller uses the return value to skip the durable release in that case.
|
|
1500
|
+
"""
|
|
1501
|
+
async with service._http_bridge_lock:
|
|
1502
|
+
current_future = service._http_bridge_inflight_sessions.get(key)
|
|
1503
|
+
replacement_in_flight = current_future is not None and current_future is not inflight_future
|
|
1504
|
+
if current_future is inflight_future:
|
|
1505
|
+
service._http_bridge_inflight_sessions.pop(key, None)
|
|
1506
|
+
if inflight_future is not None and not inflight_future.done():
|
|
1507
|
+
if isinstance(exc, asyncio.CancelledError):
|
|
1508
|
+
inflight_future.cancel()
|
|
1509
|
+
else:
|
|
1510
|
+
inflight_future.set_exception(exc)
|
|
1511
|
+
inflight_future.exception()
|
|
1512
|
+
registered_session = service._http_bridge_sessions.get(key)
|
|
1513
|
+
# A replacement that has claimed but not yet published its session is
|
|
1514
|
+
# just as much the winner as a registered one: releasing here would
|
|
1515
|
+
# close the row beneath it, and it would then register with an older
|
|
1516
|
+
# epoch and be fenced out on its first renewal (issue #1695).
|
|
1517
|
+
registered_winner = (
|
|
1518
|
+
registered_session if registered_session is not None and registered_session is not created_session else None
|
|
1519
|
+
)
|
|
1520
|
+
# A registered winner on a DIFFERENT account no longer shares this row:
|
|
1521
|
+
# our claim already rewrote its account binding and cleared its
|
|
1522
|
+
# continuity aliases, so preserving the row would leave it bound to our
|
|
1523
|
+
# account while that session keeps dispatching. Release it instead —
|
|
1524
|
+
# the winner is then fenced promptly and retries cleanly.
|
|
1525
|
+
winner_shares_row = registered_winner is not None and (
|
|
1526
|
+
created_session is None or registered_winner.account.id == created_session.account.id
|
|
1527
|
+
)
|
|
1528
|
+
superseded = replacement_in_flight or winner_shares_row
|
|
1529
|
+
if superseded and registered_session is not None and created_session is not None:
|
|
1530
|
+
# Eviction can still land DURING the claim, so this creator may
|
|
1531
|
+
# have advanced the shared row's epoch past the session that won
|
|
1532
|
+
# the slot, fencing the winner's own renewals. Both sessions are
|
|
1533
|
+
# this instance's and point at the same row, so hand the epoch we
|
|
1534
|
+
# won over to the winner rather than leaving it stranded.
|
|
1535
|
+
assert registered_session is not None
|
|
1536
|
+
claimed_id = created_session.durable_session_id
|
|
1537
|
+
claimed_epoch = created_session.durable_owner_epoch
|
|
1538
|
+
registered_epoch = registered_session.durable_owner_epoch
|
|
1539
|
+
if (
|
|
1540
|
+
claimed_id is not None
|
|
1541
|
+
and claimed_epoch is not None
|
|
1542
|
+
and registered_session.durable_session_id == claimed_id
|
|
1543
|
+
and (registered_epoch is None or claimed_epoch > registered_epoch)
|
|
1544
|
+
# Only when both sessions selected the same account: a claim
|
|
1545
|
+
# rewrites the row's account_id and clears continuity aliases,
|
|
1546
|
+
# so handing the epoch across an account change would leave the
|
|
1547
|
+
# winner renewing and dispatching on a row bound to a different
|
|
1548
|
+
# account. Fail closed there instead — the winner's renewal is
|
|
1549
|
+
# fenced, it is evicted, and the request retries cleanly.
|
|
1550
|
+
and created_session.account.id == registered_session.account.id
|
|
1551
|
+
):
|
|
1552
|
+
registered_session.durable_owner_epoch = claimed_epoch
|
|
1553
|
+
return superseded
|
|
1554
|
+
|
|
1555
|
+
|
|
1556
|
+
async def _close_http_bridge_session_resources(
|
|
1557
|
+
service: Any,
|
|
1558
|
+
session: "_HTTPBridgeSession",
|
|
1559
|
+
*,
|
|
1560
|
+
turn_state_lock_held: bool = False,
|
|
1561
|
+
release_durable_session: bool = True,
|
|
1562
|
+
) -> None:
|
|
1563
|
+
session.closed = True
|
|
1564
|
+
durable_session_id = getattr(session, "durable_session_id", None)
|
|
1565
|
+
durable_owner_epoch = getattr(session, "durable_owner_epoch", None)
|
|
1566
|
+
durable_release_allowed = release_durable_session and _http_bridge_durable_release_allowed(service, session)
|
|
1567
|
+
if turn_state_lock_held:
|
|
1568
|
+
service._unregister_http_bridge_turn_states_locked(session)
|
|
1569
|
+
service._unregister_http_bridge_previous_response_ids_locked(session)
|
|
1570
|
+
else:
|
|
1571
|
+
await service._unregister_http_bridge_turn_states(session)
|
|
1572
|
+
await service._unregister_http_bridge_previous_response_ids(session)
|
|
1573
|
+
account_lease = getattr(session, "account_lease", None)
|
|
1574
|
+
try:
|
|
1575
|
+
await service._load_balancer.release_account_lease(account_lease)
|
|
1576
|
+
except Exception:
|
|
1577
|
+
logger.warning("Failed to release HTTP bridge account lease during close", exc_info=True)
|
|
1578
|
+
finally:
|
|
1579
|
+
session.account_lease = None
|
|
1580
|
+
durable_release_succeeded = durable_owner_epoch is None
|
|
1581
|
+
if durable_release_allowed:
|
|
1582
|
+
try:
|
|
1583
|
+
released = await service._durable_bridge.release_live_session(
|
|
1584
|
+
session_id=durable_session_id,
|
|
1585
|
+
instance_id=_service_get_settings().http_responses_session_bridge_instance_id,
|
|
1586
|
+
owner_epoch=durable_owner_epoch,
|
|
1587
|
+
draining=shutdown_state.is_bridge_drain_active(),
|
|
1588
|
+
)
|
|
1589
|
+
# Fenced releases return the current owner snapshot, while a
|
|
1590
|
+
# missing row returns None. Only an ownerless snapshot (or a
|
|
1591
|
+
# missing row) means this generation no longer owns a durable lease.
|
|
1592
|
+
durable_release_succeeded = released is None or getattr(released, "owner_instance_id", None) is None
|
|
1593
|
+
except Exception:
|
|
1594
|
+
logger.warning("Failed to release durable HTTP bridge session", exc_info=True)
|
|
1595
|
+
# Closing a generation retires its process-local denial slot as well as
|
|
1596
|
+
# its routing aliases. Keep pinned requests fenced; the owner helper marks
|
|
1597
|
+
# those entries superseded and lets their final pin release remove them.
|
|
1598
|
+
# A deliberately retained lease or a failed/fenced release still belongs
|
|
1599
|
+
# to a live durable owner, so its fence must remain for a successor.
|
|
1600
|
+
if durable_release_succeeded:
|
|
1601
|
+
owner_key = durable_session_id if durable_session_id is not None else f"local:{id(session)}"
|
|
1602
|
+
pending_denied_response_ids = tuple(getattr(session, "denied_proxy_injected_anchor_cleanup_pending", ()))
|
|
1603
|
+
_forget_http_bridge_denied_anchor_fence_owner(
|
|
1604
|
+
service,
|
|
1605
|
+
owner_key,
|
|
1606
|
+
owner_epoch=durable_owner_epoch,
|
|
1607
|
+
preserve_response_ids=pending_denied_response_ids,
|
|
1608
|
+
)
|
|
1609
|
+
_retire_http_bridge_denied_anchor_predecessors_after_durable_clear(
|
|
1610
|
+
service,
|
|
1611
|
+
owner_key,
|
|
1612
|
+
owner_epoch=durable_owner_epoch,
|
|
1613
|
+
preserve_response_ids=pending_denied_response_ids,
|
|
1614
|
+
)
|
|
1615
|
+
upstream_reader = session.upstream_reader
|
|
1616
|
+
if upstream_reader is not None:
|
|
1617
|
+
if upstream_reader is asyncio.current_task():
|
|
1618
|
+
session.upstream_reader = None
|
|
1619
|
+
else:
|
|
1620
|
+
await _await_cancelled_task(
|
|
1621
|
+
upstream_reader,
|
|
1622
|
+
label="http bridge upstream reader",
|
|
1623
|
+
cleanup_tasks=service._background_cleanup_tasks,
|
|
1624
|
+
scheduler=scheduler_for(service),
|
|
1625
|
+
)
|
|
1626
|
+
if session.upstream_reader is upstream_reader:
|
|
1627
|
+
session.upstream_reader = None
|
|
1628
|
+
try:
|
|
1629
|
+
await session.upstream.close()
|
|
1630
|
+
except Exception:
|
|
1631
|
+
logger.debug("Failed to close HTTP bridge upstream websocket", exc_info=True)
|
|
1632
|
+
pending_requests = getattr(session, "pending_requests", None)
|
|
1633
|
+
pending_lock = getattr(session, "pending_lock", None)
|
|
1634
|
+
response_create_gate = getattr(session, "response_create_gate", None)
|
|
1635
|
+
if pending_requests is not None and pending_lock is not None:
|
|
1636
|
+
async with pending_lock:
|
|
1637
|
+
session.queued_request_count = 0
|
|
1638
|
+
await service._fail_pending_websocket_requests(
|
|
1639
|
+
account=session.account,
|
|
1640
|
+
account_id_value=session.account.id,
|
|
1641
|
+
pending_requests=pending_requests,
|
|
1642
|
+
pending_lock=pending_lock,
|
|
1643
|
+
error_code="stream_incomplete",
|
|
1644
|
+
error_message="HTTP bridge session closed before response.completed",
|
|
1645
|
+
api_key=None,
|
|
1646
|
+
response_create_gate=response_create_gate,
|
|
1647
|
+
)
|
|
1648
|
+
_log_http_bridge_event(
|
|
1649
|
+
"close",
|
|
1650
|
+
session.key,
|
|
1651
|
+
account_id=session.account.id,
|
|
1652
|
+
model=session.request_model,
|
|
1653
|
+
cache_key_family=session.key.affinity_kind,
|
|
1654
|
+
model_class=_extract_model_class(session.request_model) if session.request_model else None,
|
|
1655
|
+
)
|
|
1656
|
+
|
|
1657
|
+
|
|
1658
|
+
async def _close_http_bridge_session(
|
|
1659
|
+
service: Any,
|
|
1660
|
+
session: "_HTTPBridgeSession",
|
|
1661
|
+
*,
|
|
1662
|
+
turn_state_lock_held: bool = False,
|
|
1663
|
+
release_durable_session: bool = True,
|
|
1664
|
+
) -> None:
|
|
1665
|
+
# Direct close callers can be cancelled just like the bounded background
|
|
1666
|
+
# wrapper. Keep the resource owner alive until its reader, socket, and
|
|
1667
|
+
# leases are actually finalized; only then may detached-capacity tracking
|
|
1668
|
+
# disappear. Otherwise cancellation can turn a live predecessor into an
|
|
1669
|
+
# unowned generation that neither shutdown nor capacity accounting sees.
|
|
1670
|
+
def resource_close_task() -> asyncio.Task[None]:
|
|
1671
|
+
existing = session.resource_close_task
|
|
1672
|
+
if existing is not None and (
|
|
1673
|
+
not existing.done() or (not existing.cancelled() and existing.exception() is None)
|
|
1674
|
+
):
|
|
1675
|
+
return existing
|
|
1676
|
+
created = scheduler_for(service).create_task(
|
|
1677
|
+
_close_http_bridge_session_resources(
|
|
1678
|
+
service,
|
|
1679
|
+
session,
|
|
1680
|
+
turn_state_lock_held=turn_state_lock_held,
|
|
1681
|
+
release_durable_session=release_durable_session,
|
|
1682
|
+
),
|
|
1683
|
+
name=f"http-bridge-resource-close-{_hash_identifier(session.key.affinity_key)}",
|
|
1684
|
+
)
|
|
1685
|
+
session.resource_close_task = created
|
|
1686
|
+
return created
|
|
1687
|
+
|
|
1688
|
+
# Close callers can race (reader retirement, account invalidation, and
|
|
1689
|
+
# shutdown). Install one resource owner under the registry lock so leases
|
|
1690
|
+
# and pending settlement are finalized exactly once.
|
|
1691
|
+
if turn_state_lock_held:
|
|
1692
|
+
close_task = resource_close_task()
|
|
1693
|
+
else:
|
|
1694
|
+
async with service._http_bridge_lock:
|
|
1695
|
+
close_task = resource_close_task()
|
|
1696
|
+
_, cancellation = await _await_task_deferring_cancellation(close_task)
|
|
1697
|
+
|
|
1698
|
+
# Detached generations remain capacity owners until resource closure ends.
|
|
1699
|
+
# Finalize that ownership here so direct error-recovery closes and bounded
|
|
1700
|
+
# background closes cannot drift into different lifecycles. This update is
|
|
1701
|
+
# independently owned because cancellation can begin while its lock waits.
|
|
1702
|
+
async def finalize_detached_ownership() -> None:
|
|
1703
|
+
if turn_state_lock_held:
|
|
1704
|
+
if service._http_bridge_detached_sessions.get(id(session)) is session:
|
|
1705
|
+
service._http_bridge_detached_sessions.pop(id(session), None)
|
|
1706
|
+
else:
|
|
1707
|
+
async with service._http_bridge_lock:
|
|
1708
|
+
if service._http_bridge_detached_sessions.get(id(session)) is session:
|
|
1709
|
+
service._http_bridge_detached_sessions.pop(id(session), None)
|
|
1710
|
+
|
|
1711
|
+
ownership_task = scheduler_for(service).create_task(
|
|
1712
|
+
finalize_detached_ownership(),
|
|
1713
|
+
name=f"http-bridge-detached-finalize-{_hash_identifier(session.key.affinity_key)}",
|
|
1714
|
+
)
|
|
1715
|
+
_, ownership_cancellation = await _await_task_deferring_cancellation(ownership_task)
|
|
1716
|
+
cancellation = cancellation or ownership_cancellation
|
|
1717
|
+
if cancellation is not None:
|
|
1718
|
+
raise cancellation
|
|
1719
|
+
|
|
1720
|
+
|
|
1721
|
+
async def _close_http_bridge_session_bounded(
|
|
1722
|
+
service: Any,
|
|
1723
|
+
session: "_HTTPBridgeSession",
|
|
1724
|
+
*,
|
|
1725
|
+
reason: str,
|
|
1726
|
+
) -> None:
|
|
1727
|
+
if session.upstream_reader is asyncio.current_task():
|
|
1728
|
+
session.upstream_reader = None
|
|
1729
|
+
|
|
1730
|
+
close_task = scheduler_for(service).create_task(
|
|
1731
|
+
service._close_http_bridge_session(session),
|
|
1732
|
+
name=f"http-bridge-close-{_hash_identifier(session.key.affinity_key)}",
|
|
1733
|
+
)
|
|
1734
|
+
|
|
1735
|
+
def track_after_interruption(*, interruption: str) -> None:
|
|
1736
|
+
def close_done(done_task: asyncio.Task[None]) -> None:
|
|
1737
|
+
try:
|
|
1738
|
+
done_task.result()
|
|
1739
|
+
except asyncio.CancelledError:
|
|
1740
|
+
service._http_bridge_background_cleanup_failed = True
|
|
1741
|
+
logger.warning(
|
|
1742
|
+
"http_bridge_session_close_cancelled_after_%s reason=%s bridge_kind=%s "
|
|
1743
|
+
"bridge_key=%s account_id=%s model=%s",
|
|
1744
|
+
interruption,
|
|
1745
|
+
reason,
|
|
1746
|
+
session.key.affinity_kind,
|
|
1747
|
+
_hash_identifier(session.key.affinity_key),
|
|
1748
|
+
session.account.id,
|
|
1749
|
+
session.request_model,
|
|
1750
|
+
)
|
|
1751
|
+
except BaseException:
|
|
1752
|
+
service._http_bridge_background_cleanup_failed = True
|
|
1753
|
+
logger.warning(
|
|
1754
|
+
"http_bridge_session_close_failed_after_%s reason=%s bridge_kind=%s "
|
|
1755
|
+
"bridge_key=%s account_id=%s model=%s",
|
|
1756
|
+
interruption,
|
|
1757
|
+
reason,
|
|
1758
|
+
session.key.affinity_kind,
|
|
1759
|
+
_hash_identifier(session.key.affinity_key),
|
|
1760
|
+
session.account.id,
|
|
1761
|
+
session.request_model,
|
|
1762
|
+
exc_info=True,
|
|
1763
|
+
)
|
|
1764
|
+
finally:
|
|
1765
|
+
service._background_cleanup_tasks.discard(done_task)
|
|
1766
|
+
|
|
1767
|
+
if close_task.done():
|
|
1768
|
+
close_done(close_task)
|
|
1769
|
+
return
|
|
1770
|
+
service._background_cleanup_tasks.add(close_task)
|
|
1771
|
+
close_task.add_done_callback(close_done)
|
|
1772
|
+
|
|
1773
|
+
try:
|
|
1774
|
+
# Not ``wait_for(shield(...))``: ``close_task`` is the shared
|
|
1775
|
+
# per-session resource owner, and 3.14's shield leaks a done-callback
|
|
1776
|
+
# onto it for every waiter that times out or is cancelled.
|
|
1777
|
+
await wait_on_shared_future(
|
|
1778
|
+
close_task,
|
|
1779
|
+
timeout=_HTTP_BRIDGE_BACKGROUND_CLOSE_TIMEOUT_SECONDS,
|
|
1780
|
+
scheduler=scheduler_for(service),
|
|
1781
|
+
)
|
|
1782
|
+
except TimeoutError:
|
|
1783
|
+
track_after_interruption(interruption="timeout")
|
|
1784
|
+
logger.warning(
|
|
1785
|
+
"http_bridge_session_close_timeout reason=%s bridge_kind=%s bridge_key=%s "
|
|
1786
|
+
"account_id=%s model=%s timeout_seconds=%.1f background_cleanup_tasks=%d",
|
|
1787
|
+
reason,
|
|
1788
|
+
session.key.affinity_kind,
|
|
1789
|
+
_hash_identifier(session.key.affinity_key),
|
|
1790
|
+
session.account.id,
|
|
1791
|
+
session.request_model,
|
|
1792
|
+
_HTTP_BRIDGE_BACKGROUND_CLOSE_TIMEOUT_SECONDS,
|
|
1793
|
+
len(service._background_cleanup_tasks),
|
|
1794
|
+
)
|
|
1795
|
+
except asyncio.CancelledError:
|
|
1796
|
+
track_after_interruption(interruption="cancellation")
|
|
1797
|
+
raise
|
|
1798
|
+
except Exception:
|
|
1799
|
+
logger.warning(
|
|
1800
|
+
"http_bridge_session_close_failed reason=%s bridge_kind=%s bridge_key=%s account_id=%s model=%s",
|
|
1801
|
+
reason,
|
|
1802
|
+
session.key.affinity_kind,
|
|
1803
|
+
_hash_identifier(session.key.affinity_key),
|
|
1804
|
+
session.account.id,
|
|
1805
|
+
session.request_model,
|
|
1806
|
+
exc_info=True,
|
|
1807
|
+
)
|
|
1808
|
+
|
|
1809
|
+
|
|
1810
|
+
def _http_bridge_models_compatible(existing_model: str | None, request_model: str | None) -> bool:
|
|
1811
|
+
"""Never reuse a bridge across two explicitly different model slugs."""
|
|
1812
|
+
|
|
1813
|
+
if existing_model is None or request_model is None:
|
|
1814
|
+
return True
|
|
1815
|
+
return existing_model.strip().lower() == request_model.strip().lower()
|
|
1816
|
+
|
|
1817
|
+
|
|
1818
|
+
def _http_bridge_compatible(
|
|
1819
|
+
session: _HTTPBridgeSession,
|
|
1820
|
+
request_model: str | None,
|
|
1821
|
+
request_service_tier: str | None,
|
|
1822
|
+
same_model_required: bool = False,
|
|
1823
|
+
) -> bool:
|
|
1824
|
+
"""Check catalog compatibility while preserving stricter legacy reuse paths."""
|
|
1825
|
+
|
|
1826
|
+
model_compatible = not same_model_required or _http_bridge_models_compatible(
|
|
1827
|
+
session.request_model,
|
|
1828
|
+
request_model,
|
|
1829
|
+
)
|
|
1830
|
+
return model_compatible and _http_bridge_session_supports_service_tier(
|
|
1831
|
+
session,
|
|
1832
|
+
request_model=request_model,
|
|
1833
|
+
request_service_tier=request_service_tier,
|
|
1834
|
+
)
|
|
1835
|
+
|
|
1836
|
+
|
|
1837
|
+
def _http_bridge_alias_target_is_stale(session: _HTTPBridgeSession | None) -> bool:
|
|
1838
|
+
return (
|
|
1839
|
+
session is None
|
|
1840
|
+
or (session.closed and not session.handoff_in_progress)
|
|
1841
|
+
or not _http_bridge_session_account_active(session)
|
|
1842
|
+
)
|
|
1843
|
+
|
|
1844
|
+
|
|
1845
|
+
def _http_bridge_incompatible_model_fork_key(
|
|
1846
|
+
*,
|
|
1847
|
+
key: "_HTTPBridgeSessionKey",
|
|
1848
|
+
existing_model: str | None,
|
|
1849
|
+
request_model: str | None,
|
|
1850
|
+
request_scope_id: str,
|
|
1851
|
+
) -> "_HTTPBridgeSessionKey | None":
|
|
1852
|
+
if key.affinity_kind not in {
|
|
1853
|
+
"session_header",
|
|
1854
|
+
"thread_header",
|
|
1855
|
+
"turn_state_header",
|
|
1856
|
+
"internal_unanchored_parallel",
|
|
1857
|
+
"internal_model_parallel",
|
|
1858
|
+
}:
|
|
1859
|
+
return None
|
|
1860
|
+
if _http_bridge_models_compatible(existing_model, request_model):
|
|
1861
|
+
return None
|
|
1862
|
+
if key.affinity_kind == "internal_model_parallel":
|
|
1863
|
+
return None
|
|
1864
|
+
recovery_fork = is_http_bridge_account_neutral_replay(
|
|
1865
|
+
kind=key.affinity_kind,
|
|
1866
|
+
key=key.affinity_key,
|
|
1867
|
+
)
|
|
1868
|
+
fork_value = sha256(
|
|
1869
|
+
f"{key.affinity_kind}\0{key.affinity_key}\0{request_model or ''}\0{request_scope_id}".encode()
|
|
1870
|
+
).hexdigest()
|
|
1871
|
+
if recovery_fork:
|
|
1872
|
+
fork_kind, fork_value = make_http_bridge_account_neutral_replay_key(fork_value)
|
|
1873
|
+
else:
|
|
1874
|
+
fork_kind = "internal_model_parallel"
|
|
1875
|
+
fork_key = _HTTPBridgeSessionKey(
|
|
1876
|
+
fork_kind,
|
|
1877
|
+
fork_value,
|
|
1878
|
+
key.api_key_id,
|
|
1879
|
+
)
|
|
1880
|
+
_log_http_bridge_event(
|
|
1881
|
+
"model_transition_fork",
|
|
1882
|
+
fork_key,
|
|
1883
|
+
account_id=None,
|
|
1884
|
+
model=request_model,
|
|
1885
|
+
detail=f"previous_model={existing_model}",
|
|
1886
|
+
cache_key_family=key.affinity_kind,
|
|
1887
|
+
model_class=_extract_model_class(request_model) if request_model else None,
|
|
1888
|
+
owner_check_applied=False,
|
|
1889
|
+
)
|
|
1890
|
+
return fork_key
|
|
1891
|
+
|
|
1892
|
+
|
|
1893
|
+
def _http_bridge_locally_owned_fork_key(
|
|
1894
|
+
fork_key: "_HTTPBridgeSessionKey",
|
|
1895
|
+
forwarded_request: bool,
|
|
1896
|
+
forwarded_original_request_unanchored: bool,
|
|
1897
|
+
) -> "_HTTPBridgeSessionKey | None":
|
|
1898
|
+
if not forwarded_request:
|
|
1899
|
+
return None
|
|
1900
|
+
if fork_key.affinity_kind == "internal_request_parallel":
|
|
1901
|
+
return fork_key
|
|
1902
|
+
if forwarded_original_request_unanchored and fork_key.affinity_kind == "internal_unanchored_parallel":
|
|
1903
|
+
return fork_key
|
|
1904
|
+
return None
|
|
1905
|
+
|
|
1906
|
+
|
|
1907
|
+
def _http_bridge_parallel_fork_key(
|
|
1908
|
+
*,
|
|
1909
|
+
key: "_HTTPBridgeSessionKey",
|
|
1910
|
+
session: "_HTTPBridgeSession | None",
|
|
1911
|
+
inflight_creation: bool,
|
|
1912
|
+
incoming_turn_state: str | None,
|
|
1913
|
+
previous_response_id: str | None,
|
|
1914
|
+
request_model: str | None,
|
|
1915
|
+
request_service_tier: str | None,
|
|
1916
|
+
request_scope_id: str,
|
|
1917
|
+
allow_model_fork: bool = True,
|
|
1918
|
+
same_model_required: bool = False,
|
|
1919
|
+
force_canonical_replacement: bool = False,
|
|
1920
|
+
) -> "_HTTPBridgeSessionKey | None":
|
|
1921
|
+
"""Give incompatible or concurrent requests an independent websocket lane."""
|
|
1922
|
+
|
|
1923
|
+
if force_canonical_replacement:
|
|
1924
|
+
if session is not None:
|
|
1925
|
+
# A restart must replace the canonical session-header lane. Forking
|
|
1926
|
+
# would leave the old owner reusable after the restart moved affinity.
|
|
1927
|
+
session.upstream_control.reconnect_requested = True
|
|
1928
|
+
session.upstream_control.retire_after_drain = True
|
|
1929
|
+
return None
|
|
1930
|
+
|
|
1931
|
+
reason: str | None = None
|
|
1932
|
+
if (
|
|
1933
|
+
key.affinity_kind in {"session_header", "thread_header"}
|
|
1934
|
+
and incoming_turn_state is None
|
|
1935
|
+
and previous_response_id is None
|
|
1936
|
+
):
|
|
1937
|
+
if inflight_creation:
|
|
1938
|
+
reason = "session_creation_inflight"
|
|
1939
|
+
elif session is not None and not session.closed:
|
|
1940
|
+
if _http_bridge_session_has_visible_requests(session):
|
|
1941
|
+
reason = "active_request"
|
|
1942
|
+
elif (
|
|
1943
|
+
reservation_id := getattr(session, "unanchored_reservation_id", None)
|
|
1944
|
+
) is not None and reservation_id != request_scope_id:
|
|
1945
|
+
reason = "session_reserved"
|
|
1946
|
+
elif not _http_bridge_models_compatible(session.request_model, request_model):
|
|
1947
|
+
reason = "model_change"
|
|
1948
|
+
if reason is not None:
|
|
1949
|
+
fork_key = _HTTPBridgeSessionKey(
|
|
1950
|
+
"internal_unanchored_parallel",
|
|
1951
|
+
sha256(f"{key.affinity_key}\0{request_scope_id}".encode()).hexdigest(),
|
|
1952
|
+
key.api_key_id,
|
|
1953
|
+
)
|
|
1954
|
+
_log_http_bridge_event(
|
|
1955
|
+
"unanchored_parallel_fork",
|
|
1956
|
+
fork_key,
|
|
1957
|
+
account_id=None,
|
|
1958
|
+
model=request_model,
|
|
1959
|
+
detail=f"reason={reason}",
|
|
1960
|
+
cache_key_family=key.affinity_kind,
|
|
1961
|
+
model_class=_extract_model_class(request_model) if request_model else None,
|
|
1962
|
+
owner_check_applied=False,
|
|
1963
|
+
)
|
|
1964
|
+
return fork_key
|
|
1965
|
+
|
|
1966
|
+
if session is None or session.closed or not _http_bridge_session_account_active(session):
|
|
1967
|
+
return None
|
|
1968
|
+
if allow_model_fork:
|
|
1969
|
+
model_fork_key = _http_bridge_incompatible_model_fork_key(
|
|
1970
|
+
key=key,
|
|
1971
|
+
existing_model=session.request_model,
|
|
1972
|
+
request_model=request_model,
|
|
1973
|
+
request_scope_id=request_scope_id,
|
|
1974
|
+
)
|
|
1975
|
+
if model_fork_key is not None:
|
|
1976
|
+
return model_fork_key
|
|
1977
|
+
if _http_bridge_compatible(
|
|
1978
|
+
session,
|
|
1979
|
+
request_model,
|
|
1980
|
+
request_service_tier,
|
|
1981
|
+
same_model_required,
|
|
1982
|
+
):
|
|
1983
|
+
return None
|
|
1984
|
+
if incoming_turn_state is not None or previous_response_id is not None:
|
|
1985
|
+
raise ProxyResponseError(502, _http_bridge_continuity_lost_error_envelope())
|
|
1986
|
+
|
|
1987
|
+
fork_key = _HTTPBridgeSessionKey(
|
|
1988
|
+
"internal_request_parallel",
|
|
1989
|
+
sha256(f"{key.affinity_kind}\0{key.affinity_key}\0{request_scope_id}".encode()).hexdigest(),
|
|
1990
|
+
key.api_key_id,
|
|
1991
|
+
)
|
|
1992
|
+
_log_http_bridge_event(
|
|
1993
|
+
"request_compatibility_fork",
|
|
1994
|
+
fork_key,
|
|
1995
|
+
account_id=session.account.id,
|
|
1996
|
+
model=request_model,
|
|
1997
|
+
detail=f"source_kind={key.affinity_kind}",
|
|
1998
|
+
cache_key_family=key.affinity_kind,
|
|
1999
|
+
model_class=_extract_model_class(request_model) if request_model else None,
|
|
2000
|
+
owner_check_applied=False,
|
|
2001
|
+
)
|
|
2002
|
+
return fork_key
|
|
2003
|
+
|
|
2004
|
+
|
|
2005
|
+
def _http_bridge_request_needs_unanchored_handoff(
|
|
2006
|
+
key: "_HTTPBridgeSessionKey",
|
|
2007
|
+
incoming_turn_state: str | None,
|
|
2008
|
+
previous_response_id: str | None,
|
|
2009
|
+
forwarded_request: bool,
|
|
2010
|
+
forwarded_original_request_unanchored: bool,
|
|
2011
|
+
) -> bool:
|
|
2012
|
+
if forwarded_request:
|
|
2013
|
+
return forwarded_original_request_unanchored
|
|
2014
|
+
return (
|
|
2015
|
+
key.affinity_kind in {"session_header", "thread_header"}
|
|
2016
|
+
and incoming_turn_state is None
|
|
2017
|
+
and previous_response_id is None
|
|
2018
|
+
)
|
|
2019
|
+
|
|
2020
|
+
|
|
2021
|
+
def _reserve_http_bridge_unanchored_handoff(
|
|
2022
|
+
session: "_HTTPBridgeSession",
|
|
2023
|
+
*,
|
|
2024
|
+
request_scope_id: str,
|
|
2025
|
+
) -> None:
|
|
2026
|
+
current_reservation = getattr(session, "unanchored_reservation_id", None)
|
|
2027
|
+
if current_reservation is not None and current_reservation != request_scope_id:
|
|
2028
|
+
raise ProxyResponseError(
|
|
2029
|
+
409,
|
|
2030
|
+
openai_error(
|
|
2031
|
+
"bridge_session_reserved",
|
|
2032
|
+
"HTTP responses session bridge is reserved by another request",
|
|
2033
|
+
error_type="server_error",
|
|
2034
|
+
),
|
|
2035
|
+
)
|
|
2036
|
+
session.unanchored_reservation_id = request_scope_id
|
|
2037
|
+
|
|
2038
|
+
|
|
2039
|
+
def _release_http_bridge_unanchored_handoff(
|
|
2040
|
+
session: "_HTTPBridgeSession",
|
|
2041
|
+
*,
|
|
2042
|
+
request_scope_id: str,
|
|
2043
|
+
) -> None:
|
|
2044
|
+
if session.unanchored_reservation_id == request_scope_id:
|
|
2045
|
+
session.unanchored_reservation_id = None
|
|
2046
|
+
|
|
2047
|
+
|
|
2048
|
+
async def _refresh_reused_http_bridge_session_with_handoff(
|
|
2049
|
+
service: "_HTTPBridgeServiceProtocol",
|
|
2050
|
+
session: "_HTTPBridgeSession",
|
|
2051
|
+
*,
|
|
2052
|
+
key: "_HTTPBridgeSessionKey",
|
|
2053
|
+
request_scope_id: str,
|
|
2054
|
+
reserve_handoff: bool,
|
|
2055
|
+
) -> None:
|
|
2056
|
+
if reserve_handoff:
|
|
2057
|
+
_reserve_http_bridge_unanchored_handoff(session, request_scope_id=request_scope_id)
|
|
2058
|
+
try:
|
|
2059
|
+
await service._refresh_durable_http_bridge_session(session)
|
|
2060
|
+
_log_http_bridge_event(
|
|
2061
|
+
"reuse",
|
|
2062
|
+
key,
|
|
2063
|
+
account_id=session.account.id,
|
|
2064
|
+
model=session.request_model,
|
|
2065
|
+
pending_count=service._http_bridge_pending_count_nowait(session, context="reuse_log"),
|
|
2066
|
+
cache_key_family=key.affinity_kind,
|
|
2067
|
+
model_class=_extract_model_class(session.request_model) if session.request_model else None,
|
|
2068
|
+
)
|
|
2069
|
+
except BaseException:
|
|
2070
|
+
if reserve_handoff:
|
|
2071
|
+
_release_http_bridge_unanchored_handoff(session, request_scope_id=request_scope_id)
|
|
2072
|
+
raise
|
|
2073
|
+
|
|
2074
|
+
|
|
2075
|
+
def _http_bridge_session_retiring_with_visible_requests(session: "_HTTPBridgeSession") -> bool:
|
|
2076
|
+
# A reserved handoff is not queued yet, but it owns the lane just as a
|
|
2077
|
+
# visible request does and must be allowed to submit before retirement.
|
|
2078
|
+
return session.upstream_control.retire_after_drain and (
|
|
2079
|
+
_http_bridge_session_has_visible_requests(session) or session.unanchored_reservation_id is not None
|
|
2080
|
+
)
|
|
2081
|
+
|
|
2082
|
+
|
|
2083
|
+
def _http_bridge_payload_looks_like_full_resend(payload: ResponsesRequest) -> bool:
|
|
2084
|
+
input_value = payload.input
|
|
2085
|
+
if isinstance(input_value, str):
|
|
2086
|
+
return len(input_value) >= 4096
|
|
2087
|
+
if isinstance(input_value, Sequence) and not isinstance(input_value, (str, bytes, bytearray)):
|
|
2088
|
+
if len(input_value) > 1:
|
|
2089
|
+
return True
|
|
2090
|
+
if len(input_value) == 1:
|
|
2091
|
+
try:
|
|
2092
|
+
return len(json.dumps(input_value[0], ensure_ascii=True, separators=(",", ":"))) >= 4096
|
|
2093
|
+
except TypeError:
|
|
2094
|
+
return False
|
|
2095
|
+
return False
|
|
2096
|
+
|
|
2097
|
+
|
|
2098
|
+
def _preferred_http_bridge_reconnect_turn_state(session: "_HTTPBridgeSession") -> str | None:
|
|
2099
|
+
if (
|
|
2100
|
+
session.codex_session
|
|
2101
|
+
and session.downstream_turn_state is not None
|
|
2102
|
+
and session.affinity.kind == StickySessionKind.CODEX_SESSION
|
|
2103
|
+
and session.affinity.key == session.downstream_turn_state
|
|
2104
|
+
):
|
|
2105
|
+
return session.downstream_turn_state
|
|
2106
|
+
return session.upstream_turn_state
|
|
2107
|
+
|
|
2108
|
+
|
|
2109
|
+
def _http_bridge_reconnect_turn_state(
|
|
2110
|
+
session: "_HTTPBridgeSession",
|
|
2111
|
+
account_id: str,
|
|
2112
|
+
owner_rebind_affinity: _AffinityPolicy | None,
|
|
2113
|
+
) -> str | None:
|
|
2114
|
+
"""Return the turn state the replacement handshake for ``account_id`` may carry.
|
|
2115
|
+
|
|
2116
|
+
The turn state was learned from the retired socket and belongs to the
|
|
2117
|
+
account that issued it. Only a reconnect to that same account offers it
|
|
2118
|
+
again; a replacement account (or an owner rebind) opens its socket with no
|
|
2119
|
+
turn state -- the same condition under which the reconnect clears the
|
|
2120
|
+
session's turn state once the replacement socket is open, so the handshake
|
|
2121
|
+
cannot leak what the session-side cleanup is about to drop
|
|
2122
|
+
(``responses-api-compat``: "Cross-account bridge retries clear turn-state").
|
|
2123
|
+
"""
|
|
2124
|
+
if owner_rebind_affinity is not None or account_id != session.account.id:
|
|
2125
|
+
return None
|
|
2126
|
+
return _preferred_http_bridge_reconnect_turn_state(session)
|
|
2127
|
+
|
|
2128
|
+
|
|
2129
|
+
def _http_bridge_turn_state_alias_key(turn_state: str, api_key_id: str | None) -> tuple[str, str | None]:
|
|
2130
|
+
return (turn_state, api_key_id)
|
|
2131
|
+
|
|
2132
|
+
|
|
2133
|
+
def _http_bridge_live_turn_state_alias_owner(
|
|
2134
|
+
service: _HTTPBridgeServiceProtocol,
|
|
2135
|
+
session: _HTTPBridgeSession,
|
|
2136
|
+
turn_state: str,
|
|
2137
|
+
) -> _HTTPBridgeSession | None:
|
|
2138
|
+
alias_key = _http_bridge_turn_state_alias_key(turn_state, session.key.api_key_id)
|
|
2139
|
+
existing_key = service._http_bridge_turn_state_index.get(alias_key)
|
|
2140
|
+
if existing_key is None or existing_key == session.key:
|
|
2141
|
+
return None
|
|
2142
|
+
existing_session = service._http_bridge_sessions.get(existing_key)
|
|
2143
|
+
if _http_bridge_alias_target_is_stale(existing_session):
|
|
2144
|
+
return None
|
|
2145
|
+
return existing_session
|
|
2146
|
+
|
|
2147
|
+
|
|
2148
|
+
def _register_http_bridge_turn_state_aliases_locked(
|
|
2149
|
+
service: _HTTPBridgeServiceProtocol,
|
|
2150
|
+
session: _HTTPBridgeSession,
|
|
2151
|
+
) -> None:
|
|
2152
|
+
for alias in session.downstream_turn_state_aliases:
|
|
2153
|
+
alias_key = _http_bridge_turn_state_alias_key(alias, session.key.api_key_id)
|
|
2154
|
+
existing_key = service._http_bridge_turn_state_index.get(alias_key)
|
|
2155
|
+
if existing_key is not None and existing_key != session.key:
|
|
2156
|
+
existing_session = service._http_bridge_sessions.get(existing_key)
|
|
2157
|
+
if not _http_bridge_alias_target_is_stale(existing_session):
|
|
2158
|
+
continue
|
|
2159
|
+
service._http_bridge_turn_state_index[alias_key] = session.key
|
|
2160
|
+
|
|
2161
|
+
|
|
2162
|
+
def _http_bridge_previous_response_alias_key(response_id: str, api_key_id: str | None) -> tuple[str, str | None]:
|
|
2163
|
+
return (response_id.strip(), api_key_id)
|
|
2164
|
+
|
|
2165
|
+
|
|
2166
|
+
def _http_bridge_live_previous_response_alias_owner(
|
|
2167
|
+
service: _HTTPBridgeServiceProtocol,
|
|
2168
|
+
session: _HTTPBridgeSession,
|
|
2169
|
+
response_id: str,
|
|
2170
|
+
) -> _HTTPBridgeSession | None:
|
|
2171
|
+
alias_key = _http_bridge_previous_response_alias_key(response_id, session.key.api_key_id)
|
|
2172
|
+
existing_key = service._http_bridge_previous_response_index.get(alias_key)
|
|
2173
|
+
if existing_key is None or existing_key == session.key:
|
|
2174
|
+
return None
|
|
2175
|
+
existing_session = service._http_bridge_sessions.get(existing_key)
|
|
2176
|
+
if _http_bridge_alias_target_is_stale(existing_session):
|
|
2177
|
+
return None
|
|
2178
|
+
return existing_session
|
|
2179
|
+
|
|
2180
|
+
|
|
2181
|
+
def _http_bridge_session_allows_api_key(session: "_HTTPBridgeSession", api_key: ApiKeyData | None) -> bool:
|
|
2182
|
+
if api_key is None or not api_key.account_assignment_scope_enabled:
|
|
2183
|
+
return True
|
|
2184
|
+
return session.account.id in api_key.assigned_account_ids
|
|
2185
|
+
|
|
2186
|
+
|
|
2187
|
+
def _http_bridge_session_account_active(session: "_HTTPBridgeSession") -> bool:
|
|
2188
|
+
# Hot-path invariant: this reuse check MUST stay a pure in-memory lookup (no
|
|
2189
|
+
# per-request database reads). Cross-replica freshness comes from the
|
|
2190
|
+
# `account_routing` cache-invalidation namespace refreshing the routing
|
|
2191
|
+
# availability snapshot behind is_account_routing_unavailable().
|
|
2192
|
+
return (
|
|
2193
|
+
session.account.status in (AccountStatus.ACTIVE, AccountStatus.REAUTH_REQUIRED)
|
|
2194
|
+
and not reauth_access_token_is_expired(
|
|
2195
|
+
session.account.status,
|
|
2196
|
+
session.access_token_expires_at,
|
|
2197
|
+
)
|
|
2198
|
+
and not is_account_routing_unavailable(session.account.id)
|
|
2199
|
+
)
|
|
2200
|
+
|
|
2201
|
+
|
|
2202
|
+
def _http_bridge_session_reusable_for_request(
|
|
2203
|
+
*,
|
|
2204
|
+
session: "_HTTPBridgeSession",
|
|
2205
|
+
key: "_HTTPBridgeSessionKey",
|
|
2206
|
+
incoming_turn_state: str | None,
|
|
2207
|
+
previous_response_id: str | None,
|
|
2208
|
+
) -> bool:
|
|
2209
|
+
if session.upstream_control.retire_after_drain:
|
|
2210
|
+
return False
|
|
2211
|
+
if key.affinity_kind != "prompt_cache":
|
|
2212
|
+
return True
|
|
2213
|
+
if incoming_turn_state is not None:
|
|
2214
|
+
return True
|
|
2215
|
+
if previous_response_id is not None:
|
|
2216
|
+
return True
|
|
2217
|
+
return not session.codex_session
|
|
2218
|
+
|
|
2219
|
+
|
|
2220
|
+
def _http_bridge_session_matches_preferred_account(
|
|
2221
|
+
*,
|
|
2222
|
+
session: "_HTTPBridgeSession",
|
|
2223
|
+
previous_response_id: str | None,
|
|
2224
|
+
preferred_account_id: str | None,
|
|
2225
|
+
require_preferred_account: bool = False,
|
|
2226
|
+
) -> bool:
|
|
2227
|
+
if preferred_account_id is None:
|
|
2228
|
+
return True
|
|
2229
|
+
if previous_response_id is None and not require_preferred_account:
|
|
2230
|
+
return True
|
|
2231
|
+
return session.account.id == preferred_account_id
|
|
2232
|
+
|
|
2233
|
+
|
|
2234
|
+
def _http_bridge_session_reusable_for_lookup(
|
|
2235
|
+
*,
|
|
2236
|
+
session: "_HTTPBridgeSession",
|
|
2237
|
+
key: "_HTTPBridgeSessionKey",
|
|
2238
|
+
api_key: ApiKeyData | None,
|
|
2239
|
+
incoming_turn_state: str | None,
|
|
2240
|
+
previous_response_id: str | None,
|
|
2241
|
+
preferred_account_id: str | None,
|
|
2242
|
+
require_preferred_account: bool,
|
|
2243
|
+
service_tier_supported: bool,
|
|
2244
|
+
allow_closed_admission_handoff: bool,
|
|
2245
|
+
session_key_quarantined: bool,
|
|
2246
|
+
) -> bool:
|
|
2247
|
+
if session.quarantined and not session_key_quarantined:
|
|
2248
|
+
# The quarantine registry TTL is the source of truth
|
|
2249
|
+
# (``_http_bridge_session_key_quarantined``): pruning drops the
|
|
2250
|
+
# registry entry without touching the per-session flag, so a live
|
|
2251
|
+
# session that outlives its quarantine window must become reusable
|
|
2252
|
+
# again instead of staying rejected forever. Reset the stale flag so
|
|
2253
|
+
# session state agrees with the registry.
|
|
2254
|
+
session.quarantined = False
|
|
2255
|
+
live_or_retained = _http_bridge_session_account_active(session) and (
|
|
2256
|
+
not session.closed or (allow_closed_admission_handoff and _http_bridge_session_has_admission_waiter(session))
|
|
2257
|
+
)
|
|
2258
|
+
return (
|
|
2259
|
+
live_or_retained
|
|
2260
|
+
# A quarantined key has proven silent/wedged; a new request must take
|
|
2261
|
+
# the fresh session path instead of re-attaching. The registry verdict
|
|
2262
|
+
# is authoritative for the key in both directions: a freshly created
|
|
2263
|
+
# replacement session (flag still False) under a still-quarantined key
|
|
2264
|
+
# is not reusable either, so a concurrent full-resend cannot restore
|
|
2265
|
+
# the suppressed durable anchor through session hydration.
|
|
2266
|
+
and not session_key_quarantined
|
|
2267
|
+
and _http_bridge_session_allows_api_key(session, api_key)
|
|
2268
|
+
and _http_bridge_session_reusable_for_request(
|
|
2269
|
+
session=session,
|
|
2270
|
+
key=key,
|
|
2271
|
+
incoming_turn_state=incoming_turn_state,
|
|
2272
|
+
previous_response_id=previous_response_id,
|
|
2273
|
+
)
|
|
2274
|
+
and _http_bridge_session_matches_preferred_account(
|
|
2275
|
+
session=session,
|
|
2276
|
+
previous_response_id=previous_response_id,
|
|
2277
|
+
preferred_account_id=preferred_account_id,
|
|
2278
|
+
require_preferred_account=require_preferred_account,
|
|
2279
|
+
)
|
|
2280
|
+
and service_tier_supported
|
|
2281
|
+
)
|
|
2282
|
+
|
|
2283
|
+
|
|
2284
|
+
def _require_http_bridge_bound_account_not_excluded(
|
|
2285
|
+
hard_account_bound: bool,
|
|
2286
|
+
account_id: str,
|
|
2287
|
+
excluded_account_ids: set[str],
|
|
2288
|
+
) -> None:
|
|
2289
|
+
if hard_account_bound and account_id in excluded_account_ids:
|
|
2290
|
+
raise ProxyResponseError(
|
|
2291
|
+
502,
|
|
2292
|
+
openai_error(
|
|
2293
|
+
"upstream_unavailable",
|
|
2294
|
+
"HTTP responses session bridge continuity account is excluded",
|
|
2295
|
+
),
|
|
2296
|
+
)
|
|
2297
|
+
|
|
2298
|
+
|
|
2299
|
+
def _record_http_bridge_handoff_compatibility_rejection(
|
|
2300
|
+
*,
|
|
2301
|
+
session: "_HTTPBridgeSession",
|
|
2302
|
+
key: "_HTTPBridgeSessionKey",
|
|
2303
|
+
api_key: ApiKeyData | None,
|
|
2304
|
+
incoming_turn_state: str | None,
|
|
2305
|
+
previous_response_id: str | None,
|
|
2306
|
+
preferred_account_id: str | None,
|
|
2307
|
+
require_preferred_account: bool,
|
|
2308
|
+
request_service_tier: str | None,
|
|
2309
|
+
service_tier_supported: bool,
|
|
2310
|
+
) -> None:
|
|
2311
|
+
continuity_anchor = (
|
|
2312
|
+
"previous_response_id"
|
|
2313
|
+
if previous_response_id is not None
|
|
2314
|
+
else "turn_state"
|
|
2315
|
+
if incoming_turn_state is not None
|
|
2316
|
+
else "none"
|
|
2317
|
+
)
|
|
2318
|
+
preferred_account = (
|
|
2319
|
+
"required" if require_preferred_account else "preferred" if preferred_account_id is not None else "none"
|
|
2320
|
+
)
|
|
2321
|
+
service_tier = (
|
|
2322
|
+
"not_requested" if request_service_tier is None else "supported" if service_tier_supported else "unsupported"
|
|
2323
|
+
)
|
|
2324
|
+
api_key_scope = "scoped" if api_key is not None and api_key.id is not None else "unscoped"
|
|
2325
|
+
if PROMETHEUS_AVAILABLE and bridge_handoff_compatibility_rejection_total is not None:
|
|
2326
|
+
bridge_handoff_compatibility_rejection_total.labels(
|
|
2327
|
+
continuity_anchor=continuity_anchor,
|
|
2328
|
+
preferred_account=preferred_account,
|
|
2329
|
+
service_tier=service_tier,
|
|
2330
|
+
api_key_scope=api_key_scope,
|
|
2331
|
+
).inc()
|
|
2332
|
+
logger.warning(
|
|
2333
|
+
"http_bridge_handoff_compatibility_rejected bridge_kind=%s bridge_key=%s key_strength=%s "
|
|
2334
|
+
"continuity_anchor=%s preferred_account=%s preferred_account_id=%s require_preferred=%s "
|
|
2335
|
+
"service_tier=%s api_key_scope=%s existing_closed=%s admission_waiters=%s pending_count=%s",
|
|
2336
|
+
key.affinity_kind,
|
|
2337
|
+
_hash_identifier(key.affinity_key),
|
|
2338
|
+
_http_bridge_key_strength(key),
|
|
2339
|
+
continuity_anchor,
|
|
2340
|
+
preferred_account,
|
|
2341
|
+
_hash_identifier_or_none(preferred_account_id),
|
|
2342
|
+
require_preferred_account,
|
|
2343
|
+
service_tier,
|
|
2344
|
+
api_key_scope,
|
|
2345
|
+
session.closed,
|
|
2346
|
+
getattr(session, "admission_waiter_count", 0),
|
|
2347
|
+
len(session.pending_requests),
|
|
2348
|
+
)
|
|
2349
|
+
|
|
2350
|
+
|
|
2351
|
+
def _record_http_bridge_unanchored_handoff_recovery(*, reason: str) -> None:
|
|
2352
|
+
if PROMETHEUS_AVAILABLE and bridge_unanchored_handoff_recovery_total is not None:
|
|
2353
|
+
bridge_unanchored_handoff_recovery_total.labels(reason=reason).inc()
|
|
2354
|
+
|
|
2355
|
+
|
|
2356
|
+
def _raise_http_bridge_incompatible_admission_handoff(
|
|
2357
|
+
*,
|
|
2358
|
+
session: "_HTTPBridgeSession",
|
|
2359
|
+
key: "_HTTPBridgeSessionKey",
|
|
2360
|
+
api_key: ApiKeyData | None,
|
|
2361
|
+
incoming_turn_state: str | None,
|
|
2362
|
+
previous_response_id: str | None,
|
|
2363
|
+
preferred_account_id: str | None,
|
|
2364
|
+
require_preferred_account: bool,
|
|
2365
|
+
request_service_tier: str | None,
|
|
2366
|
+
service_tier_supported: bool,
|
|
2367
|
+
) -> None:
|
|
2368
|
+
_record_http_bridge_handoff_compatibility_rejection(
|
|
2369
|
+
session=session,
|
|
2370
|
+
key=key,
|
|
2371
|
+
api_key=api_key,
|
|
2372
|
+
incoming_turn_state=incoming_turn_state,
|
|
2373
|
+
previous_response_id=previous_response_id,
|
|
2374
|
+
preferred_account_id=preferred_account_id,
|
|
2375
|
+
require_preferred_account=require_preferred_account,
|
|
2376
|
+
request_service_tier=request_service_tier,
|
|
2377
|
+
service_tier_supported=service_tier_supported,
|
|
2378
|
+
)
|
|
2379
|
+
raise ProxyResponseError(
|
|
2380
|
+
503,
|
|
2381
|
+
openai_error(
|
|
2382
|
+
"upstream_unavailable",
|
|
2383
|
+
"HTTP responses session bridge is preserving an incompatible admission handoff",
|
|
2384
|
+
),
|
|
2385
|
+
)
|
|
2386
|
+
|
|
2387
|
+
|
|
2388
|
+
def _make_http_bridge_session_key(
|
|
2389
|
+
payload: ResponsesRequest,
|
|
2390
|
+
*,
|
|
2391
|
+
headers: Mapping[str, str],
|
|
2392
|
+
affinity: _AffinityPolicy,
|
|
2393
|
+
api_key: ApiKeyData | None,
|
|
2394
|
+
request_id: str,
|
|
2395
|
+
explicit_prompt_cache_key: str | None = None,
|
|
2396
|
+
allow_forwarded_affinity_headers: bool = False,
|
|
2397
|
+
forwarded_affinity_kind: str | None = None,
|
|
2398
|
+
forwarded_affinity_key: str | None = None,
|
|
2399
|
+
) -> _HTTPBridgeSessionKey:
|
|
2400
|
+
forwarded_key = (
|
|
2401
|
+
_forwarded_http_bridge_session_key(
|
|
2402
|
+
headers,
|
|
2403
|
+
api_key,
|
|
2404
|
+
forwarded_affinity_kind=forwarded_affinity_kind,
|
|
2405
|
+
forwarded_affinity_key=forwarded_affinity_key,
|
|
2406
|
+
)
|
|
2407
|
+
if allow_forwarded_affinity_headers
|
|
2408
|
+
else None
|
|
2409
|
+
)
|
|
2410
|
+
if forwarded_key is not None:
|
|
2411
|
+
return forwarded_key
|
|
2412
|
+
turn_state_key = _sticky_key_from_turn_state_header(headers)
|
|
2413
|
+
if turn_state_key is not None:
|
|
2414
|
+
affinity_key = turn_state_key
|
|
2415
|
+
affinity_kind = "turn_state_header"
|
|
2416
|
+
strength: Literal["hard", "soft"] = "hard"
|
|
2417
|
+
elif (thread_key := _codex_backend_identity(headers).thread_selection_key) is not None:
|
|
2418
|
+
# prompt_cache_key is intentionally shared by current Codex root trees.
|
|
2419
|
+
# The thread key is canonical identity; once a bridge exists it is hard
|
|
2420
|
+
# transport continuity even though pre-bridge account locality is soft.
|
|
2421
|
+
affinity_key = thread_key
|
|
2422
|
+
affinity_kind = "thread_header"
|
|
2423
|
+
strength = "hard"
|
|
2424
|
+
else:
|
|
2425
|
+
session_key = _sticky_key_from_session_header(headers)
|
|
2426
|
+
if session_key is not None:
|
|
2427
|
+
# Compatibility path for clients that do not expose thread-id.
|
|
2428
|
+
# Current Codex reaches the thread_header branch above; do not
|
|
2429
|
+
# reintroduce prompt_cache_key as thread identity here.
|
|
2430
|
+
session_header_key = _make_http_bridge_session_header_fallback_key(
|
|
2431
|
+
headers=headers,
|
|
2432
|
+
api_key=api_key,
|
|
2433
|
+
explicit_prompt_cache_key=explicit_prompt_cache_key,
|
|
2434
|
+
)
|
|
2435
|
+
assert session_header_key is not None
|
|
2436
|
+
affinity_key = session_header_key.affinity_key
|
|
2437
|
+
affinity_kind = "session_header"
|
|
2438
|
+
strength = "hard"
|
|
2439
|
+
else:
|
|
2440
|
+
inferred_key = (
|
|
2441
|
+
inferred_http_bridge_key(payload) if payload.conversation or explicit_prompt_cache_key is None else None
|
|
2442
|
+
)
|
|
2443
|
+
affinity_key = inferred_key or affinity.key or request_id
|
|
2444
|
+
affinity_kind = (
|
|
2445
|
+
"prompt_cache" if inferred_key else affinity.kind.value if affinity.kind is not None else "request"
|
|
2446
|
+
)
|
|
2447
|
+
strength = "soft"
|
|
2448
|
+
return _HTTPBridgeSessionKey(
|
|
2449
|
+
affinity_kind=affinity_kind,
|
|
2450
|
+
affinity_key=affinity_key,
|
|
2451
|
+
api_key_id=api_key.id if api_key is not None else None,
|
|
2452
|
+
strength=strength,
|
|
2453
|
+
)
|
|
2454
|
+
|
|
2455
|
+
|
|
2456
|
+
def _make_http_bridge_session_header_fallback_key(
|
|
2457
|
+
*,
|
|
2458
|
+
headers: Mapping[str, str],
|
|
2459
|
+
api_key: ApiKeyData | None,
|
|
2460
|
+
explicit_prompt_cache_key: str | None,
|
|
2461
|
+
) -> _HTTPBridgeSessionKey | None:
|
|
2462
|
+
if _codex_backend_identity(headers).thread_id is not None:
|
|
2463
|
+
# Never let a current thread attach to the legacy
|
|
2464
|
+
# (session-id, prompt_cache_key) lane: both values are shared across
|
|
2465
|
+
# siblings. Exact turn-state/previous-response aliases are handled by
|
|
2466
|
+
# durable lookup independently and remain the only safe migration path.
|
|
2467
|
+
return None
|
|
2468
|
+
session_key = _sticky_key_from_session_header(headers)
|
|
2469
|
+
if session_key is None:
|
|
2470
|
+
return None
|
|
2471
|
+
affinity_key = (
|
|
2472
|
+
sha256(f"{session_key}\0{explicit_prompt_cache_key.strip()}".encode()).hexdigest()
|
|
2473
|
+
if isinstance(explicit_prompt_cache_key, str) and explicit_prompt_cache_key.strip()
|
|
2474
|
+
else session_key
|
|
2475
|
+
)
|
|
2476
|
+
return _HTTPBridgeSessionKey(
|
|
2477
|
+
"session_header",
|
|
2478
|
+
affinity_key,
|
|
2479
|
+
api_key.id if api_key is not None else None,
|
|
2480
|
+
)
|
|
2481
|
+
|
|
2482
|
+
|
|
2483
|
+
def _turn_keys(
|
|
2484
|
+
headers: Mapping[str, str],
|
|
2485
|
+
api_key: ApiKeyData | None,
|
|
2486
|
+
requested_key: _HTTPBridgeSessionKey,
|
|
2487
|
+
fallback_key: _HTTPBridgeSessionKey | None,
|
|
2488
|
+
) -> tuple[str | None, _HTTPBridgeSessionKey | None]:
|
|
2489
|
+
thread_key = _codex_backend_identity(headers).thread_selection_key
|
|
2490
|
+
thread_fallback_key = (
|
|
2491
|
+
_HTTPBridgeSessionKey("thread_header", thread_key, api_key.id if api_key is not None else None)
|
|
2492
|
+
if thread_key is not None
|
|
2493
|
+
else None
|
|
2494
|
+
)
|
|
2495
|
+
incoming_session_key = None if thread_fallback_key is not None else _sticky_key_from_session_header(headers)
|
|
2496
|
+
initial_session_key = (
|
|
2497
|
+
fallback_key
|
|
2498
|
+
or thread_fallback_key
|
|
2499
|
+
or (requested_key if requested_key.affinity_kind == "session_header" else None)
|
|
2500
|
+
)
|
|
2501
|
+
return incoming_session_key, initial_session_key
|
|
2502
|
+
|
|
2503
|
+
|
|
2504
|
+
def _alias_fallback_key(
|
|
2505
|
+
incoming_session_key: str | None,
|
|
2506
|
+
initial_session_key: _HTTPBridgeSessionKey | None,
|
|
2507
|
+
api_key_id: str | None,
|
|
2508
|
+
) -> _HTTPBridgeSessionKey | None:
|
|
2509
|
+
if initial_session_key is not None:
|
|
2510
|
+
return initial_session_key
|
|
2511
|
+
if incoming_session_key is None:
|
|
2512
|
+
return None
|
|
2513
|
+
return _HTTPBridgeSessionKey("session_header", incoming_session_key, api_key_id)
|
|
2514
|
+
|
|
2515
|
+
|
|
2516
|
+
async def _http_bridge_should_wait_for_registration(
|
|
2517
|
+
self,
|
|
2518
|
+
key: _HTTPBridgeSessionKey,
|
|
2519
|
+
settings: Settings,
|
|
2520
|
+
) -> bool:
|
|
2521
|
+
import app.core.startup as startup_module
|
|
2522
|
+
|
|
2523
|
+
if startup_module._bridge_registration_complete:
|
|
2524
|
+
return False
|
|
2525
|
+
if key.strength != "hard":
|
|
2526
|
+
return False
|
|
2527
|
+
if _http_bridge_requires_cluster_registration(settings):
|
|
2528
|
+
return True
|
|
2529
|
+
if self._ring_membership is None:
|
|
2530
|
+
return False
|
|
2531
|
+
try:
|
|
2532
|
+
active_members = await self._ring_membership.list_active()
|
|
2533
|
+
except Exception:
|
|
2534
|
+
logger.debug("Skipping bridge registration gate because active ring lookup failed", exc_info=True)
|
|
2535
|
+
return False
|
|
2536
|
+
current_instance = settings.http_responses_session_bridge_instance_id
|
|
2537
|
+
return any(member != current_instance for member in active_members)
|
|
2538
|
+
|
|
2539
|
+
|
|
2540
|
+
def _durable_bridge_lookup_active_owner(lookup: DurableBridgeLookup | None) -> str | None:
|
|
2541
|
+
if lookup is None:
|
|
2542
|
+
return None
|
|
2543
|
+
if lookup.state == "closed":
|
|
2544
|
+
return None
|
|
2545
|
+
if lookup.owner_instance_id is None or lookup.lease_expires_at is None:
|
|
2546
|
+
return None
|
|
2547
|
+
lease_expires_at = to_utc_naive(lookup.lease_expires_at)
|
|
2548
|
+
if lease_expires_at <= utcnow():
|
|
2549
|
+
return None
|
|
2550
|
+
return lookup.owner_instance_id
|
|
2551
|
+
|
|
2552
|
+
|
|
2553
|
+
def _durable_bridge_lookup_allows_local_reuse(
|
|
2554
|
+
lookup: DurableBridgeLookup | None,
|
|
2555
|
+
*,
|
|
2556
|
+
current_instance: str,
|
|
2557
|
+
) -> bool:
|
|
2558
|
+
if lookup is None:
|
|
2559
|
+
return True
|
|
2560
|
+
owner_instance = _durable_bridge_lookup_active_owner(lookup)
|
|
2561
|
+
if owner_instance is None:
|
|
2562
|
+
return True
|
|
2563
|
+
return owner_instance == current_instance
|
|
2564
|
+
|
|
2565
|
+
|
|
2566
|
+
def _http_bridge_allow_durable_takeover(lookup: DurableBridgeLookup | None) -> bool:
|
|
2567
|
+
return _http_bridge_durable_lookup_allows_turn_state_takeover(lookup)
|
|
2568
|
+
|
|
2569
|
+
|
|
2570
|
+
def _http_bridge_claim_allows_takeover(
|
|
2571
|
+
lookup: DurableBridgeLookup | None,
|
|
2572
|
+
*,
|
|
2573
|
+
force: bool,
|
|
2574
|
+
) -> bool:
|
|
2575
|
+
if _http_bridge_allow_durable_takeover(lookup):
|
|
2576
|
+
return True
|
|
2577
|
+
if not force:
|
|
2578
|
+
return False
|
|
2579
|
+
return lookup is None or lookup.state != "draining"
|
|
2580
|
+
|
|
2581
|
+
|
|
2582
|
+
def _http_bridge_has_durable_recovery_anchor(
|
|
2583
|
+
*,
|
|
2584
|
+
previous_response_id: str | None,
|
|
2585
|
+
durable_lookup: DurableBridgeLookup | None,
|
|
2586
|
+
) -> bool:
|
|
2587
|
+
if previous_response_id is not None:
|
|
2588
|
+
return True
|
|
2589
|
+
if durable_lookup is None or durable_lookup.latest_response_id is None:
|
|
2590
|
+
return False
|
|
2591
|
+
return durable_lookup.canonical_kind in _HARD_HTTP_BRIDGE_AFFINITY_KINDS
|
|
2592
|
+
|
|
2593
|
+
|
|
2594
|
+
def _http_bridge_can_local_recover_without_ring(
|
|
2595
|
+
*,
|
|
2596
|
+
key: _HTTPBridgeSessionKey,
|
|
2597
|
+
headers: Mapping[str, str],
|
|
2598
|
+
previous_response_id: str | None,
|
|
2599
|
+
durable_lookup: DurableBridgeLookup | None,
|
|
2600
|
+
) -> bool:
|
|
2601
|
+
if _http_bridge_has_durable_recovery_anchor(
|
|
2602
|
+
previous_response_id=previous_response_id,
|
|
2603
|
+
durable_lookup=durable_lookup,
|
|
2604
|
+
):
|
|
2605
|
+
return True
|
|
2606
|
+
return (
|
|
2607
|
+
key.affinity_kind in {"session_header", "thread_header"}
|
|
2608
|
+
and previous_response_id is None
|
|
2609
|
+
and _sticky_key_from_turn_state_header(headers) is None
|
|
2610
|
+
)
|
|
2611
|
+
|
|
2612
|
+
|
|
2613
|
+
def _http_bridge_can_single_instance_owner_takeover_without_anchor(
|
|
2614
|
+
*,
|
|
2615
|
+
key: _HTTPBridgeSessionKey,
|
|
2616
|
+
owner_instance: str | None,
|
|
2617
|
+
current_instance: str,
|
|
2618
|
+
ring: tuple[str, ...],
|
|
2619
|
+
) -> bool:
|
|
2620
|
+
if key.strength != "hard":
|
|
2621
|
+
return False
|
|
2622
|
+
if owner_instance is None or owner_instance == current_instance:
|
|
2623
|
+
return False
|
|
2624
|
+
if len(ring) != 1:
|
|
2625
|
+
return False
|
|
2626
|
+
if ring[0] != current_instance:
|
|
2627
|
+
return False
|
|
2628
|
+
return owner_instance not in ring
|
|
2629
|
+
|
|
2630
|
+
|
|
2631
|
+
def _http_bridge_can_single_instance_prompt_cache_takeover_without_anchor(
|
|
2632
|
+
*,
|
|
2633
|
+
key: _HTTPBridgeSessionKey,
|
|
2634
|
+
owner_instance: str | None,
|
|
2635
|
+
current_instance: str,
|
|
2636
|
+
ring: tuple[str, ...],
|
|
2637
|
+
) -> bool:
|
|
2638
|
+
if key.affinity_kind != "prompt_cache":
|
|
2639
|
+
return False
|
|
2640
|
+
if owner_instance is None or owner_instance == current_instance:
|
|
2641
|
+
return False
|
|
2642
|
+
if len(ring) != 1:
|
|
2643
|
+
return False
|
|
2644
|
+
if ring[0] != current_instance:
|
|
2645
|
+
return False
|
|
2646
|
+
return owner_instance not in ring
|
|
2647
|
+
|
|
2648
|
+
|
|
2649
|
+
def _http_bridge_endpoint_matches_current_instance(owner_endpoint: str, settings: Settings) -> bool:
|
|
2650
|
+
current_endpoint = settings.http_responses_session_bridge_advertise_base_url
|
|
2651
|
+
if current_endpoint is None:
|
|
2652
|
+
return False
|
|
2653
|
+
return owner_endpoint.strip().rstrip("/") == current_endpoint.strip().rstrip("/")
|
|
2654
|
+
|
|
2655
|
+
|
|
2656
|
+
def _http_bridge_can_recover_during_drain(
|
|
2657
|
+
*,
|
|
2658
|
+
key: _HTTPBridgeSessionKey,
|
|
2659
|
+
headers: Mapping[str, str],
|
|
2660
|
+
previous_response_id: str | None,
|
|
2661
|
+
durable_lookup: DurableBridgeLookup | None,
|
|
2662
|
+
) -> bool:
|
|
2663
|
+
return _http_bridge_has_durable_recovery_anchor(
|
|
2664
|
+
previous_response_id=previous_response_id,
|
|
2665
|
+
durable_lookup=durable_lookup,
|
|
2666
|
+
)
|
|
2667
|
+
|
|
2668
|
+
|
|
2669
|
+
def _http_bridge_request_stage(
|
|
2670
|
+
*,
|
|
2671
|
+
headers: Mapping[str, str],
|
|
2672
|
+
payload: ResponsesRequest,
|
|
2673
|
+
durable_lookup: DurableBridgeLookup | None,
|
|
2674
|
+
) -> str:
|
|
2675
|
+
del durable_lookup
|
|
2676
|
+
if (
|
|
2677
|
+
payload.previous_response_id is not None
|
|
2678
|
+
or _sticky_key_from_turn_state_header(headers) is not None
|
|
2679
|
+
or _sticky_key_from_session_header(headers) is not None
|
|
2680
|
+
):
|
|
2681
|
+
return "follow_up"
|
|
2682
|
+
return "first_turn"
|
|
2683
|
+
|
|
2684
|
+
|
|
2685
|
+
def _record_bridge_reattach(*, path: str, outcome: str) -> None:
|
|
2686
|
+
if PROMETHEUS_AVAILABLE and bridge_reattach_total is not None:
|
|
2687
|
+
bridge_reattach_total.labels(path=path, outcome=outcome).inc()
|
|
2688
|
+
|
|
2689
|
+
|
|
2690
|
+
def _record_bridge_first_turn_timeout() -> None:
|
|
2691
|
+
if PROMETHEUS_AVAILABLE and bridge_first_turn_timeout_total is not None:
|
|
2692
|
+
bridge_first_turn_timeout_total.inc()
|
|
2693
|
+
|
|
2694
|
+
|
|
2695
|
+
def _record_bridge_drain_recovery_allowed() -> None:
|
|
2696
|
+
if PROMETHEUS_AVAILABLE and bridge_drain_recovery_allowed_total is not None:
|
|
2697
|
+
bridge_drain_recovery_allowed_total.inc()
|
|
2698
|
+
|
|
2699
|
+
|
|
2700
|
+
def _is_missing_durable_bridge_table_error(exc: Exception) -> bool:
|
|
2701
|
+
message = str(exc).lower()
|
|
2702
|
+
if "http_bridge_sessions" not in message and "http_bridge_session_aliases" not in message:
|
|
2703
|
+
return False
|
|
2704
|
+
return "no such table" in message or "does not exist" in message or "undefinedtable" in message
|
|
2705
|
+
|
|
2706
|
+
|
|
2707
|
+
def _http_bridge_durable_lease_ttl_seconds() -> float:
|
|
2708
|
+
return float(RING_STALE_THRESHOLD_SECONDS)
|
|
2709
|
+
|
|
2710
|
+
|
|
2711
|
+
def _http_bridge_durable_release_allowed(service: Any, session: Any) -> bool:
|
|
2712
|
+
session_id = getattr(session, "durable_session_id", None)
|
|
2713
|
+
owner_epoch = getattr(session, "durable_owner_epoch", None)
|
|
2714
|
+
if session_id is None or owner_epoch is None:
|
|
2715
|
+
return False
|
|
2716
|
+
return not any(
|
|
2717
|
+
not task.done() and getattr(task, "_http_bridge_recovery_session_id", None) == session_id
|
|
2718
|
+
for task in service._background_cleanup_tasks
|
|
2719
|
+
)
|
|
2720
|
+
|
|
2721
|
+
|
|
2722
|
+
async def _drain_cancelled_task(task: asyncio.Task[Any]) -> None:
|
|
2723
|
+
await asyncio.gather(task, return_exceptions=True)
|
|
2724
|
+
|
|
2725
|
+
|
|
2726
|
+
def _cancel_and_track_cancelled_task(
|
|
2727
|
+
task: asyncio.Task[Any],
|
|
2728
|
+
*,
|
|
2729
|
+
label: str,
|
|
2730
|
+
cleanup_tasks: set[asyncio.Task[None]] | None,
|
|
2731
|
+
cancel_task: bool = True,
|
|
2732
|
+
scheduler: Scheduler = REAL_SCHEDULER,
|
|
2733
|
+
) -> None:
|
|
2734
|
+
if cancel_task:
|
|
2735
|
+
task.cancel()
|
|
2736
|
+
cleanup_task = scheduler.create_task(_drain_cancelled_task(task), name=f"cancelled-task-cleanup-{label}")
|
|
2737
|
+
if cleanup_tasks is not None:
|
|
2738
|
+
cleanup_tasks.add(cleanup_task)
|
|
2739
|
+
cleanup_task.add_done_callback(cleanup_tasks.discard)
|
|
2740
|
+
|
|
2741
|
+
|
|
2742
|
+
async def _await_cancelled_task(
|
|
2743
|
+
task: asyncio.Task[_TaskResultT],
|
|
2744
|
+
*,
|
|
2745
|
+
timeout_seconds: float = _TASK_CANCEL_TIMEOUT_SECONDS,
|
|
2746
|
+
label: str,
|
|
2747
|
+
cancel: bool = True,
|
|
2748
|
+
cleanup_tasks: set[asyncio.Task[None]] | None = None,
|
|
2749
|
+
scheduler: Scheduler = REAL_SCHEDULER,
|
|
2750
|
+
) -> bool:
|
|
2751
|
+
effective_timeout = max(float(timeout_seconds), 0.0)
|
|
2752
|
+
remaining_drain_timeout = shutdown_state.remaining_drain_timeout_seconds()
|
|
2753
|
+
if remaining_drain_timeout is not None:
|
|
2754
|
+
effective_timeout = min(effective_timeout, remaining_drain_timeout)
|
|
2755
|
+
|
|
2756
|
+
# Give a new child one scheduling turn before cancellation so
|
|
2757
|
+
# cancellation-resistant tasks enter the deferred-drain path.
|
|
2758
|
+
if cancel and not task.done():
|
|
2759
|
+
try:
|
|
2760
|
+
await asyncio.sleep(0)
|
|
2761
|
+
except asyncio.CancelledError:
|
|
2762
|
+
_cancel_and_track_cancelled_task(task, label=label, cleanup_tasks=cleanup_tasks, scheduler=scheduler)
|
|
2763
|
+
raise
|
|
2764
|
+
if cancel:
|
|
2765
|
+
task.cancel()
|
|
2766
|
+
try:
|
|
2767
|
+
done, _ = await scheduler.wait({task}, timeout=effective_timeout)
|
|
2768
|
+
except asyncio.CancelledError:
|
|
2769
|
+
if not task.done():
|
|
2770
|
+
_cancel_and_track_cancelled_task(
|
|
2771
|
+
task,
|
|
2772
|
+
label=label,
|
|
2773
|
+
cleanup_tasks=cleanup_tasks,
|
|
2774
|
+
cancel_task=False,
|
|
2775
|
+
scheduler=scheduler,
|
|
2776
|
+
)
|
|
2777
|
+
raise
|
|
2778
|
+
if task not in done:
|
|
2779
|
+
logger.warning("Timed out waiting for %s cancellation", label)
|
|
2780
|
+
_cancel_and_track_cancelled_task(
|
|
2781
|
+
task,
|
|
2782
|
+
label=label,
|
|
2783
|
+
cleanup_tasks=cleanup_tasks,
|
|
2784
|
+
cancel_task=False,
|
|
2785
|
+
scheduler=scheduler,
|
|
2786
|
+
)
|
|
2787
|
+
return False
|
|
2788
|
+
try:
|
|
2789
|
+
task.result()
|
|
2790
|
+
except asyncio.CancelledError:
|
|
2791
|
+
return True
|
|
2792
|
+
return True
|
|
2793
|
+
|
|
2794
|
+
|
|
2795
|
+
async def _persist_http_bridge_replacement_account(
|
|
2796
|
+
service: _HTTPBridgeServiceProtocol,
|
|
2797
|
+
session: _HTTPBridgeSession,
|
|
2798
|
+
account_id: str,
|
|
2799
|
+
) -> None:
|
|
2800
|
+
if account_id == session.account.id or session.durable_session_id is None or session.durable_owner_epoch is None:
|
|
2801
|
+
return
|
|
2802
|
+
try:
|
|
2803
|
+
rebound = await service._durable_bridge.rebind_session_account(
|
|
2804
|
+
session_id=session.durable_session_id,
|
|
2805
|
+
api_key_id=session.key.api_key_id,
|
|
2806
|
+
instance_id=_service_get_settings().http_responses_session_bridge_instance_id,
|
|
2807
|
+
owner_epoch=session.durable_owner_epoch,
|
|
2808
|
+
account_id=account_id,
|
|
2809
|
+
clear_continuity=True,
|
|
2810
|
+
)
|
|
2811
|
+
except Exception as exc:
|
|
2812
|
+
raise ProxyResponseError(
|
|
2813
|
+
502,
|
|
2814
|
+
openai_error(
|
|
2815
|
+
"bridge_continuity_persistence_failed",
|
|
2816
|
+
"HTTP responses session account could not be persisted; retry the request.",
|
|
2817
|
+
),
|
|
2818
|
+
) from exc
|
|
2819
|
+
if not rebound:
|
|
2820
|
+
raise ProxyResponseError(
|
|
2821
|
+
502,
|
|
2822
|
+
openai_error(
|
|
2823
|
+
"bridge_continuity_persistence_failed",
|
|
2824
|
+
"HTTP responses session ownership changed during account recovery; retry the request.",
|
|
2825
|
+
),
|
|
2826
|
+
)
|
|
2827
|
+
|
|
2828
|
+
|
|
2829
|
+
async def _release_http_bridge_unanchored_handoffs_for_request(
|
|
2830
|
+
service: _HTTPBridgeServiceProtocol,
|
|
2831
|
+
*,
|
|
2832
|
+
request_scope_id: str,
|
|
2833
|
+
) -> None:
|
|
2834
|
+
"""Fail-safe cleanup for reservations published before request submission."""
|
|
2835
|
+
|
|
2836
|
+
async with service._http_bridge_lock:
|
|
2837
|
+
for session in (*service._http_bridge_sessions.values(), *service._http_bridge_detached_sessions.values()):
|
|
2838
|
+
_release_http_bridge_unanchored_handoff(session, request_scope_id=request_scope_id)
|
|
2839
|
+
# Nested stream finalizers can clear their marker before this fail-safe
|
|
2840
|
+
# sweep runs. Reconsider every detached generation so marker ordering
|
|
2841
|
+
# cannot leave a fully drained predecessor owning a socket and cap slot.
|
|
2842
|
+
detached_sessions = tuple(service._http_bridge_detached_sessions.values())
|
|
2843
|
+
deadline = clock_for(service).monotonic() + _HTTP_BRIDGE_DETACHED_RETIRE_LOCK_WAIT_SECONDS
|
|
2844
|
+
for index, session in enumerate(detached_sessions):
|
|
2845
|
+
remaining = deadline - clock_for(service).monotonic()
|
|
2846
|
+
if remaining <= 0:
|
|
2847
|
+
logger.warning(
|
|
2848
|
+
"Detached HTTP bridge retire sweep deadline exhausted: skipped_sessions=%d",
|
|
2849
|
+
len(detached_sessions) - index,
|
|
2850
|
+
)
|
|
2851
|
+
break
|
|
2852
|
+
await service._retire_http_bridge_after_drain_if_ready(
|
|
2853
|
+
session,
|
|
2854
|
+
lock_wait_timeout_seconds=remaining,
|
|
2855
|
+
)
|
|
2856
|
+
|
|
2857
|
+
|
|
2858
|
+
def _track_alias_registration(session: _HTTPBridgeSession, alias: str, *, turn_state: bool) -> int:
|
|
2859
|
+
session.alias_registration_generation += 1
|
|
2860
|
+
generation = session.alias_registration_generation
|
|
2861
|
+
registrations = (
|
|
2862
|
+
session.turn_state_alias_registration_generations
|
|
2863
|
+
if turn_state
|
|
2864
|
+
else session.previous_response_alias_registration_generations
|
|
2865
|
+
)
|
|
2866
|
+
registrations[alias] = generation
|
|
2867
|
+
return generation
|
|
2868
|
+
|
|
2869
|
+
|
|
2870
|
+
async def _persist_http_bridge_turn_state_alias(
|
|
2871
|
+
service: _HTTPBridgeServiceProtocol,
|
|
2872
|
+
session: _HTTPBridgeSession,
|
|
2873
|
+
*,
|
|
2874
|
+
turn_state: str,
|
|
2875
|
+
registration_generation: int,
|
|
2876
|
+
instance_id: str,
|
|
2877
|
+
lease_ttl_seconds: float,
|
|
2878
|
+
local_alias_was_published: bool = True,
|
|
2879
|
+
reversible: bool = False,
|
|
2880
|
+
) -> tuple[DurableBridgeAliasRegistration | None, DurableBridgeAliasRegistrationReceipt | None]:
|
|
2881
|
+
owner_epoch = session.durable_owner_epoch
|
|
2882
|
+
try:
|
|
2883
|
+
receipt = None
|
|
2884
|
+
if reversible:
|
|
2885
|
+
receipt = await service._durable_bridge.register_recovery_turn_state(
|
|
2886
|
+
session_id=session.durable_session_id,
|
|
2887
|
+
api_key_id=session.key.api_key_id,
|
|
2888
|
+
instance_id=instance_id,
|
|
2889
|
+
owner_epoch=owner_epoch,
|
|
2890
|
+
turn_state=turn_state,
|
|
2891
|
+
lease_ttl_seconds=lease_ttl_seconds,
|
|
2892
|
+
)
|
|
2893
|
+
registered = receipt.status
|
|
2894
|
+
else:
|
|
2895
|
+
registered = await service._durable_bridge.register_turn_state(
|
|
2896
|
+
session_id=session.durable_session_id,
|
|
2897
|
+
api_key_id=session.key.api_key_id,
|
|
2898
|
+
instance_id=instance_id,
|
|
2899
|
+
owner_epoch=owner_epoch,
|
|
2900
|
+
turn_state=turn_state,
|
|
2901
|
+
lease_ttl_seconds=lease_ttl_seconds,
|
|
2902
|
+
)
|
|
2903
|
+
except Exception:
|
|
2904
|
+
logger.warning("Failed to persist durable HTTP bridge turn-state alias", exc_info=True)
|
|
2905
|
+
if not local_alias_was_published:
|
|
2906
|
+
async with service._http_bridge_lock:
|
|
2907
|
+
if session.turn_state_alias_registration_generations.get(turn_state) == registration_generation:
|
|
2908
|
+
session.turn_state_alias_registration_generations.pop(turn_state, None)
|
|
2909
|
+
return None, None
|
|
2910
|
+
if registered == DurableBridgeAliasRegistration.REGISTERED:
|
|
2911
|
+
return registered, receipt
|
|
2912
|
+
|
|
2913
|
+
fenced_out_session: _HTTPBridgeSession | None = None
|
|
2914
|
+
async with service._http_bridge_lock:
|
|
2915
|
+
if session.turn_state_alias_registration_generations.get(turn_state) != registration_generation:
|
|
2916
|
+
return None, receipt
|
|
2917
|
+
session.turn_state_alias_registration_generations.pop(turn_state, None)
|
|
2918
|
+
if local_alias_was_published:
|
|
2919
|
+
session.downstream_turn_state_aliases.discard(turn_state)
|
|
2920
|
+
if session.downstream_turn_state == turn_state:
|
|
2921
|
+
session.downstream_turn_state = None
|
|
2922
|
+
alias_key = _http_bridge_turn_state_alias_key(turn_state, session.key.api_key_id)
|
|
2923
|
+
current_session = service._http_bridge_sessions.get(session.key)
|
|
2924
|
+
current_generation_owns_alias = (
|
|
2925
|
+
current_session is not None
|
|
2926
|
+
and current_session is not session
|
|
2927
|
+
and turn_state in current_session.downstream_turn_state_aliases
|
|
2928
|
+
)
|
|
2929
|
+
if (
|
|
2930
|
+
not current_generation_owns_alias
|
|
2931
|
+
and service._http_bridge_turn_state_index.get(alias_key) == session.key
|
|
2932
|
+
):
|
|
2933
|
+
service._http_bridge_turn_state_index.pop(alias_key, None)
|
|
2934
|
+
if registered == DurableBridgeAliasRegistration.OWNER_FENCED:
|
|
2935
|
+
fenced_out_session = _evict_fenced_out_http_bridge_session_locked(
|
|
2936
|
+
service,
|
|
2937
|
+
session,
|
|
2938
|
+
owner_epoch_at_write=owner_epoch,
|
|
2939
|
+
)
|
|
2940
|
+
_schedule_fenced_out_http_bridge_session_close(service, fenced_out_session, detail="turn_state_alias_fenced_out")
|
|
2941
|
+
return registered, receipt
|
|
2942
|
+
|
|
2943
|
+
|
|
2944
|
+
async def _persist_http_bridge_previous_response_alias(
|
|
2945
|
+
service: _HTTPBridgeServiceProtocol,
|
|
2946
|
+
session: _HTTPBridgeSession,
|
|
2947
|
+
*,
|
|
2948
|
+
response_id: str,
|
|
2949
|
+
registration_generation: int,
|
|
2950
|
+
input_item_count: int | None,
|
|
2951
|
+
input_full_fingerprint: str | None,
|
|
2952
|
+
pending_tool_calls: Mapping[str, str] | None,
|
|
2953
|
+
instance_id: str,
|
|
2954
|
+
lease_ttl_seconds: float,
|
|
2955
|
+
local_alias_was_published: bool = True,
|
|
2956
|
+
) -> DurableBridgeAliasRegistration | None:
|
|
2957
|
+
owner_epoch = session.durable_owner_epoch
|
|
2958
|
+
try:
|
|
2959
|
+
registered = await service._durable_bridge.register_previous_response_id(
|
|
2960
|
+
session_id=session.durable_session_id,
|
|
2961
|
+
api_key_id=session.key.api_key_id,
|
|
2962
|
+
instance_id=instance_id,
|
|
2963
|
+
owner_epoch=owner_epoch,
|
|
2964
|
+
response_id=response_id,
|
|
2965
|
+
lease_ttl_seconds=lease_ttl_seconds,
|
|
2966
|
+
input_item_count=input_item_count,
|
|
2967
|
+
input_full_fingerprint=input_full_fingerprint,
|
|
2968
|
+
pending_tool_calls=pending_tool_calls,
|
|
2969
|
+
)
|
|
2970
|
+
except Exception:
|
|
2971
|
+
logger.warning("Failed to persist durable HTTP bridge previous_response_id alias", exc_info=True)
|
|
2972
|
+
if not local_alias_was_published:
|
|
2973
|
+
async with service._http_bridge_lock:
|
|
2974
|
+
if session.previous_response_alias_registration_generations.get(response_id) == registration_generation:
|
|
2975
|
+
session.previous_response_alias_registration_generations.pop(response_id, None)
|
|
2976
|
+
return None
|
|
2977
|
+
if registered == DurableBridgeAliasRegistration.REGISTERED:
|
|
2978
|
+
return registered
|
|
2979
|
+
|
|
2980
|
+
fenced_out_session: _HTTPBridgeSession | None = None
|
|
2981
|
+
async with service._http_bridge_lock:
|
|
2982
|
+
if session.previous_response_alias_registration_generations.get(response_id) != registration_generation:
|
|
2983
|
+
return
|
|
2984
|
+
session.previous_response_alias_registration_generations.pop(response_id, None)
|
|
2985
|
+
if local_alias_was_published:
|
|
2986
|
+
session.previous_response_ids.discard(response_id)
|
|
2987
|
+
alias_key = _http_bridge_previous_response_alias_key(response_id, session.key.api_key_id)
|
|
2988
|
+
current_session = service._http_bridge_sessions.get(session.key)
|
|
2989
|
+
current_generation_owns_alias = (
|
|
2990
|
+
current_session is not None
|
|
2991
|
+
and current_session is not session
|
|
2992
|
+
and response_id in current_session.previous_response_ids
|
|
2993
|
+
)
|
|
2994
|
+
if (
|
|
2995
|
+
not current_generation_owns_alias
|
|
2996
|
+
and service._http_bridge_previous_response_index.get(alias_key) == session.key
|
|
2997
|
+
):
|
|
2998
|
+
service._http_bridge_previous_response_index.pop(alias_key, None)
|
|
2999
|
+
if registered == DurableBridgeAliasRegistration.OWNER_FENCED:
|
|
3000
|
+
fenced_out_session = _evict_fenced_out_http_bridge_session_locked(
|
|
3001
|
+
service,
|
|
3002
|
+
session,
|
|
3003
|
+
owner_epoch_at_write=owner_epoch,
|
|
3004
|
+
)
|
|
3005
|
+
_schedule_fenced_out_http_bridge_session_close(
|
|
3006
|
+
service, fenced_out_session, detail="previous_response_alias_fenced_out"
|
|
3007
|
+
)
|
|
3008
|
+
return registered
|
|
3009
|
+
|
|
3010
|
+
|
|
3011
|
+
def _evict_fenced_out_http_bridge_session_locked(
|
|
3012
|
+
service: _HTTPBridgeServiceProtocol,
|
|
3013
|
+
session: _HTTPBridgeSession,
|
|
3014
|
+
*,
|
|
3015
|
+
owner_epoch_at_write: int | None,
|
|
3016
|
+
) -> _HTTPBridgeSession | None:
|
|
3017
|
+
"""Detach a session whose fenced durable write proved lost ownership.
|
|
3018
|
+
|
|
3019
|
+
Callers must hold ``service._http_bridge_lock``. When the session's local
|
|
3020
|
+
epoch advanced concurrently (same-session re-claim), the fenced rejection
|
|
3021
|
+
is stale rather than a lost-ownership signal and no eviction happens.
|
|
3022
|
+
"""
|
|
3023
|
+
|
|
3024
|
+
if session.closed:
|
|
3025
|
+
return None
|
|
3026
|
+
if owner_epoch_at_write is None or session.durable_owner_epoch != owner_epoch_at_write:
|
|
3027
|
+
return None
|
|
3028
|
+
detached = service._detach_http_bridge_session_locked(session.key, expected_session=session)
|
|
3029
|
+
if detached is None:
|
|
3030
|
+
session.closed = True
|
|
3031
|
+
return detached or session
|
|
3032
|
+
|
|
3033
|
+
|
|
3034
|
+
def _schedule_fenced_out_http_bridge_session_close(
|
|
3035
|
+
service: _HTTPBridgeServiceProtocol,
|
|
3036
|
+
session: _HTTPBridgeSession | None,
|
|
3037
|
+
*,
|
|
3038
|
+
detail: str,
|
|
3039
|
+
) -> None:
|
|
3040
|
+
if session is None:
|
|
3041
|
+
return
|
|
3042
|
+
_log_http_bridge_event(
|
|
3043
|
+
"fenced_out_evict",
|
|
3044
|
+
session.key,
|
|
3045
|
+
account_id=session.account.id,
|
|
3046
|
+
model=session.request_model,
|
|
3047
|
+
detail=f"outcome={detail}",
|
|
3048
|
+
cache_key_family=session.key.affinity_kind,
|
|
3049
|
+
model_class=_extract_model_class(session.request_model) if session.request_model else None,
|
|
3050
|
+
owner_check_applied=True,
|
|
3051
|
+
)
|
|
3052
|
+
service._schedule_http_bridge_session_closes([session], reason="durable_fenced_out")
|
|
3053
|
+
|
|
3054
|
+
|
|
3055
|
+
def _fail_closed_http_bridge_recovery_lease(
|
|
3056
|
+
service: _HTTPBridgeServiceProtocol,
|
|
3057
|
+
session: _HTTPBridgeSession,
|
|
3058
|
+
*,
|
|
3059
|
+
detail: str,
|
|
3060
|
+
) -> ProxyResponseError:
|
|
3061
|
+
detached = service._detach_http_bridge_session_locked(session.key, expected_session=session)
|
|
3062
|
+
session.closed = True
|
|
3063
|
+
service._schedule_http_bridge_session_closes([detached or session], reason="durable_recovery_unavailable")
|
|
3064
|
+
_log_http_bridge_event(
|
|
3065
|
+
"recovery_lease_unavailable",
|
|
3066
|
+
session.key,
|
|
3067
|
+
account_id=session.account.id,
|
|
3068
|
+
model=session.request_model,
|
|
3069
|
+
detail=detail,
|
|
3070
|
+
cache_key_family=session.key.affinity_kind,
|
|
3071
|
+
model_class=_extract_model_class(session.request_model) if session.request_model else None,
|
|
3072
|
+
owner_check_applied=True,
|
|
3073
|
+
)
|
|
3074
|
+
return ProxyResponseError(
|
|
3075
|
+
502,
|
|
3076
|
+
_http_bridge_owner_lookup_unavailable_error_envelope(),
|
|
3077
|
+
)
|
|
3078
|
+
|
|
3079
|
+
|
|
3080
|
+
async def _renew_durable_http_bridge_lease(
|
|
3081
|
+
service: _HTTPBridgeServiceProtocol,
|
|
3082
|
+
session: _HTTPBridgeSession,
|
|
3083
|
+
) -> None:
|
|
3084
|
+
"""Renew the durable lease; callers must hold ``service._http_bridge_lock``.
|
|
3085
|
+
|
|
3086
|
+
Raises the retryable instance-mismatch error after evicting the local
|
|
3087
|
+
session when the renewal reveals the durable row is owned by another
|
|
3088
|
+
instance/epoch (fenced out).
|
|
3089
|
+
"""
|
|
3090
|
+
|
|
3091
|
+
account_neutral_recovery = is_http_bridge_account_neutral_replay(
|
|
3092
|
+
kind=session.key.affinity_kind,
|
|
3093
|
+
key=session.key.affinity_key,
|
|
3094
|
+
)
|
|
3095
|
+
if session.durable_session_id is None or session.durable_owner_epoch is None:
|
|
3096
|
+
if account_neutral_recovery:
|
|
3097
|
+
raise _fail_closed_http_bridge_recovery_lease(
|
|
3098
|
+
service,
|
|
3099
|
+
session,
|
|
3100
|
+
detail="outcome=missing_durable_identity",
|
|
3101
|
+
)
|
|
3102
|
+
return
|
|
3103
|
+
current_instance = _service_get_settings().http_responses_session_bridge_instance_id
|
|
3104
|
+
try:
|
|
3105
|
+
lookup = await service._durable_bridge.renew_live_session(
|
|
3106
|
+
session_id=session.durable_session_id,
|
|
3107
|
+
api_key_id=session.key.api_key_id,
|
|
3108
|
+
instance_id=current_instance,
|
|
3109
|
+
owner_epoch=session.durable_owner_epoch,
|
|
3110
|
+
lease_ttl_seconds=_http_bridge_durable_lease_ttl_seconds(),
|
|
3111
|
+
latest_turn_state=session.downstream_turn_state,
|
|
3112
|
+
latest_response_id=None,
|
|
3113
|
+
)
|
|
3114
|
+
except Exception as exc:
|
|
3115
|
+
if account_neutral_recovery:
|
|
3116
|
+
raise _fail_closed_http_bridge_recovery_lease(
|
|
3117
|
+
service,
|
|
3118
|
+
session,
|
|
3119
|
+
detail="outcome=renew_error",
|
|
3120
|
+
) from exc
|
|
3121
|
+
logger.warning("Failed to renew durable HTTP bridge session lease", exc_info=True)
|
|
3122
|
+
return
|
|
3123
|
+
if lookup is None:
|
|
3124
|
+
if account_neutral_recovery:
|
|
3125
|
+
raise _fail_closed_http_bridge_recovery_lease(
|
|
3126
|
+
service,
|
|
3127
|
+
session,
|
|
3128
|
+
detail="outcome=durable_row_missing",
|
|
3129
|
+
)
|
|
3130
|
+
return
|
|
3131
|
+
if lookup.owner_instance_id == current_instance and lookup.owner_epoch == session.durable_owner_epoch:
|
|
3132
|
+
return
|
|
3133
|
+
if (
|
|
3134
|
+
lookup.owner_instance_id == current_instance
|
|
3135
|
+
and lookup.owner_process_epoch == http_bridge_owner_process_epoch()
|
|
3136
|
+
and lookup.owner_epoch > session.durable_owner_epoch
|
|
3137
|
+
# A claim rewrites the row's account_id, so an advance that moved the
|
|
3138
|
+
# row to another account is a real ownership change for this session,
|
|
3139
|
+
# not a superseded creator: never adopt across it.
|
|
3140
|
+
and lookup.account_id == session.account.id
|
|
3141
|
+
and service._http_bridge_sessions.get(session.key) is session
|
|
3142
|
+
):
|
|
3143
|
+
# THIS process advanced the epoch while this session still holds the
|
|
3144
|
+
# registry slot for its key — a creator that was superseded mid claim,
|
|
3145
|
+
# not a real ownership loss (issue #1695). Evicting here would 409 the
|
|
3146
|
+
# session that legitimately owns the key, so adopt the epoch and keep
|
|
3147
|
+
# renewing. The process-epoch check matters because two incarnations
|
|
3148
|
+
# can share a configured instance ID across a graceful restart: the
|
|
3149
|
+
# successor's claim must still fence the predecessor out. A DIFFERENT
|
|
3150
|
+
# local session holding the slot also falls through: that one won, and
|
|
3151
|
+
# this session must be evicted.
|
|
3152
|
+
session.durable_owner_epoch = lookup.owner_epoch
|
|
3153
|
+
return
|
|
3154
|
+
# Fenced out: another instance/epoch owns the durable session. Never adopt
|
|
3155
|
+
# the foreign epoch — evict the local session so its upstream websocket and
|
|
3156
|
+
# account lease are released, and fail the request with the retryable
|
|
3157
|
+
# instance-mismatch contract used by rejected claims.
|
|
3158
|
+
detached = service._detach_http_bridge_session_locked(session.key, expected_session=session)
|
|
3159
|
+
session.closed = True
|
|
3160
|
+
service._schedule_http_bridge_session_closes([detached or session], reason="durable_fenced_out")
|
|
3161
|
+
_log_http_bridge_event(
|
|
3162
|
+
"fenced_out_evict",
|
|
3163
|
+
session.key,
|
|
3164
|
+
account_id=session.account.id,
|
|
3165
|
+
model=session.request_model,
|
|
3166
|
+
detail=(
|
|
3167
|
+
f"owner_instance={lookup.owner_instance_id}, owner_epoch={lookup.owner_epoch}, "
|
|
3168
|
+
f"current_instance={current_instance}, local_epoch={session.durable_owner_epoch}, "
|
|
3169
|
+
"outcome=renew_fenced_out"
|
|
3170
|
+
),
|
|
3171
|
+
cache_key_family=session.key.affinity_kind,
|
|
3172
|
+
model_class=_extract_model_class(session.request_model) if session.request_model else None,
|
|
3173
|
+
owner_check_applied=True,
|
|
3174
|
+
)
|
|
3175
|
+
if PROMETHEUS_AVAILABLE and bridge_instance_mismatch_total is not None:
|
|
3176
|
+
bridge_instance_mismatch_total.labels(outcome="renew_fenced_out").inc()
|
|
3177
|
+
raise ProxyResponseError(
|
|
3178
|
+
409,
|
|
3179
|
+
openai_error(
|
|
3180
|
+
"bridge_instance_mismatch",
|
|
3181
|
+
"HTTP bridge session is owned by a different instance; retry to reach the correct replica",
|
|
3182
|
+
error_type="server_error",
|
|
3183
|
+
),
|
|
3184
|
+
)
|
|
3185
|
+
|
|
3186
|
+
|
|
3187
|
+
async def _reconcile_durable_http_bridge_ownership(service: _HTTPBridgeServiceProtocol) -> int:
|
|
3188
|
+
"""Close local sessions whose durable row is owned by another instance/epoch.
|
|
3189
|
+
|
|
3190
|
+
Piggybacked on the ring-heartbeat cadence so a replica that lost
|
|
3191
|
+
ownership releases its orphaned upstream websocket and account lease
|
|
3192
|
+
within roughly the durable lease TTL instead of the idle TTL.
|
|
3193
|
+
"""
|
|
3194
|
+
|
|
3195
|
+
current_instance = _service_get_settings().http_responses_session_bridge_instance_id
|
|
3196
|
+
lease_ttl_seconds = _http_bridge_durable_lease_ttl_seconds()
|
|
3197
|
+
now = clock_for(service).monotonic()
|
|
3198
|
+
async with service._http_bridge_lock:
|
|
3199
|
+
candidates = [
|
|
3200
|
+
(key, session)
|
|
3201
|
+
for key, session in service._http_bridge_sessions.items()
|
|
3202
|
+
if not session.closed
|
|
3203
|
+
and session.durable_session_id is not None
|
|
3204
|
+
and session.durable_owner_epoch is not None
|
|
3205
|
+
and now - session.last_used_at >= lease_ttl_seconds
|
|
3206
|
+
]
|
|
3207
|
+
if not candidates:
|
|
3208
|
+
return 0
|
|
3209
|
+
lookups = await service._durable_bridge.lookup_sessions(
|
|
3210
|
+
session_ids=[session.durable_session_id for _, session in candidates if session.durable_session_id]
|
|
3211
|
+
)
|
|
3212
|
+
lookup_by_id = {lookup.session_id: lookup for lookup in lookups}
|
|
3213
|
+
fenced_sessions: list[_HTTPBridgeSession] = []
|
|
3214
|
+
async with service._http_bridge_lock:
|
|
3215
|
+
for key, session in candidates:
|
|
3216
|
+
lookup = lookup_by_id.get(session.durable_session_id or "")
|
|
3217
|
+
if lookup is None or lookup.owner_instance_id is None:
|
|
3218
|
+
continue
|
|
3219
|
+
if lookup.owner_instance_id == current_instance and lookup.owner_epoch == session.durable_owner_epoch:
|
|
3220
|
+
continue
|
|
3221
|
+
detached = service._detach_http_bridge_session_locked(key, expected_session=session)
|
|
3222
|
+
if detached is None:
|
|
3223
|
+
continue
|
|
3224
|
+
fenced_sessions.append(detached)
|
|
3225
|
+
_log_http_bridge_event(
|
|
3226
|
+
"fenced_out_evict",
|
|
3227
|
+
key,
|
|
3228
|
+
account_id=session.account.id,
|
|
3229
|
+
model=session.request_model,
|
|
3230
|
+
detail=(
|
|
3231
|
+
f"owner_instance={lookup.owner_instance_id}, owner_epoch={lookup.owner_epoch}, "
|
|
3232
|
+
f"current_instance={current_instance}, local_epoch={session.durable_owner_epoch}, "
|
|
3233
|
+
"outcome=reconcile_sweep"
|
|
3234
|
+
),
|
|
3235
|
+
cache_key_family=key.affinity_kind,
|
|
3236
|
+
model_class=_extract_model_class(session.request_model) if session.request_model else None,
|
|
3237
|
+
owner_check_applied=True,
|
|
3238
|
+
)
|
|
3239
|
+
if fenced_sessions:
|
|
3240
|
+
service._schedule_http_bridge_session_closes(fenced_sessions, reason="durable_fenced_out")
|
|
3241
|
+
return len(fenced_sessions)
|
|
3242
|
+
|
|
3243
|
+
|
|
3244
|
+
def _http_bridge_turn_state_anchor_for_owner_failure(
|
|
3245
|
+
exc: ProxyResponseError,
|
|
3246
|
+
*,
|
|
3247
|
+
headers: Mapping[str, str],
|
|
3248
|
+
previous_response_id: str | None,
|
|
3249
|
+
) -> str | None:
|
|
3250
|
+
"""Return the turn-state anchor when an owner forward failed unreachable."""
|
|
3251
|
+
|
|
3252
|
+
if previous_response_id is not None:
|
|
3253
|
+
return None
|
|
3254
|
+
turn_state = _sticky_key_from_turn_state_header(headers)
|
|
3255
|
+
if turn_state is None:
|
|
3256
|
+
return None
|
|
3257
|
+
payload = exc.payload
|
|
3258
|
+
if not isinstance(payload, dict):
|
|
3259
|
+
return None
|
|
3260
|
+
error = payload.get("error")
|
|
3261
|
+
if not isinstance(error, dict):
|
|
3262
|
+
return None
|
|
3263
|
+
if error.get("code") != "bridge_owner_unreachable":
|
|
3264
|
+
return None
|
|
3265
|
+
return turn_state
|
|
3266
|
+
|
|
3267
|
+
|
|
3268
|
+
def _http_bridge_durable_lookup_allows_turn_state_takeover(lookup: DurableBridgeLookup | None) -> bool:
|
|
3269
|
+
"""Allow local takeover only when the durable lease is not actively held.
|
|
3270
|
+
|
|
3271
|
+
Released (owner ``None``), expired-lease, CLOSED, and missing rows allow
|
|
3272
|
+
takeover. A live lease keeps failing closed even for DRAINING rows:
|
|
3273
|
+
shutdown marks rows DRAINING before releasing them, so a draining owner
|
|
3274
|
+
may still be finishing an in-flight turn until its lease is released or
|
|
3275
|
+
lapses.
|
|
3276
|
+
"""
|
|
3277
|
+
|
|
3278
|
+
return _durable_bridge_lookup_active_owner(lookup) is None
|
|
3279
|
+
|
|
3280
|
+
|
|
3281
|
+
def _forwarded_http_bridge_session_key(
|
|
3282
|
+
headers: Mapping[str, str],
|
|
3283
|
+
api_key: ApiKeyData | None,
|
|
3284
|
+
*,
|
|
3285
|
+
forwarded_affinity_kind: str | None = None,
|
|
3286
|
+
forwarded_affinity_key: str | None = None,
|
|
3287
|
+
) -> _HTTPBridgeSessionKey | None:
|
|
3288
|
+
affinity_kind = forwarded_affinity_kind or _header_value_case_insensitive(headers, "x-codex-bridge-affinity-kind")
|
|
3289
|
+
affinity_key = forwarded_affinity_key or _header_value_case_insensitive(headers, "x-codex-bridge-affinity-key")
|
|
3290
|
+
if affinity_kind is None or affinity_key is None:
|
|
3291
|
+
return None
|
|
3292
|
+
strength: Literal["hard", "soft"]
|
|
3293
|
+
if affinity_kind in _HARD_HTTP_BRIDGE_AFFINITY_KINDS:
|
|
3294
|
+
strength = "hard"
|
|
3295
|
+
else:
|
|
3296
|
+
strength = "soft"
|
|
3297
|
+
return _HTTPBridgeSessionKey(
|
|
3298
|
+
affinity_kind=affinity_kind,
|
|
3299
|
+
affinity_key=affinity_key,
|
|
3300
|
+
api_key_id=api_key.id if api_key is not None else None,
|
|
3301
|
+
strength=strength,
|
|
3302
|
+
)
|
|
3303
|
+
|
|
3304
|
+
|
|
3305
|
+
def _http_bridge_requires_cluster_registration(settings: Settings) -> bool:
|
|
3306
|
+
if len(settings.http_responses_session_bridge_instance_ring) > 1:
|
|
3307
|
+
return True
|
|
3308
|
+
advertise_base_url = settings.http_responses_session_bridge_advertise_base_url
|
|
3309
|
+
if advertise_base_url is None:
|
|
3310
|
+
return False
|
|
3311
|
+
hostname = urlparse(advertise_base_url).hostname
|
|
3312
|
+
if hostname is None:
|
|
3313
|
+
return False
|
|
3314
|
+
try:
|
|
3315
|
+
parsed_ip = ip_address(hostname)
|
|
3316
|
+
except ValueError:
|
|
3317
|
+
return True
|
|
3318
|
+
return not parsed_ip.is_loopback
|
|
3319
|
+
|
|
3320
|
+
|
|
3321
|
+
def _effective_http_bridge_idle_ttl_seconds(
|
|
3322
|
+
*,
|
|
3323
|
+
affinity: _AffinityPolicy,
|
|
3324
|
+
idle_ttl_seconds: float,
|
|
3325
|
+
codex_idle_ttl_seconds: float,
|
|
3326
|
+
prompt_cache_idle_ttl_seconds: float | None = None,
|
|
3327
|
+
) -> float:
|
|
3328
|
+
if affinity.kind == StickySessionKind.CODEX_SESSION or affinity.codex_session_source == "thread_header":
|
|
3329
|
+
# The DB row is bounded soft locality, but a live thread bridge owns
|
|
3330
|
+
# upstream socket history and therefore receives the Codex continuity
|
|
3331
|
+
# lifetime once created.
|
|
3332
|
+
return max(idle_ttl_seconds, codex_idle_ttl_seconds)
|
|
3333
|
+
if affinity.kind == StickySessionKind.PROMPT_CACHE and prompt_cache_idle_ttl_seconds is not None:
|
|
3334
|
+
return prompt_cache_idle_ttl_seconds
|
|
3335
|
+
return idle_ttl_seconds
|
|
3336
|
+
|
|
3337
|
+
|
|
3338
|
+
def _http_bridge_eviction_priority(session: _HTTPBridgeSession) -> tuple[int, float]:
|
|
3339
|
+
return (0 if not session.codex_session else 1, session.last_used_at)
|
|
3340
|
+
|
|
3341
|
+
|
|
3342
|
+
def _build_http_bridge_prewarm_text(text_data: str) -> str | None:
|
|
3343
|
+
try:
|
|
3344
|
+
payload = json.loads(text_data)
|
|
3345
|
+
except json.JSONDecodeError:
|
|
3346
|
+
return None
|
|
3347
|
+
if not isinstance(payload, dict):
|
|
3348
|
+
return None
|
|
3349
|
+
if payload.get("generate") is False:
|
|
3350
|
+
return None
|
|
3351
|
+
previous_response_id = payload.get("previous_response_id")
|
|
3352
|
+
if isinstance(previous_response_id, str) and previous_response_id.strip():
|
|
3353
|
+
return None
|
|
3354
|
+
warmup_payload = dict(payload)
|
|
3355
|
+
warmup_payload["generate"] = False
|
|
3356
|
+
return json.dumps(warmup_payload, ensure_ascii=True, separators=(",", ":"))
|
|
3357
|
+
|
|
3358
|
+
|
|
3359
|
+
def _http_bridge_prewarm_enabled(settings: Any) -> bool:
|
|
3360
|
+
"""Prewarm eligibility is the ``prewarm_enabled`` switch alone.
|
|
3361
|
+
|
|
3362
|
+
M3 codex prewarm: the switch is dashboard-managed, and it is folded into
|
|
3363
|
+
``settings`` before this is called. ``settings`` is the proxy service facade
|
|
3364
|
+
value (``_service_get_settings()``), which applies the request-bound
|
|
3365
|
+
dashboard overlay: the field is in ``DASHBOARD_OVERRIDE_SETTINGS``, so a
|
|
3366
|
+
non-NULL ``dashboard_settings`` column has already won over the deprecated
|
|
3367
|
+
``CODEX_LB_*`` env alias by the time this reads it. Outside a bound request
|
|
3368
|
+
context (startup, schedulers, unit tests) the env alias applies, and then
|
|
3369
|
+
the code default (off) -- the same precedence ``resolve_inheritable``
|
|
3370
|
+
applies for the settings API's provenance.
|
|
3371
|
+
|
|
3372
|
+
The canary percent and allow/deny cohort scaffolding was one-time
|
|
3373
|
+
rollout tooling retired by ``reduce-settings-surface-phase-4``.
|
|
3374
|
+
"""
|
|
3375
|
+
return bool(getattr(settings, "http_responses_session_bridge_codex_prewarm_enabled", False))
|
|
3376
|
+
|
|
3377
|
+
|
|
3378
|
+
def _record_http_bridge_prewarm_outcome(*, outcome: str) -> None:
|
|
3379
|
+
if not PROMETHEUS_AVAILABLE or http_bridge_prewarm_total is None:
|
|
3380
|
+
return
|
|
3381
|
+
http_bridge_prewarm_total.labels(outcome=outcome).inc()
|
|
3382
|
+
|
|
3383
|
+
|
|
3384
|
+
def _record_http_bridge_stuck_retire(
|
|
3385
|
+
*,
|
|
3386
|
+
reason: str,
|
|
3387
|
+
session: _HTTPBridgeSession,
|
|
3388
|
+
) -> None:
|
|
3389
|
+
if not PROMETHEUS_AVAILABLE or http_bridge_stuck_retire_total is None:
|
|
3390
|
+
return
|
|
3391
|
+
http_bridge_stuck_retire_total.labels(
|
|
3392
|
+
reason=reason,
|
|
3393
|
+
affinity_kind=session.key.affinity_kind,
|
|
3394
|
+
model_class=_extract_model_class(session.request_model) if session.request_model else "unknown",
|
|
3395
|
+
).inc()
|
|
3396
|
+
|
|
3397
|
+
|
|
3398
|
+
def _http_bridge_payload_without_previous_response_id(payload: ResponsesRequest) -> ResponsesRequest:
|
|
3399
|
+
if payload.previous_response_id is None:
|
|
3400
|
+
return payload
|
|
3401
|
+
return payload.model_copy(update={"previous_response_id": None})
|
|
3402
|
+
|
|
3403
|
+
|
|
3404
|
+
def _http_bridge_previous_response_error_envelope(
|
|
3405
|
+
previous_response_id: str,
|
|
3406
|
+
detail: str,
|
|
3407
|
+
) -> OpenAIErrorEnvelope:
|
|
3408
|
+
payload = openai_error(
|
|
3409
|
+
"previous_response_not_found",
|
|
3410
|
+
f"Previous response with id '{previous_response_id}' not found. {detail}",
|
|
3411
|
+
error_type="invalid_request_error",
|
|
3412
|
+
)
|
|
3413
|
+
payload["error"]["param"] = "previous_response_id"
|
|
3414
|
+
return payload
|
|
3415
|
+
|
|
3416
|
+
|
|
3417
|
+
def _http_bridge_continuity_lost_error_envelope() -> OpenAIErrorEnvelope:
|
|
3418
|
+
return previous_response_stream_incomplete_error()
|
|
3419
|
+
|
|
3420
|
+
|
|
3421
|
+
def _http_bridge_owner_lookup_unavailable_error_envelope() -> OpenAIErrorEnvelope:
|
|
3422
|
+
return openai_error(
|
|
3423
|
+
"upstream_unavailable",
|
|
3424
|
+
"HTTP bridge owner metadata unavailable; retry later.",
|
|
3425
|
+
error_type="server_error",
|
|
3426
|
+
)
|
|
3427
|
+
|
|
3428
|
+
|
|
3429
|
+
def _mark_http_bridge_reader_handoff_reconnect_failed(session: Any, old_reader: Any) -> None:
|
|
3430
|
+
if old_reader is not None:
|
|
3431
|
+
session.closed = True
|
|
3432
|
+
|
|
3433
|
+
|
|
3434
|
+
def _http_bridge_previous_response_owner_unavailable_error() -> ProxyResponseError:
|
|
3435
|
+
return ProxyResponseError(
|
|
3436
|
+
502,
|
|
3437
|
+
openai_error(
|
|
3438
|
+
"previous_response_owner_unavailable",
|
|
3439
|
+
"Previous response owner account is unavailable; retry later.",
|
|
3440
|
+
error_type="server_error",
|
|
3441
|
+
),
|
|
3442
|
+
)
|
|
3443
|
+
|
|
3444
|
+
|
|
3445
|
+
def _http_bridge_reconnect_selection_failure(
|
|
3446
|
+
selection: Any,
|
|
3447
|
+
required_preferred_account_id: str | None,
|
|
3448
|
+
) -> ProxyResponseError:
|
|
3449
|
+
if required_preferred_account_id is not None:
|
|
3450
|
+
return _http_bridge_previous_response_owner_unavailable_error()
|
|
3451
|
+
status_code, error_payload = selection_failure_response(selection)
|
|
3452
|
+
return ProxyResponseError(status_code, error_payload)
|
|
3453
|
+
|
|
3454
|
+
|
|
3455
|
+
def _http_bridge_reconnect_connect_failure(
|
|
3456
|
+
exc: BaseException,
|
|
3457
|
+
required_preferred_account_id: str | None,
|
|
3458
|
+
) -> ProxyResponseError:
|
|
3459
|
+
if required_preferred_account_id is not None:
|
|
3460
|
+
return _http_bridge_previous_response_owner_unavailable_error()
|
|
3461
|
+
if isinstance(exc, ProxyResponseError):
|
|
3462
|
+
return exc
|
|
3463
|
+
raise exc
|
|
3464
|
+
|
|
3465
|
+
|
|
3466
|
+
def _http_bridge_should_attempt_local_previous_response_recovery(exc: ProxyResponseError) -> bool:
|
|
3467
|
+
payload = exc.payload
|
|
3468
|
+
if not isinstance(payload, dict):
|
|
3469
|
+
return False
|
|
3470
|
+
error = payload.get("error")
|
|
3471
|
+
if not isinstance(error, dict):
|
|
3472
|
+
return False
|
|
3473
|
+
code_value = error.get("code")
|
|
3474
|
+
raw_code = code_value.strip() if isinstance(code_value, str) and code_value.strip() else None
|
|
3475
|
+
type_value = error.get("type")
|
|
3476
|
+
error_type = type_value.strip() if isinstance(type_value, str) and type_value.strip() else None
|
|
3477
|
+
param_state = OpenAIErrorParam.from_mapping(cast(Mapping[str, JsonValue], error))
|
|
3478
|
+
if param_state.malformed:
|
|
3479
|
+
return False
|
|
3480
|
+
# Normalize like the websocket rewrite path (#1818): upstream frames may
|
|
3481
|
+
# carry the classifiable code only in ``type`` (or omit both code and
|
|
3482
|
+
# param on the terse previous-response rejection), and a raw read would
|
|
3483
|
+
# misclassify them into the ambiguous transport class below (issue #1830).
|
|
3484
|
+
code = _normalize_error_code(raw_code, error_type)
|
|
3485
|
+
if code in {
|
|
3486
|
+
"bridge_owner_unreachable",
|
|
3487
|
+
"bridge_previous_response_not_found",
|
|
3488
|
+
"previous_response_not_found",
|
|
3489
|
+
"bridge_instance_mismatch",
|
|
3490
|
+
}:
|
|
3491
|
+
return True
|
|
3492
|
+
if code in {"stream_incomplete", "stream_idle_timeout", "upstream_request_timeout"}:
|
|
3493
|
+
# An ambiguous transport failure is never locally recoverable: upstream
|
|
3494
|
+
# may already have accepted the anchored request, and there is no
|
|
3495
|
+
# idempotency or status endpoint to prove otherwise. Fail closed.
|
|
3496
|
+
return False
|
|
3497
|
+
message_value = error.get("message")
|
|
3498
|
+
message = message_value.strip() if isinstance(message_value, str) and message_value.strip() else None
|
|
3499
|
+
return _is_previous_response_not_found_error(code=code, param=param_state, message=message)
|
|
3500
|
+
|
|
3501
|
+
|
|
3502
|
+
def _http_bridge_is_explicit_previous_response_rejection(exc: ProxyResponseError) -> bool:
|
|
3503
|
+
payload = exc.payload
|
|
3504
|
+
if not isinstance(payload, dict):
|
|
3505
|
+
return False
|
|
3506
|
+
error = payload.get("error")
|
|
3507
|
+
if not isinstance(error, dict):
|
|
3508
|
+
return False
|
|
3509
|
+
code_value = error.get("code")
|
|
3510
|
+
raw_code = code_value.strip() if isinstance(code_value, str) and code_value.strip() else None
|
|
3511
|
+
type_value = error.get("type")
|
|
3512
|
+
error_type = type_value.strip() if isinstance(type_value, str) and type_value.strip() else None
|
|
3513
|
+
param_state = OpenAIErrorParam.from_mapping(cast(Mapping[str, JsonValue], error))
|
|
3514
|
+
if param_state.malformed:
|
|
3515
|
+
return False
|
|
3516
|
+
code = _normalize_error_code(raw_code, error_type)
|
|
3517
|
+
if code == "bridge_previous_response_not_found":
|
|
3518
|
+
return True
|
|
3519
|
+
message_value = error.get("message")
|
|
3520
|
+
message = message_value.strip() if isinstance(message_value, str) and message_value.strip() else None
|
|
3521
|
+
return _is_previous_response_not_found_error(code=code, param=param_state, message=message)
|
|
3522
|
+
|
|
3523
|
+
|
|
3524
|
+
def _http_bridge_is_previous_response_owner_unavailable(exc: ProxyResponseError) -> bool:
|
|
3525
|
+
if exc.status_code != 502:
|
|
3526
|
+
return False
|
|
3527
|
+
payload = exc.payload
|
|
3528
|
+
if not isinstance(payload, dict):
|
|
3529
|
+
return False
|
|
3530
|
+
error = payload.get("error")
|
|
3531
|
+
if not isinstance(error, dict):
|
|
3532
|
+
return False
|
|
3533
|
+
return error.get("code") == "previous_response_owner_unavailable"
|
|
3534
|
+
|
|
3535
|
+
|
|
3536
|
+
def _http_bridge_should_attempt_soft_affinity_reroute(
|
|
3537
|
+
exc: ProxyResponseError,
|
|
3538
|
+
*,
|
|
3539
|
+
key: "_HTTPBridgeSessionKey",
|
|
3540
|
+
previous_response_id: str | None,
|
|
3541
|
+
) -> bool:
|
|
3542
|
+
if exc.status_code != 429:
|
|
3543
|
+
return False
|
|
3544
|
+
if key.strength == "hard" or previous_response_id is not None:
|
|
3545
|
+
return False
|
|
3546
|
+
payload = exc.payload
|
|
3547
|
+
if not isinstance(payload, dict):
|
|
3548
|
+
return False
|
|
3549
|
+
error = payload.get("error")
|
|
3550
|
+
if not isinstance(error, dict):
|
|
3551
|
+
return False
|
|
3552
|
+
# api_key_stream_fair_share is deliberately absent: the fair-share gate
|
|
3553
|
+
# is per-key and pool-wide, so rerouting to another account cannot help.
|
|
3554
|
+
return error.get("code") in {
|
|
3555
|
+
"bridge_queue_full",
|
|
3556
|
+
"response_create_gate_timeout",
|
|
3557
|
+
"account_response_create_cap",
|
|
3558
|
+
"account_stream_cap",
|
|
3559
|
+
}
|
|
3560
|
+
|
|
3561
|
+
|
|
3562
|
+
def _persistent_http_bridge_affinity(affinity: _AffinityPolicy) -> _AffinityPolicy:
|
|
3563
|
+
if not affinity.abandon_unavailable_legacy_owner:
|
|
3564
|
+
return affinity
|
|
3565
|
+
# Restart authority is proof attached to one canonical request body, not a
|
|
3566
|
+
# property of the longer-lived process session. Never let a reused bridge
|
|
3567
|
+
# grant a later ordinary request permission to retire hard ownership.
|
|
3568
|
+
return replace(affinity, abandon_unavailable_legacy_owner=False)
|
|
3569
|
+
|
|
3570
|
+
|
|
3571
|
+
def _http_bridge_is_context_overflow_error(exc: ProxyResponseError) -> bool:
|
|
3572
|
+
payload = exc.payload
|
|
3573
|
+
if not isinstance(payload, dict):
|
|
3574
|
+
return False
|
|
3575
|
+
error = payload.get("error")
|
|
3576
|
+
if not isinstance(error, dict):
|
|
3577
|
+
return False
|
|
3578
|
+
code_value = error.get("code")
|
|
3579
|
+
code = code_value.strip() if isinstance(code_value, str) and code_value.strip() else None
|
|
3580
|
+
type_value = error.get("type")
|
|
3581
|
+
error_type = type_value.strip() if isinstance(type_value, str) and type_value.strip() else None
|
|
3582
|
+
normalized_code = _normalize_error_code(code, error_type)
|
|
3583
|
+
return normalized_code == "context_length_exceeded"
|
|
3584
|
+
|
|
3585
|
+
|
|
3586
|
+
def _http_bridge_should_rollover_after_context_overflow(
|
|
3587
|
+
exc: ProxyResponseError,
|
|
3588
|
+
*,
|
|
3589
|
+
key: _HTTPBridgeSessionKey | None = None,
|
|
3590
|
+
) -> bool:
|
|
3591
|
+
if not _http_bridge_is_context_overflow_error(exc):
|
|
3592
|
+
return False
|
|
3593
|
+
if key is not None and key.strength == "hard":
|
|
3594
|
+
return False
|
|
3595
|
+
return True
|
|
3596
|
+
|
|
3597
|
+
|
|
3598
|
+
def _http_bridge_should_attempt_local_bootstrap_rebind(
|
|
3599
|
+
exc: ProxyResponseError,
|
|
3600
|
+
*,
|
|
3601
|
+
key: _HTTPBridgeSessionKey,
|
|
3602
|
+
headers: Mapping[str, str],
|
|
3603
|
+
previous_response_id: str | None,
|
|
3604
|
+
) -> bool:
|
|
3605
|
+
if key.affinity_kind not in {"session_header", "thread_header"}:
|
|
3606
|
+
return False
|
|
3607
|
+
if previous_response_id is not None:
|
|
3608
|
+
return False
|
|
3609
|
+
if _sticky_key_from_turn_state_header(headers) is not None:
|
|
3610
|
+
return False
|
|
3611
|
+
payload = exc.payload
|
|
3612
|
+
if not isinstance(payload, dict):
|
|
3613
|
+
return False
|
|
3614
|
+
error = payload.get("error")
|
|
3615
|
+
if not isinstance(error, dict):
|
|
3616
|
+
return False
|
|
3617
|
+
code = error.get("code")
|
|
3618
|
+
return code in {
|
|
3619
|
+
"bridge_owner_unreachable",
|
|
3620
|
+
"bridge_instance_mismatch",
|
|
3621
|
+
}
|
|
3622
|
+
|
|
3623
|
+
|
|
3624
|
+
def _normalized_http_bridge_instance_ring(settings: Settings) -> tuple[str, tuple[str, ...]]:
|
|
3625
|
+
instance_id = settings.http_responses_session_bridge_instance_id.strip()
|
|
3626
|
+
if not instance_id:
|
|
3627
|
+
instance_id = "codex-lb"
|
|
3628
|
+
ring_entries: list[str] = []
|
|
3629
|
+
for entry in settings.http_responses_session_bridge_instance_ring:
|
|
3630
|
+
stripped = entry.strip()
|
|
3631
|
+
if stripped:
|
|
3632
|
+
ring_entries.append(stripped)
|
|
3633
|
+
if not ring_entries:
|
|
3634
|
+
ring_entries.append(instance_id)
|
|
3635
|
+
return instance_id, tuple(sorted(set(ring_entries)))
|
|
3636
|
+
|
|
3637
|
+
|
|
3638
|
+
async def _active_http_bridge_instance_ring(
|
|
3639
|
+
settings: Settings,
|
|
3640
|
+
ring_membership: RingMembershipService | None,
|
|
3641
|
+
) -> tuple[str, tuple[str, ...]]:
|
|
3642
|
+
instance_id, static_ring = _normalized_http_bridge_instance_ring(settings)
|
|
3643
|
+
if ring_membership is None:
|
|
3644
|
+
return instance_id, static_ring
|
|
3645
|
+
try:
|
|
3646
|
+
active_members = await ring_membership.list_active(require_endpoint=True)
|
|
3647
|
+
except Exception:
|
|
3648
|
+
logger.warning("Bridge ring lookup failed — refusing to fall back to static ring", exc_info=True)
|
|
3649
|
+
raise
|
|
3650
|
+
if not active_members:
|
|
3651
|
+
return instance_id, (instance_id,)
|
|
3652
|
+
normalized_members = tuple(
|
|
3653
|
+
sorted({member.strip() for member in active_members if isinstance(member, str) and member.strip()})
|
|
3654
|
+
)
|
|
3655
|
+
if not normalized_members:
|
|
3656
|
+
return instance_id, static_ring
|
|
3657
|
+
return instance_id, normalized_members
|
|
3658
|
+
|
|
3659
|
+
|
|
3660
|
+
async def _http_bridge_owner_instance(
|
|
3661
|
+
key: _HTTPBridgeSessionKey,
|
|
3662
|
+
settings: Settings,
|
|
3663
|
+
ring_membership: RingMembershipService | None = None,
|
|
3664
|
+
) -> str | None:
|
|
3665
|
+
instance_id, ring = await _active_http_bridge_instance_ring(settings, ring_membership)
|
|
3666
|
+
if len(ring) <= 1:
|
|
3667
|
+
return instance_id
|
|
3668
|
+
hash_input = f"{key.affinity_kind}:{key.affinity_key}:{key.api_key_id or ''}"
|
|
3669
|
+
return select_node(hash_input, ring)
|
|
3670
|
+
|
|
3671
|
+
|
|
3672
|
+
def _http_bridge_runtime_config(
|
|
3673
|
+
dashboard_settings: DashboardSettings,
|
|
3674
|
+
app_settings: Settings,
|
|
3675
|
+
) -> _HTTPBridgeRuntimeConfig:
|
|
3676
|
+
return _HTTPBridgeRuntimeConfig(
|
|
3677
|
+
enabled=app_settings.http_responses_session_bridge_enabled,
|
|
3678
|
+
idle_ttl_seconds=HTTP_BRIDGE_IDLE_TTL_SECONDS,
|
|
3679
|
+
codex_idle_ttl_seconds=HTTP_BRIDGE_CODEX_IDLE_TTL_SECONDS,
|
|
3680
|
+
max_sessions=app_settings.http_responses_session_bridge_max_sessions,
|
|
3681
|
+
queue_limit=app_settings.http_responses_session_bridge_queue_limit,
|
|
3682
|
+
prompt_cache_idle_ttl_seconds=float(
|
|
3683
|
+
dashboard_settings.http_responses_session_bridge_prompt_cache_idle_ttl_seconds,
|
|
3684
|
+
),
|
|
3685
|
+
gateway_safe_mode=dashboard_settings.http_responses_session_bridge_gateway_safe_mode,
|
|
3686
|
+
)
|
|
3687
|
+
|
|
3688
|
+
|
|
3689
|
+
def _http_bridge_request_budget_seconds(settings: object) -> float:
|
|
3690
|
+
return float(
|
|
3691
|
+
getattr(
|
|
3692
|
+
settings,
|
|
3693
|
+
"http_responses_session_bridge_request_budget_seconds",
|
|
3694
|
+
getattr(settings, "proxy_request_budget_seconds", 600.0),
|
|
3695
|
+
)
|
|
3696
|
+
)
|
|
3697
|
+
|
|
3698
|
+
|
|
3699
|
+
def _http_bridge_eventless_budget_seconds(settings: object, *, fallback_seconds: float) -> float:
|
|
3700
|
+
"""Named pre-response-start silence budget for the downstream bridge stream.
|
|
3701
|
+
|
|
3702
|
+
Before ``response.created`` the downstream event queue is silent by design,
|
|
3703
|
+
so ``stream_idle_timeout_seconds`` (a *post-start* inter-event budget) does
|
|
3704
|
+
not describe this phase at all. The honest bound is the owner-side stuck
|
|
3705
|
+
gate: once ``HTTP_BRIDGE_STUCK_GATE_RETIRE_AFTER_SECONDS`` retires the
|
|
3706
|
+
pending handoff there is nothing left for the client to wait for. Clamp to
|
|
3707
|
+
the stream-idle and bridge-request budgets so the pre-response watchdog can
|
|
3708
|
+
never outlive the request it guards.
|
|
3709
|
+
|
|
3710
|
+
``fallback_seconds`` is used when a caller's settings object does not carry
|
|
3711
|
+
``stream_idle_timeout_seconds`` at all.
|
|
3712
|
+
"""
|
|
3713
|
+
|
|
3714
|
+
stuck_gate_seconds = float(HTTP_BRIDGE_STUCK_GATE_RETIRE_AFTER_SECONDS)
|
|
3715
|
+
stream_idle_timeout_seconds = float(getattr(settings, "stream_idle_timeout_seconds", fallback_seconds))
|
|
3716
|
+
return max(
|
|
3717
|
+
0.001,
|
|
3718
|
+
min(
|
|
3719
|
+
stuck_gate_seconds,
|
|
3720
|
+
stream_idle_timeout_seconds,
|
|
3721
|
+
_http_bridge_request_budget_seconds(settings),
|
|
3722
|
+
),
|
|
3723
|
+
)
|
|
3724
|
+
|
|
3725
|
+
|
|
3726
|
+
def _http_bridge_eventless_max_keepalive_count(
|
|
3727
|
+
settings: object,
|
|
3728
|
+
*,
|
|
3729
|
+
keepalive_interval_seconds: float,
|
|
3730
|
+
floor_count: int,
|
|
3731
|
+
) -> int:
|
|
3732
|
+
"""Keepalive ticks the downstream stream waits before ``response.created``.
|
|
3733
|
+
|
|
3734
|
+
Derived from :func:`_http_bridge_eventless_budget_seconds` instead of the
|
|
3735
|
+
implicit ``_STREAM_KEEPALIVE_MAX_COUNT * sse_keepalive_interval_seconds``
|
|
3736
|
+
product, which silently encoded a ~60s pre-response deadline that had no
|
|
3737
|
+
relationship to any configured timeout.
|
|
3738
|
+
"""
|
|
3739
|
+
|
|
3740
|
+
interval_seconds = max(0.001, keepalive_interval_seconds)
|
|
3741
|
+
minimum_count = max(1, floor_count)
|
|
3742
|
+
budget_seconds = _http_bridge_eventless_budget_seconds(
|
|
3743
|
+
settings,
|
|
3744
|
+
fallback_seconds=interval_seconds * minimum_count,
|
|
3745
|
+
)
|
|
3746
|
+
derived_count = max(1, math.ceil(budget_seconds / interval_seconds))
|
|
3747
|
+
if minimum_count * interval_seconds <= budget_seconds:
|
|
3748
|
+
return max(minimum_count, derived_count)
|
|
3749
|
+
return derived_count
|
|
3750
|
+
|
|
3751
|
+
|
|
3752
|
+
def _http_bridge_admission_timeout_seconds(
|
|
3753
|
+
request_state: _WebSocketRequestState,
|
|
3754
|
+
admission_timeout_seconds: float,
|
|
3755
|
+
settings: object,
|
|
3756
|
+
*,
|
|
3757
|
+
now: float,
|
|
3758
|
+
) -> float:
|
|
3759
|
+
# Bridged requests may retry response-create gate acquisition within one
|
|
3760
|
+
# bridge request budget, so every wait must be clamped to the remaining
|
|
3761
|
+
# time. Re-prepared retry states reset started_at but deliberately retain
|
|
3762
|
+
# the original deadline; using started_at alone would extend the budget.
|
|
3763
|
+
# ``now`` comes from the owner's clock so the deadline (also owner-clock
|
|
3764
|
+
# based) and the sample share one time domain.
|
|
3765
|
+
deadline = request_state.bridge_request_deadline
|
|
3766
|
+
if deadline is None:
|
|
3767
|
+
deadline = request_state.started_at + _http_bridge_request_budget_seconds(settings)
|
|
3768
|
+
remaining_budget_seconds = deadline - now
|
|
3769
|
+
return max(0.0, min(admission_timeout_seconds, remaining_budget_seconds))
|
|
3770
|
+
|
|
3771
|
+
|
|
3772
|
+
def _http_bridge_owner_check_required(
|
|
3773
|
+
key: _HTTPBridgeSessionKey,
|
|
3774
|
+
*,
|
|
3775
|
+
gateway_safe_mode: bool,
|
|
3776
|
+
) -> bool:
|
|
3777
|
+
if key.strength == "hard":
|
|
3778
|
+
return True
|
|
3779
|
+
return gateway_safe_mode and key.affinity_kind == "sticky_thread"
|
|
3780
|
+
|
|
3781
|
+
|
|
3782
|
+
def _http_bridge_key_strength(key: _HTTPBridgeSessionKey) -> str:
|
|
3783
|
+
return key.strength or "soft"
|
|
3784
|
+
|
|
3785
|
+
|
|
3786
|
+
# Frames the bridge injects into its own downstream streams. They prove the
|
|
3787
|
+
# proxy is alive, never that the upstream is.
|
|
3788
|
+
_HTTP_BRIDGE_LOCALLY_INJECTED_EVENT_TYPES = frozenset({"codex.keepalive"})
|
|
3789
|
+
|
|
3790
|
+
|
|
3791
|
+
def _http_bridge_event_proves_upstream_liveness(event_type: str | None) -> bool:
|
|
3792
|
+
"""Return whether an event type is upstream traffic rather than our own."""
|
|
3793
|
+
|
|
3794
|
+
if not event_type:
|
|
3795
|
+
return False
|
|
3796
|
+
return event_type not in _HTTP_BRIDGE_LOCALLY_INJECTED_EVENT_TYPES
|
|
3797
|
+
|
|
3798
|
+
|
|
3799
|
+
def _record_http_bridge_unmatched_upstream_liveness(
|
|
3800
|
+
session: "_HTTPBridgeSession",
|
|
3801
|
+
*,
|
|
3802
|
+
event_type: str | None,
|
|
3803
|
+
) -> int:
|
|
3804
|
+
"""Count an upstream frame that proved liveness but matched no request."""
|
|
3805
|
+
|
|
3806
|
+
if not _http_bridge_event_proves_upstream_liveness(event_type):
|
|
3807
|
+
return session.unmatched_upstream_liveness_count
|
|
3808
|
+
session.unmatched_upstream_liveness_count += 1
|
|
3809
|
+
return session.unmatched_upstream_liveness_count
|
|
3810
|
+
|
|
3811
|
+
|
|
3812
|
+
def _log_http_bridge_event(
|
|
3813
|
+
event: str,
|
|
3814
|
+
key: _HTTPBridgeSessionKey,
|
|
3815
|
+
*,
|
|
3816
|
+
account_id: str | None,
|
|
3817
|
+
model: str | None,
|
|
3818
|
+
pending_count: int | None = None,
|
|
3819
|
+
detail: str | None = None,
|
|
3820
|
+
cache_key_family: str | None = None,
|
|
3821
|
+
model_class: str | None = None,
|
|
3822
|
+
owner_check_applied: bool | None = None,
|
|
3823
|
+
error_message: str | None = None,
|
|
3824
|
+
upstream_close_code: int | None = None,
|
|
3825
|
+
response_events_seen: int | None = None,
|
|
3826
|
+
transport_classification: str | None = None,
|
|
3827
|
+
) -> None:
|
|
3828
|
+
if event in {"create", "reuse", "reconnect", "close", "evict_idle"} and http_bridge_connections_total is not None:
|
|
3829
|
+
http_bridge_connections_total.labels(event=event).inc()
|
|
3830
|
+
level = logging.INFO
|
|
3831
|
+
if event in {
|
|
3832
|
+
"queue_full",
|
|
3833
|
+
"submit_on_closed",
|
|
3834
|
+
"send_failure",
|
|
3835
|
+
"retry_fresh_upstream",
|
|
3836
|
+
"retry_precreated",
|
|
3837
|
+
"retry_precreated_clean_close",
|
|
3838
|
+
"reconnect",
|
|
3839
|
+
"terminal_error",
|
|
3840
|
+
"capacity_exhausted_active_sessions",
|
|
3841
|
+
"owner_mismatch",
|
|
3842
|
+
"owner_forward_fail",
|
|
3843
|
+
"owner_unavailable_replay_rejected",
|
|
3844
|
+
"missing_response_created_timeout",
|
|
3845
|
+
"prompt_cache_locality_miss",
|
|
3846
|
+
"reallocation_orphan",
|
|
3847
|
+
"context_overflow_rollover",
|
|
3848
|
+
"reader_failure",
|
|
3849
|
+
}:
|
|
3850
|
+
level = logging.WARNING
|
|
3851
|
+
logger.log(
|
|
3852
|
+
level,
|
|
3853
|
+
"http_bridge_event event=%s bridge_kind=%s bridge_key=%s account_id=%s"
|
|
3854
|
+
" model=%s pending=%s detail=%s cache_key_family=%s model_class=%s"
|
|
3855
|
+
" key_strength=%s owner_check_applied=%s error_message=%s upstream_close_code=%s"
|
|
3856
|
+
" response_events_seen=%s transport_classification=%s",
|
|
3857
|
+
event,
|
|
3858
|
+
key.affinity_kind,
|
|
3859
|
+
_hash_identifier(key.affinity_key),
|
|
3860
|
+
account_id,
|
|
3861
|
+
model,
|
|
3862
|
+
pending_count,
|
|
3863
|
+
detail,
|
|
3864
|
+
cache_key_family,
|
|
3865
|
+
model_class,
|
|
3866
|
+
_http_bridge_key_strength(key),
|
|
3867
|
+
owner_check_applied,
|
|
3868
|
+
error_message,
|
|
3869
|
+
upstream_close_code,
|
|
3870
|
+
response_events_seen,
|
|
3871
|
+
transport_classification,
|
|
3872
|
+
)
|
|
3873
|
+
|
|
3874
|
+
|
|
3875
|
+
def _patchable_helper(name: str, original: Callable[..., Any]) -> Callable[..., Any]:
|
|
3876
|
+
def _wrapper(*args: Any, **kwargs: Any) -> Any:
|
|
3877
|
+
service_module = sys.modules.get("app.modules.proxy.service")
|
|
3878
|
+
target = getattr(service_module, name, original) if service_module is not None else original
|
|
3879
|
+
if target is _wrapper:
|
|
3880
|
+
target = original
|
|
3881
|
+
return target(*args, **kwargs)
|
|
3882
|
+
|
|
3883
|
+
return _wrapper
|
|
3884
|
+
|
|
3885
|
+
|
|
3886
|
+
for _helper_name in (
|
|
3887
|
+
"_http_bridge_startup_wait_timeout_error",
|
|
3888
|
+
"_log_http_bridge_startup_wait_timeout",
|
|
3889
|
+
"_http_bridge_precreated_retry_failure_error",
|
|
3890
|
+
"_trim_http_bridge_previous_response_input_items",
|
|
3891
|
+
"_is_http_bridge_previous_response_output_item",
|
|
3892
|
+
"_has_http_bridge_response_output_marker",
|
|
3893
|
+
"_http_bridge_input_item_type",
|
|
3894
|
+
"_normalize_http_bridge_error_event",
|
|
3895
|
+
"_http_bridge_request_counts_against_queue",
|
|
3896
|
+
"_http_bridge_session_has_visible_requests",
|
|
3897
|
+
"_http_bridge_session_retiring_with_visible_requests",
|
|
3898
|
+
"_http_bridge_payload_looks_like_full_resend",
|
|
3899
|
+
"_preferred_http_bridge_reconnect_turn_state",
|
|
3900
|
+
"_http_bridge_reconnect_turn_state",
|
|
3901
|
+
"_http_bridge_turn_state_alias_key",
|
|
3902
|
+
"_http_bridge_previous_response_alias_key",
|
|
3903
|
+
"_http_bridge_session_allows_api_key",
|
|
3904
|
+
"_http_bridge_session_account_active",
|
|
3905
|
+
"_http_bridge_session_reusable_for_request",
|
|
3906
|
+
"_http_bridge_session_matches_preferred_account",
|
|
3907
|
+
"_make_http_bridge_session_key",
|
|
3908
|
+
"_http_bridge_should_wait_for_registration",
|
|
3909
|
+
"_durable_bridge_lookup_active_owner",
|
|
3910
|
+
"_durable_bridge_lookup_allows_local_reuse",
|
|
3911
|
+
"_http_bridge_allow_durable_takeover",
|
|
3912
|
+
"_http_bridge_claim_allows_takeover",
|
|
3913
|
+
"_http_bridge_has_durable_recovery_anchor",
|
|
3914
|
+
"_http_bridge_can_local_recover_without_ring",
|
|
3915
|
+
"_http_bridge_can_single_instance_owner_takeover_without_anchor",
|
|
3916
|
+
"_http_bridge_can_single_instance_prompt_cache_takeover_without_anchor",
|
|
3917
|
+
"_http_bridge_endpoint_matches_current_instance",
|
|
3918
|
+
"_http_bridge_can_recover_during_drain",
|
|
3919
|
+
"_http_bridge_request_stage",
|
|
3920
|
+
"_record_bridge_reattach",
|
|
3921
|
+
"_record_bridge_first_turn_timeout",
|
|
3922
|
+
"_record_bridge_drain_recovery_allowed",
|
|
3923
|
+
"_is_missing_durable_bridge_table_error",
|
|
3924
|
+
"_http_bridge_durable_lease_ttl_seconds",
|
|
3925
|
+
"_persist_http_bridge_replacement_account",
|
|
3926
|
+
"_forwarded_http_bridge_session_key",
|
|
3927
|
+
"_http_bridge_requires_cluster_registration",
|
|
3928
|
+
"_effective_http_bridge_idle_ttl_seconds",
|
|
3929
|
+
"_http_bridge_eviction_priority",
|
|
3930
|
+
"_http_bridge_capacity_after_planned_closes",
|
|
3931
|
+
"_plan_http_bridge_lru_capacity_closes",
|
|
3932
|
+
"_build_http_bridge_prewarm_text",
|
|
3933
|
+
"_http_bridge_prewarm_enabled",
|
|
3934
|
+
"_record_http_bridge_prewarm_outcome",
|
|
3935
|
+
"_record_http_bridge_stuck_retire",
|
|
3936
|
+
"_http_bridge_payload_without_previous_response_id",
|
|
3937
|
+
"_http_bridge_previous_response_error_envelope",
|
|
3938
|
+
"_http_bridge_continuity_lost_error_envelope",
|
|
3939
|
+
"_http_bridge_owner_lookup_unavailable_error_envelope",
|
|
3940
|
+
"_http_bridge_should_attempt_local_previous_response_recovery",
|
|
3941
|
+
"_http_bridge_is_explicit_previous_response_rejection",
|
|
3942
|
+
"_http_bridge_is_previous_response_owner_unavailable",
|
|
3943
|
+
"_http_bridge_should_attempt_soft_affinity_reroute",
|
|
3944
|
+
"_http_bridge_is_context_overflow_error",
|
|
3945
|
+
"_http_bridge_should_rollover_after_context_overflow",
|
|
3946
|
+
"_http_bridge_should_attempt_local_bootstrap_rebind",
|
|
3947
|
+
"_http_bridge_turn_state_anchor_for_owner_failure",
|
|
3948
|
+
"_http_bridge_durable_lookup_allows_turn_state_takeover",
|
|
3949
|
+
"_normalized_http_bridge_instance_ring",
|
|
3950
|
+
"_active_http_bridge_instance_ring",
|
|
3951
|
+
"_http_bridge_owner_instance",
|
|
3952
|
+
"_http_bridge_runtime_config",
|
|
3953
|
+
"_http_bridge_owner_check_required",
|
|
3954
|
+
"_http_bridge_key_strength",
|
|
3955
|
+
"_log_http_bridge_event",
|
|
3956
|
+
):
|
|
3957
|
+
globals()[_helper_name] = _patchable_helper(_helper_name, globals()[_helper_name])
|
|
3958
|
+
|
|
3959
|
+
del _helper_name
|