codexLB-up 1.25.0b10__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- app/__init__.py +10 -0
- app/admin_cli.py +494 -0
- app/cli.py +339 -0
- app/codex_sessions_retag.py +487 -0
- app/core/__init__.py +0 -0
- app/core/audit/__init__.py +0 -0
- app/core/audit/service.py +216 -0
- app/core/audit/types.py +94 -0
- app/core/auth/__init__.py +236 -0
- app/core/auth/api_key_cache.py +65 -0
- app/core/auth/dashboard_access.py +429 -0
- app/core/auth/dashboard_mode.py +171 -0
- app/core/auth/dashboard_session_ttl.py +74 -0
- app/core/auth/dashboard_users_cache.py +120 -0
- app/core/auth/dependencies.py +710 -0
- app/core/auth/external_identity.py +78 -0
- app/core/auth/guardian.py +374 -0
- app/core/auth/models.py +49 -0
- app/core/auth/providers/__init__.py +129 -0
- app/core/auth/providers/oidc.py +838 -0
- app/core/auth/providers/registry.py +118 -0
- app/core/auth/refresh.py +394 -0
- app/core/auth/step_up.py +117 -0
- app/core/auth/totp.py +81 -0
- app/core/balancer/__init__.py +71 -0
- app/core/balancer/logic.py +1612 -0
- app/core/balancer/rendezvous_hash.py +21 -0
- app/core/balancer/types.py +21 -0
- app/core/bootstrap.py +172 -0
- app/core/cache/invalidation.py +426 -0
- app/core/clients/__init__.py +0 -0
- app/core/clients/codex.py +996 -0
- app/core/clients/codex_version.py +187 -0
- app/core/clients/codex_version_snapshot.py +2 -0
- app/core/clients/files.py +534 -0
- app/core/clients/headers.py +29 -0
- app/core/clients/http.py +511 -0
- app/core/clients/model_fetcher.py +276 -0
- app/core/clients/native_egress.py +1304 -0
- app/core/clients/oauth.py +477 -0
- app/core/clients/proxy.py +6089 -0
- app/core/clients/proxy_websocket.py +1411 -0
- app/core/clients/rate_limit_reset_credits.py +412 -0
- app/core/clients/stream_errors.py +12 -0
- app/core/clients/thread_cache_identity.py +347 -0
- app/core/clients/usage.py +467 -0
- app/core/clock.py +170 -0
- app/core/config/__init__.py +0 -0
- app/core/config/background_jobs.py +73 -0
- app/core/config/context_window_overrides.py +166 -0
- app/core/config/dashboard_overrides.py +145 -0
- app/core/config/inheritable.py +47 -0
- app/core/config/key_fingerprint.py +115 -0
- app/core/config/settings.py +872 -0
- app/core/config/settings_cache.py +100 -0
- app/core/config/spool_retention.py +217 -0
- app/core/config/tiers.py +174 -0
- app/core/conversation_archive.py +568 -0
- app/core/crypto.py +37 -0
- app/core/draining.py +11 -0
- app/core/errors.py +365 -0
- app/core/exceptions.py +155 -0
- app/core/handlers/__init__.py +3 -0
- app/core/handlers/exceptions.py +478 -0
- app/core/http_protocol.py +176 -0
- app/core/http_protocol_httptools.py +204 -0
- app/core/ingress_limits.py +17 -0
- app/core/metadata_store.py +18 -0
- app/core/metrics/__init__.py +0 -0
- app/core/metrics/middleware.py +74 -0
- app/core/metrics/prometheus.py +604 -0
- app/core/middleware/__init__.py +25 -0
- app/core/middleware/api_firewall.py +111 -0
- app/core/middleware/app_version.py +37 -0
- app/core/middleware/dashboard_auth_proxy.py +55 -0
- app/core/middleware/dashboard_csrf.py +177 -0
- app/core/middleware/dashboard_gzip.py +41 -0
- app/core/middleware/dashboard_overrides.py +59 -0
- app/core/middleware/firewall_cache.py +91 -0
- app/core/middleware/inflight.py +126 -0
- app/core/middleware/multipart_content_encoding.py +129 -0
- app/core/middleware/path_rewrite.py +148 -0
- app/core/middleware/request_body_limit.py +196 -0
- app/core/middleware/request_decompression.py +209 -0
- app/core/middleware/request_id.py +72 -0
- app/core/middleware/required_capability_http.py +116 -0
- app/core/middleware/trusted_proxy_headers.py +36 -0
- app/core/multipart.py +237 -0
- app/core/multipart_fields.py +91 -0
- app/core/openai/__init__.py +0 -0
- app/core/openai/chat_requests.py +493 -0
- app/core/openai/chat_responses.py +931 -0
- app/core/openai/contracts.py +46 -0
- app/core/openai/exceptions.py +24 -0
- app/core/openai/host_models.py +14 -0
- app/core/openai/images.py +413 -0
- app/core/openai/message_coercion.py +376 -0
- app/core/openai/model_refresh_scheduler.py +423 -0
- app/core/openai/model_registry.py +997 -0
- app/core/openai/model_registry_store.py +548 -0
- app/core/openai/models.py +172 -0
- app/core/openai/parsing.py +97 -0
- app/core/openai/requests.py +1980 -0
- app/core/openai/strict_schema.py +199 -0
- app/core/openai/tool_call_safety.py +99 -0
- app/core/openai/v1_requests.py +187 -0
- app/core/plan_types.py +90 -0
- app/core/prestop.py +202 -0
- app/core/rate_limiter/__init__.py +0 -0
- app/core/rate_limiter/db_rate_limiter.py +215 -0
- app/core/request_locality.py +473 -0
- app/core/resilience/__init__.py +0 -0
- app/core/resilience/backpressure.py +57 -0
- app/core/resilience/bulkhead.py +202 -0
- app/core/resilience/circuit_breaker.py +150 -0
- app/core/resilience/degradation.py +55 -0
- app/core/resilience/loop_lag_monitor.py +60 -0
- app/core/resilience/memory_monitor.py +137 -0
- app/core/resilience/network_recovery.py +256 -0
- app/core/resilience/overload.py +98 -0
- app/core/resilience/toggles.py +114 -0
- app/core/retention/__init__.py +0 -0
- app/core/retention/job.py +240 -0
- app/core/retention/scheduler.py +69 -0
- app/core/runtime_logging.py +601 -0
- app/core/scheduling/__init__.py +0 -0
- app/core/scheduling/leader_election.py +993 -0
- app/core/scheduling/leader_election_handle.py +25 -0
- app/core/server.py +140 -0
- app/core/shutdown.py +326 -0
- app/core/socket_peer.py +22 -0
- app/core/startup.py +41 -0
- app/core/timeout_invariants.py +347 -0
- app/core/tracing/__init__.py +0 -0
- app/core/tracing/otel.py +207 -0
- app/core/types.py +6 -0
- app/core/upstream_proxy/__init__.py +13 -0
- app/core/upstream_proxy/cache.py +130 -0
- app/core/upstream_proxy/resolver.py +189 -0
- app/core/upstream_proxy/types.py +104 -0
- app/core/usage/__init__.py +407 -0
- app/core/usage/depletion.py +109 -0
- app/core/usage/live_hub.py +41 -0
- app/core/usage/live_snapshots.py +174 -0
- app/core/usage/logs.py +212 -0
- app/core/usage/metadata_scheduler.py +118 -0
- app/core/usage/models.py +54 -0
- app/core/usage/pricing.py +576 -0
- app/core/usage/pricing_catalog.py +241 -0
- app/core/usage/pricing_snapshot.json +821 -0
- app/core/usage/quota.py +133 -0
- app/core/usage/refresh_policy.py +21 -0
- app/core/usage/refresh_scheduler.py +713 -0
- app/core/usage/reset_credits_refresh_scheduler.py +435 -0
- app/core/usage/types.py +168 -0
- app/core/utils/__init__.py +0 -0
- app/core/utils/json_guards.py +18 -0
- app/core/utils/locks.py +33 -0
- app/core/utils/masking.py +20 -0
- app/core/utils/proxy_env.py +104 -0
- app/core/utils/request_id.py +65 -0
- app/core/utils/retry.py +77 -0
- app/core/utils/shared_future.py +172 -0
- app/core/utils/sse.py +286 -0
- app/core/utils/time.py +24 -0
- app/db/__init__.py +0 -0
- app/db/account_identity_lock.py +56 -0
- app/db/alembic/__init__.py +1 -0
- app/db/alembic/env.py +70 -0
- app/db/alembic/revision_ids.py +43 -0
- app/db/alembic/versions/20260213_000000_base_schema.py +248 -0
- app/db/alembic/versions/20260213_000100_normalize_account_plan_types.py +45 -0
- app/db/alembic/versions/20260213_000200_add_request_logs_reasoning_effort.py +45 -0
- app/db/alembic/versions/20260213_000300_add_accounts_reset_at.py +45 -0
- app/db/alembic/versions/20260213_000400_add_accounts_chatgpt_account_id.py +45 -0
- app/db/alembic/versions/20260213_000500_add_dashboard_settings.py +70 -0
- app/db/alembic/versions/20260213_000600_add_dashboard_settings_totp.py +64 -0
- app/db/alembic/versions/20260213_000700_add_dashboard_settings_password.py +59 -0
- app/db/alembic/versions/20260213_000800_add_api_keys.py +87 -0
- app/db/alembic/versions/20260214_000000_add_api_key_limits.py +143 -0
- app/db/alembic/versions/20260215_000000_add_idx_logs_requested_at.py +38 -0
- app/db/alembic/versions/20260218_000000_add_api_key_usage_reservations.py +117 -0
- app/db/alembic/versions/20260218_000100_add_import_without_overwrite_and_drop_accounts_email_unique.py +198 -0
- app/db/alembic/versions/20260225_000000_add_dashboard_settings_routing_strategy.py +53 -0
- app/db/alembic/versions/20260228_020000_align_api_key_limit_enum_types.py +104 -0
- app/db/alembic/versions/20260228_030000_add_api_firewall_allowlist.py +42 -0
- app/db/alembic/versions/20260306_000000_add_request_logs_service_tier.py +45 -0
- app/db/alembic/versions/20260307_000000_add_api_key_enforcement_fields.py +56 -0
- app/db/alembic/versions/20260308_000000_add_sqlite_performance_indexes.py +41 -0
- app/db/alembic/versions/20260309_000000_add_additional_usage_history.py +71 -0
- app/db/alembic/versions/20260309_000000_request_logs_nullable_account_id.py +28 -0
- app/db/alembic/versions/20260310_000000_add_request_logs_transport.py +45 -0
- app/db/alembic/versions/20260310_000000_fix_postgresql_enum_value_casing.py +79 -0
- app/db/alembic/versions/20260310_120000_add_sticky_session_kinds_and_affinity_ttl.py +110 -0
- app/db/alembic/versions/20260312_000000_add_additional_usage_quota_key.py +193 -0
- app/db/alembic/versions/20260312_000000_split_sticky_sessions_primary_key_by_kind.py +96 -0
- app/db/alembic/versions/20260312_010000_merge_additional_usage_and_sticky_session_heads.py +25 -0
- app/db/alembic/versions/20260312_020000_merge_request_logs_transport_and_feature_heads.py +25 -0
- app/db/alembic/versions/20260312_020000_merge_request_logs_transport_and_sticky_heads.py +25 -0
- app/db/alembic/versions/20260312_120000_add_dashboard_upstream_stream_transport.py +62 -0
- app/db/alembic/versions/20260313_024500_merge_request_log_transport_heads.py +27 -0
- app/db/alembic/versions/20260319_100937_increase_prompt_cache_ttl_to_1800s.py +73 -0
- app/db/alembic/versions/20260319_111500_merge_prompt_cache_ttl_and_upstream_transport_heads.py +25 -0
- app/db/alembic/versions/20260319_183000_normalize_sqlite_account_status_casing.py +48 -0
- app/db/alembic/versions/20260320_000000_add_request_log_requested_actual_tiers.py +51 -0
- app/db/alembic/versions/20260321_190000_tighten_dashboard_db_indexes.py +64 -0
- app/db/alembic/versions/20260321_210000_merge_request_log_tiers_and_dashboard_index_heads.py +25 -0
- app/db/alembic/versions/20260325_000000_add_request_log_cost.py +193 -0
- app/db/alembic/versions/20260328_000000_add_rate_limit_attempts.py +48 -0
- app/db/alembic/versions/20260328_120000_add_request_log_ttft.py +37 -0
- app/db/alembic/versions/20260328_130000_add_audit_logs_table.py +45 -0
- app/db/alembic/versions/20260328_140000_add_scheduler_leader_table.py +39 -0
- app/db/alembic/versions/20260330_000000_add_cache_locality_settings.py +69 -0
- app/db/alembic/versions/20260330_010000_merge_scheduler_leader_and_cache_locality_heads.py +25 -0
- app/db/alembic/versions/20260330_020000_add_bridge_ring_members.py +46 -0
- app/db/alembic/versions/20260401_000000_add_api_key_enforced_service_tier.py +52 -0
- app/db/alembic/versions/20260401_000000_add_cache_invalidation.py +41 -0
- app/db/alembic/versions/20260401_020000_merge_cache_invalidation_and_api_key_service_tier_heads.py +22 -0
- app/db/alembic/versions/20260402_000000_switch_dashboard_routing_default_to_capacity_weighted.py +66 -0
- app/db/alembic/versions/20260403_000000_add_credit_api_key_limit_values.py +44 -0
- app/db/alembic/versions/20260406_000000_add_api_key_account_assignments.py +57 -0
- app/db/alembic/versions/20260406_010000_add_api_key_assignment_scope_flag.py +75 -0
- app/db/alembic/versions/20260407_000000_add_bridge_gateway_safe_mode.py +51 -0
- app/db/alembic/versions/20260407_010000_merge_api_key_assignment_and_bridge_gateway_heads.py +25 -0
- app/db/alembic/versions/20260408_000000_switch_import_without_overwrite_default_to_true.py +73 -0
- app/db/alembic/versions/20260408_010000_merge_import_without_overwrite_and_assignment_heads.py +25 -0
- app/db/alembic/versions/20260409_000000_add_http_bridge_sessions.py +130 -0
- app/db/alembic/versions/20260409_000000_switch_sticky_threads_and_prefer_earlier_reset_defaults_to_true.py +89 -0
- app/db/alembic/versions/20260409_010000_add_dashboard_settings_bootstrap_token.py +57 -0
- app/db/alembic/versions/20260409_010000_merge_http_bridge_and_import_without_overwrite_heads.py +25 -0
- app/db/alembic/versions/20260409_020000_fix_http_bridge_last_seen_index.py +46 -0
- app/db/alembic/versions/20260410_000000_backfill_pristine_dashboard_settings_defaults.py +77 -0
- app/db/alembic/versions/20260410_030000_restore_import_without_overwrite_default_true.py +73 -0
- app/db/alembic/versions/20260410_040000_merge_dashboard_defaults_and_import_default_heads.py +25 -0
- app/db/alembic/versions/20260413_000000_add_accounts_blocked_at.py +44 -0
- app/db/alembic/versions/20260415_160000_add_automations_tables.py +189 -0
- app/db/alembic/versions/20260415_160000_add_request_logs_response_lookup_index.py +67 -0
- app/db/alembic/versions/20260417_000000_add_request_log_plan_type.py +44 -0
- app/db/alembic/versions/20260418_190000_add_automation_runs_cycle_key.py +87 -0
- app/db/alembic/versions/20260419_000000_add_automation_run_cycle_metadata.py +56 -0
- app/db/alembic/versions/20260419_020000_add_automation_run_cycles_snapshot_tables.py +342 -0
- app/db/alembic/versions/20260421_000000_add_dashboard_session_ttl_setting.py +51 -0
- app/db/alembic/versions/20260421_120000_merge_request_log_lookup_and_plan_type_heads.py +25 -0
- app/db/alembic/versions/20260421_130000_merge_automation_and_request_log_heads.py +25 -0
- app/db/alembic/versions/20260422_103000_normalize_legacy_automation_run_cycle_keys.py +648 -0
- app/db/alembic/versions/20260423_120000_add_api_key_limit_reset_at_index.py +43 -0
- app/db/alembic/versions/20260424_000000_merge_dashboard_session_ttl_and_request_log_heads.py +25 -0
- app/db/alembic/versions/20260426_000000_add_dashboard_relative_availability_settings.py +68 -0
- app/db/alembic/versions/20260427_130000_add_warmup_model_and_request_kind.py +75 -0
- app/db/alembic/versions/20260506_000000_add_account_routing_policy.py +37 -0
- app/db/alembic/versions/20260509_000000_add_api_key_traffic_class.py +37 -0
- app/db/alembic/versions/20260509_010000_add_additional_quota_routing_policies.py +37 -0
- app/db/alembic/versions/20260513_000000_add_accounts_alias.py +44 -0
- app/db/alembic/versions/20260513_000000_add_api_key_apply_to_codex_model.py +59 -0
- app/db/alembic/versions/20260514_000000_add_request_logs_api_key_time_index.py +43 -0
- app/db/alembic/versions/20260515_000000_add_prefer_earlier_reset_window.py +47 -0
- app/db/alembic/versions/20260515_000000_soft_delete_request_logs_on_account_delete.py +83 -0
- app/db/alembic/versions/20260515_020000_add_split_sticky_budget_thresholds.py +62 -0
- app/db/alembic/versions/20260516_000000_add_sqlite_hot_path_indexes.py +75 -0
- app/db/alembic/versions/20260517_000000_merge_sqlite_hot_path_and_request_log_delete_heads.py +24 -0
- app/db/alembic/versions/20260518_000000_add_http_bridge_durable_input_prefix.py +44 -0
- app/db/alembic/versions/20260518_010000_merge_http_bridge_and_sqlite_recovery_heads.py +25 -0
- app/db/alembic/versions/20260518_120000_add_dashboard_guest_access.py +57 -0
- app/db/alembic/versions/20260520_000000_merge_api_key_and_http_bridge_heads.py +26 -0
- app/db/alembic/versions/20260520_010000_add_request_logs_api_key_account_index.py +45 -0
- app/db/alembic/versions/20260520_010000_merge_automation_and_api_key_heads.py +26 -0
- app/db/alembic/versions/20260520_030000_add_quota_planner.py +194 -0
- app/db/alembic/versions/20260520_040000_merge_quota_planner_and_request_log_index_heads.py +24 -0
- app/db/alembic/versions/20260521_000000_add_account_security_work_authorized.py +46 -0
- app/db/alembic/versions/20260521_000000_merge_dashboard_guest_and_api_key_heads.py +26 -0
- app/db/alembic/versions/20260521_000000_merge_prefer_reset_window_head.py +26 -0
- app/db/alembic/versions/20260521_000000_merge_warmup_and_request_log_index_heads.py +25 -0
- app/db/alembic/versions/20260522_000000_add_limit_warmup_trigger.py +195 -0
- app/db/alembic/versions/20260524_000000_merge_accounts_alias_and_reset_window_heads.py +24 -0
- app/db/alembic/versions/20260525_000000_add_usage_raw_window_latest_index.py +31 -0
- app/db/alembic/versions/20260526_000000_add_request_log_failure_metadata.py +100 -0
- app/db/alembic/versions/20260531_000000_add_account_workspace_identity.py +39 -0
- app/db/alembic/versions/20260531_000000_add_single_account_routing.py +42 -0
- app/db/alembic/versions/20260601_000000_merge_relative_availability_and_usage_raw_heads.py +24 -0
- app/db/alembic/versions/20260601_010000_add_api_key_usage_sections.py +58 -0
- app/db/alembic/versions/20260601_010000_add_routing_policy_persistence.py +67 -0
- app/db/alembic/versions/20260601_010000_merge_security_work_and_relative_availability_heads.py +25 -0
- app/db/alembic/versions/20260601_010000_merge_split_sticky_and_relative_usage_heads.py +25 -0
- app/db/alembic/versions/20260601_010000_merge_warmup_and_relative_availability_heads.py +25 -0
- app/db/alembic/versions/20260601_020000_merge_additional_quota_routing_and_relative_availability_heads.py +25 -0
- app/db/alembic/versions/20260601_020000_merge_quota_planner_and_relative_availability_heads.py +25 -0
- app/db/alembic/versions/20260601_020000_merge_reset_window_and_relative_availability_heads.py +25 -0
- app/db/alembic/versions/20260601_020000_merge_routing_policy_and_split_threshold_heads.py +24 -0
- app/db/alembic/versions/20260601_020000_merge_warmup_and_request_log_failure_heads.py +25 -0
- app/db/alembic/versions/20260601_030000_merge_single_account_and_routing_quota_heads.py +29 -0
- app/db/alembic/versions/20260601_040000_merge_security_work_and_routing_consolidation_heads.py +25 -0
- app/db/alembic/versions/20260601_050000_merge_quota_planner_and_routing_consolidation_heads.py +25 -0
- app/db/alembic/versions/20260602_050000_add_upstream_proxy_routing.py +164 -0
- app/db/alembic/versions/20260602_060000_merge_account_workspace_and_failure_heads.py +25 -0
- app/db/alembic/versions/20260602_070000_add_request_log_useragent_fields.py +86 -0
- app/db/alembic/versions/20260602_070000_merge_quota_planner_and_account_workspace_heads.py +25 -0
- app/db/alembic/versions/20260602_080000_merge_upstream_proxy_and_quota_planner_heads.py +25 -0
- app/db/alembic/versions/20260602_080000_merge_useragent_and_upstream_proxy_heads.py +24 -0
- app/db/alembic/versions/20260603_000000_add_weekly_pace_working_days.py +52 -0
- app/db/alembic/versions/20260603_000000_free_account_monthly_window.py +60 -0
- app/db/alembic/versions/20260604_000000_add_reauth_required_account_status.py +159 -0
- app/db/alembic/versions/20260604_010000_merge_automations_and_reauth_status_heads.py +25 -0
- app/db/alembic/versions/20260604_010000_merge_dashboard_guest_and_reauth_status_heads.py +25 -0
- app/db/alembic/versions/20260604_010000_merge_useragent_and_reauth_heads.py +24 -0
- app/db/alembic/versions/20260607_000000_merge_weekly_monthly_useragent_heads.py +28 -0
- app/db/alembic/versions/20260608_000000_add_hide_upstream_quota_from_api_keys.py +50 -0
- app/db/alembic/versions/20260611_000000_add_request_log_archive_request_id.py +47 -0
- app/db/alembic/versions/20260611_000000_merge_automations_and_weekly_monthly_useragent_heads.py +26 -0
- app/db/alembic/versions/20260611_000000_merge_dashboard_guest_and_weekly_useragent_heads.py +26 -0
- app/db/alembic/versions/20260611_010000_add_request_log_client_ip.py +57 -0
- app/db/alembic/versions/20260611_020000_merge_request_log_archive_and_client_ip_heads.py +26 -0
- app/db/alembic/versions/20260611_030000_merge_dashboard_guest_and_request_log_client_ip_heads.py +26 -0
- app/db/alembic/versions/20260613_000000_add_accounts_codex_installation_id.py +57 -0
- app/db/alembic/versions/20260613_010000_merge_installation_id_and_dashboard_weekly_heads.py +26 -0
- app/db/alembic/versions/20260615_000000_merge_automations_and_dashboard_guest_heads.py +26 -0
- app/db/alembic/versions/20260623_000000_add_limit_warmup_exhausted_threshold.py +51 -0
- app/db/alembic/versions/20260626_000000_add_http_downstream_transport_policy.py +46 -0
- app/db/alembic/versions/20260626_010000_add_request_logs_upstream_transport.py +31 -0
- app/db/alembic/versions/20260629_000000_add_dashboard_query_hot_path_indexes.py +53 -0
- app/db/alembic/versions/20260630_000000_add_limit_warmup_staggered_idle.py +47 -0
- app/db/alembic/versions/20260630_000000_merge_request_log_client_ip_and_dashboard_hot_path_heads.py +26 -0
- app/db/alembic/versions/20260630_010000_merge_warmup_and_request_log_dashboard_heads.py +26 -0
- app/db/alembic/versions/20260630_020000_merge_automations_and_main_heads.py +26 -0
- app/db/alembic/versions/20260630_020000_merge_warmup_threshold_and_main_heads.py +26 -0
- app/db/alembic/versions/20260630_030000_add_automation_run_model_snapshot.py +59 -0
- app/db/alembic/versions/20260630_040000_merge_automation_snapshot_and_warmup_threshold_heads.py +26 -0
- app/db/alembic/versions/20260630_050000_add_automation_run_prompt_snapshot.py +52 -0
- app/db/alembic/versions/20260701_000000_add_weekly_pace_smoothing_minutes.py +51 -0
- app/db/alembic/versions/20260702_000000_add_model_sources.py +176 -0
- app/db/alembic/versions/20260705_000000_add_model_source_audio_transcriptions.py +50 -0
- app/db/alembic/versions/20260705_000000_harden_dashboard_session_ttl.py +62 -0
- app/db/alembic/versions/20260706_000000_add_model_source_audio_per_minute.py +43 -0
- app/db/alembic/versions/20260707_000000_merge_model_sources_and_session_ttl.py +24 -0
- app/db/alembic/versions/20260707_010000_merge_automations_and_model_sources_heads.py +24 -0
- app/db/alembic/versions/20260707_020000_add_automation_job_account_scope.py +125 -0
- app/db/alembic/versions/20260709_000000_add_ttft_phase_observability.py +66 -0
- app/db/alembic/versions/20260711_000000_add_dashboard_prohibit_fast_mode.py +42 -0
- app/db/alembic/versions/20260711_010000_add_account_chatgpt_user_id.py +40 -0
- app/db/alembic/versions/20260711_020000_add_dashboard_account_concurrency_caps.py +71 -0
- app/db/alembic/versions/20260711_030000_add_limit_warmup_idle_threshold.py +51 -0
- app/db/alembic/versions/20260712_010000_add_account_usage_rollups.py +56 -0
- app/db/alembic/versions/20260712_020000_add_api_key_usage_rollups.py +53 -0
- app/db/alembic/versions/20260713_020000_add_model_registry_snapshot.py +42 -0
- app/db/alembic/versions/20260713_040000_add_account_refresh_claims.py +42 -0
- app/db/alembic/versions/20260713_040000_add_replica_guardrails.py +64 -0
- app/db/alembic/versions/20260713_070000_add_reset_credit_redeem_tables.py +58 -0
- app/db/alembic/versions/20260714_000000_add_oauth_flow_states.py +63 -0
- app/db/alembic/versions/20260715_000000_add_request_log_queue_latency.py +44 -0
- app/db/alembic/versions/20260716_000000_add_oauth_device_flow_slots.py +41 -0
- app/db/alembic/versions/20260716_010000_add_dashboard_retention_settings.py +48 -0
- app/db/alembic/versions/20260716_010000_add_reset_credit_display_settings.py +73 -0
- app/db/alembic/versions/20260717_000000_add_http_bridge_retry_circuits.py +47 -0
- app/db/alembic/versions/20260717_000000_merge_retention_and_reset_credit_display_heads.py +26 -0
- app/db/alembic/versions/20260717_000000_optimize_dashboard_hot_path_indexes.py +162 -0
- app/db/alembic/versions/20260717_000001_merge_retry_circuits_and_dashboard_indexes.py +22 -0
- app/db/alembic/versions/20260720_000000_add_request_log_conversation_id.py +57 -0
- app/db/alembic/versions/20260722_000000_backfill_request_log_useragent_families.py +40 -0
- app/db/alembic/versions/20260724_000000_add_request_usage_time_rollups.py +176 -0
- app/db/alembic/versions/20260724_000000_merge_request_log_schema_heads.py +33 -0
- app/db/alembic/versions/20260725_000000_add_http_bridge_pending_tool_calls.py +43 -0
- app/db/alembic/versions/20260726_000000_add_account_plan_downgrade_observations.py +44 -0
- app/db/alembic/versions/20260726_000000_repair_request_usage_rollups_after_merge.py +37 -0
- app/db/alembic/versions/20260727_000000_add_sticky_session_continuity_abandoned_at.py +45 -0
- app/db/alembic/versions/20260728_000000_merge_pending_tool_calls_and_rollup_repair_heads.py +31 -0
- app/db/alembic/versions/20260730_000000_add_api_key_fair_share_threshold.py +44 -0
- app/db/alembic/versions/20260730_000000_add_http_bridge_recovery_attempts.py +60 -0
- app/db/alembic/versions/20260731_000000_add_capability_lineage_markers.py +38 -0
- app/db/alembic/versions/20260803_000000_merge_http_bridge_recovery_and_capability_lineage_heads.py +30 -0
- app/db/alembic/versions/20260804_000000_add_http_bridge_operations.py +63 -0
- app/db/alembic/versions/20260804_000001_add_global_http_bridge_operation_fingerprint.py +53 -0
- app/db/alembic/versions/20260804_230000_add_request_log_connection_request_kind.py +43 -0
- app/db/alembic/versions/20260805_000000_add_http_bridge_operation_spool.py +75 -0
- app/db/alembic/versions/20260805_000001_finalize_http_bridge_operation_spool.py +65 -0
- app/db/alembic/versions/20260806_000000_add_additional_usage_alias_probe_indexes.py +84 -0
- app/db/alembic/versions/20260806_000000_add_anonymous_telemetry.py +49 -0
- app/db/alembic/versions/20260806_010000_add_conversation_presence_rollup.py +72 -0
- app/db/alembic/versions/20260806_020000_add_usage_history_bulk_covering_indexes.py +99 -0
- app/db/alembic/versions/20260806_030000_add_api_key_allowed_reasoning_efforts.py +65 -0
- app/db/alembic/versions/20260806_120000_add_http_bridge_owner_process_epoch.py +57 -0
- app/db/alembic/versions/20260807_000000_merge_http_bridge_operation_spool_and_latest_main.py +24 -0
- app/db/alembic/versions/20260808_000000_tune_usage_history_autovacuum.py +54 -0
- app/db/alembic/versions/20260810_000000_add_http_bridge_recovery_dispatch_count.py +44 -0
- app/db/alembic/versions/20260811_000000_add_hourly_rollup_cancelled_count.py +92 -0
- app/db/alembic/versions/20260812_000000_merge_recovery_dispatch_and_hourly_cancelled_heads.py +25 -0
- app/db/alembic/versions/20260812_120000_add_sticky_abandonment_scope.py +43 -0
- app/db/alembic/versions/20260813_000000_add_file_account_pins.py +40 -0
- app/db/alembic/versions/20260816_000000_add_account_pending_deletion.py +81 -0
- app/db/alembic/versions/20260816_000000_add_model_source_embeddings.py +50 -0
- app/db/alembic/versions/20260821_000000_add_retry_circuit_admission_generation.py +49 -0
- app/db/alembic/versions/20260826_000000_add_http_bridge_event_chunks.py +107 -0
- app/db/alembic/versions/20260828_000000_add_accounts_chatgpt_identity_index.py +55 -0
- app/db/alembic/versions/20260830_000000_add_quota_warmup_claim_expiry.py +83 -0
- app/db/alembic/versions/20260908_000000_add_guest_session_generation.py +37 -0
- app/db/alembic/versions/20260908_000000_add_subscription_overflow.py +140 -0
- app/db/alembic/versions/20260908_000000_replace_upstream_stream_transport_default_sentinel.py +50 -0
- app/db/alembic/versions/20260908_020000_merge_overflow_transport_heads.py +27 -0
- app/db/alembic/versions/20260909_000000_add_dashboard_roles.py +72 -0
- app/db/alembic/versions/20260909_010000_add_dashboard_users.py +183 -0
- app/db/alembic/versions/20260909_020000_reproject_compat_admin_credentials.py +105 -0
- app/db/alembic/versions/20260909_030000_add_audit_actor_columns.py +103 -0
- app/db/alembic/versions/20260909_030000_dashboard_resilience_toggle_settings.py +63 -0
- app/db/alembic/versions/20260909_040000_add_dashboard_user_invites.py +54 -0
- app/db/alembic/versions/20260909_040000_dashboard_timeout_settings.py +68 -0
- app/db/alembic/versions/20260909_050000_dashboard_routing_overload_settings.py +65 -0
- app/db/alembic/versions/20260909_060000_add_report_rollup.py +74 -0
- app/db/alembic/versions/20260909_070000_automation_run_claim_budget.py +50 -0
- app/db/alembic/versions/20260909_080000_dashboard_stream_bridge_budgets.py +62 -0
- app/db/alembic/versions/20260909_090000_dashboard_background_job_toggles.py +63 -0
- app/db/alembic/versions/20260909_100000_dashboard_codex_prewarm.py +49 -0
- app/db/alembic/versions/20260909_110000_model_context_window_overrides.py +45 -0
- app/db/alembic/versions/20260909_120000_dashboard_conversation_archive.py +48 -0
- app/db/alembic/versions/20260909_130000_add_request_logs_live_facet_indexes.py +96 -0
- app/db/alembic/versions/20260910_000000_add_totp_required_for_admin_role.py +44 -0
- app/db/alembic/versions/20260910_000000_request_logs_missing_cost_index.py +52 -0
- app/db/alembic/versions/20260910_010000_add_dashboard_auth_providers.py +134 -0
- app/db/alembic/versions/20260910_010000_dashboard_spool_retention.py +52 -0
- app/db/alembic/versions/20260910_020000_add_dashboard_role_mappings.py +74 -0
- app/db/alembic/versions/20260910_143000_request_log_affinity.py +28 -0
- app/db/alembic/versions/20260910_180000_merge_affinity_guest_heads.py +17 -0
- app/db/alembic/versions/20260910_200000_merge_affinity_identity_heads.py +17 -0
- app/db/alembic/versions/20260910_220000_merge_affinity_invite_heads.py +17 -0
- app/db/alembic/versions/20260911_000000_model_source_pins_kind_expires_index.py +43 -0
- app/db/alembic/versions/20260911_010000_merge_pin_index_and_affinity_heads.py +17 -0
- app/db/alembic/versions/20260911_020000_add_http_bridge_terminal_append_phase.py +54 -0
- app/db/alembic/versions/20260911_030000_add_local_login_policy.py +48 -0
- app/db/alembic/versions/20260911_040000_add_thread_cache_identity_mode.py +48 -0
- app/db/alembic/versions/20260911_060000_add_bridge_session_continuity_abandonment.py +40 -0
- app/db/alembic/versions/20260912_000000_merge_thread_cache_and_bridge_retirement_heads.py +22 -0
- app/db/alembic/versions/20260912_010000_drop_legacy_dashboard_credentials.py +140 -0
- app/db/alembic/versions/20260913_000000_add_oidc_provider_flow.py +134 -0
- app/db/alembic/versions/20260914_000000_add_scim_tokens.py +78 -0
- app/db/alembic/versions/20260914_000000_drop_subscription_overflow_schema.py +132 -0
- app/db/alembic/versions/20260918_000000_merge_scim_and_overflow_heads.py +17 -0
- app/db/alembic/versions/__init__.py +1 -0
- app/db/backup.py +64 -0
- app/db/migrate.py +1249 -0
- app/db/migration_lock.py +160 -0
- app/db/migration_url.py +18 -0
- app/db/models.py +3100 -0
- app/db/recover.py +244 -0
- app/db/session.py +1295 -0
- app/db/snapshot.py +46 -0
- app/db/sqlite_lock_retry.py +190 -0
- app/db/sqlite_utils.py +549 -0
- app/dependencies.py +462 -0
- app/main.py +1240 -0
- app/modules/__init__.py +0 -0
- app/modules/accounts/__init__.py +0 -0
- app/modules/accounts/api.py +426 -0
- app/modules/accounts/auth_manager.py +1439 -0
- app/modules/accounts/background_repository.py +160 -0
- app/modules/accounts/deletion.py +540 -0
- app/modules/accounts/mappers.py +654 -0
- app/modules/accounts/refresh_claims.py +352 -0
- app/modules/accounts/repository.py +1679 -0
- app/modules/accounts/schemas.py +308 -0
- app/modules/accounts/service.py +809 -0
- app/modules/accounts/usage_rollup.py +486 -0
- app/modules/accounts/usage_rollup_scheduler.py +75 -0
- app/modules/accounts/usage_time_rollup.py +1037 -0
- app/modules/accounts/usage_time_rollup_read.py +522 -0
- app/modules/api_keys/__init__.py +1 -0
- app/modules/api_keys/api.py +344 -0
- app/modules/api_keys/last_used_coalescer.py +216 -0
- app/modules/api_keys/limit_windows.py +41 -0
- app/modules/api_keys/repository.py +1200 -0
- app/modules/api_keys/reset_scheduler.py +85 -0
- app/modules/api_keys/schemas.py +135 -0
- app/modules/api_keys/service.py +2123 -0
- app/modules/audit/__init__.py +0 -0
- app/modules/audit/api.py +88 -0
- app/modules/audit/repository.py +74 -0
- app/modules/audit/schemas.py +33 -0
- app/modules/audit/service.py +71 -0
- app/modules/auth_providers/__init__.py +0 -0
- app/modules/auth_providers/api.py +127 -0
- app/modules/auth_providers/config.py +343 -0
- app/modules/auth_providers/repository.py +101 -0
- app/modules/auth_providers/schemas.py +88 -0
- app/modules/auth_providers/seed.py +105 -0
- app/modules/auth_providers/service.py +289 -0
- app/modules/automations/__init__.py +1 -0
- app/modules/automations/api.py +369 -0
- app/modules/automations/repository.py +1896 -0
- app/modules/automations/scheduler.py +107 -0
- app/modules/automations/schemas.py +158 -0
- app/modules/automations/service.py +2548 -0
- app/modules/cache_isolation_probe/__init__.py +0 -0
- app/modules/cache_isolation_probe/api.py +188 -0
- app/modules/cache_isolation_probe/prefix.py +144 -0
- app/modules/cache_isolation_probe/schemas.py +94 -0
- app/modules/cache_isolation_probe/sender.py +229 -0
- app/modules/cache_isolation_probe/service.py +525 -0
- app/modules/conversation_archive/__init__.py +1 -0
- app/modules/conversation_archive/api.py +101 -0
- app/modules/conversation_archive/schemas.py +28 -0
- app/modules/conversation_archive/service.py +165 -0
- app/modules/dashboard/__init__.py +1 -0
- app/modules/dashboard/api.py +106 -0
- app/modules/dashboard/builders.py +107 -0
- app/modules/dashboard/repository.py +304 -0
- app/modules/dashboard/schemas.py +148 -0
- app/modules/dashboard/service.py +534 -0
- app/modules/dashboard/timeframes.py +19 -0
- app/modules/dashboard/weekly_pace.py +617 -0
- app/modules/dashboard_auth/__init__.py +1 -0
- app/modules/dashboard_auth/api.py +1344 -0
- app/modules/dashboard_auth/oidc_api.py +753 -0
- app/modules/dashboard_auth/oidc_flows.py +330 -0
- app/modules/dashboard_auth/repository.py +452 -0
- app/modules/dashboard_auth/schemas.py +232 -0
- app/modules/dashboard_auth/service.py +1384 -0
- app/modules/dashboard_roles/__init__.py +0 -0
- app/modules/dashboard_roles/api.py +64 -0
- app/modules/dashboard_roles/repository.py +54 -0
- app/modules/dashboard_roles/schemas.py +29 -0
- app/modules/dashboard_roles/seed.py +80 -0
- app/modules/dashboard_roles/service.py +142 -0
- app/modules/dashboard_users/__init__.py +0 -0
- app/modules/dashboard_users/api.py +278 -0
- app/modules/dashboard_users/break_glass.py +169 -0
- app/modules/dashboard_users/credentials.py +18 -0
- app/modules/dashboard_users/identity_resolver.py +665 -0
- app/modules/dashboard_users/repository.py +725 -0
- app/modules/dashboard_users/schemas.py +117 -0
- app/modules/dashboard_users/service.py +1054 -0
- app/modules/firewall/__init__.py +1 -0
- app/modules/firewall/api.py +78 -0
- app/modules/firewall/repository.py +51 -0
- app/modules/firewall/schemas.py +28 -0
- app/modules/firewall/service.py +93 -0
- app/modules/fleet/__init__.py +1 -0
- app/modules/fleet/api.py +177 -0
- app/modules/fleet/mappers.py +56 -0
- app/modules/fleet/observability.py +372 -0
- app/modules/fleet/schemas.py +119 -0
- app/modules/health/__init__.py +0 -0
- app/modules/health/api.py +246 -0
- app/modules/health/schemas.py +27 -0
- app/modules/limit_warmup/__init__.py +1 -0
- app/modules/limit_warmup/repository.py +159 -0
- app/modules/limit_warmup/service.py +1001 -0
- app/modules/model_sources/__init__.py +1 -0
- app/modules/model_sources/api.py +99 -0
- app/modules/model_sources/catalog.py +307 -0
- app/modules/model_sources/forwarding.py +1435 -0
- app/modules/model_sources/projection.py +55 -0
- app/modules/model_sources/repository.py +201 -0
- app/modules/model_sources/schemas.py +79 -0
- app/modules/model_sources/selection.py +166 -0
- app/modules/model_sources/service.py +236 -0
- app/modules/oauth/__init__.py +0 -0
- app/modules/oauth/api.py +101 -0
- app/modules/oauth/repository.py +238 -0
- app/modules/oauth/schemas.py +49 -0
- app/modules/oauth/service.py +1196 -0
- app/modules/oauth/templates/oauth_success.html +122 -0
- app/modules/proxy/__init__.py +0 -0
- app/modules/proxy/_load_balancer/__init__.py +1 -0
- app/modules/proxy/_load_balancer/error_rate.py +116 -0
- app/modules/proxy/_load_balancer/latency_cohort.py +154 -0
- app/modules/proxy/_load_balancer/model_eligibility.py +250 -0
- app/modules/proxy/_load_balancer/opportunistic_admission.py +350 -0
- app/modules/proxy/_load_balancer/overload_backoff.py +394 -0
- app/modules/proxy/_load_balancer/sticky_selection.py +2417 -0
- app/modules/proxy/_load_balancer/throughput_cohort.py +233 -0
- app/modules/proxy/_load_balancer/ttft_cohort.py +214 -0
- app/modules/proxy/_load_balancer/tunables.py +97 -0
- app/modules/proxy/_load_balancer/types.py +96 -0
- app/modules/proxy/_load_balancer/unbound_selection.py +534 -0
- app/modules/proxy/_service/__init__.py +6 -0
- app/modules/proxy/_service/api_key_usage.py +745 -0
- app/modules/proxy/_service/clock_budget.py +38 -0
- app/modules/proxy/_service/codex_control.py +611 -0
- app/modules/proxy/_service/compact.py +2211 -0
- app/modules/proxy/_service/file_ops.py +749 -0
- app/modules/proxy/_service/http_bridge/__init__.py +3 -0
- app/modules/proxy/_service/http_bridge/accepted_replay.py +331 -0
- app/modules/proxy/_service/http_bridge/account_sessions.py +57 -0
- app/modules/proxy/_service/http_bridge/activity.py +178 -0
- app/modules/proxy/_service/http_bridge/helpers.py +3959 -0
- app/modules/proxy/_service/http_bridge/mixin.py +2435 -0
- app/modules/proxy/_service/http_bridge/owner_forwarding.py +609 -0
- app/modules/proxy/_service/http_bridge/protocol.py +167 -0
- app/modules/proxy/_service/http_bridge/proxy_failover.py +58 -0
- app/modules/proxy/_service/http_bridge/quarantine.py +443 -0
- app/modules/proxy/_service/http_bridge/request_submit.py +4582 -0
- app/modules/proxy/_service/http_bridge/retry_circuit.py +2442 -0
- app/modules/proxy/_service/http_bridge/service_stubs.py +492 -0
- app/modules/proxy/_service/http_bridge/session_registry.py +778 -0
- app/modules/proxy/_service/http_bridge/streaming.py +5328 -0
- app/modules/proxy/_service/http_bridge/upstream_events.py +4538 -0
- app/modules/proxy/_service/observability.py +410 -0
- app/modules/proxy/_service/rate_limit.py +361 -0
- app/modules/proxy/_service/realtime_live.py +655 -0
- app/modules/proxy/_service/refresh.py +154 -0
- app/modules/proxy/_service/request_log.py +585 -0
- app/modules/proxy/_service/response_create.py +930 -0
- app/modules/proxy/_service/streaming/__init__.py +3 -0
- app/modules/proxy/_service/streaming/helpers.py +1317 -0
- app/modules/proxy/_service/streaming/mixin.py +1100 -0
- app/modules/proxy/_service/streaming/protocol.py +32 -0
- app/modules/proxy/_service/streaming/retry.py +3762 -0
- app/modules/proxy/_service/support.py +2122 -0
- app/modules/proxy/_service/transcribe.py +434 -0
- app/modules/proxy/_service/warmup.py +483 -0
- app/modules/proxy/_service/websocket/__init__.py +3 -0
- app/modules/proxy/_service/websocket/helpers.py +2360 -0
- app/modules/proxy/_service/websocket/mixin.py +7285 -0
- app/modules/proxy/_service/websocket/protocol.py +74 -0
- app/modules/proxy/account_cache.py +229 -0
- app/modules/proxy/account_eligibility.py +87 -0
- app/modules/proxy/additional_model_limits.py +40 -0
- app/modules/proxy/affinity.py +830 -0
- app/modules/proxy/affinity_observation.py +101 -0
- app/modules/proxy/api.py +10254 -0
- app/modules/proxy/api_key_usage.py +113 -0
- app/modules/proxy/cap_partitioning.py +280 -0
- app/modules/proxy/capability_lineage.py +65 -0
- app/modules/proxy/capability_lineage_repository.py +82 -0
- app/modules/proxy/capability_routing.py +253 -0
- app/modules/proxy/continuity.py +112 -0
- app/modules/proxy/downstream_delivery.py +277 -0
- app/modules/proxy/durable_bridge_coordinator.py +1217 -0
- app/modules/proxy/durable_bridge_repository.py +4516 -0
- app/modules/proxy/durable_bridge_runtime.py +9 -0
- app/modules/proxy/durable_bridge_transcript_codec.py +122 -0
- app/modules/proxy/fair_share.py +131 -0
- app/modules/proxy/file_pin_repository.py +210 -0
- app/modules/proxy/helpers.py +439 -0
- app/modules/proxy/http_bridge_event_batcher.py +613 -0
- app/modules/proxy/http_bridge_forwarding.py +797 -0
- app/modules/proxy/http_continuation.py +76 -0
- app/modules/proxy/images_observability.py +62 -0
- app/modules/proxy/images_service.py +1054 -0
- app/modules/proxy/load_balancer.py +3016 -0
- app/modules/proxy/rate_limit_cache.py +46 -0
- app/modules/proxy/replay_safety.py +1063 -0
- app/modules/proxy/repo_bundle.py +31 -0
- app/modules/proxy/request_policy.py +1054 -0
- app/modules/proxy/ring_membership.py +251 -0
- app/modules/proxy/schemas.py +345 -0
- app/modules/proxy/selection_errors.py +41 -0
- app/modules/proxy/service.py +2600 -0
- app/modules/proxy/source_admission.py +125 -0
- app/modules/proxy/source_dispatch.py +999 -0
- app/modules/proxy/sticky_repository.py +831 -0
- app/modules/proxy/thread_anchors.py +491 -0
- app/modules/proxy/tool_call_dedupe.py +802 -0
- app/modules/proxy/types.py +54 -0
- app/modules/proxy/work_admission.py +121 -0
- app/modules/quota_planner/__init__.py +1 -0
- app/modules/quota_planner/api.py +276 -0
- app/modules/quota_planner/logic.py +743 -0
- app/modules/quota_planner/repository.py +722 -0
- app/modules/quota_planner/scheduler.py +212 -0
- app/modules/quota_planner/schemas.py +102 -0
- app/modules/quota_planner/warmup.py +693 -0
- app/modules/rate_limit_reset_credits/__init__.py +0 -0
- app/modules/rate_limit_reset_credits/api.py +601 -0
- app/modules/rate_limit_reset_credits/redeem_coordination.py +274 -0
- app/modules/rate_limit_reset_credits/store.py +114 -0
- app/modules/reports/__init__.py +1 -0
- app/modules/reports/api.py +134 -0
- app/modules/reports/cache.py +74 -0
- app/modules/reports/filters.py +27 -0
- app/modules/reports/repository.py +525 -0
- app/modules/reports/rollup.py +150 -0
- app/modules/reports/rollup_read.py +148 -0
- app/modules/reports/schemas.py +123 -0
- app/modules/reports/service.py +287 -0
- app/modules/reports/thread_identity.py +256 -0
- app/modules/request_logs/__init__.py +0 -0
- app/modules/request_logs/api.py +224 -0
- app/modules/request_logs/cost_backfill.py +154 -0
- app/modules/request_logs/mappers.py +97 -0
- app/modules/request_logs/repository.py +1765 -0
- app/modules/request_logs/schemas.py +145 -0
- app/modules/request_logs/service.py +348 -0
- app/modules/role_mappings/__init__.py +0 -0
- app/modules/role_mappings/api.py +206 -0
- app/modules/role_mappings/matching.py +55 -0
- app/modules/role_mappings/repository.py +106 -0
- app/modules/role_mappings/schemas.py +67 -0
- app/modules/role_mappings/service.py +334 -0
- app/modules/runtime/__init__.py +1 -0
- app/modules/runtime/api.py +18 -0
- app/modules/runtime/schemas.py +14 -0
- app/modules/runtime/service.py +218 -0
- app/modules/scim/__init__.py +1 -0
- app/modules/scim/api.py +194 -0
- app/modules/scim/dependencies.py +112 -0
- app/modules/scim/errors.py +77 -0
- app/modules/scim/management_api.py +156 -0
- app/modules/scim/repository.py +212 -0
- app/modules/scim/schemas.py +145 -0
- app/modules/scim/service.py +786 -0
- app/modules/scim/tokens.py +43 -0
- app/modules/settings/__init__.py +1 -0
- app/modules/settings/api.py +1677 -0
- app/modules/settings/repository.py +606 -0
- app/modules/settings/schemas.py +496 -0
- app/modules/settings/service.py +769 -0
- app/modules/shared/__init__.py +0 -0
- app/modules/shared/schemas.py +22 -0
- app/modules/sticky_sessions/__init__.py +1 -0
- app/modules/sticky_sessions/api.py +118 -0
- app/modules/sticky_sessions/cleanup_scheduler.py +487 -0
- app/modules/sticky_sessions/schemas.py +78 -0
- app/modules/sticky_sessions/service.py +196 -0
- app/modules/telemetry/__init__.py +5 -0
- app/modules/telemetry/api.py +115 -0
- app/modules/telemetry/clients.py +62 -0
- app/modules/telemetry/consent.py +115 -0
- app/modules/telemetry/scheduler.py +85 -0
- app/modules/telemetry/schemas.py +163 -0
- app/modules/telemetry/sender.py +221 -0
- app/modules/telemetry/snapshot.py +490 -0
- app/modules/usage/__init__.py +0 -0
- app/modules/usage/additional_quota_keys.py +379 -0
- app/modules/usage/api.py +45 -0
- app/modules/usage/background_repository.py +147 -0
- app/modules/usage/builders.py +559 -0
- app/modules/usage/depletion_service.py +277 -0
- app/modules/usage/live_ingest.py +356 -0
- app/modules/usage/mappers.py +29 -0
- app/modules/usage/plan_downgrade_observations.py +488 -0
- app/modules/usage/repository.py +1890 -0
- app/modules/usage/schemas.py +68 -0
- app/modules/usage/service.py +102 -0
- app/modules/usage/updater.py +1488 -0
- app/static/assets/Trans-Cj-J9mVA.js +1 -0
- app/static/assets/access-page-DHzZiXZi.js +1 -0
- app/static/assets/account-cost-donut-DGhfA4U8.js +1 -0
- app/static/assets/account-identifiers-CxbdLshZ.js +1 -0
- app/static/assets/account-status-DIsU5sUS.js +1 -0
- app/static/assets/account-trend-chart-BptlfwwQ.js +1 -0
- app/static/assets/accounts-page-Bbai8ift.js +3 -0
- app/static/assets/api-Dwa7XxPe.js +1 -0
- app/static/assets/api-client-dugX1Pti.js +55 -0
- app/static/assets/api-key-create-dialog-DnRm0oYN.js +1 -0
- app/static/assets/api-key-created-dialog-B6jkDG9f.js +1 -0
- app/static/assets/api-key-edit-dialog-Ck3Bnc3V.js +1 -0
- app/static/assets/api-trend-chart-BchCw7Hj.js +1 -0
- app/static/assets/apis-page-BvXz9c6i.js +2 -0
- app/static/assets/arrow-up-Dqh_KzJo.js +1 -0
- app/static/assets/arrow-up-down-LBRgVDGY.js +1 -0
- app/static/assets/automations-page-DQgNAJOR.js +3 -0
- app/static/assets/badge-DqMb9es0.js +1 -0
- app/static/assets/blur-email-HEvKagG2.js +1 -0
- app/static/assets/button-BjVmmiGW.js +1 -0
- app/static/assets/checkbox-apsl9vzk.js +1 -0
- app/static/assets/chevron-left-1d_pHVa5.js +1 -0
- app/static/assets/circle-alert-BS0xd_o1.js +1 -0
- app/static/assets/clsx-DB0hHKMi.js +1 -0
- app/static/assets/colors-BBy8GEtr.js +1 -0
- app/static/assets/copy-button-D7nAvOKR.js +1 -0
- app/static/assets/cost-per-day-chart-YdqlkZVM.js +1 -0
- app/static/assets/createLucideIcon-DBLzLNh2.js +1 -0
- app/static/assets/dashboard-page-CSdhBy0f.js +2 -0
- app/static/assets/distribution-metric-format-yZTy1ZEu.js +1 -0
- app/static/assets/donut-chart-Bxc69cra.js +1 -0
- app/static/assets/download-kQteIcsB.js +1 -0
- app/static/assets/ellipsis-B_5OgIre.js +1 -0
- app/static/assets/empty-state-JcA7tc8m.js +1 -0
- app/static/assets/es2015-bMH8YKZT.js +41 -0
- app/static/assets/es6-B264ni6W.js +69 -0
- app/static/assets/external-link-UVAOgMQ0.js +1 -0
- app/static/assets/i18n-BWqZ9Ua2.js +1 -0
- app/static/assets/inbox-B_RM2s1J.js +1 -0
- app/static/assets/index-DIpd264Z.js +23 -0
- app/static/assets/index-DMeLKRJx.css +2 -0
- app/static/assets/inherit-badge-QhTYVaU6.js +1 -0
- app/static/assets/invite-dialog-0CYh0GiQ.js +1 -0
- app/static/assets/iso-Bpe5Ihbb.js +1 -0
- app/static/assets/label-CGaLmLNf.js +1 -0
- app/static/assets/lazy-recharts-Pv8p9ArM.js +2 -0
- app/static/assets/loading-overlay-CzKQfHLI.js +1 -0
- app/static/assets/mini-quota-bar-cGT_t8Ae.js +1 -0
- app/static/assets/model-distribution-donut-BKjAfT48.js +1 -0
- app/static/assets/model-source-multi-select-D2d06Vc7.js +1 -0
- app/static/assets/multi-select-filter-KIA5qU6Z.js +1 -0
- app/static/assets/oauth-dialog-XeaGGDjS.js +1 -0
- app/static/assets/pagination-controls-D1Ypd0RU.js +1 -0
- app/static/assets/payload-CIYnGZz9.js +1 -0
- app/static/assets/queue-wait-chart-VoV7Yg9L.js +1 -0
- app/static/assets/report-chart-card-BXDliwcc.js +1 -0
- app/static/assets/reports-page-CGCJeMHG.js +3 -0
- app/static/assets/reset-credit-confirm-dialog-QOifTnVB.js +1 -0
- app/static/assets/rolldown-runtime-hePW80VL.js +1 -0
- app/static/assets/search-HfhqwERt.js +1 -0
- app/static/assets/select-Cl33bz-g.js +1 -0
- app/static/assets/settings-page-mZj01HYl.js +2 -0
- app/static/assets/shim-CZwyM4Q7.js +1 -0
- app/static/assets/skeleton-C_EUc4Rp.js +1 -0
- app/static/assets/sparkline-chart-DdNNsddM.js +1 -0
- app/static/assets/status-badge-CHHA5X6c.js +1 -0
- app/static/assets/switch-CzxgokcM.js +1 -0
- app/static/assets/table-DXaNSQ0G.js +1 -0
- app/static/assets/time-to-first-token-chart--68L0u_v.js +1 -0
- app/static/assets/tokens-per-day-chart-Cs82BFuE.js +1 -0
- app/static/assets/tokens-per-second-chart-AD2dfOn6.js +1 -0
- app/static/assets/totp-settings-CRHtxQ7j.js +1 -0
- app/static/assets/trash-D3KyWr1h.js +1 -0
- app/static/assets/use-account-quota-display-BPmrTBTC.js +1 -0
- app/static/assets/use-accounts-PWlVGw3z.js +1 -0
- app/static/assets/use-auth-CuDlW3iC.js +1 -0
- app/static/assets/use-chart-colors-DIu6tClI.js +1 -0
- app/static/assets/use-model-sources-Bul24JBu.js +1 -0
- app/static/assets/use-models-DUCPUuZg.js +1 -0
- app/static/assets/use-reduced-motion-hAm5r6_d.js +1 -0
- app/static/assets/useTranslation--56nHhlH.js +1 -0
- app/static/assets/useragent-distribution-donut-Dup6sf_n.js +1 -0
- app/static/assets/users-uIC3TIlE.js +1 -0
- app/static/assets/vendor-query-_bHZuxEx.js +1 -0
- app/static/assets/vendor-react-BVL0YZPW.js +9 -0
- app/static/favicon.svg +10 -0
- app/static/fonts/GeistSans-Variable.woff2 +0 -0
- app/static/fonts/jetbrains-mono-latin-ext.woff2 +0 -0
- app/static/fonts/jetbrains-mono-latin.woff2 +0 -0
- app/static/index.html +29 -0
- codexlb_up-1.25.0b10.dist-info/METADATA +124 -0
- codexlb_up-1.25.0b10.dist-info/RECORD +827 -0
- codexlb_up-1.25.0b10.dist-info/WHEEL +4 -0
- codexlb_up-1.25.0b10.dist-info/entry_points.txt +3 -0
- codexlb_up-1.25.0b10.dist-info/licenses/LICENSE +21 -0
- config/additional_quota_registry.json +11 -0
|
@@ -0,0 +1,3762 @@
|
|
|
1
|
+
# pyright: reportGeneralTypeIssues=false
|
|
2
|
+
from __future__ import annotations
|
|
3
|
+
|
|
4
|
+
import asyncio
|
|
5
|
+
import json
|
|
6
|
+
import logging
|
|
7
|
+
import sys
|
|
8
|
+
from dataclasses import replace
|
|
9
|
+
from typing import Any, AsyncGenerator, AsyncIterator, Mapping, cast
|
|
10
|
+
|
|
11
|
+
import aiohttp
|
|
12
|
+
|
|
13
|
+
from app.core.auth.refresh import RefreshError, is_transient_refresh_contention, refresh_contention_kind
|
|
14
|
+
from app.core.balancer import failover_decision
|
|
15
|
+
from app.core.balancer.logic import (
|
|
16
|
+
BURST_SAME_ACCOUNT_MAX_RETRIES,
|
|
17
|
+
BURST_SURFACE_RETRY_AFTER_SECONDS,
|
|
18
|
+
burst_same_account_backoff_seconds,
|
|
19
|
+
)
|
|
20
|
+
from app.core.balancer.types import ClassifiedFailure, UpstreamError
|
|
21
|
+
from app.core.clients.proxy import (
|
|
22
|
+
ProxyResponseError,
|
|
23
|
+
_is_native_codex_request,
|
|
24
|
+
_resolve_stream_transport,
|
|
25
|
+
is_confirmed_pre_dispatch_transport_error,
|
|
26
|
+
pop_stream_timeout_overrides,
|
|
27
|
+
)
|
|
28
|
+
from app.core.clients.thread_cache_identity import (
|
|
29
|
+
ThreadCacheIdentity,
|
|
30
|
+
cache_scope_payload_overhead_bytes,
|
|
31
|
+
)
|
|
32
|
+
from app.core.clients.thread_cache_identity import (
|
|
33
|
+
effective_thread_cache_identity_mode as _effective_thread_cache_identity_mode,
|
|
34
|
+
)
|
|
35
|
+
from app.core.clock import REAL_CLOCK, REAL_SCHEDULER, Clock, Scheduler, clock_for, scheduler_for
|
|
36
|
+
from app.core.config.dashboard_overrides import with_dashboard_overrides
|
|
37
|
+
from app.core.errors import (
|
|
38
|
+
SYNTHETIC_TRANSPORT_FAILURE_CODES,
|
|
39
|
+
openai_error,
|
|
40
|
+
synthetic_transport_failure_event,
|
|
41
|
+
)
|
|
42
|
+
from app.core.errors import (
|
|
43
|
+
synthetic_stream_failure_event as response_failed_event,
|
|
44
|
+
)
|
|
45
|
+
from app.core.openai.requests import ResponsesRequest, extract_input_file_ids
|
|
46
|
+
from app.core.resilience.network_recovery import (
|
|
47
|
+
NetworkRecoveryDecision,
|
|
48
|
+
ProcessNetworkRecovery,
|
|
49
|
+
)
|
|
50
|
+
from app.core.resilience.toggles import bind_resilience_toggles, set_resilience_toggles
|
|
51
|
+
from app.core.upstream_proxy import UpstreamProxyRouteError
|
|
52
|
+
from app.core.utils.request_id import ensure_request_id
|
|
53
|
+
from app.core.utils.retry import backoff_seconds
|
|
54
|
+
from app.core.utils.shared_future import _await_task_deferring_cancellation
|
|
55
|
+
from app.core.utils.sse import format_sse_event
|
|
56
|
+
from app.db.models import Account
|
|
57
|
+
from app.modules.api_keys.service import ApiKeyData, ApiKeyUsageReservationData
|
|
58
|
+
from app.modules.proxy._load_balancer.overload_backoff import (
|
|
59
|
+
UPSTREAM_OVERLOAD_CODES,
|
|
60
|
+
record_upstream_burst_rejection,
|
|
61
|
+
)
|
|
62
|
+
from app.modules.proxy._service.observability import (
|
|
63
|
+
_maybe_log_proxy_request_shape,
|
|
64
|
+
_record_continuity_fail_closed,
|
|
65
|
+
_record_upstream_transport_decision,
|
|
66
|
+
record_http_bridge_routing,
|
|
67
|
+
)
|
|
68
|
+
from app.modules.proxy._service.streaming.protocol import _StreamingServiceProtocol
|
|
69
|
+
from app.modules.proxy._service.support import (
|
|
70
|
+
_ACCOUNT_MODEL_UNSUPPORTED_ERROR_CODE,
|
|
71
|
+
_ACCOUNT_SELECTION_RECOVERY_HEARTBEAT_SECONDS,
|
|
72
|
+
_LOCAL_ACCOUNT_CAP_ERROR_CODES,
|
|
73
|
+
_account_capacity_wait_payload,
|
|
74
|
+
_account_selection_recovery_sleep_seconds,
|
|
75
|
+
_request_log_client_fields,
|
|
76
|
+
_RetryableStreamError,
|
|
77
|
+
_signal_propagated_capacity_startup_wait,
|
|
78
|
+
_signal_propagated_responses_service_cleanup_ready,
|
|
79
|
+
_stream_settlement_error_payload,
|
|
80
|
+
_StreamSettlement,
|
|
81
|
+
_TerminalStreamError,
|
|
82
|
+
_TransientStreamError,
|
|
83
|
+
_WebSocketUpstreamControl,
|
|
84
|
+
configured_upstream_stream_transport,
|
|
85
|
+
upstream_websocket_transport_recently_failed,
|
|
86
|
+
)
|
|
87
|
+
from app.modules.proxy._service.websocket.helpers import (
|
|
88
|
+
_websocket_input_items_are_self_contained_fresh_replay,
|
|
89
|
+
)
|
|
90
|
+
from app.modules.proxy.affinity import (
|
|
91
|
+
_is_synthesized_turn_state,
|
|
92
|
+
_owner_lookup_session_id_from_headers,
|
|
93
|
+
_prompt_cache_key_from_request_model,
|
|
94
|
+
_sticky_key_for_responses_request,
|
|
95
|
+
_sticky_key_from_turn_state_header,
|
|
96
|
+
_websocket_continuity_key_from_headers,
|
|
97
|
+
)
|
|
98
|
+
from app.modules.proxy.affinity_observation import AffinityObservation
|
|
99
|
+
from app.modules.proxy.api_key_usage import estimate_api_key_request_usage
|
|
100
|
+
from app.modules.proxy.continuity import resolve_required_account_id
|
|
101
|
+
from app.modules.proxy.helpers import (
|
|
102
|
+
_apply_error_metadata,
|
|
103
|
+
_is_account_model_unsupported_error,
|
|
104
|
+
_normalize_error_code,
|
|
105
|
+
_parse_openai_error,
|
|
106
|
+
_upstream_error_from_openai,
|
|
107
|
+
classify_upstream_failure,
|
|
108
|
+
is_upstream_burst_rejection,
|
|
109
|
+
is_upstream_model_capacity_error,
|
|
110
|
+
)
|
|
111
|
+
from app.modules.proxy.http_continuation import http_continuation_signal
|
|
112
|
+
from app.modules.proxy.load_balancer import AccountLease, AccountSelection
|
|
113
|
+
from app.modules.proxy.replay_safety import responses_payload_is_account_neutral_fresh_replay
|
|
114
|
+
from app.modules.proxy.selection_errors import USAGE_LIMIT_REACHED, selection_failure_response
|
|
115
|
+
|
|
116
|
+
_REQUEST_TRANSPORT_HTTP = "http"
|
|
117
|
+
_REQUEST_TRANSPORT_WEBSOCKET = "websocket"
|
|
118
|
+
_HTTP_DOWNSTREAM_TRANSPORT_POLICY_DEFAULT = "smart"
|
|
119
|
+
_HTTP_DOWNSTREAM_TRANSPORT_POLICIES = frozenset({"smart", "always_http", "always_websocket", "pinned"})
|
|
120
|
+
logger = logging.getLogger(__name__)
|
|
121
|
+
|
|
122
|
+
|
|
123
|
+
def _facade() -> Any:
|
|
124
|
+
return sys.modules["app.modules.proxy.service"]
|
|
125
|
+
|
|
126
|
+
|
|
127
|
+
def _http_downstream_request_is_sticky(payload: ResponsesRequest, headers: Mapping[str, str]) -> bool:
|
|
128
|
+
return http_continuation_signal(payload, headers) is not None
|
|
129
|
+
|
|
130
|
+
|
|
131
|
+
_POST_REFRESH_TRANSIENT_EXHAUSTED_ATTR = "_codex_lb_post_refresh_transient_exhausted"
|
|
132
|
+
# Set on a surfaced pre-visible ``ProxyResponseError`` whose account health was
|
|
133
|
+
# already written where the failover decision was made, so the outer terminal
|
|
134
|
+
# handler does not write it a second time. Exactly-once matters most for an
|
|
135
|
+
# exhausted owner-bound burst 429 (its same-account retries engage only the
|
|
136
|
+
# replica-local burst cooldown, never ``record_error``), but every surfaced
|
|
137
|
+
# pre-visible failure is penalized once by construction.
|
|
138
|
+
_STREAM_HEALTH_RECORDED_ATTR = "_codex_lb_stream_health_recorded"
|
|
139
|
+
|
|
140
|
+
|
|
141
|
+
def _resolve_http_downstream_transport(policy: str, *, payload: ResponsesRequest, headers: Mapping[str, str]) -> str:
|
|
142
|
+
normalized_policy = policy.strip().lower()
|
|
143
|
+
if normalized_policy not in _HTTP_DOWNSTREAM_TRANSPORT_POLICIES:
|
|
144
|
+
raise ValueError(f"Unsupported HTTP downstream transport policy: {policy}")
|
|
145
|
+
if normalized_policy in ("always_http", "pinned"):
|
|
146
|
+
return "http"
|
|
147
|
+
if normalized_policy == "always_websocket":
|
|
148
|
+
return "websocket"
|
|
149
|
+
return "websocket" if _http_downstream_request_is_sticky(payload, headers) else "http"
|
|
150
|
+
|
|
151
|
+
|
|
152
|
+
def _verified_cross_transport_fresh_replay(
|
|
153
|
+
proxy: _StreamingServiceProtocol,
|
|
154
|
+
*,
|
|
155
|
+
payload: ResponsesRequest,
|
|
156
|
+
headers: Mapping[str, str],
|
|
157
|
+
api_key: ApiKeyData | None,
|
|
158
|
+
) -> ResponsesRequest | None:
|
|
159
|
+
"""Return an unanchored body only when local WS continuity proves its prefix."""
|
|
160
|
+
previous_response_id = payload.previous_response_id
|
|
161
|
+
input_value = payload.input
|
|
162
|
+
if previous_response_id is None or not isinstance(input_value, list):
|
|
163
|
+
return None
|
|
164
|
+
if extract_input_file_ids(input_value):
|
|
165
|
+
return None
|
|
166
|
+
input_items = cast(list[Any], input_value)
|
|
167
|
+
if not _websocket_input_items_are_self_contained_fresh_replay(input_items):
|
|
168
|
+
return None
|
|
169
|
+
session_id = _websocket_continuity_key_from_headers(headers)
|
|
170
|
+
if session_id is None:
|
|
171
|
+
return None
|
|
172
|
+
api_key_id = api_key.id if api_key is not None else None
|
|
173
|
+
continuity_state = proxy._websocket_continuity_index.get((session_id, api_key_id))
|
|
174
|
+
if continuity_state is None or continuity_state.last_completed_response_id != previous_response_id:
|
|
175
|
+
# HTTP and WebSocket entry points synthesize different turn-state
|
|
176
|
+
# headers. The response id remains globally specific within the API-key
|
|
177
|
+
# scope, so use its unique retained state when the direct key differs.
|
|
178
|
+
matching_states = [
|
|
179
|
+
state
|
|
180
|
+
for (_continuity_key, continuity_api_key_id), state in proxy._websocket_continuity_index.items()
|
|
181
|
+
if continuity_api_key_id == api_key_id and state.last_completed_response_id == previous_response_id
|
|
182
|
+
]
|
|
183
|
+
if len(matching_states) != 1:
|
|
184
|
+
return None
|
|
185
|
+
continuity_state = matching_states[0]
|
|
186
|
+
if not _facade()._input_prefix_matches_stored_context(
|
|
187
|
+
input_value,
|
|
188
|
+
stored_count=continuity_state.last_completed_input_count,
|
|
189
|
+
stored_fingerprint=continuity_state.last_completed_input_prefix_fingerprint,
|
|
190
|
+
):
|
|
191
|
+
return None
|
|
192
|
+
fresh_payload = payload.model_copy(update={"previous_response_id": None})
|
|
193
|
+
if not responses_payload_is_account_neutral_fresh_replay(fresh_payload.to_replay_safety_payload()):
|
|
194
|
+
return None
|
|
195
|
+
return fresh_payload
|
|
196
|
+
|
|
197
|
+
|
|
198
|
+
def _effective_http_downstream_transport_policy(
|
|
199
|
+
api_key: ApiKeyData | None,
|
|
200
|
+
dashboard_settings: Any,
|
|
201
|
+
) -> tuple[str, bool]:
|
|
202
|
+
override = getattr(api_key, "transport_policy_override", None) if api_key is not None else None
|
|
203
|
+
if override is not None:
|
|
204
|
+
return override, True
|
|
205
|
+
dashboard_policy = getattr(dashboard_settings, "http_downstream_transport_policy", None)
|
|
206
|
+
if isinstance(dashboard_policy, str) and dashboard_policy:
|
|
207
|
+
return dashboard_policy, False
|
|
208
|
+
return _HTTP_DOWNSTREAM_TRANSPORT_POLICY_DEFAULT, False
|
|
209
|
+
|
|
210
|
+
|
|
211
|
+
def _resolved_configured_stream_transport(dashboard_settings: Any) -> tuple[str, bool]:
|
|
212
|
+
configured = configured_upstream_stream_transport(dashboard_settings)
|
|
213
|
+
return configured, configured in ("http", "websocket")
|
|
214
|
+
|
|
215
|
+
|
|
216
|
+
def _http_bridge_allowed_by_transport_policy(
|
|
217
|
+
payload: ResponsesRequest,
|
|
218
|
+
headers: Mapping[str, str],
|
|
219
|
+
*,
|
|
220
|
+
api_key: ApiKeyData | None,
|
|
221
|
+
dashboard_settings: Any,
|
|
222
|
+
base_settings: Any,
|
|
223
|
+
) -> bool:
|
|
224
|
+
"""Apply ordinary HTTP transport precedence before entering the WS bridge."""
|
|
225
|
+
|
|
226
|
+
configured_transport, explicit_transport = _resolved_configured_stream_transport(dashboard_settings)
|
|
227
|
+
if upstream_websocket_transport_recently_failed():
|
|
228
|
+
reason, allowed = "recent_ws_failure", False
|
|
229
|
+
elif explicit_transport:
|
|
230
|
+
reason, allowed = f"explicit_{configured_transport}", configured_transport == "websocket"
|
|
231
|
+
else:
|
|
232
|
+
policy, _override_applied = _effective_http_downstream_transport_policy(api_key, dashboard_settings)
|
|
233
|
+
allowed = _resolve_http_downstream_transport(policy, payload=payload, headers=headers) == "websocket"
|
|
234
|
+
signal = http_continuation_signal(payload, headers)
|
|
235
|
+
reason = f"smart_{signal or 'single_turn'}" if policy == "smart" else policy
|
|
236
|
+
record_http_bridge_routing(stage="admission", reason=reason)
|
|
237
|
+
return allowed
|
|
238
|
+
|
|
239
|
+
|
|
240
|
+
async def _iter_account_capacity_recovery_wait(
|
|
241
|
+
*,
|
|
242
|
+
request_id: str,
|
|
243
|
+
model: str | None,
|
|
244
|
+
account_id: str | None,
|
|
245
|
+
error_message: str | None,
|
|
246
|
+
recovery_sleep_seconds: float,
|
|
247
|
+
remaining_budget_seconds: float,
|
|
248
|
+
emit_keepalives: bool,
|
|
249
|
+
stage: str,
|
|
250
|
+
scheduler: Scheduler = REAL_SCHEDULER,
|
|
251
|
+
clock: Clock = REAL_CLOCK,
|
|
252
|
+
) -> AsyncIterator[str]:
|
|
253
|
+
# ``remaining_budget_seconds`` is sampled by the caller through the owner's
|
|
254
|
+
# budget seam (``proxy._remaining_budget_seconds``), so this owner-less
|
|
255
|
+
# helper never reads a clock of its own for budget math; ``clock`` only
|
|
256
|
+
# stamps the wait start and ``scheduler`` owns the heartbeat sleeps.
|
|
257
|
+
if not emit_keepalives:
|
|
258
|
+
_signal_propagated_capacity_startup_wait()
|
|
259
|
+
if remaining_budget_seconds <= 0:
|
|
260
|
+
return
|
|
261
|
+
wait_started_at = clock.monotonic()
|
|
262
|
+
remaining_sleep_seconds = min(recovery_sleep_seconds, remaining_budget_seconds)
|
|
263
|
+
_facade().logger.info(
|
|
264
|
+
"Waiting for account capacity before retrying stream request_id=%s model=%s account_id=%s "
|
|
265
|
+
"stage=%s sleep_seconds=%.1f recovery_hint_seconds=%.1f error=%s",
|
|
266
|
+
request_id,
|
|
267
|
+
model,
|
|
268
|
+
account_id,
|
|
269
|
+
stage,
|
|
270
|
+
remaining_sleep_seconds,
|
|
271
|
+
recovery_sleep_seconds,
|
|
272
|
+
error_message,
|
|
273
|
+
)
|
|
274
|
+
while remaining_sleep_seconds > 0:
|
|
275
|
+
if emit_keepalives:
|
|
276
|
+
yield format_sse_event(
|
|
277
|
+
cast(
|
|
278
|
+
Mapping[str, Any],
|
|
279
|
+
_account_capacity_wait_payload(
|
|
280
|
+
None,
|
|
281
|
+
request_id=request_id,
|
|
282
|
+
reason=error_message,
|
|
283
|
+
retry_after_seconds=remaining_sleep_seconds,
|
|
284
|
+
started_at=wait_started_at,
|
|
285
|
+
now=clock.monotonic(),
|
|
286
|
+
),
|
|
287
|
+
)
|
|
288
|
+
)
|
|
289
|
+
chunk_seconds = min(
|
|
290
|
+
remaining_sleep_seconds,
|
|
291
|
+
_ACCOUNT_SELECTION_RECOVERY_HEARTBEAT_SECONDS,
|
|
292
|
+
)
|
|
293
|
+
await scheduler.sleep(chunk_seconds)
|
|
294
|
+
remaining_sleep_seconds -= chunk_seconds
|
|
295
|
+
|
|
296
|
+
|
|
297
|
+
def _retry_after_kwargs(retry_after_seconds: float | None) -> dict[str, float]:
|
|
298
|
+
"""Forward upstream ``Retry-After`` to ``_handle_stream_error`` only when one was sent.
|
|
299
|
+
|
|
300
|
+
The kw-only ``retry_after_seconds`` is additive; omitting it when there is
|
|
301
|
+
nothing to convey keeps partial doubles with the historical positional
|
|
302
|
+
signature (``account, error, code, http_status``) working.
|
|
303
|
+
"""
|
|
304
|
+
return {} if retry_after_seconds is None else {"retry_after_seconds": retry_after_seconds}
|
|
305
|
+
|
|
306
|
+
|
|
307
|
+
def _payload_size_estimate_bytes(payload: ResponsesRequest) -> int:
|
|
308
|
+
return len(json.dumps(payload.to_payload(), ensure_ascii=True, separators=(",", ":")).encode("utf-8"))
|
|
309
|
+
|
|
310
|
+
|
|
311
|
+
def _transient_retry_error_code(tex: BaseException) -> str:
|
|
312
|
+
"""Error code the same-account transient retry loop aggregates under.
|
|
313
|
+
|
|
314
|
+
Stream-framed failures keep their own code. HTTP-status failures used to
|
|
315
|
+
collapse to ``server_error``; an upstream 5xx whose body says the account
|
|
316
|
+
is overloaded keeps that code so the health write after retry exhaustion
|
|
317
|
+
still counts as an admission rejection (the overload backoff feeds on the
|
|
318
|
+
code, and ``server_is_overloaded`` classifies as the same transient class).
|
|
319
|
+
"""
|
|
320
|
+
if isinstance(tex, _TransientStreamError):
|
|
321
|
+
return tex.code
|
|
322
|
+
payload = getattr(tex, "payload", None)
|
|
323
|
+
if isinstance(payload, Mapping):
|
|
324
|
+
parsed = _parse_openai_error(payload)
|
|
325
|
+
if parsed is not None and isinstance(parsed.code, str) and parsed.code in UPSTREAM_OVERLOAD_CODES:
|
|
326
|
+
return parsed.code
|
|
327
|
+
return "server_error"
|
|
328
|
+
|
|
329
|
+
|
|
330
|
+
class _StreamingRetryMixin:
|
|
331
|
+
async def _stream_with_retry(
|
|
332
|
+
self,
|
|
333
|
+
payload: ResponsesRequest,
|
|
334
|
+
headers: Mapping[str, str],
|
|
335
|
+
*,
|
|
336
|
+
codex_session_affinity: bool,
|
|
337
|
+
propagate_http_errors: bool,
|
|
338
|
+
openai_cache_affinity: bool,
|
|
339
|
+
api_key: ApiKeyData | None,
|
|
340
|
+
api_key_reservation: ApiKeyUsageReservationData | None,
|
|
341
|
+
suppress_text_done_events: bool,
|
|
342
|
+
request_transport: str,
|
|
343
|
+
rewritten_file_account_id: str | None = None,
|
|
344
|
+
file_account_resolution_complete: bool = False,
|
|
345
|
+
upstream_stream_transport_override: str | None = None,
|
|
346
|
+
client_ip: str | None = None,
|
|
347
|
+
enforce_openai_sdk_contract: bool = True,
|
|
348
|
+
) -> AsyncIterator[str]:
|
|
349
|
+
proxy = cast(_StreamingServiceProtocol, self)
|
|
350
|
+
scheduler = scheduler_for(proxy)
|
|
351
|
+
clock = clock_for(proxy)
|
|
352
|
+
useragent, useragent_group, conversation_id = _request_log_client_fields(headers)
|
|
353
|
+
request_id = ensure_request_id()
|
|
354
|
+
start = clock.monotonic()
|
|
355
|
+
base_settings = _facade().get_settings()
|
|
356
|
+
settings = await _facade().get_settings_cache().get()
|
|
357
|
+
# C2-3 resilience toggles: resolved from this request's snapshot and
|
|
358
|
+
# bound to the task so the upstream client gates its breaker the same
|
|
359
|
+
# way; rebound before every upstream attempt because a keepalive yield
|
|
360
|
+
# can move this generator to another task (ContextVars follow tasks).
|
|
361
|
+
resilience = bind_resilience_toggles(settings, startup_settings=base_settings)
|
|
362
|
+
concurrency_caps = _facade().effective_account_concurrency_caps(settings)
|
|
363
|
+
deadline = start + _facade()._stream_request_budget_seconds(
|
|
364
|
+
base_settings,
|
|
365
|
+
request_transport=request_transport,
|
|
366
|
+
)
|
|
367
|
+
prefer_earlier_reset = settings.prefer_earlier_reset_accounts
|
|
368
|
+
# Resolved once per request: the API-key override wins, then the fleet
|
|
369
|
+
# value. ``shared`` is a strict no-op downstream. The fleet value is read
|
|
370
|
+
# off the *overlaid* settings, not the raw dashboard row: the row's column
|
|
371
|
+
# is NULL until an operator sets it, so reading it directly would keep the
|
|
372
|
+
# proxy on ``shared`` while ``GET /api/settings`` reported the environment
|
|
373
|
+
# value as effective. ``DASHBOARD_MODE_SETTINGS`` puts the column back on
|
|
374
|
+
# top where it belongs -- dashboard over environment over default.
|
|
375
|
+
thread_cache_identity_mode, thread_cache_identity_from_key = _effective_thread_cache_identity_mode(
|
|
376
|
+
api_key, with_dashboard_overrides(base_settings)
|
|
377
|
+
)
|
|
378
|
+
upstream_transport_policy_label = "explicit" if upstream_stream_transport_override is not None else "configured"
|
|
379
|
+
upstream_transport_sticky = _http_downstream_request_is_sticky(payload, headers)
|
|
380
|
+
preserve_native_failure_lifecycle = not enforce_openai_sdk_contract and _is_native_codex_request(headers)
|
|
381
|
+
|
|
382
|
+
def _must_abort_native_transport_failure(
|
|
383
|
+
exc: ProxyResponseError,
|
|
384
|
+
*,
|
|
385
|
+
downstream_visible: bool,
|
|
386
|
+
) -> bool:
|
|
387
|
+
if not preserve_native_failure_lifecycle or downstream_visible:
|
|
388
|
+
return False
|
|
389
|
+
error = _parse_openai_error(exc.payload)
|
|
390
|
+
error_code = _normalize_error_code(
|
|
391
|
+
error.code if error else None,
|
|
392
|
+
error.type if error else None,
|
|
393
|
+
)
|
|
394
|
+
return error_code in SYNTHETIC_TRANSPORT_FAILURE_CODES
|
|
395
|
+
|
|
396
|
+
upstream_stream_transport = upstream_stream_transport_override
|
|
397
|
+
if upstream_stream_transport is None:
|
|
398
|
+
configured_transport, explicit_transport = _resolved_configured_stream_transport(settings)
|
|
399
|
+
# ``has_image_generation_tool`` means exactly that: folding
|
|
400
|
+
# ``input_image`` into it pinned every image turn to upstream HTTP
|
|
401
|
+
# from inside _resolve_stream_transport, where no log or counter
|
|
402
|
+
# records the decision (#2363). An ``input_image`` request is pinned
|
|
403
|
+
# only by the narrow predicate below.
|
|
404
|
+
image_generation_bypass = _facade()._responses_request_uses_image_generation(payload)
|
|
405
|
+
payload_size_estimate = _payload_size_estimate_bytes(payload)
|
|
406
|
+
resolved_base_transport = _resolve_stream_transport(
|
|
407
|
+
transport=configured_transport,
|
|
408
|
+
transport_override=None,
|
|
409
|
+
model=payload.model,
|
|
410
|
+
headers=headers,
|
|
411
|
+
has_image_generation_tool=image_generation_bypass,
|
|
412
|
+
# Include what isolated mode will inject: that choice is passed
|
|
413
|
+
# down as an explicit override, which short-circuits the
|
|
414
|
+
# post-injection size check, so a request just under the
|
|
415
|
+
# websocket budget has to be measured at its egress size here.
|
|
416
|
+
payload_size_estimate_bytes=(
|
|
417
|
+
payload_size_estimate + cache_scope_payload_overhead_bytes(thread_cache_identity_mode)
|
|
418
|
+
),
|
|
419
|
+
)
|
|
420
|
+
upstream_stream_transport = resolved_base_transport
|
|
421
|
+
if not explicit_transport and (
|
|
422
|
+
image_generation_bypass
|
|
423
|
+
or _facade()._input_image_request_requires_http_upstream(
|
|
424
|
+
payload, payload_size_estimate_bytes=payload_size_estimate
|
|
425
|
+
)
|
|
426
|
+
):
|
|
427
|
+
upstream_stream_transport = "http"
|
|
428
|
+
if (
|
|
429
|
+
not explicit_transport
|
|
430
|
+
and request_transport == _REQUEST_TRANSPORT_HTTP
|
|
431
|
+
and upstream_stream_transport == "websocket"
|
|
432
|
+
):
|
|
433
|
+
sticky = upstream_transport_sticky
|
|
434
|
+
if upstream_websocket_transport_recently_failed():
|
|
435
|
+
policy = "recent_ws_failure"
|
|
436
|
+
override_applied = False
|
|
437
|
+
upstream_transport_policy_label = policy
|
|
438
|
+
upstream_stream_transport = "http"
|
|
439
|
+
else:
|
|
440
|
+
policy, override_applied = _effective_http_downstream_transport_policy(api_key, settings)
|
|
441
|
+
upstream_transport_policy_label = policy
|
|
442
|
+
policy_transport = _resolve_http_downstream_transport(policy, payload=payload, headers=headers)
|
|
443
|
+
upstream_stream_transport = "http" if policy_transport == "http" else configured_transport
|
|
444
|
+
logger.info(
|
|
445
|
+
"http_downstream_transport_decision policy=%s override_applied=%s sticky=%s "
|
|
446
|
+
"upstream_stream_transport=%s request_id=%s",
|
|
447
|
+
policy,
|
|
448
|
+
override_applied,
|
|
449
|
+
sticky,
|
|
450
|
+
upstream_stream_transport,
|
|
451
|
+
request_id,
|
|
452
|
+
)
|
|
453
|
+
elif request_transport == _REQUEST_TRANSPORT_HTTP:
|
|
454
|
+
logger.info(
|
|
455
|
+
"http_downstream_transport_decision policy=explicit override_applied=%s sticky=%s "
|
|
456
|
+
"upstream_stream_transport=%s request_id=%s",
|
|
457
|
+
False,
|
|
458
|
+
_http_downstream_request_is_sticky(payload, headers),
|
|
459
|
+
upstream_stream_transport,
|
|
460
|
+
request_id,
|
|
461
|
+
)
|
|
462
|
+
if request_transport == _REQUEST_TRANSPORT_HTTP and upstream_websocket_transport_recently_failed():
|
|
463
|
+
upstream_stream_transport = "http"
|
|
464
|
+
upstream_transport_policy_label = "recent_ws_failure"
|
|
465
|
+
if rewritten_file_account_id is None and not file_account_resolution_complete:
|
|
466
|
+
proxy._raise_for_unsupported_input_image_references(payload)
|
|
467
|
+
rewritten_file_account_id = await proxy._resolve_file_account_for_responses(payload, headers)
|
|
468
|
+
affinity = _sticky_key_for_responses_request(
|
|
469
|
+
payload,
|
|
470
|
+
headers,
|
|
471
|
+
codex_session_affinity=codex_session_affinity,
|
|
472
|
+
openai_cache_affinity=openai_cache_affinity,
|
|
473
|
+
openai_cache_affinity_max_age_seconds=settings.openai_cache_affinity_max_age_seconds,
|
|
474
|
+
sticky_threads_enabled=settings.sticky_threads_enabled,
|
|
475
|
+
api_key=api_key,
|
|
476
|
+
)
|
|
477
|
+
turn_state_owner_account_id: str | None = None
|
|
478
|
+
turn_state = _sticky_key_from_turn_state_header(headers)
|
|
479
|
+
if turn_state is not None:
|
|
480
|
+
# HTTP and WebSocket transports share the bridge turn-state index;
|
|
481
|
+
# treating this as ordinary sticky input would cross replicas or
|
|
482
|
+
# accounts when the token was minted by an HTTP bridge session.
|
|
483
|
+
turn_state_owner_account_id = await proxy._resolve_compact_turn_state_owner(
|
|
484
|
+
turn_state=turn_state,
|
|
485
|
+
api_key=api_key,
|
|
486
|
+
fail_on_missing=not _is_synthesized_turn_state(turn_state),
|
|
487
|
+
)
|
|
488
|
+
affinity_observation = AffinityObservation.from_policy(affinity)
|
|
489
|
+
_maybe_log_proxy_request_shape(
|
|
490
|
+
"stream",
|
|
491
|
+
payload,
|
|
492
|
+
headers,
|
|
493
|
+
sticky_kind=affinity_observation.kind,
|
|
494
|
+
sticky_key_source=affinity_observation.source,
|
|
495
|
+
derivation_outcome=affinity.prompt_cache_derivation_outcome,
|
|
496
|
+
prompt_cache_key_set=_prompt_cache_key_from_request_model(payload) is not None,
|
|
497
|
+
thread_cache_identity_mode=thread_cache_identity_mode,
|
|
498
|
+
thread_cache_identity_from_key=thread_cache_identity_from_key,
|
|
499
|
+
)
|
|
500
|
+
routing_strategy = _facade()._routing_strategy(settings)
|
|
501
|
+
max_attempts = _facade()._STREAM_MAX_ACCOUNT_ATTEMPTS
|
|
502
|
+
settled = False
|
|
503
|
+
any_attempt_logged = False
|
|
504
|
+
upstream_transport_metric_status: str | None = None
|
|
505
|
+
upstream_transport_metric_recorded = False
|
|
506
|
+
network_recovery = ProcessNetworkRecovery(transport="stream", request_id=request_id)
|
|
507
|
+
settlement = _StreamSettlement()
|
|
508
|
+
last_transient_exc: ProxyResponseError | None = None
|
|
509
|
+
last_pre_dispatch_transport_error: ProxyResponseError | None = None
|
|
510
|
+
last_account_model_rejection: ProxyResponseError | None = None
|
|
511
|
+
last_account_model_rejection_account_id: str | None = None
|
|
512
|
+
account_model_replacement_account_id: str | None = None
|
|
513
|
+
account_model_replay_attempted = False
|
|
514
|
+
current_account_lease: AccountLease | None = None
|
|
515
|
+
last_security_work_retry_error: _RetryableStreamError | None = None
|
|
516
|
+
excluded_account_ids: set[str] = set()
|
|
517
|
+
transient_failed_account_id: str | None = None
|
|
518
|
+
hard_affinity_same_owner_retry_attempted = False
|
|
519
|
+
# Owner-bound burst 429 same-account retries; reset when the counter's
|
|
520
|
+
# account differs from the one being retried (see the closures below).
|
|
521
|
+
burst_same_account_retries = 0
|
|
522
|
+
burst_same_account_retry_account_id: str | None = None
|
|
523
|
+
deferred_capacity_account: Account | None = None
|
|
524
|
+
deferred_capacity_lease: AccountLease | None = None
|
|
525
|
+
preferred_account_id: str | None = None
|
|
526
|
+
payload_replay_required_account_id: str | None = None
|
|
527
|
+
file_preferred_account_id: str | None = rewritten_file_account_id
|
|
528
|
+
require_preferred_account = False
|
|
529
|
+
last_retryable_stream_error: _RetryableStreamError | None = None
|
|
530
|
+
# (account, error, code, http_status, retry_count, retry_after_seconds,
|
|
531
|
+
# burst_cooldown_recorded): a queued keyed health write. The burst
|
|
532
|
+
# cooldown is replica-local runtime state with no reservation-ordering
|
|
533
|
+
# dependency, so it is engaged at rejection time and the flag stops the
|
|
534
|
+
# deferred write from re-engaging it after the stream settled.
|
|
535
|
+
pending_post_refresh_transient_penalties: list[
|
|
536
|
+
tuple[Account, UpstreamError, str, int | None, int, float | None, bool]
|
|
537
|
+
] = []
|
|
538
|
+
deferred_account_error_backoffs: dict[str, Account] = {}
|
|
539
|
+
post_refresh_transient_replacement_selected = False
|
|
540
|
+
require_security_work_authorized = False
|
|
541
|
+
account_leases: list[AccountLease] = []
|
|
542
|
+
estimated_lease_tokens = _facade()._estimated_lease_tokens_from_request_usage_budget(
|
|
543
|
+
estimate_api_key_request_usage(payload)
|
|
544
|
+
)
|
|
545
|
+
verified_fresh_replay_payload = _verified_cross_transport_fresh_replay(
|
|
546
|
+
proxy,
|
|
547
|
+
payload=payload,
|
|
548
|
+
headers=headers,
|
|
549
|
+
api_key=api_key,
|
|
550
|
+
)
|
|
551
|
+
|
|
552
|
+
async def _release_tracked_stream_lease(lease: AccountLease | None) -> None:
|
|
553
|
+
if lease is None:
|
|
554
|
+
return
|
|
555
|
+
try:
|
|
556
|
+
account_leases.remove(lease)
|
|
557
|
+
except ValueError:
|
|
558
|
+
pass
|
|
559
|
+
await proxy._load_balancer.release_account_lease(lease)
|
|
560
|
+
|
|
561
|
+
def _render_dispatch_transport_error(exc: ProxyResponseError) -> str:
|
|
562
|
+
# Terminal render of the preserved sanitized transport failure:
|
|
563
|
+
# the client sees the original upstream-unavailable error instead
|
|
564
|
+
# of a misleading generated ``no_accounts`` response.
|
|
565
|
+
error = _parse_openai_error(exc.payload)
|
|
566
|
+
error_code = (
|
|
567
|
+
_normalize_error_code(
|
|
568
|
+
error.code if error else None,
|
|
569
|
+
error.type if error else None,
|
|
570
|
+
)
|
|
571
|
+
or "upstream_unavailable"
|
|
572
|
+
)
|
|
573
|
+
error_message = error.message if error and error.message else "Upstream transport failed"
|
|
574
|
+
event = response_failed_event(
|
|
575
|
+
error_code,
|
|
576
|
+
error_message,
|
|
577
|
+
error_type=(error.type if error else None) or "server_error",
|
|
578
|
+
response_id=request_id,
|
|
579
|
+
error_param=error.param_state if error else None,
|
|
580
|
+
)
|
|
581
|
+
_apply_error_metadata(event["response"]["error"], error)
|
|
582
|
+
return format_sse_event(event)
|
|
583
|
+
|
|
584
|
+
async def _flush_pending_post_refresh_penalties() -> None:
|
|
585
|
+
# Consume one queued penalty at a time only after that entry's
|
|
586
|
+
# health write finishes (success or logged failure). Clearing the
|
|
587
|
+
# whole queue first would drop later accounts if one write raises
|
|
588
|
+
# or cancellation lands on an await (compact flush isolates entries
|
|
589
|
+
# the same way). Each entry runs in an owned cancellation-deferring
|
|
590
|
+
# task so CancelledError mid-write cannot leave a half-applied
|
|
591
|
+
# tuple for cleanup to replay (double error_count).
|
|
592
|
+
while pending_post_refresh_transient_penalties:
|
|
593
|
+
(
|
|
594
|
+
failed_account,
|
|
595
|
+
transient_error_payload,
|
|
596
|
+
transient_error_code,
|
|
597
|
+
transient_http_status,
|
|
598
|
+
transient_retry_count,
|
|
599
|
+
transient_retry_after_seconds,
|
|
600
|
+
transient_burst_cooldown_recorded,
|
|
601
|
+
) = pending_post_refresh_transient_penalties[0]
|
|
602
|
+
|
|
603
|
+
async def _apply_deferred_penalty(
|
|
604
|
+
account: Account = failed_account,
|
|
605
|
+
error_payload: UpstreamError = transient_error_payload,
|
|
606
|
+
error_code: str = transient_error_code,
|
|
607
|
+
http_status: int | None = transient_http_status,
|
|
608
|
+
retry_count: int = transient_retry_count,
|
|
609
|
+
retry_after_seconds: float | None = transient_retry_after_seconds,
|
|
610
|
+
burst_cooldown_recorded: bool = transient_burst_cooldown_recorded,
|
|
611
|
+
) -> None:
|
|
612
|
+
try:
|
|
613
|
+
await proxy._handle_stream_error(
|
|
614
|
+
account,
|
|
615
|
+
error_payload,
|
|
616
|
+
error_code,
|
|
617
|
+
http_status=http_status,
|
|
618
|
+
**_retry_after_kwargs(retry_after_seconds),
|
|
619
|
+
**({"burst_cooldown_recorded": True} if burst_cooldown_recorded else {}),
|
|
620
|
+
)
|
|
621
|
+
if retry_count > 1:
|
|
622
|
+
await proxy._load_balancer.record_errors(account, retry_count - 1)
|
|
623
|
+
except Exception:
|
|
624
|
+
logger.warning(
|
|
625
|
+
"Failed to flush deferred keyed stream health account_id=%s request_id=%s",
|
|
626
|
+
account.id,
|
|
627
|
+
request_id,
|
|
628
|
+
exc_info=True,
|
|
629
|
+
)
|
|
630
|
+
|
|
631
|
+
apply_task = scheduler.create_task(
|
|
632
|
+
_apply_deferred_penalty(),
|
|
633
|
+
name=f"flush-deferred-keyed-stream-health-{failed_account.id}-{request_id}",
|
|
634
|
+
)
|
|
635
|
+
_, cancellation = await _await_task_deferring_cancellation(apply_task)
|
|
636
|
+
del pending_post_refresh_transient_penalties[0]
|
|
637
|
+
if cancellation is not None:
|
|
638
|
+
raise cancellation
|
|
639
|
+
|
|
640
|
+
async def _flush_or_schedule_pending_post_refresh_penalties() -> None:
|
|
641
|
+
if not pending_post_refresh_transient_penalties:
|
|
642
|
+
return
|
|
643
|
+
flush_coro = _flush_pending_post_refresh_penalties()
|
|
644
|
+
current_task = asyncio.current_task()
|
|
645
|
+
if current_task is not None and current_task.cancelling():
|
|
646
|
+
proxy._schedule_cancel_safe_cleanup(
|
|
647
|
+
flush_coro,
|
|
648
|
+
action="flush_deferred_keyed_stream_health",
|
|
649
|
+
request_id=request_id,
|
|
650
|
+
)
|
|
651
|
+
return
|
|
652
|
+
await flush_coro
|
|
653
|
+
|
|
654
|
+
async def _settle_stream_usage_before_pending_penalty(
|
|
655
|
+
current_settlement: _StreamSettlement,
|
|
656
|
+
*,
|
|
657
|
+
settlement_order_required: bool = False,
|
|
658
|
+
) -> bool:
|
|
659
|
+
nonlocal settled
|
|
660
|
+
apply_pending_penalty = post_refresh_transient_replacement_selected and bool(
|
|
661
|
+
pending_post_refresh_transient_penalties
|
|
662
|
+
)
|
|
663
|
+
wait_for_health_write = (
|
|
664
|
+
settlement_order_required
|
|
665
|
+
or current_settlement.settlement_order_required
|
|
666
|
+
or apply_pending_penalty
|
|
667
|
+
or bool(deferred_account_error_backoffs)
|
|
668
|
+
)
|
|
669
|
+
settle_kwargs = {"wait_for_settlement": True} if wait_for_health_write else {}
|
|
670
|
+
settled_result = await proxy._settle_stream_api_key_usage(
|
|
671
|
+
api_key,
|
|
672
|
+
api_key_reservation,
|
|
673
|
+
current_settlement,
|
|
674
|
+
request_id,
|
|
675
|
+
**settle_kwargs,
|
|
676
|
+
)
|
|
677
|
+
if not settled_result:
|
|
678
|
+
return False
|
|
679
|
+
# Commit settlement visibility before any cancellable health flush.
|
|
680
|
+
# CancelledError during flush must not leave ``settled`` false while
|
|
681
|
+
# ``usage_settlement_transferred`` is already true — that combination
|
|
682
|
+
# skips both reservation cleanup and retained-queue flush.
|
|
683
|
+
settled = True
|
|
684
|
+
try:
|
|
685
|
+
await proxy._drain_deferred_account_error_backoffs(deferred_account_error_backoffs)
|
|
686
|
+
finally:
|
|
687
|
+
# Route backoffs and queued stream penalties have independent
|
|
688
|
+
# ownership after settlement. A failed backoff write must not
|
|
689
|
+
# orphan the penalty queue in detached cancellation cleanup.
|
|
690
|
+
if apply_pending_penalty:
|
|
691
|
+
await _flush_pending_post_refresh_penalties()
|
|
692
|
+
return settled_result
|
|
693
|
+
|
|
694
|
+
async def _record_or_defer_confirmed_route_backoff(account: Account) -> None:
|
|
695
|
+
if api_key is not None and api_key_reservation is not None:
|
|
696
|
+
deferred_account_error_backoffs.setdefault(account.id, account)
|
|
697
|
+
return
|
|
698
|
+
await proxy._load_balancer.record_error_backoff(account)
|
|
699
|
+
|
|
700
|
+
async def _handle_or_defer_keyed_stream_health(
|
|
701
|
+
failed_account: Account,
|
|
702
|
+
failed_error: UpstreamError,
|
|
703
|
+
failed_code: str,
|
|
704
|
+
*,
|
|
705
|
+
http_status: int | None = None,
|
|
706
|
+
transient_retry_count: int = 1,
|
|
707
|
+
retry_after_seconds: float | None = None,
|
|
708
|
+
) -> ClassifiedFailure:
|
|
709
|
+
"""Classify and either write health now or defer until after settle.
|
|
710
|
+
|
|
711
|
+
Keyed streams keep a shared ``api_key_reservation`` open across
|
|
712
|
+
mid-loop failover ``continue``/``break`` paths. Immediate
|
|
713
|
+
``_handle_stream_error`` / ``record_errors`` on those paths would
|
|
714
|
+
run while the reservation is still open; queue into
|
|
715
|
+
``pending_post_refresh_transient_penalties`` instead so
|
|
716
|
+
``_settle_stream_usage_before_pending_penalty`` flushes health
|
|
717
|
+
after settlement (same ordering as compact keyed mid-loop health).
|
|
718
|
+
"""
|
|
719
|
+
if api_key is not None and api_key_reservation is not None:
|
|
720
|
+
classified = classify_upstream_failure(
|
|
721
|
+
error_code=failed_code,
|
|
722
|
+
error=failed_error,
|
|
723
|
+
http_status=http_status,
|
|
724
|
+
phase="first_event",
|
|
725
|
+
)
|
|
726
|
+
burst_cooldown_recorded = is_upstream_burst_rejection(
|
|
727
|
+
failure_class=classified["failure_class"],
|
|
728
|
+
http_status=http_status,
|
|
729
|
+
)
|
|
730
|
+
if burst_cooldown_recorded:
|
|
731
|
+
# Replica-local runtime state only (no DB write, no
|
|
732
|
+
# reservation ordering): engage it now so other requests
|
|
733
|
+
# are steered away during the burst, not after settlement.
|
|
734
|
+
await record_upstream_burst_rejection(
|
|
735
|
+
proxy._load_balancer,
|
|
736
|
+
failed_account,
|
|
737
|
+
retry_after_seconds=retry_after_seconds,
|
|
738
|
+
)
|
|
739
|
+
pending_post_refresh_transient_penalties.append(
|
|
740
|
+
(
|
|
741
|
+
failed_account,
|
|
742
|
+
failed_error,
|
|
743
|
+
failed_code,
|
|
744
|
+
http_status,
|
|
745
|
+
transient_retry_count,
|
|
746
|
+
retry_after_seconds,
|
|
747
|
+
burst_cooldown_recorded,
|
|
748
|
+
)
|
|
749
|
+
)
|
|
750
|
+
return classified
|
|
751
|
+
classified = await proxy._handle_stream_error(
|
|
752
|
+
failed_account,
|
|
753
|
+
failed_error,
|
|
754
|
+
failed_code,
|
|
755
|
+
http_status=http_status,
|
|
756
|
+
**_retry_after_kwargs(retry_after_seconds),
|
|
757
|
+
)
|
|
758
|
+
if transient_retry_count > 1:
|
|
759
|
+
await proxy._load_balancer.record_errors(failed_account, transient_retry_count - 1)
|
|
760
|
+
return classified
|
|
761
|
+
|
|
762
|
+
def _stream_owner_bound_to(account: Account) -> bool:
|
|
763
|
+
"""True when this request cannot move off ``account``.
|
|
764
|
+
|
|
765
|
+
Mirrors the loop's ``can_try_other_account`` predicates: single-account
|
|
766
|
+
routing, a payload already dispatched to the account (account-bound
|
|
767
|
+
items such as ``reasoning``), a required previous-response /
|
|
768
|
+
turn-state owner, or an input-file owner. A required owner whose
|
|
769
|
+
locally verified full-input replay may still move
|
|
770
|
+
(``_move_verified_fresh_replay_from_owner``) is *not* bound.
|
|
771
|
+
"""
|
|
772
|
+
if routing_strategy == "single_account":
|
|
773
|
+
return True
|
|
774
|
+
if (
|
|
775
|
+
require_preferred_account
|
|
776
|
+
and preferred_account_id == account.id
|
|
777
|
+
and verified_fresh_replay_payload is not None
|
|
778
|
+
):
|
|
779
|
+
return False
|
|
780
|
+
return (
|
|
781
|
+
payload_replay_required_account_id == account.id
|
|
782
|
+
or (require_preferred_account and preferred_account_id == account.id)
|
|
783
|
+
or account.id == file_preferred_account_id
|
|
784
|
+
or turn_state_owner_account_id is not None
|
|
785
|
+
)
|
|
786
|
+
|
|
787
|
+
def _burst_same_account_retry_available(account: Account, *, burst: bool) -> bool:
|
|
788
|
+
if not burst:
|
|
789
|
+
return False
|
|
790
|
+
retries = burst_same_account_retries if burst_same_account_retry_account_id == account.id else 0
|
|
791
|
+
return retries < BURST_SAME_ACCOUNT_MAX_RETRIES and proxy._remaining_budget_seconds(deadline) > 0
|
|
792
|
+
|
|
793
|
+
def _next_burst_same_account_retry(account: Account) -> int:
|
|
794
|
+
nonlocal burst_same_account_retries, burst_same_account_retry_account_id
|
|
795
|
+
if burst_same_account_retry_account_id != account.id:
|
|
796
|
+
burst_same_account_retry_account_id = account.id
|
|
797
|
+
burst_same_account_retries = 0
|
|
798
|
+
burst_same_account_retries += 1
|
|
799
|
+
return burst_same_account_retries
|
|
800
|
+
|
|
801
|
+
def _stamp_surfaced_burst_retry_after(exc: ProxyResponseError) -> None:
|
|
802
|
+
# A surfaced burst 429 that carried no upstream Retry-After still
|
|
803
|
+
# tells the client when to come back (api.py emits the header).
|
|
804
|
+
if exc.retry_after_seconds is None:
|
|
805
|
+
exc.retry_after_seconds = BURST_SURFACE_RETRY_AFTER_SECONDS
|
|
806
|
+
|
|
807
|
+
async def _drain_pending_post_refresh_penalty_on_terminal(
|
|
808
|
+
current_settlement: _StreamSettlement,
|
|
809
|
+
) -> bool:
|
|
810
|
+
nonlocal post_refresh_transient_replacement_selected, settled
|
|
811
|
+
if pending_post_refresh_transient_penalties or deferred_account_error_backoffs:
|
|
812
|
+
# A failed replacement selection still ends the request. Mark
|
|
813
|
+
# it as terminal so the deferred failure is settled and
|
|
814
|
+
# recorded before this path returns or re-raises.
|
|
815
|
+
if pending_post_refresh_transient_penalties:
|
|
816
|
+
post_refresh_transient_replacement_selected = True
|
|
817
|
+
settled = await _settle_stream_usage_before_pending_penalty(current_settlement)
|
|
818
|
+
return settled
|
|
819
|
+
return True
|
|
820
|
+
|
|
821
|
+
async def _finalize_terminal_settlement_after_downstream_close(
|
|
822
|
+
current_settlement: _StreamSettlement,
|
|
823
|
+
account: Account,
|
|
824
|
+
*,
|
|
825
|
+
settlement_order_required: bool = False,
|
|
826
|
+
upstream_http_status: int | None = None,
|
|
827
|
+
) -> None:
|
|
828
|
+
"""Settle usage then write account health after the client went away.
|
|
829
|
+
|
|
830
|
+
``_StreamSettlement`` has no HTTP-status field, so a caller that
|
|
831
|
+
reached here from an HTTP-coded failure must pass
|
|
832
|
+
``upstream_http_status`` for the soft-overload window to see that
|
|
833
|
+
the settlement's ``error_code`` was HTTP-derived rather than a
|
|
834
|
+
status-less stream terminal.
|
|
835
|
+
"""
|
|
836
|
+
nonlocal settled
|
|
837
|
+
|
|
838
|
+
async def _finalize() -> None:
|
|
839
|
+
nonlocal settled
|
|
840
|
+
if not settled:
|
|
841
|
+
settled = await _settle_stream_usage_before_pending_penalty(
|
|
842
|
+
current_settlement,
|
|
843
|
+
settlement_order_required=settlement_order_required,
|
|
844
|
+
)
|
|
845
|
+
if not settled:
|
|
846
|
+
return
|
|
847
|
+
if current_settlement.account_health_error:
|
|
848
|
+
await proxy._handle_stream_error(
|
|
849
|
+
account,
|
|
850
|
+
_stream_settlement_error_payload(current_settlement),
|
|
851
|
+
current_settlement.error_code or "upstream_error",
|
|
852
|
+
upstream_http_status=upstream_http_status,
|
|
853
|
+
)
|
|
854
|
+
elif current_settlement.record_success:
|
|
855
|
+
await proxy._load_balancer.record_success(account)
|
|
856
|
+
|
|
857
|
+
finalize_task = scheduler.create_task(_finalize(), name=f"stream-terminal-settlement-{request_id}")
|
|
858
|
+
_, cancellation = await _await_task_deferring_cancellation(finalize_task)
|
|
859
|
+
if cancellation is not None:
|
|
860
|
+
raise cancellation
|
|
861
|
+
|
|
862
|
+
async def _wait_for_process_network_recovery(
|
|
863
|
+
account: Account,
|
|
864
|
+
*,
|
|
865
|
+
error_code: str | None,
|
|
866
|
+
retryable_same_contract: bool,
|
|
867
|
+
failed_session: aiohttp.ClientSession | None = None,
|
|
868
|
+
) -> NetworkRecoveryDecision:
|
|
869
|
+
network_recovery.account_id = account.id
|
|
870
|
+
return await network_recovery.wait(
|
|
871
|
+
error_code=error_code,
|
|
872
|
+
retryable_same_contract=retryable_same_contract,
|
|
873
|
+
deadline=deadline,
|
|
874
|
+
rotate_shared_client=True,
|
|
875
|
+
failed_session=failed_session,
|
|
876
|
+
)
|
|
877
|
+
|
|
878
|
+
async def _settle_process_network_budget_exhaustion(
|
|
879
|
+
account: Account,
|
|
880
|
+
settlement: _StreamSettlement,
|
|
881
|
+
) -> None:
|
|
882
|
+
nonlocal settled
|
|
883
|
+
settlement.status = "error"
|
|
884
|
+
settlement.record_success = False
|
|
885
|
+
settlement.account_health_error = False
|
|
886
|
+
settlement.error_code = "upstream_request_timeout"
|
|
887
|
+
settlement.error_message = "Proxy request budget exhausted"
|
|
888
|
+
settlement.error = {"message": "Proxy request budget exhausted"}
|
|
889
|
+
await proxy._write_stream_preflight_error(
|
|
890
|
+
affinity_observation=affinity_observation,
|
|
891
|
+
account_id=account.id,
|
|
892
|
+
api_key=api_key,
|
|
893
|
+
request_id=request_id,
|
|
894
|
+
model=payload.model,
|
|
895
|
+
start=start,
|
|
896
|
+
error_code="upstream_request_timeout",
|
|
897
|
+
error_message="Proxy request budget exhausted",
|
|
898
|
+
reasoning_effort=payload.reasoning.effort if payload.reasoning else None,
|
|
899
|
+
service_tier=payload.service_tier,
|
|
900
|
+
transport=request_transport,
|
|
901
|
+
upstream_transport=upstream_stream_transport,
|
|
902
|
+
useragent=useragent,
|
|
903
|
+
useragent_group=useragent_group,
|
|
904
|
+
conversation_id=conversation_id,
|
|
905
|
+
client_ip=client_ip,
|
|
906
|
+
)
|
|
907
|
+
settled = await _settle_stream_usage_before_pending_penalty(settlement)
|
|
908
|
+
|
|
909
|
+
def _authorize_payload_dispatch(account: Account) -> bool:
|
|
910
|
+
required_account_id = payload_replay_required_account_id
|
|
911
|
+
if required_account_id is not None and required_account_id != account.id:
|
|
912
|
+
raise ProxyResponseError(
|
|
913
|
+
502,
|
|
914
|
+
openai_error(
|
|
915
|
+
"previous_response_owner_unavailable",
|
|
916
|
+
"Request payload owner account is unavailable; retry later.",
|
|
917
|
+
error_type="server_error",
|
|
918
|
+
),
|
|
919
|
+
)
|
|
920
|
+
return required_account_id is None and not responses_payload_is_account_neutral_fresh_replay(
|
|
921
|
+
payload.to_replay_safety_payload()
|
|
922
|
+
)
|
|
923
|
+
|
|
924
|
+
def _move_verified_fresh_replay_from_owner(*, account_id: str, outcome: str) -> bool:
|
|
925
|
+
# Only a proxy-injected owner anchor with locally verified full
|
|
926
|
+
# input may move; the failed owner stays excluded so sticky
|
|
927
|
+
# selection cannot immediately loop back to it.
|
|
928
|
+
nonlocal affinity, payload, payload_replay_required_account_id
|
|
929
|
+
nonlocal preferred_account_id, require_preferred_account, verified_fresh_replay_payload
|
|
930
|
+
if not (
|
|
931
|
+
require_preferred_account
|
|
932
|
+
and preferred_account_id == account_id
|
|
933
|
+
and verified_fresh_replay_payload is not None
|
|
934
|
+
):
|
|
935
|
+
return False
|
|
936
|
+
if not responses_payload_is_account_neutral_fresh_replay(
|
|
937
|
+
verified_fresh_replay_payload.to_replay_safety_payload()
|
|
938
|
+
):
|
|
939
|
+
return False
|
|
940
|
+
payload = verified_fresh_replay_payload
|
|
941
|
+
payload_replay_required_account_id = None
|
|
942
|
+
verified_fresh_replay_payload = None
|
|
943
|
+
excluded_account_ids.add(account_id)
|
|
944
|
+
preferred_account_id = None
|
|
945
|
+
require_preferred_account = False
|
|
946
|
+
affinity = replace(affinity, reallocate_sticky=True)
|
|
947
|
+
logger.info(
|
|
948
|
+
"cross_transport_verified_fresh_replay request_id=%s outcome=%s account_id=%s",
|
|
949
|
+
request_id,
|
|
950
|
+
outcome,
|
|
951
|
+
account_id,
|
|
952
|
+
)
|
|
953
|
+
return True
|
|
954
|
+
|
|
955
|
+
async def _stream_post_refresh_with_capacity_recovery(
|
|
956
|
+
account: Account,
|
|
957
|
+
*,
|
|
958
|
+
settlement: _StreamSettlement,
|
|
959
|
+
can_try_other_account: bool,
|
|
960
|
+
tool_call_dedupe: _WebSocketUpstreamControl,
|
|
961
|
+
) -> AsyncGenerator[str, None]:
|
|
962
|
+
nonlocal last_transient_exc
|
|
963
|
+
transient_retries = 0
|
|
964
|
+
|
|
965
|
+
async def _iter_stream_once() -> AsyncGenerator[str, None]:
|
|
966
|
+
inner_stream = proxy._stream_once(
|
|
967
|
+
account,
|
|
968
|
+
payload,
|
|
969
|
+
headers,
|
|
970
|
+
request_id,
|
|
971
|
+
False,
|
|
972
|
+
request_started_at=start,
|
|
973
|
+
affinity_observation=affinity_observation,
|
|
974
|
+
allow_transient_retry=True,
|
|
975
|
+
api_key=api_key,
|
|
976
|
+
api_key_reservation=api_key_reservation,
|
|
977
|
+
settlement=settlement,
|
|
978
|
+
suppress_text_done_events=suppress_text_done_events,
|
|
979
|
+
upstream_stream_transport=upstream_stream_transport,
|
|
980
|
+
request_transport=request_transport,
|
|
981
|
+
concurrency_caps=concurrency_caps,
|
|
982
|
+
useragent=useragent,
|
|
983
|
+
useragent_group=useragent_group,
|
|
984
|
+
conversation_id=conversation_id,
|
|
985
|
+
client_ip=client_ip,
|
|
986
|
+
tool_call_dedupe=tool_call_dedupe,
|
|
987
|
+
enforce_openai_sdk_contract=enforce_openai_sdk_contract,
|
|
988
|
+
thread_cache_identity=ThreadCacheIdentity(thread_cache_identity_mode, account.id),
|
|
989
|
+
)
|
|
990
|
+
try:
|
|
991
|
+
try:
|
|
992
|
+
async for line in inner_stream:
|
|
993
|
+
yield line
|
|
994
|
+
finally:
|
|
995
|
+
close_task = scheduler.create_task(
|
|
996
|
+
inner_stream.aclose(),
|
|
997
|
+
name=f"stream-post-refresh-inner-close-{request_id}",
|
|
998
|
+
)
|
|
999
|
+
_, close_cancellation = await _await_task_deferring_cancellation(close_task)
|
|
1000
|
+
if close_cancellation is not None:
|
|
1001
|
+
raise close_cancellation
|
|
1002
|
+
except ProxyResponseError as exc:
|
|
1003
|
+
if is_confirmed_pre_dispatch_transport_error(exc):
|
|
1004
|
+
# Keep dispatch provenance intact for the outer account
|
|
1005
|
+
# failover handler. Converting this into the generic
|
|
1006
|
+
# transient wrapper would authorize same-account replay
|
|
1007
|
+
# and lose the confirmed dead-route backoff semantics.
|
|
1008
|
+
raise
|
|
1009
|
+
error = _parse_openai_error(exc.payload)
|
|
1010
|
+
error_code = _normalize_error_code(
|
|
1011
|
+
error.code if error else None,
|
|
1012
|
+
error.type if error else None,
|
|
1013
|
+
)
|
|
1014
|
+
error_message = error.message if error else None
|
|
1015
|
+
retryable_connect_failure = bool(
|
|
1016
|
+
exc.failure_phase == "connect"
|
|
1017
|
+
and _facade()._should_retry_transient_stream_error(error_code, error_message)
|
|
1018
|
+
)
|
|
1019
|
+
retryable_capacity_rejection = bool(
|
|
1020
|
+
is_upstream_model_capacity_error(error_message)
|
|
1021
|
+
and _facade()._should_retry_transient_stream_error(error_code, error_message)
|
|
1022
|
+
)
|
|
1023
|
+
if retryable_connect_failure or retryable_capacity_rejection:
|
|
1024
|
+
raise _TransientStreamError(
|
|
1025
|
+
error_code or "upstream_error",
|
|
1026
|
+
_upstream_error_from_openai(error),
|
|
1027
|
+
account_health_error=retryable_connect_failure,
|
|
1028
|
+
) from exc
|
|
1029
|
+
raise
|
|
1030
|
+
|
|
1031
|
+
while True:
|
|
1032
|
+
settlement.reset()
|
|
1033
|
+
# Rebind per attempt: a capacity keepalive may have handed this
|
|
1034
|
+
# generator to another task since the request-entry binding.
|
|
1035
|
+
set_resilience_toggles(resilience)
|
|
1036
|
+
stream_timeout_tokens = _facade()._push_stream_attempt_timeout_overrides(
|
|
1037
|
+
proxy._remaining_budget_seconds(deadline)
|
|
1038
|
+
)
|
|
1039
|
+
try:
|
|
1040
|
+
attempt_stream = _iter_stream_once()
|
|
1041
|
+
try:
|
|
1042
|
+
try:
|
|
1043
|
+
async for line in attempt_stream:
|
|
1044
|
+
yield line
|
|
1045
|
+
finally:
|
|
1046
|
+
close_task = scheduler.create_task(
|
|
1047
|
+
attempt_stream.aclose(),
|
|
1048
|
+
name=f"stream-post-refresh-close-{request_id}",
|
|
1049
|
+
)
|
|
1050
|
+
_, close_cancellation = await _await_task_deferring_cancellation(close_task)
|
|
1051
|
+
if close_cancellation is not None:
|
|
1052
|
+
raise close_cancellation
|
|
1053
|
+
except (asyncio.CancelledError, GeneratorExit):
|
|
1054
|
+
# A terminal frame may already have been yielded when
|
|
1055
|
+
# downstream cancellation is delivered on the next
|
|
1056
|
+
# generator resume. Finalize that terminal usage and
|
|
1057
|
+
# health result before propagating cancellation so the
|
|
1058
|
+
# reservation is not released as abandoned.
|
|
1059
|
+
if settlement.status in {"success", "error"} and not settled:
|
|
1060
|
+
await _finalize_terminal_settlement_after_downstream_close(
|
|
1061
|
+
settlement,
|
|
1062
|
+
account,
|
|
1063
|
+
settlement_order_required=True,
|
|
1064
|
+
)
|
|
1065
|
+
raise
|
|
1066
|
+
network_recovery.log_recovered()
|
|
1067
|
+
return
|
|
1068
|
+
except _TerminalStreamError:
|
|
1069
|
+
# `_stream_once()` has already yielded the terminal event.
|
|
1070
|
+
# Returning preserves fail-closed delivery: replaying here
|
|
1071
|
+
# could duplicate a request that reached the upstream.
|
|
1072
|
+
return
|
|
1073
|
+
except _TransientStreamError as exc:
|
|
1074
|
+
recovery_decision = await _wait_for_process_network_recovery(
|
|
1075
|
+
account,
|
|
1076
|
+
error_code=exc.code,
|
|
1077
|
+
retryable_same_contract=False,
|
|
1078
|
+
failed_session=None,
|
|
1079
|
+
)
|
|
1080
|
+
if recovery_decision == "retry":
|
|
1081
|
+
continue
|
|
1082
|
+
if recovery_decision == "exhausted":
|
|
1083
|
+
raise ProxyResponseError(
|
|
1084
|
+
502,
|
|
1085
|
+
openai_error("upstream_request_timeout", "Proxy request budget exhausted"),
|
|
1086
|
+
) from exc
|
|
1087
|
+
transient_retries += 1
|
|
1088
|
+
if (
|
|
1089
|
+
transient_retries < _facade()._MAX_TRANSIENT_SAME_ACCOUNT_RETRIES
|
|
1090
|
+
and proxy._remaining_budget_seconds(deadline) > 0
|
|
1091
|
+
and not settlement.downstream_visible
|
|
1092
|
+
):
|
|
1093
|
+
delay = backoff_seconds(transient_retries)
|
|
1094
|
+
_facade().logger.info(
|
|
1095
|
+
"Transient post-refresh stream error, retrying same account "
|
|
1096
|
+
"request_id=%s account_id=%s retry=%s/%s delay=%.2fs code=%s",
|
|
1097
|
+
request_id,
|
|
1098
|
+
account.id,
|
|
1099
|
+
transient_retries,
|
|
1100
|
+
_facade()._MAX_TRANSIENT_SAME_ACCOUNT_RETRIES,
|
|
1101
|
+
delay,
|
|
1102
|
+
exc.code,
|
|
1103
|
+
)
|
|
1104
|
+
await scheduler.sleep(delay)
|
|
1105
|
+
continue
|
|
1106
|
+
error_message = str(exc.error.get("message") or "Upstream error")
|
|
1107
|
+
settlement.record_success = False
|
|
1108
|
+
settlement.error_code = exc.code
|
|
1109
|
+
settlement.error_message = error_message
|
|
1110
|
+
settlement.error = exc.error
|
|
1111
|
+
settlement.account_health_error = (
|
|
1112
|
+
_facade()._should_penalize_stream_error(exc.code, error_message)
|
|
1113
|
+
or is_upstream_model_capacity_error(error_message)
|
|
1114
|
+
or exc.account_health_error
|
|
1115
|
+
)
|
|
1116
|
+
if settlement.account_health_error:
|
|
1117
|
+
pending_post_refresh_transient_penalties.append(
|
|
1118
|
+
(
|
|
1119
|
+
account,
|
|
1120
|
+
_stream_settlement_error_payload(settlement),
|
|
1121
|
+
settlement.error_code or "upstream_error",
|
|
1122
|
+
502,
|
|
1123
|
+
transient_retries,
|
|
1124
|
+
None,
|
|
1125
|
+
False,
|
|
1126
|
+
)
|
|
1127
|
+
)
|
|
1128
|
+
if can_try_other_account:
|
|
1129
|
+
retry_exc = ProxyResponseError(502, openai_error(exc.code, error_message))
|
|
1130
|
+
setattr(retry_exc, _POST_REFRESH_TRANSIENT_EXHAUSTED_ATTR, True)
|
|
1131
|
+
raise retry_exc from exc
|
|
1132
|
+
yield format_sse_event(
|
|
1133
|
+
response_failed_event(
|
|
1134
|
+
exc.code,
|
|
1135
|
+
error_message,
|
|
1136
|
+
response_id=settlement.response_id or request_id,
|
|
1137
|
+
)
|
|
1138
|
+
)
|
|
1139
|
+
return
|
|
1140
|
+
except ProxyResponseError as exc:
|
|
1141
|
+
if _must_abort_native_transport_failure(
|
|
1142
|
+
exc,
|
|
1143
|
+
downstream_visible=settlement.downstream_visible,
|
|
1144
|
+
):
|
|
1145
|
+
raise
|
|
1146
|
+
error = _parse_openai_error(exc.payload)
|
|
1147
|
+
error_code = _normalize_error_code(
|
|
1148
|
+
error.code if error else None,
|
|
1149
|
+
error.type if error else None,
|
|
1150
|
+
)
|
|
1151
|
+
recovery_decision = await _wait_for_process_network_recovery(
|
|
1152
|
+
account,
|
|
1153
|
+
error_code=error_code,
|
|
1154
|
+
retryable_same_contract=exc.retryable_same_contract,
|
|
1155
|
+
failed_session=exc.failed_session,
|
|
1156
|
+
)
|
|
1157
|
+
if recovery_decision == "retry":
|
|
1158
|
+
continue
|
|
1159
|
+
if recovery_decision == "exhausted":
|
|
1160
|
+
raise ProxyResponseError(
|
|
1161
|
+
502,
|
|
1162
|
+
openai_error("upstream_request_timeout", "Proxy request budget exhausted"),
|
|
1163
|
+
) from exc
|
|
1164
|
+
if error_code != "account_response_create_cap":
|
|
1165
|
+
raise
|
|
1166
|
+
last_transient_exc = exc
|
|
1167
|
+
if can_try_other_account:
|
|
1168
|
+
raise
|
|
1169
|
+
recovery_sleep_seconds = _account_selection_recovery_sleep_seconds(
|
|
1170
|
+
AccountSelection(
|
|
1171
|
+
account=None,
|
|
1172
|
+
error_message=error.message if error else None,
|
|
1173
|
+
error_code=error_code,
|
|
1174
|
+
)
|
|
1175
|
+
)
|
|
1176
|
+
if recovery_sleep_seconds is None or proxy._remaining_budget_seconds(deadline) <= 0:
|
|
1177
|
+
raise
|
|
1178
|
+
async for wait_event in _iter_account_capacity_recovery_wait(
|
|
1179
|
+
request_id=request_id,
|
|
1180
|
+
model=payload.model,
|
|
1181
|
+
account_id=account.id,
|
|
1182
|
+
error_message=error.message if error else None,
|
|
1183
|
+
recovery_sleep_seconds=recovery_sleep_seconds,
|
|
1184
|
+
remaining_budget_seconds=proxy._remaining_budget_seconds(deadline),
|
|
1185
|
+
emit_keepalives=not propagate_http_errors or not enforce_openai_sdk_contract,
|
|
1186
|
+
stage="post_refresh_response_create",
|
|
1187
|
+
scheduler=scheduler,
|
|
1188
|
+
clock=clock,
|
|
1189
|
+
):
|
|
1190
|
+
yield wait_event
|
|
1191
|
+
if proxy._remaining_budget_seconds(deadline) <= 0:
|
|
1192
|
+
raise
|
|
1193
|
+
finally:
|
|
1194
|
+
pop_stream_timeout_overrides(stream_timeout_tokens)
|
|
1195
|
+
|
|
1196
|
+
def _record_upstream_transport_metric_once(status: str) -> None:
|
|
1197
|
+
nonlocal upstream_transport_metric_recorded
|
|
1198
|
+
if upstream_transport_metric_recorded:
|
|
1199
|
+
return
|
|
1200
|
+
upstream_transport_metric_recorded = True
|
|
1201
|
+
_record_upstream_transport_decision(
|
|
1202
|
+
downstream_transport=request_transport,
|
|
1203
|
+
upstream_transport=upstream_stream_transport,
|
|
1204
|
+
policy=upstream_transport_policy_label,
|
|
1205
|
+
sticky=upstream_transport_sticky,
|
|
1206
|
+
status=status,
|
|
1207
|
+
)
|
|
1208
|
+
|
|
1209
|
+
async def _render_account_model_rejection(
|
|
1210
|
+
exc: ProxyResponseError,
|
|
1211
|
+
*,
|
|
1212
|
+
account_id: str | None,
|
|
1213
|
+
) -> str:
|
|
1214
|
+
error = _parse_openai_error(exc.payload)
|
|
1215
|
+
error_code = (
|
|
1216
|
+
_normalize_error_code(
|
|
1217
|
+
error.code if error else None,
|
|
1218
|
+
error.type if error else None,
|
|
1219
|
+
)
|
|
1220
|
+
or "invalid_request_error"
|
|
1221
|
+
)
|
|
1222
|
+
error_message = error.message if error and error.message else "Upstream rejected the requested model"
|
|
1223
|
+
event = response_failed_event(
|
|
1224
|
+
error_code,
|
|
1225
|
+
error_message,
|
|
1226
|
+
error_type=(error.type if error else None) or "invalid_request_error",
|
|
1227
|
+
response_id=request_id,
|
|
1228
|
+
error_param=error.param_state if error else None,
|
|
1229
|
+
)
|
|
1230
|
+
_apply_error_metadata(event["response"]["error"], error)
|
|
1231
|
+
if not any_attempt_logged:
|
|
1232
|
+
await proxy._write_request_log(
|
|
1233
|
+
affinity_observation=affinity_observation,
|
|
1234
|
+
account_id=account_id,
|
|
1235
|
+
api_key=api_key,
|
|
1236
|
+
request_id=request_id,
|
|
1237
|
+
model=payload.model,
|
|
1238
|
+
latency_ms=int((clock.monotonic() - start) * 1000),
|
|
1239
|
+
status="error",
|
|
1240
|
+
error_code=error_code,
|
|
1241
|
+
error_message=error_message,
|
|
1242
|
+
reasoning_effort=payload.reasoning.effort if payload.reasoning else None,
|
|
1243
|
+
transport=request_transport,
|
|
1244
|
+
upstream_transport=upstream_stream_transport,
|
|
1245
|
+
service_tier=payload.service_tier,
|
|
1246
|
+
requested_service_tier=payload.service_tier,
|
|
1247
|
+
useragent=useragent,
|
|
1248
|
+
useragent_group=useragent_group,
|
|
1249
|
+
conversation_id=conversation_id,
|
|
1250
|
+
client_ip=client_ip,
|
|
1251
|
+
)
|
|
1252
|
+
return format_sse_event(event)
|
|
1253
|
+
|
|
1254
|
+
async def _retry_account_model_rejection(
|
|
1255
|
+
exc: ProxyResponseError,
|
|
1256
|
+
account: Account,
|
|
1257
|
+
*,
|
|
1258
|
+
outcome: str,
|
|
1259
|
+
) -> bool | None:
|
|
1260
|
+
nonlocal affinity, current_account_lease
|
|
1261
|
+
nonlocal account_model_replay_attempted
|
|
1262
|
+
nonlocal last_account_model_rejection, last_account_model_rejection_account_id
|
|
1263
|
+
error = _parse_openai_error(exc.payload)
|
|
1264
|
+
error_code = _normalize_error_code(
|
|
1265
|
+
error.code if error else None,
|
|
1266
|
+
error.type if error else None,
|
|
1267
|
+
)
|
|
1268
|
+
if not _is_account_model_unsupported_error(
|
|
1269
|
+
code=error_code,
|
|
1270
|
+
message=error.message if error else None,
|
|
1271
|
+
model=payload.model,
|
|
1272
|
+
):
|
|
1273
|
+
return None
|
|
1274
|
+
can_move_verified_owner = bool(
|
|
1275
|
+
require_preferred_account
|
|
1276
|
+
and preferred_account_id == account.id
|
|
1277
|
+
and verified_fresh_replay_payload is not None
|
|
1278
|
+
)
|
|
1279
|
+
can_try_other_account = bool(
|
|
1280
|
+
not account_model_replay_attempted
|
|
1281
|
+
and attempt < max_attempts - 1
|
|
1282
|
+
and (
|
|
1283
|
+
can_move_verified_owner
|
|
1284
|
+
or (not require_preferred_account and account.id != file_preferred_account_id)
|
|
1285
|
+
)
|
|
1286
|
+
)
|
|
1287
|
+
if not can_try_other_account:
|
|
1288
|
+
return False
|
|
1289
|
+
logger.info(
|
|
1290
|
+
"Retrying stream after account/model rejection request_id=%s account_id=%s model=%s phase=%s reason=%s",
|
|
1291
|
+
request_id,
|
|
1292
|
+
account.id,
|
|
1293
|
+
payload.model,
|
|
1294
|
+
outcome,
|
|
1295
|
+
_ACCOUNT_MODEL_UNSUPPORTED_ERROR_CODE,
|
|
1296
|
+
)
|
|
1297
|
+
account_model_replay_attempted = True
|
|
1298
|
+
last_account_model_rejection = exc
|
|
1299
|
+
last_account_model_rejection_account_id = account.id
|
|
1300
|
+
await _release_tracked_stream_lease(current_account_lease)
|
|
1301
|
+
current_account_lease = None
|
|
1302
|
+
if not _move_verified_fresh_replay_from_owner(
|
|
1303
|
+
account_id=account.id,
|
|
1304
|
+
outcome=outcome,
|
|
1305
|
+
):
|
|
1306
|
+
excluded_account_ids.add(account.id)
|
|
1307
|
+
affinity = replace(affinity, reallocate_sticky=True)
|
|
1308
|
+
return True
|
|
1309
|
+
|
|
1310
|
+
try:
|
|
1311
|
+
# From this exact point the service finalizer below owns reservation
|
|
1312
|
+
# settlement/release. Preflight failures before this boundary are
|
|
1313
|
+
# still owned by the originating API startup guard.
|
|
1314
|
+
_signal_propagated_responses_service_cleanup_ready()
|
|
1315
|
+
if payload.previous_response_id is not None:
|
|
1316
|
+
previous_response_lookup_session_id = _owner_lookup_session_id_from_headers(headers)
|
|
1317
|
+
preferred_account_id = await proxy._resolve_websocket_previous_response_owner(
|
|
1318
|
+
previous_response_id=payload.previous_response_id,
|
|
1319
|
+
api_key=api_key,
|
|
1320
|
+
session_id=previous_response_lookup_session_id,
|
|
1321
|
+
surface="http_stream",
|
|
1322
|
+
)
|
|
1323
|
+
require_preferred_account = preferred_account_id is not None
|
|
1324
|
+
# `previous_response_id` is a stored-object continuation, so it
|
|
1325
|
+
# remains hard owner-bound even when the request also carries a
|
|
1326
|
+
# soft prompt-cache affinity key. A different account may have a
|
|
1327
|
+
# warmer cache, but it cannot safely resolve the stored response.
|
|
1328
|
+
if preferred_account_id is None:
|
|
1329
|
+
selection_inputs = await proxy._load_balancer._load_selection_inputs(
|
|
1330
|
+
model=payload.model,
|
|
1331
|
+
additional_limit_name=None,
|
|
1332
|
+
account_ids=None,
|
|
1333
|
+
)
|
|
1334
|
+
if len(selection_inputs.accounts) != 1:
|
|
1335
|
+
message = "Previous response owner account is unavailable; retry later."
|
|
1336
|
+
_record_continuity_fail_closed(
|
|
1337
|
+
surface="http_stream",
|
|
1338
|
+
reason="owner_account_unavailable",
|
|
1339
|
+
previous_response_id=payload.previous_response_id,
|
|
1340
|
+
session_id=previous_response_lookup_session_id,
|
|
1341
|
+
upstream_error_code="owner_lookup_miss",
|
|
1342
|
+
)
|
|
1343
|
+
event = response_failed_event(
|
|
1344
|
+
"previous_response_owner_unavailable",
|
|
1345
|
+
message,
|
|
1346
|
+
response_id=request_id,
|
|
1347
|
+
)
|
|
1348
|
+
yield format_sse_event(event)
|
|
1349
|
+
await proxy._write_request_log(
|
|
1350
|
+
affinity_observation=affinity_observation,
|
|
1351
|
+
account_id=None,
|
|
1352
|
+
api_key=api_key,
|
|
1353
|
+
request_id=request_id,
|
|
1354
|
+
model=payload.model,
|
|
1355
|
+
latency_ms=int((clock.monotonic() - start) * 1000),
|
|
1356
|
+
status="error",
|
|
1357
|
+
error_code="previous_response_owner_unavailable",
|
|
1358
|
+
error_message=message,
|
|
1359
|
+
reasoning_effort=payload.reasoning.effort if payload.reasoning else None,
|
|
1360
|
+
transport=request_transport,
|
|
1361
|
+
upstream_transport=upstream_stream_transport,
|
|
1362
|
+
service_tier=payload.service_tier,
|
|
1363
|
+
requested_service_tier=payload.service_tier,
|
|
1364
|
+
useragent=useragent,
|
|
1365
|
+
useragent_group=useragent_group,
|
|
1366
|
+
conversation_id=conversation_id,
|
|
1367
|
+
client_ip=client_ip,
|
|
1368
|
+
)
|
|
1369
|
+
return
|
|
1370
|
+
# File and previous-response ownership are peers, not fallback
|
|
1371
|
+
# preferences. Resolve both before selection so a conflict cannot
|
|
1372
|
+
# be hidden by whichever source happened to run first. A hard turn
|
|
1373
|
+
# state is checked against this required owner by the balancer.
|
|
1374
|
+
preferred_account_id = resolve_required_account_id(
|
|
1375
|
+
("turn state", turn_state_owner_account_id),
|
|
1376
|
+
("previous response", preferred_account_id),
|
|
1377
|
+
("input file", rewritten_file_account_id),
|
|
1378
|
+
)
|
|
1379
|
+
require_preferred_account = require_preferred_account or turn_state_owner_account_id is not None
|
|
1380
|
+
file_required_preferred_account = rewritten_file_account_id is not None
|
|
1381
|
+
for attempt in range(max_attempts):
|
|
1382
|
+
remaining_budget = proxy._remaining_budget_seconds(deadline)
|
|
1383
|
+
if remaining_budget <= 0:
|
|
1384
|
+
await _drain_pending_post_refresh_penalty_on_terminal(settlement)
|
|
1385
|
+
_facade().logger.warning(
|
|
1386
|
+
"Proxy request budget exhausted before retry request_id=%s attempt=%s",
|
|
1387
|
+
request_id,
|
|
1388
|
+
attempt + 1,
|
|
1389
|
+
)
|
|
1390
|
+
await proxy._write_stream_preflight_error(
|
|
1391
|
+
affinity_observation=affinity_observation,
|
|
1392
|
+
account_id=None,
|
|
1393
|
+
api_key=api_key,
|
|
1394
|
+
request_id=request_id,
|
|
1395
|
+
model=payload.model,
|
|
1396
|
+
start=start,
|
|
1397
|
+
error_code="upstream_request_timeout",
|
|
1398
|
+
error_message="Proxy request budget exhausted",
|
|
1399
|
+
reasoning_effort=payload.reasoning.effort if payload.reasoning else None,
|
|
1400
|
+
service_tier=payload.service_tier,
|
|
1401
|
+
transport=request_transport,
|
|
1402
|
+
upstream_transport=upstream_stream_transport,
|
|
1403
|
+
useragent=useragent,
|
|
1404
|
+
useragent_group=useragent_group,
|
|
1405
|
+
conversation_id=conversation_id,
|
|
1406
|
+
client_ip=client_ip,
|
|
1407
|
+
)
|
|
1408
|
+
if preserve_native_failure_lifecycle:
|
|
1409
|
+
return
|
|
1410
|
+
yield format_sse_event(_facade()._proxy_request_timeout_event(request_id))
|
|
1411
|
+
return
|
|
1412
|
+
while True:
|
|
1413
|
+
effective_preferred_account_id = resolve_required_account_id(
|
|
1414
|
+
("continuation", preferred_account_id),
|
|
1415
|
+
("dispatched payload", payload_replay_required_account_id),
|
|
1416
|
+
)
|
|
1417
|
+
effective_require_preferred_account = (
|
|
1418
|
+
require_preferred_account or payload_replay_required_account_id is not None
|
|
1419
|
+
)
|
|
1420
|
+
try:
|
|
1421
|
+
selection = await proxy._select_account_with_budget_compatible(
|
|
1422
|
+
deadline,
|
|
1423
|
+
request_id=request_id,
|
|
1424
|
+
kind="stream",
|
|
1425
|
+
api_key=api_key,
|
|
1426
|
+
affinity_policy=affinity,
|
|
1427
|
+
prefer_earlier_reset_accounts=prefer_earlier_reset,
|
|
1428
|
+
prefer_earlier_reset_window=_facade()._prefer_earlier_reset_window(settings),
|
|
1429
|
+
routing_strategy=routing_strategy,
|
|
1430
|
+
model=payload.model,
|
|
1431
|
+
service_tier=payload.service_tier,
|
|
1432
|
+
exclude_account_ids=excluded_account_ids,
|
|
1433
|
+
preferred_account_id=effective_preferred_account_id,
|
|
1434
|
+
require_security_work_authorized=require_security_work_authorized,
|
|
1435
|
+
lease_kind="stream",
|
|
1436
|
+
estimated_lease_tokens=estimated_lease_tokens,
|
|
1437
|
+
# Keep stored-object and file ownership strict. The
|
|
1438
|
+
# verified-fresh replay branch below removes its
|
|
1439
|
+
# anchor before it permits cross-account movement.
|
|
1440
|
+
fallback_on_preferred_account_unavailable=not (
|
|
1441
|
+
effective_require_preferred_account or file_required_preferred_account
|
|
1442
|
+
),
|
|
1443
|
+
)
|
|
1444
|
+
except ProxyResponseError as exc:
|
|
1445
|
+
await _drain_pending_post_refresh_penalty_on_terminal(settlement)
|
|
1446
|
+
error = _parse_openai_error(exc.payload)
|
|
1447
|
+
error_code = _normalize_error_code(error.code if error else None, error.type if error else None)
|
|
1448
|
+
error_message = error.message if error else None
|
|
1449
|
+
if _facade()._is_proxy_budget_exhausted_error(exc):
|
|
1450
|
+
await proxy._write_stream_preflight_error(
|
|
1451
|
+
affinity_observation=affinity_observation,
|
|
1452
|
+
account_id=None,
|
|
1453
|
+
api_key=api_key,
|
|
1454
|
+
request_id=request_id,
|
|
1455
|
+
model=payload.model,
|
|
1456
|
+
start=start,
|
|
1457
|
+
error_code="upstream_request_timeout",
|
|
1458
|
+
error_message="Proxy request budget exhausted",
|
|
1459
|
+
reasoning_effort=payload.reasoning.effort if payload.reasoning else None,
|
|
1460
|
+
service_tier=payload.service_tier,
|
|
1461
|
+
transport=request_transport,
|
|
1462
|
+
upstream_transport=upstream_stream_transport,
|
|
1463
|
+
useragent=useragent,
|
|
1464
|
+
useragent_group=useragent_group,
|
|
1465
|
+
conversation_id=conversation_id,
|
|
1466
|
+
client_ip=client_ip,
|
|
1467
|
+
)
|
|
1468
|
+
if preserve_native_failure_lifecycle:
|
|
1469
|
+
return
|
|
1470
|
+
yield format_sse_event(_facade()._proxy_request_timeout_event(request_id))
|
|
1471
|
+
return
|
|
1472
|
+
event = response_failed_event(
|
|
1473
|
+
error_code,
|
|
1474
|
+
error_message or "Upstream unavailable",
|
|
1475
|
+
error_type=(error.type or "server_error") if error else "server_error",
|
|
1476
|
+
response_id=request_id,
|
|
1477
|
+
)
|
|
1478
|
+
_apply_error_metadata(event["response"]["error"], error)
|
|
1479
|
+
yield format_sse_event(event)
|
|
1480
|
+
return
|
|
1481
|
+
account = selection.account
|
|
1482
|
+
current_account_lease = selection.lease
|
|
1483
|
+
if selection.lease is not None:
|
|
1484
|
+
account_leases.append(selection.lease)
|
|
1485
|
+
if (
|
|
1486
|
+
not account
|
|
1487
|
+
and require_security_work_authorized
|
|
1488
|
+
and selection.error_code == _facade()._NO_SECURITY_WORK_AUTHORIZED_ACCOUNTS_CODE
|
|
1489
|
+
):
|
|
1490
|
+
_facade().logger.info(
|
|
1491
|
+
"No security-work-authorized account available for stream retry; "
|
|
1492
|
+
"continuing normal account failover request_id=%s",
|
|
1493
|
+
request_id,
|
|
1494
|
+
)
|
|
1495
|
+
yield format_sse_event(
|
|
1496
|
+
_facade()._security_work_advisory_event(
|
|
1497
|
+
code=_facade()._NO_SECURITY_WORK_AUTHORIZED_ACCOUNTS_CODE,
|
|
1498
|
+
message=_facade()._SECURITY_WORK_NO_AUTHORIZED_ACCOUNTS_MESSAGE,
|
|
1499
|
+
request_id=request_id,
|
|
1500
|
+
action="continue_normal_selection",
|
|
1501
|
+
)
|
|
1502
|
+
)
|
|
1503
|
+
require_security_work_authorized = False
|
|
1504
|
+
continue
|
|
1505
|
+
if not account and deferred_capacity_account is not None:
|
|
1506
|
+
deferred_error = _parse_openai_error(last_transient_exc.payload) if last_transient_exc else None
|
|
1507
|
+
recovery_sleep_seconds = _account_selection_recovery_sleep_seconds(
|
|
1508
|
+
AccountSelection(
|
|
1509
|
+
account=None,
|
|
1510
|
+
error_message=deferred_error.message if deferred_error else None,
|
|
1511
|
+
error_code="account_response_create_cap",
|
|
1512
|
+
)
|
|
1513
|
+
)
|
|
1514
|
+
if recovery_sleep_seconds is not None:
|
|
1515
|
+
remaining_budget_seconds = proxy._remaining_budget_seconds(deadline)
|
|
1516
|
+
if remaining_budget_seconds <= 0:
|
|
1517
|
+
if propagate_http_errors and last_transient_exc is not None:
|
|
1518
|
+
raise last_transient_exc
|
|
1519
|
+
event = response_failed_event(
|
|
1520
|
+
"account_response_create_cap",
|
|
1521
|
+
(deferred_error.message if deferred_error else None)
|
|
1522
|
+
or "Account response-create concurrency limit reached",
|
|
1523
|
+
error_type=(deferred_error.type if deferred_error else None) or "server_error",
|
|
1524
|
+
response_id=request_id,
|
|
1525
|
+
)
|
|
1526
|
+
yield format_sse_event(event)
|
|
1527
|
+
return
|
|
1528
|
+
capacity_account = deferred_capacity_account
|
|
1529
|
+
capacity_account_id = capacity_account.id
|
|
1530
|
+
excluded_account_ids.discard(capacity_account_id)
|
|
1531
|
+
async for wait_event in _iter_account_capacity_recovery_wait(
|
|
1532
|
+
request_id=request_id,
|
|
1533
|
+
model=payload.model,
|
|
1534
|
+
account_id=capacity_account_id,
|
|
1535
|
+
error_message=deferred_error.message if deferred_error else None,
|
|
1536
|
+
recovery_sleep_seconds=recovery_sleep_seconds,
|
|
1537
|
+
remaining_budget_seconds=proxy._remaining_budget_seconds(deadline),
|
|
1538
|
+
emit_keepalives=not propagate_http_errors or not enforce_openai_sdk_contract,
|
|
1539
|
+
stage="response_create_no_alternate",
|
|
1540
|
+
scheduler=scheduler,
|
|
1541
|
+
clock=clock,
|
|
1542
|
+
):
|
|
1543
|
+
yield wait_event
|
|
1544
|
+
if proxy._remaining_budget_seconds(deadline) <= 0:
|
|
1545
|
+
if propagate_http_errors and last_transient_exc is not None:
|
|
1546
|
+
raise last_transient_exc
|
|
1547
|
+
event = response_failed_event(
|
|
1548
|
+
"account_response_create_cap",
|
|
1549
|
+
(deferred_error.message if deferred_error else None)
|
|
1550
|
+
or "Account response-create concurrency limit reached",
|
|
1551
|
+
error_type=(deferred_error.type if deferred_error else None) or "server_error",
|
|
1552
|
+
response_id=request_id,
|
|
1553
|
+
)
|
|
1554
|
+
yield format_sse_event(event)
|
|
1555
|
+
return
|
|
1556
|
+
account = capacity_account
|
|
1557
|
+
current_account_lease = deferred_capacity_lease
|
|
1558
|
+
deferred_capacity_account = None
|
|
1559
|
+
deferred_capacity_lease = None
|
|
1560
|
+
if account is not None and deferred_capacity_account is not None:
|
|
1561
|
+
await _release_tracked_stream_lease(deferred_capacity_lease)
|
|
1562
|
+
deferred_capacity_account = None
|
|
1563
|
+
deferred_capacity_lease = None
|
|
1564
|
+
if (
|
|
1565
|
+
not account
|
|
1566
|
+
and selection.error_code == "hard_affinity_saturated"
|
|
1567
|
+
and transient_failed_account_id is not None
|
|
1568
|
+
and transient_failed_account_id in excluded_account_ids
|
|
1569
|
+
and not hard_affinity_same_owner_retry_attempted
|
|
1570
|
+
):
|
|
1571
|
+
# A hard CODEX_SESSION row can resolve only to its owner.
|
|
1572
|
+
# If the transport just excluded that same account after
|
|
1573
|
+
# a pre-visible transient failure, alternate selection is
|
|
1574
|
+
# impossible by construction. Retry the known owner once
|
|
1575
|
+
# without weakening or rebinding the durable affinity.
|
|
1576
|
+
excluded_account_ids.discard(transient_failed_account_id)
|
|
1577
|
+
hard_affinity_same_owner_retry_attempted = True
|
|
1578
|
+
_facade().logger.info(
|
|
1579
|
+
"Retrying transient stream failure on hard affinity owner request_id=%s account_id=%s",
|
|
1580
|
+
request_id,
|
|
1581
|
+
transient_failed_account_id,
|
|
1582
|
+
)
|
|
1583
|
+
continue
|
|
1584
|
+
if (
|
|
1585
|
+
not account
|
|
1586
|
+
and selection.error_code != USAGE_LIMIT_REACHED
|
|
1587
|
+
and (
|
|
1588
|
+
selection.error_code in _LOCAL_ACCOUNT_CAP_ERROR_CODES
|
|
1589
|
+
or not (propagate_http_errors and last_transient_exc is not None)
|
|
1590
|
+
)
|
|
1591
|
+
and (
|
|
1592
|
+
selection.error_code in _LOCAL_ACCOUNT_CAP_ERROR_CODES
|
|
1593
|
+
# A preserved confirmed pre-dispatch failure is
|
|
1594
|
+
# terminal for this request: waiting for capacity
|
|
1595
|
+
# recovery cannot resurrect the dead proxy route.
|
|
1596
|
+
or last_pre_dispatch_transport_error is None
|
|
1597
|
+
)
|
|
1598
|
+
and (
|
|
1599
|
+
selection.error_code in _LOCAL_ACCOUNT_CAP_ERROR_CODES
|
|
1600
|
+
or (last_retryable_stream_error is None and last_security_work_retry_error is None)
|
|
1601
|
+
)
|
|
1602
|
+
):
|
|
1603
|
+
recovery_sleep_seconds = _account_selection_recovery_sleep_seconds(selection)
|
|
1604
|
+
if recovery_sleep_seconds is not None:
|
|
1605
|
+
remaining_budget_seconds = proxy._remaining_budget_seconds(deadline)
|
|
1606
|
+
if remaining_budget_seconds <= 0:
|
|
1607
|
+
break
|
|
1608
|
+
async for wait_event in _iter_account_capacity_recovery_wait(
|
|
1609
|
+
request_id=request_id,
|
|
1610
|
+
model=payload.model,
|
|
1611
|
+
account_id=None,
|
|
1612
|
+
error_message=selection.error_message,
|
|
1613
|
+
recovery_sleep_seconds=recovery_sleep_seconds,
|
|
1614
|
+
remaining_budget_seconds=remaining_budget_seconds,
|
|
1615
|
+
emit_keepalives=not propagate_http_errors or not enforce_openai_sdk_contract,
|
|
1616
|
+
stage="selection",
|
|
1617
|
+
scheduler=scheduler,
|
|
1618
|
+
clock=clock,
|
|
1619
|
+
):
|
|
1620
|
+
yield wait_event
|
|
1621
|
+
if proxy._remaining_budget_seconds(deadline) <= 0:
|
|
1622
|
+
break
|
|
1623
|
+
continue
|
|
1624
|
+
break
|
|
1625
|
+
if not account:
|
|
1626
|
+
if last_account_model_rejection is not None:
|
|
1627
|
+
await _drain_pending_post_refresh_penalty_on_terminal(settlement)
|
|
1628
|
+
if propagate_http_errors:
|
|
1629
|
+
raise last_account_model_rejection
|
|
1630
|
+
yield await _render_account_model_rejection(
|
|
1631
|
+
last_account_model_rejection,
|
|
1632
|
+
account_id=last_account_model_rejection_account_id,
|
|
1633
|
+
)
|
|
1634
|
+
return
|
|
1635
|
+
if last_pre_dispatch_transport_error is not None:
|
|
1636
|
+
# No eligible replacement exists: preserve the original
|
|
1637
|
+
# sanitized upstream-unavailable failure instead of
|
|
1638
|
+
# generating a misleading ``no_accounts`` response.
|
|
1639
|
+
await _drain_pending_post_refresh_penalty_on_terminal(settlement)
|
|
1640
|
+
if propagate_http_errors:
|
|
1641
|
+
raise last_pre_dispatch_transport_error
|
|
1642
|
+
yield _render_dispatch_transport_error(last_pre_dispatch_transport_error)
|
|
1643
|
+
return
|
|
1644
|
+
if selection.error_code == USAGE_LIMIT_REACHED:
|
|
1645
|
+
await _drain_pending_post_refresh_penalty_on_terminal(settlement)
|
|
1646
|
+
no_accounts_msg = selection.error_message or "Usage limit reached"
|
|
1647
|
+
status_code, error_payload = selection_failure_response(selection)
|
|
1648
|
+
await proxy._write_request_log(
|
|
1649
|
+
affinity_observation=affinity_observation,
|
|
1650
|
+
account_id=None,
|
|
1651
|
+
api_key=api_key,
|
|
1652
|
+
request_id=request_id,
|
|
1653
|
+
model=payload.model,
|
|
1654
|
+
latency_ms=int((clock.monotonic() - start) * 1000),
|
|
1655
|
+
status="error",
|
|
1656
|
+
error_code=USAGE_LIMIT_REACHED,
|
|
1657
|
+
error_message=no_accounts_msg,
|
|
1658
|
+
reasoning_effort=payload.reasoning.effort if payload.reasoning else None,
|
|
1659
|
+
transport=request_transport,
|
|
1660
|
+
upstream_transport=upstream_stream_transport,
|
|
1661
|
+
service_tier=payload.service_tier,
|
|
1662
|
+
requested_service_tier=payload.service_tier,
|
|
1663
|
+
useragent=useragent,
|
|
1664
|
+
useragent_group=useragent_group,
|
|
1665
|
+
client_ip=client_ip,
|
|
1666
|
+
)
|
|
1667
|
+
if propagate_http_errors:
|
|
1668
|
+
raise ProxyResponseError(status_code, error_payload)
|
|
1669
|
+
yield format_sse_event(
|
|
1670
|
+
response_failed_event(
|
|
1671
|
+
USAGE_LIMIT_REACHED,
|
|
1672
|
+
no_accounts_msg,
|
|
1673
|
+
error_type=USAGE_LIMIT_REACHED,
|
|
1674
|
+
response_id=request_id,
|
|
1675
|
+
resets_at=selection.resets_at,
|
|
1676
|
+
)
|
|
1677
|
+
)
|
|
1678
|
+
return
|
|
1679
|
+
if selection.error_code in _LOCAL_ACCOUNT_CAP_ERROR_CODES:
|
|
1680
|
+
await _drain_pending_post_refresh_penalty_on_terminal(settlement)
|
|
1681
|
+
no_accounts_msg = selection.error_message or "Local account capacity is exhausted"
|
|
1682
|
+
error_code = selection.error_code
|
|
1683
|
+
event = response_failed_event(
|
|
1684
|
+
error_code,
|
|
1685
|
+
no_accounts_msg,
|
|
1686
|
+
error_type="rate_limit_error",
|
|
1687
|
+
response_id=request_id,
|
|
1688
|
+
)
|
|
1689
|
+
await proxy._write_request_log(
|
|
1690
|
+
affinity_observation=affinity_observation,
|
|
1691
|
+
account_id=None,
|
|
1692
|
+
api_key=api_key,
|
|
1693
|
+
request_id=request_id,
|
|
1694
|
+
model=payload.model,
|
|
1695
|
+
latency_ms=int((clock.monotonic() - start) * 1000),
|
|
1696
|
+
status="error",
|
|
1697
|
+
error_code=error_code,
|
|
1698
|
+
error_message=no_accounts_msg,
|
|
1699
|
+
reasoning_effort=payload.reasoning.effort if payload.reasoning else None,
|
|
1700
|
+
transport=request_transport,
|
|
1701
|
+
upstream_transport=upstream_stream_transport,
|
|
1702
|
+
service_tier=payload.service_tier,
|
|
1703
|
+
requested_service_tier=payload.service_tier,
|
|
1704
|
+
useragent=useragent,
|
|
1705
|
+
useragent_group=useragent_group,
|
|
1706
|
+
conversation_id=conversation_id,
|
|
1707
|
+
client_ip=client_ip,
|
|
1708
|
+
)
|
|
1709
|
+
if propagate_http_errors:
|
|
1710
|
+
raise ProxyResponseError(
|
|
1711
|
+
429,
|
|
1712
|
+
openai_error(
|
|
1713
|
+
error_code,
|
|
1714
|
+
no_accounts_msg,
|
|
1715
|
+
error_type="rate_limit_error",
|
|
1716
|
+
),
|
|
1717
|
+
)
|
|
1718
|
+
yield format_sse_event(event)
|
|
1719
|
+
return
|
|
1720
|
+
if (
|
|
1721
|
+
require_preferred_account
|
|
1722
|
+
and preferred_account_id is not None
|
|
1723
|
+
and verified_fresh_replay_payload is not None
|
|
1724
|
+
):
|
|
1725
|
+
excluded_account_ids.add(preferred_account_id)
|
|
1726
|
+
payload = verified_fresh_replay_payload
|
|
1727
|
+
verified_fresh_replay_payload = None
|
|
1728
|
+
preferred_account_id = None
|
|
1729
|
+
require_preferred_account = False
|
|
1730
|
+
affinity = replace(affinity, reallocate_sticky=True)
|
|
1731
|
+
logger.info(
|
|
1732
|
+
"cross_transport_verified_fresh_replay request_id=%s outcome=owner_unavailable",
|
|
1733
|
+
request_id,
|
|
1734
|
+
)
|
|
1735
|
+
continue
|
|
1736
|
+
await _drain_pending_post_refresh_penalty_on_terminal(settlement)
|
|
1737
|
+
if propagate_http_errors and last_transient_exc is not None:
|
|
1738
|
+
raise last_transient_exc
|
|
1739
|
+
if last_retryable_stream_error is not None:
|
|
1740
|
+
error_message = str(last_retryable_stream_error.error.get("message") or "Upstream error")
|
|
1741
|
+
event = response_failed_event(
|
|
1742
|
+
last_retryable_stream_error.code,
|
|
1743
|
+
error_message,
|
|
1744
|
+
response_id=request_id,
|
|
1745
|
+
)
|
|
1746
|
+
yield format_sse_event(event)
|
|
1747
|
+
await proxy._write_request_log(
|
|
1748
|
+
affinity_observation=affinity_observation,
|
|
1749
|
+
account_id=None,
|
|
1750
|
+
api_key=api_key,
|
|
1751
|
+
request_id=request_id,
|
|
1752
|
+
model=payload.model,
|
|
1753
|
+
latency_ms=int((clock.monotonic() - start) * 1000),
|
|
1754
|
+
status="error",
|
|
1755
|
+
error_code=last_retryable_stream_error.code,
|
|
1756
|
+
error_message=error_message,
|
|
1757
|
+
reasoning_effort=payload.reasoning.effort if payload.reasoning else None,
|
|
1758
|
+
transport=request_transport,
|
|
1759
|
+
upstream_transport=upstream_stream_transport,
|
|
1760
|
+
service_tier=payload.service_tier,
|
|
1761
|
+
requested_service_tier=payload.service_tier,
|
|
1762
|
+
useragent=useragent,
|
|
1763
|
+
useragent_group=useragent_group,
|
|
1764
|
+
conversation_id=conversation_id,
|
|
1765
|
+
client_ip=client_ip,
|
|
1766
|
+
)
|
|
1767
|
+
return
|
|
1768
|
+
if require_preferred_account and preferred_account_id is not None:
|
|
1769
|
+
error_code = "previous_response_owner_unavailable"
|
|
1770
|
+
message = "Previous response owner account is unavailable; retry later."
|
|
1771
|
+
reason = "owner_account_unavailable"
|
|
1772
|
+
upstream_error_code = "no_accounts"
|
|
1773
|
+
if selection.error_code == "continuity_owner_conflict":
|
|
1774
|
+
error_code = "continuity_owner_conflict"
|
|
1775
|
+
message = (
|
|
1776
|
+
selection.error_message
|
|
1777
|
+
or "Account-owned continuity sources conflict; retry the logical turn"
|
|
1778
|
+
)
|
|
1779
|
+
reason = "owner_conflict"
|
|
1780
|
+
upstream_error_code = selection.error_code
|
|
1781
|
+
_record_continuity_fail_closed(
|
|
1782
|
+
surface="http_stream",
|
|
1783
|
+
reason=reason,
|
|
1784
|
+
previous_response_id=payload.previous_response_id,
|
|
1785
|
+
session_id=headers.get("x-codex-turn-state") or headers.get("session_id"),
|
|
1786
|
+
upstream_error_code=upstream_error_code,
|
|
1787
|
+
)
|
|
1788
|
+
event = response_failed_event(
|
|
1789
|
+
error_code,
|
|
1790
|
+
message,
|
|
1791
|
+
response_id=request_id,
|
|
1792
|
+
)
|
|
1793
|
+
yield format_sse_event(event)
|
|
1794
|
+
await proxy._write_request_log(
|
|
1795
|
+
affinity_observation=affinity_observation,
|
|
1796
|
+
account_id=preferred_account_id,
|
|
1797
|
+
api_key=api_key,
|
|
1798
|
+
request_id=request_id,
|
|
1799
|
+
model=payload.model,
|
|
1800
|
+
latency_ms=int((clock.monotonic() - start) * 1000),
|
|
1801
|
+
status="error",
|
|
1802
|
+
error_code=error_code,
|
|
1803
|
+
error_message=message,
|
|
1804
|
+
reasoning_effort=payload.reasoning.effort if payload.reasoning else None,
|
|
1805
|
+
transport=request_transport,
|
|
1806
|
+
upstream_transport=upstream_stream_transport,
|
|
1807
|
+
service_tier=payload.service_tier,
|
|
1808
|
+
requested_service_tier=payload.service_tier,
|
|
1809
|
+
useragent=useragent,
|
|
1810
|
+
useragent_group=useragent_group,
|
|
1811
|
+
conversation_id=conversation_id,
|
|
1812
|
+
client_ip=client_ip,
|
|
1813
|
+
)
|
|
1814
|
+
return
|
|
1815
|
+
# If a prior attempt stored a transient 500 and the caller
|
|
1816
|
+
# expects HTTP error propagation, re-raise the original error
|
|
1817
|
+
# instead of returning a generic no_accounts event.
|
|
1818
|
+
if propagate_http_errors and last_transient_exc is not None:
|
|
1819
|
+
raise last_transient_exc
|
|
1820
|
+
if last_security_work_retry_error is not None:
|
|
1821
|
+
message = (
|
|
1822
|
+
last_security_work_retry_error.error.get("message")
|
|
1823
|
+
or "Security work authorization is required"
|
|
1824
|
+
)
|
|
1825
|
+
event = response_failed_event(
|
|
1826
|
+
last_security_work_retry_error.code,
|
|
1827
|
+
message,
|
|
1828
|
+
response_id=request_id,
|
|
1829
|
+
)
|
|
1830
|
+
yield format_sse_event(event)
|
|
1831
|
+
await proxy._write_request_log(
|
|
1832
|
+
affinity_observation=affinity_observation,
|
|
1833
|
+
account_id=None,
|
|
1834
|
+
api_key=api_key,
|
|
1835
|
+
request_id=request_id,
|
|
1836
|
+
model=payload.model,
|
|
1837
|
+
latency_ms=int((clock.monotonic() - start) * 1000),
|
|
1838
|
+
status="error",
|
|
1839
|
+
error_code=last_security_work_retry_error.code,
|
|
1840
|
+
error_message=message,
|
|
1841
|
+
reasoning_effort=payload.reasoning.effort if payload.reasoning else None,
|
|
1842
|
+
transport=request_transport,
|
|
1843
|
+
upstream_transport=upstream_stream_transport,
|
|
1844
|
+
service_tier=payload.service_tier,
|
|
1845
|
+
requested_service_tier=payload.service_tier,
|
|
1846
|
+
useragent=useragent,
|
|
1847
|
+
useragent_group=useragent_group,
|
|
1848
|
+
conversation_id=conversation_id,
|
|
1849
|
+
client_ip=client_ip,
|
|
1850
|
+
)
|
|
1851
|
+
return
|
|
1852
|
+
no_accounts_msg = selection.error_message or "No active accounts available"
|
|
1853
|
+
error_code = selection.error_code or "no_accounts"
|
|
1854
|
+
event = response_failed_event(
|
|
1855
|
+
error_code,
|
|
1856
|
+
no_accounts_msg,
|
|
1857
|
+
error_type="rate_limit_error"
|
|
1858
|
+
if error_code in _LOCAL_ACCOUNT_CAP_ERROR_CODES
|
|
1859
|
+
else "server_error",
|
|
1860
|
+
response_id=request_id,
|
|
1861
|
+
)
|
|
1862
|
+
yield format_sse_event(event)
|
|
1863
|
+
await proxy._write_request_log(
|
|
1864
|
+
affinity_observation=affinity_observation,
|
|
1865
|
+
account_id=None,
|
|
1866
|
+
api_key=api_key,
|
|
1867
|
+
request_id=request_id,
|
|
1868
|
+
model=payload.model,
|
|
1869
|
+
latency_ms=int((clock.monotonic() - start) * 1000),
|
|
1870
|
+
status="error",
|
|
1871
|
+
error_code=error_code,
|
|
1872
|
+
error_message=no_accounts_msg,
|
|
1873
|
+
reasoning_effort=payload.reasoning.effort if payload.reasoning else None,
|
|
1874
|
+
transport=request_transport,
|
|
1875
|
+
upstream_transport=upstream_stream_transport,
|
|
1876
|
+
service_tier=payload.service_tier,
|
|
1877
|
+
requested_service_tier=payload.service_tier,
|
|
1878
|
+
useragent=useragent,
|
|
1879
|
+
useragent_group=useragent_group,
|
|
1880
|
+
conversation_id=conversation_id,
|
|
1881
|
+
client_ip=client_ip,
|
|
1882
|
+
)
|
|
1883
|
+
return
|
|
1884
|
+
|
|
1885
|
+
if pending_post_refresh_transient_penalties:
|
|
1886
|
+
post_refresh_transient_replacement_selected = True
|
|
1887
|
+
|
|
1888
|
+
account_id_value = account.id
|
|
1889
|
+
if last_pre_dispatch_transport_error is not None:
|
|
1890
|
+
# The preserved connect failure is only authoritative when
|
|
1891
|
+
# replacement selection is empty. Once another account is
|
|
1892
|
+
# actually attempted, its terminal outcome takes precedence.
|
|
1893
|
+
if last_transient_exc is last_pre_dispatch_transport_error:
|
|
1894
|
+
last_transient_exc = None
|
|
1895
|
+
last_pre_dispatch_transport_error = None
|
|
1896
|
+
if last_account_model_rejection is not None and account.id != last_account_model_rejection_account_id:
|
|
1897
|
+
# The original 400 is only the fallback when account
|
|
1898
|
+
# selection cannot produce a replacement. Once this
|
|
1899
|
+
# replacement attempt starts, its own failure is the one
|
|
1900
|
+
# that must reach the client. Keep the separate replay
|
|
1901
|
+
# budget so another account/model rejection cannot trigger
|
|
1902
|
+
# a second transparent replay.
|
|
1903
|
+
account_model_replacement_account_id = account.id
|
|
1904
|
+
last_account_model_rejection = None
|
|
1905
|
+
last_account_model_rejection_account_id = None
|
|
1906
|
+
if (
|
|
1907
|
+
require_preferred_account
|
|
1908
|
+
and preferred_account_id is not None
|
|
1909
|
+
and account.id != preferred_account_id
|
|
1910
|
+
):
|
|
1911
|
+
if verified_fresh_replay_payload is not None:
|
|
1912
|
+
payload = verified_fresh_replay_payload
|
|
1913
|
+
verified_fresh_replay_payload = None
|
|
1914
|
+
excluded_account_ids.add(preferred_account_id)
|
|
1915
|
+
preferred_account_id = None
|
|
1916
|
+
require_preferred_account = False
|
|
1917
|
+
affinity = replace(affinity, reallocate_sticky=True)
|
|
1918
|
+
logger.info(
|
|
1919
|
+
"cross_transport_verified_fresh_replay request_id=%s outcome=alternate_selected",
|
|
1920
|
+
request_id,
|
|
1921
|
+
)
|
|
1922
|
+
else:
|
|
1923
|
+
error_code = "previous_response_owner_unavailable"
|
|
1924
|
+
message = "Previous response owner account is unavailable; retry later."
|
|
1925
|
+
reason = "owner_account_unavailable"
|
|
1926
|
+
upstream_error_code = "upstream_unavailable"
|
|
1927
|
+
if selection.error_code == "continuity_owner_conflict":
|
|
1928
|
+
error_code = "continuity_owner_conflict"
|
|
1929
|
+
message = (
|
|
1930
|
+
selection.error_message
|
|
1931
|
+
or "Account-owned continuity sources conflict; retry the logical turn"
|
|
1932
|
+
)
|
|
1933
|
+
reason = "owner_conflict"
|
|
1934
|
+
upstream_error_code = selection.error_code
|
|
1935
|
+
_record_continuity_fail_closed(
|
|
1936
|
+
surface="http_stream",
|
|
1937
|
+
reason=reason,
|
|
1938
|
+
previous_response_id=payload.previous_response_id,
|
|
1939
|
+
session_id=headers.get("x-codex-turn-state") or headers.get("session_id"),
|
|
1940
|
+
upstream_error_code=upstream_error_code,
|
|
1941
|
+
)
|
|
1942
|
+
event = response_failed_event(
|
|
1943
|
+
error_code,
|
|
1944
|
+
message,
|
|
1945
|
+
response_id=request_id,
|
|
1946
|
+
)
|
|
1947
|
+
yield format_sse_event(event)
|
|
1948
|
+
await proxy._write_request_log(
|
|
1949
|
+
affinity_observation=affinity_observation,
|
|
1950
|
+
account_id=preferred_account_id,
|
|
1951
|
+
api_key=api_key,
|
|
1952
|
+
request_id=request_id,
|
|
1953
|
+
model=payload.model,
|
|
1954
|
+
latency_ms=int((clock.monotonic() - start) * 1000),
|
|
1955
|
+
status="error",
|
|
1956
|
+
error_code=error_code,
|
|
1957
|
+
error_message=message,
|
|
1958
|
+
reasoning_effort=payload.reasoning.effort if payload.reasoning else None,
|
|
1959
|
+
transport=request_transport,
|
|
1960
|
+
upstream_transport=upstream_stream_transport,
|
|
1961
|
+
service_tier=payload.service_tier,
|
|
1962
|
+
requested_service_tier=payload.service_tier,
|
|
1963
|
+
useragent=useragent,
|
|
1964
|
+
useragent_group=useragent_group,
|
|
1965
|
+
conversation_id=conversation_id,
|
|
1966
|
+
client_ip=client_ip,
|
|
1967
|
+
)
|
|
1968
|
+
return
|
|
1969
|
+
try:
|
|
1970
|
+
remaining_budget = proxy._remaining_budget_seconds(deadline)
|
|
1971
|
+
if remaining_budget <= 0:
|
|
1972
|
+
_facade().logger.warning(
|
|
1973
|
+
"Proxy request budget exhausted before freshness check "
|
|
1974
|
+
"request_id=%s attempt=%s account_id=%s",
|
|
1975
|
+
request_id,
|
|
1976
|
+
attempt + 1,
|
|
1977
|
+
account.id,
|
|
1978
|
+
)
|
|
1979
|
+
await proxy._write_stream_preflight_error(
|
|
1980
|
+
affinity_observation=affinity_observation,
|
|
1981
|
+
account_id=account.id,
|
|
1982
|
+
api_key=api_key,
|
|
1983
|
+
request_id=request_id,
|
|
1984
|
+
model=payload.model,
|
|
1985
|
+
start=start,
|
|
1986
|
+
error_code="upstream_request_timeout",
|
|
1987
|
+
error_message="Proxy request budget exhausted",
|
|
1988
|
+
reasoning_effort=payload.reasoning.effort if payload.reasoning else None,
|
|
1989
|
+
service_tier=payload.service_tier,
|
|
1990
|
+
transport=request_transport,
|
|
1991
|
+
upstream_transport=upstream_stream_transport,
|
|
1992
|
+
useragent=useragent,
|
|
1993
|
+
useragent_group=useragent_group,
|
|
1994
|
+
conversation_id=conversation_id,
|
|
1995
|
+
client_ip=client_ip,
|
|
1996
|
+
)
|
|
1997
|
+
if preserve_native_failure_lifecycle:
|
|
1998
|
+
return
|
|
1999
|
+
yield format_sse_event(_facade()._proxy_request_timeout_event(request_id))
|
|
2000
|
+
return
|
|
2001
|
+
try:
|
|
2002
|
+
account = await proxy._ensure_fresh_with_budget(account, timeout_seconds=remaining_budget)
|
|
2003
|
+
except UpstreamProxyRouteError as exc:
|
|
2004
|
+
message = f"Upstream proxy route unavailable: {exc.reason}"
|
|
2005
|
+
await proxy._write_stream_preflight_error(
|
|
2006
|
+
affinity_observation=affinity_observation,
|
|
2007
|
+
account_id=account.id,
|
|
2008
|
+
api_key=api_key,
|
|
2009
|
+
request_id=request_id,
|
|
2010
|
+
model=payload.model,
|
|
2011
|
+
start=start,
|
|
2012
|
+
error_code="upstream_proxy_unavailable",
|
|
2013
|
+
error_message=message,
|
|
2014
|
+
reasoning_effort=payload.reasoning.effort if payload.reasoning else None,
|
|
2015
|
+
service_tier=payload.service_tier,
|
|
2016
|
+
transport=request_transport,
|
|
2017
|
+
upstream_transport=upstream_stream_transport,
|
|
2018
|
+
upstream_proxy_fail_closed_reason=exc.reason,
|
|
2019
|
+
useragent=useragent,
|
|
2020
|
+
useragent_group=useragent_group,
|
|
2021
|
+
conversation_id=conversation_id,
|
|
2022
|
+
client_ip=client_ip,
|
|
2023
|
+
)
|
|
2024
|
+
event = response_failed_event(
|
|
2025
|
+
"upstream_proxy_unavailable",
|
|
2026
|
+
message,
|
|
2027
|
+
response_id=request_id,
|
|
2028
|
+
)
|
|
2029
|
+
yield format_sse_event(event)
|
|
2030
|
+
return
|
|
2031
|
+
except (RefreshError, aiohttp.ClientError, asyncio.TimeoutError) as exc:
|
|
2032
|
+
selected_account_model_replacement = account.id == account_model_replacement_account_id
|
|
2033
|
+
if isinstance(exc, RefreshError):
|
|
2034
|
+
if exc.is_permanent:
|
|
2035
|
+
await proxy._load_balancer.mark_permanent_failure(account, exc.code)
|
|
2036
|
+
# Keep the warning account routable for later
|
|
2037
|
+
# requests, but do not immediately reselect it in
|
|
2038
|
+
# this request after its refresh material failed.
|
|
2039
|
+
# Release its stream slot before failover too.
|
|
2040
|
+
await _release_tracked_stream_lease(current_account_lease)
|
|
2041
|
+
current_account_lease = None
|
|
2042
|
+
if not selected_account_model_replacement:
|
|
2043
|
+
excluded_account_ids.add(account.id)
|
|
2044
|
+
continue
|
|
2045
|
+
if is_transient_refresh_contention(exc):
|
|
2046
|
+
# Transient CROSS-REPLICA refresh contention: benign
|
|
2047
|
+
# claim contention (the account's refresh claim is
|
|
2048
|
+
# held by another replica) OR a post-exchange
|
|
2049
|
+
# persist/status CAS conflict. This is NOT a genuine
|
|
2050
|
+
# ``transport_error`` OAuth failure — the account's
|
|
2051
|
+
# credentials are healthy — so fail over WITHOUT an
|
|
2052
|
+
# account-health penalty (no ``_handle_stream_error``);
|
|
2053
|
+
# the genuine transport failure handled below keeps
|
|
2054
|
+
# its penalty. A post-exchange persist conflict is
|
|
2055
|
+
# logged distinctly (rarer, more-serious race).
|
|
2056
|
+
if refresh_contention_kind(exc) == "persist_conflict":
|
|
2057
|
+
logger.warning(
|
|
2058
|
+
"Stream freshness-check refresh post-exchange persist conflict "
|
|
2059
|
+
"code=%s account_id=%s",
|
|
2060
|
+
exc.code,
|
|
2061
|
+
account.id,
|
|
2062
|
+
)
|
|
2063
|
+
if (
|
|
2064
|
+
not selected_account_model_replacement
|
|
2065
|
+
and not require_preferred_account
|
|
2066
|
+
and preferred_account_id is None
|
|
2067
|
+
):
|
|
2068
|
+
# Movable request: release the stream lease and
|
|
2069
|
+
# fail over to a different account instead of
|
|
2070
|
+
# reselecting the same one until attempts are
|
|
2071
|
+
# exhausted. Record a retryable
|
|
2072
|
+
# upstream_unavailable so that if EVERY candidate
|
|
2073
|
+
# hits the held-claim condition, exhaustion
|
|
2074
|
+
# surfaces upstream_unavailable instead of a
|
|
2075
|
+
# misleading generic no_accounts response.
|
|
2076
|
+
await _release_tracked_stream_lease(current_account_lease)
|
|
2077
|
+
current_account_lease = None
|
|
2078
|
+
excluded_account_ids.add(account.id)
|
|
2079
|
+
last_retryable_stream_error = _RetryableStreamError(
|
|
2080
|
+
"upstream_unavailable",
|
|
2081
|
+
{
|
|
2082
|
+
"message": (
|
|
2083
|
+
exc.message
|
|
2084
|
+
or "Account refresh is temporarily unavailable; "
|
|
2085
|
+
"no healthy account could be reached."
|
|
2086
|
+
)
|
|
2087
|
+
},
|
|
2088
|
+
exclude_account=True,
|
|
2089
|
+
)
|
|
2090
|
+
continue
|
|
2091
|
+
# PINNED request (``previous_response_id`` /
|
|
2092
|
+
# ``input_file.file_id``): must not cross accounts.
|
|
2093
|
+
# Reselecting the same pinned account until attempts
|
|
2094
|
+
# are exhausted is pointless (it would leak the held
|
|
2095
|
+
# stream lease each iteration and then surface a
|
|
2096
|
+
# misleading ``no_accounts`` result). Stay on the
|
|
2097
|
+
# owner account, release the lease, and surface a
|
|
2098
|
+
# retryable ``upstream_unavailable`` promptly so the
|
|
2099
|
+
# client can retry once the claim clears.
|
|
2100
|
+
await _release_tracked_stream_lease(current_account_lease)
|
|
2101
|
+
current_account_lease = None
|
|
2102
|
+
message = exc.message or "Account refresh is temporarily unavailable; retry later."
|
|
2103
|
+
last_retryable_stream_error = _RetryableStreamError(
|
|
2104
|
+
"upstream_unavailable",
|
|
2105
|
+
{"message": message},
|
|
2106
|
+
)
|
|
2107
|
+
await proxy._write_stream_preflight_error(
|
|
2108
|
+
affinity_observation=affinity_observation,
|
|
2109
|
+
account_id=account.id,
|
|
2110
|
+
api_key=api_key,
|
|
2111
|
+
request_id=request_id,
|
|
2112
|
+
model=payload.model,
|
|
2113
|
+
start=start,
|
|
2114
|
+
error_code="upstream_unavailable",
|
|
2115
|
+
error_message=message,
|
|
2116
|
+
reasoning_effort=payload.reasoning.effort if payload.reasoning else None,
|
|
2117
|
+
service_tier=payload.service_tier,
|
|
2118
|
+
transport=request_transport,
|
|
2119
|
+
upstream_transport=upstream_stream_transport,
|
|
2120
|
+
useragent=useragent,
|
|
2121
|
+
useragent_group=useragent_group,
|
|
2122
|
+
conversation_id=conversation_id,
|
|
2123
|
+
client_ip=client_ip,
|
|
2124
|
+
)
|
|
2125
|
+
event = response_failed_event(
|
|
2126
|
+
"upstream_unavailable",
|
|
2127
|
+
message,
|
|
2128
|
+
response_id=request_id,
|
|
2129
|
+
)
|
|
2130
|
+
yield format_sse_event(event)
|
|
2131
|
+
return
|
|
2132
|
+
if not exc.transport_error and not selected_account_model_replacement:
|
|
2133
|
+
# Non-transport, non-permanent RefreshError: release
|
|
2134
|
+
# the stream lease and reselect (its prior behavior).
|
|
2135
|
+
await _release_tracked_stream_lease(current_account_lease)
|
|
2136
|
+
current_account_lease = None
|
|
2137
|
+
continue
|
|
2138
|
+
# A GENUINE OAuth transport failure
|
|
2139
|
+
# (``code == "transport_error"``): the account/route is
|
|
2140
|
+
# at fault, so it falls through to the shared
|
|
2141
|
+
# transport-failure handling below — identical to a raw
|
|
2142
|
+
# aiohttp/connect failure — which records the
|
|
2143
|
+
# account-health penalty (``_handle_stream_error``) so
|
|
2144
|
+
# the broken account backs off instead of being kept
|
|
2145
|
+
# healthy and reselected on the next request.
|
|
2146
|
+
_facade().logger.warning(
|
|
2147
|
+
"Stream refresh/connect failed request_id=%s attempt=%s account_id=%s",
|
|
2148
|
+
request_id,
|
|
2149
|
+
attempt + 1,
|
|
2150
|
+
account.id,
|
|
2151
|
+
exc_info=True,
|
|
2152
|
+
)
|
|
2153
|
+
message = getattr(exc, "message", None) or str(exc) or "Request to upstream timed out"
|
|
2154
|
+
if (
|
|
2155
|
+
not selected_account_model_replacement
|
|
2156
|
+
and _facade()._should_retry_transient_stream_error("upstream_unavailable", message)
|
|
2157
|
+
and attempt + 1 < max_attempts
|
|
2158
|
+
and _move_verified_fresh_replay_from_owner(
|
|
2159
|
+
account_id=account.id,
|
|
2160
|
+
outcome="owner_refresh_connect_failure",
|
|
2161
|
+
)
|
|
2162
|
+
):
|
|
2163
|
+
await _handle_or_defer_keyed_stream_health(
|
|
2164
|
+
account,
|
|
2165
|
+
{"message": message},
|
|
2166
|
+
"upstream_unavailable",
|
|
2167
|
+
)
|
|
2168
|
+
await _release_tracked_stream_lease(current_account_lease)
|
|
2169
|
+
current_account_lease = None
|
|
2170
|
+
last_retryable_stream_error = _RetryableStreamError(
|
|
2171
|
+
"upstream_unavailable",
|
|
2172
|
+
{"message": message},
|
|
2173
|
+
exclude_account=True,
|
|
2174
|
+
)
|
|
2175
|
+
continue
|
|
2176
|
+
if (
|
|
2177
|
+
not selected_account_model_replacement
|
|
2178
|
+
and not require_preferred_account
|
|
2179
|
+
and preferred_account_id is None
|
|
2180
|
+
and _facade()._should_retry_transient_stream_error("upstream_unavailable", message)
|
|
2181
|
+
and attempt + 1 < max_attempts
|
|
2182
|
+
):
|
|
2183
|
+
await _handle_or_defer_keyed_stream_health(
|
|
2184
|
+
account,
|
|
2185
|
+
{"message": message},
|
|
2186
|
+
"upstream_unavailable",
|
|
2187
|
+
)
|
|
2188
|
+
last_retryable_stream_error = _RetryableStreamError(
|
|
2189
|
+
"upstream_unavailable",
|
|
2190
|
+
{"message": message},
|
|
2191
|
+
exclude_account=True,
|
|
2192
|
+
)
|
|
2193
|
+
# The account keeps its health penalty above and is now
|
|
2194
|
+
# excluded from reselection, but its stream-concurrency
|
|
2195
|
+
# slot is still occupied by the lease appended at
|
|
2196
|
+
# selection. Release it before the failover ``continue``
|
|
2197
|
+
# (matching the claim-contention and permanent branches)
|
|
2198
|
+
# so the dead account's slot is freed immediately instead
|
|
2199
|
+
# of being held for the entire duration of the
|
|
2200
|
+
# replacement stream.
|
|
2201
|
+
await _release_tracked_stream_lease(current_account_lease)
|
|
2202
|
+
current_account_lease = None
|
|
2203
|
+
excluded_account_ids.add(account.id)
|
|
2204
|
+
continue
|
|
2205
|
+
await proxy._write_stream_preflight_error(
|
|
2206
|
+
affinity_observation=affinity_observation,
|
|
2207
|
+
account_id=account.id,
|
|
2208
|
+
api_key=api_key,
|
|
2209
|
+
request_id=request_id,
|
|
2210
|
+
model=payload.model,
|
|
2211
|
+
start=start,
|
|
2212
|
+
error_code="upstream_unavailable",
|
|
2213
|
+
error_message=message,
|
|
2214
|
+
reasoning_effort=payload.reasoning.effort if payload.reasoning else None,
|
|
2215
|
+
service_tier=payload.service_tier,
|
|
2216
|
+
transport=request_transport,
|
|
2217
|
+
upstream_transport=upstream_stream_transport,
|
|
2218
|
+
useragent=useragent,
|
|
2219
|
+
useragent_group=useragent_group,
|
|
2220
|
+
conversation_id=conversation_id,
|
|
2221
|
+
client_ip=client_ip,
|
|
2222
|
+
)
|
|
2223
|
+
event = response_failed_event(
|
|
2224
|
+
"upstream_unavailable",
|
|
2225
|
+
message,
|
|
2226
|
+
response_id=request_id,
|
|
2227
|
+
)
|
|
2228
|
+
yield format_sse_event(event)
|
|
2229
|
+
return
|
|
2230
|
+
any_attempt_logged = True
|
|
2231
|
+
settlement = _StreamSettlement()
|
|
2232
|
+
tool_call_dedupe = _WebSocketUpstreamControl()
|
|
2233
|
+
effective_attempt_timeout = proxy._remaining_budget_seconds(deadline)
|
|
2234
|
+
if effective_attempt_timeout <= 0:
|
|
2235
|
+
_facade().logger.warning(
|
|
2236
|
+
"Proxy request budget exhausted before stream attempt "
|
|
2237
|
+
"request_id=%s attempt=%s account_id=%s",
|
|
2238
|
+
request_id,
|
|
2239
|
+
attempt + 1,
|
|
2240
|
+
account.id,
|
|
2241
|
+
)
|
|
2242
|
+
await proxy._write_stream_preflight_error(
|
|
2243
|
+
affinity_observation=affinity_observation,
|
|
2244
|
+
account_id=account.id,
|
|
2245
|
+
api_key=api_key,
|
|
2246
|
+
request_id=request_id,
|
|
2247
|
+
model=payload.model,
|
|
2248
|
+
start=start,
|
|
2249
|
+
error_code="upstream_request_timeout",
|
|
2250
|
+
error_message="Proxy request budget exhausted",
|
|
2251
|
+
reasoning_effort=payload.reasoning.effort if payload.reasoning else None,
|
|
2252
|
+
service_tier=payload.service_tier,
|
|
2253
|
+
transport=request_transport,
|
|
2254
|
+
upstream_transport=upstream_stream_transport,
|
|
2255
|
+
useragent=useragent,
|
|
2256
|
+
useragent_group=useragent_group,
|
|
2257
|
+
conversation_id=conversation_id,
|
|
2258
|
+
client_ip=client_ip,
|
|
2259
|
+
)
|
|
2260
|
+
if preserve_native_failure_lifecycle:
|
|
2261
|
+
return
|
|
2262
|
+
yield format_sse_event(_facade()._proxy_request_timeout_event(request_id))
|
|
2263
|
+
return
|
|
2264
|
+
transient_retries = 0
|
|
2265
|
+
allow_retry_flag = attempt < max_attempts - 1
|
|
2266
|
+
while True:
|
|
2267
|
+
set_resilience_toggles(resilience) # rebind per attempt (task handoff)
|
|
2268
|
+
stream_timeout_tokens = _facade()._push_stream_attempt_timeout_overrides(
|
|
2269
|
+
proxy._remaining_budget_seconds(deadline),
|
|
2270
|
+
)
|
|
2271
|
+
try:
|
|
2272
|
+
settlement = _StreamSettlement()
|
|
2273
|
+
register_payload_owner = _authorize_payload_dispatch(account)
|
|
2274
|
+
inner_stream = proxy._stream_once(
|
|
2275
|
+
account,
|
|
2276
|
+
payload,
|
|
2277
|
+
headers,
|
|
2278
|
+
request_id,
|
|
2279
|
+
allow_retry_flag,
|
|
2280
|
+
request_started_at=start,
|
|
2281
|
+
affinity_observation=affinity_observation,
|
|
2282
|
+
allow_transient_retry=(
|
|
2283
|
+
transient_retries < _facade()._MAX_TRANSIENT_SAME_ACCOUNT_RETRIES - 1
|
|
2284
|
+
or allow_retry_flag
|
|
2285
|
+
),
|
|
2286
|
+
api_key=api_key,
|
|
2287
|
+
api_key_reservation=api_key_reservation,
|
|
2288
|
+
settlement=settlement,
|
|
2289
|
+
suppress_text_done_events=suppress_text_done_events,
|
|
2290
|
+
upstream_stream_transport=upstream_stream_transport,
|
|
2291
|
+
request_transport=request_transport,
|
|
2292
|
+
concurrency_caps=concurrency_caps,
|
|
2293
|
+
useragent=useragent,
|
|
2294
|
+
useragent_group=useragent_group,
|
|
2295
|
+
conversation_id=conversation_id,
|
|
2296
|
+
client_ip=client_ip,
|
|
2297
|
+
# Let the retry path observe a pre-visible
|
|
2298
|
+
# account-recovery error only for the one case
|
|
2299
|
+
# where this owner anchor has a locally
|
|
2300
|
+
# verified, unanchored replacement body. All
|
|
2301
|
+
# other continuations retain the normal
|
|
2302
|
+
# fail-closed owner-error rewrite.
|
|
2303
|
+
preferred_account_id=(
|
|
2304
|
+
None
|
|
2305
|
+
if (
|
|
2306
|
+
require_preferred_account
|
|
2307
|
+
and preferred_account_id == account.id
|
|
2308
|
+
and verified_fresh_replay_payload is not None
|
|
2309
|
+
)
|
|
2310
|
+
else preferred_account_id
|
|
2311
|
+
),
|
|
2312
|
+
tool_call_dedupe=tool_call_dedupe,
|
|
2313
|
+
enforce_openai_sdk_contract=enforce_openai_sdk_contract,
|
|
2314
|
+
thread_cache_identity=ThreadCacheIdentity(thread_cache_identity_mode, account.id),
|
|
2315
|
+
)
|
|
2316
|
+
try:
|
|
2317
|
+
try:
|
|
2318
|
+
async for line in inner_stream:
|
|
2319
|
+
if register_payload_owner:
|
|
2320
|
+
payload_replay_required_account_id = account.id
|
|
2321
|
+
register_payload_owner = False
|
|
2322
|
+
yield line
|
|
2323
|
+
if register_payload_owner:
|
|
2324
|
+
payload_replay_required_account_id = account.id
|
|
2325
|
+
except BaseException as exc:
|
|
2326
|
+
if register_payload_owner and not (
|
|
2327
|
+
isinstance(exc, ProxyResponseError)
|
|
2328
|
+
and is_confirmed_pre_dispatch_transport_error(exc)
|
|
2329
|
+
):
|
|
2330
|
+
payload_replay_required_account_id = account.id
|
|
2331
|
+
raise
|
|
2332
|
+
finally:
|
|
2333
|
+
close_task = scheduler.create_task(
|
|
2334
|
+
inner_stream.aclose(),
|
|
2335
|
+
name=f"stream-inner-close-{request_id}",
|
|
2336
|
+
)
|
|
2337
|
+
_, close_cancellation = await _await_task_deferring_cancellation(close_task)
|
|
2338
|
+
if close_cancellation is not None:
|
|
2339
|
+
raise close_cancellation
|
|
2340
|
+
except (asyncio.CancelledError, GeneratorExit):
|
|
2341
|
+
# A terminal frame may already have been yielded when
|
|
2342
|
+
# downstream cancellation is delivered on the next
|
|
2343
|
+
# generator resume. Finalize that terminal usage and
|
|
2344
|
+
# health result before propagating cancellation so the
|
|
2345
|
+
# reservation is not released as abandoned.
|
|
2346
|
+
if settlement.status in {"success", "error"} and not settled:
|
|
2347
|
+
await _finalize_terminal_settlement_after_downstream_close(settlement, account)
|
|
2348
|
+
raise
|
|
2349
|
+
except (_TransientStreamError, ProxyResponseError) as tex:
|
|
2350
|
+
if isinstance(tex, ProxyResponseError) and _must_abort_native_transport_failure(
|
|
2351
|
+
tex,
|
|
2352
|
+
downstream_visible=settlement.downstream_visible,
|
|
2353
|
+
):
|
|
2354
|
+
raise
|
|
2355
|
+
if account.id == account_model_replacement_account_id:
|
|
2356
|
+
# Account/model routing gets exactly one selected
|
|
2357
|
+
# replacement. Its own pre-visible 5xx/transport
|
|
2358
|
+
# failure is terminal; allowing the normal
|
|
2359
|
+
# transient path here would silently select a third
|
|
2360
|
+
# account. Re-raise into the outer terminal
|
|
2361
|
+
# renderer to retain the replacement details.
|
|
2362
|
+
if isinstance(tex, ProxyResponseError):
|
|
2363
|
+
raise
|
|
2364
|
+
raise ProxyResponseError(
|
|
2365
|
+
502,
|
|
2366
|
+
openai_error(
|
|
2367
|
+
tex.code,
|
|
2368
|
+
str(tex.error.get("message") or "Upstream error"),
|
|
2369
|
+
error_type="server_error",
|
|
2370
|
+
),
|
|
2371
|
+
) from tex
|
|
2372
|
+
if settlement.downstream_visible:
|
|
2373
|
+
failed_response_id = settlement.response_id or request_id
|
|
2374
|
+
if isinstance(tex, ProxyResponseError):
|
|
2375
|
+
error = _parse_openai_error(tex.payload)
|
|
2376
|
+
error_code = _normalize_error_code(
|
|
2377
|
+
error.code if error else None,
|
|
2378
|
+
error.type if error else None,
|
|
2379
|
+
)
|
|
2380
|
+
error_message = error.message if error else "Upstream error"
|
|
2381
|
+
error_type = error.type if error else None
|
|
2382
|
+
error_param = error.param_state if error else None
|
|
2383
|
+
event = response_failed_event(
|
|
2384
|
+
error_code or "upstream_error",
|
|
2385
|
+
error_message or "Upstream error",
|
|
2386
|
+
error_type=error_type or "server_error",
|
|
2387
|
+
response_id=failed_response_id,
|
|
2388
|
+
error_param=error_param,
|
|
2389
|
+
)
|
|
2390
|
+
_apply_error_metadata(event["response"]["error"], error)
|
|
2391
|
+
else:
|
|
2392
|
+
error_code = tex.code
|
|
2393
|
+
error_message = str(tex.error.get("message") or "Upstream error")
|
|
2394
|
+
event = response_failed_event(
|
|
2395
|
+
error_code or "upstream_error",
|
|
2396
|
+
error_message,
|
|
2397
|
+
response_id=failed_response_id,
|
|
2398
|
+
)
|
|
2399
|
+
if isinstance(tex, ProxyResponseError):
|
|
2400
|
+
# Downstream visibility forbids replay, but a
|
|
2401
|
+
# concrete shared HTTP/WebSocket generation
|
|
2402
|
+
# carrying a process-network failure must still
|
|
2403
|
+
# be retired before later callers lease it.
|
|
2404
|
+
await _wait_for_process_network_recovery(
|
|
2405
|
+
account,
|
|
2406
|
+
error_code=error_code,
|
|
2407
|
+
retryable_same_contract=False,
|
|
2408
|
+
failed_session=tex.failed_session,
|
|
2409
|
+
)
|
|
2410
|
+
_facade().logger.warning(
|
|
2411
|
+
"Surfacing mid-stream upstream failure without replay "
|
|
2412
|
+
"request_id=%s account_id=%s code=%s",
|
|
2413
|
+
request_id,
|
|
2414
|
+
account.id,
|
|
2415
|
+
error_code,
|
|
2416
|
+
)
|
|
2417
|
+
settlement.record_success = False
|
|
2418
|
+
settlement.error_code = error_code
|
|
2419
|
+
settlement.error_message = error_message
|
|
2420
|
+
if isinstance(tex, ProxyResponseError):
|
|
2421
|
+
settlement.error = _upstream_error_from_openai(error)
|
|
2422
|
+
else:
|
|
2423
|
+
settlement.error = tex.error
|
|
2424
|
+
settlement.account_health_error = _facade()._should_penalize_stream_error(
|
|
2425
|
+
error_code, error_message
|
|
2426
|
+
)
|
|
2427
|
+
transient_upstream_http_status = (
|
|
2428
|
+
tex.status_code if isinstance(tex, ProxyResponseError) else None
|
|
2429
|
+
)
|
|
2430
|
+
if not (
|
|
2431
|
+
preserve_native_failure_lifecycle
|
|
2432
|
+
and error_code in SYNTHETIC_TRANSPORT_FAILURE_CODES
|
|
2433
|
+
):
|
|
2434
|
+
try:
|
|
2435
|
+
yield format_sse_event(event)
|
|
2436
|
+
except (asyncio.CancelledError, GeneratorExit):
|
|
2437
|
+
await _finalize_terminal_settlement_after_downstream_close(
|
|
2438
|
+
settlement,
|
|
2439
|
+
account,
|
|
2440
|
+
upstream_http_status=transient_upstream_http_status,
|
|
2441
|
+
)
|
|
2442
|
+
raise
|
|
2443
|
+
settled = await _settle_stream_usage_before_pending_penalty(settlement)
|
|
2444
|
+
if settled and settlement.account_health_error:
|
|
2445
|
+
await proxy._handle_stream_error(
|
|
2446
|
+
account,
|
|
2447
|
+
_stream_settlement_error_payload(settlement),
|
|
2448
|
+
settlement.error_code or "upstream_error",
|
|
2449
|
+
# Evidence only. ``_TransientStreamError``
|
|
2450
|
+
# carries no status, which is correct:
|
|
2451
|
+
# those are the genuinely status-less
|
|
2452
|
+
# transients.
|
|
2453
|
+
upstream_http_status=transient_upstream_http_status,
|
|
2454
|
+
)
|
|
2455
|
+
return
|
|
2456
|
+
if isinstance(tex, ProxyResponseError) and tex.status_code != 500:
|
|
2457
|
+
error = _parse_openai_error(tex.payload)
|
|
2458
|
+
code = _normalize_error_code(
|
|
2459
|
+
error.code if error else None,
|
|
2460
|
+
error.type if error else None,
|
|
2461
|
+
)
|
|
2462
|
+
error_message = error.message if error else None
|
|
2463
|
+
account_model_retry = await _retry_account_model_rejection(
|
|
2464
|
+
tex,
|
|
2465
|
+
account,
|
|
2466
|
+
outcome="previsible",
|
|
2467
|
+
)
|
|
2468
|
+
if account_model_retry is not None:
|
|
2469
|
+
if not account_model_retry:
|
|
2470
|
+
raise
|
|
2471
|
+
# Leaving the same-account loop reaches
|
|
2472
|
+
# the outer account-selection ``continue``
|
|
2473
|
+
# below, where the rejected account is
|
|
2474
|
+
# already excluded by the helper.
|
|
2475
|
+
break
|
|
2476
|
+
if _facade()._is_security_work_authorization_required_error(code, error_message):
|
|
2477
|
+
if (
|
|
2478
|
+
account.security_work_authorized
|
|
2479
|
+
or account.id == file_preferred_account_id
|
|
2480
|
+
or require_preferred_account
|
|
2481
|
+
or attempt >= max_attempts - 1
|
|
2482
|
+
):
|
|
2483
|
+
raise
|
|
2484
|
+
_facade().logger.info(
|
|
2485
|
+
"Retrying on security-work-authorized account request_id=%s account_id=%s",
|
|
2486
|
+
request_id,
|
|
2487
|
+
account.id,
|
|
2488
|
+
)
|
|
2489
|
+
yield format_sse_event(
|
|
2490
|
+
_facade()._security_work_advisory_event(
|
|
2491
|
+
code=_facade()._SECURITY_WORK_AUTHORIZATION_REQUIRED_CODE,
|
|
2492
|
+
message=_facade()._SECURITY_WORK_RETRY_MESSAGE,
|
|
2493
|
+
request_id=request_id,
|
|
2494
|
+
action="retry_security_work_authorized",
|
|
2495
|
+
account_id=account.id,
|
|
2496
|
+
)
|
|
2497
|
+
)
|
|
2498
|
+
await _release_tracked_stream_lease(current_account_lease)
|
|
2499
|
+
current_account_lease = None
|
|
2500
|
+
excluded_account_ids.add(account.id)
|
|
2501
|
+
require_security_work_authorized = True
|
|
2502
|
+
last_security_work_retry_error = _RetryableStreamError(
|
|
2503
|
+
_facade()._SECURITY_WORK_AUTHORIZATION_REQUIRED_CODE,
|
|
2504
|
+
_upstream_error_from_openai(error),
|
|
2505
|
+
)
|
|
2506
|
+
last_transient_exc = tex
|
|
2507
|
+
break
|
|
2508
|
+
if code == "account_response_create_cap":
|
|
2509
|
+
last_transient_exc = tex
|
|
2510
|
+
recovery_sleep_seconds = _account_selection_recovery_sleep_seconds(
|
|
2511
|
+
AccountSelection(
|
|
2512
|
+
account=None,
|
|
2513
|
+
error_message=error_message,
|
|
2514
|
+
error_code=code,
|
|
2515
|
+
)
|
|
2516
|
+
)
|
|
2517
|
+
if recovery_sleep_seconds is not None:
|
|
2518
|
+
can_try_other_account = (
|
|
2519
|
+
not require_preferred_account
|
|
2520
|
+
and account.id != file_preferred_account_id
|
|
2521
|
+
and attempt < max_attempts - 1
|
|
2522
|
+
)
|
|
2523
|
+
if can_try_other_account:
|
|
2524
|
+
deferred_capacity_account = account
|
|
2525
|
+
deferred_capacity_lease = current_account_lease
|
|
2526
|
+
excluded_account_ids.add(account.id)
|
|
2527
|
+
break
|
|
2528
|
+
remaining_budget_seconds = proxy._remaining_budget_seconds(deadline)
|
|
2529
|
+
if remaining_budget_seconds <= 0:
|
|
2530
|
+
raise
|
|
2531
|
+
async for wait_event in _iter_account_capacity_recovery_wait(
|
|
2532
|
+
request_id=request_id,
|
|
2533
|
+
model=payload.model,
|
|
2534
|
+
account_id=account.id,
|
|
2535
|
+
error_message=error_message,
|
|
2536
|
+
recovery_sleep_seconds=recovery_sleep_seconds,
|
|
2537
|
+
remaining_budget_seconds=remaining_budget_seconds,
|
|
2538
|
+
emit_keepalives=not propagate_http_errors
|
|
2539
|
+
or not enforce_openai_sdk_contract,
|
|
2540
|
+
stage="response_create",
|
|
2541
|
+
scheduler=scheduler,
|
|
2542
|
+
clock=clock,
|
|
2543
|
+
):
|
|
2544
|
+
yield wait_event
|
|
2545
|
+
if proxy._remaining_budget_seconds(deadline) <= 0:
|
|
2546
|
+
raise
|
|
2547
|
+
continue
|
|
2548
|
+
last_transient_exc = tex
|
|
2549
|
+
await _release_tracked_stream_lease(current_account_lease)
|
|
2550
|
+
current_account_lease = None
|
|
2551
|
+
excluded_account_ids.add(account.id)
|
|
2552
|
+
break
|
|
2553
|
+
recovery_decision = await _wait_for_process_network_recovery(
|
|
2554
|
+
account,
|
|
2555
|
+
error_code=code,
|
|
2556
|
+
retryable_same_contract=tex.retryable_same_contract,
|
|
2557
|
+
failed_session=tex.failed_session,
|
|
2558
|
+
)
|
|
2559
|
+
if recovery_decision == "retry":
|
|
2560
|
+
continue
|
|
2561
|
+
if recovery_decision == "exhausted":
|
|
2562
|
+
_facade()._raise_proxy_budget_exhausted()
|
|
2563
|
+
if _facade()._is_account_neutral_error_code(code):
|
|
2564
|
+
raise
|
|
2565
|
+
if is_confirmed_pre_dispatch_transport_error(tex):
|
|
2566
|
+
# The transport proved the request never
|
|
2567
|
+
# dispatched: this account's proxy route is
|
|
2568
|
+
# dead. Release the account's stream lease
|
|
2569
|
+
# before recording health so its slot never
|
|
2570
|
+
# outlives the failed route, then jump
|
|
2571
|
+
# straight to the bounded transient backoff
|
|
2572
|
+
# floor so independent requests stop
|
|
2573
|
+
# rediscovering the dead route one generic
|
|
2574
|
+
# error at a time.
|
|
2575
|
+
await _release_tracked_stream_lease(current_account_lease)
|
|
2576
|
+
current_account_lease = None
|
|
2577
|
+
await _record_or_defer_confirmed_route_backoff(account)
|
|
2578
|
+
# A confirmed pre-dispatch connect failure
|
|
2579
|
+
# guarantees zero upstream bytes, so a
|
|
2580
|
+
# locally verified full-resend replay is
|
|
2581
|
+
# safe to move off the dead owner before
|
|
2582
|
+
# the hard-ownership raise (same file/
|
|
2583
|
+
# turn-state/single-account guards as the
|
|
2584
|
+
# post-visible failover path below).
|
|
2585
|
+
verified_owner_replay_moved = False
|
|
2586
|
+
if (
|
|
2587
|
+
attempt < max_attempts - 1
|
|
2588
|
+
and routing_strategy != "single_account"
|
|
2589
|
+
and file_preferred_account_id is None
|
|
2590
|
+
and turn_state_owner_account_id is None
|
|
2591
|
+
):
|
|
2592
|
+
verified_owner_replay_moved = _move_verified_fresh_replay_from_owner(
|
|
2593
|
+
account_id=account.id,
|
|
2594
|
+
outcome="owner_pre_dispatch_proxy_connect_failure",
|
|
2595
|
+
)
|
|
2596
|
+
can_try_other_account = verified_owner_replay_moved or (
|
|
2597
|
+
not require_preferred_account
|
|
2598
|
+
and account.id != file_preferred_account_id
|
|
2599
|
+
and attempt < max_attempts - 1
|
|
2600
|
+
)
|
|
2601
|
+
if not can_try_other_account:
|
|
2602
|
+
# Hard account ownership or exhausted
|
|
2603
|
+
# attempts: fail closed on the original
|
|
2604
|
+
# sanitized failure without crossing
|
|
2605
|
+
# accounts.
|
|
2606
|
+
raise
|
|
2607
|
+
last_transient_exc = tex
|
|
2608
|
+
last_pre_dispatch_transport_error = tex
|
|
2609
|
+
transient_failed_account_id = account.id
|
|
2610
|
+
excluded_account_ids.add(account.id)
|
|
2611
|
+
if not verified_owner_replay_moved:
|
|
2612
|
+
affinity = replace(affinity, reallocate_sticky=True)
|
|
2613
|
+
_facade().logger.info(
|
|
2614
|
+
"Retrying stream after confirmed pre-dispatch proxy connect failure "
|
|
2615
|
+
"request_id=%s account_id=%s attempt=%d",
|
|
2616
|
+
request_id,
|
|
2617
|
+
account.id,
|
|
2618
|
+
attempt + 1,
|
|
2619
|
+
)
|
|
2620
|
+
break
|
|
2621
|
+
classified = classify_upstream_failure(
|
|
2622
|
+
error_code=code,
|
|
2623
|
+
error=_upstream_error_from_openai(error),
|
|
2624
|
+
http_status=tex.status_code,
|
|
2625
|
+
phase="first_event",
|
|
2626
|
+
)
|
|
2627
|
+
burst = is_upstream_burst_rejection(
|
|
2628
|
+
failure_class=classified["failure_class"],
|
|
2629
|
+
http_status=tex.status_code,
|
|
2630
|
+
)
|
|
2631
|
+
if resilience.deterministic_failover_enabled:
|
|
2632
|
+
action = failover_decision(
|
|
2633
|
+
failure_class=classified["failure_class"],
|
|
2634
|
+
downstream_visible=settlement.downstream_visible,
|
|
2635
|
+
candidates_remaining=max_attempts - attempt - 1,
|
|
2636
|
+
owner_bound=_stream_owner_bound_to(account),
|
|
2637
|
+
same_account_retry_available=_burst_same_account_retry_available(
|
|
2638
|
+
account, burst=burst
|
|
2639
|
+
),
|
|
2640
|
+
)
|
|
2641
|
+
else:
|
|
2642
|
+
action = "surface"
|
|
2643
|
+
_facade().logger.info(
|
|
2644
|
+
"Failover decision request_id=%s transport=stream account_id=%s "
|
|
2645
|
+
"attempt=%d failure_class=%s action=%s",
|
|
2646
|
+
request_id,
|
|
2647
|
+
account.id,
|
|
2648
|
+
attempt + 1,
|
|
2649
|
+
classified["failure_class"],
|
|
2650
|
+
action,
|
|
2651
|
+
)
|
|
2652
|
+
if action == "retry_same_account":
|
|
2653
|
+
# Owner-bound burst 429: the request cannot
|
|
2654
|
+
# move, so back off briefly on the owner and
|
|
2655
|
+
# redispatch. Only the replica-local burst
|
|
2656
|
+
# cooldown is engaged per retry (steering
|
|
2657
|
+
# *other* unbound requests away at rejection
|
|
2658
|
+
# time, keyed or not); the transient error
|
|
2659
|
+
# penalty is written once, when the failure
|
|
2660
|
+
# is finally surfaced, so a bursting owner
|
|
2661
|
+
# is not pushed into selection error backoff
|
|
2662
|
+
# and a success is never followed by a
|
|
2663
|
+
# deferred penalty. The account is neither
|
|
2664
|
+
# excluded nor released from its dispatch
|
|
2665
|
+
# anchor, and the stream lease is kept so the
|
|
2666
|
+
# redispatch stays inside the admitted slot.
|
|
2667
|
+
await record_upstream_burst_rejection(
|
|
2668
|
+
proxy._load_balancer,
|
|
2669
|
+
account,
|
|
2670
|
+
retry_after_seconds=tex.retry_after_seconds,
|
|
2671
|
+
)
|
|
2672
|
+
retry_index = _next_burst_same_account_retry(account)
|
|
2673
|
+
delay = burst_same_account_backoff_seconds(
|
|
2674
|
+
retry_index,
|
|
2675
|
+
retry_after_seconds=tex.retry_after_seconds,
|
|
2676
|
+
)
|
|
2677
|
+
_facade().logger.info(
|
|
2678
|
+
"Burst 429 on owner-bound stream, retrying same account "
|
|
2679
|
+
"request_id=%s account_id=%s retry=%s/%s delay=%.2fs retry_after_seconds=%s",
|
|
2680
|
+
request_id,
|
|
2681
|
+
account.id,
|
|
2682
|
+
retry_index,
|
|
2683
|
+
BURST_SAME_ACCOUNT_MAX_RETRIES,
|
|
2684
|
+
delay,
|
|
2685
|
+
tex.retry_after_seconds,
|
|
2686
|
+
)
|
|
2687
|
+
async for wait_event in _iter_account_capacity_recovery_wait(
|
|
2688
|
+
request_id=request_id,
|
|
2689
|
+
model=payload.model,
|
|
2690
|
+
account_id=account.id,
|
|
2691
|
+
error_message=error_message,
|
|
2692
|
+
recovery_sleep_seconds=delay,
|
|
2693
|
+
remaining_budget_seconds=proxy._remaining_budget_seconds(deadline),
|
|
2694
|
+
# The wait is bounded (<= 10 s) so no
|
|
2695
|
+
# keepalive is needed; on the propagating
|
|
2696
|
+
# route a keepalive frame would commit
|
|
2697
|
+
# 200/SSE and make the surfaced 429
|
|
2698
|
+
# unreachable as an HTTP status.
|
|
2699
|
+
emit_keepalives=not propagate_http_errors,
|
|
2700
|
+
stage="burst_backoff",
|
|
2701
|
+
scheduler=scheduler,
|
|
2702
|
+
clock=clock,
|
|
2703
|
+
):
|
|
2704
|
+
yield wait_event
|
|
2705
|
+
if proxy._remaining_budget_seconds(deadline) > 0:
|
|
2706
|
+
continue
|
|
2707
|
+
# Budget spent during the wait: surface the
|
|
2708
|
+
# original rejection below (one health write).
|
|
2709
|
+
if action == "failover_next":
|
|
2710
|
+
await _handle_or_defer_keyed_stream_health(
|
|
2711
|
+
account,
|
|
2712
|
+
_upstream_error_from_openai(error),
|
|
2713
|
+
code,
|
|
2714
|
+
http_status=tex.status_code,
|
|
2715
|
+
retry_after_seconds=tex.retry_after_seconds,
|
|
2716
|
+
)
|
|
2717
|
+
last_transient_exc = tex
|
|
2718
|
+
transient_failed_account_id = account.id
|
|
2719
|
+
await _release_tracked_stream_lease(current_account_lease)
|
|
2720
|
+
current_account_lease = None
|
|
2721
|
+
excluded_account_ids.add(account.id)
|
|
2722
|
+
_move_verified_fresh_replay_from_owner(
|
|
2723
|
+
account_id=account.id,
|
|
2724
|
+
outcome="owner_previsible_failure",
|
|
2725
|
+
)
|
|
2726
|
+
break
|
|
2727
|
+
# Workspace failures are terminal for this
|
|
2728
|
+
# owner; defer keyed health until settlement.
|
|
2729
|
+
# Preserve the existing policy for other codes.
|
|
2730
|
+
if classified["failure_class"] == "account_unavailable":
|
|
2731
|
+
await _handle_or_defer_keyed_stream_health(
|
|
2732
|
+
account,
|
|
2733
|
+
_upstream_error_from_openai(error),
|
|
2734
|
+
code,
|
|
2735
|
+
http_status=tex.status_code,
|
|
2736
|
+
retry_after_seconds=tex.retry_after_seconds,
|
|
2737
|
+
)
|
|
2738
|
+
else:
|
|
2739
|
+
await proxy._handle_stream_error(
|
|
2740
|
+
account,
|
|
2741
|
+
_upstream_error_from_openai(error),
|
|
2742
|
+
code,
|
|
2743
|
+
http_status=tex.status_code,
|
|
2744
|
+
**_retry_after_kwargs(tex.retry_after_seconds),
|
|
2745
|
+
)
|
|
2746
|
+
setattr(tex, _STREAM_HEALTH_RECORDED_ATTR, True)
|
|
2747
|
+
if burst:
|
|
2748
|
+
_stamp_surfaced_burst_retry_after(tex)
|
|
2749
|
+
raise
|
|
2750
|
+
error_payload: UpstreamError = (
|
|
2751
|
+
tex.error
|
|
2752
|
+
if isinstance(tex, _TransientStreamError)
|
|
2753
|
+
else _upstream_error_from_openai(_parse_openai_error(tex.payload))
|
|
2754
|
+
)
|
|
2755
|
+
error_code = _transient_retry_error_code(tex)
|
|
2756
|
+
error_message = str(error_payload.get("message") or "")
|
|
2757
|
+
recovery_decision = await _wait_for_process_network_recovery(
|
|
2758
|
+
account,
|
|
2759
|
+
error_code=error_code,
|
|
2760
|
+
retryable_same_contract=(
|
|
2761
|
+
isinstance(tex, ProxyResponseError) and tex.retryable_same_contract
|
|
2762
|
+
),
|
|
2763
|
+
failed_session=tex.failed_session if isinstance(tex, ProxyResponseError) else None,
|
|
2764
|
+
)
|
|
2765
|
+
if recovery_decision == "retry":
|
|
2766
|
+
continue
|
|
2767
|
+
if recovery_decision == "exhausted":
|
|
2768
|
+
await _settle_process_network_budget_exhaustion(account, settlement)
|
|
2769
|
+
if preserve_native_failure_lifecycle:
|
|
2770
|
+
return
|
|
2771
|
+
yield format_sse_event(_facade()._proxy_request_timeout_event(request_id))
|
|
2772
|
+
return
|
|
2773
|
+
transient_retries += 1
|
|
2774
|
+
if (
|
|
2775
|
+
transient_retries < _facade()._MAX_TRANSIENT_SAME_ACCOUNT_RETRIES
|
|
2776
|
+
and proxy._remaining_budget_seconds(deadline) > 0
|
|
2777
|
+
and not settlement.downstream_visible
|
|
2778
|
+
):
|
|
2779
|
+
delay = backoff_seconds(transient_retries)
|
|
2780
|
+
_facade().logger.info(
|
|
2781
|
+
"Transient stream error, retrying same account "
|
|
2782
|
+
"request_id=%s account_id=%s retry=%s/%s delay=%.2fs code=%s",
|
|
2783
|
+
request_id,
|
|
2784
|
+
account.id,
|
|
2785
|
+
transient_retries,
|
|
2786
|
+
_facade()._MAX_TRANSIENT_SAME_ACCOUNT_RETRIES,
|
|
2787
|
+
delay,
|
|
2788
|
+
error_code,
|
|
2789
|
+
)
|
|
2790
|
+
await scheduler.sleep(delay)
|
|
2791
|
+
continue # inner loop: retry same account
|
|
2792
|
+
# Exhausted same-account retries — penalize and failover
|
|
2793
|
+
_facade().logger.warning(
|
|
2794
|
+
"Transient retries exhausted for account "
|
|
2795
|
+
"request_id=%s account_id=%s retries=%s code=%s",
|
|
2796
|
+
request_id,
|
|
2797
|
+
account.id,
|
|
2798
|
+
transient_retries,
|
|
2799
|
+
error_code,
|
|
2800
|
+
)
|
|
2801
|
+
await _handle_or_defer_keyed_stream_health(
|
|
2802
|
+
account,
|
|
2803
|
+
error_payload,
|
|
2804
|
+
error_code,
|
|
2805
|
+
http_status=(tex.status_code if isinstance(tex, ProxyResponseError) else None),
|
|
2806
|
+
transient_retry_count=transient_retries,
|
|
2807
|
+
)
|
|
2808
|
+
# Preserve last ProxyResponseError for propagate_http_errors path.
|
|
2809
|
+
if isinstance(tex, ProxyResponseError):
|
|
2810
|
+
last_transient_exc = tex
|
|
2811
|
+
if isinstance(tex, _TransientStreamError) and (
|
|
2812
|
+
tex.preserve_on_selection_exhausted or error_code == "stream_incomplete"
|
|
2813
|
+
):
|
|
2814
|
+
last_retryable_stream_error = _RetryableStreamError(
|
|
2815
|
+
error_code,
|
|
2816
|
+
error_payload,
|
|
2817
|
+
exclude_account=True,
|
|
2818
|
+
)
|
|
2819
|
+
await _release_tracked_stream_lease(current_account_lease)
|
|
2820
|
+
current_account_lease = None
|
|
2821
|
+
excluded_account_ids.add(account.id)
|
|
2822
|
+
break # outer loop: select different account
|
|
2823
|
+
finally:
|
|
2824
|
+
pop_stream_timeout_overrides(stream_timeout_tokens)
|
|
2825
|
+
if settlement.settlement_order_required:
|
|
2826
|
+
await _finalize_terminal_settlement_after_downstream_close(
|
|
2827
|
+
settlement,
|
|
2828
|
+
account,
|
|
2829
|
+
settlement_order_required=True,
|
|
2830
|
+
)
|
|
2831
|
+
else:
|
|
2832
|
+
settled = await _settle_stream_usage_before_pending_penalty(settlement)
|
|
2833
|
+
if settled and settlement.account_health_error:
|
|
2834
|
+
await proxy._handle_stream_error(
|
|
2835
|
+
account,
|
|
2836
|
+
_stream_settlement_error_payload(settlement),
|
|
2837
|
+
settlement.error_code or "upstream_error",
|
|
2838
|
+
)
|
|
2839
|
+
elif settled and settlement.record_success:
|
|
2840
|
+
await proxy._load_balancer.record_success(account)
|
|
2841
|
+
network_recovery.log_recovered()
|
|
2842
|
+
upstream_transport_metric_status = settlement.status
|
|
2843
|
+
_record_upstream_transport_metric_once(settlement.status)
|
|
2844
|
+
return
|
|
2845
|
+
continue # outer loop: account failover after transient exhaustion
|
|
2846
|
+
except _RetryableStreamError as exc:
|
|
2847
|
+
if _facade()._is_security_work_authorization_required_error(exc.code, exc.error.get("message")):
|
|
2848
|
+
if (
|
|
2849
|
+
account.security_work_authorized
|
|
2850
|
+
or account.id == file_preferred_account_id
|
|
2851
|
+
or require_preferred_account
|
|
2852
|
+
or attempt >= max_attempts - 1
|
|
2853
|
+
):
|
|
2854
|
+
event = response_failed_event(
|
|
2855
|
+
exc.code,
|
|
2856
|
+
exc.error.get("message") or "Security work authorization is required",
|
|
2857
|
+
response_id=request_id,
|
|
2858
|
+
)
|
|
2859
|
+
yield format_sse_event(event)
|
|
2860
|
+
return
|
|
2861
|
+
_facade().logger.info(
|
|
2862
|
+
"Retrying on security-work-authorized account request_id=%s account_id=%s",
|
|
2863
|
+
request_id,
|
|
2864
|
+
account.id,
|
|
2865
|
+
)
|
|
2866
|
+
yield format_sse_event(
|
|
2867
|
+
_facade()._security_work_advisory_event(
|
|
2868
|
+
code=_facade()._SECURITY_WORK_AUTHORIZATION_REQUIRED_CODE,
|
|
2869
|
+
message=_facade()._SECURITY_WORK_RETRY_MESSAGE,
|
|
2870
|
+
request_id=request_id,
|
|
2871
|
+
action="retry_security_work_authorized",
|
|
2872
|
+
account_id=account.id,
|
|
2873
|
+
)
|
|
2874
|
+
)
|
|
2875
|
+
await _release_tracked_stream_lease(current_account_lease)
|
|
2876
|
+
current_account_lease = None
|
|
2877
|
+
excluded_account_ids.add(account.id)
|
|
2878
|
+
require_security_work_authorized = True
|
|
2879
|
+
last_security_work_retry_error = exc
|
|
2880
|
+
continue
|
|
2881
|
+
await _handle_or_defer_keyed_stream_health(account, exc.error, exc.code)
|
|
2882
|
+
last_retryable_stream_error = exc
|
|
2883
|
+
if exc.exclude_account:
|
|
2884
|
+
await _release_tracked_stream_lease(current_account_lease)
|
|
2885
|
+
current_account_lease = None
|
|
2886
|
+
excluded_account_ids.add(account.id)
|
|
2887
|
+
_move_verified_fresh_replay_from_owner(
|
|
2888
|
+
account_id=account.id,
|
|
2889
|
+
outcome="owner_previsible_retryable_failure",
|
|
2890
|
+
)
|
|
2891
|
+
continue
|
|
2892
|
+
except _TerminalStreamError:
|
|
2893
|
+
if settlement.settlement_order_required:
|
|
2894
|
+
await _finalize_terminal_settlement_after_downstream_close(
|
|
2895
|
+
settlement,
|
|
2896
|
+
account,
|
|
2897
|
+
settlement_order_required=True,
|
|
2898
|
+
)
|
|
2899
|
+
else:
|
|
2900
|
+
health_write_allowed = await _settle_stream_usage_before_pending_penalty(settlement)
|
|
2901
|
+
if health_write_allowed and settlement.account_health_error:
|
|
2902
|
+
await proxy._handle_stream_error(
|
|
2903
|
+
account,
|
|
2904
|
+
_stream_settlement_error_payload(settlement),
|
|
2905
|
+
settlement.error_code or "upstream_error",
|
|
2906
|
+
)
|
|
2907
|
+
return
|
|
2908
|
+
except ProxyResponseError as exc:
|
|
2909
|
+
if _must_abort_native_transport_failure(
|
|
2910
|
+
exc,
|
|
2911
|
+
downstream_visible=settlement.downstream_visible,
|
|
2912
|
+
):
|
|
2913
|
+
# A native Codex transport failure is ambiguous once the
|
|
2914
|
+
# helper may have dispatched the request. Re-raising the
|
|
2915
|
+
# original exception aborts the downstream stream without
|
|
2916
|
+
# replaying side effects on this or another account.
|
|
2917
|
+
await _drain_pending_post_refresh_penalty_on_terminal(settlement)
|
|
2918
|
+
raise
|
|
2919
|
+
if _facade()._is_proxy_budget_exhausted_error(exc):
|
|
2920
|
+
await _settle_process_network_budget_exhaustion(account, settlement)
|
|
2921
|
+
if preserve_native_failure_lifecycle:
|
|
2922
|
+
return
|
|
2923
|
+
yield format_sse_event(_facade()._proxy_request_timeout_event(request_id))
|
|
2924
|
+
return
|
|
2925
|
+
account_model_retry = await _retry_account_model_rejection(
|
|
2926
|
+
exc,
|
|
2927
|
+
account,
|
|
2928
|
+
outcome="outer_proxy_error",
|
|
2929
|
+
)
|
|
2930
|
+
if account_model_retry:
|
|
2931
|
+
continue
|
|
2932
|
+
if account_model_retry is False:
|
|
2933
|
+
await _drain_pending_post_refresh_penalty_on_terminal(settlement)
|
|
2934
|
+
if propagate_http_errors:
|
|
2935
|
+
raise
|
|
2936
|
+
yield await _render_account_model_rejection(exc, account_id=account.id)
|
|
2937
|
+
return
|
|
2938
|
+
if exc.status_code == 401:
|
|
2939
|
+
remaining_budget = proxy._remaining_budget_seconds(deadline)
|
|
2940
|
+
if remaining_budget <= 0:
|
|
2941
|
+
_facade().logger.warning(
|
|
2942
|
+
"Proxy request budget exhausted before forced refresh retry "
|
|
2943
|
+
"request_id=%s attempt=%s account_id=%s",
|
|
2944
|
+
request_id,
|
|
2945
|
+
attempt + 1,
|
|
2946
|
+
account.id,
|
|
2947
|
+
)
|
|
2948
|
+
await proxy._write_stream_preflight_error(
|
|
2949
|
+
affinity_observation=affinity_observation,
|
|
2950
|
+
account_id=account.id,
|
|
2951
|
+
api_key=api_key,
|
|
2952
|
+
request_id=request_id,
|
|
2953
|
+
model=payload.model,
|
|
2954
|
+
start=start,
|
|
2955
|
+
error_code="upstream_request_timeout",
|
|
2956
|
+
error_message="Proxy request budget exhausted",
|
|
2957
|
+
reasoning_effort=payload.reasoning.effort if payload.reasoning else None,
|
|
2958
|
+
service_tier=payload.service_tier,
|
|
2959
|
+
transport=request_transport,
|
|
2960
|
+
upstream_transport=upstream_stream_transport,
|
|
2961
|
+
useragent=useragent,
|
|
2962
|
+
useragent_group=useragent_group,
|
|
2963
|
+
conversation_id=conversation_id,
|
|
2964
|
+
client_ip=client_ip,
|
|
2965
|
+
)
|
|
2966
|
+
if preserve_native_failure_lifecycle:
|
|
2967
|
+
return
|
|
2968
|
+
yield format_sse_event(_facade()._proxy_request_timeout_event(request_id))
|
|
2969
|
+
return
|
|
2970
|
+
try:
|
|
2971
|
+
account = await proxy._ensure_fresh_with_budget(
|
|
2972
|
+
account,
|
|
2973
|
+
force=True,
|
|
2974
|
+
timeout_seconds=remaining_budget,
|
|
2975
|
+
)
|
|
2976
|
+
except (RefreshError, aiohttp.ClientError, asyncio.TimeoutError) as refresh_exc:
|
|
2977
|
+
if isinstance(refresh_exc, RefreshError):
|
|
2978
|
+
if refresh_exc.is_permanent:
|
|
2979
|
+
await proxy._load_balancer.mark_permanent_failure(account, refresh_exc.code)
|
|
2980
|
+
# Keep the warning account routable for later
|
|
2981
|
+
# requests, but exclude it from this request's
|
|
2982
|
+
# retry pool after upstream rejected its token.
|
|
2983
|
+
await _release_tracked_stream_lease(current_account_lease)
|
|
2984
|
+
current_account_lease = None
|
|
2985
|
+
excluded_account_ids.add(account.id)
|
|
2986
|
+
continue
|
|
2987
|
+
if is_transient_refresh_contention(refresh_exc):
|
|
2988
|
+
# Transient CROSS-REPLICA refresh contention on
|
|
2989
|
+
# the post-401 forced refresh: benign claim
|
|
2990
|
+
# contention OR a post-exchange persist/status
|
|
2991
|
+
# CAS conflict. This is NOT a genuine
|
|
2992
|
+
# ``transport_error`` OAuth failure — the
|
|
2993
|
+
# account's credentials are healthy — so fail
|
|
2994
|
+
# over WITHOUT an account-health penalty (no
|
|
2995
|
+
# ``_handle_stream_error``); the genuine
|
|
2996
|
+
# transport failure handled below keeps its
|
|
2997
|
+
# penalty. A post-exchange persist conflict is
|
|
2998
|
+
# logged distinctly (rarer, more-serious race).
|
|
2999
|
+
if refresh_contention_kind(refresh_exc) == "persist_conflict":
|
|
3000
|
+
logger.warning(
|
|
3001
|
+
"Stream post-401 forced-refresh post-exchange persist conflict "
|
|
3002
|
+
"code=%s account_id=%s",
|
|
3003
|
+
refresh_exc.code,
|
|
3004
|
+
account.id,
|
|
3005
|
+
)
|
|
3006
|
+
if not require_preferred_account and preferred_account_id is None:
|
|
3007
|
+
# Movable request: release the skipped
|
|
3008
|
+
# account's stream lease and fail over to a
|
|
3009
|
+
# different account. Record a retryable
|
|
3010
|
+
# upstream_unavailable so exhaustion surfaces
|
|
3011
|
+
# upstream_unavailable instead of a
|
|
3012
|
+
# misleading generic no_accounts response.
|
|
3013
|
+
await _release_tracked_stream_lease(current_account_lease)
|
|
3014
|
+
current_account_lease = None
|
|
3015
|
+
excluded_account_ids.add(account.id)
|
|
3016
|
+
last_retryable_stream_error = _RetryableStreamError(
|
|
3017
|
+
"upstream_unavailable",
|
|
3018
|
+
{
|
|
3019
|
+
"message": (
|
|
3020
|
+
refresh_exc.message
|
|
3021
|
+
or "Account refresh is temporarily unavailable; "
|
|
3022
|
+
"no healthy account could be reached."
|
|
3023
|
+
)
|
|
3024
|
+
},
|
|
3025
|
+
exclude_account=True,
|
|
3026
|
+
)
|
|
3027
|
+
continue
|
|
3028
|
+
# PINNED request: must not cross accounts. Stay
|
|
3029
|
+
# on the owner account, release the lease, and
|
|
3030
|
+
# surface a retryable ``upstream_unavailable``
|
|
3031
|
+
# promptly so the client can retry once the claim
|
|
3032
|
+
# clears.
|
|
3033
|
+
await _release_tracked_stream_lease(current_account_lease)
|
|
3034
|
+
current_account_lease = None
|
|
3035
|
+
message = (
|
|
3036
|
+
refresh_exc.message
|
|
3037
|
+
or "Account refresh is temporarily unavailable; retry later."
|
|
3038
|
+
)
|
|
3039
|
+
last_retryable_stream_error = _RetryableStreamError(
|
|
3040
|
+
"upstream_unavailable",
|
|
3041
|
+
{"message": message},
|
|
3042
|
+
)
|
|
3043
|
+
await proxy._write_stream_preflight_error(
|
|
3044
|
+
affinity_observation=affinity_observation,
|
|
3045
|
+
account_id=account.id,
|
|
3046
|
+
api_key=api_key,
|
|
3047
|
+
request_id=request_id,
|
|
3048
|
+
model=payload.model,
|
|
3049
|
+
start=start,
|
|
3050
|
+
error_code="upstream_unavailable",
|
|
3051
|
+
error_message=message,
|
|
3052
|
+
reasoning_effort=payload.reasoning.effort if payload.reasoning else None,
|
|
3053
|
+
service_tier=payload.service_tier,
|
|
3054
|
+
transport=request_transport,
|
|
3055
|
+
upstream_transport=upstream_stream_transport,
|
|
3056
|
+
useragent=useragent,
|
|
3057
|
+
useragent_group=useragent_group,
|
|
3058
|
+
conversation_id=conversation_id,
|
|
3059
|
+
client_ip=client_ip,
|
|
3060
|
+
)
|
|
3061
|
+
event = response_failed_event(
|
|
3062
|
+
"upstream_unavailable",
|
|
3063
|
+
message,
|
|
3064
|
+
response_id=request_id,
|
|
3065
|
+
)
|
|
3066
|
+
yield format_sse_event(event)
|
|
3067
|
+
return
|
|
3068
|
+
if not refresh_exc.transport_error:
|
|
3069
|
+
# Non-transport, non-permanent RefreshError:
|
|
3070
|
+
# release the stream lease and reselect.
|
|
3071
|
+
await _release_tracked_stream_lease(current_account_lease)
|
|
3072
|
+
current_account_lease = None
|
|
3073
|
+
continue
|
|
3074
|
+
# A GENUINE OAuth transport failure
|
|
3075
|
+
# (``code == "transport_error"``): the account/route
|
|
3076
|
+
# is at fault, so it falls through to the shared
|
|
3077
|
+
# transport-failure handling below — identical to a
|
|
3078
|
+
# raw aiohttp/connect failure — which records the
|
|
3079
|
+
# account-health penalty (``_handle_stream_error``)
|
|
3080
|
+
# so the broken account backs off instead of being
|
|
3081
|
+
# kept healthy and reselected on the next request.
|
|
3082
|
+
_facade().logger.warning(
|
|
3083
|
+
"Stream forced refresh/connect failed request_id=%s attempt=%s account_id=%s",
|
|
3084
|
+
request_id,
|
|
3085
|
+
attempt + 1,
|
|
3086
|
+
account.id,
|
|
3087
|
+
exc_info=True,
|
|
3088
|
+
)
|
|
3089
|
+
message = getattr(refresh_exc, "message", None) or str(refresh_exc)
|
|
3090
|
+
message = message or "Request to upstream timed out"
|
|
3091
|
+
if (
|
|
3092
|
+
not require_preferred_account
|
|
3093
|
+
and preferred_account_id is None
|
|
3094
|
+
and _facade()._should_retry_transient_stream_error("upstream_unavailable", message)
|
|
3095
|
+
and attempt + 1 < max_attempts
|
|
3096
|
+
):
|
|
3097
|
+
await _handle_or_defer_keyed_stream_health(
|
|
3098
|
+
account,
|
|
3099
|
+
{"message": message},
|
|
3100
|
+
"upstream_unavailable",
|
|
3101
|
+
)
|
|
3102
|
+
last_retryable_stream_error = _RetryableStreamError(
|
|
3103
|
+
"upstream_unavailable",
|
|
3104
|
+
{"message": message},
|
|
3105
|
+
exclude_account=True,
|
|
3106
|
+
)
|
|
3107
|
+
# Release the excluded account's stream lease before
|
|
3108
|
+
# the failover ``continue`` so its stream-concurrency
|
|
3109
|
+
# slot is not held for the duration of the replacement
|
|
3110
|
+
# stream (matching the pre-refresh transport branch and
|
|
3111
|
+
# the claim-contention/permanent branches above).
|
|
3112
|
+
await _release_tracked_stream_lease(current_account_lease)
|
|
3113
|
+
current_account_lease = None
|
|
3114
|
+
excluded_account_ids.add(account.id)
|
|
3115
|
+
continue
|
|
3116
|
+
await proxy._write_stream_preflight_error(
|
|
3117
|
+
affinity_observation=affinity_observation,
|
|
3118
|
+
account_id=account.id,
|
|
3119
|
+
api_key=api_key,
|
|
3120
|
+
request_id=request_id,
|
|
3121
|
+
model=payload.model,
|
|
3122
|
+
start=start,
|
|
3123
|
+
error_code="upstream_unavailable",
|
|
3124
|
+
error_message=message,
|
|
3125
|
+
reasoning_effort=payload.reasoning.effort if payload.reasoning else None,
|
|
3126
|
+
service_tier=payload.service_tier,
|
|
3127
|
+
transport=request_transport,
|
|
3128
|
+
upstream_transport=upstream_stream_transport,
|
|
3129
|
+
useragent=useragent,
|
|
3130
|
+
useragent_group=useragent_group,
|
|
3131
|
+
conversation_id=conversation_id,
|
|
3132
|
+
client_ip=client_ip,
|
|
3133
|
+
)
|
|
3134
|
+
event = response_failed_event(
|
|
3135
|
+
"upstream_unavailable",
|
|
3136
|
+
message,
|
|
3137
|
+
response_id=request_id,
|
|
3138
|
+
)
|
|
3139
|
+
yield format_sse_event(event)
|
|
3140
|
+
return
|
|
3141
|
+
settlement = _StreamSettlement()
|
|
3142
|
+
effective_attempt_timeout = proxy._remaining_budget_seconds(deadline)
|
|
3143
|
+
if effective_attempt_timeout <= 0:
|
|
3144
|
+
_facade().logger.warning(
|
|
3145
|
+
"Proxy request budget exhausted before post-refresh stream attempt "
|
|
3146
|
+
"request_id=%s attempt=%s account_id=%s",
|
|
3147
|
+
request_id,
|
|
3148
|
+
attempt + 1,
|
|
3149
|
+
account.id,
|
|
3150
|
+
)
|
|
3151
|
+
await proxy._write_stream_preflight_error(
|
|
3152
|
+
affinity_observation=affinity_observation,
|
|
3153
|
+
account_id=account.id,
|
|
3154
|
+
api_key=api_key,
|
|
3155
|
+
request_id=request_id,
|
|
3156
|
+
model=payload.model,
|
|
3157
|
+
start=start,
|
|
3158
|
+
error_code="upstream_request_timeout",
|
|
3159
|
+
error_message="Proxy request budget exhausted",
|
|
3160
|
+
reasoning_effort=payload.reasoning.effort if payload.reasoning else None,
|
|
3161
|
+
service_tier=payload.service_tier,
|
|
3162
|
+
transport=request_transport,
|
|
3163
|
+
upstream_transport=upstream_stream_transport,
|
|
3164
|
+
useragent=useragent,
|
|
3165
|
+
useragent_group=useragent_group,
|
|
3166
|
+
conversation_id=conversation_id,
|
|
3167
|
+
client_ip=client_ip,
|
|
3168
|
+
)
|
|
3169
|
+
if preserve_native_failure_lifecycle:
|
|
3170
|
+
return
|
|
3171
|
+
yield format_sse_event(_facade()._proxy_request_timeout_event(request_id))
|
|
3172
|
+
return
|
|
3173
|
+
try:
|
|
3174
|
+
can_try_other_account = (
|
|
3175
|
+
not require_preferred_account
|
|
3176
|
+
and account.id != file_preferred_account_id
|
|
3177
|
+
and attempt < max_attempts - 1
|
|
3178
|
+
)
|
|
3179
|
+
post_refresh_stream = _stream_post_refresh_with_capacity_recovery(
|
|
3180
|
+
account,
|
|
3181
|
+
settlement=settlement,
|
|
3182
|
+
can_try_other_account=can_try_other_account,
|
|
3183
|
+
tool_call_dedupe=tool_call_dedupe,
|
|
3184
|
+
)
|
|
3185
|
+
try:
|
|
3186
|
+
async for line in post_refresh_stream:
|
|
3187
|
+
yield line
|
|
3188
|
+
finally:
|
|
3189
|
+
# Closing this generator runs its internal
|
|
3190
|
+
# cancellation-safe close/terminal finalization;
|
|
3191
|
+
# without an owned aclose() the child would stay
|
|
3192
|
+
# suspended after a downstream disconnect.
|
|
3193
|
+
close_task = scheduler.create_task(
|
|
3194
|
+
post_refresh_stream.aclose(),
|
|
3195
|
+
name=f"stream-post-refresh-outer-close-{request_id}",
|
|
3196
|
+
)
|
|
3197
|
+
_, close_cancellation = await _await_task_deferring_cancellation(close_task)
|
|
3198
|
+
if close_cancellation is not None:
|
|
3199
|
+
raise close_cancellation
|
|
3200
|
+
except ProxyResponseError as retry_exc:
|
|
3201
|
+
if _must_abort_native_transport_failure(
|
|
3202
|
+
retry_exc,
|
|
3203
|
+
downstream_visible=settlement.downstream_visible,
|
|
3204
|
+
):
|
|
3205
|
+
raise
|
|
3206
|
+
if _facade()._is_proxy_budget_exhausted_error(retry_exc):
|
|
3207
|
+
await _settle_process_network_budget_exhaustion(account, settlement)
|
|
3208
|
+
if preserve_native_failure_lifecycle:
|
|
3209
|
+
return
|
|
3210
|
+
yield format_sse_event(_facade()._proxy_request_timeout_event(request_id))
|
|
3211
|
+
return
|
|
3212
|
+
if settlement.downstream_visible:
|
|
3213
|
+
failed_response_id = settlement.response_id or request_id
|
|
3214
|
+
error = _parse_openai_error(retry_exc.payload)
|
|
3215
|
+
error_code = _normalize_error_code(
|
|
3216
|
+
error.code if error else None,
|
|
3217
|
+
error.type if error else None,
|
|
3218
|
+
)
|
|
3219
|
+
error_message = error.message if error else "Upstream error"
|
|
3220
|
+
event = response_failed_event(
|
|
3221
|
+
error_code or "upstream_error",
|
|
3222
|
+
error_message or "Upstream error",
|
|
3223
|
+
error_type=(error.type if error else None) or "server_error",
|
|
3224
|
+
response_id=failed_response_id,
|
|
3225
|
+
error_param=error.param_state if error else None,
|
|
3226
|
+
)
|
|
3227
|
+
_apply_error_metadata(event["response"]["error"], error)
|
|
3228
|
+
_facade().logger.warning(
|
|
3229
|
+
"Surfacing post-refresh stream failure without replay "
|
|
3230
|
+
"request_id=%s account_id=%s code=%s",
|
|
3231
|
+
request_id,
|
|
3232
|
+
account.id,
|
|
3233
|
+
error_code,
|
|
3234
|
+
)
|
|
3235
|
+
yield format_sse_event(event)
|
|
3236
|
+
settlement.record_success = False
|
|
3237
|
+
settlement.error_code = error_code
|
|
3238
|
+
settlement.error_message = error_message
|
|
3239
|
+
settlement.error = _upstream_error_from_openai(error)
|
|
3240
|
+
settlement.account_health_error = _facade()._should_penalize_stream_error(
|
|
3241
|
+
error_code, error_message
|
|
3242
|
+
)
|
|
3243
|
+
settled = await _settle_stream_usage_before_pending_penalty(settlement)
|
|
3244
|
+
if settled and settlement.account_health_error:
|
|
3245
|
+
await proxy._handle_stream_error(
|
|
3246
|
+
account,
|
|
3247
|
+
_stream_settlement_error_payload(settlement),
|
|
3248
|
+
settlement.error_code or "upstream_error",
|
|
3249
|
+
http_status=retry_exc.status_code,
|
|
3250
|
+
)
|
|
3251
|
+
return
|
|
3252
|
+
error = _parse_openai_error(retry_exc.payload)
|
|
3253
|
+
error_code = _normalize_error_code(
|
|
3254
|
+
error.code if error else None,
|
|
3255
|
+
error.type if error else None,
|
|
3256
|
+
)
|
|
3257
|
+
if getattr(retry_exc, _POST_REFRESH_TRANSIENT_EXHAUSTED_ATTR, False):
|
|
3258
|
+
transient_error_payload = _stream_settlement_error_payload(settlement)
|
|
3259
|
+
last_transient_exc = retry_exc
|
|
3260
|
+
last_retryable_stream_error = _RetryableStreamError(
|
|
3261
|
+
settlement.error_code or error_code or "upstream_error",
|
|
3262
|
+
transient_error_payload,
|
|
3263
|
+
)
|
|
3264
|
+
await _release_tracked_stream_lease(current_account_lease)
|
|
3265
|
+
current_account_lease = None
|
|
3266
|
+
excluded_account_ids.add(account.id)
|
|
3267
|
+
continue
|
|
3268
|
+
account_model_retry = await _retry_account_model_rejection(
|
|
3269
|
+
retry_exc,
|
|
3270
|
+
account,
|
|
3271
|
+
outcome="post_refresh",
|
|
3272
|
+
)
|
|
3273
|
+
if account_model_retry:
|
|
3274
|
+
continue
|
|
3275
|
+
if account_model_retry is False:
|
|
3276
|
+
await _drain_pending_post_refresh_penalty_on_terminal(settlement)
|
|
3277
|
+
if propagate_http_errors:
|
|
3278
|
+
raise
|
|
3279
|
+
yield await _render_account_model_rejection(
|
|
3280
|
+
retry_exc,
|
|
3281
|
+
account_id=account.id,
|
|
3282
|
+
)
|
|
3283
|
+
return
|
|
3284
|
+
if error_code == "account_response_create_cap":
|
|
3285
|
+
last_transient_exc = retry_exc
|
|
3286
|
+
if can_try_other_account:
|
|
3287
|
+
deferred_capacity_account = account
|
|
3288
|
+
deferred_capacity_lease = current_account_lease
|
|
3289
|
+
excluded_account_ids.add(account.id)
|
|
3290
|
+
continue
|
|
3291
|
+
# The same-account helper only re-raises this
|
|
3292
|
+
# neutral cap when recovery cannot continue
|
|
3293
|
+
# within the original budget. Exit the account
|
|
3294
|
+
# loop so the preserved cap is propagated or
|
|
3295
|
+
# rendered below, instead of replacing it with
|
|
3296
|
+
# a next-attempt timeout.
|
|
3297
|
+
await _drain_pending_post_refresh_penalty_on_terminal(settlement)
|
|
3298
|
+
break
|
|
3299
|
+
if _facade()._is_account_neutral_error_code(error_code):
|
|
3300
|
+
await _drain_pending_post_refresh_penalty_on_terminal(settlement)
|
|
3301
|
+
raise
|
|
3302
|
+
if is_confirmed_pre_dispatch_transport_error(retry_exc):
|
|
3303
|
+
# This retry still failed before dispatch. Handle
|
|
3304
|
+
# the proven dead route before the generic
|
|
3305
|
+
# failover policy: that policy does not know
|
|
3306
|
+
# about hard account ownership and may otherwise
|
|
3307
|
+
# cross a previous-response, turn-state, file, or
|
|
3308
|
+
# single-account boundary.
|
|
3309
|
+
await _release_tracked_stream_lease(current_account_lease)
|
|
3310
|
+
current_account_lease = None
|
|
3311
|
+
await _record_or_defer_confirmed_route_backoff(account)
|
|
3312
|
+
last_transient_exc = retry_exc
|
|
3313
|
+
last_pre_dispatch_transport_error = retry_exc
|
|
3314
|
+
|
|
3315
|
+
verified_owner_replay_moved = False
|
|
3316
|
+
if (
|
|
3317
|
+
attempt < max_attempts - 1
|
|
3318
|
+
and routing_strategy != "single_account"
|
|
3319
|
+
and file_preferred_account_id is None
|
|
3320
|
+
and turn_state_owner_account_id is None
|
|
3321
|
+
):
|
|
3322
|
+
verified_owner_replay_moved = _move_verified_fresh_replay_from_owner(
|
|
3323
|
+
account_id=account.id,
|
|
3324
|
+
outcome="owner_post_refresh_proxy_connect_failure",
|
|
3325
|
+
)
|
|
3326
|
+
|
|
3327
|
+
can_try_other_account = bool(
|
|
3328
|
+
attempt < max_attempts - 1
|
|
3329
|
+
and routing_strategy != "single_account"
|
|
3330
|
+
and file_preferred_account_id is None
|
|
3331
|
+
and turn_state_owner_account_id is None
|
|
3332
|
+
and not require_preferred_account
|
|
3333
|
+
)
|
|
3334
|
+
if can_try_other_account:
|
|
3335
|
+
excluded_account_ids.add(account.id)
|
|
3336
|
+
if not verified_owner_replay_moved:
|
|
3337
|
+
affinity = replace(affinity, reallocate_sticky=True)
|
|
3338
|
+
_facade().logger.info(
|
|
3339
|
+
"Retrying post-refresh stream after confirmed pre-dispatch proxy "
|
|
3340
|
+
"connect failure request_id=%s account_id=%s attempt=%d",
|
|
3341
|
+
request_id,
|
|
3342
|
+
account.id,
|
|
3343
|
+
attempt + 1,
|
|
3344
|
+
)
|
|
3345
|
+
continue
|
|
3346
|
+
|
|
3347
|
+
# Hard ownership or an exhausted attempt budget:
|
|
3348
|
+
# stop here. The shared terminal path settles the
|
|
3349
|
+
# reservation, drains the deferred backoff floor,
|
|
3350
|
+
# and preserves this sanitized failure.
|
|
3351
|
+
break
|
|
3352
|
+
current_error_payload = _upstream_error_from_openai(error)
|
|
3353
|
+
current_error_code = error_code or "upstream_error"
|
|
3354
|
+
classified = classify_upstream_failure(
|
|
3355
|
+
error_code=current_error_code,
|
|
3356
|
+
error=current_error_payload,
|
|
3357
|
+
http_status=retry_exc.status_code,
|
|
3358
|
+
phase="first_event",
|
|
3359
|
+
)
|
|
3360
|
+
candidates_remaining = max_attempts - attempt - 1
|
|
3361
|
+
burst = is_upstream_burst_rejection(
|
|
3362
|
+
failure_class=classified["failure_class"],
|
|
3363
|
+
http_status=retry_exc.status_code,
|
|
3364
|
+
)
|
|
3365
|
+
if retry_exc.status_code == 401 and candidates_remaining > 0:
|
|
3366
|
+
action = "failover_next"
|
|
3367
|
+
elif resilience.deterministic_failover_enabled:
|
|
3368
|
+
action = failover_decision(
|
|
3369
|
+
failure_class=classified["failure_class"],
|
|
3370
|
+
downstream_visible=False,
|
|
3371
|
+
candidates_remaining=candidates_remaining,
|
|
3372
|
+
owner_bound=_stream_owner_bound_to(account),
|
|
3373
|
+
# The post-refresh redispatch re-enters the
|
|
3374
|
+
# attempt loop, so it also needs an attempt.
|
|
3375
|
+
same_account_retry_available=(
|
|
3376
|
+
candidates_remaining > 0
|
|
3377
|
+
and _burst_same_account_retry_available(account, burst=burst)
|
|
3378
|
+
),
|
|
3379
|
+
)
|
|
3380
|
+
else:
|
|
3381
|
+
action = "surface"
|
|
3382
|
+
_facade().logger.info(
|
|
3383
|
+
"Failover decision request_id=%s transport=stream account_id=%s "
|
|
3384
|
+
"attempt=%d phase=post_refresh failure_class=%s action=%s",
|
|
3385
|
+
request_id,
|
|
3386
|
+
account.id,
|
|
3387
|
+
attempt + 1,
|
|
3388
|
+
classified["failure_class"],
|
|
3389
|
+
action,
|
|
3390
|
+
)
|
|
3391
|
+
if action == "retry_same_account":
|
|
3392
|
+
# Owner-bound burst 429 after the forced refresh:
|
|
3393
|
+
# keep the dispatch anchor and let the next
|
|
3394
|
+
# attempt re-select the same owner (it is not
|
|
3395
|
+
# excluded) after a short backoff. As on the
|
|
3396
|
+
# pre-visible path only the burst cooldown is
|
|
3397
|
+
# engaged here; the transient penalty is written
|
|
3398
|
+
# once if the failure is finally surfaced.
|
|
3399
|
+
await record_upstream_burst_rejection(
|
|
3400
|
+
proxy._load_balancer,
|
|
3401
|
+
account,
|
|
3402
|
+
retry_after_seconds=retry_exc.retry_after_seconds,
|
|
3403
|
+
)
|
|
3404
|
+
await _release_tracked_stream_lease(current_account_lease)
|
|
3405
|
+
current_account_lease = None
|
|
3406
|
+
retry_index = _next_burst_same_account_retry(account)
|
|
3407
|
+
delay = burst_same_account_backoff_seconds(
|
|
3408
|
+
retry_index,
|
|
3409
|
+
retry_after_seconds=retry_exc.retry_after_seconds,
|
|
3410
|
+
)
|
|
3411
|
+
# A failed re-selection re-raises this exception:
|
|
3412
|
+
# stamp Retry-After now (after the delay was
|
|
3413
|
+
# derived from the upstream value).
|
|
3414
|
+
_stamp_surfaced_burst_retry_after(retry_exc)
|
|
3415
|
+
last_transient_exc = retry_exc
|
|
3416
|
+
_facade().logger.info(
|
|
3417
|
+
"Burst 429 on owner-bound stream, retrying same account "
|
|
3418
|
+
"request_id=%s account_id=%s phase=post_refresh retry=%s/%s delay=%.2fs "
|
|
3419
|
+
"retry_after_seconds=%s",
|
|
3420
|
+
request_id,
|
|
3421
|
+
account.id,
|
|
3422
|
+
retry_index,
|
|
3423
|
+
BURST_SAME_ACCOUNT_MAX_RETRIES,
|
|
3424
|
+
delay,
|
|
3425
|
+
retry_exc.retry_after_seconds,
|
|
3426
|
+
)
|
|
3427
|
+
async for wait_event in _iter_account_capacity_recovery_wait(
|
|
3428
|
+
request_id=request_id,
|
|
3429
|
+
model=payload.model,
|
|
3430
|
+
account_id=account.id,
|
|
3431
|
+
error_message=error.message if error else None,
|
|
3432
|
+
recovery_sleep_seconds=delay,
|
|
3433
|
+
remaining_budget_seconds=proxy._remaining_budget_seconds(deadline),
|
|
3434
|
+
emit_keepalives=not propagate_http_errors,
|
|
3435
|
+
stage="burst_backoff",
|
|
3436
|
+
scheduler=scheduler,
|
|
3437
|
+
clock=clock,
|
|
3438
|
+
):
|
|
3439
|
+
yield wait_event
|
|
3440
|
+
if proxy._remaining_budget_seconds(deadline) > 0:
|
|
3441
|
+
continue
|
|
3442
|
+
# Budget spent during the wait: surface the
|
|
3443
|
+
# original rejection below (one health write).
|
|
3444
|
+
if action == "failover_next":
|
|
3445
|
+
await _handle_or_defer_keyed_stream_health(
|
|
3446
|
+
account,
|
|
3447
|
+
current_error_payload,
|
|
3448
|
+
current_error_code,
|
|
3449
|
+
http_status=retry_exc.status_code,
|
|
3450
|
+
retry_after_seconds=retry_exc.retry_after_seconds,
|
|
3451
|
+
)
|
|
3452
|
+
last_transient_exc = retry_exc
|
|
3453
|
+
await _release_tracked_stream_lease(current_account_lease)
|
|
3454
|
+
current_account_lease = None
|
|
3455
|
+
_move_verified_fresh_replay_from_owner(
|
|
3456
|
+
account_id=account.id,
|
|
3457
|
+
outcome="owner_post_refresh_failure",
|
|
3458
|
+
)
|
|
3459
|
+
excluded_account_ids.add(account.id)
|
|
3460
|
+
continue
|
|
3461
|
+
health_write_allowed = await _drain_pending_post_refresh_penalty_on_terminal(settlement)
|
|
3462
|
+
if health_write_allowed:
|
|
3463
|
+
await proxy._handle_stream_error(
|
|
3464
|
+
account,
|
|
3465
|
+
current_error_payload,
|
|
3466
|
+
current_error_code,
|
|
3467
|
+
http_status=retry_exc.status_code,
|
|
3468
|
+
**_retry_after_kwargs(retry_exc.retry_after_seconds),
|
|
3469
|
+
)
|
|
3470
|
+
setattr(retry_exc, _STREAM_HEALTH_RECORDED_ATTR, True)
|
|
3471
|
+
if burst:
|
|
3472
|
+
_stamp_surfaced_burst_retry_after(retry_exc)
|
|
3473
|
+
if propagate_http_errors:
|
|
3474
|
+
raise
|
|
3475
|
+
error_message = error.message if error else None
|
|
3476
|
+
event = response_failed_event(
|
|
3477
|
+
error_code or "upstream_error",
|
|
3478
|
+
error_message or "Upstream error",
|
|
3479
|
+
error_type=(error.type if error else None) or "server_error",
|
|
3480
|
+
response_id=request_id,
|
|
3481
|
+
error_param=error.param_state if error else None,
|
|
3482
|
+
)
|
|
3483
|
+
_apply_error_metadata(event["response"]["error"], error)
|
|
3484
|
+
yield format_sse_event(event)
|
|
3485
|
+
return
|
|
3486
|
+
current_account_penalty_queued = any(
|
|
3487
|
+
failed_account is account
|
|
3488
|
+
for failed_account, *_rest in pending_post_refresh_transient_penalties
|
|
3489
|
+
)
|
|
3490
|
+
ordered_settlement_required = bool(
|
|
3491
|
+
pending_post_refresh_transient_penalties or deferred_account_error_backoffs
|
|
3492
|
+
)
|
|
3493
|
+
if ordered_settlement_required:
|
|
3494
|
+
health_write_allowed = await _drain_pending_post_refresh_penalty_on_terminal(settlement)
|
|
3495
|
+
if (
|
|
3496
|
+
health_write_allowed
|
|
3497
|
+
and settlement.account_health_error
|
|
3498
|
+
and not current_account_penalty_queued
|
|
3499
|
+
):
|
|
3500
|
+
await proxy._handle_stream_error(
|
|
3501
|
+
account,
|
|
3502
|
+
_stream_settlement_error_payload(settlement),
|
|
3503
|
+
settlement.error_code or "upstream_error",
|
|
3504
|
+
)
|
|
3505
|
+
elif health_write_allowed and settlement.record_success:
|
|
3506
|
+
await proxy._load_balancer.record_success(account)
|
|
3507
|
+
else:
|
|
3508
|
+
await _finalize_terminal_settlement_after_downstream_close(
|
|
3509
|
+
settlement,
|
|
3510
|
+
account,
|
|
3511
|
+
settlement_order_required=True,
|
|
3512
|
+
)
|
|
3513
|
+
upstream_transport_metric_status = settlement.status
|
|
3514
|
+
_record_upstream_transport_metric_once(settlement.status)
|
|
3515
|
+
return
|
|
3516
|
+
error = _parse_openai_error(exc.payload)
|
|
3517
|
+
error_code = _normalize_error_code(error.code if error else None, error.type if error else None)
|
|
3518
|
+
error_message = error.message if error else None
|
|
3519
|
+
error_type = error.type if error else None
|
|
3520
|
+
error_param = error.param_state if error else None
|
|
3521
|
+
if _facade()._is_security_work_authorization_required_error(error_code, error_message):
|
|
3522
|
+
if (
|
|
3523
|
+
not account.security_work_authorized
|
|
3524
|
+
and account.id != file_preferred_account_id
|
|
3525
|
+
and not require_preferred_account
|
|
3526
|
+
and attempt < max_attempts - 1
|
|
3527
|
+
):
|
|
3528
|
+
_facade().logger.info(
|
|
3529
|
+
"Retrying on security-work-authorized account request_id=%s account_id=%s",
|
|
3530
|
+
request_id,
|
|
3531
|
+
account.id,
|
|
3532
|
+
)
|
|
3533
|
+
yield format_sse_event(
|
|
3534
|
+
_facade()._security_work_advisory_event(
|
|
3535
|
+
code=_facade()._SECURITY_WORK_AUTHORIZATION_REQUIRED_CODE,
|
|
3536
|
+
message=_facade()._SECURITY_WORK_RETRY_MESSAGE,
|
|
3537
|
+
request_id=request_id,
|
|
3538
|
+
action="retry_security_work_authorized",
|
|
3539
|
+
account_id=account.id,
|
|
3540
|
+
)
|
|
3541
|
+
)
|
|
3542
|
+
await _release_tracked_stream_lease(current_account_lease)
|
|
3543
|
+
current_account_lease = None
|
|
3544
|
+
excluded_account_ids.add(account.id)
|
|
3545
|
+
require_security_work_authorized = True
|
|
3546
|
+
continue
|
|
3547
|
+
health_write_allowed = await _drain_pending_post_refresh_penalty_on_terminal(settlement)
|
|
3548
|
+
if (
|
|
3549
|
+
health_write_allowed
|
|
3550
|
+
and not getattr(exc, _STREAM_HEALTH_RECORDED_ATTR, False)
|
|
3551
|
+
and _facade()._should_penalize_stream_error(error_code, error_message)
|
|
3552
|
+
):
|
|
3553
|
+
await proxy._handle_stream_error(
|
|
3554
|
+
account,
|
|
3555
|
+
_upstream_error_from_openai(error),
|
|
3556
|
+
error_code,
|
|
3557
|
+
# Evidence only: this failure carries an upstream
|
|
3558
|
+
# HTTP status, so it is not a status-less terminal
|
|
3559
|
+
# and must stay out of the soft-overload window.
|
|
3560
|
+
# Passing it positionally would change the neutral
|
|
3561
|
+
# rejection predicates and the 429 branch too.
|
|
3562
|
+
upstream_http_status=exc.status_code,
|
|
3563
|
+
)
|
|
3564
|
+
if propagate_http_errors:
|
|
3565
|
+
raise
|
|
3566
|
+
event = response_failed_event(
|
|
3567
|
+
error_code,
|
|
3568
|
+
error_message or "Upstream error",
|
|
3569
|
+
error_type=error_type or "server_error",
|
|
3570
|
+
response_id=request_id,
|
|
3571
|
+
error_param=error_param,
|
|
3572
|
+
)
|
|
3573
|
+
_apply_error_metadata(event["response"]["error"], error)
|
|
3574
|
+
yield format_sse_event(event)
|
|
3575
|
+
return
|
|
3576
|
+
except RefreshError as exc:
|
|
3577
|
+
if exc.is_permanent:
|
|
3578
|
+
await proxy._load_balancer.mark_permanent_failure(account, exc.code)
|
|
3579
|
+
await _release_tracked_stream_lease(current_account_lease)
|
|
3580
|
+
current_account_lease = None
|
|
3581
|
+
excluded_account_ids.add(account.id)
|
|
3582
|
+
continue
|
|
3583
|
+
except Exception:
|
|
3584
|
+
await _drain_pending_post_refresh_penalty_on_terminal(settlement)
|
|
3585
|
+
_facade().logger.warning(
|
|
3586
|
+
"Proxy streaming failed without retry account_id=%s request_id=%s",
|
|
3587
|
+
account_id_value,
|
|
3588
|
+
request_id,
|
|
3589
|
+
exc_info=True,
|
|
3590
|
+
)
|
|
3591
|
+
event = response_failed_event(
|
|
3592
|
+
"upstream_error",
|
|
3593
|
+
"Proxy streaming failed",
|
|
3594
|
+
response_id=request_id,
|
|
3595
|
+
)
|
|
3596
|
+
yield format_sse_event(event)
|
|
3597
|
+
return
|
|
3598
|
+
await _drain_pending_post_refresh_penalty_on_terminal(settlement)
|
|
3599
|
+
# When HTTP error propagation is enabled and the last failure was
|
|
3600
|
+
# a transient 500, re-raise to preserve the upstream status/payload.
|
|
3601
|
+
if last_account_model_rejection is not None:
|
|
3602
|
+
if propagate_http_errors:
|
|
3603
|
+
raise last_account_model_rejection
|
|
3604
|
+
yield await _render_account_model_rejection(
|
|
3605
|
+
last_account_model_rejection,
|
|
3606
|
+
account_id=last_account_model_rejection_account_id,
|
|
3607
|
+
)
|
|
3608
|
+
return
|
|
3609
|
+
if propagate_http_errors and last_transient_exc is not None:
|
|
3610
|
+
raise last_transient_exc
|
|
3611
|
+
if last_pre_dispatch_transport_error is not None:
|
|
3612
|
+
# Attempt budget exhausted after confirmed pre-dispatch route
|
|
3613
|
+
# failures: surface the original sanitized failure rather than
|
|
3614
|
+
# a generated ``no_accounts`` response.
|
|
3615
|
+
yield _render_dispatch_transport_error(last_pre_dispatch_transport_error)
|
|
3616
|
+
return
|
|
3617
|
+
if last_retryable_stream_error is not None:
|
|
3618
|
+
retries_exhausted_msg = str(last_retryable_stream_error.error.get("message") or "Upstream error")
|
|
3619
|
+
event = response_failed_event(
|
|
3620
|
+
last_retryable_stream_error.code,
|
|
3621
|
+
retries_exhausted_msg,
|
|
3622
|
+
response_id=request_id,
|
|
3623
|
+
)
|
|
3624
|
+
if last_retryable_stream_error.code in SYNTHETIC_TRANSPORT_FAILURE_CODES:
|
|
3625
|
+
event = synthetic_transport_failure_event(event)
|
|
3626
|
+
if not (
|
|
3627
|
+
preserve_native_failure_lifecycle
|
|
3628
|
+
and last_retryable_stream_error.code in SYNTHETIC_TRANSPORT_FAILURE_CODES
|
|
3629
|
+
):
|
|
3630
|
+
yield format_sse_event(event)
|
|
3631
|
+
if not any_attempt_logged:
|
|
3632
|
+
await proxy._write_request_log(
|
|
3633
|
+
affinity_observation=affinity_observation,
|
|
3634
|
+
account_id=None,
|
|
3635
|
+
api_key=api_key,
|
|
3636
|
+
request_id=request_id,
|
|
3637
|
+
model=payload.model,
|
|
3638
|
+
latency_ms=int((clock.monotonic() - start) * 1000),
|
|
3639
|
+
status="error",
|
|
3640
|
+
error_code=last_retryable_stream_error.code,
|
|
3641
|
+
error_message=retries_exhausted_msg,
|
|
3642
|
+
reasoning_effort=payload.reasoning.effort if payload.reasoning else None,
|
|
3643
|
+
transport=request_transport,
|
|
3644
|
+
upstream_transport=upstream_stream_transport,
|
|
3645
|
+
service_tier=payload.service_tier,
|
|
3646
|
+
requested_service_tier=payload.service_tier,
|
|
3647
|
+
useragent=useragent,
|
|
3648
|
+
useragent_group=useragent_group,
|
|
3649
|
+
conversation_id=conversation_id,
|
|
3650
|
+
client_ip=client_ip,
|
|
3651
|
+
)
|
|
3652
|
+
return
|
|
3653
|
+
if last_transient_exc is not None:
|
|
3654
|
+
error = _parse_openai_error(last_transient_exc.payload)
|
|
3655
|
+
error_code = _normalize_error_code(error.code if error else None, error.type if error else None)
|
|
3656
|
+
if error_code == "account_response_create_cap":
|
|
3657
|
+
error_message = error.message if error else None
|
|
3658
|
+
event = response_failed_event(
|
|
3659
|
+
error_code,
|
|
3660
|
+
error_message or "Account response-create concurrency limit reached",
|
|
3661
|
+
error_type=(error.type if error else None) or "server_error",
|
|
3662
|
+
response_id=request_id,
|
|
3663
|
+
error_param=error.param_state if error else None,
|
|
3664
|
+
)
|
|
3665
|
+
_apply_error_metadata(event["response"]["error"], error)
|
|
3666
|
+
yield format_sse_event(event)
|
|
3667
|
+
return
|
|
3668
|
+
|
|
3669
|
+
retries_exhausted_msg = "No available accounts after retries"
|
|
3670
|
+
_facade().logger.warning(
|
|
3671
|
+
"Proxy streaming exhausted accounts request_id=%s model=%s transport=%s attempts=%s "
|
|
3672
|
+
"excluded_count=%s preferred_account_id=%s api_key_present=%s",
|
|
3673
|
+
request_id,
|
|
3674
|
+
payload.model,
|
|
3675
|
+
request_transport,
|
|
3676
|
+
attempt,
|
|
3677
|
+
len(excluded_account_ids),
|
|
3678
|
+
preferred_account_id,
|
|
3679
|
+
api_key is not None,
|
|
3680
|
+
)
|
|
3681
|
+
event = response_failed_event(
|
|
3682
|
+
"no_accounts",
|
|
3683
|
+
retries_exhausted_msg,
|
|
3684
|
+
response_id=request_id,
|
|
3685
|
+
)
|
|
3686
|
+
yield format_sse_event(event)
|
|
3687
|
+
if not any_attempt_logged:
|
|
3688
|
+
await proxy._write_request_log(
|
|
3689
|
+
affinity_observation=affinity_observation,
|
|
3690
|
+
account_id=None,
|
|
3691
|
+
api_key=api_key,
|
|
3692
|
+
request_id=request_id,
|
|
3693
|
+
model=payload.model,
|
|
3694
|
+
latency_ms=int((clock.monotonic() - start) * 1000),
|
|
3695
|
+
status="error",
|
|
3696
|
+
error_code="no_accounts",
|
|
3697
|
+
error_message=retries_exhausted_msg,
|
|
3698
|
+
reasoning_effort=payload.reasoning.effort if payload.reasoning else None,
|
|
3699
|
+
transport=request_transport,
|
|
3700
|
+
upstream_transport=upstream_stream_transport,
|
|
3701
|
+
service_tier=payload.service_tier,
|
|
3702
|
+
requested_service_tier=payload.service_tier,
|
|
3703
|
+
useragent=useragent,
|
|
3704
|
+
useragent_group=useragent_group,
|
|
3705
|
+
conversation_id=conversation_id,
|
|
3706
|
+
client_ip=client_ip,
|
|
3707
|
+
)
|
|
3708
|
+
finally:
|
|
3709
|
+
if not upstream_transport_metric_recorded:
|
|
3710
|
+
_record_upstream_transport_metric_once(upstream_transport_metric_status or "error")
|
|
3711
|
+
for account_lease in account_leases:
|
|
3712
|
+
await proxy._load_balancer.release_account_lease(account_lease)
|
|
3713
|
+
if (
|
|
3714
|
+
not settled
|
|
3715
|
+
and not settlement.usage_settlement_transferred
|
|
3716
|
+
and api_key is not None
|
|
3717
|
+
and api_key_reservation is not None
|
|
3718
|
+
):
|
|
3719
|
+
|
|
3720
|
+
async def _release_reservation_then_drain_backoffs() -> None:
|
|
3721
|
+
nonlocal settled
|
|
3722
|
+
if pending_post_refresh_transient_penalties:
|
|
3723
|
+
# Mid-loop keyed health may already be queued. Prefer
|
|
3724
|
+
# settle-then-flush so cancel cleanup still applies the
|
|
3725
|
+
# deferred account penalty after the reservation closes.
|
|
3726
|
+
settled = await _drain_pending_post_refresh_penalty_on_terminal(settlement)
|
|
3727
|
+
if settled:
|
|
3728
|
+
return
|
|
3729
|
+
released = await proxy._release_unsettled_stream_api_key_usage(
|
|
3730
|
+
api_key=api_key,
|
|
3731
|
+
api_key_reservation=api_key_reservation,
|
|
3732
|
+
request_id=request_id,
|
|
3733
|
+
)
|
|
3734
|
+
if released:
|
|
3735
|
+
try:
|
|
3736
|
+
await proxy._drain_deferred_account_error_backoffs(deferred_account_error_backoffs)
|
|
3737
|
+
finally:
|
|
3738
|
+
# Confirmed release owns the same independent
|
|
3739
|
+
# post-settlement health lanes as confirmed settle.
|
|
3740
|
+
await _flush_or_schedule_pending_post_refresh_penalties()
|
|
3741
|
+
|
|
3742
|
+
release_coro = _release_reservation_then_drain_backoffs()
|
|
3743
|
+
current_task = asyncio.current_task()
|
|
3744
|
+
if current_task is not None and current_task.cancelling():
|
|
3745
|
+
proxy._schedule_cancel_safe_cleanup(
|
|
3746
|
+
release_coro,
|
|
3747
|
+
action="release_stream_api_key_reservation",
|
|
3748
|
+
request_id=request_id,
|
|
3749
|
+
)
|
|
3750
|
+
else:
|
|
3751
|
+
await release_coro
|
|
3752
|
+
elif settled and deferred_account_error_backoffs:
|
|
3753
|
+
try:
|
|
3754
|
+
await proxy._drain_deferred_account_error_backoffs(deferred_account_error_backoffs)
|
|
3755
|
+
finally:
|
|
3756
|
+
# A retained backoff failure must not block ownership of
|
|
3757
|
+
# later penalties left by a cancelled flush.
|
|
3758
|
+
await _flush_or_schedule_pending_post_refresh_penalties()
|
|
3759
|
+
elif settled:
|
|
3760
|
+
# Finish penalties left behind if a prior flush was cancelled
|
|
3761
|
+
# mid-loop after settlement already closed the reservation.
|
|
3762
|
+
await _flush_or_schedule_pending_post_refresh_penalties()
|