codexLB-up 1.25.0b10__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- app/__init__.py +10 -0
- app/admin_cli.py +494 -0
- app/cli.py +339 -0
- app/codex_sessions_retag.py +487 -0
- app/core/__init__.py +0 -0
- app/core/audit/__init__.py +0 -0
- app/core/audit/service.py +216 -0
- app/core/audit/types.py +94 -0
- app/core/auth/__init__.py +236 -0
- app/core/auth/api_key_cache.py +65 -0
- app/core/auth/dashboard_access.py +429 -0
- app/core/auth/dashboard_mode.py +171 -0
- app/core/auth/dashboard_session_ttl.py +74 -0
- app/core/auth/dashboard_users_cache.py +120 -0
- app/core/auth/dependencies.py +710 -0
- app/core/auth/external_identity.py +78 -0
- app/core/auth/guardian.py +374 -0
- app/core/auth/models.py +49 -0
- app/core/auth/providers/__init__.py +129 -0
- app/core/auth/providers/oidc.py +838 -0
- app/core/auth/providers/registry.py +118 -0
- app/core/auth/refresh.py +394 -0
- app/core/auth/step_up.py +117 -0
- app/core/auth/totp.py +81 -0
- app/core/balancer/__init__.py +71 -0
- app/core/balancer/logic.py +1612 -0
- app/core/balancer/rendezvous_hash.py +21 -0
- app/core/balancer/types.py +21 -0
- app/core/bootstrap.py +172 -0
- app/core/cache/invalidation.py +426 -0
- app/core/clients/__init__.py +0 -0
- app/core/clients/codex.py +996 -0
- app/core/clients/codex_version.py +187 -0
- app/core/clients/codex_version_snapshot.py +2 -0
- app/core/clients/files.py +534 -0
- app/core/clients/headers.py +29 -0
- app/core/clients/http.py +511 -0
- app/core/clients/model_fetcher.py +276 -0
- app/core/clients/native_egress.py +1304 -0
- app/core/clients/oauth.py +477 -0
- app/core/clients/proxy.py +6089 -0
- app/core/clients/proxy_websocket.py +1411 -0
- app/core/clients/rate_limit_reset_credits.py +412 -0
- app/core/clients/stream_errors.py +12 -0
- app/core/clients/thread_cache_identity.py +347 -0
- app/core/clients/usage.py +467 -0
- app/core/clock.py +170 -0
- app/core/config/__init__.py +0 -0
- app/core/config/background_jobs.py +73 -0
- app/core/config/context_window_overrides.py +166 -0
- app/core/config/dashboard_overrides.py +145 -0
- app/core/config/inheritable.py +47 -0
- app/core/config/key_fingerprint.py +115 -0
- app/core/config/settings.py +872 -0
- app/core/config/settings_cache.py +100 -0
- app/core/config/spool_retention.py +217 -0
- app/core/config/tiers.py +174 -0
- app/core/conversation_archive.py +568 -0
- app/core/crypto.py +37 -0
- app/core/draining.py +11 -0
- app/core/errors.py +365 -0
- app/core/exceptions.py +155 -0
- app/core/handlers/__init__.py +3 -0
- app/core/handlers/exceptions.py +478 -0
- app/core/http_protocol.py +176 -0
- app/core/http_protocol_httptools.py +204 -0
- app/core/ingress_limits.py +17 -0
- app/core/metadata_store.py +18 -0
- app/core/metrics/__init__.py +0 -0
- app/core/metrics/middleware.py +74 -0
- app/core/metrics/prometheus.py +604 -0
- app/core/middleware/__init__.py +25 -0
- app/core/middleware/api_firewall.py +111 -0
- app/core/middleware/app_version.py +37 -0
- app/core/middleware/dashboard_auth_proxy.py +55 -0
- app/core/middleware/dashboard_csrf.py +177 -0
- app/core/middleware/dashboard_gzip.py +41 -0
- app/core/middleware/dashboard_overrides.py +59 -0
- app/core/middleware/firewall_cache.py +91 -0
- app/core/middleware/inflight.py +126 -0
- app/core/middleware/multipart_content_encoding.py +129 -0
- app/core/middleware/path_rewrite.py +148 -0
- app/core/middleware/request_body_limit.py +196 -0
- app/core/middleware/request_decompression.py +209 -0
- app/core/middleware/request_id.py +72 -0
- app/core/middleware/required_capability_http.py +116 -0
- app/core/middleware/trusted_proxy_headers.py +36 -0
- app/core/multipart.py +237 -0
- app/core/multipart_fields.py +91 -0
- app/core/openai/__init__.py +0 -0
- app/core/openai/chat_requests.py +493 -0
- app/core/openai/chat_responses.py +931 -0
- app/core/openai/contracts.py +46 -0
- app/core/openai/exceptions.py +24 -0
- app/core/openai/host_models.py +14 -0
- app/core/openai/images.py +413 -0
- app/core/openai/message_coercion.py +376 -0
- app/core/openai/model_refresh_scheduler.py +423 -0
- app/core/openai/model_registry.py +997 -0
- app/core/openai/model_registry_store.py +548 -0
- app/core/openai/models.py +172 -0
- app/core/openai/parsing.py +97 -0
- app/core/openai/requests.py +1980 -0
- app/core/openai/strict_schema.py +199 -0
- app/core/openai/tool_call_safety.py +99 -0
- app/core/openai/v1_requests.py +187 -0
- app/core/plan_types.py +90 -0
- app/core/prestop.py +202 -0
- app/core/rate_limiter/__init__.py +0 -0
- app/core/rate_limiter/db_rate_limiter.py +215 -0
- app/core/request_locality.py +473 -0
- app/core/resilience/__init__.py +0 -0
- app/core/resilience/backpressure.py +57 -0
- app/core/resilience/bulkhead.py +202 -0
- app/core/resilience/circuit_breaker.py +150 -0
- app/core/resilience/degradation.py +55 -0
- app/core/resilience/loop_lag_monitor.py +60 -0
- app/core/resilience/memory_monitor.py +137 -0
- app/core/resilience/network_recovery.py +256 -0
- app/core/resilience/overload.py +98 -0
- app/core/resilience/toggles.py +114 -0
- app/core/retention/__init__.py +0 -0
- app/core/retention/job.py +240 -0
- app/core/retention/scheduler.py +69 -0
- app/core/runtime_logging.py +601 -0
- app/core/scheduling/__init__.py +0 -0
- app/core/scheduling/leader_election.py +993 -0
- app/core/scheduling/leader_election_handle.py +25 -0
- app/core/server.py +140 -0
- app/core/shutdown.py +326 -0
- app/core/socket_peer.py +22 -0
- app/core/startup.py +41 -0
- app/core/timeout_invariants.py +347 -0
- app/core/tracing/__init__.py +0 -0
- app/core/tracing/otel.py +207 -0
- app/core/types.py +6 -0
- app/core/upstream_proxy/__init__.py +13 -0
- app/core/upstream_proxy/cache.py +130 -0
- app/core/upstream_proxy/resolver.py +189 -0
- app/core/upstream_proxy/types.py +104 -0
- app/core/usage/__init__.py +407 -0
- app/core/usage/depletion.py +109 -0
- app/core/usage/live_hub.py +41 -0
- app/core/usage/live_snapshots.py +174 -0
- app/core/usage/logs.py +212 -0
- app/core/usage/metadata_scheduler.py +118 -0
- app/core/usage/models.py +54 -0
- app/core/usage/pricing.py +576 -0
- app/core/usage/pricing_catalog.py +241 -0
- app/core/usage/pricing_snapshot.json +821 -0
- app/core/usage/quota.py +133 -0
- app/core/usage/refresh_policy.py +21 -0
- app/core/usage/refresh_scheduler.py +713 -0
- app/core/usage/reset_credits_refresh_scheduler.py +435 -0
- app/core/usage/types.py +168 -0
- app/core/utils/__init__.py +0 -0
- app/core/utils/json_guards.py +18 -0
- app/core/utils/locks.py +33 -0
- app/core/utils/masking.py +20 -0
- app/core/utils/proxy_env.py +104 -0
- app/core/utils/request_id.py +65 -0
- app/core/utils/retry.py +77 -0
- app/core/utils/shared_future.py +172 -0
- app/core/utils/sse.py +286 -0
- app/core/utils/time.py +24 -0
- app/db/__init__.py +0 -0
- app/db/account_identity_lock.py +56 -0
- app/db/alembic/__init__.py +1 -0
- app/db/alembic/env.py +70 -0
- app/db/alembic/revision_ids.py +43 -0
- app/db/alembic/versions/20260213_000000_base_schema.py +248 -0
- app/db/alembic/versions/20260213_000100_normalize_account_plan_types.py +45 -0
- app/db/alembic/versions/20260213_000200_add_request_logs_reasoning_effort.py +45 -0
- app/db/alembic/versions/20260213_000300_add_accounts_reset_at.py +45 -0
- app/db/alembic/versions/20260213_000400_add_accounts_chatgpt_account_id.py +45 -0
- app/db/alembic/versions/20260213_000500_add_dashboard_settings.py +70 -0
- app/db/alembic/versions/20260213_000600_add_dashboard_settings_totp.py +64 -0
- app/db/alembic/versions/20260213_000700_add_dashboard_settings_password.py +59 -0
- app/db/alembic/versions/20260213_000800_add_api_keys.py +87 -0
- app/db/alembic/versions/20260214_000000_add_api_key_limits.py +143 -0
- app/db/alembic/versions/20260215_000000_add_idx_logs_requested_at.py +38 -0
- app/db/alembic/versions/20260218_000000_add_api_key_usage_reservations.py +117 -0
- app/db/alembic/versions/20260218_000100_add_import_without_overwrite_and_drop_accounts_email_unique.py +198 -0
- app/db/alembic/versions/20260225_000000_add_dashboard_settings_routing_strategy.py +53 -0
- app/db/alembic/versions/20260228_020000_align_api_key_limit_enum_types.py +104 -0
- app/db/alembic/versions/20260228_030000_add_api_firewall_allowlist.py +42 -0
- app/db/alembic/versions/20260306_000000_add_request_logs_service_tier.py +45 -0
- app/db/alembic/versions/20260307_000000_add_api_key_enforcement_fields.py +56 -0
- app/db/alembic/versions/20260308_000000_add_sqlite_performance_indexes.py +41 -0
- app/db/alembic/versions/20260309_000000_add_additional_usage_history.py +71 -0
- app/db/alembic/versions/20260309_000000_request_logs_nullable_account_id.py +28 -0
- app/db/alembic/versions/20260310_000000_add_request_logs_transport.py +45 -0
- app/db/alembic/versions/20260310_000000_fix_postgresql_enum_value_casing.py +79 -0
- app/db/alembic/versions/20260310_120000_add_sticky_session_kinds_and_affinity_ttl.py +110 -0
- app/db/alembic/versions/20260312_000000_add_additional_usage_quota_key.py +193 -0
- app/db/alembic/versions/20260312_000000_split_sticky_sessions_primary_key_by_kind.py +96 -0
- app/db/alembic/versions/20260312_010000_merge_additional_usage_and_sticky_session_heads.py +25 -0
- app/db/alembic/versions/20260312_020000_merge_request_logs_transport_and_feature_heads.py +25 -0
- app/db/alembic/versions/20260312_020000_merge_request_logs_transport_and_sticky_heads.py +25 -0
- app/db/alembic/versions/20260312_120000_add_dashboard_upstream_stream_transport.py +62 -0
- app/db/alembic/versions/20260313_024500_merge_request_log_transport_heads.py +27 -0
- app/db/alembic/versions/20260319_100937_increase_prompt_cache_ttl_to_1800s.py +73 -0
- app/db/alembic/versions/20260319_111500_merge_prompt_cache_ttl_and_upstream_transport_heads.py +25 -0
- app/db/alembic/versions/20260319_183000_normalize_sqlite_account_status_casing.py +48 -0
- app/db/alembic/versions/20260320_000000_add_request_log_requested_actual_tiers.py +51 -0
- app/db/alembic/versions/20260321_190000_tighten_dashboard_db_indexes.py +64 -0
- app/db/alembic/versions/20260321_210000_merge_request_log_tiers_and_dashboard_index_heads.py +25 -0
- app/db/alembic/versions/20260325_000000_add_request_log_cost.py +193 -0
- app/db/alembic/versions/20260328_000000_add_rate_limit_attempts.py +48 -0
- app/db/alembic/versions/20260328_120000_add_request_log_ttft.py +37 -0
- app/db/alembic/versions/20260328_130000_add_audit_logs_table.py +45 -0
- app/db/alembic/versions/20260328_140000_add_scheduler_leader_table.py +39 -0
- app/db/alembic/versions/20260330_000000_add_cache_locality_settings.py +69 -0
- app/db/alembic/versions/20260330_010000_merge_scheduler_leader_and_cache_locality_heads.py +25 -0
- app/db/alembic/versions/20260330_020000_add_bridge_ring_members.py +46 -0
- app/db/alembic/versions/20260401_000000_add_api_key_enforced_service_tier.py +52 -0
- app/db/alembic/versions/20260401_000000_add_cache_invalidation.py +41 -0
- app/db/alembic/versions/20260401_020000_merge_cache_invalidation_and_api_key_service_tier_heads.py +22 -0
- app/db/alembic/versions/20260402_000000_switch_dashboard_routing_default_to_capacity_weighted.py +66 -0
- app/db/alembic/versions/20260403_000000_add_credit_api_key_limit_values.py +44 -0
- app/db/alembic/versions/20260406_000000_add_api_key_account_assignments.py +57 -0
- app/db/alembic/versions/20260406_010000_add_api_key_assignment_scope_flag.py +75 -0
- app/db/alembic/versions/20260407_000000_add_bridge_gateway_safe_mode.py +51 -0
- app/db/alembic/versions/20260407_010000_merge_api_key_assignment_and_bridge_gateway_heads.py +25 -0
- app/db/alembic/versions/20260408_000000_switch_import_without_overwrite_default_to_true.py +73 -0
- app/db/alembic/versions/20260408_010000_merge_import_without_overwrite_and_assignment_heads.py +25 -0
- app/db/alembic/versions/20260409_000000_add_http_bridge_sessions.py +130 -0
- app/db/alembic/versions/20260409_000000_switch_sticky_threads_and_prefer_earlier_reset_defaults_to_true.py +89 -0
- app/db/alembic/versions/20260409_010000_add_dashboard_settings_bootstrap_token.py +57 -0
- app/db/alembic/versions/20260409_010000_merge_http_bridge_and_import_without_overwrite_heads.py +25 -0
- app/db/alembic/versions/20260409_020000_fix_http_bridge_last_seen_index.py +46 -0
- app/db/alembic/versions/20260410_000000_backfill_pristine_dashboard_settings_defaults.py +77 -0
- app/db/alembic/versions/20260410_030000_restore_import_without_overwrite_default_true.py +73 -0
- app/db/alembic/versions/20260410_040000_merge_dashboard_defaults_and_import_default_heads.py +25 -0
- app/db/alembic/versions/20260413_000000_add_accounts_blocked_at.py +44 -0
- app/db/alembic/versions/20260415_160000_add_automations_tables.py +189 -0
- app/db/alembic/versions/20260415_160000_add_request_logs_response_lookup_index.py +67 -0
- app/db/alembic/versions/20260417_000000_add_request_log_plan_type.py +44 -0
- app/db/alembic/versions/20260418_190000_add_automation_runs_cycle_key.py +87 -0
- app/db/alembic/versions/20260419_000000_add_automation_run_cycle_metadata.py +56 -0
- app/db/alembic/versions/20260419_020000_add_automation_run_cycles_snapshot_tables.py +342 -0
- app/db/alembic/versions/20260421_000000_add_dashboard_session_ttl_setting.py +51 -0
- app/db/alembic/versions/20260421_120000_merge_request_log_lookup_and_plan_type_heads.py +25 -0
- app/db/alembic/versions/20260421_130000_merge_automation_and_request_log_heads.py +25 -0
- app/db/alembic/versions/20260422_103000_normalize_legacy_automation_run_cycle_keys.py +648 -0
- app/db/alembic/versions/20260423_120000_add_api_key_limit_reset_at_index.py +43 -0
- app/db/alembic/versions/20260424_000000_merge_dashboard_session_ttl_and_request_log_heads.py +25 -0
- app/db/alembic/versions/20260426_000000_add_dashboard_relative_availability_settings.py +68 -0
- app/db/alembic/versions/20260427_130000_add_warmup_model_and_request_kind.py +75 -0
- app/db/alembic/versions/20260506_000000_add_account_routing_policy.py +37 -0
- app/db/alembic/versions/20260509_000000_add_api_key_traffic_class.py +37 -0
- app/db/alembic/versions/20260509_010000_add_additional_quota_routing_policies.py +37 -0
- app/db/alembic/versions/20260513_000000_add_accounts_alias.py +44 -0
- app/db/alembic/versions/20260513_000000_add_api_key_apply_to_codex_model.py +59 -0
- app/db/alembic/versions/20260514_000000_add_request_logs_api_key_time_index.py +43 -0
- app/db/alembic/versions/20260515_000000_add_prefer_earlier_reset_window.py +47 -0
- app/db/alembic/versions/20260515_000000_soft_delete_request_logs_on_account_delete.py +83 -0
- app/db/alembic/versions/20260515_020000_add_split_sticky_budget_thresholds.py +62 -0
- app/db/alembic/versions/20260516_000000_add_sqlite_hot_path_indexes.py +75 -0
- app/db/alembic/versions/20260517_000000_merge_sqlite_hot_path_and_request_log_delete_heads.py +24 -0
- app/db/alembic/versions/20260518_000000_add_http_bridge_durable_input_prefix.py +44 -0
- app/db/alembic/versions/20260518_010000_merge_http_bridge_and_sqlite_recovery_heads.py +25 -0
- app/db/alembic/versions/20260518_120000_add_dashboard_guest_access.py +57 -0
- app/db/alembic/versions/20260520_000000_merge_api_key_and_http_bridge_heads.py +26 -0
- app/db/alembic/versions/20260520_010000_add_request_logs_api_key_account_index.py +45 -0
- app/db/alembic/versions/20260520_010000_merge_automation_and_api_key_heads.py +26 -0
- app/db/alembic/versions/20260520_030000_add_quota_planner.py +194 -0
- app/db/alembic/versions/20260520_040000_merge_quota_planner_and_request_log_index_heads.py +24 -0
- app/db/alembic/versions/20260521_000000_add_account_security_work_authorized.py +46 -0
- app/db/alembic/versions/20260521_000000_merge_dashboard_guest_and_api_key_heads.py +26 -0
- app/db/alembic/versions/20260521_000000_merge_prefer_reset_window_head.py +26 -0
- app/db/alembic/versions/20260521_000000_merge_warmup_and_request_log_index_heads.py +25 -0
- app/db/alembic/versions/20260522_000000_add_limit_warmup_trigger.py +195 -0
- app/db/alembic/versions/20260524_000000_merge_accounts_alias_and_reset_window_heads.py +24 -0
- app/db/alembic/versions/20260525_000000_add_usage_raw_window_latest_index.py +31 -0
- app/db/alembic/versions/20260526_000000_add_request_log_failure_metadata.py +100 -0
- app/db/alembic/versions/20260531_000000_add_account_workspace_identity.py +39 -0
- app/db/alembic/versions/20260531_000000_add_single_account_routing.py +42 -0
- app/db/alembic/versions/20260601_000000_merge_relative_availability_and_usage_raw_heads.py +24 -0
- app/db/alembic/versions/20260601_010000_add_api_key_usage_sections.py +58 -0
- app/db/alembic/versions/20260601_010000_add_routing_policy_persistence.py +67 -0
- app/db/alembic/versions/20260601_010000_merge_security_work_and_relative_availability_heads.py +25 -0
- app/db/alembic/versions/20260601_010000_merge_split_sticky_and_relative_usage_heads.py +25 -0
- app/db/alembic/versions/20260601_010000_merge_warmup_and_relative_availability_heads.py +25 -0
- app/db/alembic/versions/20260601_020000_merge_additional_quota_routing_and_relative_availability_heads.py +25 -0
- app/db/alembic/versions/20260601_020000_merge_quota_planner_and_relative_availability_heads.py +25 -0
- app/db/alembic/versions/20260601_020000_merge_reset_window_and_relative_availability_heads.py +25 -0
- app/db/alembic/versions/20260601_020000_merge_routing_policy_and_split_threshold_heads.py +24 -0
- app/db/alembic/versions/20260601_020000_merge_warmup_and_request_log_failure_heads.py +25 -0
- app/db/alembic/versions/20260601_030000_merge_single_account_and_routing_quota_heads.py +29 -0
- app/db/alembic/versions/20260601_040000_merge_security_work_and_routing_consolidation_heads.py +25 -0
- app/db/alembic/versions/20260601_050000_merge_quota_planner_and_routing_consolidation_heads.py +25 -0
- app/db/alembic/versions/20260602_050000_add_upstream_proxy_routing.py +164 -0
- app/db/alembic/versions/20260602_060000_merge_account_workspace_and_failure_heads.py +25 -0
- app/db/alembic/versions/20260602_070000_add_request_log_useragent_fields.py +86 -0
- app/db/alembic/versions/20260602_070000_merge_quota_planner_and_account_workspace_heads.py +25 -0
- app/db/alembic/versions/20260602_080000_merge_upstream_proxy_and_quota_planner_heads.py +25 -0
- app/db/alembic/versions/20260602_080000_merge_useragent_and_upstream_proxy_heads.py +24 -0
- app/db/alembic/versions/20260603_000000_add_weekly_pace_working_days.py +52 -0
- app/db/alembic/versions/20260603_000000_free_account_monthly_window.py +60 -0
- app/db/alembic/versions/20260604_000000_add_reauth_required_account_status.py +159 -0
- app/db/alembic/versions/20260604_010000_merge_automations_and_reauth_status_heads.py +25 -0
- app/db/alembic/versions/20260604_010000_merge_dashboard_guest_and_reauth_status_heads.py +25 -0
- app/db/alembic/versions/20260604_010000_merge_useragent_and_reauth_heads.py +24 -0
- app/db/alembic/versions/20260607_000000_merge_weekly_monthly_useragent_heads.py +28 -0
- app/db/alembic/versions/20260608_000000_add_hide_upstream_quota_from_api_keys.py +50 -0
- app/db/alembic/versions/20260611_000000_add_request_log_archive_request_id.py +47 -0
- app/db/alembic/versions/20260611_000000_merge_automations_and_weekly_monthly_useragent_heads.py +26 -0
- app/db/alembic/versions/20260611_000000_merge_dashboard_guest_and_weekly_useragent_heads.py +26 -0
- app/db/alembic/versions/20260611_010000_add_request_log_client_ip.py +57 -0
- app/db/alembic/versions/20260611_020000_merge_request_log_archive_and_client_ip_heads.py +26 -0
- app/db/alembic/versions/20260611_030000_merge_dashboard_guest_and_request_log_client_ip_heads.py +26 -0
- app/db/alembic/versions/20260613_000000_add_accounts_codex_installation_id.py +57 -0
- app/db/alembic/versions/20260613_010000_merge_installation_id_and_dashboard_weekly_heads.py +26 -0
- app/db/alembic/versions/20260615_000000_merge_automations_and_dashboard_guest_heads.py +26 -0
- app/db/alembic/versions/20260623_000000_add_limit_warmup_exhausted_threshold.py +51 -0
- app/db/alembic/versions/20260626_000000_add_http_downstream_transport_policy.py +46 -0
- app/db/alembic/versions/20260626_010000_add_request_logs_upstream_transport.py +31 -0
- app/db/alembic/versions/20260629_000000_add_dashboard_query_hot_path_indexes.py +53 -0
- app/db/alembic/versions/20260630_000000_add_limit_warmup_staggered_idle.py +47 -0
- app/db/alembic/versions/20260630_000000_merge_request_log_client_ip_and_dashboard_hot_path_heads.py +26 -0
- app/db/alembic/versions/20260630_010000_merge_warmup_and_request_log_dashboard_heads.py +26 -0
- app/db/alembic/versions/20260630_020000_merge_automations_and_main_heads.py +26 -0
- app/db/alembic/versions/20260630_020000_merge_warmup_threshold_and_main_heads.py +26 -0
- app/db/alembic/versions/20260630_030000_add_automation_run_model_snapshot.py +59 -0
- app/db/alembic/versions/20260630_040000_merge_automation_snapshot_and_warmup_threshold_heads.py +26 -0
- app/db/alembic/versions/20260630_050000_add_automation_run_prompt_snapshot.py +52 -0
- app/db/alembic/versions/20260701_000000_add_weekly_pace_smoothing_minutes.py +51 -0
- app/db/alembic/versions/20260702_000000_add_model_sources.py +176 -0
- app/db/alembic/versions/20260705_000000_add_model_source_audio_transcriptions.py +50 -0
- app/db/alembic/versions/20260705_000000_harden_dashboard_session_ttl.py +62 -0
- app/db/alembic/versions/20260706_000000_add_model_source_audio_per_minute.py +43 -0
- app/db/alembic/versions/20260707_000000_merge_model_sources_and_session_ttl.py +24 -0
- app/db/alembic/versions/20260707_010000_merge_automations_and_model_sources_heads.py +24 -0
- app/db/alembic/versions/20260707_020000_add_automation_job_account_scope.py +125 -0
- app/db/alembic/versions/20260709_000000_add_ttft_phase_observability.py +66 -0
- app/db/alembic/versions/20260711_000000_add_dashboard_prohibit_fast_mode.py +42 -0
- app/db/alembic/versions/20260711_010000_add_account_chatgpt_user_id.py +40 -0
- app/db/alembic/versions/20260711_020000_add_dashboard_account_concurrency_caps.py +71 -0
- app/db/alembic/versions/20260711_030000_add_limit_warmup_idle_threshold.py +51 -0
- app/db/alembic/versions/20260712_010000_add_account_usage_rollups.py +56 -0
- app/db/alembic/versions/20260712_020000_add_api_key_usage_rollups.py +53 -0
- app/db/alembic/versions/20260713_020000_add_model_registry_snapshot.py +42 -0
- app/db/alembic/versions/20260713_040000_add_account_refresh_claims.py +42 -0
- app/db/alembic/versions/20260713_040000_add_replica_guardrails.py +64 -0
- app/db/alembic/versions/20260713_070000_add_reset_credit_redeem_tables.py +58 -0
- app/db/alembic/versions/20260714_000000_add_oauth_flow_states.py +63 -0
- app/db/alembic/versions/20260715_000000_add_request_log_queue_latency.py +44 -0
- app/db/alembic/versions/20260716_000000_add_oauth_device_flow_slots.py +41 -0
- app/db/alembic/versions/20260716_010000_add_dashboard_retention_settings.py +48 -0
- app/db/alembic/versions/20260716_010000_add_reset_credit_display_settings.py +73 -0
- app/db/alembic/versions/20260717_000000_add_http_bridge_retry_circuits.py +47 -0
- app/db/alembic/versions/20260717_000000_merge_retention_and_reset_credit_display_heads.py +26 -0
- app/db/alembic/versions/20260717_000000_optimize_dashboard_hot_path_indexes.py +162 -0
- app/db/alembic/versions/20260717_000001_merge_retry_circuits_and_dashboard_indexes.py +22 -0
- app/db/alembic/versions/20260720_000000_add_request_log_conversation_id.py +57 -0
- app/db/alembic/versions/20260722_000000_backfill_request_log_useragent_families.py +40 -0
- app/db/alembic/versions/20260724_000000_add_request_usage_time_rollups.py +176 -0
- app/db/alembic/versions/20260724_000000_merge_request_log_schema_heads.py +33 -0
- app/db/alembic/versions/20260725_000000_add_http_bridge_pending_tool_calls.py +43 -0
- app/db/alembic/versions/20260726_000000_add_account_plan_downgrade_observations.py +44 -0
- app/db/alembic/versions/20260726_000000_repair_request_usage_rollups_after_merge.py +37 -0
- app/db/alembic/versions/20260727_000000_add_sticky_session_continuity_abandoned_at.py +45 -0
- app/db/alembic/versions/20260728_000000_merge_pending_tool_calls_and_rollup_repair_heads.py +31 -0
- app/db/alembic/versions/20260730_000000_add_api_key_fair_share_threshold.py +44 -0
- app/db/alembic/versions/20260730_000000_add_http_bridge_recovery_attempts.py +60 -0
- app/db/alembic/versions/20260731_000000_add_capability_lineage_markers.py +38 -0
- app/db/alembic/versions/20260803_000000_merge_http_bridge_recovery_and_capability_lineage_heads.py +30 -0
- app/db/alembic/versions/20260804_000000_add_http_bridge_operations.py +63 -0
- app/db/alembic/versions/20260804_000001_add_global_http_bridge_operation_fingerprint.py +53 -0
- app/db/alembic/versions/20260804_230000_add_request_log_connection_request_kind.py +43 -0
- app/db/alembic/versions/20260805_000000_add_http_bridge_operation_spool.py +75 -0
- app/db/alembic/versions/20260805_000001_finalize_http_bridge_operation_spool.py +65 -0
- app/db/alembic/versions/20260806_000000_add_additional_usage_alias_probe_indexes.py +84 -0
- app/db/alembic/versions/20260806_000000_add_anonymous_telemetry.py +49 -0
- app/db/alembic/versions/20260806_010000_add_conversation_presence_rollup.py +72 -0
- app/db/alembic/versions/20260806_020000_add_usage_history_bulk_covering_indexes.py +99 -0
- app/db/alembic/versions/20260806_030000_add_api_key_allowed_reasoning_efforts.py +65 -0
- app/db/alembic/versions/20260806_120000_add_http_bridge_owner_process_epoch.py +57 -0
- app/db/alembic/versions/20260807_000000_merge_http_bridge_operation_spool_and_latest_main.py +24 -0
- app/db/alembic/versions/20260808_000000_tune_usage_history_autovacuum.py +54 -0
- app/db/alembic/versions/20260810_000000_add_http_bridge_recovery_dispatch_count.py +44 -0
- app/db/alembic/versions/20260811_000000_add_hourly_rollup_cancelled_count.py +92 -0
- app/db/alembic/versions/20260812_000000_merge_recovery_dispatch_and_hourly_cancelled_heads.py +25 -0
- app/db/alembic/versions/20260812_120000_add_sticky_abandonment_scope.py +43 -0
- app/db/alembic/versions/20260813_000000_add_file_account_pins.py +40 -0
- app/db/alembic/versions/20260816_000000_add_account_pending_deletion.py +81 -0
- app/db/alembic/versions/20260816_000000_add_model_source_embeddings.py +50 -0
- app/db/alembic/versions/20260821_000000_add_retry_circuit_admission_generation.py +49 -0
- app/db/alembic/versions/20260826_000000_add_http_bridge_event_chunks.py +107 -0
- app/db/alembic/versions/20260828_000000_add_accounts_chatgpt_identity_index.py +55 -0
- app/db/alembic/versions/20260830_000000_add_quota_warmup_claim_expiry.py +83 -0
- app/db/alembic/versions/20260908_000000_add_guest_session_generation.py +37 -0
- app/db/alembic/versions/20260908_000000_add_subscription_overflow.py +140 -0
- app/db/alembic/versions/20260908_000000_replace_upstream_stream_transport_default_sentinel.py +50 -0
- app/db/alembic/versions/20260908_020000_merge_overflow_transport_heads.py +27 -0
- app/db/alembic/versions/20260909_000000_add_dashboard_roles.py +72 -0
- app/db/alembic/versions/20260909_010000_add_dashboard_users.py +183 -0
- app/db/alembic/versions/20260909_020000_reproject_compat_admin_credentials.py +105 -0
- app/db/alembic/versions/20260909_030000_add_audit_actor_columns.py +103 -0
- app/db/alembic/versions/20260909_030000_dashboard_resilience_toggle_settings.py +63 -0
- app/db/alembic/versions/20260909_040000_add_dashboard_user_invites.py +54 -0
- app/db/alembic/versions/20260909_040000_dashboard_timeout_settings.py +68 -0
- app/db/alembic/versions/20260909_050000_dashboard_routing_overload_settings.py +65 -0
- app/db/alembic/versions/20260909_060000_add_report_rollup.py +74 -0
- app/db/alembic/versions/20260909_070000_automation_run_claim_budget.py +50 -0
- app/db/alembic/versions/20260909_080000_dashboard_stream_bridge_budgets.py +62 -0
- app/db/alembic/versions/20260909_090000_dashboard_background_job_toggles.py +63 -0
- app/db/alembic/versions/20260909_100000_dashboard_codex_prewarm.py +49 -0
- app/db/alembic/versions/20260909_110000_model_context_window_overrides.py +45 -0
- app/db/alembic/versions/20260909_120000_dashboard_conversation_archive.py +48 -0
- app/db/alembic/versions/20260909_130000_add_request_logs_live_facet_indexes.py +96 -0
- app/db/alembic/versions/20260910_000000_add_totp_required_for_admin_role.py +44 -0
- app/db/alembic/versions/20260910_000000_request_logs_missing_cost_index.py +52 -0
- app/db/alembic/versions/20260910_010000_add_dashboard_auth_providers.py +134 -0
- app/db/alembic/versions/20260910_010000_dashboard_spool_retention.py +52 -0
- app/db/alembic/versions/20260910_020000_add_dashboard_role_mappings.py +74 -0
- app/db/alembic/versions/20260910_143000_request_log_affinity.py +28 -0
- app/db/alembic/versions/20260910_180000_merge_affinity_guest_heads.py +17 -0
- app/db/alembic/versions/20260910_200000_merge_affinity_identity_heads.py +17 -0
- app/db/alembic/versions/20260910_220000_merge_affinity_invite_heads.py +17 -0
- app/db/alembic/versions/20260911_000000_model_source_pins_kind_expires_index.py +43 -0
- app/db/alembic/versions/20260911_010000_merge_pin_index_and_affinity_heads.py +17 -0
- app/db/alembic/versions/20260911_020000_add_http_bridge_terminal_append_phase.py +54 -0
- app/db/alembic/versions/20260911_030000_add_local_login_policy.py +48 -0
- app/db/alembic/versions/20260911_040000_add_thread_cache_identity_mode.py +48 -0
- app/db/alembic/versions/20260911_060000_add_bridge_session_continuity_abandonment.py +40 -0
- app/db/alembic/versions/20260912_000000_merge_thread_cache_and_bridge_retirement_heads.py +22 -0
- app/db/alembic/versions/20260912_010000_drop_legacy_dashboard_credentials.py +140 -0
- app/db/alembic/versions/20260913_000000_add_oidc_provider_flow.py +134 -0
- app/db/alembic/versions/20260914_000000_add_scim_tokens.py +78 -0
- app/db/alembic/versions/20260914_000000_drop_subscription_overflow_schema.py +132 -0
- app/db/alembic/versions/20260918_000000_merge_scim_and_overflow_heads.py +17 -0
- app/db/alembic/versions/__init__.py +1 -0
- app/db/backup.py +64 -0
- app/db/migrate.py +1249 -0
- app/db/migration_lock.py +160 -0
- app/db/migration_url.py +18 -0
- app/db/models.py +3100 -0
- app/db/recover.py +244 -0
- app/db/session.py +1295 -0
- app/db/snapshot.py +46 -0
- app/db/sqlite_lock_retry.py +190 -0
- app/db/sqlite_utils.py +549 -0
- app/dependencies.py +462 -0
- app/main.py +1240 -0
- app/modules/__init__.py +0 -0
- app/modules/accounts/__init__.py +0 -0
- app/modules/accounts/api.py +426 -0
- app/modules/accounts/auth_manager.py +1439 -0
- app/modules/accounts/background_repository.py +160 -0
- app/modules/accounts/deletion.py +540 -0
- app/modules/accounts/mappers.py +654 -0
- app/modules/accounts/refresh_claims.py +352 -0
- app/modules/accounts/repository.py +1679 -0
- app/modules/accounts/schemas.py +308 -0
- app/modules/accounts/service.py +809 -0
- app/modules/accounts/usage_rollup.py +486 -0
- app/modules/accounts/usage_rollup_scheduler.py +75 -0
- app/modules/accounts/usage_time_rollup.py +1037 -0
- app/modules/accounts/usage_time_rollup_read.py +522 -0
- app/modules/api_keys/__init__.py +1 -0
- app/modules/api_keys/api.py +344 -0
- app/modules/api_keys/last_used_coalescer.py +216 -0
- app/modules/api_keys/limit_windows.py +41 -0
- app/modules/api_keys/repository.py +1200 -0
- app/modules/api_keys/reset_scheduler.py +85 -0
- app/modules/api_keys/schemas.py +135 -0
- app/modules/api_keys/service.py +2123 -0
- app/modules/audit/__init__.py +0 -0
- app/modules/audit/api.py +88 -0
- app/modules/audit/repository.py +74 -0
- app/modules/audit/schemas.py +33 -0
- app/modules/audit/service.py +71 -0
- app/modules/auth_providers/__init__.py +0 -0
- app/modules/auth_providers/api.py +127 -0
- app/modules/auth_providers/config.py +343 -0
- app/modules/auth_providers/repository.py +101 -0
- app/modules/auth_providers/schemas.py +88 -0
- app/modules/auth_providers/seed.py +105 -0
- app/modules/auth_providers/service.py +289 -0
- app/modules/automations/__init__.py +1 -0
- app/modules/automations/api.py +369 -0
- app/modules/automations/repository.py +1896 -0
- app/modules/automations/scheduler.py +107 -0
- app/modules/automations/schemas.py +158 -0
- app/modules/automations/service.py +2548 -0
- app/modules/cache_isolation_probe/__init__.py +0 -0
- app/modules/cache_isolation_probe/api.py +188 -0
- app/modules/cache_isolation_probe/prefix.py +144 -0
- app/modules/cache_isolation_probe/schemas.py +94 -0
- app/modules/cache_isolation_probe/sender.py +229 -0
- app/modules/cache_isolation_probe/service.py +525 -0
- app/modules/conversation_archive/__init__.py +1 -0
- app/modules/conversation_archive/api.py +101 -0
- app/modules/conversation_archive/schemas.py +28 -0
- app/modules/conversation_archive/service.py +165 -0
- app/modules/dashboard/__init__.py +1 -0
- app/modules/dashboard/api.py +106 -0
- app/modules/dashboard/builders.py +107 -0
- app/modules/dashboard/repository.py +304 -0
- app/modules/dashboard/schemas.py +148 -0
- app/modules/dashboard/service.py +534 -0
- app/modules/dashboard/timeframes.py +19 -0
- app/modules/dashboard/weekly_pace.py +617 -0
- app/modules/dashboard_auth/__init__.py +1 -0
- app/modules/dashboard_auth/api.py +1344 -0
- app/modules/dashboard_auth/oidc_api.py +753 -0
- app/modules/dashboard_auth/oidc_flows.py +330 -0
- app/modules/dashboard_auth/repository.py +452 -0
- app/modules/dashboard_auth/schemas.py +232 -0
- app/modules/dashboard_auth/service.py +1384 -0
- app/modules/dashboard_roles/__init__.py +0 -0
- app/modules/dashboard_roles/api.py +64 -0
- app/modules/dashboard_roles/repository.py +54 -0
- app/modules/dashboard_roles/schemas.py +29 -0
- app/modules/dashboard_roles/seed.py +80 -0
- app/modules/dashboard_roles/service.py +142 -0
- app/modules/dashboard_users/__init__.py +0 -0
- app/modules/dashboard_users/api.py +278 -0
- app/modules/dashboard_users/break_glass.py +169 -0
- app/modules/dashboard_users/credentials.py +18 -0
- app/modules/dashboard_users/identity_resolver.py +665 -0
- app/modules/dashboard_users/repository.py +725 -0
- app/modules/dashboard_users/schemas.py +117 -0
- app/modules/dashboard_users/service.py +1054 -0
- app/modules/firewall/__init__.py +1 -0
- app/modules/firewall/api.py +78 -0
- app/modules/firewall/repository.py +51 -0
- app/modules/firewall/schemas.py +28 -0
- app/modules/firewall/service.py +93 -0
- app/modules/fleet/__init__.py +1 -0
- app/modules/fleet/api.py +177 -0
- app/modules/fleet/mappers.py +56 -0
- app/modules/fleet/observability.py +372 -0
- app/modules/fleet/schemas.py +119 -0
- app/modules/health/__init__.py +0 -0
- app/modules/health/api.py +246 -0
- app/modules/health/schemas.py +27 -0
- app/modules/limit_warmup/__init__.py +1 -0
- app/modules/limit_warmup/repository.py +159 -0
- app/modules/limit_warmup/service.py +1001 -0
- app/modules/model_sources/__init__.py +1 -0
- app/modules/model_sources/api.py +99 -0
- app/modules/model_sources/catalog.py +307 -0
- app/modules/model_sources/forwarding.py +1435 -0
- app/modules/model_sources/projection.py +55 -0
- app/modules/model_sources/repository.py +201 -0
- app/modules/model_sources/schemas.py +79 -0
- app/modules/model_sources/selection.py +166 -0
- app/modules/model_sources/service.py +236 -0
- app/modules/oauth/__init__.py +0 -0
- app/modules/oauth/api.py +101 -0
- app/modules/oauth/repository.py +238 -0
- app/modules/oauth/schemas.py +49 -0
- app/modules/oauth/service.py +1196 -0
- app/modules/oauth/templates/oauth_success.html +122 -0
- app/modules/proxy/__init__.py +0 -0
- app/modules/proxy/_load_balancer/__init__.py +1 -0
- app/modules/proxy/_load_balancer/error_rate.py +116 -0
- app/modules/proxy/_load_balancer/latency_cohort.py +154 -0
- app/modules/proxy/_load_balancer/model_eligibility.py +250 -0
- app/modules/proxy/_load_balancer/opportunistic_admission.py +350 -0
- app/modules/proxy/_load_balancer/overload_backoff.py +394 -0
- app/modules/proxy/_load_balancer/sticky_selection.py +2417 -0
- app/modules/proxy/_load_balancer/throughput_cohort.py +233 -0
- app/modules/proxy/_load_balancer/ttft_cohort.py +214 -0
- app/modules/proxy/_load_balancer/tunables.py +97 -0
- app/modules/proxy/_load_balancer/types.py +96 -0
- app/modules/proxy/_load_balancer/unbound_selection.py +534 -0
- app/modules/proxy/_service/__init__.py +6 -0
- app/modules/proxy/_service/api_key_usage.py +745 -0
- app/modules/proxy/_service/clock_budget.py +38 -0
- app/modules/proxy/_service/codex_control.py +611 -0
- app/modules/proxy/_service/compact.py +2211 -0
- app/modules/proxy/_service/file_ops.py +749 -0
- app/modules/proxy/_service/http_bridge/__init__.py +3 -0
- app/modules/proxy/_service/http_bridge/accepted_replay.py +331 -0
- app/modules/proxy/_service/http_bridge/account_sessions.py +57 -0
- app/modules/proxy/_service/http_bridge/activity.py +178 -0
- app/modules/proxy/_service/http_bridge/helpers.py +3959 -0
- app/modules/proxy/_service/http_bridge/mixin.py +2435 -0
- app/modules/proxy/_service/http_bridge/owner_forwarding.py +609 -0
- app/modules/proxy/_service/http_bridge/protocol.py +167 -0
- app/modules/proxy/_service/http_bridge/proxy_failover.py +58 -0
- app/modules/proxy/_service/http_bridge/quarantine.py +443 -0
- app/modules/proxy/_service/http_bridge/request_submit.py +4582 -0
- app/modules/proxy/_service/http_bridge/retry_circuit.py +2442 -0
- app/modules/proxy/_service/http_bridge/service_stubs.py +492 -0
- app/modules/proxy/_service/http_bridge/session_registry.py +778 -0
- app/modules/proxy/_service/http_bridge/streaming.py +5328 -0
- app/modules/proxy/_service/http_bridge/upstream_events.py +4538 -0
- app/modules/proxy/_service/observability.py +410 -0
- app/modules/proxy/_service/rate_limit.py +361 -0
- app/modules/proxy/_service/realtime_live.py +655 -0
- app/modules/proxy/_service/refresh.py +154 -0
- app/modules/proxy/_service/request_log.py +585 -0
- app/modules/proxy/_service/response_create.py +930 -0
- app/modules/proxy/_service/streaming/__init__.py +3 -0
- app/modules/proxy/_service/streaming/helpers.py +1317 -0
- app/modules/proxy/_service/streaming/mixin.py +1100 -0
- app/modules/proxy/_service/streaming/protocol.py +32 -0
- app/modules/proxy/_service/streaming/retry.py +3762 -0
- app/modules/proxy/_service/support.py +2122 -0
- app/modules/proxy/_service/transcribe.py +434 -0
- app/modules/proxy/_service/warmup.py +483 -0
- app/modules/proxy/_service/websocket/__init__.py +3 -0
- app/modules/proxy/_service/websocket/helpers.py +2360 -0
- app/modules/proxy/_service/websocket/mixin.py +7285 -0
- app/modules/proxy/_service/websocket/protocol.py +74 -0
- app/modules/proxy/account_cache.py +229 -0
- app/modules/proxy/account_eligibility.py +87 -0
- app/modules/proxy/additional_model_limits.py +40 -0
- app/modules/proxy/affinity.py +830 -0
- app/modules/proxy/affinity_observation.py +101 -0
- app/modules/proxy/api.py +10254 -0
- app/modules/proxy/api_key_usage.py +113 -0
- app/modules/proxy/cap_partitioning.py +280 -0
- app/modules/proxy/capability_lineage.py +65 -0
- app/modules/proxy/capability_lineage_repository.py +82 -0
- app/modules/proxy/capability_routing.py +253 -0
- app/modules/proxy/continuity.py +112 -0
- app/modules/proxy/downstream_delivery.py +277 -0
- app/modules/proxy/durable_bridge_coordinator.py +1217 -0
- app/modules/proxy/durable_bridge_repository.py +4516 -0
- app/modules/proxy/durable_bridge_runtime.py +9 -0
- app/modules/proxy/durable_bridge_transcript_codec.py +122 -0
- app/modules/proxy/fair_share.py +131 -0
- app/modules/proxy/file_pin_repository.py +210 -0
- app/modules/proxy/helpers.py +439 -0
- app/modules/proxy/http_bridge_event_batcher.py +613 -0
- app/modules/proxy/http_bridge_forwarding.py +797 -0
- app/modules/proxy/http_continuation.py +76 -0
- app/modules/proxy/images_observability.py +62 -0
- app/modules/proxy/images_service.py +1054 -0
- app/modules/proxy/load_balancer.py +3016 -0
- app/modules/proxy/rate_limit_cache.py +46 -0
- app/modules/proxy/replay_safety.py +1063 -0
- app/modules/proxy/repo_bundle.py +31 -0
- app/modules/proxy/request_policy.py +1054 -0
- app/modules/proxy/ring_membership.py +251 -0
- app/modules/proxy/schemas.py +345 -0
- app/modules/proxy/selection_errors.py +41 -0
- app/modules/proxy/service.py +2600 -0
- app/modules/proxy/source_admission.py +125 -0
- app/modules/proxy/source_dispatch.py +999 -0
- app/modules/proxy/sticky_repository.py +831 -0
- app/modules/proxy/thread_anchors.py +491 -0
- app/modules/proxy/tool_call_dedupe.py +802 -0
- app/modules/proxy/types.py +54 -0
- app/modules/proxy/work_admission.py +121 -0
- app/modules/quota_planner/__init__.py +1 -0
- app/modules/quota_planner/api.py +276 -0
- app/modules/quota_planner/logic.py +743 -0
- app/modules/quota_planner/repository.py +722 -0
- app/modules/quota_planner/scheduler.py +212 -0
- app/modules/quota_planner/schemas.py +102 -0
- app/modules/quota_planner/warmup.py +693 -0
- app/modules/rate_limit_reset_credits/__init__.py +0 -0
- app/modules/rate_limit_reset_credits/api.py +601 -0
- app/modules/rate_limit_reset_credits/redeem_coordination.py +274 -0
- app/modules/rate_limit_reset_credits/store.py +114 -0
- app/modules/reports/__init__.py +1 -0
- app/modules/reports/api.py +134 -0
- app/modules/reports/cache.py +74 -0
- app/modules/reports/filters.py +27 -0
- app/modules/reports/repository.py +525 -0
- app/modules/reports/rollup.py +150 -0
- app/modules/reports/rollup_read.py +148 -0
- app/modules/reports/schemas.py +123 -0
- app/modules/reports/service.py +287 -0
- app/modules/reports/thread_identity.py +256 -0
- app/modules/request_logs/__init__.py +0 -0
- app/modules/request_logs/api.py +224 -0
- app/modules/request_logs/cost_backfill.py +154 -0
- app/modules/request_logs/mappers.py +97 -0
- app/modules/request_logs/repository.py +1765 -0
- app/modules/request_logs/schemas.py +145 -0
- app/modules/request_logs/service.py +348 -0
- app/modules/role_mappings/__init__.py +0 -0
- app/modules/role_mappings/api.py +206 -0
- app/modules/role_mappings/matching.py +55 -0
- app/modules/role_mappings/repository.py +106 -0
- app/modules/role_mappings/schemas.py +67 -0
- app/modules/role_mappings/service.py +334 -0
- app/modules/runtime/__init__.py +1 -0
- app/modules/runtime/api.py +18 -0
- app/modules/runtime/schemas.py +14 -0
- app/modules/runtime/service.py +218 -0
- app/modules/scim/__init__.py +1 -0
- app/modules/scim/api.py +194 -0
- app/modules/scim/dependencies.py +112 -0
- app/modules/scim/errors.py +77 -0
- app/modules/scim/management_api.py +156 -0
- app/modules/scim/repository.py +212 -0
- app/modules/scim/schemas.py +145 -0
- app/modules/scim/service.py +786 -0
- app/modules/scim/tokens.py +43 -0
- app/modules/settings/__init__.py +1 -0
- app/modules/settings/api.py +1677 -0
- app/modules/settings/repository.py +606 -0
- app/modules/settings/schemas.py +496 -0
- app/modules/settings/service.py +769 -0
- app/modules/shared/__init__.py +0 -0
- app/modules/shared/schemas.py +22 -0
- app/modules/sticky_sessions/__init__.py +1 -0
- app/modules/sticky_sessions/api.py +118 -0
- app/modules/sticky_sessions/cleanup_scheduler.py +487 -0
- app/modules/sticky_sessions/schemas.py +78 -0
- app/modules/sticky_sessions/service.py +196 -0
- app/modules/telemetry/__init__.py +5 -0
- app/modules/telemetry/api.py +115 -0
- app/modules/telemetry/clients.py +62 -0
- app/modules/telemetry/consent.py +115 -0
- app/modules/telemetry/scheduler.py +85 -0
- app/modules/telemetry/schemas.py +163 -0
- app/modules/telemetry/sender.py +221 -0
- app/modules/telemetry/snapshot.py +490 -0
- app/modules/usage/__init__.py +0 -0
- app/modules/usage/additional_quota_keys.py +379 -0
- app/modules/usage/api.py +45 -0
- app/modules/usage/background_repository.py +147 -0
- app/modules/usage/builders.py +559 -0
- app/modules/usage/depletion_service.py +277 -0
- app/modules/usage/live_ingest.py +356 -0
- app/modules/usage/mappers.py +29 -0
- app/modules/usage/plan_downgrade_observations.py +488 -0
- app/modules/usage/repository.py +1890 -0
- app/modules/usage/schemas.py +68 -0
- app/modules/usage/service.py +102 -0
- app/modules/usage/updater.py +1488 -0
- app/static/assets/Trans-Cj-J9mVA.js +1 -0
- app/static/assets/access-page-DHzZiXZi.js +1 -0
- app/static/assets/account-cost-donut-DGhfA4U8.js +1 -0
- app/static/assets/account-identifiers-CxbdLshZ.js +1 -0
- app/static/assets/account-status-DIsU5sUS.js +1 -0
- app/static/assets/account-trend-chart-BptlfwwQ.js +1 -0
- app/static/assets/accounts-page-Bbai8ift.js +3 -0
- app/static/assets/api-Dwa7XxPe.js +1 -0
- app/static/assets/api-client-dugX1Pti.js +55 -0
- app/static/assets/api-key-create-dialog-DnRm0oYN.js +1 -0
- app/static/assets/api-key-created-dialog-B6jkDG9f.js +1 -0
- app/static/assets/api-key-edit-dialog-Ck3Bnc3V.js +1 -0
- app/static/assets/api-trend-chart-BchCw7Hj.js +1 -0
- app/static/assets/apis-page-BvXz9c6i.js +2 -0
- app/static/assets/arrow-up-Dqh_KzJo.js +1 -0
- app/static/assets/arrow-up-down-LBRgVDGY.js +1 -0
- app/static/assets/automations-page-DQgNAJOR.js +3 -0
- app/static/assets/badge-DqMb9es0.js +1 -0
- app/static/assets/blur-email-HEvKagG2.js +1 -0
- app/static/assets/button-BjVmmiGW.js +1 -0
- app/static/assets/checkbox-apsl9vzk.js +1 -0
- app/static/assets/chevron-left-1d_pHVa5.js +1 -0
- app/static/assets/circle-alert-BS0xd_o1.js +1 -0
- app/static/assets/clsx-DB0hHKMi.js +1 -0
- app/static/assets/colors-BBy8GEtr.js +1 -0
- app/static/assets/copy-button-D7nAvOKR.js +1 -0
- app/static/assets/cost-per-day-chart-YdqlkZVM.js +1 -0
- app/static/assets/createLucideIcon-DBLzLNh2.js +1 -0
- app/static/assets/dashboard-page-CSdhBy0f.js +2 -0
- app/static/assets/distribution-metric-format-yZTy1ZEu.js +1 -0
- app/static/assets/donut-chart-Bxc69cra.js +1 -0
- app/static/assets/download-kQteIcsB.js +1 -0
- app/static/assets/ellipsis-B_5OgIre.js +1 -0
- app/static/assets/empty-state-JcA7tc8m.js +1 -0
- app/static/assets/es2015-bMH8YKZT.js +41 -0
- app/static/assets/es6-B264ni6W.js +69 -0
- app/static/assets/external-link-UVAOgMQ0.js +1 -0
- app/static/assets/i18n-BWqZ9Ua2.js +1 -0
- app/static/assets/inbox-B_RM2s1J.js +1 -0
- app/static/assets/index-DIpd264Z.js +23 -0
- app/static/assets/index-DMeLKRJx.css +2 -0
- app/static/assets/inherit-badge-QhTYVaU6.js +1 -0
- app/static/assets/invite-dialog-0CYh0GiQ.js +1 -0
- app/static/assets/iso-Bpe5Ihbb.js +1 -0
- app/static/assets/label-CGaLmLNf.js +1 -0
- app/static/assets/lazy-recharts-Pv8p9ArM.js +2 -0
- app/static/assets/loading-overlay-CzKQfHLI.js +1 -0
- app/static/assets/mini-quota-bar-cGT_t8Ae.js +1 -0
- app/static/assets/model-distribution-donut-BKjAfT48.js +1 -0
- app/static/assets/model-source-multi-select-D2d06Vc7.js +1 -0
- app/static/assets/multi-select-filter-KIA5qU6Z.js +1 -0
- app/static/assets/oauth-dialog-XeaGGDjS.js +1 -0
- app/static/assets/pagination-controls-D1Ypd0RU.js +1 -0
- app/static/assets/payload-CIYnGZz9.js +1 -0
- app/static/assets/queue-wait-chart-VoV7Yg9L.js +1 -0
- app/static/assets/report-chart-card-BXDliwcc.js +1 -0
- app/static/assets/reports-page-CGCJeMHG.js +3 -0
- app/static/assets/reset-credit-confirm-dialog-QOifTnVB.js +1 -0
- app/static/assets/rolldown-runtime-hePW80VL.js +1 -0
- app/static/assets/search-HfhqwERt.js +1 -0
- app/static/assets/select-Cl33bz-g.js +1 -0
- app/static/assets/settings-page-mZj01HYl.js +2 -0
- app/static/assets/shim-CZwyM4Q7.js +1 -0
- app/static/assets/skeleton-C_EUc4Rp.js +1 -0
- app/static/assets/sparkline-chart-DdNNsddM.js +1 -0
- app/static/assets/status-badge-CHHA5X6c.js +1 -0
- app/static/assets/switch-CzxgokcM.js +1 -0
- app/static/assets/table-DXaNSQ0G.js +1 -0
- app/static/assets/time-to-first-token-chart--68L0u_v.js +1 -0
- app/static/assets/tokens-per-day-chart-Cs82BFuE.js +1 -0
- app/static/assets/tokens-per-second-chart-AD2dfOn6.js +1 -0
- app/static/assets/totp-settings-CRHtxQ7j.js +1 -0
- app/static/assets/trash-D3KyWr1h.js +1 -0
- app/static/assets/use-account-quota-display-BPmrTBTC.js +1 -0
- app/static/assets/use-accounts-PWlVGw3z.js +1 -0
- app/static/assets/use-auth-CuDlW3iC.js +1 -0
- app/static/assets/use-chart-colors-DIu6tClI.js +1 -0
- app/static/assets/use-model-sources-Bul24JBu.js +1 -0
- app/static/assets/use-models-DUCPUuZg.js +1 -0
- app/static/assets/use-reduced-motion-hAm5r6_d.js +1 -0
- app/static/assets/useTranslation--56nHhlH.js +1 -0
- app/static/assets/useragent-distribution-donut-Dup6sf_n.js +1 -0
- app/static/assets/users-uIC3TIlE.js +1 -0
- app/static/assets/vendor-query-_bHZuxEx.js +1 -0
- app/static/assets/vendor-react-BVL0YZPW.js +9 -0
- app/static/favicon.svg +10 -0
- app/static/fonts/GeistSans-Variable.woff2 +0 -0
- app/static/fonts/jetbrains-mono-latin-ext.woff2 +0 -0
- app/static/fonts/jetbrains-mono-latin.woff2 +0 -0
- app/static/index.html +29 -0
- codexlb_up-1.25.0b10.dist-info/METADATA +124 -0
- codexlb_up-1.25.0b10.dist-info/RECORD +827 -0
- codexlb_up-1.25.0b10.dist-info/WHEEL +4 -0
- codexlb_up-1.25.0b10.dist-info/entry_points.txt +3 -0
- codexlb_up-1.25.0b10.dist-info/licenses/LICENSE +21 -0
- config/additional_quota_registry.json +11 -0
|
@@ -0,0 +1,2417 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
import asyncio
|
|
4
|
+
import logging
|
|
5
|
+
from collections.abc import Awaitable, Callable, Collection, Iterable, Mapping
|
|
6
|
+
from dataclasses import dataclass, replace
|
|
7
|
+
from datetime import datetime, timezone
|
|
8
|
+
from typing import Generic, Literal, Protocol, TypeVar
|
|
9
|
+
|
|
10
|
+
from app.core.balancer import (
|
|
11
|
+
HEALTH_TIER_DRAINING,
|
|
12
|
+
HEALTH_TIER_HEALTHY,
|
|
13
|
+
HEALTH_TIER_PROBING,
|
|
14
|
+
ROUTING_POLICY_BURN_FIRST,
|
|
15
|
+
ROUTING_POLICY_PRESERVE,
|
|
16
|
+
TRAFFIC_CLASS_FOREGROUND,
|
|
17
|
+
AccountState,
|
|
18
|
+
ResetPreferenceWindow,
|
|
19
|
+
RoutingCostsByAccount,
|
|
20
|
+
RoutingStrategy,
|
|
21
|
+
SelectionResult,
|
|
22
|
+
TrafficClass,
|
|
23
|
+
select_account,
|
|
24
|
+
)
|
|
25
|
+
from app.core.clock import Clock
|
|
26
|
+
from app.core.crypto import TokenEncryptor
|
|
27
|
+
from app.core.utils.time import utcnow
|
|
28
|
+
from app.db.models import Account, AccountStatus, AdditionalUsageHistory, StickySessionKind, UsageHistory
|
|
29
|
+
from app.db.snapshot import clone_row
|
|
30
|
+
from app.modules.accounts.repository import AccountsRepository
|
|
31
|
+
from app.modules.proxy._load_balancer.overload_backoff import (
|
|
32
|
+
filter_overload_backoff_candidates,
|
|
33
|
+
isolation_substitute_seed,
|
|
34
|
+
overload_backoff_active,
|
|
35
|
+
overload_isolation_active,
|
|
36
|
+
sticky_owner_isolation_reroute_pool,
|
|
37
|
+
)
|
|
38
|
+
from app.modules.proxy._load_balancer.tunables import RoutingTunables
|
|
39
|
+
from app.modules.proxy._load_balancer.types import (
|
|
40
|
+
MAX_SELECTION_ATTEMPTS,
|
|
41
|
+
AccountConcurrencyCaps,
|
|
42
|
+
AccountLease,
|
|
43
|
+
AccountLeaseKind,
|
|
44
|
+
ProbeReservation,
|
|
45
|
+
RuntimeState,
|
|
46
|
+
)
|
|
47
|
+
from app.modules.proxy.affinity import _CodexSessionSource
|
|
48
|
+
from app.modules.proxy.fair_share import (
|
|
49
|
+
API_KEY_STREAM_FAIR_SHARE_ERROR_CODE,
|
|
50
|
+
FairShareDecision,
|
|
51
|
+
fair_share_denial_message,
|
|
52
|
+
)
|
|
53
|
+
from app.modules.proxy.repo_bundle import ProxyRepoFactory
|
|
54
|
+
from app.modules.proxy.sticky_repository import StickyOwnerLookup, StickySessionsRepository
|
|
55
|
+
from app.modules.quota_planner.logic import PlannerSettings, build_routing_costs
|
|
56
|
+
|
|
57
|
+
# Preserve the established observability surface while implementation moves to
|
|
58
|
+
# a private module; operators and tests filter this logger by its public owner.
|
|
59
|
+
logger = logging.getLogger("app.modules.proxy.load_balancer")
|
|
60
|
+
|
|
61
|
+
_STICKY_GRACE_PERIOD_SECONDS = 10.0
|
|
62
|
+
_STICKY_EXISTING_UNSET = object()
|
|
63
|
+
_RECOVERABLE_STATUSES = frozenset(
|
|
64
|
+
{
|
|
65
|
+
AccountStatus.ACTIVE,
|
|
66
|
+
AccountStatus.REAUTH_REQUIRED,
|
|
67
|
+
AccountStatus.RATE_LIMITED,
|
|
68
|
+
AccountStatus.QUOTA_EXCEEDED,
|
|
69
|
+
}
|
|
70
|
+
)
|
|
71
|
+
_AMBIGUOUS_CONVERSATION_OWNER_CODE = "conversation_owner_unavailable"
|
|
72
|
+
_AMBIGUOUS_CONVERSATION_OWNER_MESSAGE = "Conversation owner cannot be determined from the eligible account pool"
|
|
73
|
+
|
|
74
|
+
StickySelectionDisposition = Literal["shared_result", "direct_error"]
|
|
75
|
+
AccountCapRejectionCallback = Callable[[AccountLeaseKind | None], None]
|
|
76
|
+
|
|
77
|
+
|
|
78
|
+
class SelectionInputsProtocol(Protocol):
|
|
79
|
+
accounts: list[Account]
|
|
80
|
+
latest_primary: dict[str, UsageHistory | AdditionalUsageHistory]
|
|
81
|
+
latest_secondary: dict[str, UsageHistory | AdditionalUsageHistory]
|
|
82
|
+
latest_monthly: dict[str, UsageHistory]
|
|
83
|
+
quota_planner_settings: PlannerSettings
|
|
84
|
+
runtime_accounts: list[Account] | None
|
|
85
|
+
error_message: str | None
|
|
86
|
+
error_code: str | None
|
|
87
|
+
ignore_standard_quota_account_ids: frozenset[str]
|
|
88
|
+
routing_policy_override: str | None
|
|
89
|
+
|
|
90
|
+
@property
|
|
91
|
+
def effective_continuity_owner_candidates(self) -> list[Account]: ...
|
|
92
|
+
|
|
93
|
+
@property
|
|
94
|
+
def effective_sticky_mutation_authority_account_ids(self) -> frozenset[str]: ...
|
|
95
|
+
|
|
96
|
+
|
|
97
|
+
SelectionInputsT = TypeVar("SelectionInputsT", bound=SelectionInputsProtocol)
|
|
98
|
+
|
|
99
|
+
|
|
100
|
+
class SelectionStatesOwner(Protocol):
|
|
101
|
+
"""What ``prepare_selection_states`` needs from the balancer: runtime, clock, encryptor, lock-held maintenance."""
|
|
102
|
+
|
|
103
|
+
_clock: Clock
|
|
104
|
+
_runtime: dict[str, RuntimeState]
|
|
105
|
+
_encryptor: TokenEncryptor
|
|
106
|
+
|
|
107
|
+
def _reclaim_stale_account_leases_locked(
|
|
108
|
+
self,
|
|
109
|
+
*,
|
|
110
|
+
routing_tunables: RoutingTunables,
|
|
111
|
+
redact_sensitive_details: bool = False,
|
|
112
|
+
) -> None: ...
|
|
113
|
+
|
|
114
|
+
def _prune_runtime(self, accounts: Iterable[Account]) -> None: ...
|
|
115
|
+
|
|
116
|
+
|
|
117
|
+
def prepare_selection_states(
|
|
118
|
+
owner: SelectionStatesOwner,
|
|
119
|
+
selection_inputs: SelectionInputsProtocol,
|
|
120
|
+
*,
|
|
121
|
+
build_states: Callable[..., tuple[list[AccountState], dict[str, Account]]],
|
|
122
|
+
required_account_id: str | None,
|
|
123
|
+
redact_sensitive_details: bool,
|
|
124
|
+
routing_tunables: RoutingTunables,
|
|
125
|
+
soft_drain_enabled: bool | None = None,
|
|
126
|
+
model: str | None = None,
|
|
127
|
+
) -> tuple[list[AccountState], dict[str, Account]]:
|
|
128
|
+
"""Build the live selection states for one attempt of a selection path.
|
|
129
|
+
|
|
130
|
+
Runs under the owner's runtime lock: reclaims stale leases, prunes runtime
|
|
131
|
+
entries for accounts that left the pool, builds the states over the live
|
|
132
|
+
runtime with ``build_states`` (``load_balancer._build_states``; passed in
|
|
133
|
+
so the balancer module stays the seam tests patch) and, when the path is
|
|
134
|
+
pinned to ``required_account_id``, narrows the result to that account.
|
|
135
|
+
``model`` is the requested model the latency cohort weight is scoped to.
|
|
136
|
+
"""
|
|
137
|
+
owner._reclaim_stale_account_leases_locked(
|
|
138
|
+
routing_tunables=routing_tunables,
|
|
139
|
+
redact_sensitive_details=redact_sensitive_details,
|
|
140
|
+
)
|
|
141
|
+
owner._prune_runtime(selection_inputs.runtime_accounts or selection_inputs.accounts)
|
|
142
|
+
states, account_map = build_states(
|
|
143
|
+
accounts=selection_inputs.accounts,
|
|
144
|
+
latest_primary=selection_inputs.latest_primary,
|
|
145
|
+
latest_secondary=selection_inputs.latest_secondary,
|
|
146
|
+
latest_monthly=selection_inputs.latest_monthly,
|
|
147
|
+
runtime=owner._runtime,
|
|
148
|
+
now=owner._clock.time(),
|
|
149
|
+
routing_policy_override=selection_inputs.routing_policy_override,
|
|
150
|
+
ignore_standard_quota_account_ids=selection_inputs.ignore_standard_quota_account_ids,
|
|
151
|
+
encryptor=owner._encryptor,
|
|
152
|
+
routing_tunables=routing_tunables,
|
|
153
|
+
# C2-3 resilience toggles: an explicit value (opportunistic admission)
|
|
154
|
+
# wins; selection carries it on its inputs.
|
|
155
|
+
soft_drain_enabled=(
|
|
156
|
+
soft_drain_enabled
|
|
157
|
+
if soft_drain_enabled is not None
|
|
158
|
+
else getattr(selection_inputs, "soft_drain_enabled", None)
|
|
159
|
+
),
|
|
160
|
+
model=model,
|
|
161
|
+
)
|
|
162
|
+
if required_account_id is None:
|
|
163
|
+
return states, account_map
|
|
164
|
+
return (
|
|
165
|
+
[state for state in states if state.account_id == required_account_id],
|
|
166
|
+
{account_id: account for account_id, account in account_map.items() if account_id == required_account_id},
|
|
167
|
+
)
|
|
168
|
+
|
|
169
|
+
|
|
170
|
+
class StickySelectionOwner(Protocol):
|
|
171
|
+
_clock: Clock
|
|
172
|
+
_runtime: dict[str, RuntimeState]
|
|
173
|
+
_runtime_lock: asyncio.Lock
|
|
174
|
+
_repo_factory: ProxyRepoFactory
|
|
175
|
+
|
|
176
|
+
def _prepare_sticky_selection_states(
|
|
177
|
+
self,
|
|
178
|
+
selection_inputs: SelectionInputsProtocol,
|
|
179
|
+
*,
|
|
180
|
+
required_account_id: str | None,
|
|
181
|
+
redact_sensitive_details: bool,
|
|
182
|
+
routing_tunables: RoutingTunables,
|
|
183
|
+
model: str | None = None,
|
|
184
|
+
) -> tuple[list[AccountState], dict[str, Account]]: ...
|
|
185
|
+
|
|
186
|
+
def _sync_runtime_state(
|
|
187
|
+
self,
|
|
188
|
+
account: Account,
|
|
189
|
+
state: AccountState,
|
|
190
|
+
*,
|
|
191
|
+
selected: bool = False,
|
|
192
|
+
expected_version: int | None = None,
|
|
193
|
+
) -> bool: ...
|
|
194
|
+
|
|
195
|
+
def _account_lease_allowed_locked(
|
|
196
|
+
self,
|
|
197
|
+
account_id: str,
|
|
198
|
+
*,
|
|
199
|
+
kind: AccountLeaseKind,
|
|
200
|
+
caps: AccountConcurrencyCaps,
|
|
201
|
+
stream_reserve_slots: int = 0,
|
|
202
|
+
) -> bool: ...
|
|
203
|
+
|
|
204
|
+
def _acquire_account_lease_locked(
|
|
205
|
+
self,
|
|
206
|
+
account_id: str,
|
|
207
|
+
*,
|
|
208
|
+
kind: AccountLeaseKind,
|
|
209
|
+
estimated_tokens: float,
|
|
210
|
+
record_selection: bool = True,
|
|
211
|
+
api_key_id: str | None = None,
|
|
212
|
+
) -> AccountLease: ...
|
|
213
|
+
|
|
214
|
+
def _api_key_stream_fair_share_denial_locked(
|
|
215
|
+
self,
|
|
216
|
+
*,
|
|
217
|
+
api_key_id: str | None,
|
|
218
|
+
lease_kind: AccountLeaseKind | None,
|
|
219
|
+
candidate_account_ids: Collection[str],
|
|
220
|
+
caps: AccountConcurrencyCaps,
|
|
221
|
+
stream_reserve_slots: int,
|
|
222
|
+
threshold_pct: int,
|
|
223
|
+
redact_sensitive_details: bool = False,
|
|
224
|
+
) -> FairShareDecision | None: ...
|
|
225
|
+
|
|
226
|
+
def _reserve_due_probe_locked(
|
|
227
|
+
self,
|
|
228
|
+
states: list[AccountState],
|
|
229
|
+
*,
|
|
230
|
+
prefer_earlier_reset: bool,
|
|
231
|
+
prefer_earlier_reset_window: ResetPreferenceWindow,
|
|
232
|
+
routing_strategy: RoutingStrategy,
|
|
233
|
+
relative_availability_power: float,
|
|
234
|
+
relative_availability_top_k: int,
|
|
235
|
+
traffic_class: TrafficClass,
|
|
236
|
+
routing_costs_by_account_id: RoutingCostsByAccount | None,
|
|
237
|
+
) -> ProbeReservation | None: ...
|
|
238
|
+
|
|
239
|
+
def _probe_reservation_current_locked(self, reservation: ProbeReservation | None) -> bool: ...
|
|
240
|
+
|
|
241
|
+
def _release_due_probe_reservation_locked(self, reservation: ProbeReservation | None) -> None: ...
|
|
242
|
+
|
|
243
|
+
def _commit_due_probe_reservation_locked(self, reservation: ProbeReservation | None) -> bool: ...
|
|
244
|
+
|
|
245
|
+
def _sync_committed_probe_state_locked(
|
|
246
|
+
self,
|
|
247
|
+
reservation: ProbeReservation,
|
|
248
|
+
account_map: dict[str, Account],
|
|
249
|
+
states: list[AccountState],
|
|
250
|
+
) -> None: ...
|
|
251
|
+
|
|
252
|
+
async def _persist_selection_state(
|
|
253
|
+
self,
|
|
254
|
+
accounts_repo: AccountsRepository,
|
|
255
|
+
account_map: dict[str, Account],
|
|
256
|
+
states: list[AccountState],
|
|
257
|
+
) -> set[str]: ...
|
|
258
|
+
|
|
259
|
+
async def _select_with_stickiness(
|
|
260
|
+
self,
|
|
261
|
+
*,
|
|
262
|
+
states: list[AccountState],
|
|
263
|
+
account_map: dict[str, Account],
|
|
264
|
+
sticky_key: str | None,
|
|
265
|
+
sticky_kind: StickySessionKind | None,
|
|
266
|
+
reallocate_sticky: bool,
|
|
267
|
+
sticky_max_age_seconds: int | None,
|
|
268
|
+
budget_threshold_pct: float,
|
|
269
|
+
secondary_budget_threshold_pct: float,
|
|
270
|
+
prefer_earlier_reset_accounts: bool,
|
|
271
|
+
prefer_earlier_reset_window: ResetPreferenceWindow,
|
|
272
|
+
routing_strategy: RoutingStrategy,
|
|
273
|
+
relative_availability_power: float,
|
|
274
|
+
relative_availability_top_k: int,
|
|
275
|
+
sticky_repo: StickySessionsRepository | None,
|
|
276
|
+
routing_costs_by_account_id: RoutingCostsByAccount | None,
|
|
277
|
+
sticky_existing_account_id: str | None | object,
|
|
278
|
+
initial_preferred_account_id: str | None,
|
|
279
|
+
preserve_existing_mapping_on_fallback: bool,
|
|
280
|
+
preserve_reason_request_local: bool = False,
|
|
281
|
+
traffic_class: TrafficClass,
|
|
282
|
+
ignore_standard_quota: bool,
|
|
283
|
+
allow_usage_exhaustion_error: bool = True,
|
|
284
|
+
usage_exhaustion_states: Iterable[AccountState] | None = None,
|
|
285
|
+
sticky_refresh_skip_deadline: datetime | None = None,
|
|
286
|
+
redact_sensitive_details: bool = False,
|
|
287
|
+
) -> _StickySelectionOutcome: ...
|
|
288
|
+
|
|
289
|
+
async def release_account_lease(self, lease: AccountLease | None) -> None: ...
|
|
290
|
+
|
|
291
|
+
|
|
292
|
+
@dataclass(frozen=True, slots=True)
|
|
293
|
+
class StickySelectionRequest(Generic[SelectionInputsT]):
|
|
294
|
+
sticky_key: str
|
|
295
|
+
sticky_kind: StickySessionKind | None
|
|
296
|
+
reallocate_sticky: bool
|
|
297
|
+
sticky_source: _CodexSessionSource | None
|
|
298
|
+
legacy_sticky_key: str | None
|
|
299
|
+
legacy_existing_account_id: str | None
|
|
300
|
+
legacy_abandoned_account_id: str | None
|
|
301
|
+
sticky_seed_key: str | None
|
|
302
|
+
sticky_seed_kind: StickySessionKind | None
|
|
303
|
+
sticky_seed_account_id: str | None
|
|
304
|
+
spill_bare_session_on_account_cap: bool
|
|
305
|
+
abandon_unavailable_legacy_owner: bool
|
|
306
|
+
require_unambiguous_account: bool
|
|
307
|
+
sticky_max_age_seconds: int | None
|
|
308
|
+
prefer_earlier_reset_accounts: bool
|
|
309
|
+
prefer_earlier_reset_window: ResetPreferenceWindow
|
|
310
|
+
routing_strategy: RoutingStrategy
|
|
311
|
+
relative_availability_power: float
|
|
312
|
+
relative_availability_top_k: int
|
|
313
|
+
required_account_id: str | None
|
|
314
|
+
budget_threshold_pct: float
|
|
315
|
+
secondary_budget_threshold_pct: float
|
|
316
|
+
routing_costs_by_account_id: RoutingCostsByAccount | None
|
|
317
|
+
lease_kind: AccountLeaseKind | None
|
|
318
|
+
estimated_lease_tokens: float
|
|
319
|
+
stream_reserve_slots: int
|
|
320
|
+
traffic_class: TrafficClass
|
|
321
|
+
concurrency_caps: AccountConcurrencyCaps
|
|
322
|
+
redact_sensitive_details: bool
|
|
323
|
+
# C2-2 routing/overload: dashboard snapshot resolved once by the caller.
|
|
324
|
+
routing_tunables: RoutingTunables
|
|
325
|
+
selection_inputs: SelectionInputsT
|
|
326
|
+
reload_inputs: Callable[[], Awaitable[SelectionInputsT]]
|
|
327
|
+
record_account_cap_rejection: AccountCapRejectionCallback
|
|
328
|
+
allow_usage_exhaustion_error: bool = True
|
|
329
|
+
api_key_id: str | None = None
|
|
330
|
+
api_key_stream_fair_share_threshold_pct: int = 0
|
|
331
|
+
# Accounts this request's retry loop already failed over from. Distinct
|
|
332
|
+
# from the pre-exclusion continuity pool, which also differs from the
|
|
333
|
+
# routable pool by catalog-evidence model filtering.
|
|
334
|
+
exclude_account_ids: frozenset[str] = frozenset()
|
|
335
|
+
# Requested model; scopes the per-model latency cohort weight of fresh draws.
|
|
336
|
+
model: str | None = None
|
|
337
|
+
# First-iteration owner read performed by the caller inside its shared
|
|
338
|
+
# owner-lookup session (see load_balancer.select_account). Consumed exactly
|
|
339
|
+
# once; retries re-read fresh ownership evidence through a repo bundle.
|
|
340
|
+
initial_sticky_owner_lookup: StickyOwnerLookup | None = None
|
|
341
|
+
|
|
342
|
+
|
|
343
|
+
@dataclass(frozen=True, slots=True)
|
|
344
|
+
class _StickyMutation:
|
|
345
|
+
# ``None`` is an intentional delete; absence of a mutation means preserve
|
|
346
|
+
# the current mapping until final admission succeeds.
|
|
347
|
+
account_id: str | None
|
|
348
|
+
# Set only when this mutation is a pure same-owner freshness rewrite of a
|
|
349
|
+
# row this request's lookup observed inside the repository's refresh-skip
|
|
350
|
+
# window. The persist site revalidates the deadline against the clock at
|
|
351
|
+
# write time and may then omit the statement entirely; a mutation that
|
|
352
|
+
# rebinds, deletes, or must initialize a seed mapping never carries it.
|
|
353
|
+
refresh_skip_deadline: datetime | None = None
|
|
354
|
+
|
|
355
|
+
|
|
356
|
+
@dataclass(frozen=True, slots=True)
|
|
357
|
+
class _StickySelectionOutcome:
|
|
358
|
+
selection: SelectionResult
|
|
359
|
+
mutation: _StickyMutation | None = None
|
|
360
|
+
# The candidate pool the selection actually ran over when a NEW account
|
|
361
|
+
# was chosen for the key (overload-free first pass); ``None`` means the
|
|
362
|
+
# caller's full pool. Probe reservation must use the same pool.
|
|
363
|
+
effective_states: list[AccountState] | None = None
|
|
364
|
+
# ``(mapping, overload_free_candidates)`` when this selection released an
|
|
365
|
+
# isolated sticky owner. Carried rather than logged, because selection can
|
|
366
|
+
# still lose the candidate to a concurrent lease or be retried on stale
|
|
367
|
+
# state -- and a release that never served must not reach the counter. The
|
|
368
|
+
# caller emits it once admission has actually succeeded.
|
|
369
|
+
isolation_release: tuple[str, int] | None = None
|
|
370
|
+
|
|
371
|
+
|
|
372
|
+
@dataclass(frozen=True, slots=True)
|
|
373
|
+
class StickySelectionOutcome(Generic[SelectionInputsT]):
|
|
374
|
+
selection_inputs: SelectionInputsT
|
|
375
|
+
selected_snapshot: Account | None
|
|
376
|
+
selected_lease: AccountLease | None
|
|
377
|
+
error_message: str | None
|
|
378
|
+
error_code: str | None
|
|
379
|
+
resets_at: int | None = None
|
|
380
|
+
disposition: StickySelectionDisposition = "shared_result"
|
|
381
|
+
# Set only alongside ``hard_affinity_saturated`` when the resolved hard
|
|
382
|
+
# owner is one of the caller's own ``exclude_account_ids``
|
|
383
|
+
# (``_hard_affinity_owner_excluded_by_caller``).
|
|
384
|
+
hard_affinity_owner_excluded: bool = False
|
|
385
|
+
|
|
386
|
+
|
|
387
|
+
def _hard_affinity_owner_excluded_by_caller(
|
|
388
|
+
*,
|
|
389
|
+
error_code: str | None,
|
|
390
|
+
owner_account_id: str | None | object,
|
|
391
|
+
exclude_account_ids: frozenset[str],
|
|
392
|
+
) -> bool:
|
|
393
|
+
"""Whether this ``hard_affinity_saturated`` was caused by the caller's own exclusion.
|
|
394
|
+
|
|
395
|
+
``hard_affinity_saturated`` means "the resolved hard ``CODEX_SESSION``
|
|
396
|
+
owner is not selectable". Two causes reach the same code and the callers'
|
|
397
|
+
recovery wait (``_HARD_AFFINITY_RECOVERY_SLEEP_SECONDS``) can only help
|
|
398
|
+
one of them:
|
|
399
|
+
|
|
400
|
+
* the owner is briefly unavailable (cap, health backoff, status) -- it may
|
|
401
|
+
recover inside the wait, which is exactly why the short wait exists;
|
|
402
|
+
* the caller passed the owner in ``exclude_account_ids`` -- the exclusion
|
|
403
|
+
filter drops it from the pool before ownership narrows selection to it
|
|
404
|
+
(``select_account``), so **no** amount of waiting can produce a
|
|
405
|
+
candidate while the caller keeps excluding it. Selection is not going to
|
|
406
|
+
spill to another account either: a resolved hard row is ownership
|
|
407
|
+
evidence, never a preference.
|
|
408
|
+
|
|
409
|
+
Only the selector can tell these apart, because only it knows which
|
|
410
|
+
account the row resolved to. Reporting the distinction (rather than the
|
|
411
|
+
owner id itself) keeps the account id out of surfaces that would have to
|
|
412
|
+
redact it, and answers exactly the question every caller asks: "is my own
|
|
413
|
+
exclusion set the reason, so is waiting futile?"
|
|
414
|
+
"""
|
|
415
|
+
return (
|
|
416
|
+
error_code == "hard_affinity_saturated"
|
|
417
|
+
and isinstance(owner_account_id, str)
|
|
418
|
+
and owner_account_id in exclude_account_ids
|
|
419
|
+
)
|
|
420
|
+
|
|
421
|
+
|
|
422
|
+
async def run_sticky_selection_path(
|
|
423
|
+
owner: StickySelectionOwner,
|
|
424
|
+
*,
|
|
425
|
+
request: StickySelectionRequest[SelectionInputsT],
|
|
426
|
+
) -> StickySelectionOutcome[SelectionInputsT]:
|
|
427
|
+
selection_inputs = request.selection_inputs
|
|
428
|
+
sticky_key = request.sticky_key
|
|
429
|
+
sticky_kind = request.sticky_kind
|
|
430
|
+
reallocate_sticky = request.reallocate_sticky
|
|
431
|
+
sticky_source = request.sticky_source
|
|
432
|
+
legacy_sticky_key = request.legacy_sticky_key
|
|
433
|
+
legacy_existing_account_id = request.legacy_existing_account_id
|
|
434
|
+
legacy_abandoned_account_id = request.legacy_abandoned_account_id
|
|
435
|
+
sticky_seed_key = request.sticky_seed_key
|
|
436
|
+
sticky_seed_kind = request.sticky_seed_kind
|
|
437
|
+
sticky_seed_account_id = request.sticky_seed_account_id
|
|
438
|
+
spill_bare_session_on_account_cap = request.spill_bare_session_on_account_cap
|
|
439
|
+
abandon_unavailable_legacy_owner = request.abandon_unavailable_legacy_owner
|
|
440
|
+
require_unambiguous_account = request.require_unambiguous_account
|
|
441
|
+
sticky_max_age_seconds = request.sticky_max_age_seconds
|
|
442
|
+
prefer_earlier_reset_accounts = request.prefer_earlier_reset_accounts
|
|
443
|
+
prefer_earlier_reset_window = request.prefer_earlier_reset_window
|
|
444
|
+
routing_strategy = request.routing_strategy
|
|
445
|
+
relative_availability_power = request.relative_availability_power
|
|
446
|
+
relative_availability_top_k = request.relative_availability_top_k
|
|
447
|
+
required_account_id = request.required_account_id
|
|
448
|
+
budget_threshold_pct = request.budget_threshold_pct
|
|
449
|
+
secondary_budget_threshold_pct = request.secondary_budget_threshold_pct
|
|
450
|
+
routing_costs_by_account_id = request.routing_costs_by_account_id
|
|
451
|
+
lease_kind = request.lease_kind
|
|
452
|
+
estimated_lease_tokens = request.estimated_lease_tokens
|
|
453
|
+
stream_reserve_slots = request.stream_reserve_slots
|
|
454
|
+
traffic_class = request.traffic_class
|
|
455
|
+
caps = request.concurrency_caps
|
|
456
|
+
redact_sensitive_details = request.redact_sensitive_details
|
|
457
|
+
routing_tunables = request.routing_tunables
|
|
458
|
+
load_selection_inputs = request.reload_inputs
|
|
459
|
+
_record_account_cap_rejection = request.record_account_cap_rejection
|
|
460
|
+
allow_usage_exhaustion_error = request.allow_usage_exhaustion_error
|
|
461
|
+
api_key_id = request.api_key_id
|
|
462
|
+
fair_share_threshold_pct = request.api_key_stream_fair_share_threshold_pct
|
|
463
|
+
|
|
464
|
+
selected_snapshot: Account | None = None
|
|
465
|
+
selected_lease: AccountLease | None = None
|
|
466
|
+
error_message: str | None = None
|
|
467
|
+
selection_error_code: str | None = None
|
|
468
|
+
selection_resets_at: int | None = None
|
|
469
|
+
|
|
470
|
+
def _direct_error(
|
|
471
|
+
*,
|
|
472
|
+
account: None,
|
|
473
|
+
error_message: str | None,
|
|
474
|
+
error_code: str | None = None,
|
|
475
|
+
) -> StickySelectionOutcome[SelectionInputsT]:
|
|
476
|
+
assert account is None
|
|
477
|
+
return StickySelectionOutcome(
|
|
478
|
+
selection_inputs=selection_inputs,
|
|
479
|
+
selected_snapshot=None,
|
|
480
|
+
selected_lease=None,
|
|
481
|
+
error_message=error_message,
|
|
482
|
+
error_code=error_code,
|
|
483
|
+
disposition="direct_error",
|
|
484
|
+
)
|
|
485
|
+
|
|
486
|
+
sticky_existing_account_id: str | None | object = _STICKY_EXISTING_UNSET
|
|
487
|
+
sticky_continuity_abandoned = False
|
|
488
|
+
sticky_refresh_skip_deadline: datetime | None = None
|
|
489
|
+
# A thread row whose process seed exists but is still unowned must keep
|
|
490
|
+
# its retention write: that write doubles as the seed-initialization
|
|
491
|
+
# carrier (see the ``initialize_seed_key`` argument at the persist site
|
|
492
|
+
# below), and suppressing it would let sibling threads select divergent
|
|
493
|
+
# owners until the skip window closes. Thread-only affinity without a
|
|
494
|
+
# seed key has nothing to initialize and stays skippable.
|
|
495
|
+
seed_initialization_pending = (
|
|
496
|
+
sticky_source == "thread_header" and sticky_seed_key is not None and sticky_seed_account_id is None
|
|
497
|
+
)
|
|
498
|
+
# A source-qualified marker can be observed before this call or after a
|
|
499
|
+
# retirement CAS miss. In both cases its retained owner is authoritative
|
|
500
|
+
# exclusion evidence even though it is no longer affinity ownership for
|
|
501
|
+
# the matching session-header source.
|
|
502
|
+
retired_legacy_owner_account_ids = (
|
|
503
|
+
{legacy_abandoned_account_id} if legacy_abandoned_account_id is not None else set()
|
|
504
|
+
)
|
|
505
|
+
attempt = 0
|
|
506
|
+
suppress_recovery_probe_candidates = False
|
|
507
|
+
pending_initial_owner_lookup = request.initial_sticky_owner_lookup
|
|
508
|
+
while True:
|
|
509
|
+
attempt += 1
|
|
510
|
+
sticky_existing_is_legacy = isinstance(legacy_existing_account_id, str)
|
|
511
|
+
if sticky_kind is not None:
|
|
512
|
+
async with owner._runtime_lock:
|
|
513
|
+
pass
|
|
514
|
+
if pending_initial_owner_lookup is not None:
|
|
515
|
+
# The caller already read this iteration's owner inside its
|
|
516
|
+
# shared owner-lookup session. Consume it exactly once so
|
|
517
|
+
# every retry (including post-reset attempts that wrap
|
|
518
|
+
# ``attempt`` back to 1) still re-reads fresh evidence.
|
|
519
|
+
sticky_owner_lookup = pending_initial_owner_lookup
|
|
520
|
+
pending_initial_owner_lookup = None
|
|
521
|
+
else:
|
|
522
|
+
async with owner._repo_factory() as repos:
|
|
523
|
+
sticky_owner_lookup = await repos.sticky_sessions.get_account_id_and_abandonment(
|
|
524
|
+
sticky_key,
|
|
525
|
+
kind=sticky_kind,
|
|
526
|
+
max_age_seconds=sticky_max_age_seconds,
|
|
527
|
+
continuity_source=sticky_source,
|
|
528
|
+
)
|
|
529
|
+
sticky_existing_account_id = sticky_owner_lookup.account_id
|
|
530
|
+
# `is True` (not a truthy check): an un-configured test double
|
|
531
|
+
# for sticky_sessions may return an object whose attribute
|
|
532
|
+
# access auto-vivifies to a mock rather than a real bool, and
|
|
533
|
+
# that must fail safe as "not abandoned", the same as it
|
|
534
|
+
# always has, rather than silently bypassing the ambiguous
|
|
535
|
+
# owner check below.
|
|
536
|
+
sticky_continuity_abandoned = sticky_owner_lookup.continuity_abandoned is True
|
|
537
|
+
# ``isinstance`` for the same test-double reason as above. The
|
|
538
|
+
# deadline is only ever an optimization hint: None always
|
|
539
|
+
# falls back to today's write-on-every-request refresh
|
|
540
|
+
# behavior, and seed-needing requests never skip.
|
|
541
|
+
observed_refresh_skip_deadline = sticky_owner_lookup.refresh_skip_deadline
|
|
542
|
+
sticky_refresh_skip_deadline = (
|
|
543
|
+
observed_refresh_skip_deadline
|
|
544
|
+
if isinstance(observed_refresh_skip_deadline, datetime) and not seed_initialization_pending
|
|
545
|
+
else None
|
|
546
|
+
)
|
|
547
|
+
sticky_abandoned_account_id = sticky_owner_lookup.abandoned_account_id
|
|
548
|
+
if sticky_owner_lookup.continuity_abandoned is True and isinstance(
|
|
549
|
+
sticky_abandoned_account_id,
|
|
550
|
+
str,
|
|
551
|
+
):
|
|
552
|
+
retired_legacy_owner_account_ids.add(sticky_abandoned_account_id)
|
|
553
|
+
if sticky_existing_is_legacy:
|
|
554
|
+
# Mixed-version replicas can create both rows on
|
|
555
|
+
# different accounts. The raw row was loaded before
|
|
556
|
+
# branch selection and always wins as possible hard
|
|
557
|
+
# turn-state ownership.
|
|
558
|
+
sticky_existing_account_id = legacy_existing_account_id
|
|
559
|
+
sticky_continuity_abandoned = False
|
|
560
|
+
# The freshness observation belongs to the namespaced row,
|
|
561
|
+
# not the raw legacy owner that now shadows it.
|
|
562
|
+
sticky_refresh_skip_deadline = None
|
|
563
|
+
async with owner._runtime_lock:
|
|
564
|
+
states, account_map = owner._prepare_sticky_selection_states(
|
|
565
|
+
selection_inputs,
|
|
566
|
+
required_account_id=required_account_id,
|
|
567
|
+
redact_sensitive_details=redact_sensitive_details,
|
|
568
|
+
routing_tunables=routing_tunables,
|
|
569
|
+
model=request.model,
|
|
570
|
+
)
|
|
571
|
+
if retired_legacy_owner_account_ids:
|
|
572
|
+
# Retirement is authoritative even when this selector loaded a
|
|
573
|
+
# pre-retirement account snapshot (or another replica still has
|
|
574
|
+
# one cached). Never let that stale snapshot immediately repin
|
|
575
|
+
# the account this request just proved durably unavailable.
|
|
576
|
+
states = [state for state in states if state.account_id not in retired_legacy_owner_account_ids]
|
|
577
|
+
account_map = {
|
|
578
|
+
account_id: account
|
|
579
|
+
for account_id, account in account_map.items()
|
|
580
|
+
if account_id not in retired_legacy_owner_account_ids
|
|
581
|
+
}
|
|
582
|
+
effective_routing_costs = (
|
|
583
|
+
routing_costs_by_account_id
|
|
584
|
+
if routing_costs_by_account_id is not None
|
|
585
|
+
else build_routing_costs(
|
|
586
|
+
settings=selection_inputs.quota_planner_settings,
|
|
587
|
+
states=states,
|
|
588
|
+
now=datetime.fromtimestamp(owner._clock.time(), timezone.utc),
|
|
589
|
+
)
|
|
590
|
+
)
|
|
591
|
+
# Key shape is deliberately irrelevant here. Only typed
|
|
592
|
+
# source provenance created by the affinity parser can
|
|
593
|
+
# grant mobility; otherwise a crafted hard turn-state key
|
|
594
|
+
# could become spillable.
|
|
595
|
+
bare_session_key = (
|
|
596
|
+
sticky_kind == StickySessionKind.CODEX_SESSION
|
|
597
|
+
and sticky_source == "session_header"
|
|
598
|
+
and legacy_sticky_key is not None
|
|
599
|
+
and not sticky_existing_is_legacy
|
|
600
|
+
)
|
|
601
|
+
cap_spillover_allowed = spill_bare_session_on_account_cap and lease_kind is not None and bare_session_key
|
|
602
|
+
hard_sticky = isinstance(sticky_existing_account_id, str) and (
|
|
603
|
+
sticky_existing_is_legacy or (sticky_kind == StickySessionKind.CODEX_SESSION and not bare_session_key)
|
|
604
|
+
)
|
|
605
|
+
if hard_sticky and required_account_id is not None and sticky_existing_account_id != required_account_id:
|
|
606
|
+
return _direct_error(
|
|
607
|
+
account=None,
|
|
608
|
+
error_message="Account-owned continuity sources conflict; retry the logical turn",
|
|
609
|
+
error_code="continuity_owner_conflict",
|
|
610
|
+
)
|
|
611
|
+
# A resolved hard row proves ownership. Without one, use the
|
|
612
|
+
# same pre-health/pre-cap pool as the no-sticky path above.
|
|
613
|
+
#
|
|
614
|
+
# A tombstoned row (sticky_continuity_abandoned — see
|
|
615
|
+
# purge_stale_hard_codex_session_mappings) is exempted from this
|
|
616
|
+
# fail-closed check even though it's just as pool-ambiguous as a
|
|
617
|
+
# never-seen key: unlike a never-seen key, we know this session's
|
|
618
|
+
# owner was durably unavailable and continuity was deliberately
|
|
619
|
+
# abandoned, so picking a fresh owner here is what lets the
|
|
620
|
+
# request recover instead of failing closed forever with no path
|
|
621
|
+
# back to a resolved hard row.
|
|
622
|
+
if (
|
|
623
|
+
require_unambiguous_account
|
|
624
|
+
and not hard_sticky
|
|
625
|
+
and not sticky_continuity_abandoned
|
|
626
|
+
and len(selection_inputs.effective_continuity_owner_candidates) != 1
|
|
627
|
+
):
|
|
628
|
+
return _direct_error(
|
|
629
|
+
account=None,
|
|
630
|
+
error_message=_AMBIGUOUS_CONVERSATION_OWNER_MESSAGE,
|
|
631
|
+
error_code=_AMBIGUOUS_CONVERSATION_OWNER_CODE,
|
|
632
|
+
)
|
|
633
|
+
# Fair share is measured against the full candidate pool, before
|
|
634
|
+
# hard-sticky narrows selection to the owner account.
|
|
635
|
+
fair_share_candidate_ids = [state.account_id for state in states]
|
|
636
|
+
fair_share_denial = owner._api_key_stream_fair_share_denial_locked(
|
|
637
|
+
api_key_id=api_key_id,
|
|
638
|
+
lease_kind=lease_kind,
|
|
639
|
+
candidate_account_ids=fair_share_candidate_ids,
|
|
640
|
+
caps=caps,
|
|
641
|
+
stream_reserve_slots=stream_reserve_slots,
|
|
642
|
+
threshold_pct=fair_share_threshold_pct,
|
|
643
|
+
redact_sensitive_details=redact_sensitive_details,
|
|
644
|
+
)
|
|
645
|
+
# An isolated soft owner may be released to a sibling by the
|
|
646
|
+
# overload isolation stage (see ``_run_select_with_stickiness``).
|
|
647
|
+
owner_overload_isolated = isinstance(sticky_existing_account_id, str) and overload_isolation_active(
|
|
648
|
+
owner._runtime.get(sticky_existing_account_id), owner._clock.time()
|
|
649
|
+
)
|
|
650
|
+
if hard_sticky:
|
|
651
|
+
# A resolved hard Codex mapping is an ownership
|
|
652
|
+
# constraint, not a preference. Scope, exclusions,
|
|
653
|
+
# health, and caps may make it unavailable, but must
|
|
654
|
+
# never delete or rebind it.
|
|
655
|
+
selection_states = [state for state in states if state.account_id == sticky_existing_account_id]
|
|
656
|
+
elif bare_session_key and isinstance(sticky_existing_account_id, str) and not cap_spillover_allowed:
|
|
657
|
+
# Mobility was revoked by owner-bearing payload or
|
|
658
|
+
# recovery stage. Keep the old cap exception for this
|
|
659
|
+
# soft hint; the authoritative preferred-owner path
|
|
660
|
+
# normally bypasses it.
|
|
661
|
+
selection_states = states
|
|
662
|
+
if owner_overload_isolated:
|
|
663
|
+
# The owner keeps its cap exemption, but a sibling it
|
|
664
|
+
# may be released to must pass the caps: otherwise the
|
|
665
|
+
# reroute could pick a saturated sibling that lease
|
|
666
|
+
# admission then rejects while the owner had capacity.
|
|
667
|
+
cap_eligible_ids = {
|
|
668
|
+
state.account_id
|
|
669
|
+
for state in _filter_states_for_account_caps(
|
|
670
|
+
states,
|
|
671
|
+
lease_kind=lease_kind,
|
|
672
|
+
caps=caps,
|
|
673
|
+
stream_reserve_slots=stream_reserve_slots,
|
|
674
|
+
)
|
|
675
|
+
}
|
|
676
|
+
selection_states = [
|
|
677
|
+
state
|
|
678
|
+
for state in states
|
|
679
|
+
if state.account_id == sticky_existing_account_id or state.account_id in cap_eligible_ids
|
|
680
|
+
]
|
|
681
|
+
else:
|
|
682
|
+
selection_states = _filter_states_for_account_caps(
|
|
683
|
+
states,
|
|
684
|
+
lease_kind=lease_kind,
|
|
685
|
+
caps=caps,
|
|
686
|
+
stream_reserve_slots=stream_reserve_slots,
|
|
687
|
+
)
|
|
688
|
+
if cap_spillover_allowed and lease_kind == "stream":
|
|
689
|
+
# Stream selection immediately precedes response-create
|
|
690
|
+
# admission. Prefer an account that can satisfy both,
|
|
691
|
+
# while preserving the later create-cap error when all
|
|
692
|
+
# are full.
|
|
693
|
+
response_create_states = _filter_states_for_account_caps(
|
|
694
|
+
selection_states,
|
|
695
|
+
lease_kind="response_create",
|
|
696
|
+
caps=caps,
|
|
697
|
+
stream_reserve_slots=0,
|
|
698
|
+
)
|
|
699
|
+
selection_states = response_create_states or selection_states
|
|
700
|
+
# Cap spillover is request-local: the mapping is preserved so the
|
|
701
|
+
# session returns to its owner once the cap clears. Overload
|
|
702
|
+
# isolation is request-local for the same reason (see
|
|
703
|
+
# ``_run_select_with_stickiness``), so an owner that is both
|
|
704
|
+
# capped and isolated preserves too -- rebinding it would strand
|
|
705
|
+
# the thread on the sibling after both conditions clear.
|
|
706
|
+
# Why the mapping is being kept matters downstream: only pressure
|
|
707
|
+
# that is *request-local* (a cap, this request's exclusion list)
|
|
708
|
+
# means "this owner can serve the next turn". An owner kept because
|
|
709
|
+
# the conversation's ownership is ambiguous
|
|
710
|
+
# (``require_unambiguous_account``) may be out of this request's
|
|
711
|
+
# routable or security scope entirely, and must not be treated as a
|
|
712
|
+
# warm owner waiting out an isolation window.
|
|
713
|
+
preserve_reason_request_local = (
|
|
714
|
+
bare_session_key
|
|
715
|
+
and isinstance(sticky_existing_account_id, str)
|
|
716
|
+
and (
|
|
717
|
+
cap_spillover_allowed
|
|
718
|
+
# An explicit reallocation is an instruction to retire the
|
|
719
|
+
# mapping, and it outranks cap spillover exactly as it
|
|
720
|
+
# already does on the TTL-bounded branch below. Without
|
|
721
|
+
# this the owner is absent from ``selection_states``, so
|
|
722
|
+
# the later ``caller_requested_reallocation`` guard is
|
|
723
|
+
# never reached and the row survives a reallocation that
|
|
724
|
+
# used to rebind it. The ``require_unambiguous_account``
|
|
725
|
+
# arm below is not gated: it is a correctness requirement
|
|
726
|
+
# about an ambiguous conversation owner, not a locality
|
|
727
|
+
# preference, so a caller cannot waive it.
|
|
728
|
+
and not reallocate_sticky
|
|
729
|
+
and any(state.account_id == sticky_existing_account_id for state in states)
|
|
730
|
+
and not any(state.account_id == sticky_existing_account_id for state in selection_states)
|
|
731
|
+
)
|
|
732
|
+
)
|
|
733
|
+
preserve_existing_mapping = preserve_reason_request_local or (
|
|
734
|
+
bare_session_key and isinstance(sticky_existing_account_id, str) and require_unambiguous_account
|
|
735
|
+
)
|
|
736
|
+
if (
|
|
737
|
+
not preserve_existing_mapping
|
|
738
|
+
and isinstance(sticky_existing_account_id, str)
|
|
739
|
+
and not hard_sticky
|
|
740
|
+
and not reallocate_sticky
|
|
741
|
+
and sticky_max_age_seconds is not None
|
|
742
|
+
and all(state.account_id != sticky_existing_account_id for state in selection_states)
|
|
743
|
+
):
|
|
744
|
+
# A soft TTL-bounded owner (prompt-cache thread row) missing
|
|
745
|
+
# from this request's candidates only because of request-local
|
|
746
|
+
# pressure -- the per-account cap filter dropped it, or the
|
|
747
|
+
# retry loop excluded it after a transient upstream failure
|
|
748
|
+
# while its persisted status is still recoverable -- keeps its
|
|
749
|
+
# mapping. Same rule as bare-session cap spillover: serve the
|
|
750
|
+
# alternate for this request so the conversation returns to
|
|
751
|
+
# its warm owner next turn. A PAUSED/DEACTIVATED owner or one
|
|
752
|
+
# outside the request's continuity scope is still rebound.
|
|
753
|
+
# The PAUSED/DEACTIVATED half of that rule needs no test here:
|
|
754
|
+
# ``_selectable_accounts`` drops both before ``_build_states``,
|
|
755
|
+
# so such an owner never reaches ``states`` and this predicate
|
|
756
|
+
# is already false for it. Only the exclusion arm below can see
|
|
757
|
+
# an owner outside that filter, and it checks the status.
|
|
758
|
+
preserve_reason_request_local = any(
|
|
759
|
+
state.account_id == sticky_existing_account_id for state in states
|
|
760
|
+
) or (
|
|
761
|
+
sticky_existing_account_id in request.exclude_account_ids
|
|
762
|
+
and any(
|
|
763
|
+
account.id == sticky_existing_account_id and account.status in _RECOVERABLE_STATUSES
|
|
764
|
+
for account in selection_inputs.effective_continuity_owner_candidates
|
|
765
|
+
)
|
|
766
|
+
)
|
|
767
|
+
preserve_existing_mapping = preserve_reason_request_local
|
|
768
|
+
if suppress_recovery_probe_candidates:
|
|
769
|
+
selection_states = _filter_recovery_probe_candidates(
|
|
770
|
+
selection_states,
|
|
771
|
+
traffic_class=traffic_class,
|
|
772
|
+
now=owner._clock.time(),
|
|
773
|
+
)
|
|
774
|
+
probe_reservation: ProbeReservation | None = None
|
|
775
|
+
# Raw sticky rows are global, while account-assigned API keys and
|
|
776
|
+
# other authenticated policies narrow a request's mutation authority.
|
|
777
|
+
# Keep this check on the pre-health continuity pool: quota exhaustion
|
|
778
|
+
# may authorize retirement, but being outside policy scope never does.
|
|
779
|
+
legacy_owner_in_effective_policy_scope = (
|
|
780
|
+
isinstance(sticky_existing_account_id, str)
|
|
781
|
+
and sticky_existing_account_id in selection_inputs.effective_sticky_mutation_authority_account_ids
|
|
782
|
+
)
|
|
783
|
+
if (
|
|
784
|
+
abandon_unavailable_legacy_owner
|
|
785
|
+
and hard_sticky
|
|
786
|
+
and sticky_existing_is_legacy
|
|
787
|
+
and sticky_source in {"session_header", "thread_header"}
|
|
788
|
+
and legacy_sticky_key is not None
|
|
789
|
+
and isinstance(sticky_existing_account_id, str)
|
|
790
|
+
and legacy_owner_in_effective_policy_scope
|
|
791
|
+
):
|
|
792
|
+
async with owner._repo_factory() as repos:
|
|
793
|
+
owner_retired = await repos.sticky_sessions.abandon_legacy_session_header_owner_if_unavailable(
|
|
794
|
+
legacy_sticky_key,
|
|
795
|
+
kind=StickySessionKind.CODEX_SESSION,
|
|
796
|
+
expected_account_id=sticky_existing_account_id,
|
|
797
|
+
)
|
|
798
|
+
authoritative_legacy_owner = None
|
|
799
|
+
if not owner_retired:
|
|
800
|
+
authoritative_legacy_owner = await repos.sticky_sessions.get_account_id_and_abandonment(
|
|
801
|
+
legacy_sticky_key,
|
|
802
|
+
kind=StickySessionKind.CODEX_SESSION,
|
|
803
|
+
continuity_source="session_header",
|
|
804
|
+
)
|
|
805
|
+
# One guarded write is authoritative for this selection. Repeating
|
|
806
|
+
# it in capacity-wait retries would add write pressure and could
|
|
807
|
+
# reinterpret a later status transition as restart authorization.
|
|
808
|
+
abandon_unavailable_legacy_owner = False
|
|
809
|
+
if owner_retired:
|
|
810
|
+
# The raw compatibility row is now a tombstone. Drop only the
|
|
811
|
+
# selection loop's cached legacy owner and run the normal path
|
|
812
|
+
# again so namespaced affinity, leases, and admission checks
|
|
813
|
+
# are established through the existing selection path.
|
|
814
|
+
logger.info(
|
|
815
|
+
"Legacy Codex session-header owner abandoned for self-contained goal restart account_id=%s",
|
|
816
|
+
"<redacted>" if redact_sensitive_details else sticky_existing_account_id,
|
|
817
|
+
)
|
|
818
|
+
retired_legacy_owner_account_ids.add(sticky_existing_account_id)
|
|
819
|
+
legacy_existing_account_id = None
|
|
820
|
+
continue
|
|
821
|
+
# A failed compare-and-set means the cached owner is no longer
|
|
822
|
+
# authoritative: it may have recovered, another request may have
|
|
823
|
+
# rebound the raw row, or another worker may already have
|
|
824
|
+
# tombstoned it. Re-read under a fresh transaction and restart the
|
|
825
|
+
# loop so each outcome is handled by normal selection. Retaining
|
|
826
|
+
# the stale owner here would defeat the CAS and can fail a restart
|
|
827
|
+
# even though a concurrent operation already established a valid
|
|
828
|
+
# replacement.
|
|
829
|
+
assert authoritative_legacy_owner is not None
|
|
830
|
+
legacy_existing_account_id = authoritative_legacy_owner.account_id
|
|
831
|
+
if authoritative_legacy_owner.continuity_abandoned is True:
|
|
832
|
+
abandoned_account_id = authoritative_legacy_owner.abandoned_account_id
|
|
833
|
+
if isinstance(abandoned_account_id, str):
|
|
834
|
+
retired_legacy_owner_account_ids.add(abandoned_account_id)
|
|
835
|
+
continue
|
|
836
|
+
sticky_outcome = _StickySelectionOutcome(selection=SelectionResult(None, None))
|
|
837
|
+
if fair_share_denial is not None:
|
|
838
|
+
# Denial parks in the transport capacity-wait loop like a cap
|
|
839
|
+
# denial. Sticky DB work, mapping mutation, and probe
|
|
840
|
+
# reservation are all skipped so mappings are preserved.
|
|
841
|
+
selection_error_code = API_KEY_STREAM_FAIR_SHARE_ERROR_CODE
|
|
842
|
+
result = SelectionResult(None, fair_share_denial_message(fair_share_denial))
|
|
843
|
+
elif hard_sticky and not selection_states:
|
|
844
|
+
selection_error_code = "hard_affinity_saturated"
|
|
845
|
+
selection_resets_at = None
|
|
846
|
+
result = SelectionResult(None, "Hard affinity owner account is unavailable")
|
|
847
|
+
elif not selection_states and states:
|
|
848
|
+
selection_error_code = _account_cap_error_code(lease_kind)
|
|
849
|
+
selection_resets_at = None
|
|
850
|
+
result = SelectionResult(None, _account_cap_error_message(lease_kind, caps))
|
|
851
|
+
logger.warning(
|
|
852
|
+
"Account cap exhausted during sticky selection lease_kind=%s reason=%s candidates=%s",
|
|
853
|
+
lease_kind,
|
|
854
|
+
selection_error_code,
|
|
855
|
+
len(states),
|
|
856
|
+
)
|
|
857
|
+
_record_account_cap_rejection(lease_kind)
|
|
858
|
+
elif hard_sticky:
|
|
859
|
+
# Hard rows are ownership evidence. Select only from the
|
|
860
|
+
# resolved owner state and never enter sticky fallback code,
|
|
861
|
+
# which may delete or rebind soft mappings under pressure.
|
|
862
|
+
result = _select_account_preferring_budget_safe(
|
|
863
|
+
selection_states,
|
|
864
|
+
prefer_earlier_reset=prefer_earlier_reset_accounts,
|
|
865
|
+
prefer_earlier_reset_window=prefer_earlier_reset_window,
|
|
866
|
+
routing_strategy=routing_strategy,
|
|
867
|
+
relative_availability_power=relative_availability_power,
|
|
868
|
+
relative_availability_top_k=relative_availability_top_k,
|
|
869
|
+
budget_threshold_pct=budget_threshold_pct,
|
|
870
|
+
secondary_budget_threshold_pct=secondary_budget_threshold_pct,
|
|
871
|
+
traffic_class=traffic_class,
|
|
872
|
+
ignore_standard_quota=False,
|
|
873
|
+
routing_costs_by_account_id=effective_routing_costs,
|
|
874
|
+
allow_usage_exhaustion_error=allow_usage_exhaustion_error,
|
|
875
|
+
usage_exhaustion_states=states,
|
|
876
|
+
)
|
|
877
|
+
if result.account is None:
|
|
878
|
+
selection_error_code = "hard_affinity_saturated"
|
|
879
|
+
selection_resets_at = None
|
|
880
|
+
result = SelectionResult(
|
|
881
|
+
None,
|
|
882
|
+
result.error_message or "Hard affinity owner account is unavailable",
|
|
883
|
+
)
|
|
884
|
+
else:
|
|
885
|
+
selection_error_code = None
|
|
886
|
+
selection_resets_at = None
|
|
887
|
+
else:
|
|
888
|
+
selection_error_code = None
|
|
889
|
+
selection_resets_at = None
|
|
890
|
+
try:
|
|
891
|
+
async with owner._repo_factory() as repos:
|
|
892
|
+
sticky_outcome = await owner._select_with_stickiness(
|
|
893
|
+
states=selection_states,
|
|
894
|
+
account_map=account_map,
|
|
895
|
+
sticky_key=sticky_key,
|
|
896
|
+
sticky_kind=sticky_kind,
|
|
897
|
+
reallocate_sticky=reallocate_sticky,
|
|
898
|
+
sticky_max_age_seconds=sticky_max_age_seconds,
|
|
899
|
+
budget_threshold_pct=budget_threshold_pct,
|
|
900
|
+
secondary_budget_threshold_pct=secondary_budget_threshold_pct,
|
|
901
|
+
prefer_earlier_reset_accounts=prefer_earlier_reset_accounts,
|
|
902
|
+
prefer_earlier_reset_window=prefer_earlier_reset_window,
|
|
903
|
+
routing_strategy=routing_strategy,
|
|
904
|
+
relative_availability_power=relative_availability_power,
|
|
905
|
+
relative_availability_top_k=relative_availability_top_k,
|
|
906
|
+
sticky_repo=repos.sticky_sessions,
|
|
907
|
+
sticky_existing_account_id=sticky_existing_account_id,
|
|
908
|
+
initial_preferred_account_id=(
|
|
909
|
+
sticky_seed_account_id
|
|
910
|
+
if not isinstance(sticky_existing_account_id, str) and not sticky_continuity_abandoned
|
|
911
|
+
else None
|
|
912
|
+
),
|
|
913
|
+
preserve_existing_mapping_on_fallback=preserve_existing_mapping,
|
|
914
|
+
preserve_reason_request_local=preserve_reason_request_local,
|
|
915
|
+
traffic_class=traffic_class,
|
|
916
|
+
ignore_standard_quota=False,
|
|
917
|
+
routing_costs_by_account_id=effective_routing_costs,
|
|
918
|
+
allow_usage_exhaustion_error=allow_usage_exhaustion_error,
|
|
919
|
+
usage_exhaustion_states=states,
|
|
920
|
+
sticky_refresh_skip_deadline=sticky_refresh_skip_deadline,
|
|
921
|
+
redact_sensitive_details=redact_sensitive_details,
|
|
922
|
+
)
|
|
923
|
+
result = sticky_outcome.selection
|
|
924
|
+
if (
|
|
925
|
+
result.account is None
|
|
926
|
+
and result.error_code is None
|
|
927
|
+
and lease_kind is not None
|
|
928
|
+
and len(selection_states) < len(states)
|
|
929
|
+
and any(
|
|
930
|
+
state.status in (AccountStatus.ACTIVE, AccountStatus.REAUTH_REQUIRED)
|
|
931
|
+
for state in states
|
|
932
|
+
if state not in selection_states
|
|
933
|
+
)
|
|
934
|
+
):
|
|
935
|
+
selection_error_code = _account_cap_error_code(lease_kind)
|
|
936
|
+
result = SelectionResult(
|
|
937
|
+
None,
|
|
938
|
+
_account_cap_error_message(lease_kind, caps),
|
|
939
|
+
error_code=selection_error_code,
|
|
940
|
+
)
|
|
941
|
+
except BaseException:
|
|
942
|
+
async with owner._runtime_lock:
|
|
943
|
+
owner._release_due_probe_reservation_locked(probe_reservation)
|
|
944
|
+
raise
|
|
945
|
+
selected_account_map = account_map
|
|
946
|
+
selected_states = []
|
|
947
|
+
probe_reservation_invalidated = False
|
|
948
|
+
reserved_probe_admitted = False
|
|
949
|
+
async with owner._runtime_lock:
|
|
950
|
+
selected: Account | None = None
|
|
951
|
+
selection_admitted = False
|
|
952
|
+
selected_reserved_probe = False
|
|
953
|
+
should_reserve_probe = bool(
|
|
954
|
+
result.account is not None
|
|
955
|
+
and (
|
|
956
|
+
not isinstance(sticky_existing_account_id, str)
|
|
957
|
+
or result.account.account_id != sticky_existing_account_id
|
|
958
|
+
or reallocate_sticky
|
|
959
|
+
)
|
|
960
|
+
)
|
|
961
|
+
# A fresh binding may have been chosen from the overload-free
|
|
962
|
+
# subset; reserve the recovery probe from that same pool so an
|
|
963
|
+
# older due probe the pass skipped cannot invalidate the match.
|
|
964
|
+
probe_states = sticky_outcome.effective_states or selection_states
|
|
965
|
+
probing_result_requires_reservation = _probing_result_requires_recovery_reservation(
|
|
966
|
+
probe_states,
|
|
967
|
+
result.account,
|
|
968
|
+
routing_strategy=routing_strategy,
|
|
969
|
+
traffic_class=traffic_class,
|
|
970
|
+
now=owner._clock.time(),
|
|
971
|
+
)
|
|
972
|
+
if should_reserve_probe and probing_result_requires_reservation:
|
|
973
|
+
# Sticky persistence happens outside the runtime lock.
|
|
974
|
+
# Delay the reversible reservation until after sticky
|
|
975
|
+
# selection proves we are not simply retaining a
|
|
976
|
+
# selectable owner; otherwise an owner-retaining request
|
|
977
|
+
# can temporarily consume the only due probing slot and
|
|
978
|
+
# make concurrent unbound traffic miss recovery.
|
|
979
|
+
probe_reservation = owner._reserve_due_probe_locked(
|
|
980
|
+
probe_states,
|
|
981
|
+
prefer_earlier_reset=prefer_earlier_reset_accounts,
|
|
982
|
+
prefer_earlier_reset_window=prefer_earlier_reset_window,
|
|
983
|
+
routing_strategy=routing_strategy,
|
|
984
|
+
relative_availability_power=relative_availability_power,
|
|
985
|
+
relative_availability_top_k=relative_availability_top_k,
|
|
986
|
+
traffic_class=traffic_class,
|
|
987
|
+
routing_costs_by_account_id=effective_routing_costs,
|
|
988
|
+
)
|
|
989
|
+
if (
|
|
990
|
+
probe_reservation is not None
|
|
991
|
+
and result.account is not None
|
|
992
|
+
and probe_reservation.account_id != result.account.account_id
|
|
993
|
+
):
|
|
994
|
+
owner._release_due_probe_reservation_locked(probe_reservation)
|
|
995
|
+
probe_reservation = None
|
|
996
|
+
if probe_reservation is None and result.account is not None:
|
|
997
|
+
# The result came from a pre-DB snapshot, but the
|
|
998
|
+
# current runtime no longer admits that probing
|
|
999
|
+
# candidate. Rebuild from fresh state instead of
|
|
1000
|
+
# returning or persisting stale recovery affinity.
|
|
1001
|
+
probe_reservation_invalidated = True
|
|
1002
|
+
if result.account is None:
|
|
1003
|
+
error_message = result.error_message
|
|
1004
|
+
selection_error_code = result.error_code or selection_error_code
|
|
1005
|
+
selection_resets_at = result.resets_at or selection_resets_at
|
|
1006
|
+
elif probe_reservation_invalidated:
|
|
1007
|
+
selected = None
|
|
1008
|
+
else:
|
|
1009
|
+
selected = account_map.get(result.account.account_id)
|
|
1010
|
+
selected_reserved_probe = bool(
|
|
1011
|
+
selected is not None
|
|
1012
|
+
and probe_reservation is not None
|
|
1013
|
+
and selected.id == probe_reservation.account_id
|
|
1014
|
+
)
|
|
1015
|
+
if selected_reserved_probe and not owner._probe_reservation_current_locked(probe_reservation):
|
|
1016
|
+
# A health mutation won the CAS while sticky DB work
|
|
1017
|
+
# was in flight. Do not return or persist affinity to
|
|
1018
|
+
# the stale probing snapshot; rebuild and select from
|
|
1019
|
+
# the newer runtime state instead.
|
|
1020
|
+
owner._release_due_probe_reservation_locked(probe_reservation)
|
|
1021
|
+
selected = None
|
|
1022
|
+
probe_reservation_invalidated = True
|
|
1023
|
+
elif selected is None:
|
|
1024
|
+
error_message = result.error_message
|
|
1025
|
+
elif lease_kind is not None and not owner._account_lease_allowed_locked(
|
|
1026
|
+
selected.id,
|
|
1027
|
+
kind=lease_kind,
|
|
1028
|
+
caps=caps,
|
|
1029
|
+
stream_reserve_slots=stream_reserve_slots,
|
|
1030
|
+
):
|
|
1031
|
+
selection_error_code = _account_cap_error_code(lease_kind)
|
|
1032
|
+
error_message = _account_cap_error_message(lease_kind, caps)
|
|
1033
|
+
elif (
|
|
1034
|
+
lease_kind is not None
|
|
1035
|
+
and (
|
|
1036
|
+
fair_share_recheck := owner._api_key_stream_fair_share_denial_locked(
|
|
1037
|
+
api_key_id=api_key_id,
|
|
1038
|
+
lease_kind=lease_kind,
|
|
1039
|
+
candidate_account_ids=fair_share_candidate_ids,
|
|
1040
|
+
caps=caps,
|
|
1041
|
+
stream_reserve_slots=stream_reserve_slots,
|
|
1042
|
+
threshold_pct=fair_share_threshold_pct,
|
|
1043
|
+
redact_sensitive_details=redact_sensitive_details,
|
|
1044
|
+
)
|
|
1045
|
+
)
|
|
1046
|
+
is not None
|
|
1047
|
+
):
|
|
1048
|
+
# Sticky DB work runs between the filter-phase gate and
|
|
1049
|
+
# this commit section; concurrent selections for one key
|
|
1050
|
+
# could otherwise overshoot the share.
|
|
1051
|
+
selection_error_code = API_KEY_STREAM_FAIR_SHARE_ERROR_CODE
|
|
1052
|
+
error_message = fair_share_denial_message(fair_share_recheck)
|
|
1053
|
+
else:
|
|
1054
|
+
selection_admitted = True
|
|
1055
|
+
if lease_kind is not None:
|
|
1056
|
+
selected_lease = owner._acquire_account_lease_locked(
|
|
1057
|
+
selected.id,
|
|
1058
|
+
kind=lease_kind,
|
|
1059
|
+
estimated_tokens=estimated_lease_tokens,
|
|
1060
|
+
# Keep the reservation token intact until
|
|
1061
|
+
# persistence commits the recovery admission.
|
|
1062
|
+
record_selection=not selected_reserved_probe,
|
|
1063
|
+
api_key_id=api_key_id,
|
|
1064
|
+
)
|
|
1065
|
+
|
|
1066
|
+
if not probe_reservation_invalidated:
|
|
1067
|
+
reserved_probe_admitted = selection_admitted and selected_reserved_probe
|
|
1068
|
+
if not reserved_probe_admitted:
|
|
1069
|
+
owner._release_due_probe_reservation_locked(probe_reservation)
|
|
1070
|
+
for state in states:
|
|
1071
|
+
account = account_map.get(state.account_id)
|
|
1072
|
+
if account is None:
|
|
1073
|
+
continue
|
|
1074
|
+
state_is_reserved_probe = False
|
|
1075
|
+
if reserved_probe_admitted:
|
|
1076
|
+
assert probe_reservation is not None
|
|
1077
|
+
state_is_reserved_probe = state.account_id == probe_reservation.account_id
|
|
1078
|
+
if not state_is_reserved_probe:
|
|
1079
|
+
owner._sync_runtime_state(
|
|
1080
|
+
account,
|
|
1081
|
+
state,
|
|
1082
|
+
# A selected probe remains provisional through DB
|
|
1083
|
+
# persistence. Its reservation is committed below;
|
|
1084
|
+
# advancing last_selected_at here would make later
|
|
1085
|
+
# admission failures impossible to roll back.
|
|
1086
|
+
selected=(
|
|
1087
|
+
selection_admitted
|
|
1088
|
+
and result.account is not None
|
|
1089
|
+
and state.account_id == result.account.account_id
|
|
1090
|
+
),
|
|
1091
|
+
)
|
|
1092
|
+
selected_states.append(state)
|
|
1093
|
+
if selection_admitted and selected is not None and result.account is not None:
|
|
1094
|
+
selected_reset_at = selected.reset_at
|
|
1095
|
+
for state in selected_states:
|
|
1096
|
+
if state.account_id == result.account.account_id:
|
|
1097
|
+
state.status = result.account.status
|
|
1098
|
+
state.deactivation_reason = result.account.deactivation_reason
|
|
1099
|
+
selected_reset_at = int(state.reset_at) if state.reset_at else None
|
|
1100
|
+
break
|
|
1101
|
+
selected_snapshot = _clone_account(selected)
|
|
1102
|
+
selected_snapshot.status = result.account.status
|
|
1103
|
+
selected_snapshot.deactivation_reason = result.account.deactivation_reason
|
|
1104
|
+
selected_snapshot.reset_at = selected_reset_at
|
|
1105
|
+
|
|
1106
|
+
if probe_reservation_invalidated:
|
|
1107
|
+
selected_snapshot = None
|
|
1108
|
+
error_message = None
|
|
1109
|
+
selected_states = []
|
|
1110
|
+
selected_account_map = {}
|
|
1111
|
+
if attempt >= MAX_SELECTION_ATTEMPTS:
|
|
1112
|
+
suppress_recovery_probe_candidates = True
|
|
1113
|
+
attempt = 0
|
|
1114
|
+
selection_inputs = await load_selection_inputs()
|
|
1115
|
+
if selection_inputs.error_code is not None and not selection_inputs.accounts:
|
|
1116
|
+
return _direct_error(
|
|
1117
|
+
account=None,
|
|
1118
|
+
error_message=selection_inputs.error_message,
|
|
1119
|
+
error_code=selection_inputs.error_code,
|
|
1120
|
+
)
|
|
1121
|
+
await asyncio.sleep(0)
|
|
1122
|
+
continue
|
|
1123
|
+
selection_inputs = await load_selection_inputs()
|
|
1124
|
+
if selection_inputs.error_code is not None and not selection_inputs.accounts:
|
|
1125
|
+
return _direct_error(
|
|
1126
|
+
account=None,
|
|
1127
|
+
error_message=selection_inputs.error_message,
|
|
1128
|
+
error_code=selection_inputs.error_code,
|
|
1129
|
+
)
|
|
1130
|
+
await asyncio.sleep(0)
|
|
1131
|
+
continue
|
|
1132
|
+
|
|
1133
|
+
try:
|
|
1134
|
+
async with owner._repo_factory() as repos:
|
|
1135
|
+
stale_account_ids = await owner._persist_selection_state(
|
|
1136
|
+
repos.accounts,
|
|
1137
|
+
selected_account_map,
|
|
1138
|
+
selected_states,
|
|
1139
|
+
)
|
|
1140
|
+
except BaseException:
|
|
1141
|
+
await owner.release_account_lease(selected_lease)
|
|
1142
|
+
selected_lease = None
|
|
1143
|
+
async with owner._runtime_lock:
|
|
1144
|
+
owner._release_due_probe_reservation_locked(probe_reservation)
|
|
1145
|
+
raise
|
|
1146
|
+
stale_account_ids = stale_account_ids or set()
|
|
1147
|
+
if selected_snapshot is not None and selected_snapshot.id in stale_account_ids:
|
|
1148
|
+
await owner.release_account_lease(selected_lease)
|
|
1149
|
+
selected_lease = None
|
|
1150
|
+
async with owner._runtime_lock:
|
|
1151
|
+
owner._release_due_probe_reservation_locked(probe_reservation)
|
|
1152
|
+
selected_snapshot = None
|
|
1153
|
+
error_message = None
|
|
1154
|
+
selected_states = []
|
|
1155
|
+
selected_account_map = {}
|
|
1156
|
+
if attempt >= MAX_SELECTION_ATTEMPTS:
|
|
1157
|
+
break
|
|
1158
|
+
selection_inputs = await load_selection_inputs()
|
|
1159
|
+
if selection_inputs.error_code is not None and not selection_inputs.accounts:
|
|
1160
|
+
return _direct_error(
|
|
1161
|
+
account=None,
|
|
1162
|
+
error_message=selection_inputs.error_message,
|
|
1163
|
+
error_code=selection_inputs.error_code,
|
|
1164
|
+
)
|
|
1165
|
+
await asyncio.sleep(0)
|
|
1166
|
+
continue
|
|
1167
|
+
if (
|
|
1168
|
+
selected_snapshot is None
|
|
1169
|
+
and selection_error_code is not None
|
|
1170
|
+
and not hard_sticky
|
|
1171
|
+
and attempt < MAX_SELECTION_ATTEMPTS
|
|
1172
|
+
):
|
|
1173
|
+
selection_inputs = await load_selection_inputs()
|
|
1174
|
+
if selection_inputs.error_code is not None and not selection_inputs.accounts:
|
|
1175
|
+
return _direct_error(
|
|
1176
|
+
account=None,
|
|
1177
|
+
error_message=selection_inputs.error_message,
|
|
1178
|
+
error_code=selection_inputs.error_code,
|
|
1179
|
+
)
|
|
1180
|
+
error_message = None
|
|
1181
|
+
selected_states = []
|
|
1182
|
+
selected_account_map = {}
|
|
1183
|
+
await asyncio.sleep(0)
|
|
1184
|
+
continue
|
|
1185
|
+
should_persist_sticky_mutation = (
|
|
1186
|
+
sticky_outcome.mutation is not None
|
|
1187
|
+
and selection_error_code is None
|
|
1188
|
+
and (selected_snapshot is not None or result.account is None)
|
|
1189
|
+
)
|
|
1190
|
+
if selected_snapshot is not None and reserved_probe_admitted and should_persist_sticky_mutation:
|
|
1191
|
+
reservation_committed = False
|
|
1192
|
+
assert sticky_kind is not None
|
|
1193
|
+
sticky_mutation = sticky_outcome.mutation
|
|
1194
|
+
assert sticky_mutation is not None
|
|
1195
|
+
# A pure same-owner freshness rewrite may be omitted here exactly
|
|
1196
|
+
# as on the non-probe path below (the row already holds this
|
|
1197
|
+
# owner, so the rollback restores become no-ops and are skipped
|
|
1198
|
+
# symmetrically). The probe path deliberately never initializes a
|
|
1199
|
+
# seed, so only the deadline gates the skip.
|
|
1200
|
+
probe_refresh_write_skipped = _sticky_refresh_write_skippable(
|
|
1201
|
+
sticky_mutation,
|
|
1202
|
+
initialize_seed_key=None,
|
|
1203
|
+
)
|
|
1204
|
+
if not probe_refresh_write_skipped:
|
|
1205
|
+
try:
|
|
1206
|
+
async with owner._repo_factory() as repos:
|
|
1207
|
+
# A recovery-probe reservation is still reversible until
|
|
1208
|
+
# the runtime CAS below succeeds. Persist its thread row so
|
|
1209
|
+
# existing rollback machinery can restore it, but do not
|
|
1210
|
+
# publish an immutable process seed that cannot be safely
|
|
1211
|
+
# deleted after a concurrent sibling observes it.
|
|
1212
|
+
await _persist_sticky_mutation(
|
|
1213
|
+
sticky_repo=repos.sticky_sessions,
|
|
1214
|
+
sticky_key=sticky_key,
|
|
1215
|
+
sticky_kind=sticky_kind,
|
|
1216
|
+
mutation=sticky_mutation,
|
|
1217
|
+
)
|
|
1218
|
+
except BaseException:
|
|
1219
|
+
await owner.release_account_lease(selected_lease)
|
|
1220
|
+
selected_lease = None
|
|
1221
|
+
async with owner._runtime_lock:
|
|
1222
|
+
owner._release_due_probe_reservation_locked(probe_reservation)
|
|
1223
|
+
raise
|
|
1224
|
+
try:
|
|
1225
|
+
async with owner._runtime_lock:
|
|
1226
|
+
assert probe_reservation is not None
|
|
1227
|
+
reservation_committed = owner._commit_due_probe_reservation_locked(probe_reservation)
|
|
1228
|
+
if reservation_committed:
|
|
1229
|
+
owner._sync_committed_probe_state_locked(
|
|
1230
|
+
probe_reservation,
|
|
1231
|
+
selected_account_map,
|
|
1232
|
+
selected_states,
|
|
1233
|
+
)
|
|
1234
|
+
else:
|
|
1235
|
+
owner._release_due_probe_reservation_locked(probe_reservation)
|
|
1236
|
+
except BaseException:
|
|
1237
|
+
await owner.release_account_lease(selected_lease)
|
|
1238
|
+
selected_lease = None
|
|
1239
|
+
async with owner._runtime_lock:
|
|
1240
|
+
owner._release_due_probe_reservation_locked(probe_reservation)
|
|
1241
|
+
if not probe_refresh_write_skipped:
|
|
1242
|
+
async with owner._repo_factory() as repos:
|
|
1243
|
+
await _restore_sticky_mutation(
|
|
1244
|
+
sticky_repo=repos.sticky_sessions,
|
|
1245
|
+
sticky_key=sticky_key,
|
|
1246
|
+
sticky_kind=sticky_kind,
|
|
1247
|
+
expected_account_id=sticky_mutation.account_id,
|
|
1248
|
+
sticky_existing_account_id=sticky_existing_account_id,
|
|
1249
|
+
)
|
|
1250
|
+
raise
|
|
1251
|
+
if not reservation_committed:
|
|
1252
|
+
# Runtime health changed while account-state persistence
|
|
1253
|
+
# was in flight. The lease, probe quiet interval, and
|
|
1254
|
+
# provisional affinity must not escape; restore the
|
|
1255
|
+
# previous sticky owner before retrying against a fresh
|
|
1256
|
+
# runtime snapshot.
|
|
1257
|
+
await owner.release_account_lease(selected_lease)
|
|
1258
|
+
selected_lease = None
|
|
1259
|
+
if not probe_refresh_write_skipped:
|
|
1260
|
+
async with owner._repo_factory() as repos:
|
|
1261
|
+
await _restore_sticky_mutation(
|
|
1262
|
+
sticky_repo=repos.sticky_sessions,
|
|
1263
|
+
sticky_key=sticky_key,
|
|
1264
|
+
sticky_kind=sticky_kind,
|
|
1265
|
+
expected_account_id=sticky_mutation.account_id,
|
|
1266
|
+
sticky_existing_account_id=sticky_existing_account_id,
|
|
1267
|
+
)
|
|
1268
|
+
selected_snapshot = None
|
|
1269
|
+
error_message = None
|
|
1270
|
+
selected_states = []
|
|
1271
|
+
selected_account_map = {}
|
|
1272
|
+
if attempt >= MAX_SELECTION_ATTEMPTS:
|
|
1273
|
+
suppress_recovery_probe_candidates = True
|
|
1274
|
+
attempt = 0
|
|
1275
|
+
selection_inputs = await load_selection_inputs()
|
|
1276
|
+
if selection_inputs.error_code is not None and not selection_inputs.accounts:
|
|
1277
|
+
return _direct_error(
|
|
1278
|
+
account=None,
|
|
1279
|
+
error_message=selection_inputs.error_message,
|
|
1280
|
+
error_code=selection_inputs.error_code,
|
|
1281
|
+
)
|
|
1282
|
+
await asyncio.sleep(0)
|
|
1283
|
+
continue
|
|
1284
|
+
selection_inputs = await load_selection_inputs()
|
|
1285
|
+
if selection_inputs.error_code is not None and not selection_inputs.accounts:
|
|
1286
|
+
return _direct_error(
|
|
1287
|
+
account=None,
|
|
1288
|
+
error_message=selection_inputs.error_message,
|
|
1289
|
+
error_code=selection_inputs.error_code,
|
|
1290
|
+
)
|
|
1291
|
+
await asyncio.sleep(0)
|
|
1292
|
+
continue
|
|
1293
|
+
elif selected_snapshot is not None and reserved_probe_admitted:
|
|
1294
|
+
reservation_committed = False
|
|
1295
|
+
try:
|
|
1296
|
+
assert probe_reservation is not None
|
|
1297
|
+
async with owner._runtime_lock:
|
|
1298
|
+
reservation_committed = owner._commit_due_probe_reservation_locked(probe_reservation)
|
|
1299
|
+
if reservation_committed:
|
|
1300
|
+
owner._sync_committed_probe_state_locked(
|
|
1301
|
+
probe_reservation,
|
|
1302
|
+
selected_account_map,
|
|
1303
|
+
selected_states,
|
|
1304
|
+
)
|
|
1305
|
+
else:
|
|
1306
|
+
owner._release_due_probe_reservation_locked(probe_reservation)
|
|
1307
|
+
except BaseException:
|
|
1308
|
+
await owner.release_account_lease(selected_lease)
|
|
1309
|
+
selected_lease = None
|
|
1310
|
+
async with owner._runtime_lock:
|
|
1311
|
+
owner._release_due_probe_reservation_locked(probe_reservation)
|
|
1312
|
+
raise
|
|
1313
|
+
if not reservation_committed:
|
|
1314
|
+
# Runtime health changed while account-state persistence
|
|
1315
|
+
# was in flight. The lease and provisional affinity must
|
|
1316
|
+
# not escape; retry against a fresh runtime snapshot.
|
|
1317
|
+
await owner.release_account_lease(selected_lease)
|
|
1318
|
+
selected_lease = None
|
|
1319
|
+
selected_snapshot = None
|
|
1320
|
+
error_message = None
|
|
1321
|
+
selected_states = []
|
|
1322
|
+
selected_account_map = {}
|
|
1323
|
+
if attempt >= MAX_SELECTION_ATTEMPTS:
|
|
1324
|
+
suppress_recovery_probe_candidates = True
|
|
1325
|
+
attempt = 0
|
|
1326
|
+
selection_inputs = await load_selection_inputs()
|
|
1327
|
+
if selection_inputs.error_code is not None and not selection_inputs.accounts:
|
|
1328
|
+
return _direct_error(
|
|
1329
|
+
account=None,
|
|
1330
|
+
error_message=selection_inputs.error_message,
|
|
1331
|
+
error_code=selection_inputs.error_code,
|
|
1332
|
+
)
|
|
1333
|
+
await asyncio.sleep(0)
|
|
1334
|
+
continue
|
|
1335
|
+
selection_inputs = await load_selection_inputs()
|
|
1336
|
+
if selection_inputs.error_code is not None and not selection_inputs.accounts:
|
|
1337
|
+
return _direct_error(
|
|
1338
|
+
account=None,
|
|
1339
|
+
error_message=selection_inputs.error_message,
|
|
1340
|
+
error_code=selection_inputs.error_code,
|
|
1341
|
+
)
|
|
1342
|
+
await asyncio.sleep(0)
|
|
1343
|
+
continue
|
|
1344
|
+
if should_persist_sticky_mutation and not reserved_probe_admitted:
|
|
1345
|
+
# Sticky decisions stay provisional until cap classification,
|
|
1346
|
+
# final lease admission, account-state persistence, and the
|
|
1347
|
+
# probe CAS (when present) all succeed. Applying one final
|
|
1348
|
+
# desired-state mutation avoids unsafe compensating writes.
|
|
1349
|
+
assert sticky_kind is not None
|
|
1350
|
+
sticky_mutation = sticky_outcome.mutation
|
|
1351
|
+
assert sticky_mutation is not None
|
|
1352
|
+
initialize_seed_key = (
|
|
1353
|
+
sticky_seed_key if sticky_source == "thread_header" and sticky_seed_account_id is None else None
|
|
1354
|
+
)
|
|
1355
|
+
if not _sticky_refresh_write_skippable(sticky_mutation, initialize_seed_key=initialize_seed_key):
|
|
1356
|
+
try:
|
|
1357
|
+
async with owner._repo_factory() as repos:
|
|
1358
|
+
await _persist_sticky_mutation(
|
|
1359
|
+
sticky_repo=repos.sticky_sessions,
|
|
1360
|
+
sticky_key=sticky_key,
|
|
1361
|
+
sticky_kind=sticky_kind,
|
|
1362
|
+
mutation=sticky_mutation,
|
|
1363
|
+
initialize_seed_key=initialize_seed_key,
|
|
1364
|
+
initialize_seed_kind=sticky_seed_kind,
|
|
1365
|
+
)
|
|
1366
|
+
except BaseException:
|
|
1367
|
+
# Runtime admission may already be committed. Preserve
|
|
1368
|
+
# its selection timestamp, but never leak the local
|
|
1369
|
+
# concurrency lease when sticky persistence fails.
|
|
1370
|
+
await owner.release_account_lease(selected_lease)
|
|
1371
|
+
selected_lease = None
|
|
1372
|
+
raise
|
|
1373
|
+
if sticky_outcome.isolation_release is not None:
|
|
1374
|
+
# Here, not at selection: the candidate can still be lost to a
|
|
1375
|
+
# concurrent lease or the attempt retried on stale state, and this
|
|
1376
|
+
# counter is the one the accounts-per-conversation change is judged
|
|
1377
|
+
# by -- a release that never served must not appear in it, and a
|
|
1378
|
+
# retried attempt must not appear twice. Account identifiers are
|
|
1379
|
+
# deliberately omitted: this path has no privacy flag and private
|
|
1380
|
+
# realtime diagnostics must not expose them.
|
|
1381
|
+
release_mapping, overload_free_candidates = sticky_outcome.isolation_release
|
|
1382
|
+
logger.info(
|
|
1383
|
+
"sticky_owner_overload_isolation_reroute sticky_kind=%s overload_free_candidates=%d mapping=%s",
|
|
1384
|
+
sticky_kind.value if sticky_kind is not None else "unknown",
|
|
1385
|
+
overload_free_candidates,
|
|
1386
|
+
release_mapping,
|
|
1387
|
+
)
|
|
1388
|
+
break
|
|
1389
|
+
|
|
1390
|
+
return StickySelectionOutcome(
|
|
1391
|
+
selection_inputs=selection_inputs,
|
|
1392
|
+
selected_snapshot=selected_snapshot,
|
|
1393
|
+
selected_lease=selected_lease,
|
|
1394
|
+
error_message=error_message,
|
|
1395
|
+
error_code=selection_error_code,
|
|
1396
|
+
resets_at=selection_resets_at,
|
|
1397
|
+
hard_affinity_owner_excluded=_hard_affinity_owner_excluded_by_caller(
|
|
1398
|
+
error_code=selection_error_code,
|
|
1399
|
+
owner_account_id=sticky_existing_account_id,
|
|
1400
|
+
exclude_account_ids=request.exclude_account_ids,
|
|
1401
|
+
),
|
|
1402
|
+
)
|
|
1403
|
+
|
|
1404
|
+
|
|
1405
|
+
async def _select_with_stickiness(
|
|
1406
|
+
*,
|
|
1407
|
+
states: list[AccountState],
|
|
1408
|
+
account_map: dict[str, Account],
|
|
1409
|
+
sticky_key: str | None,
|
|
1410
|
+
sticky_kind: StickySessionKind | None,
|
|
1411
|
+
reallocate_sticky: bool,
|
|
1412
|
+
sticky_max_age_seconds: int | None,
|
|
1413
|
+
budget_threshold_pct: float = 95.0,
|
|
1414
|
+
secondary_budget_threshold_pct: float = 100.0,
|
|
1415
|
+
prefer_earlier_reset_accounts: bool,
|
|
1416
|
+
prefer_earlier_reset_window: ResetPreferenceWindow,
|
|
1417
|
+
routing_strategy: RoutingStrategy,
|
|
1418
|
+
relative_availability_power: float = 2.0,
|
|
1419
|
+
relative_availability_top_k: int = 5,
|
|
1420
|
+
sticky_repo: StickySessionsRepository | None,
|
|
1421
|
+
routing_costs_by_account_id: RoutingCostsByAccount | None = None,
|
|
1422
|
+
sticky_existing_account_id: str | None | object = _STICKY_EXISTING_UNSET,
|
|
1423
|
+
initial_preferred_account_id: str | None = None,
|
|
1424
|
+
preserve_existing_mapping_on_fallback: bool = False,
|
|
1425
|
+
preserve_reason_request_local: bool = False,
|
|
1426
|
+
traffic_class: TrafficClass = TRAFFIC_CLASS_FOREGROUND,
|
|
1427
|
+
ignore_standard_quota: bool = False,
|
|
1428
|
+
allow_usage_exhaustion_error: bool = True,
|
|
1429
|
+
usage_exhaustion_states: Iterable[AccountState] | None = None,
|
|
1430
|
+
sticky_refresh_skip_deadline: datetime | None = None,
|
|
1431
|
+
overload_backoff_runtime: Mapping[str, RuntimeState] | None = None,
|
|
1432
|
+
clock: Clock,
|
|
1433
|
+
redact_sensitive_details: bool = False,
|
|
1434
|
+
) -> _StickySelectionOutcome:
|
|
1435
|
+
if not sticky_key or not sticky_repo:
|
|
1436
|
+
return _StickySelectionOutcome(
|
|
1437
|
+
selection=_select_account_preferring_budget_safe(
|
|
1438
|
+
states,
|
|
1439
|
+
prefer_earlier_reset=prefer_earlier_reset_accounts,
|
|
1440
|
+
prefer_earlier_reset_window=prefer_earlier_reset_window,
|
|
1441
|
+
routing_strategy=routing_strategy,
|
|
1442
|
+
relative_availability_power=relative_availability_power,
|
|
1443
|
+
relative_availability_top_k=relative_availability_top_k,
|
|
1444
|
+
budget_threshold_pct=budget_threshold_pct,
|
|
1445
|
+
traffic_class=traffic_class,
|
|
1446
|
+
ignore_standard_quota=ignore_standard_quota,
|
|
1447
|
+
routing_costs_by_account_id=routing_costs_by_account_id,
|
|
1448
|
+
allow_usage_exhaustion_error=allow_usage_exhaustion_error,
|
|
1449
|
+
usage_exhaustion_states=usage_exhaustion_states,
|
|
1450
|
+
)
|
|
1451
|
+
)
|
|
1452
|
+
if sticky_kind is None:
|
|
1453
|
+
raise ValueError("sticky_kind is required when sticky_key is provided")
|
|
1454
|
+
|
|
1455
|
+
pending_mutation: _StickyMutation | None = None
|
|
1456
|
+
|
|
1457
|
+
def finish_selection(
|
|
1458
|
+
selection: SelectionResult,
|
|
1459
|
+
*,
|
|
1460
|
+
persist_account_id: str | None = None,
|
|
1461
|
+
refresh_skip_deadline: datetime | None = None,
|
|
1462
|
+
effective_states: list[AccountState] | None = None,
|
|
1463
|
+
) -> _StickySelectionOutcome:
|
|
1464
|
+
mutation = pending_mutation
|
|
1465
|
+
if persist_account_id is not None:
|
|
1466
|
+
mutation = _StickyMutation(
|
|
1467
|
+
account_id=persist_account_id,
|
|
1468
|
+
refresh_skip_deadline=refresh_skip_deadline,
|
|
1469
|
+
)
|
|
1470
|
+
return _StickySelectionOutcome(
|
|
1471
|
+
selection=selection,
|
|
1472
|
+
mutation=mutation,
|
|
1473
|
+
effective_states=effective_states,
|
|
1474
|
+
isolation_release=isolation_release,
|
|
1475
|
+
)
|
|
1476
|
+
|
|
1477
|
+
if sticky_existing_account_id is _STICKY_EXISTING_UNSET:
|
|
1478
|
+
existing = await sticky_repo.get_account_id(
|
|
1479
|
+
sticky_key,
|
|
1480
|
+
kind=sticky_kind,
|
|
1481
|
+
max_age_seconds=sticky_max_age_seconds,
|
|
1482
|
+
)
|
|
1483
|
+
# The skip deadline is only valid for the lookup that produced the
|
|
1484
|
+
# caller's ``sticky_existing_account_id``; this fresh lookup did not
|
|
1485
|
+
# observe row freshness, so fall back to write-through refresh.
|
|
1486
|
+
sticky_refresh_skip_deadline = None
|
|
1487
|
+
else:
|
|
1488
|
+
existing = sticky_existing_account_id if isinstance(sticky_existing_account_id, str) else None
|
|
1489
|
+
# When the pinned account is temporarily unavailable (rate-limited,
|
|
1490
|
+
# error backoff) but still in the pool, pick a fallback WITHOUT
|
|
1491
|
+
# overwriting the sticky mapping so the next request returns to the
|
|
1492
|
+
# original account — and its warm OpenAI prompt cache — once it
|
|
1493
|
+
# recovers. Only reallocate_sticky=True opts in to permanent
|
|
1494
|
+
# reassignment.
|
|
1495
|
+
persist_fallback = not preserve_existing_mapping_on_fallback
|
|
1496
|
+
apply_sticky_secondary_budget_threshold = False
|
|
1497
|
+
# Set when an isolated soft owner is released: the replacement pick and the
|
|
1498
|
+
# overload-free pool it came from (probe reservation must see that pool).
|
|
1499
|
+
overload_reroute: SelectionResult | None = None
|
|
1500
|
+
overload_reroute_pool: list[AccountState] | None = None
|
|
1501
|
+
usage_exhaustion_state_list = list(usage_exhaustion_states) if usage_exhaustion_states is not None else states
|
|
1502
|
+
# True when that release is request-local: the substitute serves this turn
|
|
1503
|
+
# and the sticky row keeps pointing at the isolated owner, so the thread
|
|
1504
|
+
# returns home when isolation lifts instead of accumulating one permanent
|
|
1505
|
+
# new owner per isolation episode it touches. Never true when the caller
|
|
1506
|
+
# asked for reallocation: ``reallocate_sticky`` is an explicit instruction
|
|
1507
|
+
# to retire the mapping and the isolation reroute reuses the same local
|
|
1508
|
+
# further down, so the caller's intent is captured before that happens.
|
|
1509
|
+
#
|
|
1510
|
+
# ``STICKY_THREAD`` is the exception, because on that kind the flag carries
|
|
1511
|
+
# no per-request intent to respect: ``affinity.py`` sets it on *every*
|
|
1512
|
+
# sticky-thread policy (see ``_resolve_affinity_policy``), so reading it as
|
|
1513
|
+
# "this caller asked to retire the mapping" would exempt the whole kind
|
|
1514
|
+
# from retention and leave it accumulating an owner per isolation episode --
|
|
1515
|
+
# while the two kinds either side of it stopped.
|
|
1516
|
+
caller_requested_reallocation = reallocate_sticky and sticky_kind != StickySessionKind.STICKY_THREAD
|
|
1517
|
+
overload_reroute_request_local = False
|
|
1518
|
+
isolation_release: tuple[str, int] | None = None
|
|
1519
|
+
# A mapping kept because the conversation's owner is *ambiguous* is not a
|
|
1520
|
+
# warm owner waiting out isolation: it may sit outside this request's
|
|
1521
|
+
# routable or security scope. It still keeps its row -- ambiguity is a
|
|
1522
|
+
# reason not to rebind -- but it must not have its TTL extended by a turn
|
|
1523
|
+
# served elsewhere, and it is not an isolation release to count.
|
|
1524
|
+
retention_may_write = not preserve_existing_mapping_on_fallback or preserve_reason_request_local
|
|
1525
|
+
|
|
1526
|
+
def _choose_from(candidates: list[AccountState], *, selection_seed: str | None = None) -> SelectionResult:
|
|
1527
|
+
return _select_account_preferring_budget_safe(
|
|
1528
|
+
candidates,
|
|
1529
|
+
selection_seed=selection_seed,
|
|
1530
|
+
prefer_earlier_reset=prefer_earlier_reset_accounts,
|
|
1531
|
+
prefer_earlier_reset_window=prefer_earlier_reset_window,
|
|
1532
|
+
routing_strategy=routing_strategy,
|
|
1533
|
+
relative_availability_power=relative_availability_power,
|
|
1534
|
+
relative_availability_top_k=relative_availability_top_k,
|
|
1535
|
+
budget_threshold_pct=budget_threshold_pct,
|
|
1536
|
+
secondary_budget_threshold_pct=secondary_budget_threshold_pct,
|
|
1537
|
+
apply_secondary_budget_threshold=apply_sticky_secondary_budget_threshold,
|
|
1538
|
+
traffic_class=traffic_class,
|
|
1539
|
+
ignore_standard_quota=ignore_standard_quota,
|
|
1540
|
+
routing_costs_by_account_id=routing_costs_by_account_id,
|
|
1541
|
+
allow_usage_exhaustion_error=allow_usage_exhaustion_error,
|
|
1542
|
+
usage_exhaustion_states=usage_exhaustion_state_list,
|
|
1543
|
+
)
|
|
1544
|
+
|
|
1545
|
+
if not existing and initial_preferred_account_id is not None:
|
|
1546
|
+
initial_preferred = next(
|
|
1547
|
+
(state for state in states if state.account_id == initial_preferred_account_id),
|
|
1548
|
+
None,
|
|
1549
|
+
)
|
|
1550
|
+
# The process-session preference is a fresh upstream admission on that
|
|
1551
|
+
# account: do not honor it while the account is in overload backoff and
|
|
1552
|
+
# the strategy can actually select an overload-free candidate. A pool
|
|
1553
|
+
# whose only alternatives are unselectable (cooldown, exhausted) keeps
|
|
1554
|
+
# the preference, so the bypass can never fail a request the
|
|
1555
|
+
# preferred account would have served.
|
|
1556
|
+
if (
|
|
1557
|
+
initial_preferred is not None
|
|
1558
|
+
and overload_backoff_runtime is not None
|
|
1559
|
+
and overload_backoff_active(overload_backoff_runtime.get(initial_preferred.account_id), clock.time())
|
|
1560
|
+
):
|
|
1561
|
+
preference_free_pool = filter_overload_backoff_candidates(
|
|
1562
|
+
states, overload_backoff_runtime, now=clock.time()
|
|
1563
|
+
)
|
|
1564
|
+
if preference_free_pool is not states:
|
|
1565
|
+
alternative = _choose_from(preference_free_pool)
|
|
1566
|
+
if alternative.account is not None and alternative.account.account_id != initial_preferred.account_id:
|
|
1567
|
+
overload_reroute = alternative
|
|
1568
|
+
overload_reroute_pool = preference_free_pool
|
|
1569
|
+
if initial_preferred is not None and overload_reroute is None:
|
|
1570
|
+
initial_result = select_account(
|
|
1571
|
+
[initial_preferred],
|
|
1572
|
+
prefer_earlier_reset=prefer_earlier_reset_accounts,
|
|
1573
|
+
prefer_earlier_reset_window=prefer_earlier_reset_window,
|
|
1574
|
+
routing_strategy=routing_strategy,
|
|
1575
|
+
allow_backoff_fallback=False,
|
|
1576
|
+
relative_availability_power=relative_availability_power,
|
|
1577
|
+
relative_availability_top_k=relative_availability_top_k,
|
|
1578
|
+
traffic_class=traffic_class,
|
|
1579
|
+
ignore_standard_quota=ignore_standard_quota,
|
|
1580
|
+
routing_costs=routing_costs_by_account_id,
|
|
1581
|
+
)
|
|
1582
|
+
if initial_result.account is not None:
|
|
1583
|
+
# Persist only the new thread row. The process mapping supplied
|
|
1584
|
+
# the preference but is deliberately outside this mutation.
|
|
1585
|
+
return finish_selection(
|
|
1586
|
+
initial_result,
|
|
1587
|
+
persist_account_id=initial_preferred.account_id,
|
|
1588
|
+
)
|
|
1589
|
+
|
|
1590
|
+
if existing:
|
|
1591
|
+
pinned = next((state for state in states if state.account_id == existing), None)
|
|
1592
|
+
if pinned is not None:
|
|
1593
|
+
# Retaining the pinned owner persists only to advance
|
|
1594
|
+
# ``updated_at`` on TTL-based kinds. When this request's lookup
|
|
1595
|
+
# already observed the row inside the repository's refresh-skip
|
|
1596
|
+
# window, the persist site may skip that write after revalidating
|
|
1597
|
+
# the observed deadline against the clock: concurrent requests on
|
|
1598
|
+
# a hot session otherwise serialize on the same row's upsert
|
|
1599
|
+
# lock. Rebinds and deletes never carry the deadline and always
|
|
1600
|
+
# write immediately.
|
|
1601
|
+
pinned_refresh_account_id = pinned.account_id if sticky_max_age_seconds is not None else None
|
|
1602
|
+
pinned_refresh_skip_deadline = (
|
|
1603
|
+
sticky_refresh_skip_deadline if pinned_refresh_account_id is not None else None
|
|
1604
|
+
)
|
|
1605
|
+
# Proactively rebind session affinity for any sticky kind
|
|
1606
|
+
# once the pinned account is already above the configured
|
|
1607
|
+
# budget threshold. That preserves continuity below the
|
|
1608
|
+
# threshold while avoiding obvious short-window failures once
|
|
1609
|
+
# the session is skating on the edge of exhaustion.
|
|
1610
|
+
now = clock.time()
|
|
1611
|
+
budget_pressured = (
|
|
1612
|
+
sticky_kind
|
|
1613
|
+
in (
|
|
1614
|
+
StickySessionKind.PROMPT_CACHE,
|
|
1615
|
+
StickySessionKind.STICKY_THREAD,
|
|
1616
|
+
StickySessionKind.CODEX_SESSION,
|
|
1617
|
+
)
|
|
1618
|
+
and routing_strategy not in ("sequential_drain", "reset_drain", "single_account")
|
|
1619
|
+
and pinned.status != AccountStatus.RATE_LIMITED
|
|
1620
|
+
and _state_above_sticky_budget_threshold(
|
|
1621
|
+
pinned,
|
|
1622
|
+
budget_threshold_pct,
|
|
1623
|
+
secondary_budget_threshold_pct,
|
|
1624
|
+
)
|
|
1625
|
+
)
|
|
1626
|
+
rate_limit_far_away = (
|
|
1627
|
+
sticky_kind == StickySessionKind.PROMPT_CACHE
|
|
1628
|
+
and pinned.status == AccountStatus.RATE_LIMITED
|
|
1629
|
+
and pinned.reset_at is not None
|
|
1630
|
+
and pinned.reset_at - now >= 600 # 10 minutes
|
|
1631
|
+
)
|
|
1632
|
+
|
|
1633
|
+
burn_first_reallocate = pinned.routing_policy != ROUTING_POLICY_BURN_FIRST
|
|
1634
|
+
if burn_first_reallocate:
|
|
1635
|
+
burn_first_candidates = [state for state in states if state.routing_policy == ROUTING_POLICY_BURN_FIRST]
|
|
1636
|
+
if burn_first_candidates:
|
|
1637
|
+
burn_first = select_account(
|
|
1638
|
+
burn_first_candidates,
|
|
1639
|
+
prefer_earlier_reset=prefer_earlier_reset_accounts,
|
|
1640
|
+
routing_strategy=routing_strategy,
|
|
1641
|
+
allow_backoff_fallback=False,
|
|
1642
|
+
deterministic_probe=True,
|
|
1643
|
+
relative_availability_power=relative_availability_power,
|
|
1644
|
+
relative_availability_top_k=relative_availability_top_k,
|
|
1645
|
+
traffic_class=traffic_class,
|
|
1646
|
+
ignore_standard_quota=ignore_standard_quota,
|
|
1647
|
+
)
|
|
1648
|
+
burn_first_reallocate = burn_first.account is not None
|
|
1649
|
+
|
|
1650
|
+
# Isolation stage of the overload backoff: the pinned owner is a
|
|
1651
|
+
# *soft* mapping (hard continuity owners never reach this path)
|
|
1652
|
+
# and every request re-entering it is a fresh admission upstream
|
|
1653
|
+
# keeps rejecting, so release it while the strategy can still pick
|
|
1654
|
+
# an overload-free sibling. Soft backoff levels below isolation
|
|
1655
|
+
# keep the owner, so a short burst never churns warm sessions.
|
|
1656
|
+
if (
|
|
1657
|
+
sticky_kind
|
|
1658
|
+
in (
|
|
1659
|
+
StickySessionKind.PROMPT_CACHE,
|
|
1660
|
+
StickySessionKind.STICKY_THREAD,
|
|
1661
|
+
StickySessionKind.CODEX_SESSION,
|
|
1662
|
+
)
|
|
1663
|
+
# Retention is only ever applied to an owner that can come
|
|
1664
|
+
# back. A PAUSED/DEACTIVATED owner is not "isolated, will
|
|
1665
|
+
# recover" -- it is gone, and it must fall through to the
|
|
1666
|
+
# unchanged rebind path below so the mapping is released on
|
|
1667
|
+
# this turn rather than held for the isolation window.
|
|
1668
|
+
and pinned.status in _RECOVERABLE_STATUSES
|
|
1669
|
+
):
|
|
1670
|
+
overload_reroute_pool = sticky_owner_isolation_reroute_pool(
|
|
1671
|
+
states,
|
|
1672
|
+
overload_backoff_runtime,
|
|
1673
|
+
owner_account_id=pinned.account_id,
|
|
1674
|
+
now=now,
|
|
1675
|
+
)
|
|
1676
|
+
|
|
1677
|
+
if overload_reroute_pool is not None:
|
|
1678
|
+
# A budget-pressured owner's replacement honors the same
|
|
1679
|
+
# secondary-budget filter the budget reallocation applies, so
|
|
1680
|
+
# the substitute is not an equally pressured sibling that the
|
|
1681
|
+
# next turn would reallocate again.
|
|
1682
|
+
if budget_pressured:
|
|
1683
|
+
apply_sticky_secondary_budget_threshold = True
|
|
1684
|
+
# The release is request-local, so this pick is repeated on
|
|
1685
|
+
# every turn of the thread while isolation holds. A plain
|
|
1686
|
+
# weighted draw would answer differently each time and rotate
|
|
1687
|
+
# the conversation across the siblings -- more churn than the
|
|
1688
|
+
# single rebind this change replaces -- so the pick is seeded
|
|
1689
|
+
# by the thread instead.
|
|
1690
|
+
#
|
|
1691
|
+
# The seed is spent *inside* the selector, among the accounts
|
|
1692
|
+
# it would otherwise draw from, rather than picking a
|
|
1693
|
+
# substitute here and asking the selector to ratify it: every
|
|
1694
|
+
# gate the selector applies is pool-relative (it accepts an
|
|
1695
|
+
# over-budget account only when no safe one is visible, a
|
|
1696
|
+
# draining one only when nothing healthier is, a ``preserve``
|
|
1697
|
+
# one only when no ``normal`` one is), and validating a single
|
|
1698
|
+
# candidate hides all of them.
|
|
1699
|
+
candidate = _choose_from(
|
|
1700
|
+
overload_reroute_pool,
|
|
1701
|
+
selection_seed=isolation_substitute_seed(
|
|
1702
|
+
sticky_key=sticky_key,
|
|
1703
|
+
owner_account_id=pinned.account_id,
|
|
1704
|
+
),
|
|
1705
|
+
)
|
|
1706
|
+
if candidate.account is not None and candidate.account.account_id != pinned.account_id:
|
|
1707
|
+
overload_reroute = candidate
|
|
1708
|
+
overload_reroute_request_local = not caller_requested_reallocation
|
|
1709
|
+
# Account identifiers are deliberately omitted: this path
|
|
1710
|
+
# has no privacy flag and private realtime diagnostics
|
|
1711
|
+
# must not expose them. The isolation-engaged warning
|
|
1712
|
+
# already names the account under the redaction policy.
|
|
1713
|
+
#
|
|
1714
|
+
# ``mapping=retained`` is the attributable marker for the
|
|
1715
|
+
# accounts-per-conversation factor: it says this turn went
|
|
1716
|
+
# to a sibling *without* adding an owner to the thread.
|
|
1717
|
+
if retention_may_write or not overload_reroute_request_local:
|
|
1718
|
+
isolation_release = (
|
|
1719
|
+
"retained" if overload_reroute_request_local else "rebound",
|
|
1720
|
+
len(overload_reroute_pool),
|
|
1721
|
+
)
|
|
1722
|
+
else:
|
|
1723
|
+
overload_reroute_pool = None
|
|
1724
|
+
|
|
1725
|
+
if overload_reroute is not None:
|
|
1726
|
+
# Skips the pinned-owner return and the rate-limit grace
|
|
1727
|
+
# retry below; the mutation block honors
|
|
1728
|
+
# ``overload_reroute_request_local`` and keeps the mapping.
|
|
1729
|
+
reallocate_sticky = True
|
|
1730
|
+
elif not ((budget_pressured or rate_limit_far_away) and burn_first_reallocate):
|
|
1731
|
+
pinned_result = select_account(
|
|
1732
|
+
[pinned],
|
|
1733
|
+
prefer_earlier_reset=prefer_earlier_reset_accounts,
|
|
1734
|
+
prefer_earlier_reset_window=prefer_earlier_reset_window,
|
|
1735
|
+
routing_strategy=routing_strategy,
|
|
1736
|
+
allow_backoff_fallback=False,
|
|
1737
|
+
relative_availability_power=relative_availability_power,
|
|
1738
|
+
relative_availability_top_k=relative_availability_top_k,
|
|
1739
|
+
traffic_class=traffic_class,
|
|
1740
|
+
ignore_standard_quota=ignore_standard_quota,
|
|
1741
|
+
routing_costs=routing_costs_by_account_id,
|
|
1742
|
+
)
|
|
1743
|
+
if pinned_result.account is not None:
|
|
1744
|
+
return finish_selection(
|
|
1745
|
+
pinned_result,
|
|
1746
|
+
persist_account_id=pinned_refresh_account_id,
|
|
1747
|
+
refresh_skip_deadline=pinned_refresh_skip_deadline,
|
|
1748
|
+
)
|
|
1749
|
+
else:
|
|
1750
|
+
# Reallocate only when a burn-first target exists and can
|
|
1751
|
+
# currently be selected, avoiding sticky churn to
|
|
1752
|
+
# ineligible targets.
|
|
1753
|
+
# Before reallocating, check whether the pool has a
|
|
1754
|
+
# meaningfully better candidate. When every account
|
|
1755
|
+
# is above the budget threshold, reallocating just
|
|
1756
|
+
# wastes DB writes and destroys prompt-cache locality
|
|
1757
|
+
# (thrashing).
|
|
1758
|
+
if budget_pressured:
|
|
1759
|
+
apply_sticky_secondary_budget_threshold = True
|
|
1760
|
+
pool_best = _select_account_preferring_budget_safe(
|
|
1761
|
+
states,
|
|
1762
|
+
prefer_earlier_reset=prefer_earlier_reset_accounts,
|
|
1763
|
+
prefer_earlier_reset_window=prefer_earlier_reset_window,
|
|
1764
|
+
routing_strategy=routing_strategy,
|
|
1765
|
+
relative_availability_power=relative_availability_power,
|
|
1766
|
+
relative_availability_top_k=relative_availability_top_k,
|
|
1767
|
+
deterministic_probe=True,
|
|
1768
|
+
budget_threshold_pct=budget_threshold_pct,
|
|
1769
|
+
secondary_budget_threshold_pct=secondary_budget_threshold_pct,
|
|
1770
|
+
apply_secondary_budget_threshold=True,
|
|
1771
|
+
traffic_class=traffic_class,
|
|
1772
|
+
ignore_standard_quota=ignore_standard_quota,
|
|
1773
|
+
routing_costs_by_account_id=routing_costs_by_account_id,
|
|
1774
|
+
allow_usage_exhaustion_error=allow_usage_exhaustion_error,
|
|
1775
|
+
usage_exhaustion_states=usage_exhaustion_states,
|
|
1776
|
+
)
|
|
1777
|
+
pool_also_exhausted = pool_best.account is not None and (
|
|
1778
|
+
pool_best.account.account_id == pinned.account_id
|
|
1779
|
+
or _state_above_sticky_budget_threshold(
|
|
1780
|
+
pool_best.account,
|
|
1781
|
+
budget_threshold_pct,
|
|
1782
|
+
secondary_budget_threshold_pct,
|
|
1783
|
+
)
|
|
1784
|
+
)
|
|
1785
|
+
if pool_also_exhausted:
|
|
1786
|
+
pinned_result = select_account(
|
|
1787
|
+
[pinned],
|
|
1788
|
+
prefer_earlier_reset=prefer_earlier_reset_accounts,
|
|
1789
|
+
prefer_earlier_reset_window=prefer_earlier_reset_window,
|
|
1790
|
+
routing_strategy=routing_strategy,
|
|
1791
|
+
allow_backoff_fallback=False,
|
|
1792
|
+
relative_availability_power=relative_availability_power,
|
|
1793
|
+
relative_availability_top_k=relative_availability_top_k,
|
|
1794
|
+
traffic_class=traffic_class,
|
|
1795
|
+
ignore_standard_quota=ignore_standard_quota,
|
|
1796
|
+
routing_costs=routing_costs_by_account_id,
|
|
1797
|
+
)
|
|
1798
|
+
if pinned_result.account is not None:
|
|
1799
|
+
return finish_selection(
|
|
1800
|
+
pinned_result,
|
|
1801
|
+
persist_account_id=pinned_refresh_account_id,
|
|
1802
|
+
refresh_skip_deadline=pinned_refresh_skip_deadline,
|
|
1803
|
+
)
|
|
1804
|
+
reallocate_sticky = True
|
|
1805
|
+
# Grace period: if the pinned account is rate-limited with a
|
|
1806
|
+
# known reset time within a short window, retry selection
|
|
1807
|
+
# with a small time advance to preserve prompt cache.
|
|
1808
|
+
# A shallow copy is used so the time-advanced selection does
|
|
1809
|
+
# not mutate the original state (which is later synced to DB
|
|
1810
|
+
# by _sync_state for all accounts).
|
|
1811
|
+
if not reallocate_sticky and pinned.status == AccountStatus.RATE_LIMITED:
|
|
1812
|
+
grace_copy = replace(pinned)
|
|
1813
|
+
grace_result = select_account(
|
|
1814
|
+
[grace_copy],
|
|
1815
|
+
now=clock.time() + _STICKY_GRACE_PERIOD_SECONDS,
|
|
1816
|
+
prefer_earlier_reset=prefer_earlier_reset_accounts,
|
|
1817
|
+
prefer_earlier_reset_window=prefer_earlier_reset_window,
|
|
1818
|
+
routing_strategy=routing_strategy,
|
|
1819
|
+
allow_backoff_fallback=False,
|
|
1820
|
+
relative_availability_power=relative_availability_power,
|
|
1821
|
+
relative_availability_top_k=relative_availability_top_k,
|
|
1822
|
+
traffic_class=traffic_class,
|
|
1823
|
+
ignore_standard_quota=ignore_standard_quota,
|
|
1824
|
+
routing_costs=routing_costs_by_account_id,
|
|
1825
|
+
)
|
|
1826
|
+
if grace_result.account is not None:
|
|
1827
|
+
return finish_selection(
|
|
1828
|
+
grace_result,
|
|
1829
|
+
persist_account_id=pinned_refresh_account_id,
|
|
1830
|
+
refresh_skip_deadline=pinned_refresh_skip_deadline,
|
|
1831
|
+
)
|
|
1832
|
+
if overload_reroute_request_local:
|
|
1833
|
+
# Overload isolation is a property of the *account*, not of
|
|
1834
|
+
# the thread, and it lifts. Deleting or rebinding the mapping
|
|
1835
|
+
# here is what made one conversation accumulate a new account
|
|
1836
|
+
# per isolation episode; preserving it costs nothing in
|
|
1837
|
+
# availability because the turn is already being served by
|
|
1838
|
+
# the substitute.
|
|
1839
|
+
persist_fallback = False
|
|
1840
|
+
# Preserving must not mean going silent. A TTL-based kind
|
|
1841
|
+
# (PROMPT_CACHE) expires on ``updated_at``, and the default
|
|
1842
|
+
# affinity TTL and the default isolation window are both 1800
|
|
1843
|
+
# seconds -- so suppressing every write would let the retained
|
|
1844
|
+
# row die *before* isolation lifts, and the next turn would
|
|
1845
|
+
# persist the substitute as a brand-new owner. That is the
|
|
1846
|
+
# accumulation this change exists to remove, arriving through
|
|
1847
|
+
# the back door. Rewrite the same owner instead: the row stays
|
|
1848
|
+
# on the warm account and its freshness tracks the thread.
|
|
1849
|
+
if sticky_max_age_seconds is not None and retention_may_write:
|
|
1850
|
+
pending_mutation = _StickyMutation(
|
|
1851
|
+
account_id=pinned.account_id,
|
|
1852
|
+
refresh_skip_deadline=sticky_refresh_skip_deadline,
|
|
1853
|
+
)
|
|
1854
|
+
elif reallocate_sticky:
|
|
1855
|
+
pending_mutation = _StickyMutation(account_id=None)
|
|
1856
|
+
elif pinned.status not in _RECOVERABLE_STATUSES:
|
|
1857
|
+
# Permanently down (PAUSED/DEACTIVATED) — let the
|
|
1858
|
+
# fallback be persisted to rebind the mapping.
|
|
1859
|
+
pass
|
|
1860
|
+
elif sticky_max_age_seconds is not None:
|
|
1861
|
+
# TTL-based kind (PROMPT_CACHE): preserve the original
|
|
1862
|
+
# mapping so the next request returns to the warm-cache
|
|
1863
|
+
# account once it recovers. The TTL will naturally
|
|
1864
|
+
# expire the mapping if recovery takes too long.
|
|
1865
|
+
persist_fallback = False
|
|
1866
|
+
# else: durable kind without TTL (CODEX_SESSION) — persist
|
|
1867
|
+
# fallback so the session sticks to one account during
|
|
1868
|
+
# the outage instead of bouncing across random fallbacks.
|
|
1869
|
+
else:
|
|
1870
|
+
if not preserve_existing_mapping_on_fallback:
|
|
1871
|
+
pending_mutation = _StickyMutation(account_id=None)
|
|
1872
|
+
|
|
1873
|
+
# Reaching here means a NEW account is being chosen for this key (no
|
|
1874
|
+
# owner, an unusable owner, or a reallocation): a fresh upstream
|
|
1875
|
+
# admission, not warm-session reuse. Prefer accounts upstream is not
|
|
1876
|
+
# currently rejecting as overloaded; fall back to the full pool when the
|
|
1877
|
+
# strategy rejects every overload-free candidate. The pinned-owner paths
|
|
1878
|
+
# above never consult the overload window, so an established owner keeps
|
|
1879
|
+
# serving its session even while backed off.
|
|
1880
|
+
# The owner is isolated *and* absent from selection -- a concurrency cap or
|
|
1881
|
+
# this request's exclusion list removed it before selection ran, so the
|
|
1882
|
+
# isolation branch above never saw a ``pinned`` state for it. Everything
|
|
1883
|
+
# that branch does for a retained owner has to happen here too.
|
|
1884
|
+
owner_isolated_off_pool = (
|
|
1885
|
+
preserve_existing_mapping_on_fallback
|
|
1886
|
+
# ...for request-local pressure only. A mapping kept because the
|
|
1887
|
+
# conversation's owner is ambiguous may be outside this request's
|
|
1888
|
+
# routable or security scope, and refreshing it would pin the thread
|
|
1889
|
+
# to an account that cannot serve it, turn after turn.
|
|
1890
|
+
and preserve_reason_request_local
|
|
1891
|
+
and isinstance(existing, str)
|
|
1892
|
+
and not persist_fallback
|
|
1893
|
+
and not reallocate_sticky
|
|
1894
|
+
and overload_backoff_runtime is not None
|
|
1895
|
+
and overload_isolation_active(overload_backoff_runtime.get(existing), clock.time())
|
|
1896
|
+
)
|
|
1897
|
+
if owner_isolated_off_pool and isinstance(existing, str):
|
|
1898
|
+
existing_owner_state = next(
|
|
1899
|
+
(state for state in usage_exhaustion_state_list if state.account_id == existing),
|
|
1900
|
+
None,
|
|
1901
|
+
)
|
|
1902
|
+
if (
|
|
1903
|
+
sticky_kind
|
|
1904
|
+
in (
|
|
1905
|
+
StickySessionKind.PROMPT_CACHE,
|
|
1906
|
+
StickySessionKind.STICKY_THREAD,
|
|
1907
|
+
StickySessionKind.CODEX_SESSION,
|
|
1908
|
+
)
|
|
1909
|
+
and existing_owner_state is not None
|
|
1910
|
+
and routing_strategy not in ("sequential_drain", "reset_drain", "single_account")
|
|
1911
|
+
# Unlike the pinned branch, no RATE_LIMITED carve-out. There the
|
|
1912
|
+
# exclusion exists because a rate-limited owner takes the separate
|
|
1913
|
+
# grace-retry path instead of budget reallocation; here the owner
|
|
1914
|
+
# never reached selection at all, so skipping the filter would just
|
|
1915
|
+
# hand its replacement to a sibling the budget rule excludes.
|
|
1916
|
+
and _state_above_sticky_budget_threshold(
|
|
1917
|
+
existing_owner_state,
|
|
1918
|
+
budget_threshold_pct,
|
|
1919
|
+
secondary_budget_threshold_pct,
|
|
1920
|
+
)
|
|
1921
|
+
):
|
|
1922
|
+
apply_sticky_secondary_budget_threshold = True
|
|
1923
|
+
fallback_candidates = states
|
|
1924
|
+
if overload_reroute is not None and overload_reroute_pool is not None:
|
|
1925
|
+
fallback_candidates = overload_reroute_pool
|
|
1926
|
+
chosen = overload_reroute
|
|
1927
|
+
else:
|
|
1928
|
+
if overload_backoff_runtime is not None:
|
|
1929
|
+
fallback_candidates = filter_overload_backoff_candidates(states, overload_backoff_runtime, now=clock.time())
|
|
1930
|
+
chosen = _choose_from(fallback_candidates)
|
|
1931
|
+
if chosen.account is None and fallback_candidates is not states:
|
|
1932
|
+
fallback_candidates = states
|
|
1933
|
+
chosen = _choose_from(states)
|
|
1934
|
+
# When an *isolated* owner's mapping is being kept while it sits out
|
|
1935
|
+
# this turn -- cap-filtered or excluded by this request's retry loop,
|
|
1936
|
+
# so it never reached ``selection_states`` and the isolation branch
|
|
1937
|
+
# above could not see a ``pinned`` state -- the substitute is re-picked
|
|
1938
|
+
# on every turn just as it is there, so it gets the same thread-seeded
|
|
1939
|
+
# pick. The seed is spent inside the selector, so the pool-relative
|
|
1940
|
+
# gates all hold.
|
|
1941
|
+
#
|
|
1942
|
+
# Only for isolation. Ordinary cap or retry spillover is a one-turn
|
|
1943
|
+
# detour on a healthy pool, and seeding it would trade that pool's
|
|
1944
|
+
# load-proportional draw (and its recovery probes) for stability
|
|
1945
|
+
# nothing is asking for.
|
|
1946
|
+
if owner_isolated_off_pool and isinstance(existing, str) and chosen.account is not None:
|
|
1947
|
+
seeded = _choose_from(
|
|
1948
|
+
fallback_candidates,
|
|
1949
|
+
selection_seed=isolation_substitute_seed(sticky_key=sticky_key, owner_account_id=existing),
|
|
1950
|
+
)
|
|
1951
|
+
if seeded.account is not None:
|
|
1952
|
+
chosen = seeded
|
|
1953
|
+
serving = chosen.account
|
|
1954
|
+
if serving is not None and serving.account_id != existing:
|
|
1955
|
+
# This is an isolation release too -- the owner is isolated and
|
|
1956
|
+
# a sibling is serving its turn -- so it belongs in the same
|
|
1957
|
+
# counter. Leaving it to the generic spillover line would
|
|
1958
|
+
# undercount exactly the releases this change is measured by.
|
|
1959
|
+
# Account identifiers stay out, as on the branch above.
|
|
1960
|
+
isolation_release = ("retained", len(fallback_candidates))
|
|
1961
|
+
if (
|
|
1962
|
+
isinstance(existing, str)
|
|
1963
|
+
and chosen.account is not None
|
|
1964
|
+
and chosen.account.account_id != existing
|
|
1965
|
+
# The reroute branch already logged its own outcome, retained or not.
|
|
1966
|
+
and overload_reroute is None
|
|
1967
|
+
# ...as did the off-pool retention path. And a mapping that is being
|
|
1968
|
+
# *kept* for any reason -- including an ambiguous owner, which is
|
|
1969
|
+
# preserved but deliberately neither refreshed nor counted -- is not a
|
|
1970
|
+
# rebind, whatever account served this turn.
|
|
1971
|
+
and not owner_isolated_off_pool
|
|
1972
|
+
and persist_fallback
|
|
1973
|
+
and overload_backoff_runtime is not None
|
|
1974
|
+
and overload_isolation_active(overload_backoff_runtime.get(existing), clock.time())
|
|
1975
|
+
):
|
|
1976
|
+
# An isolated owner whose mapping is being rebound rather than kept:
|
|
1977
|
+
# past recovering, dropped before selection ran (``_selectable_accounts``
|
|
1978
|
+
# removes paused and deactivated accounts), or outside this request's
|
|
1979
|
+
# scope. Still an isolation release, and the counter must carry both
|
|
1980
|
+
# halves of what isolation does to a conversation or it reports only the
|
|
1981
|
+
# reversible one.
|
|
1982
|
+
#
|
|
1983
|
+
# Emitted here, where the replacement is known: the request may instead
|
|
1984
|
+
# have failed, or kept its owner, and a release that did not happen must
|
|
1985
|
+
# not be counted. Account identifiers stay out, as on the retained line.
|
|
1986
|
+
isolation_release = ("rebound", len(fallback_candidates))
|
|
1987
|
+
if pending_mutation is None and sticky_max_age_seconds is not None and owner_isolated_off_pool:
|
|
1988
|
+
# The owner is isolated and its mapping is being kept, but it never
|
|
1989
|
+
# reached ``selection_states`` -- a cap or this request's exclusion
|
|
1990
|
+
# list removed it -- so the retention branch above could not see a
|
|
1991
|
+
# ``pinned`` state and scheduled no write. Preserving must not mean
|
|
1992
|
+
# going silent on a TTL-based kind: the default affinity TTL and the
|
|
1993
|
+
# default isolation window are both 1800 seconds, so a retained row
|
|
1994
|
+
# left unwritten expires *during* the episode and the next turn
|
|
1995
|
+
# persists the substitute as a brand-new owner -- the accumulation
|
|
1996
|
+
# this change removes, arriving through the back door.
|
|
1997
|
+
#
|
|
1998
|
+
# Scoped to isolation on purpose. When an owner is merely rate-limited
|
|
1999
|
+
# the TTL expiring *is* the intended escape, and refreshing it would
|
|
2000
|
+
# hold a conversation on an account that may never come back.
|
|
2001
|
+
pending_mutation = _StickyMutation(
|
|
2002
|
+
account_id=existing,
|
|
2003
|
+
refresh_skip_deadline=sticky_refresh_skip_deadline,
|
|
2004
|
+
)
|
|
2005
|
+
chosen_pool = fallback_candidates if fallback_candidates is not states else None
|
|
2006
|
+
if persist_fallback and chosen.account is not None and chosen.account.account_id in account_map:
|
|
2007
|
+
return finish_selection(chosen, persist_account_id=chosen.account.account_id, effective_states=chosen_pool)
|
|
2008
|
+
if preserve_existing_mapping_on_fallback and chosen.account is not None and existing is not None:
|
|
2009
|
+
# Spillover is deliberately request-local. The alternate may create
|
|
2010
|
+
# its own hard response/file/bridge owner, but local cap pressure
|
|
2011
|
+
# alone never turns this soft mapping into a distributed commit.
|
|
2012
|
+
logger.info(
|
|
2013
|
+
"internal_soft_affinity_spillover old_account_id=%s new_account_id=%s sticky_kind=%s",
|
|
2014
|
+
"<redacted>" if redact_sensitive_details else existing,
|
|
2015
|
+
"<redacted>" if redact_sensitive_details else chosen.account.account_id,
|
|
2016
|
+
sticky_kind.value,
|
|
2017
|
+
)
|
|
2018
|
+
return finish_selection(chosen, effective_states=chosen_pool)
|
|
2019
|
+
|
|
2020
|
+
|
|
2021
|
+
def _sticky_refresh_write_skippable(
|
|
2022
|
+
mutation: _StickyMutation,
|
|
2023
|
+
*,
|
|
2024
|
+
initialize_seed_key: str | None,
|
|
2025
|
+
) -> bool:
|
|
2026
|
+
"""Whether this mutation's write may be omitted at persist time.
|
|
2027
|
+
|
|
2028
|
+
True only for a pure same-owner freshness rewrite whose observed skip
|
|
2029
|
+
deadline still holds now, at the moment the statement would otherwise be
|
|
2030
|
+
issued — admission and account-state persistence sit between selection
|
|
2031
|
+
and this point, so the deadline computed at lookup time must be
|
|
2032
|
+
revalidated to keep the mapping's effective expiry within the documented
|
|
2033
|
+
skip-window bound. Deletes and seed-initializing writes are never
|
|
2034
|
+
skippable.
|
|
2035
|
+
"""
|
|
2036
|
+
if mutation.account_id is None or initialize_seed_key is not None:
|
|
2037
|
+
return False
|
|
2038
|
+
deadline = mutation.refresh_skip_deadline
|
|
2039
|
+
return isinstance(deadline, datetime) and utcnow() <= deadline
|
|
2040
|
+
|
|
2041
|
+
|
|
2042
|
+
async def _persist_sticky_mutation(
|
|
2043
|
+
*,
|
|
2044
|
+
sticky_repo: StickySessionsRepository,
|
|
2045
|
+
sticky_key: str,
|
|
2046
|
+
sticky_kind: StickySessionKind,
|
|
2047
|
+
mutation: _StickyMutation,
|
|
2048
|
+
initialize_seed_key: str | None = None,
|
|
2049
|
+
initialize_seed_kind: StickySessionKind | None = None,
|
|
2050
|
+
) -> None:
|
|
2051
|
+
if mutation.account_id is None:
|
|
2052
|
+
await sticky_repo.delete(sticky_key, kind=sticky_kind)
|
|
2053
|
+
return
|
|
2054
|
+
if initialize_seed_key is not None:
|
|
2055
|
+
if initialize_seed_kind is None:
|
|
2056
|
+
raise ValueError("initialize_seed_kind is required when initialize_seed_key is provided")
|
|
2057
|
+
# Current Codex sends thread-id on the first root request, so a fresh
|
|
2058
|
+
# process has no older bare-session request available to create its
|
|
2059
|
+
# default. Initialize it exactly once from the first admitted thread.
|
|
2060
|
+
# insert-if-absent is essential: failover or a later child may move its
|
|
2061
|
+
# own bounded row but can never rewrite the process/sibling default.
|
|
2062
|
+
# The repository operation is intentionally atomic; splitting it into
|
|
2063
|
+
# the public insert/upsert methods would commit a process default even
|
|
2064
|
+
# when persistence of the initiating thread fails.
|
|
2065
|
+
await sticky_repo.upsert_with_seed_if_absent(
|
|
2066
|
+
sticky_key,
|
|
2067
|
+
mutation.account_id,
|
|
2068
|
+
kind=sticky_kind,
|
|
2069
|
+
seed_key=initialize_seed_key,
|
|
2070
|
+
seed_kind=initialize_seed_kind,
|
|
2071
|
+
)
|
|
2072
|
+
return
|
|
2073
|
+
await sticky_repo.upsert(sticky_key, mutation.account_id, kind=sticky_kind)
|
|
2074
|
+
|
|
2075
|
+
|
|
2076
|
+
async def _restore_sticky_mutation(
|
|
2077
|
+
*,
|
|
2078
|
+
sticky_repo: StickySessionsRepository,
|
|
2079
|
+
sticky_key: str,
|
|
2080
|
+
sticky_kind: StickySessionKind,
|
|
2081
|
+
expected_account_id: str | None,
|
|
2082
|
+
sticky_existing_account_id: str | None | object,
|
|
2083
|
+
) -> None:
|
|
2084
|
+
if sticky_existing_account_id is _STICKY_EXISTING_UNSET:
|
|
2085
|
+
return
|
|
2086
|
+
await sticky_repo.restore_if_current(
|
|
2087
|
+
sticky_key,
|
|
2088
|
+
kind=sticky_kind,
|
|
2089
|
+
expected_account_id=expected_account_id,
|
|
2090
|
+
restore_account_id=sticky_existing_account_id if isinstance(sticky_existing_account_id, str) else None,
|
|
2091
|
+
)
|
|
2092
|
+
|
|
2093
|
+
|
|
2094
|
+
def _filter_states_for_account_caps(
|
|
2095
|
+
states: Iterable[AccountState],
|
|
2096
|
+
*,
|
|
2097
|
+
lease_kind: AccountLeaseKind | None,
|
|
2098
|
+
caps: AccountConcurrencyCaps,
|
|
2099
|
+
stream_reserve_slots: int = 0,
|
|
2100
|
+
) -> list[AccountState]:
|
|
2101
|
+
if lease_kind is None:
|
|
2102
|
+
return list(states)
|
|
2103
|
+
filtered: list[AccountState] = []
|
|
2104
|
+
for state in states:
|
|
2105
|
+
if lease_kind == "response_create":
|
|
2106
|
+
cap = caps.response_create_limit
|
|
2107
|
+
if cap > 0 and state.inflight_response_creates >= cap:
|
|
2108
|
+
continue
|
|
2109
|
+
else:
|
|
2110
|
+
cap = caps.stream_limit
|
|
2111
|
+
effective_cap = max(1, cap - max(0, stream_reserve_slots))
|
|
2112
|
+
if cap > 0 and state.inflight_streams >= effective_cap:
|
|
2113
|
+
continue
|
|
2114
|
+
filtered.append(state)
|
|
2115
|
+
return filtered
|
|
2116
|
+
|
|
2117
|
+
|
|
2118
|
+
def _probing_result_requires_recovery_reservation(
|
|
2119
|
+
states: Collection[AccountState],
|
|
2120
|
+
result_account: AccountState | None,
|
|
2121
|
+
*,
|
|
2122
|
+
routing_strategy: str,
|
|
2123
|
+
traffic_class: TrafficClass,
|
|
2124
|
+
now: float,
|
|
2125
|
+
) -> bool:
|
|
2126
|
+
if routing_strategy in ("sequential_drain", "reset_drain", "single_account"):
|
|
2127
|
+
return False
|
|
2128
|
+
if result_account is None or result_account.health_tier != HEALTH_TIER_PROBING:
|
|
2129
|
+
return False
|
|
2130
|
+
return _pool_has_available_healthy_account_without_backoff(states, traffic_class=traffic_class, now=now)
|
|
2131
|
+
|
|
2132
|
+
|
|
2133
|
+
def _filter_recovery_probe_candidates(
|
|
2134
|
+
states: list[AccountState],
|
|
2135
|
+
*,
|
|
2136
|
+
traffic_class: TrafficClass,
|
|
2137
|
+
now: float,
|
|
2138
|
+
) -> list[AccountState]:
|
|
2139
|
+
if not _pool_has_available_healthy_account_without_backoff(states, traffic_class=traffic_class, now=now):
|
|
2140
|
+
return states
|
|
2141
|
+
return [state for state in states if state.health_tier != HEALTH_TIER_PROBING]
|
|
2142
|
+
|
|
2143
|
+
|
|
2144
|
+
def _pool_has_available_healthy_account_without_backoff(
|
|
2145
|
+
states: Iterable[AccountState],
|
|
2146
|
+
*,
|
|
2147
|
+
traffic_class: TrafficClass,
|
|
2148
|
+
now: float,
|
|
2149
|
+
) -> bool:
|
|
2150
|
+
return _pool_has_available_account_without_backoff(
|
|
2151
|
+
(state for state in states if state.health_tier == HEALTH_TIER_HEALTHY),
|
|
2152
|
+
traffic_class=traffic_class,
|
|
2153
|
+
now=now,
|
|
2154
|
+
)
|
|
2155
|
+
|
|
2156
|
+
|
|
2157
|
+
def _pool_has_available_account_without_backoff(
|
|
2158
|
+
states: Iterable[AccountState],
|
|
2159
|
+
*,
|
|
2160
|
+
traffic_class: TrafficClass,
|
|
2161
|
+
now: float,
|
|
2162
|
+
) -> bool:
|
|
2163
|
+
"""Return whether the complete pool passes non-cap routing eligibility."""
|
|
2164
|
+
# ``select_account`` normalizes expired quota/cooldown fields in place;
|
|
2165
|
+
# classify on copies so cap-error reporting cannot mutate the real
|
|
2166
|
+
# selection snapshot before sticky persistence. Keep the pool intact:
|
|
2167
|
+
# opportunistic admission compares candidates with one another.
|
|
2168
|
+
result = select_account(
|
|
2169
|
+
[replace(state) for state in states],
|
|
2170
|
+
now=now,
|
|
2171
|
+
routing_strategy="single_account",
|
|
2172
|
+
allow_backoff_fallback=False,
|
|
2173
|
+
traffic_class=traffic_class,
|
|
2174
|
+
)
|
|
2175
|
+
return result.account is not None
|
|
2176
|
+
|
|
2177
|
+
|
|
2178
|
+
def _account_cap_error_code(lease_kind: AccountLeaseKind | None) -> str | None:
|
|
2179
|
+
if lease_kind == "response_create":
|
|
2180
|
+
return "account_response_create_cap"
|
|
2181
|
+
if lease_kind == "stream":
|
|
2182
|
+
return "account_stream_cap"
|
|
2183
|
+
return None
|
|
2184
|
+
|
|
2185
|
+
|
|
2186
|
+
def _account_cap_error_message(lease_kind: AccountLeaseKind | None, caps: AccountConcurrencyCaps) -> str:
|
|
2187
|
+
if lease_kind == "response_create":
|
|
2188
|
+
cap = caps.response_create_limit
|
|
2189
|
+
if caps.replica_count > 1 and caps.configured_response_create_limit is not None:
|
|
2190
|
+
return (
|
|
2191
|
+
f"Account response-create capacity is exhausted; this replica's share is {cap} "
|
|
2192
|
+
f"of the per-account limit {caps.configured_response_create_limit} "
|
|
2193
|
+
f"across {caps.replica_count} replicas"
|
|
2194
|
+
)
|
|
2195
|
+
return f"Account response-create capacity is exhausted; per-account limit is {cap}"
|
|
2196
|
+
if lease_kind == "stream":
|
|
2197
|
+
cap = caps.stream_limit
|
|
2198
|
+
if caps.replica_count > 1 and caps.configured_stream_limit is not None:
|
|
2199
|
+
return (
|
|
2200
|
+
f"Account stream capacity is exhausted; this replica's share is {cap} "
|
|
2201
|
+
f"of the per-account limit {caps.configured_stream_limit} "
|
|
2202
|
+
f"across {caps.replica_count} replicas. "
|
|
2203
|
+
"Increase the dashboard stream limit or wait for active streams to finish."
|
|
2204
|
+
)
|
|
2205
|
+
return (
|
|
2206
|
+
f"Account stream capacity is exhausted; per-account limit is {cap}. "
|
|
2207
|
+
"Increase the dashboard stream limit or wait for active streams to finish."
|
|
2208
|
+
)
|
|
2209
|
+
return "Account capacity is exhausted"
|
|
2210
|
+
|
|
2211
|
+
|
|
2212
|
+
def _state_above_budget_threshold(state: AccountState, budget_threshold_pct: float) -> bool:
|
|
2213
|
+
used_percent = state.priority_used_percent if state.priority_used_percent is not None else state.used_percent
|
|
2214
|
+
return used_percent is not None and used_percent > budget_threshold_pct
|
|
2215
|
+
|
|
2216
|
+
|
|
2217
|
+
def _state_above_sticky_budget_threshold(
|
|
2218
|
+
state: AccountState,
|
|
2219
|
+
budget_threshold_pct: float,
|
|
2220
|
+
secondary_budget_threshold_pct: float | None = None,
|
|
2221
|
+
) -> bool:
|
|
2222
|
+
secondary_threshold = (
|
|
2223
|
+
budget_threshold_pct if secondary_budget_threshold_pct is None else secondary_budget_threshold_pct
|
|
2224
|
+
)
|
|
2225
|
+
used_percent = state.priority_used_percent if state.priority_used_percent is not None else state.used_percent
|
|
2226
|
+
if state.limit_scoped_usage and state.priority_secondary_used_percent is None:
|
|
2227
|
+
secondary_used_percent = used_percent
|
|
2228
|
+
else:
|
|
2229
|
+
secondary_used_percent = (
|
|
2230
|
+
state.priority_secondary_used_percent
|
|
2231
|
+
if state.priority_secondary_used_percent is not None
|
|
2232
|
+
else state.secondary_used_percent
|
|
2233
|
+
)
|
|
2234
|
+
return (used_percent is not None and used_percent > budget_threshold_pct) or (
|
|
2235
|
+
secondary_used_percent is not None and secondary_used_percent > secondary_threshold
|
|
2236
|
+
)
|
|
2237
|
+
|
|
2238
|
+
|
|
2239
|
+
def _select_account_preferring_budget_safe(
|
|
2240
|
+
states: Iterable[AccountState],
|
|
2241
|
+
*,
|
|
2242
|
+
prefer_earlier_reset: bool,
|
|
2243
|
+
prefer_earlier_reset_window: ResetPreferenceWindow = "secondary",
|
|
2244
|
+
routing_strategy: RoutingStrategy,
|
|
2245
|
+
relative_availability_power: float = 2.0,
|
|
2246
|
+
relative_availability_top_k: int = 5,
|
|
2247
|
+
budget_threshold_pct: float,
|
|
2248
|
+
secondary_budget_threshold_pct: float = 100.0,
|
|
2249
|
+
apply_secondary_budget_threshold: bool = False,
|
|
2250
|
+
allow_backoff_fallback: bool = True,
|
|
2251
|
+
deterministic_probe: bool = False,
|
|
2252
|
+
traffic_class: TrafficClass = TRAFFIC_CLASS_FOREGROUND,
|
|
2253
|
+
ignore_standard_quota: bool = False,
|
|
2254
|
+
routing_costs_by_account_id: RoutingCostsByAccount | None = None,
|
|
2255
|
+
allow_usage_exhaustion_error: bool = True,
|
|
2256
|
+
usage_exhaustion_states: Iterable[AccountState] | None = None,
|
|
2257
|
+
selection_seed: str | None = None,
|
|
2258
|
+
) -> SelectionResult:
|
|
2259
|
+
state_list = list(states)
|
|
2260
|
+
if selection_seed is None and routing_strategy not in ("sequential_drain", "reset_drain", "single_account"):
|
|
2261
|
+
# This pass must precede budget-safe and routing-policy shortcuts below;
|
|
2262
|
+
# otherwise a healthy preferred account can starve PROBING indefinitely.
|
|
2263
|
+
#
|
|
2264
|
+
# A seeded caller is exempt. The due probe is whichever probing account
|
|
2265
|
+
# went quiet longest, so admitting one advances its clock and hands the
|
|
2266
|
+
# next turn to a different sibling -- the rotation the seed exists to
|
|
2267
|
+
# prevent. Probes ride on unbound traffic and on every other thread
|
|
2268
|
+
# instead, so nothing here starves them; only the handful of
|
|
2269
|
+
# conversations being held warm through an isolation window stop
|
|
2270
|
+
# carrying them.
|
|
2271
|
+
recovery_probe = select_account(
|
|
2272
|
+
state_list,
|
|
2273
|
+
prefer_earlier_reset=prefer_earlier_reset,
|
|
2274
|
+
prefer_earlier_reset_window=prefer_earlier_reset_window,
|
|
2275
|
+
routing_strategy=routing_strategy,
|
|
2276
|
+
allow_backoff_fallback=allow_backoff_fallback,
|
|
2277
|
+
deterministic_probe=deterministic_probe,
|
|
2278
|
+
recovery_probe_only=True,
|
|
2279
|
+
relative_availability_power=relative_availability_power,
|
|
2280
|
+
relative_availability_top_k=relative_availability_top_k,
|
|
2281
|
+
traffic_class=traffic_class,
|
|
2282
|
+
ignore_standard_quota=ignore_standard_quota,
|
|
2283
|
+
routing_costs=routing_costs_by_account_id,
|
|
2284
|
+
allow_usage_exhaustion_error=False,
|
|
2285
|
+
)
|
|
2286
|
+
if recovery_probe.account is not None:
|
|
2287
|
+
return recovery_probe
|
|
2288
|
+
state_budget_threshold = (
|
|
2289
|
+
(
|
|
2290
|
+
lambda state: _state_above_sticky_budget_threshold(
|
|
2291
|
+
state,
|
|
2292
|
+
budget_threshold_pct,
|
|
2293
|
+
secondary_budget_threshold_pct,
|
|
2294
|
+
)
|
|
2295
|
+
)
|
|
2296
|
+
if apply_secondary_budget_threshold
|
|
2297
|
+
else (lambda state: _state_above_budget_threshold(state, budget_threshold_pct))
|
|
2298
|
+
)
|
|
2299
|
+
if routing_strategy in ("sequential_drain", "reset_drain", "single_account"):
|
|
2300
|
+
budget_safe_states = [
|
|
2301
|
+
state
|
|
2302
|
+
for state in state_list
|
|
2303
|
+
if state.routing_policy != ROUTING_POLICY_PRESERVE and not state_budget_threshold(state)
|
|
2304
|
+
]
|
|
2305
|
+
return select_account(
|
|
2306
|
+
budget_safe_states or state_list,
|
|
2307
|
+
prefer_earlier_reset=prefer_earlier_reset,
|
|
2308
|
+
prefer_earlier_reset_window=prefer_earlier_reset_window,
|
|
2309
|
+
routing_strategy=routing_strategy,
|
|
2310
|
+
allow_backoff_fallback=allow_backoff_fallback,
|
|
2311
|
+
deterministic_probe=deterministic_probe,
|
|
2312
|
+
relative_availability_power=relative_availability_power,
|
|
2313
|
+
relative_availability_top_k=relative_availability_top_k,
|
|
2314
|
+
traffic_class=traffic_class,
|
|
2315
|
+
ignore_standard_quota=ignore_standard_quota,
|
|
2316
|
+
routing_costs=routing_costs_by_account_id,
|
|
2317
|
+
allow_usage_exhaustion_error=allow_usage_exhaustion_error,
|
|
2318
|
+
usage_exhaustion_states=usage_exhaustion_states,
|
|
2319
|
+
selection_seed=selection_seed,
|
|
2320
|
+
)
|
|
2321
|
+
|
|
2322
|
+
best_health_states = _best_health_tier_states(state_list)
|
|
2323
|
+
burn_first_states = [state for state in best_health_states if state.routing_policy == ROUTING_POLICY_BURN_FIRST]
|
|
2324
|
+
if burn_first_states:
|
|
2325
|
+
burn_first = select_account(
|
|
2326
|
+
burn_first_states,
|
|
2327
|
+
prefer_earlier_reset=prefer_earlier_reset,
|
|
2328
|
+
prefer_earlier_reset_window=prefer_earlier_reset_window,
|
|
2329
|
+
routing_strategy=routing_strategy,
|
|
2330
|
+
allow_backoff_fallback=False,
|
|
2331
|
+
deterministic_probe=deterministic_probe,
|
|
2332
|
+
relative_availability_power=relative_availability_power,
|
|
2333
|
+
relative_availability_top_k=relative_availability_top_k,
|
|
2334
|
+
traffic_class=traffic_class,
|
|
2335
|
+
ignore_standard_quota=ignore_standard_quota,
|
|
2336
|
+
routing_costs=routing_costs_by_account_id,
|
|
2337
|
+
allow_usage_exhaustion_error=allow_usage_exhaustion_error,
|
|
2338
|
+
usage_exhaustion_states=usage_exhaustion_states,
|
|
2339
|
+
selection_seed=selection_seed,
|
|
2340
|
+
)
|
|
2341
|
+
if burn_first.account is not None:
|
|
2342
|
+
return burn_first
|
|
2343
|
+
|
|
2344
|
+
preferred_states = [
|
|
2345
|
+
state
|
|
2346
|
+
for state in state_list
|
|
2347
|
+
if state.routing_policy != ROUTING_POLICY_PRESERVE and not state_budget_threshold(state)
|
|
2348
|
+
]
|
|
2349
|
+
if preferred_states:
|
|
2350
|
+
selection_pool = preferred_states if len(preferred_states) != len(state_list) else state_list
|
|
2351
|
+
preferred = select_account(
|
|
2352
|
+
selection_pool,
|
|
2353
|
+
prefer_earlier_reset=prefer_earlier_reset,
|
|
2354
|
+
prefer_earlier_reset_window=prefer_earlier_reset_window,
|
|
2355
|
+
routing_strategy=routing_strategy,
|
|
2356
|
+
allow_backoff_fallback=allow_backoff_fallback,
|
|
2357
|
+
deterministic_probe=deterministic_probe,
|
|
2358
|
+
relative_availability_power=relative_availability_power,
|
|
2359
|
+
relative_availability_top_k=relative_availability_top_k,
|
|
2360
|
+
traffic_class=traffic_class,
|
|
2361
|
+
ignore_standard_quota=ignore_standard_quota,
|
|
2362
|
+
routing_costs=routing_costs_by_account_id,
|
|
2363
|
+
allow_usage_exhaustion_error=allow_usage_exhaustion_error,
|
|
2364
|
+
usage_exhaustion_states=usage_exhaustion_states,
|
|
2365
|
+
selection_seed=selection_seed,
|
|
2366
|
+
)
|
|
2367
|
+
if preferred.account is not None:
|
|
2368
|
+
return preferred
|
|
2369
|
+
if len(preferred_states) == len(state_list):
|
|
2370
|
+
return preferred
|
|
2371
|
+
if routing_strategy == "usage_weighted" and state_list:
|
|
2372
|
+
return select_account(
|
|
2373
|
+
state_list,
|
|
2374
|
+
prefer_earlier_reset=prefer_earlier_reset,
|
|
2375
|
+
prefer_earlier_reset_window=prefer_earlier_reset_window,
|
|
2376
|
+
routing_strategy=routing_strategy,
|
|
2377
|
+
allow_backoff_fallback=allow_backoff_fallback,
|
|
2378
|
+
deterministic_probe=deterministic_probe,
|
|
2379
|
+
usage_weighted_order="primary_first",
|
|
2380
|
+
traffic_class=traffic_class,
|
|
2381
|
+
ignore_standard_quota=ignore_standard_quota,
|
|
2382
|
+
routing_costs=routing_costs_by_account_id,
|
|
2383
|
+
allow_usage_exhaustion_error=allow_usage_exhaustion_error,
|
|
2384
|
+
usage_exhaustion_states=usage_exhaustion_states,
|
|
2385
|
+
selection_seed=selection_seed,
|
|
2386
|
+
)
|
|
2387
|
+
return select_account(
|
|
2388
|
+
state_list,
|
|
2389
|
+
prefer_earlier_reset=prefer_earlier_reset,
|
|
2390
|
+
prefer_earlier_reset_window=prefer_earlier_reset_window,
|
|
2391
|
+
routing_strategy=routing_strategy,
|
|
2392
|
+
allow_backoff_fallback=allow_backoff_fallback,
|
|
2393
|
+
deterministic_probe=deterministic_probe,
|
|
2394
|
+
relative_availability_power=relative_availability_power,
|
|
2395
|
+
relative_availability_top_k=relative_availability_top_k,
|
|
2396
|
+
traffic_class=traffic_class,
|
|
2397
|
+
ignore_standard_quota=ignore_standard_quota,
|
|
2398
|
+
routing_costs=routing_costs_by_account_id,
|
|
2399
|
+
allow_usage_exhaustion_error=allow_usage_exhaustion_error,
|
|
2400
|
+
usage_exhaustion_states=usage_exhaustion_states,
|
|
2401
|
+
selection_seed=selection_seed,
|
|
2402
|
+
)
|
|
2403
|
+
|
|
2404
|
+
|
|
2405
|
+
def _best_health_tier_states(states: list[AccountState]) -> list[AccountState]:
|
|
2406
|
+
healthy = [state for state in states if state.health_tier == HEALTH_TIER_HEALTHY]
|
|
2407
|
+
if healthy:
|
|
2408
|
+
return healthy
|
|
2409
|
+
probing = [state for state in states if state.health_tier == HEALTH_TIER_PROBING]
|
|
2410
|
+
if probing:
|
|
2411
|
+
return probing
|
|
2412
|
+
draining = [state for state in states if state.health_tier == HEALTH_TIER_DRAINING]
|
|
2413
|
+
return draining or states
|
|
2414
|
+
|
|
2415
|
+
|
|
2416
|
+
def _clone_account(account: Account) -> Account:
|
|
2417
|
+
return clone_row(account)
|