codexLB-up 1.25.0b10__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- app/__init__.py +10 -0
- app/admin_cli.py +494 -0
- app/cli.py +339 -0
- app/codex_sessions_retag.py +487 -0
- app/core/__init__.py +0 -0
- app/core/audit/__init__.py +0 -0
- app/core/audit/service.py +216 -0
- app/core/audit/types.py +94 -0
- app/core/auth/__init__.py +236 -0
- app/core/auth/api_key_cache.py +65 -0
- app/core/auth/dashboard_access.py +429 -0
- app/core/auth/dashboard_mode.py +171 -0
- app/core/auth/dashboard_session_ttl.py +74 -0
- app/core/auth/dashboard_users_cache.py +120 -0
- app/core/auth/dependencies.py +710 -0
- app/core/auth/external_identity.py +78 -0
- app/core/auth/guardian.py +374 -0
- app/core/auth/models.py +49 -0
- app/core/auth/providers/__init__.py +129 -0
- app/core/auth/providers/oidc.py +838 -0
- app/core/auth/providers/registry.py +118 -0
- app/core/auth/refresh.py +394 -0
- app/core/auth/step_up.py +117 -0
- app/core/auth/totp.py +81 -0
- app/core/balancer/__init__.py +71 -0
- app/core/balancer/logic.py +1612 -0
- app/core/balancer/rendezvous_hash.py +21 -0
- app/core/balancer/types.py +21 -0
- app/core/bootstrap.py +172 -0
- app/core/cache/invalidation.py +426 -0
- app/core/clients/__init__.py +0 -0
- app/core/clients/codex.py +996 -0
- app/core/clients/codex_version.py +187 -0
- app/core/clients/codex_version_snapshot.py +2 -0
- app/core/clients/files.py +534 -0
- app/core/clients/headers.py +29 -0
- app/core/clients/http.py +511 -0
- app/core/clients/model_fetcher.py +276 -0
- app/core/clients/native_egress.py +1304 -0
- app/core/clients/oauth.py +477 -0
- app/core/clients/proxy.py +6089 -0
- app/core/clients/proxy_websocket.py +1411 -0
- app/core/clients/rate_limit_reset_credits.py +412 -0
- app/core/clients/stream_errors.py +12 -0
- app/core/clients/thread_cache_identity.py +347 -0
- app/core/clients/usage.py +467 -0
- app/core/clock.py +170 -0
- app/core/config/__init__.py +0 -0
- app/core/config/background_jobs.py +73 -0
- app/core/config/context_window_overrides.py +166 -0
- app/core/config/dashboard_overrides.py +145 -0
- app/core/config/inheritable.py +47 -0
- app/core/config/key_fingerprint.py +115 -0
- app/core/config/settings.py +872 -0
- app/core/config/settings_cache.py +100 -0
- app/core/config/spool_retention.py +217 -0
- app/core/config/tiers.py +174 -0
- app/core/conversation_archive.py +568 -0
- app/core/crypto.py +37 -0
- app/core/draining.py +11 -0
- app/core/errors.py +365 -0
- app/core/exceptions.py +155 -0
- app/core/handlers/__init__.py +3 -0
- app/core/handlers/exceptions.py +478 -0
- app/core/http_protocol.py +176 -0
- app/core/http_protocol_httptools.py +204 -0
- app/core/ingress_limits.py +17 -0
- app/core/metadata_store.py +18 -0
- app/core/metrics/__init__.py +0 -0
- app/core/metrics/middleware.py +74 -0
- app/core/metrics/prometheus.py +604 -0
- app/core/middleware/__init__.py +25 -0
- app/core/middleware/api_firewall.py +111 -0
- app/core/middleware/app_version.py +37 -0
- app/core/middleware/dashboard_auth_proxy.py +55 -0
- app/core/middleware/dashboard_csrf.py +177 -0
- app/core/middleware/dashboard_gzip.py +41 -0
- app/core/middleware/dashboard_overrides.py +59 -0
- app/core/middleware/firewall_cache.py +91 -0
- app/core/middleware/inflight.py +126 -0
- app/core/middleware/multipart_content_encoding.py +129 -0
- app/core/middleware/path_rewrite.py +148 -0
- app/core/middleware/request_body_limit.py +196 -0
- app/core/middleware/request_decompression.py +209 -0
- app/core/middleware/request_id.py +72 -0
- app/core/middleware/required_capability_http.py +116 -0
- app/core/middleware/trusted_proxy_headers.py +36 -0
- app/core/multipart.py +237 -0
- app/core/multipart_fields.py +91 -0
- app/core/openai/__init__.py +0 -0
- app/core/openai/chat_requests.py +493 -0
- app/core/openai/chat_responses.py +931 -0
- app/core/openai/contracts.py +46 -0
- app/core/openai/exceptions.py +24 -0
- app/core/openai/host_models.py +14 -0
- app/core/openai/images.py +413 -0
- app/core/openai/message_coercion.py +376 -0
- app/core/openai/model_refresh_scheduler.py +423 -0
- app/core/openai/model_registry.py +997 -0
- app/core/openai/model_registry_store.py +548 -0
- app/core/openai/models.py +172 -0
- app/core/openai/parsing.py +97 -0
- app/core/openai/requests.py +1980 -0
- app/core/openai/strict_schema.py +199 -0
- app/core/openai/tool_call_safety.py +99 -0
- app/core/openai/v1_requests.py +187 -0
- app/core/plan_types.py +90 -0
- app/core/prestop.py +202 -0
- app/core/rate_limiter/__init__.py +0 -0
- app/core/rate_limiter/db_rate_limiter.py +215 -0
- app/core/request_locality.py +473 -0
- app/core/resilience/__init__.py +0 -0
- app/core/resilience/backpressure.py +57 -0
- app/core/resilience/bulkhead.py +202 -0
- app/core/resilience/circuit_breaker.py +150 -0
- app/core/resilience/degradation.py +55 -0
- app/core/resilience/loop_lag_monitor.py +60 -0
- app/core/resilience/memory_monitor.py +137 -0
- app/core/resilience/network_recovery.py +256 -0
- app/core/resilience/overload.py +98 -0
- app/core/resilience/toggles.py +114 -0
- app/core/retention/__init__.py +0 -0
- app/core/retention/job.py +240 -0
- app/core/retention/scheduler.py +69 -0
- app/core/runtime_logging.py +601 -0
- app/core/scheduling/__init__.py +0 -0
- app/core/scheduling/leader_election.py +993 -0
- app/core/scheduling/leader_election_handle.py +25 -0
- app/core/server.py +140 -0
- app/core/shutdown.py +326 -0
- app/core/socket_peer.py +22 -0
- app/core/startup.py +41 -0
- app/core/timeout_invariants.py +347 -0
- app/core/tracing/__init__.py +0 -0
- app/core/tracing/otel.py +207 -0
- app/core/types.py +6 -0
- app/core/upstream_proxy/__init__.py +13 -0
- app/core/upstream_proxy/cache.py +130 -0
- app/core/upstream_proxy/resolver.py +189 -0
- app/core/upstream_proxy/types.py +104 -0
- app/core/usage/__init__.py +407 -0
- app/core/usage/depletion.py +109 -0
- app/core/usage/live_hub.py +41 -0
- app/core/usage/live_snapshots.py +174 -0
- app/core/usage/logs.py +212 -0
- app/core/usage/metadata_scheduler.py +118 -0
- app/core/usage/models.py +54 -0
- app/core/usage/pricing.py +576 -0
- app/core/usage/pricing_catalog.py +241 -0
- app/core/usage/pricing_snapshot.json +821 -0
- app/core/usage/quota.py +133 -0
- app/core/usage/refresh_policy.py +21 -0
- app/core/usage/refresh_scheduler.py +713 -0
- app/core/usage/reset_credits_refresh_scheduler.py +435 -0
- app/core/usage/types.py +168 -0
- app/core/utils/__init__.py +0 -0
- app/core/utils/json_guards.py +18 -0
- app/core/utils/locks.py +33 -0
- app/core/utils/masking.py +20 -0
- app/core/utils/proxy_env.py +104 -0
- app/core/utils/request_id.py +65 -0
- app/core/utils/retry.py +77 -0
- app/core/utils/shared_future.py +172 -0
- app/core/utils/sse.py +286 -0
- app/core/utils/time.py +24 -0
- app/db/__init__.py +0 -0
- app/db/account_identity_lock.py +56 -0
- app/db/alembic/__init__.py +1 -0
- app/db/alembic/env.py +70 -0
- app/db/alembic/revision_ids.py +43 -0
- app/db/alembic/versions/20260213_000000_base_schema.py +248 -0
- app/db/alembic/versions/20260213_000100_normalize_account_plan_types.py +45 -0
- app/db/alembic/versions/20260213_000200_add_request_logs_reasoning_effort.py +45 -0
- app/db/alembic/versions/20260213_000300_add_accounts_reset_at.py +45 -0
- app/db/alembic/versions/20260213_000400_add_accounts_chatgpt_account_id.py +45 -0
- app/db/alembic/versions/20260213_000500_add_dashboard_settings.py +70 -0
- app/db/alembic/versions/20260213_000600_add_dashboard_settings_totp.py +64 -0
- app/db/alembic/versions/20260213_000700_add_dashboard_settings_password.py +59 -0
- app/db/alembic/versions/20260213_000800_add_api_keys.py +87 -0
- app/db/alembic/versions/20260214_000000_add_api_key_limits.py +143 -0
- app/db/alembic/versions/20260215_000000_add_idx_logs_requested_at.py +38 -0
- app/db/alembic/versions/20260218_000000_add_api_key_usage_reservations.py +117 -0
- app/db/alembic/versions/20260218_000100_add_import_without_overwrite_and_drop_accounts_email_unique.py +198 -0
- app/db/alembic/versions/20260225_000000_add_dashboard_settings_routing_strategy.py +53 -0
- app/db/alembic/versions/20260228_020000_align_api_key_limit_enum_types.py +104 -0
- app/db/alembic/versions/20260228_030000_add_api_firewall_allowlist.py +42 -0
- app/db/alembic/versions/20260306_000000_add_request_logs_service_tier.py +45 -0
- app/db/alembic/versions/20260307_000000_add_api_key_enforcement_fields.py +56 -0
- app/db/alembic/versions/20260308_000000_add_sqlite_performance_indexes.py +41 -0
- app/db/alembic/versions/20260309_000000_add_additional_usage_history.py +71 -0
- app/db/alembic/versions/20260309_000000_request_logs_nullable_account_id.py +28 -0
- app/db/alembic/versions/20260310_000000_add_request_logs_transport.py +45 -0
- app/db/alembic/versions/20260310_000000_fix_postgresql_enum_value_casing.py +79 -0
- app/db/alembic/versions/20260310_120000_add_sticky_session_kinds_and_affinity_ttl.py +110 -0
- app/db/alembic/versions/20260312_000000_add_additional_usage_quota_key.py +193 -0
- app/db/alembic/versions/20260312_000000_split_sticky_sessions_primary_key_by_kind.py +96 -0
- app/db/alembic/versions/20260312_010000_merge_additional_usage_and_sticky_session_heads.py +25 -0
- app/db/alembic/versions/20260312_020000_merge_request_logs_transport_and_feature_heads.py +25 -0
- app/db/alembic/versions/20260312_020000_merge_request_logs_transport_and_sticky_heads.py +25 -0
- app/db/alembic/versions/20260312_120000_add_dashboard_upstream_stream_transport.py +62 -0
- app/db/alembic/versions/20260313_024500_merge_request_log_transport_heads.py +27 -0
- app/db/alembic/versions/20260319_100937_increase_prompt_cache_ttl_to_1800s.py +73 -0
- app/db/alembic/versions/20260319_111500_merge_prompt_cache_ttl_and_upstream_transport_heads.py +25 -0
- app/db/alembic/versions/20260319_183000_normalize_sqlite_account_status_casing.py +48 -0
- app/db/alembic/versions/20260320_000000_add_request_log_requested_actual_tiers.py +51 -0
- app/db/alembic/versions/20260321_190000_tighten_dashboard_db_indexes.py +64 -0
- app/db/alembic/versions/20260321_210000_merge_request_log_tiers_and_dashboard_index_heads.py +25 -0
- app/db/alembic/versions/20260325_000000_add_request_log_cost.py +193 -0
- app/db/alembic/versions/20260328_000000_add_rate_limit_attempts.py +48 -0
- app/db/alembic/versions/20260328_120000_add_request_log_ttft.py +37 -0
- app/db/alembic/versions/20260328_130000_add_audit_logs_table.py +45 -0
- app/db/alembic/versions/20260328_140000_add_scheduler_leader_table.py +39 -0
- app/db/alembic/versions/20260330_000000_add_cache_locality_settings.py +69 -0
- app/db/alembic/versions/20260330_010000_merge_scheduler_leader_and_cache_locality_heads.py +25 -0
- app/db/alembic/versions/20260330_020000_add_bridge_ring_members.py +46 -0
- app/db/alembic/versions/20260401_000000_add_api_key_enforced_service_tier.py +52 -0
- app/db/alembic/versions/20260401_000000_add_cache_invalidation.py +41 -0
- app/db/alembic/versions/20260401_020000_merge_cache_invalidation_and_api_key_service_tier_heads.py +22 -0
- app/db/alembic/versions/20260402_000000_switch_dashboard_routing_default_to_capacity_weighted.py +66 -0
- app/db/alembic/versions/20260403_000000_add_credit_api_key_limit_values.py +44 -0
- app/db/alembic/versions/20260406_000000_add_api_key_account_assignments.py +57 -0
- app/db/alembic/versions/20260406_010000_add_api_key_assignment_scope_flag.py +75 -0
- app/db/alembic/versions/20260407_000000_add_bridge_gateway_safe_mode.py +51 -0
- app/db/alembic/versions/20260407_010000_merge_api_key_assignment_and_bridge_gateway_heads.py +25 -0
- app/db/alembic/versions/20260408_000000_switch_import_without_overwrite_default_to_true.py +73 -0
- app/db/alembic/versions/20260408_010000_merge_import_without_overwrite_and_assignment_heads.py +25 -0
- app/db/alembic/versions/20260409_000000_add_http_bridge_sessions.py +130 -0
- app/db/alembic/versions/20260409_000000_switch_sticky_threads_and_prefer_earlier_reset_defaults_to_true.py +89 -0
- app/db/alembic/versions/20260409_010000_add_dashboard_settings_bootstrap_token.py +57 -0
- app/db/alembic/versions/20260409_010000_merge_http_bridge_and_import_without_overwrite_heads.py +25 -0
- app/db/alembic/versions/20260409_020000_fix_http_bridge_last_seen_index.py +46 -0
- app/db/alembic/versions/20260410_000000_backfill_pristine_dashboard_settings_defaults.py +77 -0
- app/db/alembic/versions/20260410_030000_restore_import_without_overwrite_default_true.py +73 -0
- app/db/alembic/versions/20260410_040000_merge_dashboard_defaults_and_import_default_heads.py +25 -0
- app/db/alembic/versions/20260413_000000_add_accounts_blocked_at.py +44 -0
- app/db/alembic/versions/20260415_160000_add_automations_tables.py +189 -0
- app/db/alembic/versions/20260415_160000_add_request_logs_response_lookup_index.py +67 -0
- app/db/alembic/versions/20260417_000000_add_request_log_plan_type.py +44 -0
- app/db/alembic/versions/20260418_190000_add_automation_runs_cycle_key.py +87 -0
- app/db/alembic/versions/20260419_000000_add_automation_run_cycle_metadata.py +56 -0
- app/db/alembic/versions/20260419_020000_add_automation_run_cycles_snapshot_tables.py +342 -0
- app/db/alembic/versions/20260421_000000_add_dashboard_session_ttl_setting.py +51 -0
- app/db/alembic/versions/20260421_120000_merge_request_log_lookup_and_plan_type_heads.py +25 -0
- app/db/alembic/versions/20260421_130000_merge_automation_and_request_log_heads.py +25 -0
- app/db/alembic/versions/20260422_103000_normalize_legacy_automation_run_cycle_keys.py +648 -0
- app/db/alembic/versions/20260423_120000_add_api_key_limit_reset_at_index.py +43 -0
- app/db/alembic/versions/20260424_000000_merge_dashboard_session_ttl_and_request_log_heads.py +25 -0
- app/db/alembic/versions/20260426_000000_add_dashboard_relative_availability_settings.py +68 -0
- app/db/alembic/versions/20260427_130000_add_warmup_model_and_request_kind.py +75 -0
- app/db/alembic/versions/20260506_000000_add_account_routing_policy.py +37 -0
- app/db/alembic/versions/20260509_000000_add_api_key_traffic_class.py +37 -0
- app/db/alembic/versions/20260509_010000_add_additional_quota_routing_policies.py +37 -0
- app/db/alembic/versions/20260513_000000_add_accounts_alias.py +44 -0
- app/db/alembic/versions/20260513_000000_add_api_key_apply_to_codex_model.py +59 -0
- app/db/alembic/versions/20260514_000000_add_request_logs_api_key_time_index.py +43 -0
- app/db/alembic/versions/20260515_000000_add_prefer_earlier_reset_window.py +47 -0
- app/db/alembic/versions/20260515_000000_soft_delete_request_logs_on_account_delete.py +83 -0
- app/db/alembic/versions/20260515_020000_add_split_sticky_budget_thresholds.py +62 -0
- app/db/alembic/versions/20260516_000000_add_sqlite_hot_path_indexes.py +75 -0
- app/db/alembic/versions/20260517_000000_merge_sqlite_hot_path_and_request_log_delete_heads.py +24 -0
- app/db/alembic/versions/20260518_000000_add_http_bridge_durable_input_prefix.py +44 -0
- app/db/alembic/versions/20260518_010000_merge_http_bridge_and_sqlite_recovery_heads.py +25 -0
- app/db/alembic/versions/20260518_120000_add_dashboard_guest_access.py +57 -0
- app/db/alembic/versions/20260520_000000_merge_api_key_and_http_bridge_heads.py +26 -0
- app/db/alembic/versions/20260520_010000_add_request_logs_api_key_account_index.py +45 -0
- app/db/alembic/versions/20260520_010000_merge_automation_and_api_key_heads.py +26 -0
- app/db/alembic/versions/20260520_030000_add_quota_planner.py +194 -0
- app/db/alembic/versions/20260520_040000_merge_quota_planner_and_request_log_index_heads.py +24 -0
- app/db/alembic/versions/20260521_000000_add_account_security_work_authorized.py +46 -0
- app/db/alembic/versions/20260521_000000_merge_dashboard_guest_and_api_key_heads.py +26 -0
- app/db/alembic/versions/20260521_000000_merge_prefer_reset_window_head.py +26 -0
- app/db/alembic/versions/20260521_000000_merge_warmup_and_request_log_index_heads.py +25 -0
- app/db/alembic/versions/20260522_000000_add_limit_warmup_trigger.py +195 -0
- app/db/alembic/versions/20260524_000000_merge_accounts_alias_and_reset_window_heads.py +24 -0
- app/db/alembic/versions/20260525_000000_add_usage_raw_window_latest_index.py +31 -0
- app/db/alembic/versions/20260526_000000_add_request_log_failure_metadata.py +100 -0
- app/db/alembic/versions/20260531_000000_add_account_workspace_identity.py +39 -0
- app/db/alembic/versions/20260531_000000_add_single_account_routing.py +42 -0
- app/db/alembic/versions/20260601_000000_merge_relative_availability_and_usage_raw_heads.py +24 -0
- app/db/alembic/versions/20260601_010000_add_api_key_usage_sections.py +58 -0
- app/db/alembic/versions/20260601_010000_add_routing_policy_persistence.py +67 -0
- app/db/alembic/versions/20260601_010000_merge_security_work_and_relative_availability_heads.py +25 -0
- app/db/alembic/versions/20260601_010000_merge_split_sticky_and_relative_usage_heads.py +25 -0
- app/db/alembic/versions/20260601_010000_merge_warmup_and_relative_availability_heads.py +25 -0
- app/db/alembic/versions/20260601_020000_merge_additional_quota_routing_and_relative_availability_heads.py +25 -0
- app/db/alembic/versions/20260601_020000_merge_quota_planner_and_relative_availability_heads.py +25 -0
- app/db/alembic/versions/20260601_020000_merge_reset_window_and_relative_availability_heads.py +25 -0
- app/db/alembic/versions/20260601_020000_merge_routing_policy_and_split_threshold_heads.py +24 -0
- app/db/alembic/versions/20260601_020000_merge_warmup_and_request_log_failure_heads.py +25 -0
- app/db/alembic/versions/20260601_030000_merge_single_account_and_routing_quota_heads.py +29 -0
- app/db/alembic/versions/20260601_040000_merge_security_work_and_routing_consolidation_heads.py +25 -0
- app/db/alembic/versions/20260601_050000_merge_quota_planner_and_routing_consolidation_heads.py +25 -0
- app/db/alembic/versions/20260602_050000_add_upstream_proxy_routing.py +164 -0
- app/db/alembic/versions/20260602_060000_merge_account_workspace_and_failure_heads.py +25 -0
- app/db/alembic/versions/20260602_070000_add_request_log_useragent_fields.py +86 -0
- app/db/alembic/versions/20260602_070000_merge_quota_planner_and_account_workspace_heads.py +25 -0
- app/db/alembic/versions/20260602_080000_merge_upstream_proxy_and_quota_planner_heads.py +25 -0
- app/db/alembic/versions/20260602_080000_merge_useragent_and_upstream_proxy_heads.py +24 -0
- app/db/alembic/versions/20260603_000000_add_weekly_pace_working_days.py +52 -0
- app/db/alembic/versions/20260603_000000_free_account_monthly_window.py +60 -0
- app/db/alembic/versions/20260604_000000_add_reauth_required_account_status.py +159 -0
- app/db/alembic/versions/20260604_010000_merge_automations_and_reauth_status_heads.py +25 -0
- app/db/alembic/versions/20260604_010000_merge_dashboard_guest_and_reauth_status_heads.py +25 -0
- app/db/alembic/versions/20260604_010000_merge_useragent_and_reauth_heads.py +24 -0
- app/db/alembic/versions/20260607_000000_merge_weekly_monthly_useragent_heads.py +28 -0
- app/db/alembic/versions/20260608_000000_add_hide_upstream_quota_from_api_keys.py +50 -0
- app/db/alembic/versions/20260611_000000_add_request_log_archive_request_id.py +47 -0
- app/db/alembic/versions/20260611_000000_merge_automations_and_weekly_monthly_useragent_heads.py +26 -0
- app/db/alembic/versions/20260611_000000_merge_dashboard_guest_and_weekly_useragent_heads.py +26 -0
- app/db/alembic/versions/20260611_010000_add_request_log_client_ip.py +57 -0
- app/db/alembic/versions/20260611_020000_merge_request_log_archive_and_client_ip_heads.py +26 -0
- app/db/alembic/versions/20260611_030000_merge_dashboard_guest_and_request_log_client_ip_heads.py +26 -0
- app/db/alembic/versions/20260613_000000_add_accounts_codex_installation_id.py +57 -0
- app/db/alembic/versions/20260613_010000_merge_installation_id_and_dashboard_weekly_heads.py +26 -0
- app/db/alembic/versions/20260615_000000_merge_automations_and_dashboard_guest_heads.py +26 -0
- app/db/alembic/versions/20260623_000000_add_limit_warmup_exhausted_threshold.py +51 -0
- app/db/alembic/versions/20260626_000000_add_http_downstream_transport_policy.py +46 -0
- app/db/alembic/versions/20260626_010000_add_request_logs_upstream_transport.py +31 -0
- app/db/alembic/versions/20260629_000000_add_dashboard_query_hot_path_indexes.py +53 -0
- app/db/alembic/versions/20260630_000000_add_limit_warmup_staggered_idle.py +47 -0
- app/db/alembic/versions/20260630_000000_merge_request_log_client_ip_and_dashboard_hot_path_heads.py +26 -0
- app/db/alembic/versions/20260630_010000_merge_warmup_and_request_log_dashboard_heads.py +26 -0
- app/db/alembic/versions/20260630_020000_merge_automations_and_main_heads.py +26 -0
- app/db/alembic/versions/20260630_020000_merge_warmup_threshold_and_main_heads.py +26 -0
- app/db/alembic/versions/20260630_030000_add_automation_run_model_snapshot.py +59 -0
- app/db/alembic/versions/20260630_040000_merge_automation_snapshot_and_warmup_threshold_heads.py +26 -0
- app/db/alembic/versions/20260630_050000_add_automation_run_prompt_snapshot.py +52 -0
- app/db/alembic/versions/20260701_000000_add_weekly_pace_smoothing_minutes.py +51 -0
- app/db/alembic/versions/20260702_000000_add_model_sources.py +176 -0
- app/db/alembic/versions/20260705_000000_add_model_source_audio_transcriptions.py +50 -0
- app/db/alembic/versions/20260705_000000_harden_dashboard_session_ttl.py +62 -0
- app/db/alembic/versions/20260706_000000_add_model_source_audio_per_minute.py +43 -0
- app/db/alembic/versions/20260707_000000_merge_model_sources_and_session_ttl.py +24 -0
- app/db/alembic/versions/20260707_010000_merge_automations_and_model_sources_heads.py +24 -0
- app/db/alembic/versions/20260707_020000_add_automation_job_account_scope.py +125 -0
- app/db/alembic/versions/20260709_000000_add_ttft_phase_observability.py +66 -0
- app/db/alembic/versions/20260711_000000_add_dashboard_prohibit_fast_mode.py +42 -0
- app/db/alembic/versions/20260711_010000_add_account_chatgpt_user_id.py +40 -0
- app/db/alembic/versions/20260711_020000_add_dashboard_account_concurrency_caps.py +71 -0
- app/db/alembic/versions/20260711_030000_add_limit_warmup_idle_threshold.py +51 -0
- app/db/alembic/versions/20260712_010000_add_account_usage_rollups.py +56 -0
- app/db/alembic/versions/20260712_020000_add_api_key_usage_rollups.py +53 -0
- app/db/alembic/versions/20260713_020000_add_model_registry_snapshot.py +42 -0
- app/db/alembic/versions/20260713_040000_add_account_refresh_claims.py +42 -0
- app/db/alembic/versions/20260713_040000_add_replica_guardrails.py +64 -0
- app/db/alembic/versions/20260713_070000_add_reset_credit_redeem_tables.py +58 -0
- app/db/alembic/versions/20260714_000000_add_oauth_flow_states.py +63 -0
- app/db/alembic/versions/20260715_000000_add_request_log_queue_latency.py +44 -0
- app/db/alembic/versions/20260716_000000_add_oauth_device_flow_slots.py +41 -0
- app/db/alembic/versions/20260716_010000_add_dashboard_retention_settings.py +48 -0
- app/db/alembic/versions/20260716_010000_add_reset_credit_display_settings.py +73 -0
- app/db/alembic/versions/20260717_000000_add_http_bridge_retry_circuits.py +47 -0
- app/db/alembic/versions/20260717_000000_merge_retention_and_reset_credit_display_heads.py +26 -0
- app/db/alembic/versions/20260717_000000_optimize_dashboard_hot_path_indexes.py +162 -0
- app/db/alembic/versions/20260717_000001_merge_retry_circuits_and_dashboard_indexes.py +22 -0
- app/db/alembic/versions/20260720_000000_add_request_log_conversation_id.py +57 -0
- app/db/alembic/versions/20260722_000000_backfill_request_log_useragent_families.py +40 -0
- app/db/alembic/versions/20260724_000000_add_request_usage_time_rollups.py +176 -0
- app/db/alembic/versions/20260724_000000_merge_request_log_schema_heads.py +33 -0
- app/db/alembic/versions/20260725_000000_add_http_bridge_pending_tool_calls.py +43 -0
- app/db/alembic/versions/20260726_000000_add_account_plan_downgrade_observations.py +44 -0
- app/db/alembic/versions/20260726_000000_repair_request_usage_rollups_after_merge.py +37 -0
- app/db/alembic/versions/20260727_000000_add_sticky_session_continuity_abandoned_at.py +45 -0
- app/db/alembic/versions/20260728_000000_merge_pending_tool_calls_and_rollup_repair_heads.py +31 -0
- app/db/alembic/versions/20260730_000000_add_api_key_fair_share_threshold.py +44 -0
- app/db/alembic/versions/20260730_000000_add_http_bridge_recovery_attempts.py +60 -0
- app/db/alembic/versions/20260731_000000_add_capability_lineage_markers.py +38 -0
- app/db/alembic/versions/20260803_000000_merge_http_bridge_recovery_and_capability_lineage_heads.py +30 -0
- app/db/alembic/versions/20260804_000000_add_http_bridge_operations.py +63 -0
- app/db/alembic/versions/20260804_000001_add_global_http_bridge_operation_fingerprint.py +53 -0
- app/db/alembic/versions/20260804_230000_add_request_log_connection_request_kind.py +43 -0
- app/db/alembic/versions/20260805_000000_add_http_bridge_operation_spool.py +75 -0
- app/db/alembic/versions/20260805_000001_finalize_http_bridge_operation_spool.py +65 -0
- app/db/alembic/versions/20260806_000000_add_additional_usage_alias_probe_indexes.py +84 -0
- app/db/alembic/versions/20260806_000000_add_anonymous_telemetry.py +49 -0
- app/db/alembic/versions/20260806_010000_add_conversation_presence_rollup.py +72 -0
- app/db/alembic/versions/20260806_020000_add_usage_history_bulk_covering_indexes.py +99 -0
- app/db/alembic/versions/20260806_030000_add_api_key_allowed_reasoning_efforts.py +65 -0
- app/db/alembic/versions/20260806_120000_add_http_bridge_owner_process_epoch.py +57 -0
- app/db/alembic/versions/20260807_000000_merge_http_bridge_operation_spool_and_latest_main.py +24 -0
- app/db/alembic/versions/20260808_000000_tune_usage_history_autovacuum.py +54 -0
- app/db/alembic/versions/20260810_000000_add_http_bridge_recovery_dispatch_count.py +44 -0
- app/db/alembic/versions/20260811_000000_add_hourly_rollup_cancelled_count.py +92 -0
- app/db/alembic/versions/20260812_000000_merge_recovery_dispatch_and_hourly_cancelled_heads.py +25 -0
- app/db/alembic/versions/20260812_120000_add_sticky_abandonment_scope.py +43 -0
- app/db/alembic/versions/20260813_000000_add_file_account_pins.py +40 -0
- app/db/alembic/versions/20260816_000000_add_account_pending_deletion.py +81 -0
- app/db/alembic/versions/20260816_000000_add_model_source_embeddings.py +50 -0
- app/db/alembic/versions/20260821_000000_add_retry_circuit_admission_generation.py +49 -0
- app/db/alembic/versions/20260826_000000_add_http_bridge_event_chunks.py +107 -0
- app/db/alembic/versions/20260828_000000_add_accounts_chatgpt_identity_index.py +55 -0
- app/db/alembic/versions/20260830_000000_add_quota_warmup_claim_expiry.py +83 -0
- app/db/alembic/versions/20260908_000000_add_guest_session_generation.py +37 -0
- app/db/alembic/versions/20260908_000000_add_subscription_overflow.py +140 -0
- app/db/alembic/versions/20260908_000000_replace_upstream_stream_transport_default_sentinel.py +50 -0
- app/db/alembic/versions/20260908_020000_merge_overflow_transport_heads.py +27 -0
- app/db/alembic/versions/20260909_000000_add_dashboard_roles.py +72 -0
- app/db/alembic/versions/20260909_010000_add_dashboard_users.py +183 -0
- app/db/alembic/versions/20260909_020000_reproject_compat_admin_credentials.py +105 -0
- app/db/alembic/versions/20260909_030000_add_audit_actor_columns.py +103 -0
- app/db/alembic/versions/20260909_030000_dashboard_resilience_toggle_settings.py +63 -0
- app/db/alembic/versions/20260909_040000_add_dashboard_user_invites.py +54 -0
- app/db/alembic/versions/20260909_040000_dashboard_timeout_settings.py +68 -0
- app/db/alembic/versions/20260909_050000_dashboard_routing_overload_settings.py +65 -0
- app/db/alembic/versions/20260909_060000_add_report_rollup.py +74 -0
- app/db/alembic/versions/20260909_070000_automation_run_claim_budget.py +50 -0
- app/db/alembic/versions/20260909_080000_dashboard_stream_bridge_budgets.py +62 -0
- app/db/alembic/versions/20260909_090000_dashboard_background_job_toggles.py +63 -0
- app/db/alembic/versions/20260909_100000_dashboard_codex_prewarm.py +49 -0
- app/db/alembic/versions/20260909_110000_model_context_window_overrides.py +45 -0
- app/db/alembic/versions/20260909_120000_dashboard_conversation_archive.py +48 -0
- app/db/alembic/versions/20260909_130000_add_request_logs_live_facet_indexes.py +96 -0
- app/db/alembic/versions/20260910_000000_add_totp_required_for_admin_role.py +44 -0
- app/db/alembic/versions/20260910_000000_request_logs_missing_cost_index.py +52 -0
- app/db/alembic/versions/20260910_010000_add_dashboard_auth_providers.py +134 -0
- app/db/alembic/versions/20260910_010000_dashboard_spool_retention.py +52 -0
- app/db/alembic/versions/20260910_020000_add_dashboard_role_mappings.py +74 -0
- app/db/alembic/versions/20260910_143000_request_log_affinity.py +28 -0
- app/db/alembic/versions/20260910_180000_merge_affinity_guest_heads.py +17 -0
- app/db/alembic/versions/20260910_200000_merge_affinity_identity_heads.py +17 -0
- app/db/alembic/versions/20260910_220000_merge_affinity_invite_heads.py +17 -0
- app/db/alembic/versions/20260911_000000_model_source_pins_kind_expires_index.py +43 -0
- app/db/alembic/versions/20260911_010000_merge_pin_index_and_affinity_heads.py +17 -0
- app/db/alembic/versions/20260911_020000_add_http_bridge_terminal_append_phase.py +54 -0
- app/db/alembic/versions/20260911_030000_add_local_login_policy.py +48 -0
- app/db/alembic/versions/20260911_040000_add_thread_cache_identity_mode.py +48 -0
- app/db/alembic/versions/20260911_060000_add_bridge_session_continuity_abandonment.py +40 -0
- app/db/alembic/versions/20260912_000000_merge_thread_cache_and_bridge_retirement_heads.py +22 -0
- app/db/alembic/versions/20260912_010000_drop_legacy_dashboard_credentials.py +140 -0
- app/db/alembic/versions/20260913_000000_add_oidc_provider_flow.py +134 -0
- app/db/alembic/versions/20260914_000000_add_scim_tokens.py +78 -0
- app/db/alembic/versions/20260914_000000_drop_subscription_overflow_schema.py +132 -0
- app/db/alembic/versions/20260918_000000_merge_scim_and_overflow_heads.py +17 -0
- app/db/alembic/versions/__init__.py +1 -0
- app/db/backup.py +64 -0
- app/db/migrate.py +1249 -0
- app/db/migration_lock.py +160 -0
- app/db/migration_url.py +18 -0
- app/db/models.py +3100 -0
- app/db/recover.py +244 -0
- app/db/session.py +1295 -0
- app/db/snapshot.py +46 -0
- app/db/sqlite_lock_retry.py +190 -0
- app/db/sqlite_utils.py +549 -0
- app/dependencies.py +462 -0
- app/main.py +1240 -0
- app/modules/__init__.py +0 -0
- app/modules/accounts/__init__.py +0 -0
- app/modules/accounts/api.py +426 -0
- app/modules/accounts/auth_manager.py +1439 -0
- app/modules/accounts/background_repository.py +160 -0
- app/modules/accounts/deletion.py +540 -0
- app/modules/accounts/mappers.py +654 -0
- app/modules/accounts/refresh_claims.py +352 -0
- app/modules/accounts/repository.py +1679 -0
- app/modules/accounts/schemas.py +308 -0
- app/modules/accounts/service.py +809 -0
- app/modules/accounts/usage_rollup.py +486 -0
- app/modules/accounts/usage_rollup_scheduler.py +75 -0
- app/modules/accounts/usage_time_rollup.py +1037 -0
- app/modules/accounts/usage_time_rollup_read.py +522 -0
- app/modules/api_keys/__init__.py +1 -0
- app/modules/api_keys/api.py +344 -0
- app/modules/api_keys/last_used_coalescer.py +216 -0
- app/modules/api_keys/limit_windows.py +41 -0
- app/modules/api_keys/repository.py +1200 -0
- app/modules/api_keys/reset_scheduler.py +85 -0
- app/modules/api_keys/schemas.py +135 -0
- app/modules/api_keys/service.py +2123 -0
- app/modules/audit/__init__.py +0 -0
- app/modules/audit/api.py +88 -0
- app/modules/audit/repository.py +74 -0
- app/modules/audit/schemas.py +33 -0
- app/modules/audit/service.py +71 -0
- app/modules/auth_providers/__init__.py +0 -0
- app/modules/auth_providers/api.py +127 -0
- app/modules/auth_providers/config.py +343 -0
- app/modules/auth_providers/repository.py +101 -0
- app/modules/auth_providers/schemas.py +88 -0
- app/modules/auth_providers/seed.py +105 -0
- app/modules/auth_providers/service.py +289 -0
- app/modules/automations/__init__.py +1 -0
- app/modules/automations/api.py +369 -0
- app/modules/automations/repository.py +1896 -0
- app/modules/automations/scheduler.py +107 -0
- app/modules/automations/schemas.py +158 -0
- app/modules/automations/service.py +2548 -0
- app/modules/cache_isolation_probe/__init__.py +0 -0
- app/modules/cache_isolation_probe/api.py +188 -0
- app/modules/cache_isolation_probe/prefix.py +144 -0
- app/modules/cache_isolation_probe/schemas.py +94 -0
- app/modules/cache_isolation_probe/sender.py +229 -0
- app/modules/cache_isolation_probe/service.py +525 -0
- app/modules/conversation_archive/__init__.py +1 -0
- app/modules/conversation_archive/api.py +101 -0
- app/modules/conversation_archive/schemas.py +28 -0
- app/modules/conversation_archive/service.py +165 -0
- app/modules/dashboard/__init__.py +1 -0
- app/modules/dashboard/api.py +106 -0
- app/modules/dashboard/builders.py +107 -0
- app/modules/dashboard/repository.py +304 -0
- app/modules/dashboard/schemas.py +148 -0
- app/modules/dashboard/service.py +534 -0
- app/modules/dashboard/timeframes.py +19 -0
- app/modules/dashboard/weekly_pace.py +617 -0
- app/modules/dashboard_auth/__init__.py +1 -0
- app/modules/dashboard_auth/api.py +1344 -0
- app/modules/dashboard_auth/oidc_api.py +753 -0
- app/modules/dashboard_auth/oidc_flows.py +330 -0
- app/modules/dashboard_auth/repository.py +452 -0
- app/modules/dashboard_auth/schemas.py +232 -0
- app/modules/dashboard_auth/service.py +1384 -0
- app/modules/dashboard_roles/__init__.py +0 -0
- app/modules/dashboard_roles/api.py +64 -0
- app/modules/dashboard_roles/repository.py +54 -0
- app/modules/dashboard_roles/schemas.py +29 -0
- app/modules/dashboard_roles/seed.py +80 -0
- app/modules/dashboard_roles/service.py +142 -0
- app/modules/dashboard_users/__init__.py +0 -0
- app/modules/dashboard_users/api.py +278 -0
- app/modules/dashboard_users/break_glass.py +169 -0
- app/modules/dashboard_users/credentials.py +18 -0
- app/modules/dashboard_users/identity_resolver.py +665 -0
- app/modules/dashboard_users/repository.py +725 -0
- app/modules/dashboard_users/schemas.py +117 -0
- app/modules/dashboard_users/service.py +1054 -0
- app/modules/firewall/__init__.py +1 -0
- app/modules/firewall/api.py +78 -0
- app/modules/firewall/repository.py +51 -0
- app/modules/firewall/schemas.py +28 -0
- app/modules/firewall/service.py +93 -0
- app/modules/fleet/__init__.py +1 -0
- app/modules/fleet/api.py +177 -0
- app/modules/fleet/mappers.py +56 -0
- app/modules/fleet/observability.py +372 -0
- app/modules/fleet/schemas.py +119 -0
- app/modules/health/__init__.py +0 -0
- app/modules/health/api.py +246 -0
- app/modules/health/schemas.py +27 -0
- app/modules/limit_warmup/__init__.py +1 -0
- app/modules/limit_warmup/repository.py +159 -0
- app/modules/limit_warmup/service.py +1001 -0
- app/modules/model_sources/__init__.py +1 -0
- app/modules/model_sources/api.py +99 -0
- app/modules/model_sources/catalog.py +307 -0
- app/modules/model_sources/forwarding.py +1435 -0
- app/modules/model_sources/projection.py +55 -0
- app/modules/model_sources/repository.py +201 -0
- app/modules/model_sources/schemas.py +79 -0
- app/modules/model_sources/selection.py +166 -0
- app/modules/model_sources/service.py +236 -0
- app/modules/oauth/__init__.py +0 -0
- app/modules/oauth/api.py +101 -0
- app/modules/oauth/repository.py +238 -0
- app/modules/oauth/schemas.py +49 -0
- app/modules/oauth/service.py +1196 -0
- app/modules/oauth/templates/oauth_success.html +122 -0
- app/modules/proxy/__init__.py +0 -0
- app/modules/proxy/_load_balancer/__init__.py +1 -0
- app/modules/proxy/_load_balancer/error_rate.py +116 -0
- app/modules/proxy/_load_balancer/latency_cohort.py +154 -0
- app/modules/proxy/_load_balancer/model_eligibility.py +250 -0
- app/modules/proxy/_load_balancer/opportunistic_admission.py +350 -0
- app/modules/proxy/_load_balancer/overload_backoff.py +394 -0
- app/modules/proxy/_load_balancer/sticky_selection.py +2417 -0
- app/modules/proxy/_load_balancer/throughput_cohort.py +233 -0
- app/modules/proxy/_load_balancer/ttft_cohort.py +214 -0
- app/modules/proxy/_load_balancer/tunables.py +97 -0
- app/modules/proxy/_load_balancer/types.py +96 -0
- app/modules/proxy/_load_balancer/unbound_selection.py +534 -0
- app/modules/proxy/_service/__init__.py +6 -0
- app/modules/proxy/_service/api_key_usage.py +745 -0
- app/modules/proxy/_service/clock_budget.py +38 -0
- app/modules/proxy/_service/codex_control.py +611 -0
- app/modules/proxy/_service/compact.py +2211 -0
- app/modules/proxy/_service/file_ops.py +749 -0
- app/modules/proxy/_service/http_bridge/__init__.py +3 -0
- app/modules/proxy/_service/http_bridge/accepted_replay.py +331 -0
- app/modules/proxy/_service/http_bridge/account_sessions.py +57 -0
- app/modules/proxy/_service/http_bridge/activity.py +178 -0
- app/modules/proxy/_service/http_bridge/helpers.py +3959 -0
- app/modules/proxy/_service/http_bridge/mixin.py +2435 -0
- app/modules/proxy/_service/http_bridge/owner_forwarding.py +609 -0
- app/modules/proxy/_service/http_bridge/protocol.py +167 -0
- app/modules/proxy/_service/http_bridge/proxy_failover.py +58 -0
- app/modules/proxy/_service/http_bridge/quarantine.py +443 -0
- app/modules/proxy/_service/http_bridge/request_submit.py +4582 -0
- app/modules/proxy/_service/http_bridge/retry_circuit.py +2442 -0
- app/modules/proxy/_service/http_bridge/service_stubs.py +492 -0
- app/modules/proxy/_service/http_bridge/session_registry.py +778 -0
- app/modules/proxy/_service/http_bridge/streaming.py +5328 -0
- app/modules/proxy/_service/http_bridge/upstream_events.py +4538 -0
- app/modules/proxy/_service/observability.py +410 -0
- app/modules/proxy/_service/rate_limit.py +361 -0
- app/modules/proxy/_service/realtime_live.py +655 -0
- app/modules/proxy/_service/refresh.py +154 -0
- app/modules/proxy/_service/request_log.py +585 -0
- app/modules/proxy/_service/response_create.py +930 -0
- app/modules/proxy/_service/streaming/__init__.py +3 -0
- app/modules/proxy/_service/streaming/helpers.py +1317 -0
- app/modules/proxy/_service/streaming/mixin.py +1100 -0
- app/modules/proxy/_service/streaming/protocol.py +32 -0
- app/modules/proxy/_service/streaming/retry.py +3762 -0
- app/modules/proxy/_service/support.py +2122 -0
- app/modules/proxy/_service/transcribe.py +434 -0
- app/modules/proxy/_service/warmup.py +483 -0
- app/modules/proxy/_service/websocket/__init__.py +3 -0
- app/modules/proxy/_service/websocket/helpers.py +2360 -0
- app/modules/proxy/_service/websocket/mixin.py +7285 -0
- app/modules/proxy/_service/websocket/protocol.py +74 -0
- app/modules/proxy/account_cache.py +229 -0
- app/modules/proxy/account_eligibility.py +87 -0
- app/modules/proxy/additional_model_limits.py +40 -0
- app/modules/proxy/affinity.py +830 -0
- app/modules/proxy/affinity_observation.py +101 -0
- app/modules/proxy/api.py +10254 -0
- app/modules/proxy/api_key_usage.py +113 -0
- app/modules/proxy/cap_partitioning.py +280 -0
- app/modules/proxy/capability_lineage.py +65 -0
- app/modules/proxy/capability_lineage_repository.py +82 -0
- app/modules/proxy/capability_routing.py +253 -0
- app/modules/proxy/continuity.py +112 -0
- app/modules/proxy/downstream_delivery.py +277 -0
- app/modules/proxy/durable_bridge_coordinator.py +1217 -0
- app/modules/proxy/durable_bridge_repository.py +4516 -0
- app/modules/proxy/durable_bridge_runtime.py +9 -0
- app/modules/proxy/durable_bridge_transcript_codec.py +122 -0
- app/modules/proxy/fair_share.py +131 -0
- app/modules/proxy/file_pin_repository.py +210 -0
- app/modules/proxy/helpers.py +439 -0
- app/modules/proxy/http_bridge_event_batcher.py +613 -0
- app/modules/proxy/http_bridge_forwarding.py +797 -0
- app/modules/proxy/http_continuation.py +76 -0
- app/modules/proxy/images_observability.py +62 -0
- app/modules/proxy/images_service.py +1054 -0
- app/modules/proxy/load_balancer.py +3016 -0
- app/modules/proxy/rate_limit_cache.py +46 -0
- app/modules/proxy/replay_safety.py +1063 -0
- app/modules/proxy/repo_bundle.py +31 -0
- app/modules/proxy/request_policy.py +1054 -0
- app/modules/proxy/ring_membership.py +251 -0
- app/modules/proxy/schemas.py +345 -0
- app/modules/proxy/selection_errors.py +41 -0
- app/modules/proxy/service.py +2600 -0
- app/modules/proxy/source_admission.py +125 -0
- app/modules/proxy/source_dispatch.py +999 -0
- app/modules/proxy/sticky_repository.py +831 -0
- app/modules/proxy/thread_anchors.py +491 -0
- app/modules/proxy/tool_call_dedupe.py +802 -0
- app/modules/proxy/types.py +54 -0
- app/modules/proxy/work_admission.py +121 -0
- app/modules/quota_planner/__init__.py +1 -0
- app/modules/quota_planner/api.py +276 -0
- app/modules/quota_planner/logic.py +743 -0
- app/modules/quota_planner/repository.py +722 -0
- app/modules/quota_planner/scheduler.py +212 -0
- app/modules/quota_planner/schemas.py +102 -0
- app/modules/quota_planner/warmup.py +693 -0
- app/modules/rate_limit_reset_credits/__init__.py +0 -0
- app/modules/rate_limit_reset_credits/api.py +601 -0
- app/modules/rate_limit_reset_credits/redeem_coordination.py +274 -0
- app/modules/rate_limit_reset_credits/store.py +114 -0
- app/modules/reports/__init__.py +1 -0
- app/modules/reports/api.py +134 -0
- app/modules/reports/cache.py +74 -0
- app/modules/reports/filters.py +27 -0
- app/modules/reports/repository.py +525 -0
- app/modules/reports/rollup.py +150 -0
- app/modules/reports/rollup_read.py +148 -0
- app/modules/reports/schemas.py +123 -0
- app/modules/reports/service.py +287 -0
- app/modules/reports/thread_identity.py +256 -0
- app/modules/request_logs/__init__.py +0 -0
- app/modules/request_logs/api.py +224 -0
- app/modules/request_logs/cost_backfill.py +154 -0
- app/modules/request_logs/mappers.py +97 -0
- app/modules/request_logs/repository.py +1765 -0
- app/modules/request_logs/schemas.py +145 -0
- app/modules/request_logs/service.py +348 -0
- app/modules/role_mappings/__init__.py +0 -0
- app/modules/role_mappings/api.py +206 -0
- app/modules/role_mappings/matching.py +55 -0
- app/modules/role_mappings/repository.py +106 -0
- app/modules/role_mappings/schemas.py +67 -0
- app/modules/role_mappings/service.py +334 -0
- app/modules/runtime/__init__.py +1 -0
- app/modules/runtime/api.py +18 -0
- app/modules/runtime/schemas.py +14 -0
- app/modules/runtime/service.py +218 -0
- app/modules/scim/__init__.py +1 -0
- app/modules/scim/api.py +194 -0
- app/modules/scim/dependencies.py +112 -0
- app/modules/scim/errors.py +77 -0
- app/modules/scim/management_api.py +156 -0
- app/modules/scim/repository.py +212 -0
- app/modules/scim/schemas.py +145 -0
- app/modules/scim/service.py +786 -0
- app/modules/scim/tokens.py +43 -0
- app/modules/settings/__init__.py +1 -0
- app/modules/settings/api.py +1677 -0
- app/modules/settings/repository.py +606 -0
- app/modules/settings/schemas.py +496 -0
- app/modules/settings/service.py +769 -0
- app/modules/shared/__init__.py +0 -0
- app/modules/shared/schemas.py +22 -0
- app/modules/sticky_sessions/__init__.py +1 -0
- app/modules/sticky_sessions/api.py +118 -0
- app/modules/sticky_sessions/cleanup_scheduler.py +487 -0
- app/modules/sticky_sessions/schemas.py +78 -0
- app/modules/sticky_sessions/service.py +196 -0
- app/modules/telemetry/__init__.py +5 -0
- app/modules/telemetry/api.py +115 -0
- app/modules/telemetry/clients.py +62 -0
- app/modules/telemetry/consent.py +115 -0
- app/modules/telemetry/scheduler.py +85 -0
- app/modules/telemetry/schemas.py +163 -0
- app/modules/telemetry/sender.py +221 -0
- app/modules/telemetry/snapshot.py +490 -0
- app/modules/usage/__init__.py +0 -0
- app/modules/usage/additional_quota_keys.py +379 -0
- app/modules/usage/api.py +45 -0
- app/modules/usage/background_repository.py +147 -0
- app/modules/usage/builders.py +559 -0
- app/modules/usage/depletion_service.py +277 -0
- app/modules/usage/live_ingest.py +356 -0
- app/modules/usage/mappers.py +29 -0
- app/modules/usage/plan_downgrade_observations.py +488 -0
- app/modules/usage/repository.py +1890 -0
- app/modules/usage/schemas.py +68 -0
- app/modules/usage/service.py +102 -0
- app/modules/usage/updater.py +1488 -0
- app/static/assets/Trans-Cj-J9mVA.js +1 -0
- app/static/assets/access-page-DHzZiXZi.js +1 -0
- app/static/assets/account-cost-donut-DGhfA4U8.js +1 -0
- app/static/assets/account-identifiers-CxbdLshZ.js +1 -0
- app/static/assets/account-status-DIsU5sUS.js +1 -0
- app/static/assets/account-trend-chart-BptlfwwQ.js +1 -0
- app/static/assets/accounts-page-Bbai8ift.js +3 -0
- app/static/assets/api-Dwa7XxPe.js +1 -0
- app/static/assets/api-client-dugX1Pti.js +55 -0
- app/static/assets/api-key-create-dialog-DnRm0oYN.js +1 -0
- app/static/assets/api-key-created-dialog-B6jkDG9f.js +1 -0
- app/static/assets/api-key-edit-dialog-Ck3Bnc3V.js +1 -0
- app/static/assets/api-trend-chart-BchCw7Hj.js +1 -0
- app/static/assets/apis-page-BvXz9c6i.js +2 -0
- app/static/assets/arrow-up-Dqh_KzJo.js +1 -0
- app/static/assets/arrow-up-down-LBRgVDGY.js +1 -0
- app/static/assets/automations-page-DQgNAJOR.js +3 -0
- app/static/assets/badge-DqMb9es0.js +1 -0
- app/static/assets/blur-email-HEvKagG2.js +1 -0
- app/static/assets/button-BjVmmiGW.js +1 -0
- app/static/assets/checkbox-apsl9vzk.js +1 -0
- app/static/assets/chevron-left-1d_pHVa5.js +1 -0
- app/static/assets/circle-alert-BS0xd_o1.js +1 -0
- app/static/assets/clsx-DB0hHKMi.js +1 -0
- app/static/assets/colors-BBy8GEtr.js +1 -0
- app/static/assets/copy-button-D7nAvOKR.js +1 -0
- app/static/assets/cost-per-day-chart-YdqlkZVM.js +1 -0
- app/static/assets/createLucideIcon-DBLzLNh2.js +1 -0
- app/static/assets/dashboard-page-CSdhBy0f.js +2 -0
- app/static/assets/distribution-metric-format-yZTy1ZEu.js +1 -0
- app/static/assets/donut-chart-Bxc69cra.js +1 -0
- app/static/assets/download-kQteIcsB.js +1 -0
- app/static/assets/ellipsis-B_5OgIre.js +1 -0
- app/static/assets/empty-state-JcA7tc8m.js +1 -0
- app/static/assets/es2015-bMH8YKZT.js +41 -0
- app/static/assets/es6-B264ni6W.js +69 -0
- app/static/assets/external-link-UVAOgMQ0.js +1 -0
- app/static/assets/i18n-BWqZ9Ua2.js +1 -0
- app/static/assets/inbox-B_RM2s1J.js +1 -0
- app/static/assets/index-DIpd264Z.js +23 -0
- app/static/assets/index-DMeLKRJx.css +2 -0
- app/static/assets/inherit-badge-QhTYVaU6.js +1 -0
- app/static/assets/invite-dialog-0CYh0GiQ.js +1 -0
- app/static/assets/iso-Bpe5Ihbb.js +1 -0
- app/static/assets/label-CGaLmLNf.js +1 -0
- app/static/assets/lazy-recharts-Pv8p9ArM.js +2 -0
- app/static/assets/loading-overlay-CzKQfHLI.js +1 -0
- app/static/assets/mini-quota-bar-cGT_t8Ae.js +1 -0
- app/static/assets/model-distribution-donut-BKjAfT48.js +1 -0
- app/static/assets/model-source-multi-select-D2d06Vc7.js +1 -0
- app/static/assets/multi-select-filter-KIA5qU6Z.js +1 -0
- app/static/assets/oauth-dialog-XeaGGDjS.js +1 -0
- app/static/assets/pagination-controls-D1Ypd0RU.js +1 -0
- app/static/assets/payload-CIYnGZz9.js +1 -0
- app/static/assets/queue-wait-chart-VoV7Yg9L.js +1 -0
- app/static/assets/report-chart-card-BXDliwcc.js +1 -0
- app/static/assets/reports-page-CGCJeMHG.js +3 -0
- app/static/assets/reset-credit-confirm-dialog-QOifTnVB.js +1 -0
- app/static/assets/rolldown-runtime-hePW80VL.js +1 -0
- app/static/assets/search-HfhqwERt.js +1 -0
- app/static/assets/select-Cl33bz-g.js +1 -0
- app/static/assets/settings-page-mZj01HYl.js +2 -0
- app/static/assets/shim-CZwyM4Q7.js +1 -0
- app/static/assets/skeleton-C_EUc4Rp.js +1 -0
- app/static/assets/sparkline-chart-DdNNsddM.js +1 -0
- app/static/assets/status-badge-CHHA5X6c.js +1 -0
- app/static/assets/switch-CzxgokcM.js +1 -0
- app/static/assets/table-DXaNSQ0G.js +1 -0
- app/static/assets/time-to-first-token-chart--68L0u_v.js +1 -0
- app/static/assets/tokens-per-day-chart-Cs82BFuE.js +1 -0
- app/static/assets/tokens-per-second-chart-AD2dfOn6.js +1 -0
- app/static/assets/totp-settings-CRHtxQ7j.js +1 -0
- app/static/assets/trash-D3KyWr1h.js +1 -0
- app/static/assets/use-account-quota-display-BPmrTBTC.js +1 -0
- app/static/assets/use-accounts-PWlVGw3z.js +1 -0
- app/static/assets/use-auth-CuDlW3iC.js +1 -0
- app/static/assets/use-chart-colors-DIu6tClI.js +1 -0
- app/static/assets/use-model-sources-Bul24JBu.js +1 -0
- app/static/assets/use-models-DUCPUuZg.js +1 -0
- app/static/assets/use-reduced-motion-hAm5r6_d.js +1 -0
- app/static/assets/useTranslation--56nHhlH.js +1 -0
- app/static/assets/useragent-distribution-donut-Dup6sf_n.js +1 -0
- app/static/assets/users-uIC3TIlE.js +1 -0
- app/static/assets/vendor-query-_bHZuxEx.js +1 -0
- app/static/assets/vendor-react-BVL0YZPW.js +9 -0
- app/static/favicon.svg +10 -0
- app/static/fonts/GeistSans-Variable.woff2 +0 -0
- app/static/fonts/jetbrains-mono-latin-ext.woff2 +0 -0
- app/static/fonts/jetbrains-mono-latin.woff2 +0 -0
- app/static/index.html +29 -0
- codexlb_up-1.25.0b10.dist-info/METADATA +124 -0
- codexlb_up-1.25.0b10.dist-info/RECORD +827 -0
- codexlb_up-1.25.0b10.dist-info/WHEEL +4 -0
- codexlb_up-1.25.0b10.dist-info/entry_points.txt +3 -0
- codexlb_up-1.25.0b10.dist-info/licenses/LICENSE +21 -0
- config/additional_quota_registry.json +11 -0
|
@@ -0,0 +1,2442 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
import asyncio
|
|
4
|
+
import logging
|
|
5
|
+
from dataclasses import dataclass
|
|
6
|
+
from types import SimpleNamespace
|
|
7
|
+
from typing import Any
|
|
8
|
+
|
|
9
|
+
import anyio
|
|
10
|
+
|
|
11
|
+
from app.core.clock import clock_for, scheduler_for
|
|
12
|
+
from app.core.errors import HTTP_BRIDGE_EVENTLESS_TIMEOUT_CODE
|
|
13
|
+
from app.core.metrics.prometheus import PROMETHEUS_AVAILABLE, http_bridge_retry_circuit_total
|
|
14
|
+
from app.modules.proxy._service.http_bridge.quarantine import (
|
|
15
|
+
_HTTP_BRIDGE_QUARANTINE_POISONED_ANCHOR_REASON,
|
|
16
|
+
_http_bridge_quarantine_clear_fence,
|
|
17
|
+
_http_bridge_quarantine_generation,
|
|
18
|
+
_http_bridge_quarantine_registry,
|
|
19
|
+
_http_bridge_session_key_poison_quarantined,
|
|
20
|
+
_quarantine_http_bridge_session,
|
|
21
|
+
_revoke_http_bridge_poison_quarantine,
|
|
22
|
+
)
|
|
23
|
+
from app.modules.proxy._service.observability import _hash_identifier
|
|
24
|
+
from app.modules.proxy._service.support import (
|
|
25
|
+
_HTTPBridgeResponseCreateAttempt,
|
|
26
|
+
_HTTPBridgeRetryCircuitAttemptSelection,
|
|
27
|
+
_HTTPBridgeSession,
|
|
28
|
+
_HTTPBridgeSessionKey,
|
|
29
|
+
)
|
|
30
|
+
from app.modules.proxy.durable_bridge_repository import DURABLE_BRIDGE_RETRY_CIRCUIT_STATE_TTL_SECONDS
|
|
31
|
+
|
|
32
|
+
logger = logging.getLogger(__name__)
|
|
33
|
+
|
|
34
|
+
# Consecutive failures that open the circuit. The anchor-poison threshold is
|
|
35
|
+
# this same value: once the circuit opens it refuses the key for 60-600s per
|
|
36
|
+
# strike, so a higher poison threshold could never be reached at any useful
|
|
37
|
+
# rate (#1830/#1852), and the quarantine armed at the opening is process-local,
|
|
38
|
+
# so clearing the durable anchor no later than the opening is what keeps the
|
|
39
|
+
# decision replica-safe.
|
|
40
|
+
_HTTP_BRIDGE_RETRY_CIRCUIT_FAILURE_THRESHOLD = 2
|
|
41
|
+
_HTTP_BRIDGE_RETRY_CIRCUIT_BASE_BACKOFF_SECONDS = 60.0
|
|
42
|
+
_HTTP_BRIDGE_RETRY_CIRCUIT_MAX_BACKOFF_SECONDS = 600.0
|
|
43
|
+
_HTTP_BRIDGE_RETRY_CIRCUIT_CLEAN_CLOSE_MAX_BACKOFF_SECONDS = 30.0
|
|
44
|
+
_HTTP_BRIDGE_RETRY_CIRCUIT_HALF_OPEN_LEASE_SECONDS = 600.0
|
|
45
|
+
_HTTP_BRIDGE_RETRY_CIRCUIT_CLAIM_TIMEOUT_SECONDS = 5.0
|
|
46
|
+
_HTTP_BRIDGE_RETRY_CIRCUIT_PLANNING_MISS_CAP = 4096
|
|
47
|
+
_HTTP_BRIDGE_RETRY_CIRCUIT_FAILURE_DETAILS = frozenset(
|
|
48
|
+
{
|
|
49
|
+
"stream_incomplete",
|
|
50
|
+
"clean_close",
|
|
51
|
+
"stream_idle_timeout",
|
|
52
|
+
# Pre-response-start bridge silence. Deliberately *not* aliased onto
|
|
53
|
+
# stream_idle_timeout: the whole point is that a circuit opened before
|
|
54
|
+
# any response existed must be distinguishable from one opened by a
|
|
55
|
+
# started stream going quiet.
|
|
56
|
+
HTTP_BRIDGE_EVENTLESS_TIMEOUT_CODE,
|
|
57
|
+
}
|
|
58
|
+
)
|
|
59
|
+
_HTTP_BRIDGE_RETRY_CIRCUIT_DETAIL_ALIASES = {
|
|
60
|
+
# These diagnostics describe the same ambiguous idle/incomplete
|
|
61
|
+
# transport class. Keep the durable contract to the documented
|
|
62
|
+
# failure classes while retaining the more specific event in logs.
|
|
63
|
+
"upstream_keepalive_timeout": "stream_idle_timeout",
|
|
64
|
+
"missing_response_created_timeout": "stream_idle_timeout",
|
|
65
|
+
"response_create_gate_timeout_stuck_pending": "stream_idle_timeout",
|
|
66
|
+
}
|
|
67
|
+
# Sentinel: the consult could not capture the durable anchor; the
|
|
68
|
+
# abandonment then clears continuity unfenced (degraded, logged).
|
|
69
|
+
_POISON_ANCHOR_CAPTURE_UNAVAILABLE: object = object()
|
|
70
|
+
_HTTP_BRIDGE_ANCHOR_POISON_DETAILS = {
|
|
71
|
+
"stream_idle_timeout": "repeated_zero_event_idle_timeout",
|
|
72
|
+
"stream_incomplete": "repeated_zero_event_stream_incomplete",
|
|
73
|
+
# The pre-response eventless timeout keeps its distinguishable durable
|
|
74
|
+
# detail (deliberately not aliased onto stream_idle_timeout), but a
|
|
75
|
+
# repeatedly eventless anchored request is the original silent-wedge
|
|
76
|
+
# signature, so at threshold it authorizes the same fenced abandonment.
|
|
77
|
+
HTTP_BRIDGE_EVENTLESS_TIMEOUT_CODE: "repeated_zero_event_idle_timeout",
|
|
78
|
+
}
|
|
79
|
+
# Written over a surviving at-threshold row when a completion registered a
|
|
80
|
+
# fresh anchor but its settlement failed. Deliberately outside the poison
|
|
81
|
+
# detail map: a replica loading the row keeps the cooldown but neither arms
|
|
82
|
+
# quarantine against the fresh anchor nor authorizes an abandonment for
|
|
83
|
+
# failures recorded against the superseded one.
|
|
84
|
+
_HTTP_BRIDGE_RETRY_CIRCUIT_ANCHOR_SUPERSEDED_DETAIL = "anchor_superseded"
|
|
85
|
+
# Written onto the zeroed row when an ABANDONMENT settles the circuit: the
|
|
86
|
+
# continuity columns are already empty, so without a durable marker a
|
|
87
|
+
# restarted worker or another replica would dispatch a delta-only follow-up
|
|
88
|
+
# as a brand-new conversation and silently drop its context. A completion's
|
|
89
|
+
# settle writes None instead, erasing the tombstone on real recovery.
|
|
90
|
+
_HTTP_BRIDGE_RETRY_CIRCUIT_ANCHOR_ABANDONED_DETAIL = "anchor_abandoned"
|
|
91
|
+
|
|
92
|
+
|
|
93
|
+
def _http_bridge_anchor_poison_detail(detail: str | None) -> str | None:
|
|
94
|
+
"""Map an eventless retry-circuit failure class to its anchor-poison detail.
|
|
95
|
+
|
|
96
|
+
Consecutive eventless failures on one bridge key are same-anchor failures:
|
|
97
|
+
the durable anchor only advances on a completed response, which resets the
|
|
98
|
+
circuit. Both ambiguous transport classes therefore count toward anchor
|
|
99
|
+
poison (issue #1830); ``clean_close`` never does.
|
|
100
|
+
"""
|
|
101
|
+
if detail is None:
|
|
102
|
+
return None
|
|
103
|
+
aliased = _HTTP_BRIDGE_RETRY_CIRCUIT_DETAIL_ALIASES.get(detail, detail)
|
|
104
|
+
return _HTTP_BRIDGE_ANCHOR_POISON_DETAILS.get(aliased)
|
|
105
|
+
|
|
106
|
+
|
|
107
|
+
def _http_bridge_poison_quarantine_minimum_seconds(cooldown_remaining: float) -> float:
|
|
108
|
+
"""Keep a poison quarantine alive across the cooldown and its probe window.
|
|
109
|
+
|
|
110
|
+
The probe that has to be planned unanchored is only admitted once the
|
|
111
|
+
cooldown expires, and it may then be admitted anywhere inside the
|
|
112
|
+
half-open lease that follows. Quarantine's default TTL equals the maximum
|
|
113
|
+
cooldown, so at that cooldown both would lapse in the same instant and the
|
|
114
|
+
probe would be planned with the anchor the circuit opened on. This is the
|
|
115
|
+
same span the durable retry-circuit row already reserves for itself.
|
|
116
|
+
"""
|
|
117
|
+
return max(0.0, cooldown_remaining) + _HTTP_BRIDGE_RETRY_CIRCUIT_HALF_OPEN_LEASE_SECONDS
|
|
118
|
+
|
|
119
|
+
|
|
120
|
+
def _http_bridge_retry_circuit_suppression_message(block_reason: str, retry_after_seconds: int) -> str:
|
|
121
|
+
"""Describe the timer that is actually refusing a suppressed submission.
|
|
122
|
+
|
|
123
|
+
Naming the cooldown while the half-open lease is what refuses the request
|
|
124
|
+
tells the client to come back in about a second when it is barred for the
|
|
125
|
+
rest of the lease, which turns one wedged key into a retry storm.
|
|
126
|
+
"""
|
|
127
|
+
if block_reason == "hard_key_half_open":
|
|
128
|
+
return (
|
|
129
|
+
"HTTP responses session bridge is probing recovery for this conversation; "
|
|
130
|
+
f"retry after {retry_after_seconds}s."
|
|
131
|
+
)
|
|
132
|
+
return (
|
|
133
|
+
"HTTP responses session bridge is recovering from repeated upstream failures; "
|
|
134
|
+
f"retry after {retry_after_seconds}s."
|
|
135
|
+
)
|
|
136
|
+
|
|
137
|
+
|
|
138
|
+
@dataclass(slots=True)
|
|
139
|
+
class _HTTPBridgeRetryCircuitState:
|
|
140
|
+
consecutive_failures: int = 0
|
|
141
|
+
cooldown_until: float = 0.0
|
|
142
|
+
last_detail: str | None = None
|
|
143
|
+
last_touched_monotonic: float = 0.0
|
|
144
|
+
persisted_updated_at_epoch: float = 0.0
|
|
145
|
+
# Admission generation observed with the epoch above: a replica's replay
|
|
146
|
+
# claim bumps only this column, so the settle fence must carry it or a
|
|
147
|
+
# reset can clear the circuit beneath a claim it never observed.
|
|
148
|
+
persisted_admission_generation: int = 0
|
|
149
|
+
last_failure_monotonic: float = 0.0
|
|
150
|
+
last_durable_load_monotonic: float = 0.0
|
|
151
|
+
half_open_until: float = 0.0
|
|
152
|
+
# One poisoned anchor is abandoned once. Capping the poison threshold at
|
|
153
|
+
# the circuit threshold makes every later strike in the same episode meet
|
|
154
|
+
# it too, so without this marker each one re-issues the durable clear. A
|
|
155
|
+
# failed clear leaves it unset, which is what lets the next strike retry.
|
|
156
|
+
poison_anchor_cleared: bool = False
|
|
157
|
+
# The poison class this episode still owes an abandonment for. A later
|
|
158
|
+
# non-poison strike (an unanchored probe closing clean) overwrites
|
|
159
|
+
# ``last_detail`` durably, and without this record the owed clear would
|
|
160
|
+
# never be retried and other replicas would stop arming against it.
|
|
161
|
+
owed_poison_detail: str | None = None
|
|
162
|
+
|
|
163
|
+
|
|
164
|
+
@dataclass(slots=True)
|
|
165
|
+
class _HTTPBridgeRetryCircuitKeyProbe:
|
|
166
|
+
"""Key-scoped stand-in for the planning-time first-touch load.
|
|
167
|
+
|
|
168
|
+
Planning runs before any bridge session exists, but the load path and the
|
|
169
|
+
quarantine arm only need the key plus logging context; the probe carries
|
|
170
|
+
exactly that so an at-threshold poison row recorded by another replica can
|
|
171
|
+
arm this worker's quarantine before the anchor is planned into a payload.
|
|
172
|
+
"""
|
|
173
|
+
|
|
174
|
+
key: _HTTPBridgeSessionKey
|
|
175
|
+
request_model: str | None = None
|
|
176
|
+
account: Any = None
|
|
177
|
+
quarantined: bool = False
|
|
178
|
+
|
|
179
|
+
def __post_init__(self) -> None:
|
|
180
|
+
if self.account is None:
|
|
181
|
+
self.account = SimpleNamespace(id=None)
|
|
182
|
+
|
|
183
|
+
|
|
184
|
+
def _initialize_http_bridge_retry_circuit(service: Any, reset_transient_cache: Any = None) -> None:
|
|
185
|
+
if reset_transient_cache is not None:
|
|
186
|
+
reset_transient_cache()
|
|
187
|
+
service._http_bridge_retry_circuits = {}
|
|
188
|
+
service._http_bridge_retry_circuit_loaded_keys = set()
|
|
189
|
+
service._http_bridge_retry_circuit_persisted_keys = set()
|
|
190
|
+
service._http_bridge_retry_circuit_lock = anyio.Lock()
|
|
191
|
+
service._http_bridge_retry_circuit_key_locks = {}
|
|
192
|
+
# A settlement pops the state object, taking its stale-load watermark
|
|
193
|
+
# with it; this per-key watermark survives the pop so a lookup that
|
|
194
|
+
# began before the settlement cannot adopt the pre-settlement row into
|
|
195
|
+
# a fresh state and resurrect the settled cooldown.
|
|
196
|
+
service._http_bridge_retry_circuit_reconcile_watermarks = {}
|
|
197
|
+
# Timestamped negative cache for confirmed durable misses: healthy hard
|
|
198
|
+
# keys have no circuit row, and without this every planning pass pays a
|
|
199
|
+
# durable read on top of the submit-time load. Entries live for the
|
|
200
|
+
# planning window only; a row another replica creates after a cached
|
|
201
|
+
# miss is still enforced at submission while its cooldown runs.
|
|
202
|
+
service._http_bridge_retry_circuit_planning_misses = {}
|
|
203
|
+
|
|
204
|
+
|
|
205
|
+
def _record_http_bridge_retry_circuit_duplicate_suppressed(
|
|
206
|
+
session: _HTTPBridgeSession,
|
|
207
|
+
*,
|
|
208
|
+
attempt: _HTTPBridgeResponseCreateAttempt,
|
|
209
|
+
consecutive_failures: int,
|
|
210
|
+
detail: str,
|
|
211
|
+
) -> None:
|
|
212
|
+
if PROMETHEUS_AVAILABLE and http_bridge_retry_circuit_total is not None:
|
|
213
|
+
http_bridge_retry_circuit_total.labels(outcome="duplicate_suppressed").inc()
|
|
214
|
+
logger.info(
|
|
215
|
+
"http_bridge_retry_circuit event=duplicate_suppressed bridge_kind=%s bridge_key=%s "
|
|
216
|
+
"failures=%s detail=%s attempt=%s",
|
|
217
|
+
session.key.affinity_kind,
|
|
218
|
+
_hash_identifier(session.key.affinity_key),
|
|
219
|
+
consecutive_failures,
|
|
220
|
+
detail,
|
|
221
|
+
attempt.ordinal,
|
|
222
|
+
)
|
|
223
|
+
|
|
224
|
+
|
|
225
|
+
class _HTTPBridgeRetryCircuitMixin:
|
|
226
|
+
async def _http_bridge_retry_circuit_generation(
|
|
227
|
+
self: Any,
|
|
228
|
+
session: _HTTPBridgeSession,
|
|
229
|
+
) -> tuple[bool, tuple[int, float, int, float, int, float, float] | None]:
|
|
230
|
+
return await self._http_bridge_retry_circuit_generation_for_key(session.key)
|
|
231
|
+
|
|
232
|
+
async def _http_bridge_retry_circuit_generation_for_key(
|
|
233
|
+
self: Any,
|
|
234
|
+
key: _HTTPBridgeSessionKey,
|
|
235
|
+
) -> tuple[bool, tuple[int, float, int, float, int, float, float] | None]:
|
|
236
|
+
try:
|
|
237
|
+
persisted = await self._durable_bridge.lookup_retry_circuit(
|
|
238
|
+
session_key_kind=key.affinity_kind,
|
|
239
|
+
session_key_value=key.affinity_key,
|
|
240
|
+
api_key_id=key.api_key_id,
|
|
241
|
+
)
|
|
242
|
+
except Exception:
|
|
243
|
+
logger.warning(
|
|
244
|
+
"Failed to inspect HTTP bridge retry circuit generation bridge_kind=%s bridge_key=%s",
|
|
245
|
+
key.affinity_kind,
|
|
246
|
+
_hash_identifier(key.affinity_key),
|
|
247
|
+
exc_info=True,
|
|
248
|
+
)
|
|
249
|
+
return False, None
|
|
250
|
+
async with self._http_bridge_retry_circuit_lock:
|
|
251
|
+
state = self._http_bridge_retry_circuits.get(key)
|
|
252
|
+
if state is None and persisted is None:
|
|
253
|
+
return True, None
|
|
254
|
+
persisted_updated_at_epoch = max(
|
|
255
|
+
state.persisted_updated_at_epoch if state is not None else 0.0,
|
|
256
|
+
persisted.updated_at_epoch if persisted is not None else 0.0,
|
|
257
|
+
)
|
|
258
|
+
persisted_consecutive_failures = persisted.consecutive_failures if persisted is not None else 0
|
|
259
|
+
durable_cooldown_until_epoch = persisted.cooldown_until_epoch if persisted is not None else 0.0
|
|
260
|
+
local_consecutive_failures = state.consecutive_failures if state is not None else 0
|
|
261
|
+
last_failure_monotonic = state.last_failure_monotonic if state is not None else 0.0
|
|
262
|
+
local_cooldown_until = state.cooldown_until if state is not None else 0.0
|
|
263
|
+
return True, (
|
|
264
|
+
getattr(persisted, "admission_generation", 0) if persisted is not None else 0,
|
|
265
|
+
persisted_updated_at_epoch,
|
|
266
|
+
persisted_consecutive_failures,
|
|
267
|
+
durable_cooldown_until_epoch,
|
|
268
|
+
local_consecutive_failures,
|
|
269
|
+
last_failure_monotonic,
|
|
270
|
+
local_cooldown_until,
|
|
271
|
+
)
|
|
272
|
+
|
|
273
|
+
async def _http_bridge_retry_circuit_generation_is_not_newer(
|
|
274
|
+
self: Any,
|
|
275
|
+
*,
|
|
276
|
+
key: _HTTPBridgeSessionKey,
|
|
277
|
+
captured: bool,
|
|
278
|
+
generation: tuple[int, float, int, float, int, float, float] | None,
|
|
279
|
+
) -> bool:
|
|
280
|
+
if not captured:
|
|
281
|
+
return False
|
|
282
|
+
load_succeeded, current_generation = await self._http_bridge_retry_circuit_generation_for_key(key)
|
|
283
|
+
if not load_succeeded:
|
|
284
|
+
return False
|
|
285
|
+
if generation is None:
|
|
286
|
+
return current_generation is None
|
|
287
|
+
if current_generation is None:
|
|
288
|
+
return True
|
|
289
|
+
return all(
|
|
290
|
+
current <= captured_value for current, captured_value in zip(current_generation, generation, strict=True)
|
|
291
|
+
)
|
|
292
|
+
|
|
293
|
+
async def _claim_http_bridge_retry_circuit_generation(
|
|
294
|
+
self: Any,
|
|
295
|
+
*,
|
|
296
|
+
key: _HTTPBridgeSessionKey,
|
|
297
|
+
captured: bool,
|
|
298
|
+
generation: tuple[int, float, int, float, int, float, float] | None,
|
|
299
|
+
) -> bool | None:
|
|
300
|
+
"""Atomically linearize replay admission against the captured circuit.
|
|
301
|
+
|
|
302
|
+
``True`` means claimed; ``False`` means a confirmed loss — the durable
|
|
303
|
+
CAS answered and matched nothing, or local state already advanced past
|
|
304
|
+
the capture; ``None`` means the claim could not be decided (timeout,
|
|
305
|
+
durable error, or the capability being absent), which is
|
|
306
|
+
infrastructure trouble rather than evidence that a remote probe holds
|
|
307
|
+
the lease.
|
|
308
|
+
"""
|
|
309
|
+
if not captured:
|
|
310
|
+
return False
|
|
311
|
+
expected_admission_generation = generation[0] if generation is not None else 0
|
|
312
|
+
expected_persisted_updated_at = generation[1] if generation is not None else 0.0
|
|
313
|
+
expected_persisted_failures = generation[2] if generation is not None else 0
|
|
314
|
+
expected_persisted_cooldown = generation[3] if generation is not None else 0.0
|
|
315
|
+
expected_local_failures = generation[4] if generation is not None else 0
|
|
316
|
+
expected_last_failure = generation[5] if generation is not None else 0.0
|
|
317
|
+
expected_local_cooldown = generation[6] if generation is not None else 0.0
|
|
318
|
+
claim_generation = getattr(self._durable_bridge, "claim_retry_circuit_generation", None)
|
|
319
|
+
if not callable(claim_generation):
|
|
320
|
+
return None
|
|
321
|
+
|
|
322
|
+
# Local failure recording for this key serializes on the key lock,
|
|
323
|
+
# so holding it across the durable CAS keeps the claim linearized
|
|
324
|
+
# against local strikes without parking every unrelated hard key
|
|
325
|
+
# behind the global registry lock for up to the claim timeout. The
|
|
326
|
+
# global lock is held only for the short state checks.
|
|
327
|
+
key_lock = await self._acquire_http_bridge_retry_circuit_key_lock(key)
|
|
328
|
+
try:
|
|
329
|
+
async with self._http_bridge_retry_circuit_lock:
|
|
330
|
+
state = self._http_bridge_retry_circuits.get(key)
|
|
331
|
+
if state is not None and (
|
|
332
|
+
state.consecutive_failures > expected_local_failures
|
|
333
|
+
or state.last_failure_monotonic > expected_last_failure
|
|
334
|
+
or state.cooldown_until > expected_local_cooldown
|
|
335
|
+
):
|
|
336
|
+
return False
|
|
337
|
+
try:
|
|
338
|
+
claimed = await scheduler_for(self).wait_for(
|
|
339
|
+
claim_generation(
|
|
340
|
+
session_key_kind=key.affinity_kind,
|
|
341
|
+
session_key_value=key.affinity_key,
|
|
342
|
+
api_key_id=key.api_key_id,
|
|
343
|
+
expected_updated_at_epoch=(
|
|
344
|
+
expected_persisted_updated_at if expected_persisted_updated_at > 0 else None
|
|
345
|
+
),
|
|
346
|
+
expected_admission_generation=expected_admission_generation,
|
|
347
|
+
expected_consecutive_failures=expected_persisted_failures,
|
|
348
|
+
expected_cooldown_until_epoch=expected_persisted_cooldown,
|
|
349
|
+
),
|
|
350
|
+
timeout=_HTTP_BRIDGE_RETRY_CIRCUIT_CLAIM_TIMEOUT_SECONDS,
|
|
351
|
+
)
|
|
352
|
+
except Exception:
|
|
353
|
+
logger.warning(
|
|
354
|
+
"Failed to claim HTTP bridge retry circuit generation bridge_kind=%s bridge_key=%s",
|
|
355
|
+
key.affinity_kind,
|
|
356
|
+
_hash_identifier(key.affinity_key),
|
|
357
|
+
exc_info=True,
|
|
358
|
+
)
|
|
359
|
+
return None
|
|
360
|
+
if claimed is None:
|
|
361
|
+
return False
|
|
362
|
+
async with self._http_bridge_retry_circuit_lock:
|
|
363
|
+
self._http_bridge_retry_circuit_loaded_keys.add(key)
|
|
364
|
+
self._http_bridge_retry_circuit_persisted_keys.add(key)
|
|
365
|
+
return True
|
|
366
|
+
finally:
|
|
367
|
+
key_lock.release()
|
|
368
|
+
|
|
369
|
+
async def _ensure_http_bridge_retry_circuit_loaded_for_key(
|
|
370
|
+
self: Any,
|
|
371
|
+
key: _HTTPBridgeSessionKey,
|
|
372
|
+
*,
|
|
373
|
+
request_model: str | None,
|
|
374
|
+
) -> None:
|
|
375
|
+
"""Planning-time durable load so a poison row arms quarantine pre-planning.
|
|
376
|
+
|
|
377
|
+
The submit-time load runs after the payload is already planned and
|
|
378
|
+
serialized, so on an expired at-threshold poison key the probe would
|
|
379
|
+
carry the poisoned anchor. The cached view is honored only while it
|
|
380
|
+
is younger than the minimum cooldown: a circuit another replica
|
|
381
|
+
opens after this worker's last load is then either still cooling
|
|
382
|
+
(the submit-time gate suppresses the request before dispatch) or
|
|
383
|
+
refreshed here before its expired cooldown can admit a probe, so a
|
|
384
|
+
cached below-threshold or reset row cannot hide a remote opening
|
|
385
|
+
from the anchor decision.
|
|
386
|
+
"""
|
|
387
|
+
if key.strength != "hard":
|
|
388
|
+
return
|
|
389
|
+
async with self._http_bridge_retry_circuit_lock:
|
|
390
|
+
if key in self._http_bridge_retry_circuit_loaded_keys:
|
|
391
|
+
state = self._http_bridge_retry_circuits.get(key)
|
|
392
|
+
if (
|
|
393
|
+
state is not None
|
|
394
|
+
and clock_for(self).monotonic() - state.last_durable_load_monotonic
|
|
395
|
+
<= _HTTP_BRIDGE_RETRY_CIRCUIT_BASE_BACKOFF_SECONDS
|
|
396
|
+
):
|
|
397
|
+
return
|
|
398
|
+
missed_at = self._http_bridge_retry_circuit_planning_misses.get(key)
|
|
399
|
+
if (
|
|
400
|
+
missed_at is not None
|
|
401
|
+
and clock_for(self).monotonic() - missed_at <= _HTTP_BRIDGE_RETRY_CIRCUIT_BASE_BACKOFF_SECONDS
|
|
402
|
+
):
|
|
403
|
+
# A confirmed durable miss within the planning window: a row
|
|
404
|
+
# another replica created since is still enforced by the
|
|
405
|
+
# submit-time load while its cooldown runs, so planning does
|
|
406
|
+
# not need a second round trip per healthy request.
|
|
407
|
+
return
|
|
408
|
+
probe: Any = _HTTPBridgeRetryCircuitKeyProbe(key=key, request_model=request_model)
|
|
409
|
+
await self._load_http_bridge_retry_circuit(probe)
|
|
410
|
+
|
|
411
|
+
async def _http_bridge_retry_circuit_current_count(self: Any, session: _HTTPBridgeSession) -> int:
|
|
412
|
+
async with self._http_bridge_retry_circuit_lock:
|
|
413
|
+
current_state = self._http_bridge_retry_circuits.get(session.key)
|
|
414
|
+
return current_state.consecutive_failures if current_state is not None else 0
|
|
415
|
+
|
|
416
|
+
async def _await_http_bridge_retry_circuit_attempt_settlement(
|
|
417
|
+
self: Any,
|
|
418
|
+
session: _HTTPBridgeSession,
|
|
419
|
+
*,
|
|
420
|
+
attempt: _HTTPBridgeResponseCreateAttempt,
|
|
421
|
+
detail: str,
|
|
422
|
+
) -> int:
|
|
423
|
+
settled = attempt.retry_circuit_failure_settled
|
|
424
|
+
if settled is not None:
|
|
425
|
+
await settled.wait()
|
|
426
|
+
consecutive_failures = await self._http_bridge_retry_circuit_current_count(session)
|
|
427
|
+
_record_http_bridge_retry_circuit_duplicate_suppressed(
|
|
428
|
+
session,
|
|
429
|
+
attempt=attempt,
|
|
430
|
+
consecutive_failures=consecutive_failures,
|
|
431
|
+
detail=detail,
|
|
432
|
+
)
|
|
433
|
+
return consecutive_failures
|
|
434
|
+
|
|
435
|
+
async def _record_http_bridge_retry_circuit_failure_for_attempt_selection(
|
|
436
|
+
self: Any,
|
|
437
|
+
session: _HTTPBridgeSession,
|
|
438
|
+
*,
|
|
439
|
+
detail: str,
|
|
440
|
+
selection: _HTTPBridgeRetryCircuitAttemptSelection,
|
|
441
|
+
) -> int | None:
|
|
442
|
+
attempt = selection.attempt
|
|
443
|
+
if attempt is not None:
|
|
444
|
+
return await self._record_http_bridge_retry_circuit_failure(
|
|
445
|
+
session,
|
|
446
|
+
detail=detail,
|
|
447
|
+
attempt=attempt,
|
|
448
|
+
)
|
|
449
|
+
if selection.kind == "absent":
|
|
450
|
+
return await self._record_http_bridge_retry_circuit_failure(session, detail=detail)
|
|
451
|
+
if selection.kind == "recorded":
|
|
452
|
+
for recorded_attempt in selection.attempts:
|
|
453
|
+
settled = recorded_attempt.retry_circuit_failure_settled
|
|
454
|
+
if settled is not None:
|
|
455
|
+
await settled.wait()
|
|
456
|
+
consecutive_failures = await self._http_bridge_retry_circuit_current_count(session)
|
|
457
|
+
for recorded_attempt in selection.attempts:
|
|
458
|
+
_record_http_bridge_retry_circuit_duplicate_suppressed(
|
|
459
|
+
session,
|
|
460
|
+
attempt=recorded_attempt,
|
|
461
|
+
consecutive_failures=consecutive_failures,
|
|
462
|
+
detail=detail,
|
|
463
|
+
)
|
|
464
|
+
return consecutive_failures
|
|
465
|
+
|
|
466
|
+
outcome = "ambiguous_suppressed" if selection.ambiguous else "ineligible_suppressed"
|
|
467
|
+
if PROMETHEUS_AVAILABLE and http_bridge_retry_circuit_total is not None:
|
|
468
|
+
http_bridge_retry_circuit_total.labels(outcome=outcome).inc()
|
|
469
|
+
logger.info(
|
|
470
|
+
"http_bridge_retry_circuit event=%s bridge_kind=%s bridge_key=%s detail=%s candidate_attempts=%s",
|
|
471
|
+
outcome,
|
|
472
|
+
session.key.affinity_kind,
|
|
473
|
+
_hash_identifier(session.key.affinity_key),
|
|
474
|
+
detail,
|
|
475
|
+
len(selection.attempts),
|
|
476
|
+
)
|
|
477
|
+
return None
|
|
478
|
+
|
|
479
|
+
def _prune_http_bridge_retry_circuit_state(self: Any, now: float) -> None:
|
|
480
|
+
expiry = now - DURABLE_BRIDGE_RETRY_CIRCUIT_STATE_TTL_SECONDS
|
|
481
|
+
for key, state in list(self._http_bridge_retry_circuits.items()):
|
|
482
|
+
if state.last_touched_monotonic > expiry:
|
|
483
|
+
continue
|
|
484
|
+
self._http_bridge_retry_circuits.pop(key, None)
|
|
485
|
+
self._http_bridge_retry_circuit_loaded_keys.discard(key)
|
|
486
|
+
self._http_bridge_retry_circuit_persisted_keys.discard(key)
|
|
487
|
+
for key, watermark in list(self._http_bridge_retry_circuit_reconcile_watermarks.items()):
|
|
488
|
+
if watermark <= expiry:
|
|
489
|
+
self._http_bridge_retry_circuit_reconcile_watermarks.pop(key, None)
|
|
490
|
+
miss_expiry = now - _HTTP_BRIDGE_RETRY_CIRCUIT_BASE_BACKOFF_SECONDS
|
|
491
|
+
misses = self._http_bridge_retry_circuit_planning_misses
|
|
492
|
+
# Entries are kept in insertion order (a re-record pops first), so
|
|
493
|
+
# expiry only ever sweeps the front. A full-map scan here would run
|
|
494
|
+
# under the shared circuit lock on every load and cost R²·window
|
|
495
|
+
# entry checks per second at R new healthy keys per second.
|
|
496
|
+
while misses:
|
|
497
|
+
oldest_key = next(iter(misses))
|
|
498
|
+
if misses[oldest_key] > miss_expiry:
|
|
499
|
+
break
|
|
500
|
+
del misses[oldest_key]
|
|
501
|
+
while len(misses) > _HTTP_BRIDGE_RETRY_CIRCUIT_PLANNING_MISS_CAP:
|
|
502
|
+
# The cap sheds the oldest confirmed misses first; an evicted
|
|
503
|
+
# key merely pays one planning-time load again.
|
|
504
|
+
del misses[next(iter(misses))]
|
|
505
|
+
for key, key_lock in list(self._http_bridge_retry_circuit_key_locks.items()):
|
|
506
|
+
if key not in self._http_bridge_retry_circuits and not key_lock.locked():
|
|
507
|
+
self._http_bridge_retry_circuit_key_locks.pop(key, None)
|
|
508
|
+
|
|
509
|
+
async def _acquire_http_bridge_retry_circuit_key_lock(
|
|
510
|
+
self: Any,
|
|
511
|
+
key: _HTTPBridgeSessionKey,
|
|
512
|
+
) -> asyncio.Lock:
|
|
513
|
+
"""Serialize durable circuit writes and settles for one key.
|
|
514
|
+
|
|
515
|
+
A strike write and a settle for the same key must not interleave
|
|
516
|
+
across their durable awaits: a settle landing under an in-flight
|
|
517
|
+
write resurrected the row, and a stale write landing after a settle
|
|
518
|
+
merged a finished episode's strike into whatever episode owned the
|
|
519
|
+
row next. The lock is per key, so unrelated keys never wait on each
|
|
520
|
+
other's durable I/O. The acquire loop re-checks registration because
|
|
521
|
+
pruning may drop an idle lock between the fetch and the acquire.
|
|
522
|
+
"""
|
|
523
|
+
while True:
|
|
524
|
+
async with self._http_bridge_retry_circuit_lock:
|
|
525
|
+
key_lock = self._http_bridge_retry_circuit_key_locks.setdefault(key, asyncio.Lock())
|
|
526
|
+
await key_lock.acquire()
|
|
527
|
+
# Ownership transfers to the caller only on return. The
|
|
528
|
+
# registration re-check is itself a cancellable await; a
|
|
529
|
+
# cancellation landing there would otherwise leave the key lock
|
|
530
|
+
# held forever, wedging every later persist and settle for the
|
|
531
|
+
# key.
|
|
532
|
+
try:
|
|
533
|
+
async with self._http_bridge_retry_circuit_lock:
|
|
534
|
+
if self._http_bridge_retry_circuit_key_locks.get(key) is key_lock:
|
|
535
|
+
return key_lock
|
|
536
|
+
except BaseException:
|
|
537
|
+
key_lock.release()
|
|
538
|
+
raise
|
|
539
|
+
key_lock.release()
|
|
540
|
+
|
|
541
|
+
async def _load_http_bridge_retry_circuit(self: Any, session: _HTTPBridgeSession) -> bool:
|
|
542
|
+
key = session.key
|
|
543
|
+
if key.strength != "hard":
|
|
544
|
+
return True
|
|
545
|
+
|
|
546
|
+
now_monotonic = clock_for(self).monotonic()
|
|
547
|
+
async with self._http_bridge_retry_circuit_lock:
|
|
548
|
+
self._prune_http_bridge_retry_circuit_state(now_monotonic)
|
|
549
|
+
local_state = self._http_bridge_retry_circuits.get(key)
|
|
550
|
+
if local_state is not None:
|
|
551
|
+
local_state.last_touched_monotonic = now_monotonic
|
|
552
|
+
try:
|
|
553
|
+
persisted = await self._durable_bridge.lookup_retry_circuit(
|
|
554
|
+
session_key_kind=key.affinity_kind,
|
|
555
|
+
session_key_value=key.affinity_key,
|
|
556
|
+
api_key_id=key.api_key_id,
|
|
557
|
+
)
|
|
558
|
+
except Exception:
|
|
559
|
+
if PROMETHEUS_AVAILABLE and http_bridge_retry_circuit_total is not None:
|
|
560
|
+
http_bridge_retry_circuit_total.labels(outcome="lookup_failed").inc()
|
|
561
|
+
logger.warning(
|
|
562
|
+
"Failed to load persisted HTTP bridge retry circuit bridge_kind=%s bridge_key=%s",
|
|
563
|
+
key.affinity_kind,
|
|
564
|
+
_hash_identifier(key.affinity_key),
|
|
565
|
+
exc_info=True,
|
|
566
|
+
)
|
|
567
|
+
return False
|
|
568
|
+
|
|
569
|
+
if persisted is None:
|
|
570
|
+
# A durable miss clears state loaded from another replica, but it
|
|
571
|
+
# must not discard a failure recorded locally after the last
|
|
572
|
+
# durable read. That local circuit is the only protection against
|
|
573
|
+
# immediately replaying the same failing upstream request.
|
|
574
|
+
async with self._http_bridge_retry_circuit_lock:
|
|
575
|
+
local_state = self._http_bridge_retry_circuits.get(key)
|
|
576
|
+
key_watermark = self._http_bridge_retry_circuit_reconcile_watermarks.get(key, 0.0)
|
|
577
|
+
if now_monotonic < key_watermark or (
|
|
578
|
+
local_state is not None and now_monotonic < local_state.last_durable_load_monotonic
|
|
579
|
+
):
|
|
580
|
+
# This lookup began before a same-key write completed:
|
|
581
|
+
# its miss predates the row that write created, and
|
|
582
|
+
# popping the episode here would let the admission
|
|
583
|
+
# decision that follows bypass the active cooldown the
|
|
584
|
+
# completed write just opened.
|
|
585
|
+
return True
|
|
586
|
+
locally_updated = bool(
|
|
587
|
+
local_state is not None
|
|
588
|
+
and local_state.last_failure_monotonic > local_state.last_durable_load_monotonic
|
|
589
|
+
)
|
|
590
|
+
# A confirmed miss past the stale-lookup guards is durable
|
|
591
|
+
# knowledge worth caching for the planning window; stamped
|
|
592
|
+
# with the load's start so it expires conservatively, and
|
|
593
|
+
# re-inserted so the map stays in insertion order for the
|
|
594
|
+
# front-only expiry sweep.
|
|
595
|
+
self._http_bridge_retry_circuit_planning_misses.pop(key, None)
|
|
596
|
+
self._http_bridge_retry_circuit_planning_misses[key] = now_monotonic
|
|
597
|
+
while (
|
|
598
|
+
len(self._http_bridge_retry_circuit_planning_misses) > _HTTP_BRIDGE_RETRY_CIRCUIT_PLANNING_MISS_CAP
|
|
599
|
+
):
|
|
600
|
+
# The pre-lookup prune ran before this load's durable
|
|
601
|
+
# await, so a burst of concurrent first-touch misses can
|
|
602
|
+
# land past it; the cap is enforced where entries are
|
|
603
|
+
# added.
|
|
604
|
+
del self._http_bridge_retry_circuit_planning_misses[
|
|
605
|
+
next(iter(self._http_bridge_retry_circuit_planning_misses))
|
|
606
|
+
]
|
|
607
|
+
if key in self._http_bridge_retry_circuit_persisted_keys and not locally_updated:
|
|
608
|
+
self._http_bridge_retry_circuits.pop(key, None)
|
|
609
|
+
self._http_bridge_retry_circuit_loaded_keys.discard(key)
|
|
610
|
+
self._http_bridge_retry_circuit_persisted_keys.discard(key)
|
|
611
|
+
if _http_bridge_session_key_poison_quarantined(self, key):
|
|
612
|
+
# The durable episode this quarantine fenced is gone
|
|
613
|
+
# — settled or purged by another replica — and a
|
|
614
|
+
# remotely recovered key must not stay suppressed
|
|
615
|
+
# for the stale deadline.
|
|
616
|
+
_revoke_http_bridge_poison_quarantine(
|
|
617
|
+
self,
|
|
618
|
+
key,
|
|
619
|
+
generation=_http_bridge_quarantine_clear_fence(self, key),
|
|
620
|
+
)
|
|
621
|
+
return True
|
|
622
|
+
|
|
623
|
+
now_epoch = clock_for(self).time()
|
|
624
|
+
row_age = now_epoch - persisted.updated_at_epoch
|
|
625
|
+
if (
|
|
626
|
+
row_age > DURABLE_BRIDGE_RETRY_CIRCUIT_STATE_TTL_SECONDS
|
|
627
|
+
# An abandonment tombstone guards continuity that outlives the
|
|
628
|
+
# circuit TTL: purging it here would let the unanchored-delta
|
|
629
|
+
# gate dispatch a context-free request. Its lifetime belongs to
|
|
630
|
+
# the scheduled purge's bridge-retention cutoff.
|
|
631
|
+
and persisted.last_detail != _HTTP_BRIDGE_RETRY_CIRCUIT_ANCHOR_ABANDONED_DETAIL
|
|
632
|
+
and (
|
|
633
|
+
getattr(persisted, "admission_generation", 0) == 0
|
|
634
|
+
or row_age > 2 * DURABLE_BRIDGE_RETRY_CIRCUIT_STATE_TTL_SECONDS
|
|
635
|
+
)
|
|
636
|
+
):
|
|
637
|
+
async with self._http_bridge_retry_circuit_lock:
|
|
638
|
+
stale_local_state = self._http_bridge_retry_circuits.get(key)
|
|
639
|
+
try:
|
|
640
|
+
stale_row_purged = await self._durable_bridge.purge_retry_circuit(
|
|
641
|
+
session_key_kind=key.affinity_kind,
|
|
642
|
+
session_key_value=key.affinity_key,
|
|
643
|
+
api_key_id=key.api_key_id,
|
|
644
|
+
expected_updated_at_epoch=persisted.updated_at_epoch,
|
|
645
|
+
expected_admission_generation=getattr(persisted, "admission_generation", 0),
|
|
646
|
+
# Detail-only transitions (the transitional tombstone
|
|
647
|
+
# supersede) and lagging-clock merges move neither the
|
|
648
|
+
# epoch nor the admission generation; fencing on the
|
|
649
|
+
# observed count and detail keeps this purge from
|
|
650
|
+
# deleting a row another replica just rewrote.
|
|
651
|
+
expected_consecutive_failures=persisted.consecutive_failures,
|
|
652
|
+
fence_last_detail=True,
|
|
653
|
+
expected_last_detail=persisted.last_detail,
|
|
654
|
+
)
|
|
655
|
+
except Exception:
|
|
656
|
+
logger.warning(
|
|
657
|
+
"Failed to remove stale HTTP bridge retry circuit bridge_kind=%s bridge_key=%s",
|
|
658
|
+
key.affinity_kind,
|
|
659
|
+
_hash_identifier(key.affinity_key),
|
|
660
|
+
exc_info=True,
|
|
661
|
+
)
|
|
662
|
+
# Keep a newer process-local circuit when persistence is
|
|
663
|
+
# unavailable. The next failure can still open the local
|
|
664
|
+
# circuit even though the expired durable row remains.
|
|
665
|
+
return False
|
|
666
|
+
if stale_row_purged is False:
|
|
667
|
+
# The fenced purge matched nothing: another replica moved
|
|
668
|
+
# the row after this worker's lookup — a fresh strike or
|
|
669
|
+
# opening — and proceeding as though it was deleted would
|
|
670
|
+
# pop the local circuit, revoke the quarantine, and let the
|
|
671
|
+
# very next admission dispatch with the poisoned anchor.
|
|
672
|
+
# Reconcile against the surviving row instead.
|
|
673
|
+
try:
|
|
674
|
+
refreshed = await self._durable_bridge.lookup_retry_circuit(
|
|
675
|
+
session_key_kind=key.affinity_kind,
|
|
676
|
+
session_key_value=key.affinity_key,
|
|
677
|
+
api_key_id=key.api_key_id,
|
|
678
|
+
)
|
|
679
|
+
except Exception:
|
|
680
|
+
logger.warning(
|
|
681
|
+
"Failed to reload HTTP bridge retry circuit after purge miss bridge_kind=%s bridge_key=%s",
|
|
682
|
+
key.affinity_kind,
|
|
683
|
+
_hash_identifier(key.affinity_key),
|
|
684
|
+
exc_info=True,
|
|
685
|
+
)
|
|
686
|
+
return False
|
|
687
|
+
if refreshed is not None and (
|
|
688
|
+
# A detail-only tombstone rewrite keeps the old epoch, so
|
|
689
|
+
# an age check would reject the very fence the missed
|
|
690
|
+
# purge just proved exists — and planning would inject
|
|
691
|
+
# the dead anchor the tombstone guards. Tombstones are
|
|
692
|
+
# adopted regardless of circuit age; their lifetime
|
|
693
|
+
# belongs to the bridge-retention purge.
|
|
694
|
+
refreshed.last_detail == _HTTP_BRIDGE_RETRY_CIRCUIT_ANCHOR_ABANDONED_DETAIL
|
|
695
|
+
or now_epoch - refreshed.updated_at_epoch
|
|
696
|
+
<= (
|
|
697
|
+
DURABLE_BRIDGE_RETRY_CIRCUIT_STATE_TTL_SECONDS
|
|
698
|
+
* (2 if getattr(refreshed, "admission_generation", 0) > 0 else 1)
|
|
699
|
+
)
|
|
700
|
+
):
|
|
701
|
+
persisted = refreshed
|
|
702
|
+
else:
|
|
703
|
+
# The surviving row vanished again or is itself stale:
|
|
704
|
+
# nothing current to adopt, and nothing was deleted this
|
|
705
|
+
# worker can vouch for.
|
|
706
|
+
return True
|
|
707
|
+
if stale_row_purged is not False:
|
|
708
|
+
async with self._http_bridge_retry_circuit_lock:
|
|
709
|
+
current_local_state = self._http_bridge_retry_circuits.get(key)
|
|
710
|
+
local_state_is_newer = bool(
|
|
711
|
+
current_local_state is not None
|
|
712
|
+
and current_local_state.last_failure_monotonic > current_local_state.last_durable_load_monotonic
|
|
713
|
+
)
|
|
714
|
+
if current_local_state is None or (
|
|
715
|
+
current_local_state is stale_local_state and not local_state_is_newer
|
|
716
|
+
):
|
|
717
|
+
self._http_bridge_retry_circuits.pop(key, None)
|
|
718
|
+
self._http_bridge_retry_circuit_loaded_keys.discard(key)
|
|
719
|
+
self._http_bridge_retry_circuit_persisted_keys.discard(key)
|
|
720
|
+
if _http_bridge_session_key_poison_quarantined(self, key):
|
|
721
|
+
# The expired row just purged is the episode this
|
|
722
|
+
# quarantine fenced; it ended long ago.
|
|
723
|
+
_revoke_http_bridge_poison_quarantine(
|
|
724
|
+
self,
|
|
725
|
+
key,
|
|
726
|
+
generation=_http_bridge_quarantine_clear_fence(self, key),
|
|
727
|
+
)
|
|
728
|
+
return True
|
|
729
|
+
|
|
730
|
+
cooldown_remaining = max(0.0, persisted.cooldown_until_epoch - now_epoch)
|
|
731
|
+
persisted_cooldown_until = now_monotonic + cooldown_remaining
|
|
732
|
+
arm_poison_quarantine = False
|
|
733
|
+
poison_cooldown_remaining = 0.0
|
|
734
|
+
async with self._http_bridge_retry_circuit_lock:
|
|
735
|
+
self._http_bridge_retry_circuit_persisted_keys.add(key)
|
|
736
|
+
self._http_bridge_retry_circuit_planning_misses.pop(key, None)
|
|
737
|
+
state = self._http_bridge_retry_circuits.get(key)
|
|
738
|
+
if state is None:
|
|
739
|
+
state = _HTTPBridgeRetryCircuitState(last_touched_monotonic=now_monotonic)
|
|
740
|
+
self._http_bridge_retry_circuits[key] = state
|
|
741
|
+
key_watermark = self._http_bridge_retry_circuit_reconcile_watermarks.get(key, 0.0)
|
|
742
|
+
if now_monotonic < state.last_durable_load_monotonic or now_monotonic < key_watermark:
|
|
743
|
+
# This load's lookup began before a same-key strike or
|
|
744
|
+
# settlement completed its durable write, so its row snapshot
|
|
745
|
+
# may predate that write: adopting it would erase a
|
|
746
|
+
# just-opened cooldown or resurrect a just-settled episode.
|
|
747
|
+
# The completed operation already reconciled the state from
|
|
748
|
+
# its own returned row (or popped it at settlement, which
|
|
749
|
+
# the per-key watermark records past the pop); an older
|
|
750
|
+
# snapshot has nothing newer to add.
|
|
751
|
+
self._http_bridge_retry_circuit_loaded_keys.add(key)
|
|
752
|
+
return True
|
|
753
|
+
local_failure_is_newer = state.last_failure_monotonic > state.last_durable_load_monotonic
|
|
754
|
+
# A foreign write is identified by ANY observed column moving,
|
|
755
|
+
# not the timestamp alone: a lagging-clock strike merges through
|
|
756
|
+
# greatest() without moving the epoch while incrementing the
|
|
757
|
+
# count and rewriting the detail, and an anchor supersession
|
|
758
|
+
# rewrites the detail in place by design. Epoch-only detection
|
|
759
|
+
# let a marker survive a new failure against the advanced
|
|
760
|
+
# anchor and left an active quarantine unextended.
|
|
761
|
+
episode_replaced = (
|
|
762
|
+
persisted.updated_at_epoch != state.persisted_updated_at_epoch
|
|
763
|
+
or persisted.consecutive_failures != state.consecutive_failures
|
|
764
|
+
or persisted.last_detail != state.last_detail
|
|
765
|
+
)
|
|
766
|
+
if not local_failure_is_newer:
|
|
767
|
+
# No local strike is waiting on its durable write, so the row
|
|
768
|
+
# is strictly newer knowledge and is adopted wholesale.
|
|
769
|
+
# Adoption compares no wall clocks: a reset stamped by a
|
|
770
|
+
# lagging replica clock still replaces the local episode
|
|
771
|
+
# instead of leaving a settled key suppressed for the rest
|
|
772
|
+
# of its stale cooldown with a base that no longer exists.
|
|
773
|
+
if episode_replaced:
|
|
774
|
+
# A write this worker did not produce: either the same
|
|
775
|
+
# episode struck elsewhere (then a set marker means the
|
|
776
|
+
# anchor is already cleared and the empty-continuity
|
|
777
|
+
# consult refuses regardless), or the episode was
|
|
778
|
+
# replaced — possibly at an equal or higher count — and
|
|
779
|
+
# the replacement's newly poisoned anchor must be
|
|
780
|
+
# allowed its one abandonment. Resetting on lineage
|
|
781
|
+
# rather than count ordering covers the equal-count
|
|
782
|
+
# replacement the count check below cannot see.
|
|
783
|
+
state.poison_anchor_cleared = False
|
|
784
|
+
# The local debt dies with every foreign write: counts
|
|
785
|
+
# and epochs cannot distinguish same-lineage advancement
|
|
786
|
+
# from a reset-and-overtaken replacement, and guessing
|
|
787
|
+
# wrong clears a replacement's fresh continuity. The
|
|
788
|
+
# durable row itself carries the cross-replica debt —
|
|
789
|
+
# an at-threshold poison detail is sticky against
|
|
790
|
+
# non-poison strikes in the merge — so the arm below
|
|
791
|
+
# re-derives the debt from the adopted row.
|
|
792
|
+
state.owed_poison_detail = None
|
|
793
|
+
if persisted.consecutive_failures < state.consecutive_failures:
|
|
794
|
+
# The row does not carry this worker's failures: the
|
|
795
|
+
# lineage was reset, purged, or replaced, and the marker
|
|
796
|
+
# belonged to the ended episode.
|
|
797
|
+
state.poison_anchor_cleared = False
|
|
798
|
+
state.owed_poison_detail = None
|
|
799
|
+
state.consecutive_failures = max(0, persisted.consecutive_failures)
|
|
800
|
+
state.cooldown_until = persisted_cooldown_until
|
|
801
|
+
state.last_detail = persisted.last_detail
|
|
802
|
+
if state.consecutive_failures == 0:
|
|
803
|
+
# A zero-failure row is a durable reset: the episode the
|
|
804
|
+
# marker belonged to is over, and the next poison episode
|
|
805
|
+
# on this state object must be allowed its one
|
|
806
|
+
# abandonment.
|
|
807
|
+
state.poison_anchor_cleared = False
|
|
808
|
+
state.owed_poison_detail = None
|
|
809
|
+
else:
|
|
810
|
+
# A local strike sits between its record and its durable
|
|
811
|
+
# write; keep it dominant here and let that write's own
|
|
812
|
+
# merge reconcile clock-free against the returned row.
|
|
813
|
+
state.consecutive_failures = max(state.consecutive_failures, max(0, persisted.consecutive_failures))
|
|
814
|
+
state.cooldown_until = max(state.cooldown_until, persisted_cooldown_until)
|
|
815
|
+
state.last_detail = state.last_detail or persisted.last_detail
|
|
816
|
+
if state.cooldown_until > now_monotonic:
|
|
817
|
+
# A cooling key is not probing. Without this, a merged
|
|
818
|
+
# persisted cooldown leaves a leftover half-open lease
|
|
819
|
+
# armed, and once the cooldown expires the admission gate
|
|
820
|
+
# reads that stale lease as an in-flight probe and keeps
|
|
821
|
+
# suppressing the key for the rest of the lease.
|
|
822
|
+
state.half_open_until = 0.0
|
|
823
|
+
elif episode_replaced and not local_failure_is_newer:
|
|
824
|
+
# The adopted row belongs to a replacement episode whose
|
|
825
|
+
# cooldown has already elapsed. A lease left over from the
|
|
826
|
+
# ended episode would read as this worker's in-flight probe
|
|
827
|
+
# for the new one and suppress the key for the rest of a
|
|
828
|
+
# window it never opened.
|
|
829
|
+
state.half_open_until = 0.0
|
|
830
|
+
state.persisted_updated_at_epoch = persisted.updated_at_epoch
|
|
831
|
+
state.persisted_admission_generation = getattr(persisted, "admission_generation", 0)
|
|
832
|
+
state.last_touched_monotonic = now_monotonic
|
|
833
|
+
state.last_durable_load_monotonic = now_monotonic
|
|
834
|
+
self._http_bridge_retry_circuit_loaded_keys.add(key)
|
|
835
|
+
# A poison opening recorded by another replica reaches this
|
|
836
|
+
# worker only through this load, and its quarantine is
|
|
837
|
+
# process-local: without re-arming here, this worker's probe is
|
|
838
|
+
# planned with the anchor the row's failures were recorded
|
|
839
|
+
# against.
|
|
840
|
+
effective_poison_threshold = max(1, _HTTP_BRIDGE_RETRY_CIRCUIT_FAILURE_THRESHOLD)
|
|
841
|
+
arm_poison_quarantine = (
|
|
842
|
+
not local_failure_is_newer
|
|
843
|
+
# An episode whose one-clear marker is set already abandoned
|
|
844
|
+
# its anchor (a completed verified replay deliberately keeps
|
|
845
|
+
# the row and clears the quarantine); re-arming off the
|
|
846
|
+
# unchanged row would re-fence the recovered key for another
|
|
847
|
+
# full deadline.
|
|
848
|
+
and not state.poison_anchor_cleared
|
|
849
|
+
and state.consecutive_failures >= effective_poison_threshold
|
|
850
|
+
and _http_bridge_anchor_poison_detail(state.last_detail) is not None
|
|
851
|
+
)
|
|
852
|
+
if arm_poison_quarantine:
|
|
853
|
+
state.owed_poison_detail = state.last_detail
|
|
854
|
+
# The adopted row can also disprove the episode a local poison
|
|
855
|
+
# quarantine fenced: a zero-failure reset, a deliberate anchor
|
|
856
|
+
# supersession, or a replaced lineage that has not proven poison
|
|
857
|
+
# ends it, and a remotely recovered key must not stay suppressed
|
|
858
|
+
# for the stale deadline. A replaced lineage at or past the
|
|
859
|
+
# threshold with a non-poison last strike is left fenced — its
|
|
860
|
+
# earlier strikes may still be the poison evidence.
|
|
861
|
+
revoke_stale_poison_quarantine = (
|
|
862
|
+
not local_failure_is_newer
|
|
863
|
+
# A transitional abandonment tombstone is not disproof: the
|
|
864
|
+
# fresh anchor's registration has not been confirmed, the
|
|
865
|
+
# old poisoned anchor may still be the stored one, and the
|
|
866
|
+
# surviving quarantine is what keeps full-resend planning
|
|
867
|
+
# from injecting it.
|
|
868
|
+
and persisted.last_detail != _HTTP_BRIDGE_RETRY_CIRCUIT_ANCHOR_ABANDONED_DETAIL
|
|
869
|
+
and (
|
|
870
|
+
persisted.consecutive_failures == 0
|
|
871
|
+
or persisted.last_detail == _HTTP_BRIDGE_RETRY_CIRCUIT_ANCHOR_SUPERSEDED_DETAIL
|
|
872
|
+
or (
|
|
873
|
+
episode_replaced
|
|
874
|
+
and _http_bridge_anchor_poison_detail(persisted.last_detail) is None
|
|
875
|
+
and persisted.consecutive_failures < effective_poison_threshold
|
|
876
|
+
)
|
|
877
|
+
)
|
|
878
|
+
)
|
|
879
|
+
poison_cooldown_remaining = max(0.0, state.cooldown_until - now_monotonic)
|
|
880
|
+
if arm_poison_quarantine and (episode_replaced or not _http_bridge_session_key_poison_quarantined(self, key)):
|
|
881
|
+
# Skipped only for a truly unchanged episode, so ordinary loads
|
|
882
|
+
# do not bump the quarantine generation that recovery fences
|
|
883
|
+
# observe. An adopted foreign write re-arms even while a
|
|
884
|
+
# quarantine is active: a later durable strike can extend the
|
|
885
|
+
# cooldown past the old deadline, and leaving that deadline in
|
|
886
|
+
# place would let the quarantine lapse mid-cooldown — with the
|
|
887
|
+
# planning cache still fresh — and hand the next half-open probe
|
|
888
|
+
# the poisoned anchor. The re-arm extends the deadline against
|
|
889
|
+
# the adopted cooldown and refreshes the poison provenance to
|
|
890
|
+
# the lineage that now owns the row.
|
|
891
|
+
_quarantine_http_bridge_session(
|
|
892
|
+
self,
|
|
893
|
+
session,
|
|
894
|
+
reason=_HTTP_BRIDGE_QUARANTINE_POISONED_ANCHOR_REASON,
|
|
895
|
+
minimum_seconds=_http_bridge_poison_quarantine_minimum_seconds(poison_cooldown_remaining),
|
|
896
|
+
)
|
|
897
|
+
elif revoke_stale_poison_quarantine and _http_bridge_session_key_poison_quarantined(self, key):
|
|
898
|
+
_revoke_http_bridge_poison_quarantine(
|
|
899
|
+
self,
|
|
900
|
+
key,
|
|
901
|
+
generation=_http_bridge_quarantine_clear_fence(self, key),
|
|
902
|
+
)
|
|
903
|
+
return True
|
|
904
|
+
|
|
905
|
+
async def _persist_http_bridge_retry_circuit(
|
|
906
|
+
self: Any,
|
|
907
|
+
session: _HTTPBridgeSession,
|
|
908
|
+
state: _HTTPBridgeRetryCircuitState,
|
|
909
|
+
) -> None:
|
|
910
|
+
now_monotonic = clock_for(self).monotonic()
|
|
911
|
+
now_wall = clock_for(self).time()
|
|
912
|
+
threshold = max(1, _HTTP_BRIDGE_RETRY_CIRCUIT_FAILURE_THRESHOLD)
|
|
913
|
+
key_lock = await self._acquire_http_bridge_retry_circuit_key_lock(session.key)
|
|
914
|
+
try:
|
|
915
|
+
await self._persist_http_bridge_retry_circuit_serialized(
|
|
916
|
+
session,
|
|
917
|
+
state,
|
|
918
|
+
now_monotonic=now_monotonic,
|
|
919
|
+
now_wall=now_wall,
|
|
920
|
+
threshold=threshold,
|
|
921
|
+
)
|
|
922
|
+
finally:
|
|
923
|
+
key_lock.release()
|
|
924
|
+
|
|
925
|
+
async def _persist_http_bridge_retry_circuit_serialized(
|
|
926
|
+
self: Any,
|
|
927
|
+
session: _HTTPBridgeSession,
|
|
928
|
+
state: _HTTPBridgeRetryCircuitState,
|
|
929
|
+
*,
|
|
930
|
+
now_monotonic: float,
|
|
931
|
+
now_wall: float,
|
|
932
|
+
threshold: int,
|
|
933
|
+
) -> None:
|
|
934
|
+
async with self._http_bridge_retry_circuit_lock:
|
|
935
|
+
# Re-checked under the key lock: a settle or a replacement
|
|
936
|
+
# episode that took the key while this writer waited means this
|
|
937
|
+
# strike belongs to a finished episode, and writing it would
|
|
938
|
+
# merge that stale failure into whatever episode owns the row
|
|
939
|
+
# now.
|
|
940
|
+
if self._http_bridge_retry_circuits.get(session.key) is not state:
|
|
941
|
+
return
|
|
942
|
+
consecutive_failures = state.consecutive_failures
|
|
943
|
+
cooldown_until = state.cooldown_until
|
|
944
|
+
last_detail = state.last_detail
|
|
945
|
+
persisted_updated_at_epoch = state.persisted_updated_at_epoch
|
|
946
|
+
poison_sticky_threshold = threshold
|
|
947
|
+
base_backoff = max(0.001, _HTTP_BRIDGE_RETRY_CIRCUIT_BASE_BACKOFF_SECONDS)
|
|
948
|
+
if last_detail == "clean_close":
|
|
949
|
+
base_backoff = min(
|
|
950
|
+
base_backoff,
|
|
951
|
+
max(0.001, _HTTP_BRIDGE_RETRY_CIRCUIT_CLEAN_CLOSE_MAX_BACKOFF_SECONDS),
|
|
952
|
+
)
|
|
953
|
+
try:
|
|
954
|
+
persisted = await self._durable_bridge.persist_retry_circuit(
|
|
955
|
+
session_key_kind=session.key.affinity_kind,
|
|
956
|
+
session_key_value=session.key.affinity_key,
|
|
957
|
+
api_key_id=session.key.api_key_id,
|
|
958
|
+
consecutive_failures=consecutive_failures,
|
|
959
|
+
cooldown_until_epoch=now_wall + max(0.0, cooldown_until - now_monotonic),
|
|
960
|
+
last_detail=last_detail,
|
|
961
|
+
updated_at_epoch=now_wall,
|
|
962
|
+
base_updated_at_epoch=persisted_updated_at_epoch,
|
|
963
|
+
failure_threshold=threshold,
|
|
964
|
+
poison_sticky_threshold=poison_sticky_threshold,
|
|
965
|
+
conflict_cooldown_until_epoch=now_wall + base_backoff,
|
|
966
|
+
base_backoff_seconds=max(0.001, _HTTP_BRIDGE_RETRY_CIRCUIT_BASE_BACKOFF_SECONDS),
|
|
967
|
+
max_backoff_seconds=max(0.001, _HTTP_BRIDGE_RETRY_CIRCUIT_MAX_BACKOFF_SECONDS),
|
|
968
|
+
clean_close_max_backoff_seconds=max(
|
|
969
|
+
0.001,
|
|
970
|
+
_HTTP_BRIDGE_RETRY_CIRCUIT_CLEAN_CLOSE_MAX_BACKOFF_SECONDS,
|
|
971
|
+
),
|
|
972
|
+
)
|
|
973
|
+
if persisted is not None and persisted_updated_at_epoch <= 0 and persisted.updated_at_epoch != now_wall:
|
|
974
|
+
# Blind base: the pre-strike load failed, so this write
|
|
975
|
+
# carried no valid base and dropped against a row that
|
|
976
|
+
# already existed. A stale-episode drop loses at most its
|
|
977
|
+
# own already-counted strike; a blind drop loses the only
|
|
978
|
+
# record of a genuinely new failure, both durably and — via
|
|
979
|
+
# the wholesale adoption below — locally. Re-strike exactly
|
|
980
|
+
# this one failure on top of the observed lineage: strikes
|
|
981
|
+
# are serialized per key, so this write carries one new
|
|
982
|
+
# strike, and earlier local strikes either already landed
|
|
983
|
+
# unobserved or were lost to their own failed writes. A
|
|
984
|
+
# second drop means the row moved again and the ordinary
|
|
985
|
+
# undercount trade applies.
|
|
986
|
+
persisted = await self._durable_bridge.persist_retry_circuit(
|
|
987
|
+
session_key_kind=session.key.affinity_kind,
|
|
988
|
+
session_key_value=session.key.affinity_key,
|
|
989
|
+
api_key_id=session.key.api_key_id,
|
|
990
|
+
consecutive_failures=max(0, persisted.consecutive_failures) + 1,
|
|
991
|
+
cooldown_until_epoch=now_wall + max(0.0, cooldown_until - now_monotonic),
|
|
992
|
+
last_detail=last_detail,
|
|
993
|
+
updated_at_epoch=now_wall,
|
|
994
|
+
base_updated_at_epoch=persisted.updated_at_epoch,
|
|
995
|
+
failure_threshold=threshold,
|
|
996
|
+
poison_sticky_threshold=poison_sticky_threshold,
|
|
997
|
+
conflict_cooldown_until_epoch=now_wall + base_backoff,
|
|
998
|
+
base_backoff_seconds=max(0.001, _HTTP_BRIDGE_RETRY_CIRCUIT_BASE_BACKOFF_SECONDS),
|
|
999
|
+
max_backoff_seconds=max(0.001, _HTTP_BRIDGE_RETRY_CIRCUIT_MAX_BACKOFF_SECONDS),
|
|
1000
|
+
clean_close_max_backoff_seconds=max(
|
|
1001
|
+
0.001,
|
|
1002
|
+
_HTTP_BRIDGE_RETRY_CIRCUIT_CLEAN_CLOSE_MAX_BACKOFF_SECONDS,
|
|
1003
|
+
),
|
|
1004
|
+
)
|
|
1005
|
+
if persisted is not None:
|
|
1006
|
+
persisted_cooldown_until = now_monotonic + max(0.0, persisted.cooldown_until_epoch - now_wall)
|
|
1007
|
+
async with self._http_bridge_retry_circuit_lock:
|
|
1008
|
+
current = self._http_bridge_retry_circuits.get(session.key)
|
|
1009
|
+
if current is state:
|
|
1010
|
+
# The upsert returns the post-write row: when this
|
|
1011
|
+
# write landed the row reflects it, and when its base
|
|
1012
|
+
# mismatched the row is the lineage that owns the key
|
|
1013
|
+
# now and this writer must reconcile from it. Strikes
|
|
1014
|
+
# for one key are serialized across their durable
|
|
1015
|
+
# awaits, so no local failure can be recorded while
|
|
1016
|
+
# this persist is in flight, and the returned row is
|
|
1017
|
+
# adopted wholesale. Adoption compares no wall
|
|
1018
|
+
# clocks: a reset stamped by a lagging replica clock
|
|
1019
|
+
# still replaces the local episode, and taking the
|
|
1020
|
+
# row's epoch exactly — never the max — is what lets
|
|
1021
|
+
# the next strike carry a base that actually exists.
|
|
1022
|
+
if persisted.consecutive_failures < state.consecutive_failures:
|
|
1023
|
+
# The row does not carry this worker's failures:
|
|
1024
|
+
# the lineage was reset, purged, or replaced.
|
|
1025
|
+
# The marker belonged to the ended episode; a
|
|
1026
|
+
# spurious trip only allows one extra fenced
|
|
1027
|
+
# abandonment.
|
|
1028
|
+
state.poison_anchor_cleared = False
|
|
1029
|
+
state.owed_poison_detail = None
|
|
1030
|
+
if (
|
|
1031
|
+
persisted.consecutive_failures != consecutive_failures
|
|
1032
|
+
or persisted.last_detail != last_detail
|
|
1033
|
+
):
|
|
1034
|
+
# The returned row carries a contribution this
|
|
1035
|
+
# write did not submit: a same-base foreign
|
|
1036
|
+
# strike merged under a lagging clock keeps the
|
|
1037
|
+
# epoch while moving the count or detail, and
|
|
1038
|
+
# its evidence targets whatever anchor is
|
|
1039
|
+
# current — the marker cannot survive it.
|
|
1040
|
+
state.poison_anchor_cleared = False
|
|
1041
|
+
state.owed_poison_detail = (
|
|
1042
|
+
persisted.last_detail
|
|
1043
|
+
if _http_bridge_anchor_poison_detail(persisted.last_detail) is not None
|
|
1044
|
+
else None
|
|
1045
|
+
)
|
|
1046
|
+
if (
|
|
1047
|
+
persisted.updated_at_epoch != now_wall
|
|
1048
|
+
and persisted.updated_at_epoch != persisted_updated_at_epoch
|
|
1049
|
+
):
|
|
1050
|
+
# This write lost to a foreign replacement row —
|
|
1051
|
+
# a landed write stamps max(base, now_wall), and
|
|
1052
|
+
# an unchanged row keeps the base. Adopting the
|
|
1053
|
+
# foreign epoch below makes later loads see the
|
|
1054
|
+
# episode as unchanged, so this is the only point
|
|
1055
|
+
# that can see the replacement; an equal-or-higher
|
|
1056
|
+
# count hides it from the check above, and a
|
|
1057
|
+
# marker carried across it would permanently
|
|
1058
|
+
# refuse the replacement episode its one
|
|
1059
|
+
# abandonment.
|
|
1060
|
+
state.poison_anchor_cleared = False
|
|
1061
|
+
state.owed_poison_detail = None
|
|
1062
|
+
state.consecutive_failures = max(0, persisted.consecutive_failures)
|
|
1063
|
+
state.cooldown_until = persisted_cooldown_until
|
|
1064
|
+
state.last_detail = persisted.last_detail
|
|
1065
|
+
if state.consecutive_failures == 0:
|
|
1066
|
+
# A zero-failure row is a durable reset ending
|
|
1067
|
+
# the marker's episode.
|
|
1068
|
+
state.poison_anchor_cleared = False
|
|
1069
|
+
state.owed_poison_detail = None
|
|
1070
|
+
if state.cooldown_until > now_monotonic:
|
|
1071
|
+
# A cooling key is not probing: drop any leftover
|
|
1072
|
+
# half-open lease so the admission gate cannot
|
|
1073
|
+
# read it as an in-flight probe after this
|
|
1074
|
+
# cooldown expires.
|
|
1075
|
+
state.half_open_until = 0.0
|
|
1076
|
+
state.persisted_updated_at_epoch = persisted.updated_at_epoch
|
|
1077
|
+
state.persisted_admission_generation = getattr(persisted, "admission_generation", 0)
|
|
1078
|
+
# Post-write time, not this persist's entry time: a
|
|
1079
|
+
# load that started while the durable write was in
|
|
1080
|
+
# flight can carry a start stamp later than the entry
|
|
1081
|
+
# time while its row snapshot predates the write, and
|
|
1082
|
+
# the stale-load guard compares against this value.
|
|
1083
|
+
state.last_durable_load_monotonic = max(
|
|
1084
|
+
state.last_durable_load_monotonic,
|
|
1085
|
+
clock_for(self).monotonic(),
|
|
1086
|
+
)
|
|
1087
|
+
async with self._http_bridge_retry_circuit_lock:
|
|
1088
|
+
if self._http_bridge_retry_circuits.get(session.key) is state:
|
|
1089
|
+
self._http_bridge_retry_circuit_persisted_keys.add(session.key)
|
|
1090
|
+
except Exception:
|
|
1091
|
+
if PROMETHEUS_AVAILABLE and http_bridge_retry_circuit_total is not None:
|
|
1092
|
+
http_bridge_retry_circuit_total.labels(outcome="persist_failed").inc()
|
|
1093
|
+
logger.warning(
|
|
1094
|
+
"Failed to persist HTTP bridge retry circuit bridge_kind=%s bridge_key=%s",
|
|
1095
|
+
session.key.affinity_kind,
|
|
1096
|
+
_hash_identifier(session.key.affinity_key),
|
|
1097
|
+
exc_info=True,
|
|
1098
|
+
)
|
|
1099
|
+
|
|
1100
|
+
async def _http_bridge_precreated_retry_allowed(
|
|
1101
|
+
self: Any,
|
|
1102
|
+
session: _HTTPBridgeSession,
|
|
1103
|
+
*,
|
|
1104
|
+
allow_fresh_hard_account_switch: bool = False,
|
|
1105
|
+
allow_proof_gated_continuity_replay: bool = False,
|
|
1106
|
+
claimed_lease_out: list[float] | None = None,
|
|
1107
|
+
) -> bool:
|
|
1108
|
+
"""Avoid replaying a repeatedly failing hard-affinity request in a tight loop."""
|
|
1109
|
+
if session.key.strength != "hard":
|
|
1110
|
+
return True
|
|
1111
|
+
|
|
1112
|
+
await self._load_http_bridge_retry_circuit(session)
|
|
1113
|
+
now = clock_for(self).monotonic()
|
|
1114
|
+
async with self._http_bridge_retry_circuit_lock:
|
|
1115
|
+
state = self._http_bridge_retry_circuits.get(session.key)
|
|
1116
|
+
if state is None or state.cooldown_until <= now:
|
|
1117
|
+
if (
|
|
1118
|
+
state is not None
|
|
1119
|
+
and state.consecutive_failures >= _HTTP_BRIDGE_RETRY_CIRCUIT_FAILURE_THRESHOLD
|
|
1120
|
+
and state.half_open_until > now
|
|
1121
|
+
and not allow_fresh_hard_account_switch
|
|
1122
|
+
and not allow_proof_gated_continuity_replay
|
|
1123
|
+
):
|
|
1124
|
+
if PROMETHEUS_AVAILABLE and http_bridge_retry_circuit_total is not None:
|
|
1125
|
+
http_bridge_retry_circuit_total.labels(outcome="suppressed").inc()
|
|
1126
|
+
return False
|
|
1127
|
+
if state is not None and state.cooldown_until > 0:
|
|
1128
|
+
state.cooldown_until = 0.0
|
|
1129
|
+
state.half_open_until = now + _HTTP_BRIDGE_RETRY_CIRCUIT_HALF_OPEN_LEASE_SECONDS
|
|
1130
|
+
if claimed_lease_out is not None:
|
|
1131
|
+
# The exact lease this admission claimed, handed out
|
|
1132
|
+
# under the same lock that installed it, so a
|
|
1133
|
+
# fail-closed caller can return precisely its own
|
|
1134
|
+
# probe under any interleaving.
|
|
1135
|
+
claimed_lease_out.append(state.half_open_until)
|
|
1136
|
+
logger.info(
|
|
1137
|
+
"http_bridge_retry_circuit event=half_open bridge_kind=%s bridge_key=%s failures=%s",
|
|
1138
|
+
session.key.affinity_kind,
|
|
1139
|
+
_hash_identifier(session.key.affinity_key),
|
|
1140
|
+
state.consecutive_failures,
|
|
1141
|
+
)
|
|
1142
|
+
return True
|
|
1143
|
+
|
|
1144
|
+
retry_after = max(0.0, state.cooldown_until - now)
|
|
1145
|
+
if allow_fresh_hard_account_switch:
|
|
1146
|
+
logger.info(
|
|
1147
|
+
"http_bridge_retry_circuit event=bypass_fresh_account_switch bridge_kind=%s "
|
|
1148
|
+
"bridge_key=%s failures=%s retry_after_seconds=%.1f",
|
|
1149
|
+
session.key.affinity_kind,
|
|
1150
|
+
_hash_identifier(session.key.affinity_key),
|
|
1151
|
+
state.consecutive_failures,
|
|
1152
|
+
retry_after,
|
|
1153
|
+
)
|
|
1154
|
+
return True
|
|
1155
|
+
if allow_proof_gated_continuity_replay:
|
|
1156
|
+
logger.info(
|
|
1157
|
+
"http_bridge_retry_circuit event=bypass_proof_gated_continuity_replay bridge_kind=%s "
|
|
1158
|
+
"bridge_key=%s failures=%s retry_after_seconds=%.1f",
|
|
1159
|
+
session.key.affinity_kind,
|
|
1160
|
+
_hash_identifier(session.key.affinity_key),
|
|
1161
|
+
state.consecutive_failures,
|
|
1162
|
+
retry_after,
|
|
1163
|
+
)
|
|
1164
|
+
return True
|
|
1165
|
+
if PROMETHEUS_AVAILABLE and http_bridge_retry_circuit_total is not None:
|
|
1166
|
+
http_bridge_retry_circuit_total.labels(outcome="suppressed").inc()
|
|
1167
|
+
logger.info(
|
|
1168
|
+
"http_bridge_retry_circuit event=suppressed bridge_kind=%s bridge_key=%s "
|
|
1169
|
+
"failures=%s retry_after_seconds=%.1f detail=%s",
|
|
1170
|
+
session.key.affinity_kind,
|
|
1171
|
+
_hash_identifier(session.key.affinity_key),
|
|
1172
|
+
state.consecutive_failures,
|
|
1173
|
+
retry_after,
|
|
1174
|
+
state.last_detail,
|
|
1175
|
+
)
|
|
1176
|
+
return False
|
|
1177
|
+
|
|
1178
|
+
async def _http_bridge_poison_anchor_clear_owed(
|
|
1179
|
+
self: Any,
|
|
1180
|
+
session: _HTTPBridgeSession,
|
|
1181
|
+
*,
|
|
1182
|
+
consecutive_failures: int | None,
|
|
1183
|
+
) -> "tuple[_HTTPBridgeRetryCircuitState | None, object]":
|
|
1184
|
+
"""Return the owed episode and the durable anchor captured with it.
|
|
1185
|
+
|
|
1186
|
+
Returns ``(episode, captured_anchor)`` — the validated registered
|
|
1187
|
+
episode (or ``None`` when nothing is owed) and the durable session's
|
|
1188
|
+
continuity anchor sampled before the validation reads. The caller
|
|
1189
|
+
passes exactly this episode to the post-abandonment marker and fences
|
|
1190
|
+
the continuity clear on the captured anchor: with the completion path
|
|
1191
|
+
settling the circuit before it registers a fresh anchor, any
|
|
1192
|
+
completion this consult did not veto changes the anchor after this
|
|
1193
|
+
capture, and the fenced clear then matches nothing.
|
|
1194
|
+
|
|
1195
|
+
The poison threshold is the circuit threshold, which is what makes the
|
|
1196
|
+
clear reachable at all, but it also means every later strike in the
|
|
1197
|
+
same episode meets the threshold too. Only the first successful
|
|
1198
|
+
abandonment settles the anchor; a failed one leaves the marker unset so
|
|
1199
|
+
the next strike retries it, which is the contract the retirement
|
|
1200
|
+
funnels already advertise in their failure telemetry.
|
|
1201
|
+
|
|
1202
|
+
The decision reads the currently registered episode, not the caller's
|
|
1203
|
+
captured count. Between the strike and this consult a multiplexed
|
|
1204
|
+
sibling can complete, settle the circuit, and persist a fresh valid
|
|
1205
|
+
anchor; the caller's detached state still reports the old count, and
|
|
1206
|
+
clearing on it would delete the anchor the sibling just stored. An
|
|
1207
|
+
absent or below-threshold registered episode therefore owes nothing.
|
|
1208
|
+
"""
|
|
1209
|
+
if consecutive_failures is None:
|
|
1210
|
+
return None, _POISON_ANCHOR_CAPTURE_UNAVAILABLE
|
|
1211
|
+
effective_threshold = max(1, _HTTP_BRIDGE_RETRY_CIRCUIT_FAILURE_THRESHOLD)
|
|
1212
|
+
async with self._http_bridge_retry_circuit_lock:
|
|
1213
|
+
state = self._http_bridge_retry_circuits.get(session.key)
|
|
1214
|
+
live_episode_owes = (
|
|
1215
|
+
state is not None
|
|
1216
|
+
and state.consecutive_failures >= effective_threshold
|
|
1217
|
+
and not state.poison_anchor_cleared
|
|
1218
|
+
)
|
|
1219
|
+
if not live_episode_owes:
|
|
1220
|
+
return None, _POISON_ANCHOR_CAPTURE_UNAVAILABLE
|
|
1221
|
+
# An abandonment without a captured fence is refused outright: an
|
|
1222
|
+
# unfenced clear on this degraded path could delete a fresh anchor a
|
|
1223
|
+
# sibling registered after the recheck below. A later strike retries
|
|
1224
|
+
# the consult with a working capture.
|
|
1225
|
+
anchor_reader = getattr(self._durable_bridge, "session_latest_continuity", None)
|
|
1226
|
+
durable_session_id = getattr(session, "durable_session_id", None)
|
|
1227
|
+
if anchor_reader is None or durable_session_id is None:
|
|
1228
|
+
return None, _POISON_ANCHOR_CAPTURE_UNAVAILABLE
|
|
1229
|
+
try:
|
|
1230
|
+
expected_anchor = await anchor_reader(session_id=durable_session_id)
|
|
1231
|
+
except Exception:
|
|
1232
|
+
logger.warning(
|
|
1233
|
+
"Failed to capture durable anchor before poison clear bridge_kind=%s bridge_key=%s",
|
|
1234
|
+
session.key.affinity_kind,
|
|
1235
|
+
_hash_identifier(session.key.affinity_key),
|
|
1236
|
+
exc_info=True,
|
|
1237
|
+
)
|
|
1238
|
+
return None, _POISON_ANCHOR_CAPTURE_UNAVAILABLE
|
|
1239
|
+
if expected_anchor is None:
|
|
1240
|
+
# The durable session row itself is gone; there is no continuity
|
|
1241
|
+
# to clear and no columns for a fence to protect.
|
|
1242
|
+
return None, _POISON_ANCHOR_CAPTURE_UNAVAILABLE
|
|
1243
|
+
if expected_anchor == (None, None):
|
|
1244
|
+
# Both continuity columns are already empty: there is no anchor
|
|
1245
|
+
# left to abandon, so a clear here removes no failure cause.
|
|
1246
|
+
# Authorizing it anyway would let the settle-on-abandon path
|
|
1247
|
+
# reset a circuit that is cooling on genuinely unanchored
|
|
1248
|
+
# upstream failures, and that cooldown is the only protection
|
|
1249
|
+
# those failures have.
|
|
1250
|
+
return None, _POISON_ANCHOR_CAPTURE_UNAVAILABLE
|
|
1251
|
+
# The one-clear marker lives in process memory, so a restart or
|
|
1252
|
+
# another replica cannot see it. The durable circuit row is the
|
|
1253
|
+
# episode's replica-visible record instead: a completed response
|
|
1254
|
+
# settles the circuit and deletes the row, and the fresh anchor that
|
|
1255
|
+
# completion persisted must not be deleted by a stale local episode
|
|
1256
|
+
# that survived the reset. The settle updates the row to zero rather
|
|
1257
|
+
# than deleting it, so a reset row proves the episode ended exactly
|
|
1258
|
+
# as an absent one does; only a row still at the effective threshold
|
|
1259
|
+
# is a live episode. A lookup failure proves nothing either, and the
|
|
1260
|
+
# next strike retries.
|
|
1261
|
+
try:
|
|
1262
|
+
persisted = await self._durable_bridge.lookup_retry_circuit(
|
|
1263
|
+
session_key_kind=session.key.affinity_kind,
|
|
1264
|
+
session_key_value=session.key.affinity_key,
|
|
1265
|
+
api_key_id=session.key.api_key_id,
|
|
1266
|
+
)
|
|
1267
|
+
except Exception:
|
|
1268
|
+
logger.warning(
|
|
1269
|
+
"Failed to confirm durable retry-circuit episode before anchor clear bridge_kind=%s bridge_key=%s",
|
|
1270
|
+
session.key.affinity_kind,
|
|
1271
|
+
_hash_identifier(session.key.affinity_key),
|
|
1272
|
+
exc_info=True,
|
|
1273
|
+
)
|
|
1274
|
+
return None, _POISON_ANCHOR_CAPTURE_UNAVAILABLE
|
|
1275
|
+
if persisted is None or persisted.consecutive_failures < effective_threshold:
|
|
1276
|
+
return None, _POISON_ANCHOR_CAPTURE_UNAVAILABLE
|
|
1277
|
+
# The row must be a poison episode of its own: another replica can
|
|
1278
|
+
# reset the old episode, register a fresh anchor, and open a new
|
|
1279
|
+
# at-threshold episode on clean_close, and a count-only check would
|
|
1280
|
+
# let the stale local episode clear that fresh anchor. An episode
|
|
1281
|
+
# that still OWES a poison clear stays authorized even after a later
|
|
1282
|
+
# clean probe failure overwrote the durable detail — fenced on the
|
|
1283
|
+
# exact reconciled lineage, so a replaced row never inherits the
|
|
1284
|
+
# stale owed state.
|
|
1285
|
+
row_detail_is_poison = _http_bridge_anchor_poison_detail(persisted.last_detail) is not None
|
|
1286
|
+
# Re-check the live episode after the durable await: a multiplexed
|
|
1287
|
+
# sibling can complete, settle the registry, and reset the row while
|
|
1288
|
+
# the lookup was in flight, and the stale snapshot must not authorize
|
|
1289
|
+
# a clear against the fresh anchor that completion persisted.
|
|
1290
|
+
async with self._http_bridge_retry_circuit_lock:
|
|
1291
|
+
state = self._http_bridge_retry_circuits.get(session.key)
|
|
1292
|
+
if (
|
|
1293
|
+
state is not None
|
|
1294
|
+
and state.consecutive_failures >= effective_threshold
|
|
1295
|
+
and not state.poison_anchor_cleared
|
|
1296
|
+
and (
|
|
1297
|
+
row_detail_is_poison
|
|
1298
|
+
or (
|
|
1299
|
+
state.owed_poison_detail is not None
|
|
1300
|
+
and persisted.updated_at_epoch == state.persisted_updated_at_epoch
|
|
1301
|
+
)
|
|
1302
|
+
)
|
|
1303
|
+
):
|
|
1304
|
+
if row_detail_is_poison:
|
|
1305
|
+
# The consulted poison row IS the authorization: the
|
|
1306
|
+
# abandonment fences from this episode snapshot, and a
|
|
1307
|
+
# stale local identity — an unpersisted write (epoch
|
|
1308
|
+
# zero) or a cross-replica strike that moved the row
|
|
1309
|
+
# after this worker's own — would make both settlement
|
|
1310
|
+
# attempts reject the newer fence and leave the removed
|
|
1311
|
+
# anchor's cooldown and quarantine standing. Adopt the
|
|
1312
|
+
# row's epoch, admission generation, and higher count
|
|
1313
|
+
# unconditionally.
|
|
1314
|
+
state.persisted_updated_at_epoch = persisted.updated_at_epoch
|
|
1315
|
+
state.persisted_admission_generation = getattr(persisted, "admission_generation", 0)
|
|
1316
|
+
if persisted.consecutive_failures > state.consecutive_failures:
|
|
1317
|
+
state.consecutive_failures = persisted.consecutive_failures
|
|
1318
|
+
return state, expected_anchor
|
|
1319
|
+
return None, _POISON_ANCHOR_CAPTURE_UNAVAILABLE
|
|
1320
|
+
|
|
1321
|
+
async def _http_bridge_effective_anchor_poison_detail(
|
|
1322
|
+
self: Any,
|
|
1323
|
+
session: _HTTPBridgeSession,
|
|
1324
|
+
detail: str | None,
|
|
1325
|
+
) -> str | None:
|
|
1326
|
+
"""Map a strike to its poison class, falling back to the owed one.
|
|
1327
|
+
|
|
1328
|
+
A failed abandonment leaves the episode's clear owed, and the spec
|
|
1329
|
+
promises the NEXT strike retries it — not only the next poison-class
|
|
1330
|
+
strike. A clean probe failure would otherwise overwrite the durable
|
|
1331
|
+
detail and strand the owed clear forever.
|
|
1332
|
+
"""
|
|
1333
|
+
mapped = _http_bridge_anchor_poison_detail(detail)
|
|
1334
|
+
if mapped is not None:
|
|
1335
|
+
return mapped
|
|
1336
|
+
async with self._http_bridge_retry_circuit_lock:
|
|
1337
|
+
state = self._http_bridge_retry_circuits.get(session.key)
|
|
1338
|
+
owed = state.owed_poison_detail if state is not None else None
|
|
1339
|
+
return _http_bridge_anchor_poison_detail(owed)
|
|
1340
|
+
|
|
1341
|
+
async def _http_bridge_mark_poison_anchor_cleared(
|
|
1342
|
+
self: Any,
|
|
1343
|
+
session: _HTTPBridgeSession,
|
|
1344
|
+
*,
|
|
1345
|
+
episode: _HTTPBridgeRetryCircuitState,
|
|
1346
|
+
) -> None:
|
|
1347
|
+
"""Record that this episode's poisoned anchor has been abandoned.
|
|
1348
|
+
|
|
1349
|
+
The marker belongs to the episode that performed the abandonment. If
|
|
1350
|
+
that episode was settled and replaced while the durable rebind was in
|
|
1351
|
+
flight, marking whatever is registered now would suppress the new
|
|
1352
|
+
episode's own required abandonment and leave its poisoned anchor
|
|
1353
|
+
reusable, so a caller that captured its episode marks only that one.
|
|
1354
|
+
"""
|
|
1355
|
+
async with self._http_bridge_retry_circuit_lock:
|
|
1356
|
+
state = self._http_bridge_retry_circuits.get(session.key)
|
|
1357
|
+
if state is not None and state is episode:
|
|
1358
|
+
state.poison_anchor_cleared = True
|
|
1359
|
+
state.owed_poison_detail = None
|
|
1360
|
+
|
|
1361
|
+
async def _http_bridge_precreated_retry_block(
|
|
1362
|
+
self: Any,
|
|
1363
|
+
session: _HTTPBridgeSession,
|
|
1364
|
+
) -> tuple[float, str]:
|
|
1365
|
+
"""Return ``(seconds_blocked, reason)`` for a suppressed submission.
|
|
1366
|
+
|
|
1367
|
+
The cooldown is not always what is refusing the request: once it has
|
|
1368
|
+
expired, the half-open lease keeps refusing everything except the one
|
|
1369
|
+
admitted probe. Reporting the cooldown in that window advertises a
|
|
1370
|
+
retry-after of ~1s while the caller is barred for the rest of the
|
|
1371
|
+
lease, which turns a wedged key into a client retry storm.
|
|
1372
|
+
"""
|
|
1373
|
+
if session.key.strength != "hard":
|
|
1374
|
+
return 0.0, "none"
|
|
1375
|
+
|
|
1376
|
+
await self._load_http_bridge_retry_circuit(session)
|
|
1377
|
+
now = clock_for(self).monotonic()
|
|
1378
|
+
async with self._http_bridge_retry_circuit_lock:
|
|
1379
|
+
state = self._http_bridge_retry_circuits.get(session.key)
|
|
1380
|
+
if state is None:
|
|
1381
|
+
return 0.0, "none"
|
|
1382
|
+
cooldown_remaining = max(0.0, state.cooldown_until - now)
|
|
1383
|
+
half_open_remaining = (
|
|
1384
|
+
max(0.0, state.half_open_until - now)
|
|
1385
|
+
if state.consecutive_failures >= _HTTP_BRIDGE_RETRY_CIRCUIT_FAILURE_THRESHOLD
|
|
1386
|
+
else 0.0
|
|
1387
|
+
)
|
|
1388
|
+
if half_open_remaining > cooldown_remaining:
|
|
1389
|
+
return half_open_remaining, "hard_key_half_open"
|
|
1390
|
+
return cooldown_remaining, "hard_key_cooldown"
|
|
1391
|
+
|
|
1392
|
+
async def _http_bridge_precreated_retry_block_for_key(
|
|
1393
|
+
self: Any,
|
|
1394
|
+
key: _HTTPBridgeSessionKey,
|
|
1395
|
+
*,
|
|
1396
|
+
assume_remote_half_open_lease: bool = False,
|
|
1397
|
+
) -> tuple[float, str]:
|
|
1398
|
+
"""Return ``(seconds_blocked, reason)`` for a suppressed replacement.
|
|
1399
|
+
|
|
1400
|
+
The same-owner stale-anchor recovery dispatches on a unique internal
|
|
1401
|
+
session while the circuit that refused it lives on the source hard
|
|
1402
|
+
key, so the block has to be read from that key. During the half-open
|
|
1403
|
+
lease the cooldown is already zero; reading only the replacement
|
|
1404
|
+
session (which has no circuit) and the source key's cooldown
|
|
1405
|
+
advertises a ~1s retry-after while the caller is barred for the rest
|
|
1406
|
+
of the lease.
|
|
1407
|
+
|
|
1408
|
+
``assume_remote_half_open_lease`` covers the timer this process
|
|
1409
|
+
cannot see: a caller that just lost its dispatch claim was refused by
|
|
1410
|
+
a probe admitted somewhere, and when neither a local cooldown nor a
|
|
1411
|
+
local lease is visible that probe's lease lives in another process
|
|
1412
|
+
(or died with a replaced local state). The lease deadline is not
|
|
1413
|
+
persisted, so the block is reported at the configured lease duration
|
|
1414
|
+
— the upper bound of the timer actually refusing the caller — rather
|
|
1415
|
+
than a fabricated ~1s.
|
|
1416
|
+
"""
|
|
1417
|
+
if key.strength != "hard":
|
|
1418
|
+
return 0.0, "none"
|
|
1419
|
+
now = clock_for(self).monotonic()
|
|
1420
|
+
async with self._http_bridge_retry_circuit_lock:
|
|
1421
|
+
state = self._http_bridge_retry_circuits.get(key)
|
|
1422
|
+
local_cooldown_remaining = max(0.0, state.cooldown_until - now) if state is not None else 0.0
|
|
1423
|
+
half_open_remaining = (
|
|
1424
|
+
max(0.0, state.half_open_until - now)
|
|
1425
|
+
if state is not None and state.consecutive_failures >= _HTTP_BRIDGE_RETRY_CIRCUIT_FAILURE_THRESHOLD
|
|
1426
|
+
else 0.0
|
|
1427
|
+
)
|
|
1428
|
+
cooldown_remaining = max(
|
|
1429
|
+
local_cooldown_remaining,
|
|
1430
|
+
await self._http_bridge_retry_circuit_cooldown_seconds_for_key(key),
|
|
1431
|
+
)
|
|
1432
|
+
if cooldown_remaining <= 0.0 and half_open_remaining <= 0.0:
|
|
1433
|
+
if assume_remote_half_open_lease:
|
|
1434
|
+
return _HTTP_BRIDGE_RETRY_CIRCUIT_HALF_OPEN_LEASE_SECONDS, "hard_key_half_open"
|
|
1435
|
+
return 0.0, "none"
|
|
1436
|
+
if half_open_remaining > cooldown_remaining:
|
|
1437
|
+
return half_open_remaining, "hard_key_half_open"
|
|
1438
|
+
return cooldown_remaining, "hard_key_cooldown"
|
|
1439
|
+
|
|
1440
|
+
async def _http_bridge_precreated_retry_cooldown_seconds(self: Any, session: _HTTPBridgeSession) -> float:
|
|
1441
|
+
if session.key.strength != "hard":
|
|
1442
|
+
return 0.0
|
|
1443
|
+
|
|
1444
|
+
await self._load_http_bridge_retry_circuit(session)
|
|
1445
|
+
now = clock_for(self).monotonic()
|
|
1446
|
+
async with self._http_bridge_retry_circuit_lock:
|
|
1447
|
+
state = self._http_bridge_retry_circuits.get(session.key)
|
|
1448
|
+
if state is None:
|
|
1449
|
+
return 0.0
|
|
1450
|
+
return max(0.0, state.cooldown_until - now)
|
|
1451
|
+
|
|
1452
|
+
async def _http_bridge_retry_circuit_cooldown_seconds_for_key(
|
|
1453
|
+
self: Any,
|
|
1454
|
+
key: _HTTPBridgeSessionKey,
|
|
1455
|
+
) -> float:
|
|
1456
|
+
"""Return the source-key cooldown used to suppress a replacement."""
|
|
1457
|
+
load_succeeded, generation = await self._http_bridge_retry_circuit_generation_for_key(key)
|
|
1458
|
+
if not load_succeeded or generation is None:
|
|
1459
|
+
return 0.0
|
|
1460
|
+
return max(
|
|
1461
|
+
0.0,
|
|
1462
|
+
generation[3] - clock_for(self).time(),
|
|
1463
|
+
generation[6] - clock_for(self).monotonic(),
|
|
1464
|
+
)
|
|
1465
|
+
|
|
1466
|
+
async def _record_http_bridge_retry_circuit_failure(
|
|
1467
|
+
self: Any,
|
|
1468
|
+
session: _HTTPBridgeSession,
|
|
1469
|
+
*,
|
|
1470
|
+
detail: str,
|
|
1471
|
+
attempt: _HTTPBridgeResponseCreateAttempt | None = None,
|
|
1472
|
+
terminal_pre_response_frame: bool = False,
|
|
1473
|
+
) -> int | None:
|
|
1474
|
+
"""Count one hard-key failure against the retry circuit.
|
|
1475
|
+
|
|
1476
|
+
``terminal_pre_response_frame`` is asserted only by the settlement path
|
|
1477
|
+
for an upstream terminal frame that failed the request before any
|
|
1478
|
+
response event. ``response.failed``/``response.incomplete`` mark the
|
|
1479
|
+
attempt observed without counting a response event, so without that
|
|
1480
|
+
assertion a native terminal envelope is rejected below as already
|
|
1481
|
+
settled and never consumes a strike. The eventless retirement funnel
|
|
1482
|
+
never asserts it, so its "upstream answered" guard is unchanged.
|
|
1483
|
+
"""
|
|
1484
|
+
detail = _HTTP_BRIDGE_RETRY_CIRCUIT_DETAIL_ALIASES.get(detail, detail)
|
|
1485
|
+
if session.key.strength != "hard" or detail not in _HTTP_BRIDGE_RETRY_CIRCUIT_FAILURE_DETAILS:
|
|
1486
|
+
return None
|
|
1487
|
+
|
|
1488
|
+
scoped_attempt = attempt
|
|
1489
|
+
if scoped_attempt is not None:
|
|
1490
|
+
if scoped_attempt.retry_circuit_failure_recorded:
|
|
1491
|
+
return await self._await_http_bridge_retry_circuit_attempt_settlement(
|
|
1492
|
+
session,
|
|
1493
|
+
attempt=scoped_attempt,
|
|
1494
|
+
detail=detail,
|
|
1495
|
+
)
|
|
1496
|
+
if (
|
|
1497
|
+
scoped_attempt.disarmed
|
|
1498
|
+
or (scoped_attempt.response_observed and not terminal_pre_response_frame)
|
|
1499
|
+
or (terminal_pre_response_frame and scoped_attempt.non_terminal_response_observed)
|
|
1500
|
+
):
|
|
1501
|
+
return None
|
|
1502
|
+
|
|
1503
|
+
threshold = max(1, _HTTP_BRIDGE_RETRY_CIRCUIT_FAILURE_THRESHOLD)
|
|
1504
|
+
base_backoff = max(0.001, _HTTP_BRIDGE_RETRY_CIRCUIT_BASE_BACKOFF_SECONDS)
|
|
1505
|
+
max_backoff = max(base_backoff, _HTTP_BRIDGE_RETRY_CIRCUIT_MAX_BACKOFF_SECONDS)
|
|
1506
|
+
clean_close_max_backoff = max(0.001, _HTTP_BRIDGE_RETRY_CIRCUIT_CLEAN_CLOSE_MAX_BACKOFF_SECONDS)
|
|
1507
|
+
duplicate_attempt: _HTTPBridgeResponseCreateAttempt | None = None
|
|
1508
|
+
state: _HTTPBridgeRetryCircuitState | None = None
|
|
1509
|
+
poison_class_failure = _http_bridge_anchor_poison_detail(detail) is not None
|
|
1510
|
+
quarantine_poisoned_anchor = False
|
|
1511
|
+
quarantine_cooldown_remaining = 0.0
|
|
1512
|
+
recorded_failures: int | None = None
|
|
1513
|
+
# The registry mutation and its durable write hold the key lock
|
|
1514
|
+
# together: a settle for this key either completes before the strike
|
|
1515
|
+
# registers (the strike then opens a fresh episode) or waits until
|
|
1516
|
+
# the strike's write has landed. Without this, a failure recorded
|
|
1517
|
+
# after a settlement linearized could be dropped as superseded and
|
|
1518
|
+
# disappear instead of opening the circuit.
|
|
1519
|
+
key_lock = await self._acquire_http_bridge_retry_circuit_key_lock(session.key)
|
|
1520
|
+
try:
|
|
1521
|
+
# The load runs under the key lock too: outside it, a settle in
|
|
1522
|
+
# flight lets the load re-hydrate the just-popped counts from the
|
|
1523
|
+
# not-yet-reset row, and this strike would then extend the ended
|
|
1524
|
+
# episode instead of opening a fresh one.
|
|
1525
|
+
await self._load_http_bridge_retry_circuit(session)
|
|
1526
|
+
# Sampled after the keyed wait and the load: a wait approaching
|
|
1527
|
+
# the base backoff would otherwise persist an already-aged
|
|
1528
|
+
# cooldown and make the fresh failure look older than the
|
|
1529
|
+
# durable load for merge bookkeeping.
|
|
1530
|
+
now = clock_for(self).monotonic()
|
|
1531
|
+
async with self._http_bridge_retry_circuit_lock:
|
|
1532
|
+
if scoped_attempt is not None and scoped_attempt.retry_circuit_failure_recorded:
|
|
1533
|
+
duplicate_attempt = scoped_attempt
|
|
1534
|
+
elif scoped_attempt is not None and (
|
|
1535
|
+
scoped_attempt.disarmed
|
|
1536
|
+
or (scoped_attempt.response_observed and not terminal_pre_response_frame)
|
|
1537
|
+
# ``terminal_pre_response_frame`` exists because the terminal
|
|
1538
|
+
# frame itself marks the attempt observed. It must not also
|
|
1539
|
+
# steamroll genuine midstream evidence: a deferred-reasoning
|
|
1540
|
+
# prelude observed a non-terminal response event without
|
|
1541
|
+
# counting it, so a later terminal failure is a midstream
|
|
1542
|
+
# failure, not a pre-response strike.
|
|
1543
|
+
or (terminal_pre_response_frame and scoped_attempt.non_terminal_response_observed)
|
|
1544
|
+
):
|
|
1545
|
+
return None
|
|
1546
|
+
else:
|
|
1547
|
+
state = self._http_bridge_retry_circuits.setdefault(
|
|
1548
|
+
session.key,
|
|
1549
|
+
_HTTPBridgeRetryCircuitState(last_touched_monotonic=now),
|
|
1550
|
+
)
|
|
1551
|
+
state.last_touched_monotonic = now
|
|
1552
|
+
state.last_failure_monotonic = now
|
|
1553
|
+
state.half_open_until = 0.0
|
|
1554
|
+
if scoped_attempt is not None:
|
|
1555
|
+
scoped_attempt.retry_circuit_failure_recorded = True
|
|
1556
|
+
scoped_attempt.retry_circuit_failure_settled = anyio.Event()
|
|
1557
|
+
state.consecutive_failures += 1
|
|
1558
|
+
prior_recorded_detail = state.last_detail
|
|
1559
|
+
state.last_detail = detail
|
|
1560
|
+
if _http_bridge_anchor_poison_detail(detail) is not None:
|
|
1561
|
+
if prior_recorded_detail == _HTTP_BRIDGE_RETRY_CIRCUIT_ANCHOR_SUPERSEDED_DETAIL:
|
|
1562
|
+
# The supersession marked the OLD anchor's clear
|
|
1563
|
+
# as no longer owed; this strike is evidence
|
|
1564
|
+
# against the freshly registered anchor and
|
|
1565
|
+
# starts a new abandonment story of its own.
|
|
1566
|
+
state.poison_anchor_cleared = False
|
|
1567
|
+
if state.consecutive_failures >= threshold:
|
|
1568
|
+
# Debt exists only for an episode that reached the
|
|
1569
|
+
# poison threshold on poison evidence — the same
|
|
1570
|
+
# threshold that authorizes the abandonment, so
|
|
1571
|
+
# the strike that opens the circuit arms the debt
|
|
1572
|
+
# its failed clear leaves owed.
|
|
1573
|
+
# Arming below it let a clean_close opener
|
|
1574
|
+
# resurrect an earlier poison detail and clear a
|
|
1575
|
+
# valid anchor with no quarantine covering the
|
|
1576
|
+
# race.
|
|
1577
|
+
state.owed_poison_detail = detail
|
|
1578
|
+
if state.consecutive_failures >= threshold:
|
|
1579
|
+
backoff = min(
|
|
1580
|
+
max_backoff,
|
|
1581
|
+
base_backoff * (2 ** min(state.consecutive_failures - threshold, 30)),
|
|
1582
|
+
)
|
|
1583
|
+
if detail == "clean_close":
|
|
1584
|
+
backoff = min(backoff, clean_close_max_backoff)
|
|
1585
|
+
state.cooldown_until = max(state.cooldown_until, now + backoff)
|
|
1586
|
+
# The probe admitted after this cooldown is planned before
|
|
1587
|
+
# it reaches the gate, so an anchor the circuit opened on
|
|
1588
|
+
# has to be suppressed at planning time. Quarantining the
|
|
1589
|
+
# key routes a full-resend probe through the existing
|
|
1590
|
+
# unanchored fresh path; delta-only payloads keep their
|
|
1591
|
+
# anchor there, because it is their only context.
|
|
1592
|
+
quarantine_poisoned_anchor = poison_class_failure
|
|
1593
|
+
quarantine_cooldown_remaining = max(0.0, state.cooldown_until - now)
|
|
1594
|
+
if PROMETHEUS_AVAILABLE and http_bridge_retry_circuit_total is not None:
|
|
1595
|
+
http_bridge_retry_circuit_total.labels(outcome="opened").inc()
|
|
1596
|
+
logger.warning(
|
|
1597
|
+
"http_bridge_retry_circuit event=opened bridge_kind=%s bridge_key=%s "
|
|
1598
|
+
"failures=%s cooldown_seconds=%.1f detail=%s",
|
|
1599
|
+
session.key.affinity_kind,
|
|
1600
|
+
_hash_identifier(session.key.affinity_key),
|
|
1601
|
+
state.consecutive_failures,
|
|
1602
|
+
backoff,
|
|
1603
|
+
detail,
|
|
1604
|
+
)
|
|
1605
|
+
if duplicate_attempt is None:
|
|
1606
|
+
assert state is not None
|
|
1607
|
+
armed_quarantine_generation: int | None = None
|
|
1608
|
+
pre_arm_quarantine_reason: str | None = None
|
|
1609
|
+
pre_arm_quarantine_until = 0.0
|
|
1610
|
+
if quarantine_poisoned_anchor:
|
|
1611
|
+
# A revocation after a lost cross-replica race must not
|
|
1612
|
+
# evict a weaker fence this arm merely upgraded, so the
|
|
1613
|
+
# prior active entry is captured for restoration.
|
|
1614
|
+
prior_entry = _http_bridge_quarantine_registry(self).get(session.key)
|
|
1615
|
+
if (
|
|
1616
|
+
prior_entry is not None
|
|
1617
|
+
and prior_entry.quarantined_until > clock_for(self).monotonic()
|
|
1618
|
+
and prior_entry.reason != _HTTP_BRIDGE_QUARANTINE_POISONED_ANCHOR_REASON
|
|
1619
|
+
):
|
|
1620
|
+
pre_arm_quarantine_reason = prior_entry.reason
|
|
1621
|
+
pre_arm_quarantine_until = prior_entry.quarantined_until
|
|
1622
|
+
_quarantine_http_bridge_session(
|
|
1623
|
+
self,
|
|
1624
|
+
session,
|
|
1625
|
+
reason=_HTTP_BRIDGE_QUARANTINE_POISONED_ANCHOR_REASON,
|
|
1626
|
+
minimum_seconds=_http_bridge_poison_quarantine_minimum_seconds(quarantine_cooldown_remaining),
|
|
1627
|
+
)
|
|
1628
|
+
# Captured so a lost cross-replica race can revoke exactly
|
|
1629
|
+
# this speculative arm and nothing re-armed after it.
|
|
1630
|
+
armed_quarantine_generation = _http_bridge_quarantine_generation(self, session.key)
|
|
1631
|
+
recorded_failures = await self._record_http_bridge_retry_circuit_failure_locked(
|
|
1632
|
+
session,
|
|
1633
|
+
state,
|
|
1634
|
+
scoped_attempt=scoped_attempt,
|
|
1635
|
+
threshold=threshold,
|
|
1636
|
+
quarantine_poisoned_anchor=quarantine_poisoned_anchor,
|
|
1637
|
+
quarantine_cooldown_remaining=quarantine_cooldown_remaining,
|
|
1638
|
+
armed_quarantine_generation=armed_quarantine_generation,
|
|
1639
|
+
pre_arm_quarantine_reason=pre_arm_quarantine_reason,
|
|
1640
|
+
pre_arm_quarantine_until=pre_arm_quarantine_until,
|
|
1641
|
+
)
|
|
1642
|
+
finally:
|
|
1643
|
+
key_lock.release()
|
|
1644
|
+
if duplicate_attempt is not None:
|
|
1645
|
+
return await self._await_http_bridge_retry_circuit_attempt_settlement(
|
|
1646
|
+
session,
|
|
1647
|
+
attempt=duplicate_attempt,
|
|
1648
|
+
detail=detail,
|
|
1649
|
+
)
|
|
1650
|
+
return recorded_failures
|
|
1651
|
+
|
|
1652
|
+
async def _record_http_bridge_retry_circuit_failure_locked(
|
|
1653
|
+
self: Any,
|
|
1654
|
+
session: _HTTPBridgeSession,
|
|
1655
|
+
state: _HTTPBridgeRetryCircuitState,
|
|
1656
|
+
*,
|
|
1657
|
+
scoped_attempt: _HTTPBridgeResponseCreateAttempt | None,
|
|
1658
|
+
threshold: int,
|
|
1659
|
+
quarantine_poisoned_anchor: bool,
|
|
1660
|
+
quarantine_cooldown_remaining: float,
|
|
1661
|
+
armed_quarantine_generation: int | None = None,
|
|
1662
|
+
pre_arm_quarantine_reason: str | None = None,
|
|
1663
|
+
pre_arm_quarantine_until: float = 0.0,
|
|
1664
|
+
) -> int | None:
|
|
1665
|
+
try:
|
|
1666
|
+
await self._persist_http_bridge_retry_circuit_serialized(
|
|
1667
|
+
session,
|
|
1668
|
+
state,
|
|
1669
|
+
now_monotonic=clock_for(self).monotonic(),
|
|
1670
|
+
now_wall=clock_for(self).time(),
|
|
1671
|
+
threshold=threshold,
|
|
1672
|
+
)
|
|
1673
|
+
merged_cooldown_remaining = 0.0
|
|
1674
|
+
merged_poison_opened = False
|
|
1675
|
+
merged_poison_revoked = False
|
|
1676
|
+
async with self._http_bridge_retry_circuit_lock:
|
|
1677
|
+
if self._http_bridge_retry_circuits.get(session.key) is state:
|
|
1678
|
+
self._http_bridge_retry_circuit_loaded_keys.add(session.key)
|
|
1679
|
+
consecutive_failures = state.consecutive_failures
|
|
1680
|
+
# Replicas that each record their locally-first failure stay
|
|
1681
|
+
# below the threshold under their own lock, and it is the
|
|
1682
|
+
# durable merge that opens the circuit. Neither worker ever saw
|
|
1683
|
+
# an open circuit above, so the quarantine decision has to be
|
|
1684
|
+
# revisited against the merged state or the probe admitted
|
|
1685
|
+
# after this cooldown is planned with the poisoned anchor.
|
|
1686
|
+
#
|
|
1687
|
+
# The merged cooldown can already have elapsed — another replica
|
|
1688
|
+
# may have opened the circuit long enough ago that its deadline
|
|
1689
|
+
# is in the past by the time this write returns. That key is at
|
|
1690
|
+
# its threshold with no cooldown left, so the very next request
|
|
1691
|
+
# is the half-open probe: it needs the quarantine most, not
|
|
1692
|
+
# least. Track the opening itself, and let a zero remainder fall
|
|
1693
|
+
# through to the bare half-open lease.
|
|
1694
|
+
#
|
|
1695
|
+
# This deliberately does not skip keys quarantined from the
|
|
1696
|
+
# local opening. A local open arms the floor from its own
|
|
1697
|
+
# backoff, which can be 60s, and the merge can then replace
|
|
1698
|
+
# `cooldown_until` with a deadline up to 600s out. Re-arming is
|
|
1699
|
+
# idempotent because the entry keeps the later of the two
|
|
1700
|
+
# deadlines, so recomputing against the merged cooldown can only
|
|
1701
|
+
# extend a floor that would otherwise expire mid-cooldown.
|
|
1702
|
+
#
|
|
1703
|
+
# Only when the registry still maps this key to this state. A
|
|
1704
|
+
# multiplexed sibling that completed while the persist was in
|
|
1705
|
+
# flight clears the circuit and drops this object, and it also
|
|
1706
|
+
# advanced the anchor; re-arming off the detached state would
|
|
1707
|
+
# resurrect a poisoned key whose anchor is now valid and let a
|
|
1708
|
+
# later explicit rejection discard it. Nor is there anything to
|
|
1709
|
+
# re-arm when this call already quarantined the key from its own
|
|
1710
|
+
# opening and the merge did not move the deadline: that arm is a
|
|
1711
|
+
# no-op except for the generation bump, which is the fence the
|
|
1712
|
+
# verified stale-anchor replay claims at dispatch (#1863).
|
|
1713
|
+
if self._http_bridge_retry_circuits.get(session.key) is state:
|
|
1714
|
+
merged_cooldown_remaining = max(0.0, state.cooldown_until - clock_for(self).monotonic())
|
|
1715
|
+
# The persist merge adopted the returned row, so the
|
|
1716
|
+
# verdict must come from the adopted detail and count,
|
|
1717
|
+
# not the local strike's class: a clean_close losing to
|
|
1718
|
+
# a poison opening must still quarantine, and a poison
|
|
1719
|
+
# strike losing to a reset or clean lineage must not
|
|
1720
|
+
# leave its speculative quarantine suppressing a valid
|
|
1721
|
+
# anchor.
|
|
1722
|
+
adopted_poison_class = _http_bridge_anchor_poison_detail(state.last_detail) is not None
|
|
1723
|
+
# The anchor-poison threshold is the circuit threshold:
|
|
1724
|
+
# a non-poison local strike adopting a poison row that
|
|
1725
|
+
# reaches it must still arm the quarantine, or the
|
|
1726
|
+
# loaded-key cache hands the next request the dead
|
|
1727
|
+
# anchor.
|
|
1728
|
+
poison_arm_threshold = threshold
|
|
1729
|
+
merged_poison_opened = (
|
|
1730
|
+
adopted_poison_class
|
|
1731
|
+
and consecutive_failures >= poison_arm_threshold
|
|
1732
|
+
and (
|
|
1733
|
+
not quarantine_poisoned_anchor or merged_cooldown_remaining > quarantine_cooldown_remaining
|
|
1734
|
+
)
|
|
1735
|
+
)
|
|
1736
|
+
# Revocation mirrors the arm's own justification: the
|
|
1737
|
+
# quarantine survives whenever the adopted state still
|
|
1738
|
+
# satisfies that same bar.
|
|
1739
|
+
merged_poison_revoked = quarantine_poisoned_anchor and not (
|
|
1740
|
+
adopted_poison_class and consecutive_failures >= poison_arm_threshold
|
|
1741
|
+
)
|
|
1742
|
+
if merged_poison_opened:
|
|
1743
|
+
_quarantine_http_bridge_session(
|
|
1744
|
+
self,
|
|
1745
|
+
session,
|
|
1746
|
+
reason=_HTTP_BRIDGE_QUARANTINE_POISONED_ANCHOR_REASON,
|
|
1747
|
+
minimum_seconds=_http_bridge_poison_quarantine_minimum_seconds(merged_cooldown_remaining),
|
|
1748
|
+
)
|
|
1749
|
+
elif merged_poison_revoked:
|
|
1750
|
+
_revoke_http_bridge_poison_quarantine(
|
|
1751
|
+
self,
|
|
1752
|
+
session.key,
|
|
1753
|
+
generation=armed_quarantine_generation,
|
|
1754
|
+
restore_reason=pre_arm_quarantine_reason,
|
|
1755
|
+
restore_until=pre_arm_quarantine_until,
|
|
1756
|
+
)
|
|
1757
|
+
return consecutive_failures
|
|
1758
|
+
finally:
|
|
1759
|
+
if scoped_attempt is not None and scoped_attempt.retry_circuit_failure_settled is not None:
|
|
1760
|
+
scoped_attempt.retry_circuit_failure_settled.set()
|
|
1761
|
+
|
|
1762
|
+
async def _http_bridge_suppress_poison_clear_after_anchor_advance(
|
|
1763
|
+
self: Any,
|
|
1764
|
+
session: _HTTPBridgeSession,
|
|
1765
|
+
) -> tuple[Any, str | None, str | None, float, int, bool] | None:
|
|
1766
|
+
"""Mark the surviving episode's clear as no longer owed.
|
|
1767
|
+
|
|
1768
|
+
A completed response just advanced the durable anchor while its
|
|
1769
|
+
circuit settlement failed and the old episode was restored. That
|
|
1770
|
+
episode's recorded failures were all against the superseded anchor,
|
|
1771
|
+
so its owed abandonment must not fire against the fresh one: one
|
|
1772
|
+
later eventless failure would otherwise ride the restored
|
|
1773
|
+
at-threshold count straight into clearing continuity the episode
|
|
1774
|
+
never proved dead. The cooldown itself is left standing and settles
|
|
1775
|
+
at the next opportunity.
|
|
1776
|
+
|
|
1777
|
+
The suppression is also persisted by rewriting the surviving row's
|
|
1778
|
+
failure detail under the row's version fence: the local marker
|
|
1779
|
+
protects only this worker, and another replica loading the
|
|
1780
|
+
surviving at-threshold poison row would otherwise arm quarantine
|
|
1781
|
+
against the fresh anchor and authorize an abandonment for failures
|
|
1782
|
+
recorded against the superseded one. The durable transitional
|
|
1783
|
+
detail is the fail-closed ``anchor_abandoned`` tombstone, not
|
|
1784
|
+
``anchor_superseded``: the registration has not landed yet, so a
|
|
1785
|
+
crash inside the window must read as "deltas fail closed", never
|
|
1786
|
+
as a disproved episode that hands replicas the old poisoned
|
|
1787
|
+
anchor. The caller promotes the tombstone to ``anchor_superseded``
|
|
1788
|
+
once the fresh anchor commits. A concurrent strike outracing the
|
|
1789
|
+
rewrite keeps its poison class — it is new evidence — and a failed
|
|
1790
|
+
rewrite leaves the suppression process-local, no worse than
|
|
1791
|
+
before.
|
|
1792
|
+
"""
|
|
1793
|
+
key_lock = await self._acquire_http_bridge_retry_circuit_key_lock(session.key)
|
|
1794
|
+
try:
|
|
1795
|
+
async with self._http_bridge_retry_circuit_lock:
|
|
1796
|
+
state = self._http_bridge_retry_circuits.get(session.key)
|
|
1797
|
+
if state is None:
|
|
1798
|
+
return None
|
|
1799
|
+
if (
|
|
1800
|
+
_http_bridge_anchor_poison_detail(state.last_detail) is None
|
|
1801
|
+
and state.last_detail != _HTTP_BRIDGE_RETRY_CIRCUIT_ANCHOR_ABANDONED_DETAIL
|
|
1802
|
+
# Owed debt IS poison evidence even when a later
|
|
1803
|
+
# non-poison strike overwrote the local detail (its
|
|
1804
|
+
# durable persist and reload can both fail): without a
|
|
1805
|
+
# token the debt survives the fresh registration and a
|
|
1806
|
+
# later strike abandons the anchor just registered.
|
|
1807
|
+
and state.owed_poison_detail is None
|
|
1808
|
+
):
|
|
1809
|
+
# A clean episode owes no abandonment; marking it would
|
|
1810
|
+
# refuse the clear a LATER poison strike genuinely owes,
|
|
1811
|
+
# since only the supersession sentinel resets the marker
|
|
1812
|
+
# on a fresh strike. An existing tombstone is NOT clean:
|
|
1813
|
+
# it is already the transitional durable state (a sticky
|
|
1814
|
+
# tombstone can carry later strikes onto a positive
|
|
1815
|
+
# count), and returning no token here would leave a
|
|
1816
|
+
# committed registration with nothing to promote — every
|
|
1817
|
+
# later submission carrying the freshly registered
|
|
1818
|
+
# anchor would then be rejected against the lingering
|
|
1819
|
+
# tombstone.
|
|
1820
|
+
return None
|
|
1821
|
+
prior_owed = state.owed_poison_detail
|
|
1822
|
+
state.poison_anchor_cleared = True
|
|
1823
|
+
state.owed_poison_detail = None
|
|
1824
|
+
if state.persisted_updated_at_epoch <= 0:
|
|
1825
|
+
return (state, None, prior_owed, state.persisted_updated_at_epoch, 0, False)
|
|
1826
|
+
captured_state = state
|
|
1827
|
+
captured_epoch = state.persisted_updated_at_epoch
|
|
1828
|
+
prior_detail = state.last_detail
|
|
1829
|
+
supersede_base_epoch = state.persisted_updated_at_epoch
|
|
1830
|
+
supersede_base_count = state.consecutive_failures
|
|
1831
|
+
if state.last_detail != _HTTP_BRIDGE_RETRY_CIRCUIT_ANCHOR_ABANDONED_DETAIL:
|
|
1832
|
+
# A local tombstone stays a tombstone until the fresh
|
|
1833
|
+
# anchor's registration commits and the promotion
|
|
1834
|
+
# rewrites the durable row: flipping the local cache to
|
|
1835
|
+
# the superseded sentinel early would let a concurrent
|
|
1836
|
+
# proof-gated resend that cannot reload bypass the
|
|
1837
|
+
# fail-closed gates and send the still-stored poisoned
|
|
1838
|
+
# anchor.
|
|
1839
|
+
state.last_detail = _HTTP_BRIDGE_RETRY_CIRCUIT_ANCHOR_SUPERSEDED_DETAIL
|
|
1840
|
+
superseded = False
|
|
1841
|
+
try:
|
|
1842
|
+
# A detail-only fenced update, not a strike write: the strike
|
|
1843
|
+
# upsert increments the merged count by design, and a phantom
|
|
1844
|
+
# strike here would extend the cooldown. The row's version is
|
|
1845
|
+
# left unchanged so a concurrent strike still merges onto it
|
|
1846
|
+
# and its poison class overwrites this sentinel — new
|
|
1847
|
+
# evidence outranks the supersession in either order. The
|
|
1848
|
+
# fence carries the observed count as well: a lagging-clock
|
|
1849
|
+
# strike can merge without moving the version, and every
|
|
1850
|
+
# landed merge increments the count, so the count is what
|
|
1851
|
+
# makes such a strike outrank a supersession that follows it.
|
|
1852
|
+
superseded = await self._durable_bridge.supersede_retry_circuit_detail(
|
|
1853
|
+
session_key_kind=session.key.affinity_kind,
|
|
1854
|
+
session_key_value=session.key.affinity_key,
|
|
1855
|
+
api_key_id=session.key.api_key_id,
|
|
1856
|
+
expected_updated_at_epoch=supersede_base_epoch,
|
|
1857
|
+
expected_consecutive_failures=supersede_base_count,
|
|
1858
|
+
expected_last_detail=prior_detail,
|
|
1859
|
+
last_detail=_HTTP_BRIDGE_RETRY_CIRCUIT_ANCHOR_ABANDONED_DETAIL,
|
|
1860
|
+
)
|
|
1861
|
+
except Exception:
|
|
1862
|
+
logger.warning(
|
|
1863
|
+
"Failed to persist HTTP bridge anchor supersession bridge_kind=%s bridge_key=%s",
|
|
1864
|
+
session.key.affinity_kind,
|
|
1865
|
+
_hash_identifier(session.key.affinity_key),
|
|
1866
|
+
exc_info=True,
|
|
1867
|
+
)
|
|
1868
|
+
if not superseded:
|
|
1869
|
+
async with self._http_bridge_retry_circuit_lock:
|
|
1870
|
+
current = self._http_bridge_retry_circuits.get(session.key)
|
|
1871
|
+
if current is state and state.last_detail == _HTTP_BRIDGE_RETRY_CIRCUIT_ANCHOR_SUPERSEDED_DETAIL:
|
|
1872
|
+
# The fenced rewrite missed (the row moved) or the
|
|
1873
|
+
# write failed: keep the local view matching the
|
|
1874
|
+
# durable row; the suppression stays process-local,
|
|
1875
|
+
# no worse than before.
|
|
1876
|
+
state.last_detail = prior_detail
|
|
1877
|
+
return (captured_state, prior_detail, prior_owed, captured_epoch, supersede_base_count, superseded)
|
|
1878
|
+
finally:
|
|
1879
|
+
key_lock.release()
|
|
1880
|
+
|
|
1881
|
+
async def _http_bridge_restore_poison_row_after_failed_registration(
|
|
1882
|
+
self: Any,
|
|
1883
|
+
session: _HTTPBridgeSession,
|
|
1884
|
+
poison_detail: str,
|
|
1885
|
+
) -> None:
|
|
1886
|
+
"""Re-seed durable poison evidence after a failed anchor registration.
|
|
1887
|
+
|
|
1888
|
+
The settle-before-registration ordering is fence-critical, so a
|
|
1889
|
+
registration failure lands after the circuit was already zeroed
|
|
1890
|
+
while the old poisoned anchor is still the stored one. The kept
|
|
1891
|
+
local quarantine alone cannot hold: the next planning load reads
|
|
1892
|
+
the zeroed row as a disproved episode and revokes it, and other
|
|
1893
|
+
replicas never arm. Re-opening the row at the circuit threshold
|
|
1894
|
+
with the prior poison class restores the protection everywhere; a
|
|
1895
|
+
CAS drop against a row that moved concurrently defers to that
|
|
1896
|
+
newer evidence.
|
|
1897
|
+
"""
|
|
1898
|
+
key = session.key
|
|
1899
|
+
now_wall = clock_for(self).time()
|
|
1900
|
+
now_monotonic = clock_for(self).monotonic()
|
|
1901
|
+
try:
|
|
1902
|
+
row = await self._durable_bridge.lookup_retry_circuit(
|
|
1903
|
+
session_key_kind=key.affinity_kind,
|
|
1904
|
+
session_key_value=key.affinity_key,
|
|
1905
|
+
api_key_id=key.api_key_id,
|
|
1906
|
+
)
|
|
1907
|
+
except Exception:
|
|
1908
|
+
logger.warning(
|
|
1909
|
+
"Failed to reload HTTP bridge retry circuit before poison restore bridge_kind=%s bridge_key=%s",
|
|
1910
|
+
key.affinity_kind,
|
|
1911
|
+
_hash_identifier(key.affinity_key),
|
|
1912
|
+
exc_info=True,
|
|
1913
|
+
)
|
|
1914
|
+
return
|
|
1915
|
+
threshold = max(1, _HTTP_BRIDGE_RETRY_CIRCUIT_FAILURE_THRESHOLD)
|
|
1916
|
+
if row is not None and row.last_detail == _HTTP_BRIDGE_RETRY_CIRCUIT_ANCHOR_ABANDONED_DETAIL:
|
|
1917
|
+
# The failed registration follows this completion's OWN settle,
|
|
1918
|
+
# which wrote the transitional tombstone; the strike merge's
|
|
1919
|
+
# sticky-tombstone rule would silently refuse to write the
|
|
1920
|
+
# poison class back over it, degrading the re-seed to a
|
|
1921
|
+
# threshold tombstone no replica arms a quarantine from. The
|
|
1922
|
+
# fenced detail-only supersede is the one path allowed through
|
|
1923
|
+
# that stickiness; a CAS miss means newer evidence owns the row
|
|
1924
|
+
# and the merge below defers to it.
|
|
1925
|
+
try:
|
|
1926
|
+
await self._durable_bridge.supersede_retry_circuit_detail(
|
|
1927
|
+
session_key_kind=key.affinity_kind,
|
|
1928
|
+
session_key_value=key.affinity_key,
|
|
1929
|
+
api_key_id=key.api_key_id,
|
|
1930
|
+
expected_updated_at_epoch=row.updated_at_epoch,
|
|
1931
|
+
expected_consecutive_failures=row.consecutive_failures,
|
|
1932
|
+
expected_last_detail=_HTTP_BRIDGE_RETRY_CIRCUIT_ANCHOR_ABANDONED_DETAIL,
|
|
1933
|
+
last_detail=poison_detail,
|
|
1934
|
+
)
|
|
1935
|
+
except Exception:
|
|
1936
|
+
logger.warning(
|
|
1937
|
+
"Failed to transition HTTP bridge tombstone before poison restore bridge_kind=%s bridge_key=%s",
|
|
1938
|
+
key.affinity_kind,
|
|
1939
|
+
_hash_identifier(key.affinity_key),
|
|
1940
|
+
exc_info=True,
|
|
1941
|
+
)
|
|
1942
|
+
if row.consecutive_failures >= threshold:
|
|
1943
|
+
return
|
|
1944
|
+
elif row is not None and row.consecutive_failures >= threshold:
|
|
1945
|
+
return
|
|
1946
|
+
base_backoff = max(0.001, _HTTP_BRIDGE_RETRY_CIRCUIT_BASE_BACKOFF_SECONDS)
|
|
1947
|
+
try:
|
|
1948
|
+
await self._durable_bridge.persist_retry_circuit(
|
|
1949
|
+
session_key_kind=key.affinity_kind,
|
|
1950
|
+
session_key_value=key.affinity_key,
|
|
1951
|
+
api_key_id=key.api_key_id,
|
|
1952
|
+
consecutive_failures=threshold,
|
|
1953
|
+
cooldown_until_epoch=now_wall + base_backoff,
|
|
1954
|
+
last_detail=poison_detail,
|
|
1955
|
+
updated_at_epoch=now_wall,
|
|
1956
|
+
base_updated_at_epoch=row.updated_at_epoch if row is not None else 0.0,
|
|
1957
|
+
failure_threshold=threshold,
|
|
1958
|
+
poison_sticky_threshold=threshold,
|
|
1959
|
+
conflict_cooldown_until_epoch=now_wall + base_backoff,
|
|
1960
|
+
base_backoff_seconds=base_backoff,
|
|
1961
|
+
max_backoff_seconds=max(0.001, _HTTP_BRIDGE_RETRY_CIRCUIT_MAX_BACKOFF_SECONDS),
|
|
1962
|
+
clean_close_max_backoff_seconds=max(
|
|
1963
|
+
0.001,
|
|
1964
|
+
_HTTP_BRIDGE_RETRY_CIRCUIT_CLEAN_CLOSE_MAX_BACKOFF_SECONDS,
|
|
1965
|
+
),
|
|
1966
|
+
)
|
|
1967
|
+
except Exception:
|
|
1968
|
+
logger.warning(
|
|
1969
|
+
"Failed to restore HTTP bridge poison evidence bridge_kind=%s bridge_key=%s",
|
|
1970
|
+
key.affinity_kind,
|
|
1971
|
+
_hash_identifier(key.affinity_key),
|
|
1972
|
+
exc_info=True,
|
|
1973
|
+
)
|
|
1974
|
+
return
|
|
1975
|
+
del now_monotonic
|
|
1976
|
+
# Adopt the restored row locally and re-arm the quarantine through
|
|
1977
|
+
# the ordinary load path.
|
|
1978
|
+
await self._load_http_bridge_retry_circuit(session)
|
|
1979
|
+
|
|
1980
|
+
async def _http_bridge_restore_poison_clear_after_failed_anchor_advance(
|
|
1981
|
+
self: Any,
|
|
1982
|
+
session: _HTTPBridgeSession,
|
|
1983
|
+
supersession: tuple[Any, str | None, str | None, float, int, bool] | None,
|
|
1984
|
+
) -> None:
|
|
1985
|
+
"""Roll a transitional anchor-advance suppression back.
|
|
1986
|
+
|
|
1987
|
+
The suppression is applied before the fresh anchor is registered, so
|
|
1988
|
+
a concurrent consult whose continuity read lands after publication
|
|
1989
|
+
cannot validate the old poison row against the just-published
|
|
1990
|
+
anchor. When that registration then fails, the old anchor is still
|
|
1991
|
+
the stored one, and leaving the suppression in place would strand it
|
|
1992
|
+
with no funnel willing to clear it — the failure the
|
|
1993
|
+
registration-gated ordering existed to prevent.
|
|
1994
|
+
"""
|
|
1995
|
+
if supersession is None:
|
|
1996
|
+
return
|
|
1997
|
+
captured_state, prior_detail, prior_owed, base_epoch, base_count, superseded = supersession
|
|
1998
|
+
key_lock = await self._acquire_http_bridge_retry_circuit_key_lock(session.key)
|
|
1999
|
+
try:
|
|
2000
|
+
async with self._http_bridge_retry_circuit_lock:
|
|
2001
|
+
state = self._http_bridge_retry_circuits.get(session.key)
|
|
2002
|
+
if (
|
|
2003
|
+
state is not None
|
|
2004
|
+
# Fenced to the exact captured episode and lineage: a
|
|
2005
|
+
# same-key load, strike, or settlement replacing or
|
|
2006
|
+
# reconciling the state while the registration was in
|
|
2007
|
+
# flight owns the key now, and copying the ended
|
|
2008
|
+
# lineage's owed evidence into it would let a later
|
|
2009
|
+
# strike clear continuity that episode never proved
|
|
2010
|
+
# dead.
|
|
2011
|
+
and state is captured_state
|
|
2012
|
+
and state.persisted_updated_at_epoch == base_epoch
|
|
2013
|
+
):
|
|
2014
|
+
state.poison_anchor_cleared = False
|
|
2015
|
+
if prior_owed is not None:
|
|
2016
|
+
state.owed_poison_detail = prior_owed
|
|
2017
|
+
if (
|
|
2018
|
+
prior_detail is not None
|
|
2019
|
+
and state.last_detail == _HTTP_BRIDGE_RETRY_CIRCUIT_ANCHOR_SUPERSEDED_DETAIL
|
|
2020
|
+
):
|
|
2021
|
+
state.last_detail = prior_detail
|
|
2022
|
+
if superseded and prior_detail is not None:
|
|
2023
|
+
try:
|
|
2024
|
+
# A fence miss here means a strike moved the row and its
|
|
2025
|
+
# own poison class already replaced the sentinel.
|
|
2026
|
+
await self._durable_bridge.supersede_retry_circuit_detail(
|
|
2027
|
+
session_key_kind=session.key.affinity_kind,
|
|
2028
|
+
session_key_value=session.key.affinity_key,
|
|
2029
|
+
api_key_id=session.key.api_key_id,
|
|
2030
|
+
expected_updated_at_epoch=base_epoch,
|
|
2031
|
+
expected_consecutive_failures=base_count,
|
|
2032
|
+
expected_last_detail=_HTTP_BRIDGE_RETRY_CIRCUIT_ANCHOR_ABANDONED_DETAIL,
|
|
2033
|
+
last_detail=prior_detail,
|
|
2034
|
+
)
|
|
2035
|
+
except Exception:
|
|
2036
|
+
logger.warning(
|
|
2037
|
+
"Failed to roll back HTTP bridge anchor supersession bridge_kind=%s bridge_key=%s",
|
|
2038
|
+
session.key.affinity_kind,
|
|
2039
|
+
_hash_identifier(session.key.affinity_key),
|
|
2040
|
+
exc_info=True,
|
|
2041
|
+
)
|
|
2042
|
+
finally:
|
|
2043
|
+
key_lock.release()
|
|
2044
|
+
|
|
2045
|
+
async def _http_bridge_reconcile_transitional_tombstone(self: Any, session: _HTTPBridgeSession) -> None:
|
|
2046
|
+
"""Rewrite a lingering transitional tombstone off the row's own values.
|
|
2047
|
+
|
|
2048
|
+
Strike merges keep ``anchor_abandoned`` sticky, so a CAS miss on a
|
|
2049
|
+
promote or erase does NOT mean the tombstone was replaced — it means
|
|
2050
|
+
the count moved underneath the fence while the tombstone stood, and
|
|
2051
|
+
the freshly committed anchor would stay rejected on every replica.
|
|
2052
|
+
One reload-and-rewrite round converges the interleavings: a zeroed
|
|
2053
|
+
row erases to a plain reset, a positive count promotes to the
|
|
2054
|
+
superseded sentinel, and a second miss defers to newer evidence and
|
|
2055
|
+
the next completion's settle-and-erase.
|
|
2056
|
+
"""
|
|
2057
|
+
for reconcile_attempt in range(2):
|
|
2058
|
+
try:
|
|
2059
|
+
row = await self._durable_bridge.lookup_retry_circuit(
|
|
2060
|
+
session_key_kind=session.key.affinity_kind,
|
|
2061
|
+
session_key_value=session.key.affinity_key,
|
|
2062
|
+
api_key_id=session.key.api_key_id,
|
|
2063
|
+
)
|
|
2064
|
+
if row is None or row.last_detail != _HTTP_BRIDGE_RETRY_CIRCUIT_ANCHOR_ABANDONED_DETAIL:
|
|
2065
|
+
return
|
|
2066
|
+
rewrote = await self._durable_bridge.supersede_retry_circuit_detail(
|
|
2067
|
+
session_key_kind=session.key.affinity_kind,
|
|
2068
|
+
session_key_value=session.key.affinity_key,
|
|
2069
|
+
api_key_id=session.key.api_key_id,
|
|
2070
|
+
expected_updated_at_epoch=row.updated_at_epoch,
|
|
2071
|
+
expected_consecutive_failures=row.consecutive_failures,
|
|
2072
|
+
expected_last_detail=_HTTP_BRIDGE_RETRY_CIRCUIT_ANCHOR_ABANDONED_DETAIL,
|
|
2073
|
+
last_detail=(
|
|
2074
|
+
None if row.consecutive_failures == 0 else _HTTP_BRIDGE_RETRY_CIRCUIT_ANCHOR_SUPERSEDED_DETAIL
|
|
2075
|
+
),
|
|
2076
|
+
)
|
|
2077
|
+
if rewrote:
|
|
2078
|
+
return
|
|
2079
|
+
except Exception:
|
|
2080
|
+
if reconcile_attempt == 0:
|
|
2081
|
+
continue
|
|
2082
|
+
logger.debug(
|
|
2083
|
+
"Failed to reconcile transitional circuit tombstone",
|
|
2084
|
+
exc_info=True,
|
|
2085
|
+
)
|
|
2086
|
+
return
|
|
2087
|
+
|
|
2088
|
+
async def _http_bridge_promote_transitional_supersession(
|
|
2089
|
+
self: Any,
|
|
2090
|
+
session: _HTTPBridgeSession,
|
|
2091
|
+
supersession: tuple[Any, str | None, str | None, float, int, bool] | None,
|
|
2092
|
+
) -> None:
|
|
2093
|
+
"""Promote a landed transitional tombstone to ``anchor_superseded``.
|
|
2094
|
+
|
|
2095
|
+
The fresh anchor just committed, so the fail-closed tombstone the
|
|
2096
|
+
suppression persisted is no longer the right durable record: left
|
|
2097
|
+
in place it would 404 healthy delta follow-ups riding the valid
|
|
2098
|
+
fresh anchor, and other replicas' stale quarantines would never
|
|
2099
|
+
see the disproof. A fence miss means a concurrent strike moved the
|
|
2100
|
+
row and its newer evidence stands; a failed promote leaves the
|
|
2101
|
+
conservative tombstone, which only fails closed.
|
|
2102
|
+
"""
|
|
2103
|
+
if supersession is None:
|
|
2104
|
+
return
|
|
2105
|
+
_captured_state, _prior_detail, _prior_owed, base_epoch, base_count, superseded = supersession
|
|
2106
|
+
if not superseded:
|
|
2107
|
+
return
|
|
2108
|
+
for promote_attempt in range(2):
|
|
2109
|
+
try:
|
|
2110
|
+
promoted = await self._durable_bridge.supersede_retry_circuit_detail(
|
|
2111
|
+
session_key_kind=session.key.affinity_kind,
|
|
2112
|
+
session_key_value=session.key.affinity_key,
|
|
2113
|
+
api_key_id=session.key.api_key_id,
|
|
2114
|
+
expected_updated_at_epoch=base_epoch,
|
|
2115
|
+
expected_consecutive_failures=base_count,
|
|
2116
|
+
expected_last_detail=_HTTP_BRIDGE_RETRY_CIRCUIT_ANCHOR_ABANDONED_DETAIL,
|
|
2117
|
+
last_detail=_HTTP_BRIDGE_RETRY_CIRCUIT_ANCHOR_SUPERSEDED_DETAIL,
|
|
2118
|
+
)
|
|
2119
|
+
if promoted is False:
|
|
2120
|
+
# A concurrent strike moved the count while stickiness
|
|
2121
|
+
# kept the tombstone detail: reporting success here
|
|
2122
|
+
# would leave the freshly committed anchor rejected on
|
|
2123
|
+
# every replica. Reconcile on the row's own values.
|
|
2124
|
+
await self._http_bridge_reconcile_transitional_tombstone(session)
|
|
2125
|
+
return
|
|
2126
|
+
except Exception:
|
|
2127
|
+
# One immediate retry covers a transient durable blip: a
|
|
2128
|
+
# skipped promotion leaves the fresh anchor rejected on
|
|
2129
|
+
# every replica until the next completion's settle-and-erase
|
|
2130
|
+
# heals it. A second failure defers to that healing.
|
|
2131
|
+
if promote_attempt == 0:
|
|
2132
|
+
continue
|
|
2133
|
+
logger.warning(
|
|
2134
|
+
"Failed to promote HTTP bridge anchor supersession bridge_kind=%s bridge_key=%s",
|
|
2135
|
+
session.key.affinity_kind,
|
|
2136
|
+
_hash_identifier(session.key.affinity_key),
|
|
2137
|
+
exc_info=True,
|
|
2138
|
+
)
|
|
2139
|
+
|
|
2140
|
+
async def _clear_http_bridge_retry_circuit(
|
|
2141
|
+
self: Any,
|
|
2142
|
+
session: _HTTPBridgeSession,
|
|
2143
|
+
*,
|
|
2144
|
+
settled_detail: str | None = None,
|
|
2145
|
+
settled_detail_authoritative: bool = False,
|
|
2146
|
+
expected_episode: tuple[float, int, int] | None = None,
|
|
2147
|
+
) -> bool:
|
|
2148
|
+
"""Settle a hard key's retry circuit; ``True`` when settlement held.
|
|
2149
|
+
|
|
2150
|
+
The load below re-hydrates any persisted row into the local state, so
|
|
2151
|
+
after a successful load the fence is absent only when no durable row
|
|
2152
|
+
was observed at all — and an unfenced delete there could only remove a
|
|
2153
|
+
row another writer created concurrently. A strike write still in
|
|
2154
|
+
flight while this clear runs is undone by its own writer:
|
|
2155
|
+
``_persist_http_bridge_retry_circuit`` re-checks the registry after
|
|
2156
|
+
its durable write lands and deletes exactly the row it created.
|
|
2157
|
+
"""
|
|
2158
|
+
key = session.key
|
|
2159
|
+
if key.strength != "hard":
|
|
2160
|
+
return True
|
|
2161
|
+
key_lock = await self._acquire_http_bridge_retry_circuit_key_lock(key)
|
|
2162
|
+
try:
|
|
2163
|
+
return await self._clear_http_bridge_retry_circuit_serialized(
|
|
2164
|
+
session,
|
|
2165
|
+
settled_detail=settled_detail,
|
|
2166
|
+
settled_detail_authoritative=settled_detail_authoritative,
|
|
2167
|
+
expected_episode=expected_episode,
|
|
2168
|
+
)
|
|
2169
|
+
finally:
|
|
2170
|
+
key_lock.release()
|
|
2171
|
+
|
|
2172
|
+
async def _clear_http_bridge_retry_circuit_serialized(
|
|
2173
|
+
self: Any,
|
|
2174
|
+
session: _HTTPBridgeSession,
|
|
2175
|
+
*,
|
|
2176
|
+
settled_detail: str | None = None,
|
|
2177
|
+
settled_detail_authoritative: bool = False,
|
|
2178
|
+
expected_episode: tuple[float, int, int] | None = None,
|
|
2179
|
+
) -> bool:
|
|
2180
|
+
key = session.key
|
|
2181
|
+
durable_load_succeeded = await self._load_http_bridge_retry_circuit(session)
|
|
2182
|
+
if expected_episode is not None:
|
|
2183
|
+
# The caller settles a specific authorized episode — an
|
|
2184
|
+
# abandonment clears only the circuit its abandonment
|
|
2185
|
+
# invalidated. A replacement episode opened between the
|
|
2186
|
+
# authorization and this settle (fresh strikes against a newly
|
|
2187
|
+
# registered anchor) carries its own valid cooldown; resetting
|
|
2188
|
+
# it would let the newly poisoned anchor retry immediately.
|
|
2189
|
+
# The count discriminates lagging-clock merges the epoch alone
|
|
2190
|
+
# cannot see. Nothing is popped on a mismatch: the settlement
|
|
2191
|
+
# stays owed and the newer row stands untouched.
|
|
2192
|
+
async with self._http_bridge_retry_circuit_lock:
|
|
2193
|
+
fenced_state = self._http_bridge_retry_circuits.get(key)
|
|
2194
|
+
if fenced_state is not None and (
|
|
2195
|
+
fenced_state.persisted_updated_at_epoch != expected_episode[0]
|
|
2196
|
+
or fenced_state.consecutive_failures != expected_episode[1]
|
|
2197
|
+
# A replay claim advances only the admission generation:
|
|
2198
|
+
# resetting beneath it would clear the circuit a claimed
|
|
2199
|
+
# replay depends on and admit a second dispatch beside it.
|
|
2200
|
+
or fenced_state.persisted_admission_generation != expected_episode[2]
|
|
2201
|
+
):
|
|
2202
|
+
return False
|
|
2203
|
+
async with self._http_bridge_retry_circuit_lock:
|
|
2204
|
+
state = self._http_bridge_retry_circuits.pop(key, None)
|
|
2205
|
+
self._http_bridge_retry_circuit_loaded_keys.discard(key)
|
|
2206
|
+
self._http_bridge_retry_circuit_persisted_keys.discard(key)
|
|
2207
|
+
if state is not None or not durable_load_succeeded:
|
|
2208
|
+
# The pop takes the state's stale-load watermark with it;
|
|
2209
|
+
# keep a per-key watermark so a lookup that began before
|
|
2210
|
+
# this settlement cannot adopt the pre-settlement row into a
|
|
2211
|
+
# fresh state and resurrect the settled cooldown. A healthy
|
|
2212
|
+
# key with no state and a confirmed durable miss has nothing
|
|
2213
|
+
# to resurrect, and stamping every completed conversation
|
|
2214
|
+
# would grow this map — and its global-lock prune scan —
|
|
2215
|
+
# with every key the process ever serves.
|
|
2216
|
+
self._http_bridge_retry_circuit_reconcile_watermarks[key] = clock_for(self).monotonic()
|
|
2217
|
+
expected_updated_at_epoch = (
|
|
2218
|
+
state.persisted_updated_at_epoch if state is not None and state.persisted_updated_at_epoch > 0 else None
|
|
2219
|
+
)
|
|
2220
|
+
expected_admission_generation = state.persisted_admission_generation if state is not None else None
|
|
2221
|
+
# A lagging-clock strike merges a higher count without moving the
|
|
2222
|
+
# epoch; the reset CAS carries the observed count so it cannot
|
|
2223
|
+
# zero a newer episode's cooldown it never saw. A completion
|
|
2224
|
+
# settle defeated this way still wins through the chase.
|
|
2225
|
+
expected_consecutive_failures = state.consecutive_failures if state is not None else None
|
|
2226
|
+
if (
|
|
2227
|
+
settled_detail is None
|
|
2228
|
+
# An authoritative None was chosen after consulting continuity —
|
|
2229
|
+
# the caller SAW fresh evidence replacing the poisoned anchor —
|
|
2230
|
+
# and upgrading it back to a tombstone would fail the fresh
|
|
2231
|
+
# continuity closed. Only a blind None is upgraded.
|
|
2232
|
+
and not settled_detail_authoritative
|
|
2233
|
+
and state is not None
|
|
2234
|
+
and (
|
|
2235
|
+
_http_bridge_anchor_poison_detail(state.last_detail) is not None
|
|
2236
|
+
or state.owed_poison_detail is not None
|
|
2237
|
+
or state.last_detail == _HTTP_BRIDGE_RETRY_CIRCUIT_ANCHOR_ABANDONED_DETAIL
|
|
2238
|
+
)
|
|
2239
|
+
):
|
|
2240
|
+
# The caller's pre-settle capture can be blind — a transient
|
|
2241
|
+
# read failure before this call — while this settle's own load
|
|
2242
|
+
# just adopted a poison episode or an existing tombstone.
|
|
2243
|
+
# Resetting such a row plain would leave the old poisoned
|
|
2244
|
+
# anchor behind a reset no replica reads as suspect, so the
|
|
2245
|
+
# reset detail is derived from the state actually settled: the
|
|
2246
|
+
# fail-closed tombstone, healed by a later completion's
|
|
2247
|
+
# fence-aware settle-and-erase.
|
|
2248
|
+
settled_detail = _HTTP_BRIDGE_RETRY_CIRCUIT_ANCHOR_ABANDONED_DETAIL
|
|
2249
|
+
# A confirmed miss has no version fence to protect a row created
|
|
2250
|
+
# concurrently, so leave the durable row untouched when no state was
|
|
2251
|
+
# observed. Preserve the existing best-effort clear on read failures,
|
|
2252
|
+
# which is still useful for settling a row after a transient outage.
|
|
2253
|
+
if durable_load_succeeded and (state is None or expected_updated_at_epoch is None):
|
|
2254
|
+
return True
|
|
2255
|
+
if state is None and not durable_load_succeeded:
|
|
2256
|
+
# No local episode and no durable observation: this worker holds
|
|
2257
|
+
# no fence at all, and an unfenced reset here could clear a
|
|
2258
|
+
# poison episode or a claimed admission generation another
|
|
2259
|
+
# replica created during the outage. The best-effort clear is
|
|
2260
|
+
# reserved for a worker that at least carries a local episode;
|
|
2261
|
+
# with nothing observed, the settlement stays owed.
|
|
2262
|
+
return False
|
|
2263
|
+
try:
|
|
2264
|
+
# Clearing is idempotent and must be attempted even when the
|
|
2265
|
+
# preceding lookup failed; a successful request should settle
|
|
2266
|
+
# a previously persisted circuit after a transient read error.
|
|
2267
|
+
reset_matched = await self._durable_bridge.clear_retry_circuit(
|
|
2268
|
+
session_key_kind=key.affinity_kind,
|
|
2269
|
+
session_key_value=key.affinity_key,
|
|
2270
|
+
api_key_id=key.api_key_id,
|
|
2271
|
+
expected_updated_at_epoch=expected_updated_at_epoch,
|
|
2272
|
+
expected_admission_generation=expected_admission_generation,
|
|
2273
|
+
expected_consecutive_failures=expected_consecutive_failures,
|
|
2274
|
+
reset_detail=settled_detail,
|
|
2275
|
+
)
|
|
2276
|
+
if reset_matched is False and expected_updated_at_epoch is not None:
|
|
2277
|
+
# The fenced reset matched no row: another writer moved it
|
|
2278
|
+
# after this worker's lookup. This settlement carries the
|
|
2279
|
+
# newer evidence — the completed response proved the key
|
|
2280
|
+
# works — so reload the moved row once and retry the fence
|
|
2281
|
+
# against its current version, the same settle-wins rule the
|
|
2282
|
+
# in-process race already follows (a strike writer undoes
|
|
2283
|
+
# its own row when the registry no longer maps its state).
|
|
2284
|
+
moved = await self._durable_bridge.lookup_retry_circuit(
|
|
2285
|
+
session_key_kind=key.affinity_kind,
|
|
2286
|
+
session_key_value=key.affinity_key,
|
|
2287
|
+
api_key_id=key.api_key_id,
|
|
2288
|
+
)
|
|
2289
|
+
if moved is None or moved.consecutive_failures <= 0:
|
|
2290
|
+
# The moved row was purged or already reset: settled.
|
|
2291
|
+
reset_matched = True
|
|
2292
|
+
elif (
|
|
2293
|
+
expected_admission_generation is not None
|
|
2294
|
+
and moved.updated_at_epoch == expected_updated_at_epoch
|
|
2295
|
+
and getattr(moved, "admission_generation", 0) != expected_admission_generation
|
|
2296
|
+
):
|
|
2297
|
+
# Same failure observation, advanced admission
|
|
2298
|
+
# generation: a replica claimed the replay admission
|
|
2299
|
+
# this circuit fences, and the claim deliberately holds
|
|
2300
|
+
# the circuit open for its dispatch. Chasing the moved
|
|
2301
|
+
# generation here would clear it anyway — leave the
|
|
2302
|
+
# settlement owed instead.
|
|
2303
|
+
pass
|
|
2304
|
+
elif expected_episode is not None and (
|
|
2305
|
+
moved.updated_at_epoch != expected_episode[0]
|
|
2306
|
+
# Strictly greater, not unequal: merges only ever
|
|
2307
|
+
# increment the count, so a LOWER count at the same
|
|
2308
|
+
# epoch and admission generation cannot be foreign — it
|
|
2309
|
+
# is this worker's own episode observed before local
|
|
2310
|
+
# strikes whose durable writes failed. Refusing to
|
|
2311
|
+
# settle it would leave a confirmed abandonment's
|
|
2312
|
+
# cooldown standing against an anchor already removed.
|
|
2313
|
+
or moved.consecutive_failures > expected_episode[1]
|
|
2314
|
+
or getattr(moved, "admission_generation", 0) != expected_episode[2]
|
|
2315
|
+
):
|
|
2316
|
+
# An episode-fenced settle never chases a moved row: the
|
|
2317
|
+
# caller authorized clearing exactly one episode, and a
|
|
2318
|
+
# row that no longer matches the captured epoch, count,
|
|
2319
|
+
# and admission generation is a replacement lineage —
|
|
2320
|
+
# re-fencing on ITS version would durably zero the valid
|
|
2321
|
+
# cooldown a cross-replica completion-plus-strikes just
|
|
2322
|
+
# opened against the fresh anchor. The settlement stays
|
|
2323
|
+
# owed; the settle-wins chase is for completion callers,
|
|
2324
|
+
# whose own evidence outranks any concurrent strike.
|
|
2325
|
+
pass
|
|
2326
|
+
else:
|
|
2327
|
+
reset_matched = await self._durable_bridge.clear_retry_circuit(
|
|
2328
|
+
session_key_kind=key.affinity_kind,
|
|
2329
|
+
session_key_value=key.affinity_key,
|
|
2330
|
+
api_key_id=key.api_key_id,
|
|
2331
|
+
expected_updated_at_epoch=moved.updated_at_epoch,
|
|
2332
|
+
expected_admission_generation=getattr(moved, "admission_generation", None),
|
|
2333
|
+
expected_consecutive_failures=moved.consecutive_failures,
|
|
2334
|
+
reset_detail=settled_detail,
|
|
2335
|
+
)
|
|
2336
|
+
if reset_matched is False and expected_updated_at_epoch is not None:
|
|
2337
|
+
# Two fenced attempts both matched nothing: the row is
|
|
2338
|
+
# moving faster than this settlement can chase it, so
|
|
2339
|
+
# dropping the local state would report settlement while the
|
|
2340
|
+
# durable episode survives to be reloaded. Keep the episode
|
|
2341
|
+
# for the next clear opportunity — but reconcile it onto the
|
|
2342
|
+
# row that actually survived first: restoring the original
|
|
2343
|
+
# snapshot would hand the anchor-supersession rewrite an
|
|
2344
|
+
# obsolete fence, and the next load would then read the
|
|
2345
|
+
# surviving row as a foreign episode, reset the one-clear
|
|
2346
|
+
# marker, and authorize abandoning the fresh anchor this
|
|
2347
|
+
# completion just registered.
|
|
2348
|
+
surviving = None
|
|
2349
|
+
refresh_failed = False
|
|
2350
|
+
try:
|
|
2351
|
+
surviving = await self._durable_bridge.lookup_retry_circuit(
|
|
2352
|
+
session_key_kind=key.affinity_kind,
|
|
2353
|
+
session_key_value=key.affinity_key,
|
|
2354
|
+
api_key_id=key.api_key_id,
|
|
2355
|
+
)
|
|
2356
|
+
except Exception:
|
|
2357
|
+
refresh_failed = True
|
|
2358
|
+
logger.warning(
|
|
2359
|
+
"Failed to reload HTTP bridge retry circuit after settle misses bridge_kind=%s bridge_key=%s",
|
|
2360
|
+
key.affinity_kind,
|
|
2361
|
+
_hash_identifier(key.affinity_key),
|
|
2362
|
+
exc_info=True,
|
|
2363
|
+
)
|
|
2364
|
+
if not refresh_failed and (surviving is None or surviving.consecutive_failures <= 0):
|
|
2365
|
+
# The chase ended settled after all: the row was purged
|
|
2366
|
+
# or reset by another writer while this settlement was
|
|
2367
|
+
# missing its fences.
|
|
2368
|
+
reset_matched = True
|
|
2369
|
+
else:
|
|
2370
|
+
async with self._http_bridge_retry_circuit_lock:
|
|
2371
|
+
if state is not None and self._http_bridge_retry_circuits.get(key) is None:
|
|
2372
|
+
if surviving is not None and surviving.consecutive_failures > 0:
|
|
2373
|
+
if surviving.updated_at_epoch != state.persisted_updated_at_epoch:
|
|
2374
|
+
# Adopting a lineage this worker did not
|
|
2375
|
+
# produce: same marker rule as the load
|
|
2376
|
+
# and persist merges.
|
|
2377
|
+
state.poison_anchor_cleared = False
|
|
2378
|
+
state.owed_poison_detail = None
|
|
2379
|
+
state.consecutive_failures = max(0, surviving.consecutive_failures)
|
|
2380
|
+
state.cooldown_until = clock_for(self).monotonic() + max(
|
|
2381
|
+
0.0, surviving.cooldown_until_epoch - clock_for(self).time()
|
|
2382
|
+
)
|
|
2383
|
+
state.last_detail = surviving.last_detail
|
|
2384
|
+
if state.cooldown_until > clock_for(self).monotonic():
|
|
2385
|
+
state.half_open_until = 0.0
|
|
2386
|
+
state.persisted_updated_at_epoch = surviving.updated_at_epoch
|
|
2387
|
+
state.persisted_admission_generation = getattr(surviving, "admission_generation", 0)
|
|
2388
|
+
state.last_durable_load_monotonic = max(
|
|
2389
|
+
state.last_durable_load_monotonic,
|
|
2390
|
+
clock_for(self).monotonic(),
|
|
2391
|
+
)
|
|
2392
|
+
self._http_bridge_retry_circuits[key] = state
|
|
2393
|
+
self._http_bridge_retry_circuit_loaded_keys.add(key)
|
|
2394
|
+
self._http_bridge_retry_circuit_persisted_keys.add(key)
|
|
2395
|
+
return False
|
|
2396
|
+
except Exception:
|
|
2397
|
+
logger.warning(
|
|
2398
|
+
"Failed to clear persisted HTTP bridge retry circuit bridge_kind=%s bridge_key=%s",
|
|
2399
|
+
key.affinity_kind,
|
|
2400
|
+
_hash_identifier(key.affinity_key),
|
|
2401
|
+
exc_info=True,
|
|
2402
|
+
)
|
|
2403
|
+
# A fenced delete failing means a durable row this worker wrote
|
|
2404
|
+
# is known to have survived, so this is not a completed
|
|
2405
|
+
# settlement: put the popped episode back (unless a newer one
|
|
2406
|
+
# already took the key) so the process keeps its version fence
|
|
2407
|
+
# and the next clear opportunity retries the fenced delete,
|
|
2408
|
+
# instead of a later load resurrecting the row as a fresh
|
|
2409
|
+
# cooldown against a cause that is already gone. An unfenced
|
|
2410
|
+
# best-effort delete after a failed load proves nothing about
|
|
2411
|
+
# any row and keeps its old settle-anyway semantics.
|
|
2412
|
+
if state is not None and expected_updated_at_epoch is not None:
|
|
2413
|
+
async with self._http_bridge_retry_circuit_lock:
|
|
2414
|
+
if self._http_bridge_retry_circuits.get(key) is None:
|
|
2415
|
+
self._http_bridge_retry_circuits[key] = state
|
|
2416
|
+
self._http_bridge_retry_circuit_loaded_keys.add(key)
|
|
2417
|
+
self._http_bridge_retry_circuit_persisted_keys.add(key)
|
|
2418
|
+
return False
|
|
2419
|
+
async with self._http_bridge_retry_circuit_lock:
|
|
2420
|
+
# Re-stamp past the durable delete: the pop-time watermark above
|
|
2421
|
+
# predates the commit, so a load that started between them passes
|
|
2422
|
+
# the stale-load guard while its snapshot still shows the deleted
|
|
2423
|
+
# row. Strikes hold this key's lock for the whole settle, so any
|
|
2424
|
+
# state present here was resurrected by exactly such a load —
|
|
2425
|
+
# drop it with the same fence.
|
|
2426
|
+
self._http_bridge_retry_circuit_reconcile_watermarks[key] = clock_for(self).monotonic()
|
|
2427
|
+
resurrected = self._http_bridge_retry_circuits.get(key)
|
|
2428
|
+
if resurrected is not None and resurrected is not state:
|
|
2429
|
+
self._http_bridge_retry_circuits.pop(key, None)
|
|
2430
|
+
self._http_bridge_retry_circuit_loaded_keys.discard(key)
|
|
2431
|
+
self._http_bridge_retry_circuit_persisted_keys.discard(key)
|
|
2432
|
+
if state is None:
|
|
2433
|
+
return True
|
|
2434
|
+
if PROMETHEUS_AVAILABLE and http_bridge_retry_circuit_total is not None:
|
|
2435
|
+
http_bridge_retry_circuit_total.labels(outcome="reset").inc()
|
|
2436
|
+
logger.info(
|
|
2437
|
+
"http_bridge_retry_circuit event=reset bridge_kind=%s bridge_key=%s failures=%s",
|
|
2438
|
+
key.affinity_kind,
|
|
2439
|
+
_hash_identifier(key.affinity_key),
|
|
2440
|
+
state.consecutive_failures,
|
|
2441
|
+
)
|
|
2442
|
+
return True
|