codexLB-up 1.25.0b10__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of codexLB-up might be problematic. Click here for more details.
- app/__init__.py +10 -0
- app/admin_cli.py +494 -0
- app/cli.py +339 -0
- app/codex_sessions_retag.py +487 -0
- app/core/__init__.py +0 -0
- app/core/audit/__init__.py +0 -0
- app/core/audit/service.py +216 -0
- app/core/audit/types.py +94 -0
- app/core/auth/__init__.py +236 -0
- app/core/auth/api_key_cache.py +65 -0
- app/core/auth/dashboard_access.py +429 -0
- app/core/auth/dashboard_mode.py +171 -0
- app/core/auth/dashboard_session_ttl.py +74 -0
- app/core/auth/dashboard_users_cache.py +120 -0
- app/core/auth/dependencies.py +710 -0
- app/core/auth/external_identity.py +78 -0
- app/core/auth/guardian.py +374 -0
- app/core/auth/models.py +49 -0
- app/core/auth/providers/__init__.py +129 -0
- app/core/auth/providers/oidc.py +838 -0
- app/core/auth/providers/registry.py +118 -0
- app/core/auth/refresh.py +394 -0
- app/core/auth/step_up.py +117 -0
- app/core/auth/totp.py +81 -0
- app/core/balancer/__init__.py +71 -0
- app/core/balancer/logic.py +1612 -0
- app/core/balancer/rendezvous_hash.py +21 -0
- app/core/balancer/types.py +21 -0
- app/core/bootstrap.py +172 -0
- app/core/cache/invalidation.py +426 -0
- app/core/clients/__init__.py +0 -0
- app/core/clients/codex.py +996 -0
- app/core/clients/codex_version.py +187 -0
- app/core/clients/codex_version_snapshot.py +2 -0
- app/core/clients/files.py +534 -0
- app/core/clients/headers.py +29 -0
- app/core/clients/http.py +511 -0
- app/core/clients/model_fetcher.py +276 -0
- app/core/clients/native_egress.py +1304 -0
- app/core/clients/oauth.py +477 -0
- app/core/clients/proxy.py +6089 -0
- app/core/clients/proxy_websocket.py +1411 -0
- app/core/clients/rate_limit_reset_credits.py +412 -0
- app/core/clients/stream_errors.py +12 -0
- app/core/clients/thread_cache_identity.py +347 -0
- app/core/clients/usage.py +467 -0
- app/core/clock.py +170 -0
- app/core/config/__init__.py +0 -0
- app/core/config/background_jobs.py +73 -0
- app/core/config/context_window_overrides.py +166 -0
- app/core/config/dashboard_overrides.py +145 -0
- app/core/config/inheritable.py +47 -0
- app/core/config/key_fingerprint.py +115 -0
- app/core/config/settings.py +872 -0
- app/core/config/settings_cache.py +100 -0
- app/core/config/spool_retention.py +217 -0
- app/core/config/tiers.py +174 -0
- app/core/conversation_archive.py +568 -0
- app/core/crypto.py +37 -0
- app/core/draining.py +11 -0
- app/core/errors.py +365 -0
- app/core/exceptions.py +155 -0
- app/core/handlers/__init__.py +3 -0
- app/core/handlers/exceptions.py +478 -0
- app/core/http_protocol.py +176 -0
- app/core/http_protocol_httptools.py +204 -0
- app/core/ingress_limits.py +17 -0
- app/core/metadata_store.py +18 -0
- app/core/metrics/__init__.py +0 -0
- app/core/metrics/middleware.py +74 -0
- app/core/metrics/prometheus.py +604 -0
- app/core/middleware/__init__.py +25 -0
- app/core/middleware/api_firewall.py +111 -0
- app/core/middleware/app_version.py +37 -0
- app/core/middleware/dashboard_auth_proxy.py +55 -0
- app/core/middleware/dashboard_csrf.py +177 -0
- app/core/middleware/dashboard_gzip.py +41 -0
- app/core/middleware/dashboard_overrides.py +59 -0
- app/core/middleware/firewall_cache.py +91 -0
- app/core/middleware/inflight.py +126 -0
- app/core/middleware/multipart_content_encoding.py +129 -0
- app/core/middleware/path_rewrite.py +148 -0
- app/core/middleware/request_body_limit.py +196 -0
- app/core/middleware/request_decompression.py +209 -0
- app/core/middleware/request_id.py +72 -0
- app/core/middleware/required_capability_http.py +116 -0
- app/core/middleware/trusted_proxy_headers.py +36 -0
- app/core/multipart.py +237 -0
- app/core/multipart_fields.py +91 -0
- app/core/openai/__init__.py +0 -0
- app/core/openai/chat_requests.py +493 -0
- app/core/openai/chat_responses.py +931 -0
- app/core/openai/contracts.py +46 -0
- app/core/openai/exceptions.py +24 -0
- app/core/openai/host_models.py +14 -0
- app/core/openai/images.py +413 -0
- app/core/openai/message_coercion.py +376 -0
- app/core/openai/model_refresh_scheduler.py +423 -0
- app/core/openai/model_registry.py +997 -0
- app/core/openai/model_registry_store.py +548 -0
- app/core/openai/models.py +172 -0
- app/core/openai/parsing.py +97 -0
- app/core/openai/requests.py +1980 -0
- app/core/openai/strict_schema.py +199 -0
- app/core/openai/tool_call_safety.py +99 -0
- app/core/openai/v1_requests.py +187 -0
- app/core/plan_types.py +90 -0
- app/core/prestop.py +202 -0
- app/core/rate_limiter/__init__.py +0 -0
- app/core/rate_limiter/db_rate_limiter.py +215 -0
- app/core/request_locality.py +473 -0
- app/core/resilience/__init__.py +0 -0
- app/core/resilience/backpressure.py +57 -0
- app/core/resilience/bulkhead.py +202 -0
- app/core/resilience/circuit_breaker.py +150 -0
- app/core/resilience/degradation.py +55 -0
- app/core/resilience/loop_lag_monitor.py +60 -0
- app/core/resilience/memory_monitor.py +137 -0
- app/core/resilience/network_recovery.py +256 -0
- app/core/resilience/overload.py +98 -0
- app/core/resilience/toggles.py +114 -0
- app/core/retention/__init__.py +0 -0
- app/core/retention/job.py +240 -0
- app/core/retention/scheduler.py +69 -0
- app/core/runtime_logging.py +601 -0
- app/core/scheduling/__init__.py +0 -0
- app/core/scheduling/leader_election.py +993 -0
- app/core/scheduling/leader_election_handle.py +25 -0
- app/core/server.py +140 -0
- app/core/shutdown.py +326 -0
- app/core/socket_peer.py +22 -0
- app/core/startup.py +41 -0
- app/core/timeout_invariants.py +347 -0
- app/core/tracing/__init__.py +0 -0
- app/core/tracing/otel.py +207 -0
- app/core/types.py +6 -0
- app/core/upstream_proxy/__init__.py +13 -0
- app/core/upstream_proxy/cache.py +130 -0
- app/core/upstream_proxy/resolver.py +189 -0
- app/core/upstream_proxy/types.py +104 -0
- app/core/usage/__init__.py +407 -0
- app/core/usage/depletion.py +109 -0
- app/core/usage/live_hub.py +41 -0
- app/core/usage/live_snapshots.py +174 -0
- app/core/usage/logs.py +212 -0
- app/core/usage/metadata_scheduler.py +118 -0
- app/core/usage/models.py +54 -0
- app/core/usage/pricing.py +576 -0
- app/core/usage/pricing_catalog.py +241 -0
- app/core/usage/pricing_snapshot.json +821 -0
- app/core/usage/quota.py +133 -0
- app/core/usage/refresh_policy.py +21 -0
- app/core/usage/refresh_scheduler.py +713 -0
- app/core/usage/reset_credits_refresh_scheduler.py +435 -0
- app/core/usage/types.py +168 -0
- app/core/utils/__init__.py +0 -0
- app/core/utils/json_guards.py +18 -0
- app/core/utils/locks.py +33 -0
- app/core/utils/masking.py +20 -0
- app/core/utils/proxy_env.py +104 -0
- app/core/utils/request_id.py +65 -0
- app/core/utils/retry.py +77 -0
- app/core/utils/shared_future.py +172 -0
- app/core/utils/sse.py +286 -0
- app/core/utils/time.py +24 -0
- app/db/__init__.py +0 -0
- app/db/account_identity_lock.py +56 -0
- app/db/alembic/__init__.py +1 -0
- app/db/alembic/env.py +70 -0
- app/db/alembic/revision_ids.py +43 -0
- app/db/alembic/versions/20260213_000000_base_schema.py +248 -0
- app/db/alembic/versions/20260213_000100_normalize_account_plan_types.py +45 -0
- app/db/alembic/versions/20260213_000200_add_request_logs_reasoning_effort.py +45 -0
- app/db/alembic/versions/20260213_000300_add_accounts_reset_at.py +45 -0
- app/db/alembic/versions/20260213_000400_add_accounts_chatgpt_account_id.py +45 -0
- app/db/alembic/versions/20260213_000500_add_dashboard_settings.py +70 -0
- app/db/alembic/versions/20260213_000600_add_dashboard_settings_totp.py +64 -0
- app/db/alembic/versions/20260213_000700_add_dashboard_settings_password.py +59 -0
- app/db/alembic/versions/20260213_000800_add_api_keys.py +87 -0
- app/db/alembic/versions/20260214_000000_add_api_key_limits.py +143 -0
- app/db/alembic/versions/20260215_000000_add_idx_logs_requested_at.py +38 -0
- app/db/alembic/versions/20260218_000000_add_api_key_usage_reservations.py +117 -0
- app/db/alembic/versions/20260218_000100_add_import_without_overwrite_and_drop_accounts_email_unique.py +198 -0
- app/db/alembic/versions/20260225_000000_add_dashboard_settings_routing_strategy.py +53 -0
- app/db/alembic/versions/20260228_020000_align_api_key_limit_enum_types.py +104 -0
- app/db/alembic/versions/20260228_030000_add_api_firewall_allowlist.py +42 -0
- app/db/alembic/versions/20260306_000000_add_request_logs_service_tier.py +45 -0
- app/db/alembic/versions/20260307_000000_add_api_key_enforcement_fields.py +56 -0
- app/db/alembic/versions/20260308_000000_add_sqlite_performance_indexes.py +41 -0
- app/db/alembic/versions/20260309_000000_add_additional_usage_history.py +71 -0
- app/db/alembic/versions/20260309_000000_request_logs_nullable_account_id.py +28 -0
- app/db/alembic/versions/20260310_000000_add_request_logs_transport.py +45 -0
- app/db/alembic/versions/20260310_000000_fix_postgresql_enum_value_casing.py +79 -0
- app/db/alembic/versions/20260310_120000_add_sticky_session_kinds_and_affinity_ttl.py +110 -0
- app/db/alembic/versions/20260312_000000_add_additional_usage_quota_key.py +193 -0
- app/db/alembic/versions/20260312_000000_split_sticky_sessions_primary_key_by_kind.py +96 -0
- app/db/alembic/versions/20260312_010000_merge_additional_usage_and_sticky_session_heads.py +25 -0
- app/db/alembic/versions/20260312_020000_merge_request_logs_transport_and_feature_heads.py +25 -0
- app/db/alembic/versions/20260312_020000_merge_request_logs_transport_and_sticky_heads.py +25 -0
- app/db/alembic/versions/20260312_120000_add_dashboard_upstream_stream_transport.py +62 -0
- app/db/alembic/versions/20260313_024500_merge_request_log_transport_heads.py +27 -0
- app/db/alembic/versions/20260319_100937_increase_prompt_cache_ttl_to_1800s.py +73 -0
- app/db/alembic/versions/20260319_111500_merge_prompt_cache_ttl_and_upstream_transport_heads.py +25 -0
- app/db/alembic/versions/20260319_183000_normalize_sqlite_account_status_casing.py +48 -0
- app/db/alembic/versions/20260320_000000_add_request_log_requested_actual_tiers.py +51 -0
- app/db/alembic/versions/20260321_190000_tighten_dashboard_db_indexes.py +64 -0
- app/db/alembic/versions/20260321_210000_merge_request_log_tiers_and_dashboard_index_heads.py +25 -0
- app/db/alembic/versions/20260325_000000_add_request_log_cost.py +193 -0
- app/db/alembic/versions/20260328_000000_add_rate_limit_attempts.py +48 -0
- app/db/alembic/versions/20260328_120000_add_request_log_ttft.py +37 -0
- app/db/alembic/versions/20260328_130000_add_audit_logs_table.py +45 -0
- app/db/alembic/versions/20260328_140000_add_scheduler_leader_table.py +39 -0
- app/db/alembic/versions/20260330_000000_add_cache_locality_settings.py +69 -0
- app/db/alembic/versions/20260330_010000_merge_scheduler_leader_and_cache_locality_heads.py +25 -0
- app/db/alembic/versions/20260330_020000_add_bridge_ring_members.py +46 -0
- app/db/alembic/versions/20260401_000000_add_api_key_enforced_service_tier.py +52 -0
- app/db/alembic/versions/20260401_000000_add_cache_invalidation.py +41 -0
- app/db/alembic/versions/20260401_020000_merge_cache_invalidation_and_api_key_service_tier_heads.py +22 -0
- app/db/alembic/versions/20260402_000000_switch_dashboard_routing_default_to_capacity_weighted.py +66 -0
- app/db/alembic/versions/20260403_000000_add_credit_api_key_limit_values.py +44 -0
- app/db/alembic/versions/20260406_000000_add_api_key_account_assignments.py +57 -0
- app/db/alembic/versions/20260406_010000_add_api_key_assignment_scope_flag.py +75 -0
- app/db/alembic/versions/20260407_000000_add_bridge_gateway_safe_mode.py +51 -0
- app/db/alembic/versions/20260407_010000_merge_api_key_assignment_and_bridge_gateway_heads.py +25 -0
- app/db/alembic/versions/20260408_000000_switch_import_without_overwrite_default_to_true.py +73 -0
- app/db/alembic/versions/20260408_010000_merge_import_without_overwrite_and_assignment_heads.py +25 -0
- app/db/alembic/versions/20260409_000000_add_http_bridge_sessions.py +130 -0
- app/db/alembic/versions/20260409_000000_switch_sticky_threads_and_prefer_earlier_reset_defaults_to_true.py +89 -0
- app/db/alembic/versions/20260409_010000_add_dashboard_settings_bootstrap_token.py +57 -0
- app/db/alembic/versions/20260409_010000_merge_http_bridge_and_import_without_overwrite_heads.py +25 -0
- app/db/alembic/versions/20260409_020000_fix_http_bridge_last_seen_index.py +46 -0
- app/db/alembic/versions/20260410_000000_backfill_pristine_dashboard_settings_defaults.py +77 -0
- app/db/alembic/versions/20260410_030000_restore_import_without_overwrite_default_true.py +73 -0
- app/db/alembic/versions/20260410_040000_merge_dashboard_defaults_and_import_default_heads.py +25 -0
- app/db/alembic/versions/20260413_000000_add_accounts_blocked_at.py +44 -0
- app/db/alembic/versions/20260415_160000_add_automations_tables.py +189 -0
- app/db/alembic/versions/20260415_160000_add_request_logs_response_lookup_index.py +67 -0
- app/db/alembic/versions/20260417_000000_add_request_log_plan_type.py +44 -0
- app/db/alembic/versions/20260418_190000_add_automation_runs_cycle_key.py +87 -0
- app/db/alembic/versions/20260419_000000_add_automation_run_cycle_metadata.py +56 -0
- app/db/alembic/versions/20260419_020000_add_automation_run_cycles_snapshot_tables.py +342 -0
- app/db/alembic/versions/20260421_000000_add_dashboard_session_ttl_setting.py +51 -0
- app/db/alembic/versions/20260421_120000_merge_request_log_lookup_and_plan_type_heads.py +25 -0
- app/db/alembic/versions/20260421_130000_merge_automation_and_request_log_heads.py +25 -0
- app/db/alembic/versions/20260422_103000_normalize_legacy_automation_run_cycle_keys.py +648 -0
- app/db/alembic/versions/20260423_120000_add_api_key_limit_reset_at_index.py +43 -0
- app/db/alembic/versions/20260424_000000_merge_dashboard_session_ttl_and_request_log_heads.py +25 -0
- app/db/alembic/versions/20260426_000000_add_dashboard_relative_availability_settings.py +68 -0
- app/db/alembic/versions/20260427_130000_add_warmup_model_and_request_kind.py +75 -0
- app/db/alembic/versions/20260506_000000_add_account_routing_policy.py +37 -0
- app/db/alembic/versions/20260509_000000_add_api_key_traffic_class.py +37 -0
- app/db/alembic/versions/20260509_010000_add_additional_quota_routing_policies.py +37 -0
- app/db/alembic/versions/20260513_000000_add_accounts_alias.py +44 -0
- app/db/alembic/versions/20260513_000000_add_api_key_apply_to_codex_model.py +59 -0
- app/db/alembic/versions/20260514_000000_add_request_logs_api_key_time_index.py +43 -0
- app/db/alembic/versions/20260515_000000_add_prefer_earlier_reset_window.py +47 -0
- app/db/alembic/versions/20260515_000000_soft_delete_request_logs_on_account_delete.py +83 -0
- app/db/alembic/versions/20260515_020000_add_split_sticky_budget_thresholds.py +62 -0
- app/db/alembic/versions/20260516_000000_add_sqlite_hot_path_indexes.py +75 -0
- app/db/alembic/versions/20260517_000000_merge_sqlite_hot_path_and_request_log_delete_heads.py +24 -0
- app/db/alembic/versions/20260518_000000_add_http_bridge_durable_input_prefix.py +44 -0
- app/db/alembic/versions/20260518_010000_merge_http_bridge_and_sqlite_recovery_heads.py +25 -0
- app/db/alembic/versions/20260518_120000_add_dashboard_guest_access.py +57 -0
- app/db/alembic/versions/20260520_000000_merge_api_key_and_http_bridge_heads.py +26 -0
- app/db/alembic/versions/20260520_010000_add_request_logs_api_key_account_index.py +45 -0
- app/db/alembic/versions/20260520_010000_merge_automation_and_api_key_heads.py +26 -0
- app/db/alembic/versions/20260520_030000_add_quota_planner.py +194 -0
- app/db/alembic/versions/20260520_040000_merge_quota_planner_and_request_log_index_heads.py +24 -0
- app/db/alembic/versions/20260521_000000_add_account_security_work_authorized.py +46 -0
- app/db/alembic/versions/20260521_000000_merge_dashboard_guest_and_api_key_heads.py +26 -0
- app/db/alembic/versions/20260521_000000_merge_prefer_reset_window_head.py +26 -0
- app/db/alembic/versions/20260521_000000_merge_warmup_and_request_log_index_heads.py +25 -0
- app/db/alembic/versions/20260522_000000_add_limit_warmup_trigger.py +195 -0
- app/db/alembic/versions/20260524_000000_merge_accounts_alias_and_reset_window_heads.py +24 -0
- app/db/alembic/versions/20260525_000000_add_usage_raw_window_latest_index.py +31 -0
- app/db/alembic/versions/20260526_000000_add_request_log_failure_metadata.py +100 -0
- app/db/alembic/versions/20260531_000000_add_account_workspace_identity.py +39 -0
- app/db/alembic/versions/20260531_000000_add_single_account_routing.py +42 -0
- app/db/alembic/versions/20260601_000000_merge_relative_availability_and_usage_raw_heads.py +24 -0
- app/db/alembic/versions/20260601_010000_add_api_key_usage_sections.py +58 -0
- app/db/alembic/versions/20260601_010000_add_routing_policy_persistence.py +67 -0
- app/db/alembic/versions/20260601_010000_merge_security_work_and_relative_availability_heads.py +25 -0
- app/db/alembic/versions/20260601_010000_merge_split_sticky_and_relative_usage_heads.py +25 -0
- app/db/alembic/versions/20260601_010000_merge_warmup_and_relative_availability_heads.py +25 -0
- app/db/alembic/versions/20260601_020000_merge_additional_quota_routing_and_relative_availability_heads.py +25 -0
- app/db/alembic/versions/20260601_020000_merge_quota_planner_and_relative_availability_heads.py +25 -0
- app/db/alembic/versions/20260601_020000_merge_reset_window_and_relative_availability_heads.py +25 -0
- app/db/alembic/versions/20260601_020000_merge_routing_policy_and_split_threshold_heads.py +24 -0
- app/db/alembic/versions/20260601_020000_merge_warmup_and_request_log_failure_heads.py +25 -0
- app/db/alembic/versions/20260601_030000_merge_single_account_and_routing_quota_heads.py +29 -0
- app/db/alembic/versions/20260601_040000_merge_security_work_and_routing_consolidation_heads.py +25 -0
- app/db/alembic/versions/20260601_050000_merge_quota_planner_and_routing_consolidation_heads.py +25 -0
- app/db/alembic/versions/20260602_050000_add_upstream_proxy_routing.py +164 -0
- app/db/alembic/versions/20260602_060000_merge_account_workspace_and_failure_heads.py +25 -0
- app/db/alembic/versions/20260602_070000_add_request_log_useragent_fields.py +86 -0
- app/db/alembic/versions/20260602_070000_merge_quota_planner_and_account_workspace_heads.py +25 -0
- app/db/alembic/versions/20260602_080000_merge_upstream_proxy_and_quota_planner_heads.py +25 -0
- app/db/alembic/versions/20260602_080000_merge_useragent_and_upstream_proxy_heads.py +24 -0
- app/db/alembic/versions/20260603_000000_add_weekly_pace_working_days.py +52 -0
- app/db/alembic/versions/20260603_000000_free_account_monthly_window.py +60 -0
- app/db/alembic/versions/20260604_000000_add_reauth_required_account_status.py +159 -0
- app/db/alembic/versions/20260604_010000_merge_automations_and_reauth_status_heads.py +25 -0
- app/db/alembic/versions/20260604_010000_merge_dashboard_guest_and_reauth_status_heads.py +25 -0
- app/db/alembic/versions/20260604_010000_merge_useragent_and_reauth_heads.py +24 -0
- app/db/alembic/versions/20260607_000000_merge_weekly_monthly_useragent_heads.py +28 -0
- app/db/alembic/versions/20260608_000000_add_hide_upstream_quota_from_api_keys.py +50 -0
- app/db/alembic/versions/20260611_000000_add_request_log_archive_request_id.py +47 -0
- app/db/alembic/versions/20260611_000000_merge_automations_and_weekly_monthly_useragent_heads.py +26 -0
- app/db/alembic/versions/20260611_000000_merge_dashboard_guest_and_weekly_useragent_heads.py +26 -0
- app/db/alembic/versions/20260611_010000_add_request_log_client_ip.py +57 -0
- app/db/alembic/versions/20260611_020000_merge_request_log_archive_and_client_ip_heads.py +26 -0
- app/db/alembic/versions/20260611_030000_merge_dashboard_guest_and_request_log_client_ip_heads.py +26 -0
- app/db/alembic/versions/20260613_000000_add_accounts_codex_installation_id.py +57 -0
- app/db/alembic/versions/20260613_010000_merge_installation_id_and_dashboard_weekly_heads.py +26 -0
- app/db/alembic/versions/20260615_000000_merge_automations_and_dashboard_guest_heads.py +26 -0
- app/db/alembic/versions/20260623_000000_add_limit_warmup_exhausted_threshold.py +51 -0
- app/db/alembic/versions/20260626_000000_add_http_downstream_transport_policy.py +46 -0
- app/db/alembic/versions/20260626_010000_add_request_logs_upstream_transport.py +31 -0
- app/db/alembic/versions/20260629_000000_add_dashboard_query_hot_path_indexes.py +53 -0
- app/db/alembic/versions/20260630_000000_add_limit_warmup_staggered_idle.py +47 -0
- app/db/alembic/versions/20260630_000000_merge_request_log_client_ip_and_dashboard_hot_path_heads.py +26 -0
- app/db/alembic/versions/20260630_010000_merge_warmup_and_request_log_dashboard_heads.py +26 -0
- app/db/alembic/versions/20260630_020000_merge_automations_and_main_heads.py +26 -0
- app/db/alembic/versions/20260630_020000_merge_warmup_threshold_and_main_heads.py +26 -0
- app/db/alembic/versions/20260630_030000_add_automation_run_model_snapshot.py +59 -0
- app/db/alembic/versions/20260630_040000_merge_automation_snapshot_and_warmup_threshold_heads.py +26 -0
- app/db/alembic/versions/20260630_050000_add_automation_run_prompt_snapshot.py +52 -0
- app/db/alembic/versions/20260701_000000_add_weekly_pace_smoothing_minutes.py +51 -0
- app/db/alembic/versions/20260702_000000_add_model_sources.py +176 -0
- app/db/alembic/versions/20260705_000000_add_model_source_audio_transcriptions.py +50 -0
- app/db/alembic/versions/20260705_000000_harden_dashboard_session_ttl.py +62 -0
- app/db/alembic/versions/20260706_000000_add_model_source_audio_per_minute.py +43 -0
- app/db/alembic/versions/20260707_000000_merge_model_sources_and_session_ttl.py +24 -0
- app/db/alembic/versions/20260707_010000_merge_automations_and_model_sources_heads.py +24 -0
- app/db/alembic/versions/20260707_020000_add_automation_job_account_scope.py +125 -0
- app/db/alembic/versions/20260709_000000_add_ttft_phase_observability.py +66 -0
- app/db/alembic/versions/20260711_000000_add_dashboard_prohibit_fast_mode.py +42 -0
- app/db/alembic/versions/20260711_010000_add_account_chatgpt_user_id.py +40 -0
- app/db/alembic/versions/20260711_020000_add_dashboard_account_concurrency_caps.py +71 -0
- app/db/alembic/versions/20260711_030000_add_limit_warmup_idle_threshold.py +51 -0
- app/db/alembic/versions/20260712_010000_add_account_usage_rollups.py +56 -0
- app/db/alembic/versions/20260712_020000_add_api_key_usage_rollups.py +53 -0
- app/db/alembic/versions/20260713_020000_add_model_registry_snapshot.py +42 -0
- app/db/alembic/versions/20260713_040000_add_account_refresh_claims.py +42 -0
- app/db/alembic/versions/20260713_040000_add_replica_guardrails.py +64 -0
- app/db/alembic/versions/20260713_070000_add_reset_credit_redeem_tables.py +58 -0
- app/db/alembic/versions/20260714_000000_add_oauth_flow_states.py +63 -0
- app/db/alembic/versions/20260715_000000_add_request_log_queue_latency.py +44 -0
- app/db/alembic/versions/20260716_000000_add_oauth_device_flow_slots.py +41 -0
- app/db/alembic/versions/20260716_010000_add_dashboard_retention_settings.py +48 -0
- app/db/alembic/versions/20260716_010000_add_reset_credit_display_settings.py +73 -0
- app/db/alembic/versions/20260717_000000_add_http_bridge_retry_circuits.py +47 -0
- app/db/alembic/versions/20260717_000000_merge_retention_and_reset_credit_display_heads.py +26 -0
- app/db/alembic/versions/20260717_000000_optimize_dashboard_hot_path_indexes.py +162 -0
- app/db/alembic/versions/20260717_000001_merge_retry_circuits_and_dashboard_indexes.py +22 -0
- app/db/alembic/versions/20260720_000000_add_request_log_conversation_id.py +57 -0
- app/db/alembic/versions/20260722_000000_backfill_request_log_useragent_families.py +40 -0
- app/db/alembic/versions/20260724_000000_add_request_usage_time_rollups.py +176 -0
- app/db/alembic/versions/20260724_000000_merge_request_log_schema_heads.py +33 -0
- app/db/alembic/versions/20260725_000000_add_http_bridge_pending_tool_calls.py +43 -0
- app/db/alembic/versions/20260726_000000_add_account_plan_downgrade_observations.py +44 -0
- app/db/alembic/versions/20260726_000000_repair_request_usage_rollups_after_merge.py +37 -0
- app/db/alembic/versions/20260727_000000_add_sticky_session_continuity_abandoned_at.py +45 -0
- app/db/alembic/versions/20260728_000000_merge_pending_tool_calls_and_rollup_repair_heads.py +31 -0
- app/db/alembic/versions/20260730_000000_add_api_key_fair_share_threshold.py +44 -0
- app/db/alembic/versions/20260730_000000_add_http_bridge_recovery_attempts.py +60 -0
- app/db/alembic/versions/20260731_000000_add_capability_lineage_markers.py +38 -0
- app/db/alembic/versions/20260803_000000_merge_http_bridge_recovery_and_capability_lineage_heads.py +30 -0
- app/db/alembic/versions/20260804_000000_add_http_bridge_operations.py +63 -0
- app/db/alembic/versions/20260804_000001_add_global_http_bridge_operation_fingerprint.py +53 -0
- app/db/alembic/versions/20260804_230000_add_request_log_connection_request_kind.py +43 -0
- app/db/alembic/versions/20260805_000000_add_http_bridge_operation_spool.py +75 -0
- app/db/alembic/versions/20260805_000001_finalize_http_bridge_operation_spool.py +65 -0
- app/db/alembic/versions/20260806_000000_add_additional_usage_alias_probe_indexes.py +84 -0
- app/db/alembic/versions/20260806_000000_add_anonymous_telemetry.py +49 -0
- app/db/alembic/versions/20260806_010000_add_conversation_presence_rollup.py +72 -0
- app/db/alembic/versions/20260806_020000_add_usage_history_bulk_covering_indexes.py +99 -0
- app/db/alembic/versions/20260806_030000_add_api_key_allowed_reasoning_efforts.py +65 -0
- app/db/alembic/versions/20260806_120000_add_http_bridge_owner_process_epoch.py +57 -0
- app/db/alembic/versions/20260807_000000_merge_http_bridge_operation_spool_and_latest_main.py +24 -0
- app/db/alembic/versions/20260808_000000_tune_usage_history_autovacuum.py +54 -0
- app/db/alembic/versions/20260810_000000_add_http_bridge_recovery_dispatch_count.py +44 -0
- app/db/alembic/versions/20260811_000000_add_hourly_rollup_cancelled_count.py +92 -0
- app/db/alembic/versions/20260812_000000_merge_recovery_dispatch_and_hourly_cancelled_heads.py +25 -0
- app/db/alembic/versions/20260812_120000_add_sticky_abandonment_scope.py +43 -0
- app/db/alembic/versions/20260813_000000_add_file_account_pins.py +40 -0
- app/db/alembic/versions/20260816_000000_add_account_pending_deletion.py +81 -0
- app/db/alembic/versions/20260816_000000_add_model_source_embeddings.py +50 -0
- app/db/alembic/versions/20260821_000000_add_retry_circuit_admission_generation.py +49 -0
- app/db/alembic/versions/20260826_000000_add_http_bridge_event_chunks.py +107 -0
- app/db/alembic/versions/20260828_000000_add_accounts_chatgpt_identity_index.py +55 -0
- app/db/alembic/versions/20260830_000000_add_quota_warmup_claim_expiry.py +83 -0
- app/db/alembic/versions/20260908_000000_add_guest_session_generation.py +37 -0
- app/db/alembic/versions/20260908_000000_add_subscription_overflow.py +140 -0
- app/db/alembic/versions/20260908_000000_replace_upstream_stream_transport_default_sentinel.py +50 -0
- app/db/alembic/versions/20260908_020000_merge_overflow_transport_heads.py +27 -0
- app/db/alembic/versions/20260909_000000_add_dashboard_roles.py +72 -0
- app/db/alembic/versions/20260909_010000_add_dashboard_users.py +183 -0
- app/db/alembic/versions/20260909_020000_reproject_compat_admin_credentials.py +105 -0
- app/db/alembic/versions/20260909_030000_add_audit_actor_columns.py +103 -0
- app/db/alembic/versions/20260909_030000_dashboard_resilience_toggle_settings.py +63 -0
- app/db/alembic/versions/20260909_040000_add_dashboard_user_invites.py +54 -0
- app/db/alembic/versions/20260909_040000_dashboard_timeout_settings.py +68 -0
- app/db/alembic/versions/20260909_050000_dashboard_routing_overload_settings.py +65 -0
- app/db/alembic/versions/20260909_060000_add_report_rollup.py +74 -0
- app/db/alembic/versions/20260909_070000_automation_run_claim_budget.py +50 -0
- app/db/alembic/versions/20260909_080000_dashboard_stream_bridge_budgets.py +62 -0
- app/db/alembic/versions/20260909_090000_dashboard_background_job_toggles.py +63 -0
- app/db/alembic/versions/20260909_100000_dashboard_codex_prewarm.py +49 -0
- app/db/alembic/versions/20260909_110000_model_context_window_overrides.py +45 -0
- app/db/alembic/versions/20260909_120000_dashboard_conversation_archive.py +48 -0
- app/db/alembic/versions/20260909_130000_add_request_logs_live_facet_indexes.py +96 -0
- app/db/alembic/versions/20260910_000000_add_totp_required_for_admin_role.py +44 -0
- app/db/alembic/versions/20260910_000000_request_logs_missing_cost_index.py +52 -0
- app/db/alembic/versions/20260910_010000_add_dashboard_auth_providers.py +134 -0
- app/db/alembic/versions/20260910_010000_dashboard_spool_retention.py +52 -0
- app/db/alembic/versions/20260910_020000_add_dashboard_role_mappings.py +74 -0
- app/db/alembic/versions/20260910_143000_request_log_affinity.py +28 -0
- app/db/alembic/versions/20260910_180000_merge_affinity_guest_heads.py +17 -0
- app/db/alembic/versions/20260910_200000_merge_affinity_identity_heads.py +17 -0
- app/db/alembic/versions/20260910_220000_merge_affinity_invite_heads.py +17 -0
- app/db/alembic/versions/20260911_000000_model_source_pins_kind_expires_index.py +43 -0
- app/db/alembic/versions/20260911_010000_merge_pin_index_and_affinity_heads.py +17 -0
- app/db/alembic/versions/20260911_020000_add_http_bridge_terminal_append_phase.py +54 -0
- app/db/alembic/versions/20260911_030000_add_local_login_policy.py +48 -0
- app/db/alembic/versions/20260911_040000_add_thread_cache_identity_mode.py +48 -0
- app/db/alembic/versions/20260911_060000_add_bridge_session_continuity_abandonment.py +40 -0
- app/db/alembic/versions/20260912_000000_merge_thread_cache_and_bridge_retirement_heads.py +22 -0
- app/db/alembic/versions/20260912_010000_drop_legacy_dashboard_credentials.py +140 -0
- app/db/alembic/versions/20260913_000000_add_oidc_provider_flow.py +134 -0
- app/db/alembic/versions/20260914_000000_add_scim_tokens.py +78 -0
- app/db/alembic/versions/20260914_000000_drop_subscription_overflow_schema.py +132 -0
- app/db/alembic/versions/20260918_000000_merge_scim_and_overflow_heads.py +17 -0
- app/db/alembic/versions/__init__.py +1 -0
- app/db/backup.py +64 -0
- app/db/migrate.py +1249 -0
- app/db/migration_lock.py +160 -0
- app/db/migration_url.py +18 -0
- app/db/models.py +3100 -0
- app/db/recover.py +244 -0
- app/db/session.py +1295 -0
- app/db/snapshot.py +46 -0
- app/db/sqlite_lock_retry.py +190 -0
- app/db/sqlite_utils.py +549 -0
- app/dependencies.py +462 -0
- app/main.py +1240 -0
- app/modules/__init__.py +0 -0
- app/modules/accounts/__init__.py +0 -0
- app/modules/accounts/api.py +426 -0
- app/modules/accounts/auth_manager.py +1439 -0
- app/modules/accounts/background_repository.py +160 -0
- app/modules/accounts/deletion.py +540 -0
- app/modules/accounts/mappers.py +654 -0
- app/modules/accounts/refresh_claims.py +352 -0
- app/modules/accounts/repository.py +1679 -0
- app/modules/accounts/schemas.py +308 -0
- app/modules/accounts/service.py +809 -0
- app/modules/accounts/usage_rollup.py +486 -0
- app/modules/accounts/usage_rollup_scheduler.py +75 -0
- app/modules/accounts/usage_time_rollup.py +1037 -0
- app/modules/accounts/usage_time_rollup_read.py +522 -0
- app/modules/api_keys/__init__.py +1 -0
- app/modules/api_keys/api.py +344 -0
- app/modules/api_keys/last_used_coalescer.py +216 -0
- app/modules/api_keys/limit_windows.py +41 -0
- app/modules/api_keys/repository.py +1200 -0
- app/modules/api_keys/reset_scheduler.py +85 -0
- app/modules/api_keys/schemas.py +135 -0
- app/modules/api_keys/service.py +2123 -0
- app/modules/audit/__init__.py +0 -0
- app/modules/audit/api.py +88 -0
- app/modules/audit/repository.py +74 -0
- app/modules/audit/schemas.py +33 -0
- app/modules/audit/service.py +71 -0
- app/modules/auth_providers/__init__.py +0 -0
- app/modules/auth_providers/api.py +127 -0
- app/modules/auth_providers/config.py +343 -0
- app/modules/auth_providers/repository.py +101 -0
- app/modules/auth_providers/schemas.py +88 -0
- app/modules/auth_providers/seed.py +105 -0
- app/modules/auth_providers/service.py +289 -0
- app/modules/automations/__init__.py +1 -0
- app/modules/automations/api.py +369 -0
- app/modules/automations/repository.py +1896 -0
- app/modules/automations/scheduler.py +107 -0
- app/modules/automations/schemas.py +158 -0
- app/modules/automations/service.py +2548 -0
- app/modules/cache_isolation_probe/__init__.py +0 -0
- app/modules/cache_isolation_probe/api.py +188 -0
- app/modules/cache_isolation_probe/prefix.py +144 -0
- app/modules/cache_isolation_probe/schemas.py +94 -0
- app/modules/cache_isolation_probe/sender.py +229 -0
- app/modules/cache_isolation_probe/service.py +525 -0
- app/modules/conversation_archive/__init__.py +1 -0
- app/modules/conversation_archive/api.py +101 -0
- app/modules/conversation_archive/schemas.py +28 -0
- app/modules/conversation_archive/service.py +165 -0
- app/modules/dashboard/__init__.py +1 -0
- app/modules/dashboard/api.py +106 -0
- app/modules/dashboard/builders.py +107 -0
- app/modules/dashboard/repository.py +304 -0
- app/modules/dashboard/schemas.py +148 -0
- app/modules/dashboard/service.py +534 -0
- app/modules/dashboard/timeframes.py +19 -0
- app/modules/dashboard/weekly_pace.py +617 -0
- app/modules/dashboard_auth/__init__.py +1 -0
- app/modules/dashboard_auth/api.py +1344 -0
- app/modules/dashboard_auth/oidc_api.py +753 -0
- app/modules/dashboard_auth/oidc_flows.py +330 -0
- app/modules/dashboard_auth/repository.py +452 -0
- app/modules/dashboard_auth/schemas.py +232 -0
- app/modules/dashboard_auth/service.py +1384 -0
- app/modules/dashboard_roles/__init__.py +0 -0
- app/modules/dashboard_roles/api.py +64 -0
- app/modules/dashboard_roles/repository.py +54 -0
- app/modules/dashboard_roles/schemas.py +29 -0
- app/modules/dashboard_roles/seed.py +80 -0
- app/modules/dashboard_roles/service.py +142 -0
- app/modules/dashboard_users/__init__.py +0 -0
- app/modules/dashboard_users/api.py +278 -0
- app/modules/dashboard_users/break_glass.py +169 -0
- app/modules/dashboard_users/credentials.py +18 -0
- app/modules/dashboard_users/identity_resolver.py +665 -0
- app/modules/dashboard_users/repository.py +725 -0
- app/modules/dashboard_users/schemas.py +117 -0
- app/modules/dashboard_users/service.py +1054 -0
- app/modules/firewall/__init__.py +1 -0
- app/modules/firewall/api.py +78 -0
- app/modules/firewall/repository.py +51 -0
- app/modules/firewall/schemas.py +28 -0
- app/modules/firewall/service.py +93 -0
- app/modules/fleet/__init__.py +1 -0
- app/modules/fleet/api.py +177 -0
- app/modules/fleet/mappers.py +56 -0
- app/modules/fleet/observability.py +372 -0
- app/modules/fleet/schemas.py +119 -0
- app/modules/health/__init__.py +0 -0
- app/modules/health/api.py +246 -0
- app/modules/health/schemas.py +27 -0
- app/modules/limit_warmup/__init__.py +1 -0
- app/modules/limit_warmup/repository.py +159 -0
- app/modules/limit_warmup/service.py +1001 -0
- app/modules/model_sources/__init__.py +1 -0
- app/modules/model_sources/api.py +99 -0
- app/modules/model_sources/catalog.py +307 -0
- app/modules/model_sources/forwarding.py +1435 -0
- app/modules/model_sources/projection.py +55 -0
- app/modules/model_sources/repository.py +201 -0
- app/modules/model_sources/schemas.py +79 -0
- app/modules/model_sources/selection.py +166 -0
- app/modules/model_sources/service.py +236 -0
- app/modules/oauth/__init__.py +0 -0
- app/modules/oauth/api.py +101 -0
- app/modules/oauth/repository.py +238 -0
- app/modules/oauth/schemas.py +49 -0
- app/modules/oauth/service.py +1196 -0
- app/modules/oauth/templates/oauth_success.html +122 -0
- app/modules/proxy/__init__.py +0 -0
- app/modules/proxy/_load_balancer/__init__.py +1 -0
- app/modules/proxy/_load_balancer/error_rate.py +116 -0
- app/modules/proxy/_load_balancer/latency_cohort.py +154 -0
- app/modules/proxy/_load_balancer/model_eligibility.py +250 -0
- app/modules/proxy/_load_balancer/opportunistic_admission.py +350 -0
- app/modules/proxy/_load_balancer/overload_backoff.py +394 -0
- app/modules/proxy/_load_balancer/sticky_selection.py +2417 -0
- app/modules/proxy/_load_balancer/throughput_cohort.py +233 -0
- app/modules/proxy/_load_balancer/ttft_cohort.py +214 -0
- app/modules/proxy/_load_balancer/tunables.py +97 -0
- app/modules/proxy/_load_balancer/types.py +96 -0
- app/modules/proxy/_load_balancer/unbound_selection.py +534 -0
- app/modules/proxy/_service/__init__.py +6 -0
- app/modules/proxy/_service/api_key_usage.py +745 -0
- app/modules/proxy/_service/clock_budget.py +38 -0
- app/modules/proxy/_service/codex_control.py +611 -0
- app/modules/proxy/_service/compact.py +2211 -0
- app/modules/proxy/_service/file_ops.py +749 -0
- app/modules/proxy/_service/http_bridge/__init__.py +3 -0
- app/modules/proxy/_service/http_bridge/accepted_replay.py +331 -0
- app/modules/proxy/_service/http_bridge/account_sessions.py +57 -0
- app/modules/proxy/_service/http_bridge/activity.py +178 -0
- app/modules/proxy/_service/http_bridge/helpers.py +3959 -0
- app/modules/proxy/_service/http_bridge/mixin.py +2435 -0
- app/modules/proxy/_service/http_bridge/owner_forwarding.py +609 -0
- app/modules/proxy/_service/http_bridge/protocol.py +167 -0
- app/modules/proxy/_service/http_bridge/proxy_failover.py +58 -0
- app/modules/proxy/_service/http_bridge/quarantine.py +443 -0
- app/modules/proxy/_service/http_bridge/request_submit.py +4582 -0
- app/modules/proxy/_service/http_bridge/retry_circuit.py +2442 -0
- app/modules/proxy/_service/http_bridge/service_stubs.py +492 -0
- app/modules/proxy/_service/http_bridge/session_registry.py +778 -0
- app/modules/proxy/_service/http_bridge/streaming.py +5328 -0
- app/modules/proxy/_service/http_bridge/upstream_events.py +4538 -0
- app/modules/proxy/_service/observability.py +410 -0
- app/modules/proxy/_service/rate_limit.py +361 -0
- app/modules/proxy/_service/realtime_live.py +655 -0
- app/modules/proxy/_service/refresh.py +154 -0
- app/modules/proxy/_service/request_log.py +585 -0
- app/modules/proxy/_service/response_create.py +930 -0
- app/modules/proxy/_service/streaming/__init__.py +3 -0
- app/modules/proxy/_service/streaming/helpers.py +1317 -0
- app/modules/proxy/_service/streaming/mixin.py +1100 -0
- app/modules/proxy/_service/streaming/protocol.py +32 -0
- app/modules/proxy/_service/streaming/retry.py +3762 -0
- app/modules/proxy/_service/support.py +2122 -0
- app/modules/proxy/_service/transcribe.py +434 -0
- app/modules/proxy/_service/warmup.py +483 -0
- app/modules/proxy/_service/websocket/__init__.py +3 -0
- app/modules/proxy/_service/websocket/helpers.py +2360 -0
- app/modules/proxy/_service/websocket/mixin.py +7285 -0
- app/modules/proxy/_service/websocket/protocol.py +74 -0
- app/modules/proxy/account_cache.py +229 -0
- app/modules/proxy/account_eligibility.py +87 -0
- app/modules/proxy/additional_model_limits.py +40 -0
- app/modules/proxy/affinity.py +830 -0
- app/modules/proxy/affinity_observation.py +101 -0
- app/modules/proxy/api.py +10254 -0
- app/modules/proxy/api_key_usage.py +113 -0
- app/modules/proxy/cap_partitioning.py +280 -0
- app/modules/proxy/capability_lineage.py +65 -0
- app/modules/proxy/capability_lineage_repository.py +82 -0
- app/modules/proxy/capability_routing.py +253 -0
- app/modules/proxy/continuity.py +112 -0
- app/modules/proxy/downstream_delivery.py +277 -0
- app/modules/proxy/durable_bridge_coordinator.py +1217 -0
- app/modules/proxy/durable_bridge_repository.py +4516 -0
- app/modules/proxy/durable_bridge_runtime.py +9 -0
- app/modules/proxy/durable_bridge_transcript_codec.py +122 -0
- app/modules/proxy/fair_share.py +131 -0
- app/modules/proxy/file_pin_repository.py +210 -0
- app/modules/proxy/helpers.py +439 -0
- app/modules/proxy/http_bridge_event_batcher.py +613 -0
- app/modules/proxy/http_bridge_forwarding.py +797 -0
- app/modules/proxy/http_continuation.py +76 -0
- app/modules/proxy/images_observability.py +62 -0
- app/modules/proxy/images_service.py +1054 -0
- app/modules/proxy/load_balancer.py +3016 -0
- app/modules/proxy/rate_limit_cache.py +46 -0
- app/modules/proxy/replay_safety.py +1063 -0
- app/modules/proxy/repo_bundle.py +31 -0
- app/modules/proxy/request_policy.py +1054 -0
- app/modules/proxy/ring_membership.py +251 -0
- app/modules/proxy/schemas.py +345 -0
- app/modules/proxy/selection_errors.py +41 -0
- app/modules/proxy/service.py +2600 -0
- app/modules/proxy/source_admission.py +125 -0
- app/modules/proxy/source_dispatch.py +999 -0
- app/modules/proxy/sticky_repository.py +831 -0
- app/modules/proxy/thread_anchors.py +491 -0
- app/modules/proxy/tool_call_dedupe.py +802 -0
- app/modules/proxy/types.py +54 -0
- app/modules/proxy/work_admission.py +121 -0
- app/modules/quota_planner/__init__.py +1 -0
- app/modules/quota_planner/api.py +276 -0
- app/modules/quota_planner/logic.py +743 -0
- app/modules/quota_planner/repository.py +722 -0
- app/modules/quota_planner/scheduler.py +212 -0
- app/modules/quota_planner/schemas.py +102 -0
- app/modules/quota_planner/warmup.py +693 -0
- app/modules/rate_limit_reset_credits/__init__.py +0 -0
- app/modules/rate_limit_reset_credits/api.py +601 -0
- app/modules/rate_limit_reset_credits/redeem_coordination.py +274 -0
- app/modules/rate_limit_reset_credits/store.py +114 -0
- app/modules/reports/__init__.py +1 -0
- app/modules/reports/api.py +134 -0
- app/modules/reports/cache.py +74 -0
- app/modules/reports/filters.py +27 -0
- app/modules/reports/repository.py +525 -0
- app/modules/reports/rollup.py +150 -0
- app/modules/reports/rollup_read.py +148 -0
- app/modules/reports/schemas.py +123 -0
- app/modules/reports/service.py +287 -0
- app/modules/reports/thread_identity.py +256 -0
- app/modules/request_logs/__init__.py +0 -0
- app/modules/request_logs/api.py +224 -0
- app/modules/request_logs/cost_backfill.py +154 -0
- app/modules/request_logs/mappers.py +97 -0
- app/modules/request_logs/repository.py +1765 -0
- app/modules/request_logs/schemas.py +145 -0
- app/modules/request_logs/service.py +348 -0
- app/modules/role_mappings/__init__.py +0 -0
- app/modules/role_mappings/api.py +206 -0
- app/modules/role_mappings/matching.py +55 -0
- app/modules/role_mappings/repository.py +106 -0
- app/modules/role_mappings/schemas.py +67 -0
- app/modules/role_mappings/service.py +334 -0
- app/modules/runtime/__init__.py +1 -0
- app/modules/runtime/api.py +18 -0
- app/modules/runtime/schemas.py +14 -0
- app/modules/runtime/service.py +218 -0
- app/modules/scim/__init__.py +1 -0
- app/modules/scim/api.py +194 -0
- app/modules/scim/dependencies.py +112 -0
- app/modules/scim/errors.py +77 -0
- app/modules/scim/management_api.py +156 -0
- app/modules/scim/repository.py +212 -0
- app/modules/scim/schemas.py +145 -0
- app/modules/scim/service.py +786 -0
- app/modules/scim/tokens.py +43 -0
- app/modules/settings/__init__.py +1 -0
- app/modules/settings/api.py +1677 -0
- app/modules/settings/repository.py +606 -0
- app/modules/settings/schemas.py +496 -0
- app/modules/settings/service.py +769 -0
- app/modules/shared/__init__.py +0 -0
- app/modules/shared/schemas.py +22 -0
- app/modules/sticky_sessions/__init__.py +1 -0
- app/modules/sticky_sessions/api.py +118 -0
- app/modules/sticky_sessions/cleanup_scheduler.py +487 -0
- app/modules/sticky_sessions/schemas.py +78 -0
- app/modules/sticky_sessions/service.py +196 -0
- app/modules/telemetry/__init__.py +5 -0
- app/modules/telemetry/api.py +115 -0
- app/modules/telemetry/clients.py +62 -0
- app/modules/telemetry/consent.py +115 -0
- app/modules/telemetry/scheduler.py +85 -0
- app/modules/telemetry/schemas.py +163 -0
- app/modules/telemetry/sender.py +221 -0
- app/modules/telemetry/snapshot.py +490 -0
- app/modules/usage/__init__.py +0 -0
- app/modules/usage/additional_quota_keys.py +379 -0
- app/modules/usage/api.py +45 -0
- app/modules/usage/background_repository.py +147 -0
- app/modules/usage/builders.py +559 -0
- app/modules/usage/depletion_service.py +277 -0
- app/modules/usage/live_ingest.py +356 -0
- app/modules/usage/mappers.py +29 -0
- app/modules/usage/plan_downgrade_observations.py +488 -0
- app/modules/usage/repository.py +1890 -0
- app/modules/usage/schemas.py +68 -0
- app/modules/usage/service.py +102 -0
- app/modules/usage/updater.py +1488 -0
- app/static/assets/Trans-Cj-J9mVA.js +1 -0
- app/static/assets/access-page-DHzZiXZi.js +1 -0
- app/static/assets/account-cost-donut-DGhfA4U8.js +1 -0
- app/static/assets/account-identifiers-CxbdLshZ.js +1 -0
- app/static/assets/account-status-DIsU5sUS.js +1 -0
- app/static/assets/account-trend-chart-BptlfwwQ.js +1 -0
- app/static/assets/accounts-page-Bbai8ift.js +3 -0
- app/static/assets/api-Dwa7XxPe.js +1 -0
- app/static/assets/api-client-dugX1Pti.js +55 -0
- app/static/assets/api-key-create-dialog-DnRm0oYN.js +1 -0
- app/static/assets/api-key-created-dialog-B6jkDG9f.js +1 -0
- app/static/assets/api-key-edit-dialog-Ck3Bnc3V.js +1 -0
- app/static/assets/api-trend-chart-BchCw7Hj.js +1 -0
- app/static/assets/apis-page-BvXz9c6i.js +2 -0
- app/static/assets/arrow-up-Dqh_KzJo.js +1 -0
- app/static/assets/arrow-up-down-LBRgVDGY.js +1 -0
- app/static/assets/automations-page-DQgNAJOR.js +3 -0
- app/static/assets/badge-DqMb9es0.js +1 -0
- app/static/assets/blur-email-HEvKagG2.js +1 -0
- app/static/assets/button-BjVmmiGW.js +1 -0
- app/static/assets/checkbox-apsl9vzk.js +1 -0
- app/static/assets/chevron-left-1d_pHVa5.js +1 -0
- app/static/assets/circle-alert-BS0xd_o1.js +1 -0
- app/static/assets/clsx-DB0hHKMi.js +1 -0
- app/static/assets/colors-BBy8GEtr.js +1 -0
- app/static/assets/copy-button-D7nAvOKR.js +1 -0
- app/static/assets/cost-per-day-chart-YdqlkZVM.js +1 -0
- app/static/assets/createLucideIcon-DBLzLNh2.js +1 -0
- app/static/assets/dashboard-page-CSdhBy0f.js +2 -0
- app/static/assets/distribution-metric-format-yZTy1ZEu.js +1 -0
- app/static/assets/donut-chart-Bxc69cra.js +1 -0
- app/static/assets/download-kQteIcsB.js +1 -0
- app/static/assets/ellipsis-B_5OgIre.js +1 -0
- app/static/assets/empty-state-JcA7tc8m.js +1 -0
- app/static/assets/es2015-bMH8YKZT.js +41 -0
- app/static/assets/es6-B264ni6W.js +69 -0
- app/static/assets/external-link-UVAOgMQ0.js +1 -0
- app/static/assets/i18n-BWqZ9Ua2.js +1 -0
- app/static/assets/inbox-B_RM2s1J.js +1 -0
- app/static/assets/index-DIpd264Z.js +23 -0
- app/static/assets/index-DMeLKRJx.css +2 -0
- app/static/assets/inherit-badge-QhTYVaU6.js +1 -0
- app/static/assets/invite-dialog-0CYh0GiQ.js +1 -0
- app/static/assets/iso-Bpe5Ihbb.js +1 -0
- app/static/assets/label-CGaLmLNf.js +1 -0
- app/static/assets/lazy-recharts-Pv8p9ArM.js +2 -0
- app/static/assets/loading-overlay-CzKQfHLI.js +1 -0
- app/static/assets/mini-quota-bar-cGT_t8Ae.js +1 -0
- app/static/assets/model-distribution-donut-BKjAfT48.js +1 -0
- app/static/assets/model-source-multi-select-D2d06Vc7.js +1 -0
- app/static/assets/multi-select-filter-KIA5qU6Z.js +1 -0
- app/static/assets/oauth-dialog-XeaGGDjS.js +1 -0
- app/static/assets/pagination-controls-D1Ypd0RU.js +1 -0
- app/static/assets/payload-CIYnGZz9.js +1 -0
- app/static/assets/queue-wait-chart-VoV7Yg9L.js +1 -0
- app/static/assets/report-chart-card-BXDliwcc.js +1 -0
- app/static/assets/reports-page-CGCJeMHG.js +3 -0
- app/static/assets/reset-credit-confirm-dialog-QOifTnVB.js +1 -0
- app/static/assets/rolldown-runtime-hePW80VL.js +1 -0
- app/static/assets/search-HfhqwERt.js +1 -0
- app/static/assets/select-Cl33bz-g.js +1 -0
- app/static/assets/settings-page-mZj01HYl.js +2 -0
- app/static/assets/shim-CZwyM4Q7.js +1 -0
- app/static/assets/skeleton-C_EUc4Rp.js +1 -0
- app/static/assets/sparkline-chart-DdNNsddM.js +1 -0
- app/static/assets/status-badge-CHHA5X6c.js +1 -0
- app/static/assets/switch-CzxgokcM.js +1 -0
- app/static/assets/table-DXaNSQ0G.js +1 -0
- app/static/assets/time-to-first-token-chart--68L0u_v.js +1 -0
- app/static/assets/tokens-per-day-chart-Cs82BFuE.js +1 -0
- app/static/assets/tokens-per-second-chart-AD2dfOn6.js +1 -0
- app/static/assets/totp-settings-CRHtxQ7j.js +1 -0
- app/static/assets/trash-D3KyWr1h.js +1 -0
- app/static/assets/use-account-quota-display-BPmrTBTC.js +1 -0
- app/static/assets/use-accounts-PWlVGw3z.js +1 -0
- app/static/assets/use-auth-CuDlW3iC.js +1 -0
- app/static/assets/use-chart-colors-DIu6tClI.js +1 -0
- app/static/assets/use-model-sources-Bul24JBu.js +1 -0
- app/static/assets/use-models-DUCPUuZg.js +1 -0
- app/static/assets/use-reduced-motion-hAm5r6_d.js +1 -0
- app/static/assets/useTranslation--56nHhlH.js +1 -0
- app/static/assets/useragent-distribution-donut-Dup6sf_n.js +1 -0
- app/static/assets/users-uIC3TIlE.js +1 -0
- app/static/assets/vendor-query-_bHZuxEx.js +1 -0
- app/static/assets/vendor-react-BVL0YZPW.js +9 -0
- app/static/favicon.svg +10 -0
- app/static/fonts/GeistSans-Variable.woff2 +0 -0
- app/static/fonts/jetbrains-mono-latin-ext.woff2 +0 -0
- app/static/fonts/jetbrains-mono-latin.woff2 +0 -0
- app/static/index.html +29 -0
- codexlb_up-1.25.0b10.dist-info/METADATA +124 -0
- codexlb_up-1.25.0b10.dist-info/RECORD +827 -0
- codexlb_up-1.25.0b10.dist-info/WHEEL +4 -0
- codexlb_up-1.25.0b10.dist-info/entry_points.txt +3 -0
- codexlb_up-1.25.0b10.dist-info/licenses/LICENSE +21 -0
- config/additional_quota_registry.json +11 -0
|
@@ -0,0 +1,4516 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
import asyncio
|
|
4
|
+
import json
|
|
5
|
+
import time
|
|
6
|
+
from collections.abc import Collection, Mapping, Sequence
|
|
7
|
+
from dataclasses import dataclass
|
|
8
|
+
from datetime import datetime, timedelta
|
|
9
|
+
from enum import StrEnum
|
|
10
|
+
from hashlib import sha256
|
|
11
|
+
from typing import Any, cast
|
|
12
|
+
|
|
13
|
+
from sqlalchemy import Row, and_, case, delete, exists, func, or_, select, text, true, update
|
|
14
|
+
from sqlalchemy.dialects.postgresql import insert as pg_insert
|
|
15
|
+
from sqlalchemy.dialects.sqlite import insert as sqlite_insert
|
|
16
|
+
from sqlalchemy.exc import IntegrityError
|
|
17
|
+
from sqlalchemy.ext.asyncio import AsyncSession
|
|
18
|
+
|
|
19
|
+
from app.core.utils.time import naive_utc_to_epoch, to_utc_naive, utcnow
|
|
20
|
+
from app.db.models import (
|
|
21
|
+
HTTP_BRIDGE_SPOOL_FORMAT_CHUNKS_V2,
|
|
22
|
+
HTTP_BRIDGE_SPOOL_FORMAT_ROWS_V1,
|
|
23
|
+
HTTP_BRIDGE_TERMINAL_APPEND_PHASE_APPENDED,
|
|
24
|
+
HTTP_BRIDGE_TERMINAL_APPEND_PHASE_PENDING,
|
|
25
|
+
HTTP_BRIDGE_TERMINAL_APPEND_PHASE_SETTLED,
|
|
26
|
+
Account,
|
|
27
|
+
HttpBridgeOperationEvent,
|
|
28
|
+
HttpBridgeOperationEventChunk,
|
|
29
|
+
HttpBridgeOperationRecord,
|
|
30
|
+
HttpBridgeRecoveryAttemptRecord,
|
|
31
|
+
HttpBridgeRecoveryAttemptState,
|
|
32
|
+
HttpBridgeRetryCircuit,
|
|
33
|
+
HttpBridgeSessionAlias,
|
|
34
|
+
HttpBridgeSessionRecord,
|
|
35
|
+
HttpBridgeSessionState,
|
|
36
|
+
)
|
|
37
|
+
from app.db.session import sqlite_writer_section
|
|
38
|
+
from app.modules.proxy.account_eligibility import HARD_OWNER_UNAVAILABLE_STATUSES
|
|
39
|
+
from app.modules.proxy.continuity import (
|
|
40
|
+
HTTP_BRIDGE_ACCOUNT_NEUTRAL_REPLAY_KEY_PREFIX,
|
|
41
|
+
HTTP_BRIDGE_ACCOUNT_NEUTRAL_REPLAY_KIND,
|
|
42
|
+
HTTP_BRIDGE_ACCOUNT_NEUTRAL_REPLAY_REBINDABLE_KINDS,
|
|
43
|
+
is_http_bridge_account_neutral_replay,
|
|
44
|
+
)
|
|
45
|
+
from app.modules.proxy.durable_bridge_transcript_codec import (
|
|
46
|
+
DURABLE_BRIDGE_TRANSCRIPT_MAX_EVENTS,
|
|
47
|
+
DurableBridgeTranscriptDecodeError,
|
|
48
|
+
decode_durable_bridge_transcript_chunk,
|
|
49
|
+
encode_durable_bridge_transcript_chunk,
|
|
50
|
+
)
|
|
51
|
+
|
|
52
|
+
_ANONYMOUS_API_KEY_SCOPE = "__anonymous__"
|
|
53
|
+
REQUIRED_DURABLE_BRIDGE_TABLES = (
|
|
54
|
+
"http_bridge_sessions",
|
|
55
|
+
"http_bridge_session_aliases",
|
|
56
|
+
"http_bridge_retry_circuits",
|
|
57
|
+
"http_bridge_recovery_attempts",
|
|
58
|
+
"http_bridge_operations",
|
|
59
|
+
"http_bridge_operation_events",
|
|
60
|
+
"http_bridge_operation_event_chunks",
|
|
61
|
+
)
|
|
62
|
+
DURABLE_BRIDGE_RETRY_CIRCUIT_STATE_TTL_SECONDS = 3600.0
|
|
63
|
+
# Mirrors the HTTP bridge retry circuit's abandonment tombstone detail; the
|
|
64
|
+
# scheduled purge preserves such rows until the bridge-retention cutoff.
|
|
65
|
+
_RETRY_CIRCUIT_ABANDONED_TOMBSTONE_DETAIL = "anchor_abandoned"
|
|
66
|
+
DURABLE_BRIDGE_OPERATION_SPOOL_PURGE_BATCH_SIZE = 50
|
|
67
|
+
_PURGE_CLOSED_BATCH_SIZE = 500
|
|
68
|
+
# Marks a retirement taken on the request path rather than by the sweep.
|
|
69
|
+
# The sweep writes the global timestamp form instead; both read as retired.
|
|
70
|
+
_REQUEST_PATH_ABANDONMENT_SCOPE = "request_path"
|
|
71
|
+
# Claim retry budget: insert races and epoch-CAS losses re-read and retry;
|
|
72
|
+
# each round has a winner, so a small budget converges under any realistic
|
|
73
|
+
# same-row claim contention.
|
|
74
|
+
_CLAIM_CAS_ATTEMPTS = 5
|
|
75
|
+
_SESSION_ID_LOOKUP_CHUNK_SIZE = 500
|
|
76
|
+
# Keep expanding ``IN`` predicates below the smallest supported SQLite bind
|
|
77
|
+
# budget. The bridge heartbeat can collect IDs from every pending request and
|
|
78
|
+
# event-spool context, so the caller's protection snapshot is not inherently
|
|
79
|
+
# bounded by this repository's batch size.
|
|
80
|
+
_PROTECTED_OPERATION_ID_SAFE_LIMIT = _SESSION_ID_LOOKUP_CHUNK_SIZE
|
|
81
|
+
# An oversized protection snapshot is scanned in small maintenance slices. The
|
|
82
|
+
# coordinator resumes from the returned keyset cursor on the next heartbeat so
|
|
83
|
+
# a large protected prefix cannot hold the SQLite writer section indefinitely.
|
|
84
|
+
_PROTECTED_OPERATION_SCAN_BUDGET = 128
|
|
85
|
+
_ABANDONMENT_LOG_AGE_CAP_SECONDS = 30 * 24 * 60 * 60
|
|
86
|
+
# Operation states that publish a final upstream outcome. ``abandoned`` is a
|
|
87
|
+
# duplicate-suppression fence rather than an outcome and is handled separately.
|
|
88
|
+
_HTTP_BRIDGE_TERMINAL_OPERATION_STATES = frozenset({"completed", "incomplete", "failed"})
|
|
89
|
+
# A terminal transcript outcome was already recorded for this dispatch, so a
|
|
90
|
+
# second terminal append must not rewrite it.
|
|
91
|
+
_HTTP_BRIDGE_TERMINAL_APPEND_RECORDED_PHASES = frozenset(
|
|
92
|
+
{HTTP_BRIDGE_TERMINAL_APPEND_PHASE_APPENDED, HTTP_BRIDGE_TERMINAL_APPEND_PHASE_SETTLED}
|
|
93
|
+
)
|
|
94
|
+
|
|
95
|
+
|
|
96
|
+
# Sentinel: rebind continuity clears without an anchor fence (legacy callers).
|
|
97
|
+
REBIND_ANCHOR_UNFENCED: object = object()
|
|
98
|
+
|
|
99
|
+
|
|
100
|
+
class DurableBridgeAliasRegistration(StrEnum):
|
|
101
|
+
REGISTERED = "registered"
|
|
102
|
+
OWNER_FENCED = "owner_fenced"
|
|
103
|
+
ALIAS_PROTECTED = "alias_protected"
|
|
104
|
+
|
|
105
|
+
|
|
106
|
+
@dataclass(frozen=True, slots=True)
|
|
107
|
+
class DurableBridgeAliasRegistrationReceipt:
|
|
108
|
+
"""Rollback data for a continuity alias published before dispatch."""
|
|
109
|
+
|
|
110
|
+
status: DurableBridgeAliasRegistration
|
|
111
|
+
session_id: str
|
|
112
|
+
api_key_scope: str
|
|
113
|
+
alias_kind: str
|
|
114
|
+
alias_value: str
|
|
115
|
+
instance_id: str
|
|
116
|
+
owner_epoch: int
|
|
117
|
+
previous_alias_session_id: str | None
|
|
118
|
+
previous_alias_owner_epoch: int | None
|
|
119
|
+
previous_alias_account_id: str | None
|
|
120
|
+
previous_latest_turn_state: str | None
|
|
121
|
+
|
|
122
|
+
|
|
123
|
+
def durable_bridge_api_key_scope(api_key_id: str | None) -> str:
|
|
124
|
+
if api_key_id is None:
|
|
125
|
+
return _ANONYMOUS_API_KEY_SCOPE
|
|
126
|
+
stripped = api_key_id.strip()
|
|
127
|
+
return stripped or _ANONYMOUS_API_KEY_SCOPE
|
|
128
|
+
|
|
129
|
+
|
|
130
|
+
def durable_bridge_hash(value: str) -> str:
|
|
131
|
+
return sha256(value.encode("utf-8")).hexdigest()
|
|
132
|
+
|
|
133
|
+
|
|
134
|
+
def durable_bridge_operation_fingerprint(*, api_key_scope: str, request_text: str) -> str:
|
|
135
|
+
"""Hash the logical turn together with its authorization namespace."""
|
|
136
|
+
return durable_bridge_hash(f"{api_key_scope}:{request_text}")
|
|
137
|
+
|
|
138
|
+
|
|
139
|
+
def durable_bridge_operation_id(session_id: str, request_fingerprint: str) -> str:
|
|
140
|
+
"""Derive a stable, non-secret operation key for a continuity-bound turn."""
|
|
141
|
+
return f"op_{durable_bridge_hash(f'{session_id}:{request_fingerprint}')[:64]}"
|
|
142
|
+
|
|
143
|
+
|
|
144
|
+
def _encode_pending_tool_calls(response_id: str, value: Mapping[str, str] | None) -> str | None:
|
|
145
|
+
if value is None:
|
|
146
|
+
return None
|
|
147
|
+
return json.dumps(
|
|
148
|
+
{"response_id": response_id, "calls": dict(sorted(value.items()))},
|
|
149
|
+
separators=(",", ":"),
|
|
150
|
+
)
|
|
151
|
+
|
|
152
|
+
|
|
153
|
+
def _decode_pending_tool_calls(response_id: str | None, value: str | None) -> dict[str, str] | None:
|
|
154
|
+
if response_id is None or value is None:
|
|
155
|
+
return None
|
|
156
|
+
try:
|
|
157
|
+
payload = json.loads(value)
|
|
158
|
+
except (TypeError, ValueError):
|
|
159
|
+
return None
|
|
160
|
+
if not isinstance(payload, dict) or payload.get("response_id") != response_id:
|
|
161
|
+
return None
|
|
162
|
+
calls = payload.get("calls")
|
|
163
|
+
if not isinstance(calls, dict):
|
|
164
|
+
return None
|
|
165
|
+
result: dict[str, str] = {}
|
|
166
|
+
for call_id, call_type in calls.items():
|
|
167
|
+
if not isinstance(call_id, str) or not call_id.strip():
|
|
168
|
+
return None
|
|
169
|
+
if not isinstance(call_type, str) or not call_type.strip():
|
|
170
|
+
return None
|
|
171
|
+
result[call_id] = call_type
|
|
172
|
+
return result
|
|
173
|
+
|
|
174
|
+
|
|
175
|
+
@dataclass(frozen=True, slots=True)
|
|
176
|
+
class DurableBridgeSessionSnapshot:
|
|
177
|
+
id: str
|
|
178
|
+
session_key_kind: str
|
|
179
|
+
session_key_value: str
|
|
180
|
+
session_key_hash: str
|
|
181
|
+
api_key_scope: str
|
|
182
|
+
owner_instance_id: str | None
|
|
183
|
+
owner_epoch: int
|
|
184
|
+
lease_expires_at: datetime | None
|
|
185
|
+
state: HttpBridgeSessionState
|
|
186
|
+
account_id: str | None
|
|
187
|
+
model: str | None
|
|
188
|
+
service_tier: str | None
|
|
189
|
+
latest_turn_state: str | None
|
|
190
|
+
latest_response_id: str | None
|
|
191
|
+
latest_input_item_count: int | None
|
|
192
|
+
latest_input_full_fingerprint: str | None
|
|
193
|
+
last_seen_at: datetime
|
|
194
|
+
closed_at: datetime | None
|
|
195
|
+
latest_pending_tool_calls: dict[str, str] | None = None
|
|
196
|
+
owner_process_epoch: str | None = None
|
|
197
|
+
# True once a writer retired this row's continuity owner. Readers must
|
|
198
|
+
# treat ``account_id`` as absent and may use ``abandoned_account_id`` only
|
|
199
|
+
# as exclusion evidence, never as a routing target.
|
|
200
|
+
continuity_abandoned: bool = False
|
|
201
|
+
abandoned_account_id: str | None = None
|
|
202
|
+
|
|
203
|
+
|
|
204
|
+
@dataclass(frozen=True, slots=True)
|
|
205
|
+
class DurableBridgeRetryCircuitSnapshot:
|
|
206
|
+
session_key_kind: str
|
|
207
|
+
session_key_hash: str
|
|
208
|
+
api_key_scope: str
|
|
209
|
+
consecutive_failures: int
|
|
210
|
+
cooldown_until_epoch: float
|
|
211
|
+
last_detail: str | None
|
|
212
|
+
updated_at_epoch: float
|
|
213
|
+
admission_generation: int
|
|
214
|
+
|
|
215
|
+
|
|
216
|
+
@dataclass(frozen=True, slots=True)
|
|
217
|
+
class DurableBridgeRecoveryAttemptSnapshot:
|
|
218
|
+
session_id: str
|
|
219
|
+
request_fingerprint: str
|
|
220
|
+
request_id: str
|
|
221
|
+
account_id: str | None
|
|
222
|
+
model: str | None
|
|
223
|
+
replay_safe: bool
|
|
224
|
+
state: HttpBridgeRecoveryAttemptState
|
|
225
|
+
response_id: str | None
|
|
226
|
+
|
|
227
|
+
|
|
228
|
+
@dataclass(frozen=True, slots=True)
|
|
229
|
+
class DurableBridgeOperationSnapshot:
|
|
230
|
+
operation_id: str
|
|
231
|
+
session_id: str
|
|
232
|
+
request_fingerprint: str
|
|
233
|
+
account_id: str | None
|
|
234
|
+
model: str | None
|
|
235
|
+
parent_response_id: str | None
|
|
236
|
+
state: str
|
|
237
|
+
response_id: str | None
|
|
238
|
+
recovery_dispatch_count: int = 0
|
|
239
|
+
request_text: str | None = None
|
|
240
|
+
event_spool_complete: bool = True
|
|
241
|
+
created: bool = False
|
|
242
|
+
rebound: bool = False
|
|
243
|
+
rebound_from_session_id: str | None = None
|
|
244
|
+
rebound_from_account_id: str | None = None
|
|
245
|
+
rebound_from_model: str | None = None
|
|
246
|
+
rebound_from_parent_response_id: str | None = None
|
|
247
|
+
|
|
248
|
+
|
|
249
|
+
@dataclass(frozen=True, slots=True)
|
|
250
|
+
class DurableBridgeOperationAbandonment:
|
|
251
|
+
"""Low-cardinality evidence for one operation abandoned by maintenance."""
|
|
252
|
+
|
|
253
|
+
source_state: str
|
|
254
|
+
age_seconds: float
|
|
255
|
+
owner_lease_outcome: str
|
|
256
|
+
session_hash: str
|
|
257
|
+
|
|
258
|
+
|
|
259
|
+
@dataclass(frozen=True, slots=True)
|
|
260
|
+
class DurableBridgeOperationAbandonmentScanCursor:
|
|
261
|
+
"""Keyset position for the next oversized-protection maintenance slice."""
|
|
262
|
+
|
|
263
|
+
updated_at: datetime
|
|
264
|
+
operation_id: str
|
|
265
|
+
|
|
266
|
+
|
|
267
|
+
@dataclass(frozen=True, slots=True)
|
|
268
|
+
class DurableBridgeOperationAbandonmentSweep:
|
|
269
|
+
"""Abandonments plus the cursor needed to continue a bounded sweep."""
|
|
270
|
+
|
|
271
|
+
abandonments: tuple[DurableBridgeOperationAbandonment, ...]
|
|
272
|
+
next_cursor: DurableBridgeOperationAbandonmentScanCursor | None
|
|
273
|
+
|
|
274
|
+
|
|
275
|
+
@dataclass(frozen=True, slots=True)
|
|
276
|
+
class DurableBridgeTranscriptTurn:
|
|
277
|
+
operation: DurableBridgeOperationSnapshot
|
|
278
|
+
events: tuple[str, ...]
|
|
279
|
+
|
|
280
|
+
|
|
281
|
+
@dataclass(frozen=True, slots=True)
|
|
282
|
+
class DurableBridgeOperationEventInput:
|
|
283
|
+
operation_id: str
|
|
284
|
+
session_id: str
|
|
285
|
+
instance_id: str
|
|
286
|
+
owner_epoch: int
|
|
287
|
+
event_text: str
|
|
288
|
+
|
|
289
|
+
|
|
290
|
+
@dataclass(frozen=True, slots=True)
|
|
291
|
+
class DurableBridgeOperationPurgeBatchResult:
|
|
292
|
+
selected_operations: int
|
|
293
|
+
deleted_operations: int
|
|
294
|
+
|
|
295
|
+
|
|
296
|
+
class DurableBridgeRepository:
|
|
297
|
+
def __init__(self, session: AsyncSession) -> None:
|
|
298
|
+
self._session = session
|
|
299
|
+
|
|
300
|
+
async def _commit_writer_section(self) -> None:
|
|
301
|
+
async with sqlite_writer_section():
|
|
302
|
+
await self._session.commit()
|
|
303
|
+
|
|
304
|
+
async def _delete_operation_spool_material(self, operation_ids: Sequence[str]) -> None:
|
|
305
|
+
if not operation_ids:
|
|
306
|
+
return
|
|
307
|
+
await self._session.execute(
|
|
308
|
+
delete(HttpBridgeOperationEvent).where(HttpBridgeOperationEvent.operation_id.in_(operation_ids))
|
|
309
|
+
)
|
|
310
|
+
await self._session.execute(
|
|
311
|
+
delete(HttpBridgeOperationEventChunk).where(HttpBridgeOperationEventChunk.operation_id.in_(operation_ids))
|
|
312
|
+
)
|
|
313
|
+
|
|
314
|
+
async def _operation_has_spool_material(self, operation_id: str) -> bool:
|
|
315
|
+
legacy_event = await self._session.scalar(
|
|
316
|
+
select(HttpBridgeOperationEvent.event_id)
|
|
317
|
+
.where(HttpBridgeOperationEvent.operation_id == operation_id)
|
|
318
|
+
.limit(1)
|
|
319
|
+
)
|
|
320
|
+
if legacy_event is not None:
|
|
321
|
+
return True
|
|
322
|
+
chunk_sequence = await self._session.scalar(
|
|
323
|
+
select(HttpBridgeOperationEventChunk.first_sequence_number)
|
|
324
|
+
.where(HttpBridgeOperationEventChunk.operation_id == operation_id)
|
|
325
|
+
.limit(1)
|
|
326
|
+
)
|
|
327
|
+
return chunk_sequence is not None
|
|
328
|
+
|
|
329
|
+
async def _operation_has_legacy_events(self, operation_id: str) -> bool:
|
|
330
|
+
event_id = await self._session.scalar(
|
|
331
|
+
select(HttpBridgeOperationEvent.event_id)
|
|
332
|
+
.where(HttpBridgeOperationEvent.operation_id == operation_id)
|
|
333
|
+
.limit(1)
|
|
334
|
+
)
|
|
335
|
+
return event_id is not None
|
|
336
|
+
|
|
337
|
+
async def _lock_operation_for_chunk_append(
|
|
338
|
+
self,
|
|
339
|
+
*,
|
|
340
|
+
operation_id: str,
|
|
341
|
+
session_id: str,
|
|
342
|
+
instance_id: str,
|
|
343
|
+
owner_epoch: int,
|
|
344
|
+
expected_recovery_dispatch_count: int | None = None,
|
|
345
|
+
) -> tuple[HttpBridgeOperationRecord, bool] | None:
|
|
346
|
+
owner_exists = await self._session.scalar(
|
|
347
|
+
select(HttpBridgeSessionRecord.id)
|
|
348
|
+
.where(
|
|
349
|
+
HttpBridgeSessionRecord.id == session_id,
|
|
350
|
+
HttpBridgeSessionRecord.owner_instance_id == instance_id,
|
|
351
|
+
HttpBridgeSessionRecord.owner_epoch == owner_epoch,
|
|
352
|
+
)
|
|
353
|
+
.with_for_update()
|
|
354
|
+
)
|
|
355
|
+
operation_statement = select(HttpBridgeOperationRecord).where(
|
|
356
|
+
HttpBridgeOperationRecord.operation_id == operation_id,
|
|
357
|
+
HttpBridgeOperationRecord.session_id == session_id,
|
|
358
|
+
)
|
|
359
|
+
if expected_recovery_dispatch_count is not None:
|
|
360
|
+
operation_statement = operation_statement.where(
|
|
361
|
+
HttpBridgeOperationRecord.recovery_dispatch_count == expected_recovery_dispatch_count
|
|
362
|
+
)
|
|
363
|
+
operation = await self._session.scalar(operation_statement.with_for_update())
|
|
364
|
+
# ``abandoned`` is a terminal duplicate-suppression fence that the
|
|
365
|
+
# maintenance sweep applies without clearing session ownership, so the
|
|
366
|
+
# original owner still passes the instance/epoch fence above. Refuse
|
|
367
|
+
# the lock like the rows_v1 writers do; otherwise a late terminal chunk
|
|
368
|
+
# rewrites ``state`` and a late batch grows the abandoned spool.
|
|
369
|
+
if owner_exists is None or operation is None or operation.state == "abandoned":
|
|
370
|
+
await self._session.rollback()
|
|
371
|
+
return None
|
|
372
|
+
# This dispatch already recorded a terminal transcript outcome (an
|
|
373
|
+
# append committed, or a fallback settlement published it), so the row
|
|
374
|
+
# is final: refuse the lock rather than let a duplicate or late
|
|
375
|
+
# terminal write rewrite it. The predicate is the dedicated phase
|
|
376
|
+
# marker, never ``event_spool_complete`` — an ordinary operation is
|
|
377
|
+
# ``state = completed`` with an incomplete spool for the whole window
|
|
378
|
+
# in which its terminal append runs, because the relay publishes the
|
|
379
|
+
# operation state before appending.
|
|
380
|
+
if operation.terminal_append_phase in _HTTP_BRIDGE_TERMINAL_APPEND_RECORDED_PHASES:
|
|
381
|
+
return operation, False
|
|
382
|
+
if operation.spool_format == HTTP_BRIDGE_SPOOL_FORMAT_CHUNKS_V2:
|
|
383
|
+
if await self._operation_has_legacy_events(operation_id):
|
|
384
|
+
return operation, False
|
|
385
|
+
return operation, True
|
|
386
|
+
if operation.spool_format != HTTP_BRIDGE_SPOOL_FORMAT_ROWS_V1:
|
|
387
|
+
return operation, False
|
|
388
|
+
if int(operation.event_bytes or 0) != 0 or await self._operation_has_spool_material(operation_id):
|
|
389
|
+
return operation, False
|
|
390
|
+
return operation, True
|
|
391
|
+
|
|
392
|
+
async def _next_operation_chunk_sequence(self, operation_id: str) -> int:
|
|
393
|
+
latest = (
|
|
394
|
+
await self._session.execute(
|
|
395
|
+
select(
|
|
396
|
+
HttpBridgeOperationEventChunk.first_sequence_number,
|
|
397
|
+
HttpBridgeOperationEventChunk.event_count,
|
|
398
|
+
)
|
|
399
|
+
.where(HttpBridgeOperationEventChunk.operation_id == operation_id)
|
|
400
|
+
.order_by(HttpBridgeOperationEventChunk.first_sequence_number.desc())
|
|
401
|
+
.limit(1)
|
|
402
|
+
)
|
|
403
|
+
).one_or_none()
|
|
404
|
+
if latest is None:
|
|
405
|
+
return 1
|
|
406
|
+
first_sequence_number, event_count = latest
|
|
407
|
+
return int(first_sequence_number) + int(event_count)
|
|
408
|
+
|
|
409
|
+
async def _chunk_append_owner_exists(
|
|
410
|
+
self,
|
|
411
|
+
*,
|
|
412
|
+
session_id: str,
|
|
413
|
+
instance_id: str,
|
|
414
|
+
owner_epoch: int,
|
|
415
|
+
) -> bool:
|
|
416
|
+
return (
|
|
417
|
+
await self._session.scalar(
|
|
418
|
+
select(HttpBridgeSessionRecord.id).where(
|
|
419
|
+
HttpBridgeSessionRecord.id == session_id,
|
|
420
|
+
HttpBridgeSessionRecord.owner_instance_id == instance_id,
|
|
421
|
+
HttpBridgeSessionRecord.owner_epoch == owner_epoch,
|
|
422
|
+
)
|
|
423
|
+
)
|
|
424
|
+
) is not None
|
|
425
|
+
|
|
426
|
+
async def _chunk_append_preflight_allows_encoding(
|
|
427
|
+
self,
|
|
428
|
+
*,
|
|
429
|
+
operation_id: str,
|
|
430
|
+
session_id: str,
|
|
431
|
+
instance_id: str,
|
|
432
|
+
owner_epoch: int,
|
|
433
|
+
event_bytes: int,
|
|
434
|
+
event_count: int,
|
|
435
|
+
max_bytes: int,
|
|
436
|
+
) -> bool:
|
|
437
|
+
if not await self._chunk_append_owner_exists(
|
|
438
|
+
session_id=session_id,
|
|
439
|
+
instance_id=instance_id,
|
|
440
|
+
owner_epoch=owner_epoch,
|
|
441
|
+
):
|
|
442
|
+
return False
|
|
443
|
+
current_event_bytes = await self._session.scalar(
|
|
444
|
+
select(HttpBridgeOperationRecord.event_bytes).where(
|
|
445
|
+
HttpBridgeOperationRecord.operation_id == operation_id,
|
|
446
|
+
HttpBridgeOperationRecord.session_id == session_id,
|
|
447
|
+
)
|
|
448
|
+
)
|
|
449
|
+
if current_event_bytes is None:
|
|
450
|
+
return False
|
|
451
|
+
first_sequence_number = await self._next_operation_chunk_sequence(operation_id)
|
|
452
|
+
return (
|
|
453
|
+
int(current_event_bytes) + event_bytes <= max_bytes
|
|
454
|
+
and first_sequence_number - 1 + event_count <= DURABLE_BRIDGE_TRANSCRIPT_MAX_EVENTS
|
|
455
|
+
)
|
|
456
|
+
|
|
457
|
+
async def get_session(
|
|
458
|
+
self,
|
|
459
|
+
*,
|
|
460
|
+
session_key_kind: str,
|
|
461
|
+
session_key_value: str,
|
|
462
|
+
api_key_scope: str,
|
|
463
|
+
) -> DurableBridgeSessionSnapshot | None:
|
|
464
|
+
statement = select(HttpBridgeSessionRecord).where(
|
|
465
|
+
HttpBridgeSessionRecord.session_key_kind == session_key_kind,
|
|
466
|
+
HttpBridgeSessionRecord.session_key_hash == durable_bridge_hash(session_key_value),
|
|
467
|
+
HttpBridgeSessionRecord.api_key_scope == api_key_scope,
|
|
468
|
+
)
|
|
469
|
+
result = await self._session.execute(statement)
|
|
470
|
+
row = result.scalar_one_or_none()
|
|
471
|
+
return _to_snapshot(row)
|
|
472
|
+
|
|
473
|
+
async def get_retry_circuit(
|
|
474
|
+
self,
|
|
475
|
+
*,
|
|
476
|
+
session_key_kind: str,
|
|
477
|
+
session_key_value: str,
|
|
478
|
+
api_key_scope: str,
|
|
479
|
+
) -> DurableBridgeRetryCircuitSnapshot | None:
|
|
480
|
+
result = await self._session.execute(
|
|
481
|
+
select(HttpBridgeRetryCircuit).where(
|
|
482
|
+
HttpBridgeRetryCircuit.session_key_kind == session_key_kind,
|
|
483
|
+
HttpBridgeRetryCircuit.session_key_hash == durable_bridge_hash(session_key_value),
|
|
484
|
+
HttpBridgeRetryCircuit.api_key_scope == api_key_scope,
|
|
485
|
+
)
|
|
486
|
+
)
|
|
487
|
+
return _to_retry_circuit_snapshot(result.scalar_one_or_none())
|
|
488
|
+
|
|
489
|
+
async def upsert_retry_circuit(
|
|
490
|
+
self,
|
|
491
|
+
*,
|
|
492
|
+
session_key_kind: str,
|
|
493
|
+
session_key_value: str,
|
|
494
|
+
api_key_scope: str,
|
|
495
|
+
consecutive_failures: int,
|
|
496
|
+
cooldown_until_epoch: float,
|
|
497
|
+
last_detail: str | None,
|
|
498
|
+
updated_at_epoch: float,
|
|
499
|
+
base_updated_at_epoch: float = 0.0,
|
|
500
|
+
failure_threshold: int = 1,
|
|
501
|
+
poison_sticky_threshold: int | None = None,
|
|
502
|
+
conflict_cooldown_until_epoch: float | None = None,
|
|
503
|
+
base_backoff_seconds: float = 60.0,
|
|
504
|
+
max_backoff_seconds: float = 600.0,
|
|
505
|
+
clean_close_max_backoff_seconds: float = 30.0,
|
|
506
|
+
) -> None:
|
|
507
|
+
sticky_threshold = poison_sticky_threshold if poison_sticky_threshold is not None else failure_threshold
|
|
508
|
+
values = {
|
|
509
|
+
"session_key_kind": session_key_kind,
|
|
510
|
+
"session_key_hash": durable_bridge_hash(session_key_value),
|
|
511
|
+
"api_key_scope": api_key_scope,
|
|
512
|
+
"consecutive_failures": consecutive_failures,
|
|
513
|
+
"cooldown_until_epoch": cooldown_until_epoch,
|
|
514
|
+
"last_detail": last_detail,
|
|
515
|
+
"updated_at_epoch": updated_at_epoch,
|
|
516
|
+
}
|
|
517
|
+
threshold = max(1, failure_threshold)
|
|
518
|
+
cooldown_floor = (
|
|
519
|
+
max(0.0, conflict_cooldown_until_epoch)
|
|
520
|
+
if conflict_cooldown_until_epoch is not None
|
|
521
|
+
else max(0.0, cooldown_until_epoch)
|
|
522
|
+
)
|
|
523
|
+
# A reset starts a new failure lineage. Never carry the incoming
|
|
524
|
+
# cooldown into that fresh lineage, even when the threshold is one.
|
|
525
|
+
base_backoff = max(0.001, base_backoff_seconds)
|
|
526
|
+
max_backoff = max(base_backoff, max_backoff_seconds)
|
|
527
|
+
clean_close_max_backoff = max(0.001, clean_close_max_backoff_seconds)
|
|
528
|
+
|
|
529
|
+
def cooldown_for_failure_count(failure_count: Any, last_detail: Any) -> Any:
|
|
530
|
+
regular_cooldown = case(
|
|
531
|
+
(failure_count < threshold, 0.0),
|
|
532
|
+
(failure_count == threshold, base_backoff),
|
|
533
|
+
(failure_count == threshold + 1, min(max_backoff, base_backoff * 2.0)),
|
|
534
|
+
(failure_count == threshold + 2, min(max_backoff, base_backoff * 4.0)),
|
|
535
|
+
else_=max_backoff,
|
|
536
|
+
)
|
|
537
|
+
clean_cooldown = case(
|
|
538
|
+
(failure_count < threshold, 0.0),
|
|
539
|
+
else_=clean_close_max_backoff,
|
|
540
|
+
)
|
|
541
|
+
return case((last_detail == "clean_close", clean_cooldown), else_=regular_cooldown)
|
|
542
|
+
|
|
543
|
+
dialect = self._session.get_bind().dialect.name
|
|
544
|
+
if dialect == "postgresql":
|
|
545
|
+
insert_statement = pg_insert(HttpBridgeRetryCircuit).values(**values)
|
|
546
|
+
excluded = insert_statement.excluded
|
|
547
|
+
# ``updated_at_epoch`` is an observation timestamp, not a
|
|
548
|
+
# concurrency version. Equality with the loaded base is the CAS
|
|
549
|
+
# match, even when a replica's wall clock lags it; the failure
|
|
550
|
+
# count guard still rejects an older snapshot that was loaded
|
|
551
|
+
# from the same row after a newer failure had already been
|
|
552
|
+
# merged. Every base-mismatched write is stale — its episode was
|
|
553
|
+
# settled, replaced, or outrun while the write waited — and
|
|
554
|
+
# drops, leaving the row byte-identical so in-flight fences and
|
|
555
|
+
# bases stay valid; the writer reconciles from the returned row.
|
|
556
|
+
# Wall-clock recency cannot stand in for lineage: a delayed
|
|
557
|
+
# write always carries a newer timestamp than the base it
|
|
558
|
+
# loaded, so admitting "newer than base" writes would merge a
|
|
559
|
+
# finished episode's count into whatever lineage owns the row
|
|
560
|
+
# now, including a reset row another replica has already
|
|
561
|
+
# re-struck. A concurrent same-lineage strike that loses this
|
|
562
|
+
# race undercounts by exactly its own strike, which is the
|
|
563
|
+
# accepted direction: the circuit opens at most one failure
|
|
564
|
+
# later, and a stale count can never reopen a settled episode's
|
|
565
|
+
# cooldown against a fresh lineage.
|
|
566
|
+
failure_from_current_row = and_(
|
|
567
|
+
HttpBridgeRetryCircuit.updated_at_epoch == base_updated_at_epoch,
|
|
568
|
+
excluded.consecutive_failures >= HttpBridgeRetryCircuit.consecutive_failures,
|
|
569
|
+
)
|
|
570
|
+
# An at-threshold poison detail is sticky against non-poison
|
|
571
|
+
# strikes: the row is the cross-replica record that a poisoned
|
|
572
|
+
# anchor's clear is still owed, and letting a clean probe
|
|
573
|
+
# failure overwrite it would strand that debt for every worker.
|
|
574
|
+
# Only a reset/settle (which rewrites the row wholesale) or
|
|
575
|
+
# another poison-class strike may change it.
|
|
576
|
+
sticky_poison_detail = and_(
|
|
577
|
+
HttpBridgeRetryCircuit.last_detail.in_(
|
|
578
|
+
("stream_incomplete", "stream_idle_timeout", "bridge_eventless_timeout")
|
|
579
|
+
),
|
|
580
|
+
# The effective anchor-poison threshold, not the circuit
|
|
581
|
+
# threshold: a configured threshold of one authorizes the
|
|
582
|
+
# abandonment at the first poison strike, and its failed
|
|
583
|
+
# clear leaves a one-failure debt this predicate must keep.
|
|
584
|
+
HttpBridgeRetryCircuit.consecutive_failures >= sticky_threshold,
|
|
585
|
+
excluded.last_detail.notin_(("stream_incomplete", "stream_idle_timeout", "bridge_eventless_timeout")),
|
|
586
|
+
)
|
|
587
|
+
# An abandonment tombstone is sticky against every strike
|
|
588
|
+
# detail: continuity is already gone, the tombstone is what
|
|
589
|
+
# fails anchorless deltas closed on every replica, and only the
|
|
590
|
+
# fenced settle/supersede paths — a completion establishing
|
|
591
|
+
# fresh continuity — may rewrite it.
|
|
592
|
+
# NULL-safe: a NULL detail must make this term FALSE (so its
|
|
593
|
+
# negation stays TRUE), not NULL — three-valued logic would
|
|
594
|
+
# otherwise freeze the detail of every reset row.
|
|
595
|
+
sticky_tombstone = and_(
|
|
596
|
+
HttpBridgeRetryCircuit.last_detail.is_not(None),
|
|
597
|
+
HttpBridgeRetryCircuit.last_detail == _RETRY_CIRCUIT_ABANDONED_TOMBSTONE_DETAIL,
|
|
598
|
+
)
|
|
599
|
+
conflict_failures = case(
|
|
600
|
+
(
|
|
601
|
+
failure_from_current_row,
|
|
602
|
+
func.greatest(
|
|
603
|
+
HttpBridgeRetryCircuit.consecutive_failures + 1,
|
|
604
|
+
excluded.consecutive_failures,
|
|
605
|
+
),
|
|
606
|
+
),
|
|
607
|
+
else_=HttpBridgeRetryCircuit.consecutive_failures,
|
|
608
|
+
)
|
|
609
|
+
merged_updated_at = func.greatest(
|
|
610
|
+
HttpBridgeRetryCircuit.updated_at_epoch,
|
|
611
|
+
excluded.updated_at_epoch,
|
|
612
|
+
)
|
|
613
|
+
merged_cooldown = case(
|
|
614
|
+
(
|
|
615
|
+
conflict_failures >= threshold,
|
|
616
|
+
func.greatest(
|
|
617
|
+
cooldown_floor,
|
|
618
|
+
merged_updated_at + cooldown_for_failure_count(conflict_failures, excluded.last_detail),
|
|
619
|
+
),
|
|
620
|
+
),
|
|
621
|
+
else_=0.0,
|
|
622
|
+
)
|
|
623
|
+
statement = insert_statement.on_conflict_do_update(
|
|
624
|
+
index_elements=[
|
|
625
|
+
HttpBridgeRetryCircuit.session_key_kind,
|
|
626
|
+
HttpBridgeRetryCircuit.session_key_hash,
|
|
627
|
+
HttpBridgeRetryCircuit.api_key_scope,
|
|
628
|
+
],
|
|
629
|
+
set_={
|
|
630
|
+
"consecutive_failures": conflict_failures,
|
|
631
|
+
"cooldown_until_epoch": case(
|
|
632
|
+
(
|
|
633
|
+
failure_from_current_row,
|
|
634
|
+
func.greatest(
|
|
635
|
+
HttpBridgeRetryCircuit.cooldown_until_epoch,
|
|
636
|
+
excluded.cooldown_until_epoch,
|
|
637
|
+
merged_cooldown,
|
|
638
|
+
),
|
|
639
|
+
),
|
|
640
|
+
else_=HttpBridgeRetryCircuit.cooldown_until_epoch,
|
|
641
|
+
),
|
|
642
|
+
"last_detail": case(
|
|
643
|
+
(
|
|
644
|
+
and_(failure_from_current_row, ~sticky_poison_detail, ~sticky_tombstone),
|
|
645
|
+
excluded.last_detail,
|
|
646
|
+
),
|
|
647
|
+
else_=HttpBridgeRetryCircuit.last_detail,
|
|
648
|
+
),
|
|
649
|
+
"updated_at_epoch": case(
|
|
650
|
+
(failure_from_current_row, merged_updated_at),
|
|
651
|
+
else_=HttpBridgeRetryCircuit.updated_at_epoch,
|
|
652
|
+
),
|
|
653
|
+
},
|
|
654
|
+
)
|
|
655
|
+
elif dialect == "sqlite":
|
|
656
|
+
insert_statement = sqlite_insert(HttpBridgeRetryCircuit).values(**values)
|
|
657
|
+
excluded = insert_statement.excluded
|
|
658
|
+
# ``updated_at_epoch`` is an observation timestamp, not a
|
|
659
|
+
# concurrency version. Equality with the loaded base is the CAS
|
|
660
|
+
# match, even when a replica's wall clock lags it; the failure
|
|
661
|
+
# count guard still rejects an older snapshot that was loaded
|
|
662
|
+
# from the same row after a newer failure had already been
|
|
663
|
+
# merged. Every base-mismatched write is stale — its episode was
|
|
664
|
+
# settled, replaced, or outrun while the write waited — and
|
|
665
|
+
# drops, leaving the row byte-identical so in-flight fences and
|
|
666
|
+
# bases stay valid; the writer reconciles from the returned row.
|
|
667
|
+
# Wall-clock recency cannot stand in for lineage: a delayed
|
|
668
|
+
# write always carries a newer timestamp than the base it
|
|
669
|
+
# loaded, so admitting "newer than base" writes would merge a
|
|
670
|
+
# finished episode's count into whatever lineage owns the row
|
|
671
|
+
# now, including a reset row another replica has already
|
|
672
|
+
# re-struck. A concurrent same-lineage strike that loses this
|
|
673
|
+
# race undercounts by exactly its own strike, which is the
|
|
674
|
+
# accepted direction: the circuit opens at most one failure
|
|
675
|
+
# later, and a stale count can never reopen a settled episode's
|
|
676
|
+
# cooldown against a fresh lineage.
|
|
677
|
+
failure_from_current_row = and_(
|
|
678
|
+
HttpBridgeRetryCircuit.updated_at_epoch == base_updated_at_epoch,
|
|
679
|
+
excluded.consecutive_failures >= HttpBridgeRetryCircuit.consecutive_failures,
|
|
680
|
+
)
|
|
681
|
+
# An at-threshold poison detail is sticky against non-poison
|
|
682
|
+
# strikes: the row is the cross-replica record that a poisoned
|
|
683
|
+
# anchor's clear is still owed, and letting a clean probe
|
|
684
|
+
# failure overwrite it would strand that debt for every worker.
|
|
685
|
+
# Only a reset/settle (which rewrites the row wholesale) or
|
|
686
|
+
# another poison-class strike may change it.
|
|
687
|
+
sticky_poison_detail = and_(
|
|
688
|
+
HttpBridgeRetryCircuit.last_detail.in_(
|
|
689
|
+
("stream_incomplete", "stream_idle_timeout", "bridge_eventless_timeout")
|
|
690
|
+
),
|
|
691
|
+
# The effective anchor-poison threshold, not the circuit
|
|
692
|
+
# threshold: a configured threshold of one authorizes the
|
|
693
|
+
# abandonment at the first poison strike, and its failed
|
|
694
|
+
# clear leaves a one-failure debt this predicate must keep.
|
|
695
|
+
HttpBridgeRetryCircuit.consecutive_failures >= sticky_threshold,
|
|
696
|
+
excluded.last_detail.notin_(("stream_incomplete", "stream_idle_timeout", "bridge_eventless_timeout")),
|
|
697
|
+
)
|
|
698
|
+
# An abandonment tombstone is sticky against every strike
|
|
699
|
+
# detail: continuity is already gone, the tombstone is what
|
|
700
|
+
# fails anchorless deltas closed on every replica, and only the
|
|
701
|
+
# fenced settle/supersede paths — a completion establishing
|
|
702
|
+
# fresh continuity — may rewrite it.
|
|
703
|
+
# NULL-safe: a NULL detail must make this term FALSE (so its
|
|
704
|
+
# negation stays TRUE), not NULL — three-valued logic would
|
|
705
|
+
# otherwise freeze the detail of every reset row.
|
|
706
|
+
sticky_tombstone = and_(
|
|
707
|
+
HttpBridgeRetryCircuit.last_detail.is_not(None),
|
|
708
|
+
HttpBridgeRetryCircuit.last_detail == _RETRY_CIRCUIT_ABANDONED_TOMBSTONE_DETAIL,
|
|
709
|
+
)
|
|
710
|
+
conflict_failures = case(
|
|
711
|
+
(
|
|
712
|
+
failure_from_current_row,
|
|
713
|
+
func.max(
|
|
714
|
+
HttpBridgeRetryCircuit.consecutive_failures + 1,
|
|
715
|
+
excluded.consecutive_failures,
|
|
716
|
+
),
|
|
717
|
+
),
|
|
718
|
+
else_=HttpBridgeRetryCircuit.consecutive_failures,
|
|
719
|
+
)
|
|
720
|
+
merged_updated_at = func.max(
|
|
721
|
+
HttpBridgeRetryCircuit.updated_at_epoch,
|
|
722
|
+
excluded.updated_at_epoch,
|
|
723
|
+
)
|
|
724
|
+
merged_cooldown = case(
|
|
725
|
+
(
|
|
726
|
+
conflict_failures >= threshold,
|
|
727
|
+
func.max(
|
|
728
|
+
cooldown_floor,
|
|
729
|
+
merged_updated_at + cooldown_for_failure_count(conflict_failures, excluded.last_detail),
|
|
730
|
+
),
|
|
731
|
+
),
|
|
732
|
+
else_=0.0,
|
|
733
|
+
)
|
|
734
|
+
statement = insert_statement.on_conflict_do_update(
|
|
735
|
+
index_elements=[
|
|
736
|
+
HttpBridgeRetryCircuit.session_key_kind,
|
|
737
|
+
HttpBridgeRetryCircuit.session_key_hash,
|
|
738
|
+
HttpBridgeRetryCircuit.api_key_scope,
|
|
739
|
+
],
|
|
740
|
+
set_={
|
|
741
|
+
"consecutive_failures": conflict_failures,
|
|
742
|
+
"cooldown_until_epoch": case(
|
|
743
|
+
(
|
|
744
|
+
failure_from_current_row,
|
|
745
|
+
func.max(
|
|
746
|
+
HttpBridgeRetryCircuit.cooldown_until_epoch,
|
|
747
|
+
excluded.cooldown_until_epoch,
|
|
748
|
+
merged_cooldown,
|
|
749
|
+
),
|
|
750
|
+
),
|
|
751
|
+
else_=HttpBridgeRetryCircuit.cooldown_until_epoch,
|
|
752
|
+
),
|
|
753
|
+
"last_detail": case(
|
|
754
|
+
(
|
|
755
|
+
and_(failure_from_current_row, ~sticky_poison_detail, ~sticky_tombstone),
|
|
756
|
+
excluded.last_detail,
|
|
757
|
+
),
|
|
758
|
+
else_=HttpBridgeRetryCircuit.last_detail,
|
|
759
|
+
),
|
|
760
|
+
"updated_at_epoch": case(
|
|
761
|
+
(failure_from_current_row, merged_updated_at),
|
|
762
|
+
else_=HttpBridgeRetryCircuit.updated_at_epoch,
|
|
763
|
+
),
|
|
764
|
+
},
|
|
765
|
+
)
|
|
766
|
+
else:
|
|
767
|
+
raise RuntimeError(f"DurableBridgeRepository retry circuit upsert unsupported for dialect={dialect!r}")
|
|
768
|
+
async with sqlite_writer_section():
|
|
769
|
+
await self._session.execute(statement)
|
|
770
|
+
await self._session.commit()
|
|
771
|
+
|
|
772
|
+
async def claim_retry_circuit_generation(
|
|
773
|
+
self,
|
|
774
|
+
*,
|
|
775
|
+
session_key_kind: str,
|
|
776
|
+
session_key_value: str,
|
|
777
|
+
api_key_scope: str,
|
|
778
|
+
expected_updated_at_epoch: float | None,
|
|
779
|
+
expected_admission_generation: int,
|
|
780
|
+
expected_consecutive_failures: int,
|
|
781
|
+
expected_cooldown_until_epoch: float,
|
|
782
|
+
) -> DurableBridgeRetryCircuitSnapshot | None:
|
|
783
|
+
"""Linearize replay admission against a retry-circuit generation.
|
|
784
|
+
|
|
785
|
+
``admission_generation`` is independent from the failure observation
|
|
786
|
+
timestamp. A failure committed first makes this claim fail; a delayed
|
|
787
|
+
failure committed afterward still sees its original ``updated_at``
|
|
788
|
+
baseline and is merged after the already-admitted dispatch.
|
|
789
|
+
"""
|
|
790
|
+
values = {
|
|
791
|
+
"session_key_kind": session_key_kind,
|
|
792
|
+
"session_key_hash": durable_bridge_hash(session_key_value),
|
|
793
|
+
"api_key_scope": api_key_scope,
|
|
794
|
+
"consecutive_failures": 0,
|
|
795
|
+
"cooldown_until_epoch": 0.0,
|
|
796
|
+
"last_detail": None,
|
|
797
|
+
"updated_at_epoch": time.time(),
|
|
798
|
+
"admission_generation": 1,
|
|
799
|
+
}
|
|
800
|
+
dialect = self._session.get_bind().dialect.name
|
|
801
|
+
async with sqlite_writer_section():
|
|
802
|
+
if expected_updated_at_epoch is None:
|
|
803
|
+
if expected_admission_generation != 0:
|
|
804
|
+
return None
|
|
805
|
+
if dialect == "postgresql":
|
|
806
|
+
statement = pg_insert(HttpBridgeRetryCircuit).values(**values).on_conflict_do_nothing()
|
|
807
|
+
elif dialect == "sqlite":
|
|
808
|
+
statement = sqlite_insert(HttpBridgeRetryCircuit).values(**values).on_conflict_do_nothing()
|
|
809
|
+
else:
|
|
810
|
+
raise RuntimeError(
|
|
811
|
+
f"DurableBridgeRepository retry circuit claim unsupported for dialect={dialect!r}"
|
|
812
|
+
)
|
|
813
|
+
else:
|
|
814
|
+
statement = (
|
|
815
|
+
update(HttpBridgeRetryCircuit)
|
|
816
|
+
.where(
|
|
817
|
+
HttpBridgeRetryCircuit.session_key_kind == session_key_kind,
|
|
818
|
+
HttpBridgeRetryCircuit.session_key_hash == durable_bridge_hash(session_key_value),
|
|
819
|
+
HttpBridgeRetryCircuit.api_key_scope == api_key_scope,
|
|
820
|
+
HttpBridgeRetryCircuit.admission_generation == expected_admission_generation,
|
|
821
|
+
HttpBridgeRetryCircuit.updated_at_epoch == expected_updated_at_epoch,
|
|
822
|
+
HttpBridgeRetryCircuit.consecutive_failures == expected_consecutive_failures,
|
|
823
|
+
HttpBridgeRetryCircuit.cooldown_until_epoch == expected_cooldown_until_epoch,
|
|
824
|
+
)
|
|
825
|
+
.values(admission_generation=expected_admission_generation + 1)
|
|
826
|
+
)
|
|
827
|
+
result = await self._session.execute(statement)
|
|
828
|
+
if getattr(result, "rowcount", 0) != 1:
|
|
829
|
+
await self._session.rollback()
|
|
830
|
+
return None
|
|
831
|
+
await self._session.commit()
|
|
832
|
+
return await self.get_retry_circuit(
|
|
833
|
+
session_key_kind=session_key_kind,
|
|
834
|
+
session_key_value=session_key_value,
|
|
835
|
+
api_key_scope=api_key_scope,
|
|
836
|
+
)
|
|
837
|
+
|
|
838
|
+
async def supersede_retry_circuit_detail(
|
|
839
|
+
self,
|
|
840
|
+
*,
|
|
841
|
+
session_key_kind: str,
|
|
842
|
+
session_key_value: str,
|
|
843
|
+
api_key_scope: str,
|
|
844
|
+
expected_updated_at_epoch: float,
|
|
845
|
+
expected_consecutive_failures: int,
|
|
846
|
+
expected_last_detail: str | None,
|
|
847
|
+
last_detail: str | None,
|
|
848
|
+
) -> bool:
|
|
849
|
+
"""Rewrite a surviving row's failure detail under its version fence.
|
|
850
|
+
|
|
851
|
+
Detail-only by design: the strike upsert increments the merged count,
|
|
852
|
+
so persisting an anchor supersession through it would charge a
|
|
853
|
+
phantom failure. The version is deliberately left unchanged so a
|
|
854
|
+
concurrent strike still merges onto the row and its failure class
|
|
855
|
+
overwrites this one. The count is part of the fence because a
|
|
856
|
+
lagging-clock strike can merge without moving the version — every
|
|
857
|
+
landed merge increments the count, so a strike that slipped in
|
|
858
|
+
ahead of this rewrite makes it miss instead of being overwritten.
|
|
859
|
+
"""
|
|
860
|
+
async with sqlite_writer_section():
|
|
861
|
+
result = await self._session.execute(
|
|
862
|
+
update(HttpBridgeRetryCircuit)
|
|
863
|
+
.where(
|
|
864
|
+
HttpBridgeRetryCircuit.session_key_kind == session_key_kind,
|
|
865
|
+
HttpBridgeRetryCircuit.session_key_hash == durable_bridge_hash(session_key_value),
|
|
866
|
+
HttpBridgeRetryCircuit.api_key_scope == api_key_scope,
|
|
867
|
+
HttpBridgeRetryCircuit.updated_at_epoch == expected_updated_at_epoch,
|
|
868
|
+
HttpBridgeRetryCircuit.consecutive_failures == expected_consecutive_failures,
|
|
869
|
+
# The prior detail is part of the fence: two completions
|
|
870
|
+
# can otherwise both believe they own the supersession of
|
|
871
|
+
# one shared row, and the loser's rollback would destroy
|
|
872
|
+
# the winner's — re-poisoning a freshly registered
|
|
873
|
+
# anchor. With this fence exactly one write owns each
|
|
874
|
+
# transition, forward and back.
|
|
875
|
+
HttpBridgeRetryCircuit.last_detail == expected_last_detail,
|
|
876
|
+
)
|
|
877
|
+
.values(last_detail=last_detail)
|
|
878
|
+
)
|
|
879
|
+
await self._session.commit()
|
|
880
|
+
return bool(getattr(result, "rowcount", 0))
|
|
881
|
+
|
|
882
|
+
async def delete_retry_circuit(
|
|
883
|
+
self,
|
|
884
|
+
*,
|
|
885
|
+
session_key_kind: str,
|
|
886
|
+
session_key_value: str,
|
|
887
|
+
api_key_scope: str,
|
|
888
|
+
expected_updated_at_epoch: float | None = None,
|
|
889
|
+
expected_admission_generation: int | None = None,
|
|
890
|
+
expected_consecutive_failures: int | None = None,
|
|
891
|
+
reset_detail: str | None = None,
|
|
892
|
+
) -> bool:
|
|
893
|
+
conditions = [
|
|
894
|
+
HttpBridgeRetryCircuit.session_key_kind == session_key_kind,
|
|
895
|
+
HttpBridgeRetryCircuit.session_key_hash == durable_bridge_hash(session_key_value),
|
|
896
|
+
HttpBridgeRetryCircuit.api_key_scope == api_key_scope,
|
|
897
|
+
]
|
|
898
|
+
if expected_consecutive_failures is not None:
|
|
899
|
+
# A lagging-clock strike merges a higher count without moving
|
|
900
|
+
# the epoch; the count keeps this reset from zeroing a newer
|
|
901
|
+
# episode's cooldown the caller never observed.
|
|
902
|
+
conditions.append(HttpBridgeRetryCircuit.consecutive_failures == expected_consecutive_failures)
|
|
903
|
+
if expected_updated_at_epoch is not None:
|
|
904
|
+
conditions.append(HttpBridgeRetryCircuit.updated_at_epoch == expected_updated_at_epoch)
|
|
905
|
+
if expected_admission_generation is not None:
|
|
906
|
+
# A replay claim bumps only ``admission_generation``, leaving
|
|
907
|
+
# the failure-observation epoch untouched; without this fence
|
|
908
|
+
# a reset authorized before the claim would clear the circuit
|
|
909
|
+
# beneath the admitted replay that depends on it.
|
|
910
|
+
conditions.append(HttpBridgeRetryCircuit.admission_generation == expected_admission_generation)
|
|
911
|
+
async with sqlite_writer_section():
|
|
912
|
+
result = await self._session.execute(
|
|
913
|
+
update(HttpBridgeRetryCircuit)
|
|
914
|
+
.where(*conditions)
|
|
915
|
+
.values(
|
|
916
|
+
consecutive_failures=0,
|
|
917
|
+
cooldown_until_epoch=0.0,
|
|
918
|
+
# An abandonment-driven settle leaves a durable tombstone
|
|
919
|
+
# so restarted workers and other replicas fail
|
|
920
|
+
# delta-only follow-ups closed; a completion's settle
|
|
921
|
+
# writes None, erasing it on real recovery.
|
|
922
|
+
last_detail=reset_detail,
|
|
923
|
+
updated_at_epoch=time.time(),
|
|
924
|
+
)
|
|
925
|
+
)
|
|
926
|
+
await self._session.commit()
|
|
927
|
+
# A fenced reset that matched no row is a CAS miss, not a settlement:
|
|
928
|
+
# another writer moved the row after the caller's lookup.
|
|
929
|
+
return bool(getattr(result, "rowcount", 0))
|
|
930
|
+
|
|
931
|
+
async def purge_retry_circuit(
|
|
932
|
+
self,
|
|
933
|
+
*,
|
|
934
|
+
session_key_kind: str,
|
|
935
|
+
session_key_value: str,
|
|
936
|
+
api_key_scope: str,
|
|
937
|
+
expected_updated_at_epoch: float | None = None,
|
|
938
|
+
expected_admission_generation: int | None = None,
|
|
939
|
+
expected_consecutive_failures: int | None = None,
|
|
940
|
+
fence_last_detail: bool = False,
|
|
941
|
+
expected_last_detail: str | None = None,
|
|
942
|
+
reset_detail: str | None = None,
|
|
943
|
+
) -> bool:
|
|
944
|
+
conditions = [
|
|
945
|
+
HttpBridgeRetryCircuit.session_key_kind == session_key_kind,
|
|
946
|
+
HttpBridgeRetryCircuit.session_key_hash == durable_bridge_hash(session_key_value),
|
|
947
|
+
HttpBridgeRetryCircuit.api_key_scope == api_key_scope,
|
|
948
|
+
]
|
|
949
|
+
if expected_consecutive_failures is not None:
|
|
950
|
+
conditions.append(HttpBridgeRetryCircuit.consecutive_failures == expected_consecutive_failures)
|
|
951
|
+
if fence_last_detail:
|
|
952
|
+
# A detail-only rewrite — the transitional tombstone supersede —
|
|
953
|
+
# deliberately moves neither the timestamp nor the admission
|
|
954
|
+
# generation, so the observed detail is the only fence that can
|
|
955
|
+
# see it. NULL-safe: an expected NULL matches only NULL.
|
|
956
|
+
conditions.append(
|
|
957
|
+
HttpBridgeRetryCircuit.last_detail.is_(None)
|
|
958
|
+
if expected_last_detail is None
|
|
959
|
+
else HttpBridgeRetryCircuit.last_detail == expected_last_detail
|
|
960
|
+
)
|
|
961
|
+
if expected_updated_at_epoch is not None:
|
|
962
|
+
conditions.append(HttpBridgeRetryCircuit.updated_at_epoch == expected_updated_at_epoch)
|
|
963
|
+
if expected_admission_generation is not None:
|
|
964
|
+
# A replay claim advances only the admission generation and
|
|
965
|
+
# leaves the timestamp untouched; without this fence a
|
|
966
|
+
# stale-row purge racing the claim would delete the active
|
|
967
|
+
# claimed generation and a later recovery could dispatch a
|
|
968
|
+
# second replay beside the first.
|
|
969
|
+
conditions.append(HttpBridgeRetryCircuit.admission_generation == expected_admission_generation)
|
|
970
|
+
async with sqlite_writer_section():
|
|
971
|
+
result = await self._session.execute(delete(HttpBridgeRetryCircuit).where(*conditions))
|
|
972
|
+
await self._session.commit()
|
|
973
|
+
# A fenced purge that matched nothing is a CAS miss: another replica
|
|
974
|
+
# moved the row after this worker's lookup, and the caller must
|
|
975
|
+
# reconcile against the surviving row instead of assuming deletion.
|
|
976
|
+
return bool(getattr(result, "rowcount", 0)) or expected_updated_at_epoch is None
|
|
977
|
+
|
|
978
|
+
async def get_session_by_id(self, session_id: str) -> DurableBridgeSessionSnapshot | None:
|
|
979
|
+
row = await self._session.get(HttpBridgeSessionRecord, session_id)
|
|
980
|
+
return _to_snapshot(row)
|
|
981
|
+
|
|
982
|
+
async def resolve_alias(
|
|
983
|
+
self,
|
|
984
|
+
*,
|
|
985
|
+
alias_kind: str,
|
|
986
|
+
alias_value: str,
|
|
987
|
+
api_key_scope: str,
|
|
988
|
+
) -> DurableBridgeSessionSnapshot | None:
|
|
989
|
+
statement = (
|
|
990
|
+
select(HttpBridgeSessionRecord)
|
|
991
|
+
.join(HttpBridgeSessionAlias, HttpBridgeSessionAlias.session_id == HttpBridgeSessionRecord.id)
|
|
992
|
+
.where(
|
|
993
|
+
HttpBridgeSessionAlias.alias_kind == alias_kind,
|
|
994
|
+
HttpBridgeSessionAlias.alias_hash == durable_bridge_hash(alias_value),
|
|
995
|
+
HttpBridgeSessionAlias.api_key_scope == api_key_scope,
|
|
996
|
+
)
|
|
997
|
+
.limit(1)
|
|
998
|
+
)
|
|
999
|
+
result = await self._session.execute(statement)
|
|
1000
|
+
row = result.scalar_one_or_none()
|
|
1001
|
+
return _to_snapshot(row)
|
|
1002
|
+
|
|
1003
|
+
async def find_session_by_latest_turn_state(
|
|
1004
|
+
self,
|
|
1005
|
+
*,
|
|
1006
|
+
turn_state: str,
|
|
1007
|
+
api_key_scope: str,
|
|
1008
|
+
) -> DurableBridgeSessionSnapshot | None:
|
|
1009
|
+
statement = (
|
|
1010
|
+
select(HttpBridgeSessionRecord)
|
|
1011
|
+
.where(
|
|
1012
|
+
HttpBridgeSessionRecord.latest_turn_state == turn_state,
|
|
1013
|
+
HttpBridgeSessionRecord.api_key_scope == api_key_scope,
|
|
1014
|
+
HttpBridgeSessionRecord.state.in_((HttpBridgeSessionState.ACTIVE, HttpBridgeSessionState.DRAINING)),
|
|
1015
|
+
)
|
|
1016
|
+
.order_by(
|
|
1017
|
+
case((HttpBridgeSessionRecord.state == HttpBridgeSessionState.ACTIVE, 0), else_=1),
|
|
1018
|
+
HttpBridgeSessionRecord.last_seen_at.desc(),
|
|
1019
|
+
HttpBridgeSessionRecord.updated_at.desc(),
|
|
1020
|
+
)
|
|
1021
|
+
.limit(1)
|
|
1022
|
+
)
|
|
1023
|
+
result = await self._session.execute(statement)
|
|
1024
|
+
row = result.scalar_one_or_none()
|
|
1025
|
+
return _to_snapshot(row)
|
|
1026
|
+
|
|
1027
|
+
async def find_session_by_latest_response_id(
|
|
1028
|
+
self,
|
|
1029
|
+
*,
|
|
1030
|
+
response_id: str,
|
|
1031
|
+
api_key_scope: str,
|
|
1032
|
+
) -> DurableBridgeSessionSnapshot | None:
|
|
1033
|
+
statement = (
|
|
1034
|
+
select(HttpBridgeSessionRecord)
|
|
1035
|
+
.where(
|
|
1036
|
+
HttpBridgeSessionRecord.latest_response_id == response_id,
|
|
1037
|
+
HttpBridgeSessionRecord.api_key_scope == api_key_scope,
|
|
1038
|
+
HttpBridgeSessionRecord.state.in_((HttpBridgeSessionState.ACTIVE, HttpBridgeSessionState.DRAINING)),
|
|
1039
|
+
)
|
|
1040
|
+
.order_by(
|
|
1041
|
+
case((HttpBridgeSessionRecord.state == HttpBridgeSessionState.ACTIVE, 0), else_=1),
|
|
1042
|
+
HttpBridgeSessionRecord.last_seen_at.desc(),
|
|
1043
|
+
HttpBridgeSessionRecord.updated_at.desc(),
|
|
1044
|
+
)
|
|
1045
|
+
.limit(1)
|
|
1046
|
+
)
|
|
1047
|
+
result = await self._session.execute(statement)
|
|
1048
|
+
row = result.scalar_one_or_none()
|
|
1049
|
+
return _to_snapshot(row)
|
|
1050
|
+
|
|
1051
|
+
async def claim_session(
|
|
1052
|
+
self,
|
|
1053
|
+
*,
|
|
1054
|
+
session_key_kind: str,
|
|
1055
|
+
session_key_value: str,
|
|
1056
|
+
api_key_scope: str,
|
|
1057
|
+
instance_id: str,
|
|
1058
|
+
lease_ttl_seconds: float,
|
|
1059
|
+
account_id: str | None,
|
|
1060
|
+
model: str | None,
|
|
1061
|
+
service_tier: str | None,
|
|
1062
|
+
latest_turn_state: str | None,
|
|
1063
|
+
latest_response_id: str | None,
|
|
1064
|
+
allow_takeover: bool,
|
|
1065
|
+
owner_process_epoch: str,
|
|
1066
|
+
force_owner_epoch_advance: bool = False,
|
|
1067
|
+
) -> DurableBridgeSessionSnapshot:
|
|
1068
|
+
session_key_hash = durable_bridge_hash(session_key_value)
|
|
1069
|
+
# ``allow_takeover`` was decided by the caller against a pre-claim
|
|
1070
|
+
# lookup. Once another claimant has demonstrably written this row under
|
|
1071
|
+
# us (lost CAS, or lost insert race), that decision is stale: the row
|
|
1072
|
+
# we re-read may now carry the winner's live lease, and reusing the
|
|
1073
|
+
# permission would let the loser steal it. Revalidate from the fresh
|
|
1074
|
+
# read instead — a live foreign owner then fails closed exactly like a
|
|
1075
|
+
# non-takeover claim, which surfaces as the correct cross-replica
|
|
1076
|
+
# "retry to reach the correct replica" response.
|
|
1077
|
+
contended = False
|
|
1078
|
+
# Bounded retry budget shared by the insert race (IntegrityError) and
|
|
1079
|
+
# the epoch CAS: every round has a winner, so a loser converges after
|
|
1080
|
+
# at most one fresh read per concurrent claimant.
|
|
1081
|
+
for attempt in range(_CLAIM_CAS_ATTEMPTS):
|
|
1082
|
+
now = utcnow()
|
|
1083
|
+
lease_expires_at = now + timedelta(seconds=max(1.0, lease_ttl_seconds))
|
|
1084
|
+
row = await self._session.execute(
|
|
1085
|
+
select(HttpBridgeSessionRecord)
|
|
1086
|
+
.where(
|
|
1087
|
+
HttpBridgeSessionRecord.session_key_kind == session_key_kind,
|
|
1088
|
+
HttpBridgeSessionRecord.session_key_hash == session_key_hash,
|
|
1089
|
+
HttpBridgeSessionRecord.api_key_scope == api_key_scope,
|
|
1090
|
+
)
|
|
1091
|
+
.with_for_update()
|
|
1092
|
+
)
|
|
1093
|
+
existing = row.scalar_one_or_none()
|
|
1094
|
+
if existing is None:
|
|
1095
|
+
record = HttpBridgeSessionRecord(
|
|
1096
|
+
session_key_kind=session_key_kind,
|
|
1097
|
+
session_key_value=session_key_value,
|
|
1098
|
+
session_key_hash=session_key_hash,
|
|
1099
|
+
api_key_scope=api_key_scope,
|
|
1100
|
+
owner_instance_id=instance_id,
|
|
1101
|
+
owner_process_epoch=owner_process_epoch,
|
|
1102
|
+
owner_epoch=1,
|
|
1103
|
+
lease_expires_at=lease_expires_at,
|
|
1104
|
+
state=HttpBridgeSessionState.ACTIVE,
|
|
1105
|
+
account_id=account_id,
|
|
1106
|
+
model=model,
|
|
1107
|
+
service_tier=service_tier,
|
|
1108
|
+
latest_turn_state=latest_turn_state,
|
|
1109
|
+
latest_response_id=latest_response_id,
|
|
1110
|
+
last_seen_at=now,
|
|
1111
|
+
closed_at=None,
|
|
1112
|
+
)
|
|
1113
|
+
self._session.add(record)
|
|
1114
|
+
try:
|
|
1115
|
+
await self._commit_writer_section()
|
|
1116
|
+
except IntegrityError:
|
|
1117
|
+
await self._session.rollback()
|
|
1118
|
+
if attempt < _CLAIM_CAS_ATTEMPTS - 1:
|
|
1119
|
+
contended = True
|
|
1120
|
+
continue
|
|
1121
|
+
raise
|
|
1122
|
+
# Same reason the CAS path builds its own snapshot: another
|
|
1123
|
+
# same-instance claimant can advance this brand-new row before
|
|
1124
|
+
# a refresh runs, and returning that epoch would hand two
|
|
1125
|
+
# claimants the same fence.
|
|
1126
|
+
inserted_id = record.id
|
|
1127
|
+
return DurableBridgeSessionSnapshot(
|
|
1128
|
+
id=inserted_id,
|
|
1129
|
+
session_key_kind=session_key_kind,
|
|
1130
|
+
session_key_value=session_key_value,
|
|
1131
|
+
session_key_hash=session_key_hash,
|
|
1132
|
+
api_key_scope=api_key_scope,
|
|
1133
|
+
owner_instance_id=instance_id,
|
|
1134
|
+
owner_process_epoch=owner_process_epoch,
|
|
1135
|
+
owner_epoch=1,
|
|
1136
|
+
lease_expires_at=lease_expires_at,
|
|
1137
|
+
state=HttpBridgeSessionState.ACTIVE,
|
|
1138
|
+
account_id=account_id,
|
|
1139
|
+
model=model,
|
|
1140
|
+
service_tier=service_tier,
|
|
1141
|
+
latest_turn_state=latest_turn_state,
|
|
1142
|
+
latest_response_id=latest_response_id,
|
|
1143
|
+
latest_input_item_count=None,
|
|
1144
|
+
latest_input_full_fingerprint=None,
|
|
1145
|
+
latest_pending_tool_calls=None,
|
|
1146
|
+
last_seen_at=now,
|
|
1147
|
+
closed_at=None,
|
|
1148
|
+
)
|
|
1149
|
+
|
|
1150
|
+
state_closed = existing.state == HttpBridgeSessionState.CLOSED
|
|
1151
|
+
owner_absent = existing.owner_instance_id is None
|
|
1152
|
+
account_changed = existing.account_id != account_id
|
|
1153
|
+
# Reclaiming a retired row discards continuity for the same reason
|
|
1154
|
+
# an account change does. ``_to_lookup`` masked the anchors while
|
|
1155
|
+
# the marker stood, so the request that triggered this claim was
|
|
1156
|
+
# planned as an unanchored fresh start. Clearing only the marker
|
|
1157
|
+
# would leave the old response id, turn state, fingerprints and
|
|
1158
|
+
# pending calls behind, and the next lookup would serve that
|
|
1159
|
+
# abandoned anchor as ordinary continuity — including when the
|
|
1160
|
+
# retired account itself recovers and is selected again, where
|
|
1161
|
+
# ``account_changed`` is False.
|
|
1162
|
+
continuity_retired = _bridge_continuity_is_abandoned(
|
|
1163
|
+
existing.continuity_abandoned_at,
|
|
1164
|
+
existing.continuity_abandonment_scope,
|
|
1165
|
+
)
|
|
1166
|
+
owner_changed = existing.owner_instance_id != instance_id
|
|
1167
|
+
if owner_changed:
|
|
1168
|
+
lease_expired = existing.lease_expires_at is None or to_utc_naive(existing.lease_expires_at) <= now
|
|
1169
|
+
live_owned_draining = (
|
|
1170
|
+
existing.state == HttpBridgeSessionState.DRAINING and not lease_expired and not owner_absent
|
|
1171
|
+
)
|
|
1172
|
+
takeover_permitted = allow_takeover and not contended
|
|
1173
|
+
if live_owned_draining or (
|
|
1174
|
+
not takeover_permitted and not lease_expired and not owner_absent and not state_closed
|
|
1175
|
+
):
|
|
1176
|
+
return _to_snapshot_required(existing)
|
|
1177
|
+
# Every claim advances the owner epoch, including a same-owner
|
|
1178
|
+
# reclaim: claims come only from a successor in-memory session (a
|
|
1179
|
+
# reused session renews instead of claiming), so a live same-owner
|
|
1180
|
+
# row means the predecessor local session is retiring concurrently
|
|
1181
|
+
# and its outstanding fenced release/renewals must no-op rather
|
|
1182
|
+
# than race this claim into a closed, ownerless row (issue #1695).
|
|
1183
|
+
next_epoch = existing.owner_epoch + 1
|
|
1184
|
+
|
|
1185
|
+
# Write through an explicit UPDATE that sets every ownership field
|
|
1186
|
+
# unconditionally. Mutating ORM attributes lets SQLAlchemy omit
|
|
1187
|
+
# fields whose values match this transaction's (possibly stale)
|
|
1188
|
+
# read, so a release committing between the SELECT and this write
|
|
1189
|
+
# survived the claim and the refresh below returned a closed,
|
|
1190
|
+
# ownerless row to a claimant that believed it had succeeded
|
|
1191
|
+
# (issue #1695; SQLite's with_for_update is a no-op).
|
|
1192
|
+
values: dict[str, object] = {
|
|
1193
|
+
"owner_instance_id": instance_id,
|
|
1194
|
+
"owner_process_epoch": owner_process_epoch,
|
|
1195
|
+
"owner_epoch": next_epoch,
|
|
1196
|
+
"lease_expires_at": lease_expires_at,
|
|
1197
|
+
"state": HttpBridgeSessionState.ACTIVE,
|
|
1198
|
+
"account_id": account_id,
|
|
1199
|
+
"model": model,
|
|
1200
|
+
"service_tier": service_tier,
|
|
1201
|
+
"last_seen_at": now,
|
|
1202
|
+
"closed_at": None,
|
|
1203
|
+
# A successful claim re-establishes ownership, so any prior
|
|
1204
|
+
# retirement is over. Clearing both markers here is what makes
|
|
1205
|
+
# retirement reversible: the same row, re-pinned to a healthy
|
|
1206
|
+
# account, becomes ordinary hard ownership again.
|
|
1207
|
+
"continuity_abandoned_at": None,
|
|
1208
|
+
"continuity_abandonment_scope": None,
|
|
1209
|
+
}
|
|
1210
|
+
if account_changed or continuity_retired:
|
|
1211
|
+
values["latest_turn_state"] = latest_turn_state
|
|
1212
|
+
values["latest_response_id"] = latest_response_id
|
|
1213
|
+
values["latest_input_item_count"] = None
|
|
1214
|
+
values["latest_input_full_fingerprint"] = None
|
|
1215
|
+
values["latest_pending_tool_calls_json"] = None
|
|
1216
|
+
else:
|
|
1217
|
+
if latest_turn_state is not None:
|
|
1218
|
+
values["latest_turn_state"] = latest_turn_state
|
|
1219
|
+
if latest_response_id is not None:
|
|
1220
|
+
values["latest_response_id"] = latest_response_id
|
|
1221
|
+
values["latest_input_item_count"] = None
|
|
1222
|
+
values["latest_input_full_fingerprint"] = None
|
|
1223
|
+
values["latest_pending_tool_calls_json"] = None
|
|
1224
|
+
async with sqlite_writer_section():
|
|
1225
|
+
# Compare-and-set on the epoch read above: SQLite's
|
|
1226
|
+
# with_for_update is a no-op, so two successor claims can both
|
|
1227
|
+
# read epoch N; without the guard both would write N+1 and both
|
|
1228
|
+
# believe they own the row with colliding fences. The loser's
|
|
1229
|
+
# update matches zero rows and retries against fresh state.
|
|
1230
|
+
result = await self._session.execute(
|
|
1231
|
+
update(HttpBridgeSessionRecord)
|
|
1232
|
+
.where(
|
|
1233
|
+
HttpBridgeSessionRecord.id == existing.id,
|
|
1234
|
+
HttpBridgeSessionRecord.owner_epoch == existing.owner_epoch,
|
|
1235
|
+
)
|
|
1236
|
+
.values(**values)
|
|
1237
|
+
)
|
|
1238
|
+
if not bool(getattr(result, "rowcount", 0)):
|
|
1239
|
+
await self._session.rollback()
|
|
1240
|
+
if attempt < _CLAIM_CAS_ATTEMPTS - 1:
|
|
1241
|
+
contended = True
|
|
1242
|
+
continue
|
|
1243
|
+
raise RuntimeError("Failed to claim durable bridge session after retry")
|
|
1244
|
+
if account_changed or continuity_retired:
|
|
1245
|
+
await self._clear_aliases_for_session(existing.id)
|
|
1246
|
+
await self._session.commit()
|
|
1247
|
+
# Build the snapshot from the values THIS CAS wrote rather than a
|
|
1248
|
+
# post-commit refresh: another successor can commit its own CAS
|
|
1249
|
+
# between this commit and a refresh, and returning that later epoch
|
|
1250
|
+
# would hand this claimant a fence that collides with the winner's.
|
|
1251
|
+
written_turn_state = values.get("latest_turn_state", existing.latest_turn_state)
|
|
1252
|
+
written_response_id = values.get("latest_response_id", existing.latest_response_id)
|
|
1253
|
+
written_pending_json = values.get("latest_pending_tool_calls_json", existing.latest_pending_tool_calls_json)
|
|
1254
|
+
return DurableBridgeSessionSnapshot(
|
|
1255
|
+
id=existing.id,
|
|
1256
|
+
session_key_kind=existing.session_key_kind,
|
|
1257
|
+
session_key_value=existing.session_key_value,
|
|
1258
|
+
session_key_hash=existing.session_key_hash,
|
|
1259
|
+
api_key_scope=existing.api_key_scope,
|
|
1260
|
+
owner_instance_id=instance_id,
|
|
1261
|
+
owner_process_epoch=owner_process_epoch,
|
|
1262
|
+
owner_epoch=next_epoch,
|
|
1263
|
+
lease_expires_at=lease_expires_at,
|
|
1264
|
+
state=HttpBridgeSessionState.ACTIVE,
|
|
1265
|
+
account_id=account_id,
|
|
1266
|
+
model=model,
|
|
1267
|
+
service_tier=service_tier,
|
|
1268
|
+
latest_turn_state=cast("str | None", written_turn_state),
|
|
1269
|
+
latest_response_id=cast("str | None", written_response_id),
|
|
1270
|
+
latest_input_item_count=cast(
|
|
1271
|
+
"int | None", values.get("latest_input_item_count", existing.latest_input_item_count)
|
|
1272
|
+
),
|
|
1273
|
+
latest_input_full_fingerprint=cast(
|
|
1274
|
+
"str | None",
|
|
1275
|
+
values.get("latest_input_full_fingerprint", existing.latest_input_full_fingerprint),
|
|
1276
|
+
),
|
|
1277
|
+
latest_pending_tool_calls=_decode_pending_tool_calls(
|
|
1278
|
+
cast("str | None", written_response_id),
|
|
1279
|
+
cast("str | None", written_pending_json),
|
|
1280
|
+
),
|
|
1281
|
+
last_seen_at=now,
|
|
1282
|
+
closed_at=None,
|
|
1283
|
+
)
|
|
1284
|
+
raise RuntimeError("Failed to claim durable bridge session after retry")
|
|
1285
|
+
|
|
1286
|
+
async def renew_session(
|
|
1287
|
+
self,
|
|
1288
|
+
*,
|
|
1289
|
+
session_id: str,
|
|
1290
|
+
instance_id: str,
|
|
1291
|
+
owner_epoch: int,
|
|
1292
|
+
lease_ttl_seconds: float,
|
|
1293
|
+
latest_turn_state: str | None = None,
|
|
1294
|
+
latest_response_id: str | None = None,
|
|
1295
|
+
latest_input_item_count: int | None = None,
|
|
1296
|
+
latest_input_full_fingerprint: str | None = None,
|
|
1297
|
+
latest_pending_tool_calls: Mapping[str, str] | None = None,
|
|
1298
|
+
state: HttpBridgeSessionState | None = None,
|
|
1299
|
+
) -> DurableBridgeSessionSnapshot | None:
|
|
1300
|
+
"""Renew the lease with a single fenced UPDATE.
|
|
1301
|
+
|
|
1302
|
+
Fenced-out callers mutate nothing and receive the current owner snapshot.
|
|
1303
|
+
"""
|
|
1304
|
+
|
|
1305
|
+
now = utcnow()
|
|
1306
|
+
values: dict[str, object] = {
|
|
1307
|
+
"lease_expires_at": now + timedelta(seconds=max(1.0, lease_ttl_seconds)),
|
|
1308
|
+
"last_seen_at": now,
|
|
1309
|
+
}
|
|
1310
|
+
if latest_turn_state is not None:
|
|
1311
|
+
values["latest_turn_state"] = latest_turn_state
|
|
1312
|
+
if latest_response_id is not None:
|
|
1313
|
+
values["latest_response_id"] = latest_response_id
|
|
1314
|
+
values["latest_pending_tool_calls_json"] = _encode_pending_tool_calls(
|
|
1315
|
+
latest_response_id,
|
|
1316
|
+
latest_pending_tool_calls,
|
|
1317
|
+
)
|
|
1318
|
+
if latest_input_item_count is None or latest_input_full_fingerprint is None:
|
|
1319
|
+
values["latest_input_item_count"] = None
|
|
1320
|
+
values["latest_input_full_fingerprint"] = None
|
|
1321
|
+
if latest_input_item_count is not None and latest_input_full_fingerprint is not None:
|
|
1322
|
+
values["latest_input_item_count"] = latest_input_item_count
|
|
1323
|
+
values["latest_input_full_fingerprint"] = latest_input_full_fingerprint
|
|
1324
|
+
if state is not None:
|
|
1325
|
+
values["state"] = state
|
|
1326
|
+
return await self._execute_fenced_session_update(
|
|
1327
|
+
session_id=session_id,
|
|
1328
|
+
instance_id=instance_id,
|
|
1329
|
+
owner_epoch=owner_epoch,
|
|
1330
|
+
values=values,
|
|
1331
|
+
)
|
|
1332
|
+
|
|
1333
|
+
async def latest_session_continuity(self, *, session_id: str) -> tuple[str | None, str | None] | None:
|
|
1334
|
+
"""Read the durable session's current continuity anchors.
|
|
1335
|
+
|
|
1336
|
+
Returns ``(latest_response_id, latest_turn_state)``, or ``None`` when
|
|
1337
|
+
the session row does not exist. Both anchors are read together because
|
|
1338
|
+
a continuity clear removes both: fencing on the response id alone
|
|
1339
|
+
would let the clear match while a concurrently registered turn state
|
|
1340
|
+
still occupies the row, deleting continuity the caller never proved
|
|
1341
|
+
dead.
|
|
1342
|
+
"""
|
|
1343
|
+
result = await self._session.execute(
|
|
1344
|
+
select(
|
|
1345
|
+
HttpBridgeSessionRecord.latest_response_id,
|
|
1346
|
+
HttpBridgeSessionRecord.latest_turn_state,
|
|
1347
|
+
).where(HttpBridgeSessionRecord.id == session_id)
|
|
1348
|
+
)
|
|
1349
|
+
row = result.first()
|
|
1350
|
+
return None if row is None else (row[0], row[1])
|
|
1351
|
+
|
|
1352
|
+
async def rebind_session_account(
|
|
1353
|
+
self,
|
|
1354
|
+
*,
|
|
1355
|
+
session_id: str,
|
|
1356
|
+
instance_id: str,
|
|
1357
|
+
owner_epoch: int,
|
|
1358
|
+
account_id: str,
|
|
1359
|
+
clear_continuity: bool = False,
|
|
1360
|
+
expected_latest_response_id: object = REBIND_ANCHOR_UNFENCED,
|
|
1361
|
+
expected_latest_turn_state: object = REBIND_ANCHOR_UNFENCED,
|
|
1362
|
+
) -> bool:
|
|
1363
|
+
"""Persist a replacement account only while this worker owns the lease.
|
|
1364
|
+
|
|
1365
|
+
``expected_latest_response_id`` and ``expected_latest_turn_state``
|
|
1366
|
+
fence a continuity clear on the anchors the caller validated: fresh
|
|
1367
|
+
continuity registered by a concurrent completion or turn-state write
|
|
1368
|
+
changes a column, the fenced UPDATE matches zero rows, and the caller
|
|
1369
|
+
observes a failed clear instead of deleting continuity its episode
|
|
1370
|
+
never proved dead.
|
|
1371
|
+
"""
|
|
1372
|
+
|
|
1373
|
+
async with sqlite_writer_section():
|
|
1374
|
+
values: dict[str, object] = {"account_id": account_id}
|
|
1375
|
+
conditions = [
|
|
1376
|
+
HttpBridgeSessionRecord.id == session_id,
|
|
1377
|
+
HttpBridgeSessionRecord.owner_instance_id == instance_id,
|
|
1378
|
+
HttpBridgeSessionRecord.owner_epoch == owner_epoch,
|
|
1379
|
+
]
|
|
1380
|
+
if clear_continuity:
|
|
1381
|
+
values.update(
|
|
1382
|
+
latest_turn_state=None,
|
|
1383
|
+
latest_response_id=None,
|
|
1384
|
+
latest_input_item_count=None,
|
|
1385
|
+
latest_input_full_fingerprint=None,
|
|
1386
|
+
latest_pending_tool_calls_json=None,
|
|
1387
|
+
)
|
|
1388
|
+
if expected_latest_response_id is not REBIND_ANCHOR_UNFENCED:
|
|
1389
|
+
if expected_latest_response_id is None:
|
|
1390
|
+
conditions.append(HttpBridgeSessionRecord.latest_response_id.is_(None))
|
|
1391
|
+
else:
|
|
1392
|
+
conditions.append(HttpBridgeSessionRecord.latest_response_id == expected_latest_response_id)
|
|
1393
|
+
if expected_latest_turn_state is not REBIND_ANCHOR_UNFENCED:
|
|
1394
|
+
if expected_latest_turn_state is None:
|
|
1395
|
+
conditions.append(HttpBridgeSessionRecord.latest_turn_state.is_(None))
|
|
1396
|
+
else:
|
|
1397
|
+
conditions.append(HttpBridgeSessionRecord.latest_turn_state == expected_latest_turn_state)
|
|
1398
|
+
result = await self._session.execute(update(HttpBridgeSessionRecord).where(*conditions).values(**values))
|
|
1399
|
+
if clear_continuity and bool(getattr(result, "rowcount", 0)):
|
|
1400
|
+
await self._clear_aliases_for_session(session_id)
|
|
1401
|
+
await self._session.commit()
|
|
1402
|
+
return bool(getattr(result, "rowcount", 0))
|
|
1403
|
+
|
|
1404
|
+
async def release_session(
|
|
1405
|
+
self,
|
|
1406
|
+
*,
|
|
1407
|
+
session_id: str,
|
|
1408
|
+
instance_id: str,
|
|
1409
|
+
owner_epoch: int,
|
|
1410
|
+
draining: bool,
|
|
1411
|
+
) -> DurableBridgeSessionSnapshot | None:
|
|
1412
|
+
"""Release the lease with a single fenced UPDATE.
|
|
1413
|
+
|
|
1414
|
+
Fenced-out callers mutate nothing and receive the current owner snapshot.
|
|
1415
|
+
"""
|
|
1416
|
+
|
|
1417
|
+
now = utcnow()
|
|
1418
|
+
values: dict[str, object] = {
|
|
1419
|
+
"owner_instance_id": None,
|
|
1420
|
+
"lease_expires_at": now,
|
|
1421
|
+
"last_seen_at": now,
|
|
1422
|
+
"state": HttpBridgeSessionState.DRAINING if draining else HttpBridgeSessionState.CLOSED,
|
|
1423
|
+
"closed_at": None if draining else now,
|
|
1424
|
+
}
|
|
1425
|
+
return await self._execute_fenced_session_update(
|
|
1426
|
+
session_id=session_id,
|
|
1427
|
+
instance_id=instance_id,
|
|
1428
|
+
owner_epoch=owner_epoch,
|
|
1429
|
+
values=values,
|
|
1430
|
+
)
|
|
1431
|
+
|
|
1432
|
+
async def clear_latest_response_anchor(
|
|
1433
|
+
self,
|
|
1434
|
+
*,
|
|
1435
|
+
session_id: str,
|
|
1436
|
+
instance_id: str,
|
|
1437
|
+
owner_epoch: int,
|
|
1438
|
+
) -> DurableBridgeSessionSnapshot | None:
|
|
1439
|
+
"""Invalidate a stuck eventless anchor with a single fenced UPDATE.
|
|
1440
|
+
|
|
1441
|
+
Clears only the response-id anchor and the state bound to it
|
|
1442
|
+
(input fingerprint/count, pending tool-call manifest). Leaves
|
|
1443
|
+
``latest_turn_state`` and aliases untouched so the durable session
|
|
1444
|
+
remains reattachable without the stale anchor. Fenced-out callers
|
|
1445
|
+
mutate nothing and receive the current owner snapshot.
|
|
1446
|
+
"""
|
|
1447
|
+
|
|
1448
|
+
values: dict[str, object] = {
|
|
1449
|
+
"latest_response_id": None,
|
|
1450
|
+
"latest_input_item_count": None,
|
|
1451
|
+
"latest_input_full_fingerprint": None,
|
|
1452
|
+
"latest_pending_tool_calls_json": None,
|
|
1453
|
+
}
|
|
1454
|
+
return await self._execute_fenced_session_update(
|
|
1455
|
+
session_id=session_id,
|
|
1456
|
+
instance_id=instance_id,
|
|
1457
|
+
owner_epoch=owner_epoch,
|
|
1458
|
+
values=values,
|
|
1459
|
+
)
|
|
1460
|
+
|
|
1461
|
+
async def clear_latest_response_anchor_if_matches(
|
|
1462
|
+
self,
|
|
1463
|
+
*,
|
|
1464
|
+
session_id: str,
|
|
1465
|
+
api_key_scope: str,
|
|
1466
|
+
instance_id: str,
|
|
1467
|
+
owner_epoch: int,
|
|
1468
|
+
response_id: str,
|
|
1469
|
+
) -> DurableBridgeSessionSnapshot | None:
|
|
1470
|
+
"""Clear one response anchor and its matching alias under the owner fence.
|
|
1471
|
+
|
|
1472
|
+
The response-id predicate makes a concurrent newer completion win over
|
|
1473
|
+
a stale denial. Only the denied response alias is removed; turn-state
|
|
1474
|
+
and other response aliases remain routable.
|
|
1475
|
+
"""
|
|
1476
|
+
|
|
1477
|
+
values: dict[str, object] = {
|
|
1478
|
+
"latest_response_id": None,
|
|
1479
|
+
"latest_input_item_count": None,
|
|
1480
|
+
"latest_input_full_fingerprint": None,
|
|
1481
|
+
"latest_pending_tool_calls_json": None,
|
|
1482
|
+
}
|
|
1483
|
+
async with sqlite_writer_section():
|
|
1484
|
+
cleared = await self._session.execute(
|
|
1485
|
+
update(HttpBridgeSessionRecord)
|
|
1486
|
+
.where(
|
|
1487
|
+
HttpBridgeSessionRecord.id == session_id,
|
|
1488
|
+
HttpBridgeSessionRecord.api_key_scope == api_key_scope,
|
|
1489
|
+
HttpBridgeSessionRecord.owner_instance_id == instance_id,
|
|
1490
|
+
HttpBridgeSessionRecord.owner_epoch == owner_epoch,
|
|
1491
|
+
HttpBridgeSessionRecord.latest_response_id == response_id,
|
|
1492
|
+
)
|
|
1493
|
+
.values(**values)
|
|
1494
|
+
.returning(HttpBridgeSessionRecord.id)
|
|
1495
|
+
)
|
|
1496
|
+
if cleared.scalar_one_or_none() is None:
|
|
1497
|
+
await self._session.rollback()
|
|
1498
|
+
return None
|
|
1499
|
+
await self._session.execute(
|
|
1500
|
+
delete(HttpBridgeSessionAlias).where(
|
|
1501
|
+
HttpBridgeSessionAlias.session_id == session_id,
|
|
1502
|
+
HttpBridgeSessionAlias.alias_kind == "previous_response_id",
|
|
1503
|
+
HttpBridgeSessionAlias.alias_hash == durable_bridge_hash(response_id),
|
|
1504
|
+
HttpBridgeSessionAlias.alias_value == response_id,
|
|
1505
|
+
HttpBridgeSessionAlias.api_key_scope == api_key_scope,
|
|
1506
|
+
)
|
|
1507
|
+
)
|
|
1508
|
+
row = await self._session.get(
|
|
1509
|
+
HttpBridgeSessionRecord,
|
|
1510
|
+
session_id,
|
|
1511
|
+
populate_existing=True,
|
|
1512
|
+
)
|
|
1513
|
+
await self._session.commit()
|
|
1514
|
+
return _to_snapshot(row)
|
|
1515
|
+
|
|
1516
|
+
async def record_recovery_attempt(
|
|
1517
|
+
self,
|
|
1518
|
+
*,
|
|
1519
|
+
session_id: str,
|
|
1520
|
+
instance_id: str,
|
|
1521
|
+
owner_epoch: int,
|
|
1522
|
+
request_fingerprint: str,
|
|
1523
|
+
request_id: str,
|
|
1524
|
+
account_id: str | None,
|
|
1525
|
+
model: str | None,
|
|
1526
|
+
replay_safe: bool,
|
|
1527
|
+
) -> DurableBridgeRecoveryAttemptSnapshot | None:
|
|
1528
|
+
"""Record a safe request before dispatch so an ambiguous outcome is recoverable."""
|
|
1529
|
+
async with sqlite_writer_section():
|
|
1530
|
+
# Lock the owner row through the journal write so a takeover
|
|
1531
|
+
# cannot advance the epoch after this check but before dispatch.
|
|
1532
|
+
owner_exists = await self._session.scalar(
|
|
1533
|
+
select(HttpBridgeSessionRecord.id)
|
|
1534
|
+
.where(
|
|
1535
|
+
HttpBridgeSessionRecord.id == session_id,
|
|
1536
|
+
HttpBridgeSessionRecord.owner_instance_id == instance_id,
|
|
1537
|
+
HttpBridgeSessionRecord.owner_epoch == owner_epoch,
|
|
1538
|
+
)
|
|
1539
|
+
.with_for_update()
|
|
1540
|
+
)
|
|
1541
|
+
if owner_exists is None:
|
|
1542
|
+
await self._session.rollback()
|
|
1543
|
+
return None
|
|
1544
|
+
attempt = await self._session.scalar(
|
|
1545
|
+
select(HttpBridgeRecoveryAttemptRecord)
|
|
1546
|
+
.where(HttpBridgeRecoveryAttemptRecord.session_id == session_id)
|
|
1547
|
+
.where(HttpBridgeRecoveryAttemptRecord.request_fingerprint == request_fingerprint)
|
|
1548
|
+
.with_for_update()
|
|
1549
|
+
)
|
|
1550
|
+
if attempt is None:
|
|
1551
|
+
attempt = HttpBridgeRecoveryAttemptRecord(
|
|
1552
|
+
session_id=session_id,
|
|
1553
|
+
request_fingerprint=request_fingerprint,
|
|
1554
|
+
request_id=request_id,
|
|
1555
|
+
account_id=account_id,
|
|
1556
|
+
model=model,
|
|
1557
|
+
replay_safe=replay_safe,
|
|
1558
|
+
state=HttpBridgeRecoveryAttemptState.UNKNOWN,
|
|
1559
|
+
)
|
|
1560
|
+
self._session.add(attempt)
|
|
1561
|
+
elif attempt.state == HttpBridgeRecoveryAttemptState.REPLAYED:
|
|
1562
|
+
snapshot = _to_recovery_attempt_snapshot(attempt)
|
|
1563
|
+
await self._session.rollback()
|
|
1564
|
+
return snapshot
|
|
1565
|
+
elif attempt.request_id != request_id:
|
|
1566
|
+
# A different request already owns the UNKNOWN checkpoint.
|
|
1567
|
+
# Do not overwrite it while that request may still be between
|
|
1568
|
+
# admission and dispatch; the caller must fail closed rather
|
|
1569
|
+
# than sharing a journal generation.
|
|
1570
|
+
snapshot = _to_recovery_attempt_snapshot(attempt)
|
|
1571
|
+
await self._session.rollback()
|
|
1572
|
+
return snapshot
|
|
1573
|
+
else:
|
|
1574
|
+
attempt.request_id = request_id
|
|
1575
|
+
attempt.account_id = account_id
|
|
1576
|
+
attempt.model = model
|
|
1577
|
+
attempt.replay_safe = replay_safe
|
|
1578
|
+
attempt.state = HttpBridgeRecoveryAttemptState.UNKNOWN
|
|
1579
|
+
attempt.response_id = None
|
|
1580
|
+
try:
|
|
1581
|
+
await self._session.commit()
|
|
1582
|
+
except IntegrityError:
|
|
1583
|
+
# A concurrent owner may have inserted the same fingerprint
|
|
1584
|
+
# after our initial SELECT (the absent-row case cannot be
|
|
1585
|
+
# locked by SQLite). Re-read the winner and use its durable
|
|
1586
|
+
# state instead of surfacing a transient uniqueness failure.
|
|
1587
|
+
await self._session.rollback()
|
|
1588
|
+
owner_exists = await self._session.scalar(
|
|
1589
|
+
select(HttpBridgeSessionRecord.id)
|
|
1590
|
+
.where(
|
|
1591
|
+
HttpBridgeSessionRecord.id == session_id,
|
|
1592
|
+
HttpBridgeSessionRecord.owner_instance_id == instance_id,
|
|
1593
|
+
HttpBridgeSessionRecord.owner_epoch == owner_epoch,
|
|
1594
|
+
)
|
|
1595
|
+
.with_for_update()
|
|
1596
|
+
)
|
|
1597
|
+
if owner_exists is None:
|
|
1598
|
+
await self._session.rollback()
|
|
1599
|
+
return None
|
|
1600
|
+
attempt = await self._session.scalar(
|
|
1601
|
+
select(HttpBridgeRecoveryAttemptRecord)
|
|
1602
|
+
.where(HttpBridgeRecoveryAttemptRecord.session_id == session_id)
|
|
1603
|
+
.where(HttpBridgeRecoveryAttemptRecord.request_fingerprint == request_fingerprint)
|
|
1604
|
+
)
|
|
1605
|
+
if attempt is None:
|
|
1606
|
+
raise
|
|
1607
|
+
if attempt.state == HttpBridgeRecoveryAttemptState.REPLAYED:
|
|
1608
|
+
snapshot = _to_recovery_attempt_snapshot(attempt)
|
|
1609
|
+
await self._session.rollback()
|
|
1610
|
+
return snapshot
|
|
1611
|
+
if attempt.request_id != request_id:
|
|
1612
|
+
snapshot = _to_recovery_attempt_snapshot(attempt)
|
|
1613
|
+
await self._session.rollback()
|
|
1614
|
+
return snapshot
|
|
1615
|
+
attempt.request_id = request_id
|
|
1616
|
+
attempt.account_id = account_id
|
|
1617
|
+
attempt.model = model
|
|
1618
|
+
attempt.replay_safe = replay_safe
|
|
1619
|
+
attempt.state = HttpBridgeRecoveryAttemptState.UNKNOWN
|
|
1620
|
+
attempt.response_id = None
|
|
1621
|
+
await self._session.commit()
|
|
1622
|
+
await self._session.refresh(attempt)
|
|
1623
|
+
return _to_recovery_attempt_snapshot(attempt)
|
|
1624
|
+
|
|
1625
|
+
async def lookup_recovery_attempt(
|
|
1626
|
+
self,
|
|
1627
|
+
*,
|
|
1628
|
+
session_id: str,
|
|
1629
|
+
request_fingerprint: str,
|
|
1630
|
+
) -> DurableBridgeRecoveryAttemptSnapshot | None:
|
|
1631
|
+
attempt = await self._session.scalar(
|
|
1632
|
+
select(HttpBridgeRecoveryAttemptRecord)
|
|
1633
|
+
.where(HttpBridgeRecoveryAttemptRecord.session_id == session_id)
|
|
1634
|
+
.where(HttpBridgeRecoveryAttemptRecord.request_fingerprint == request_fingerprint)
|
|
1635
|
+
.where(HttpBridgeRecoveryAttemptRecord.state == HttpBridgeRecoveryAttemptState.UNKNOWN)
|
|
1636
|
+
.where(HttpBridgeRecoveryAttemptRecord.replay_safe.is_(True))
|
|
1637
|
+
)
|
|
1638
|
+
return _to_recovery_attempt_snapshot(attempt) if attempt is not None else None
|
|
1639
|
+
|
|
1640
|
+
async def mark_recovery_attempt_replayed(
|
|
1641
|
+
self,
|
|
1642
|
+
*,
|
|
1643
|
+
session_id: str,
|
|
1644
|
+
instance_id: str,
|
|
1645
|
+
owner_epoch: int,
|
|
1646
|
+
request_fingerprint: str,
|
|
1647
|
+
response_id: str | None = None,
|
|
1648
|
+
) -> bool:
|
|
1649
|
+
async with sqlite_writer_section():
|
|
1650
|
+
# Keep the owner fence and journal transition in one transaction.
|
|
1651
|
+
# PostgreSQL's row lock prevents a concurrent takeover from
|
|
1652
|
+
# advancing the epoch between the check and the state update;
|
|
1653
|
+
# sqlite_writer_section provides the equivalent writer
|
|
1654
|
+
# serialization for SQLite.
|
|
1655
|
+
owner_exists = await self._session.scalar(
|
|
1656
|
+
select(HttpBridgeSessionRecord.id)
|
|
1657
|
+
.where(
|
|
1658
|
+
HttpBridgeSessionRecord.id == session_id,
|
|
1659
|
+
HttpBridgeSessionRecord.owner_instance_id == instance_id,
|
|
1660
|
+
HttpBridgeSessionRecord.owner_epoch == owner_epoch,
|
|
1661
|
+
)
|
|
1662
|
+
.with_for_update()
|
|
1663
|
+
)
|
|
1664
|
+
if owner_exists is None:
|
|
1665
|
+
await self._session.rollback()
|
|
1666
|
+
return False
|
|
1667
|
+
values: dict[str, object] = {"state": HttpBridgeRecoveryAttemptState.REPLAYED}
|
|
1668
|
+
if response_id is not None:
|
|
1669
|
+
values["response_id"] = response_id
|
|
1670
|
+
# A claim authorizes one replay and must only transition UNKNOWN
|
|
1671
|
+
# rows. Settlement (which supplies response_id) remains idempotent
|
|
1672
|
+
# for a REPLAYED row after the replay completes.
|
|
1673
|
+
claimable_states = (
|
|
1674
|
+
(HttpBridgeRecoveryAttemptState.UNKNOWN,)
|
|
1675
|
+
if response_id is None
|
|
1676
|
+
else (HttpBridgeRecoveryAttemptState.UNKNOWN, HttpBridgeRecoveryAttemptState.REPLAYED)
|
|
1677
|
+
)
|
|
1678
|
+
result = await self._session.execute(
|
|
1679
|
+
update(HttpBridgeRecoveryAttemptRecord)
|
|
1680
|
+
.where(
|
|
1681
|
+
HttpBridgeRecoveryAttemptRecord.session_id == session_id,
|
|
1682
|
+
HttpBridgeRecoveryAttemptRecord.request_fingerprint == request_fingerprint,
|
|
1683
|
+
HttpBridgeRecoveryAttemptRecord.state.in_(claimable_states),
|
|
1684
|
+
)
|
|
1685
|
+
.values(**values)
|
|
1686
|
+
)
|
|
1687
|
+
await self._session.commit()
|
|
1688
|
+
return bool(getattr(result, "rowcount", 0))
|
|
1689
|
+
|
|
1690
|
+
async def rollback_recovery_attempt_replayed(
|
|
1691
|
+
self,
|
|
1692
|
+
*,
|
|
1693
|
+
session_id: str,
|
|
1694
|
+
instance_id: str,
|
|
1695
|
+
owner_epoch: int,
|
|
1696
|
+
request_fingerprint: str,
|
|
1697
|
+
) -> bool:
|
|
1698
|
+
"""Return a pre-dispatch replay claim to UNKNOWN under the owner fence."""
|
|
1699
|
+
async with sqlite_writer_section():
|
|
1700
|
+
owner_exists = await self._session.scalar(
|
|
1701
|
+
select(HttpBridgeSessionRecord.id)
|
|
1702
|
+
.where(
|
|
1703
|
+
HttpBridgeSessionRecord.id == session_id,
|
|
1704
|
+
HttpBridgeSessionRecord.owner_instance_id == instance_id,
|
|
1705
|
+
HttpBridgeSessionRecord.owner_epoch == owner_epoch,
|
|
1706
|
+
)
|
|
1707
|
+
.with_for_update()
|
|
1708
|
+
)
|
|
1709
|
+
if owner_exists is None:
|
|
1710
|
+
await self._session.rollback()
|
|
1711
|
+
return False
|
|
1712
|
+
result = await self._session.execute(
|
|
1713
|
+
update(HttpBridgeRecoveryAttemptRecord)
|
|
1714
|
+
.where(
|
|
1715
|
+
HttpBridgeRecoveryAttemptRecord.session_id == session_id,
|
|
1716
|
+
HttpBridgeRecoveryAttemptRecord.request_fingerprint == request_fingerprint,
|
|
1717
|
+
HttpBridgeRecoveryAttemptRecord.state == HttpBridgeRecoveryAttemptState.REPLAYED,
|
|
1718
|
+
HttpBridgeRecoveryAttemptRecord.response_id.is_(None),
|
|
1719
|
+
)
|
|
1720
|
+
.values(state=HttpBridgeRecoveryAttemptState.UNKNOWN)
|
|
1721
|
+
)
|
|
1722
|
+
await self._session.commit()
|
|
1723
|
+
return bool(getattr(result, "rowcount", 0))
|
|
1724
|
+
|
|
1725
|
+
async def rollback_recovery_attempt_before_dispatch(
|
|
1726
|
+
self,
|
|
1727
|
+
*,
|
|
1728
|
+
session_id: str,
|
|
1729
|
+
instance_id: str,
|
|
1730
|
+
owner_epoch: int,
|
|
1731
|
+
request_fingerprint: str,
|
|
1732
|
+
) -> bool:
|
|
1733
|
+
"""Delete an UNKNOWN checkpoint proven not to have reached upstream."""
|
|
1734
|
+
async with sqlite_writer_section():
|
|
1735
|
+
owner_exists = await self._session.scalar(
|
|
1736
|
+
select(HttpBridgeSessionRecord.id)
|
|
1737
|
+
.where(
|
|
1738
|
+
HttpBridgeSessionRecord.id == session_id,
|
|
1739
|
+
HttpBridgeSessionRecord.owner_instance_id == instance_id,
|
|
1740
|
+
HttpBridgeSessionRecord.owner_epoch == owner_epoch,
|
|
1741
|
+
)
|
|
1742
|
+
.with_for_update()
|
|
1743
|
+
)
|
|
1744
|
+
if owner_exists is None:
|
|
1745
|
+
await self._session.rollback()
|
|
1746
|
+
return False
|
|
1747
|
+
result = await self._session.execute(
|
|
1748
|
+
delete(HttpBridgeRecoveryAttemptRecord).where(
|
|
1749
|
+
HttpBridgeRecoveryAttemptRecord.session_id == session_id,
|
|
1750
|
+
HttpBridgeRecoveryAttemptRecord.request_fingerprint == request_fingerprint,
|
|
1751
|
+
HttpBridgeRecoveryAttemptRecord.state == HttpBridgeRecoveryAttemptState.UNKNOWN,
|
|
1752
|
+
)
|
|
1753
|
+
)
|
|
1754
|
+
await self._session.commit()
|
|
1755
|
+
return bool(getattr(result, "rowcount", 0))
|
|
1756
|
+
|
|
1757
|
+
async def record_operation(
|
|
1758
|
+
self,
|
|
1759
|
+
*,
|
|
1760
|
+
operation_id: str,
|
|
1761
|
+
session_id: str,
|
|
1762
|
+
instance_id: str,
|
|
1763
|
+
owner_epoch: int,
|
|
1764
|
+
request_fingerprint: str,
|
|
1765
|
+
account_id: str | None,
|
|
1766
|
+
model: str | None,
|
|
1767
|
+
parent_response_id: str | None,
|
|
1768
|
+
api_key_scope: str | None = None,
|
|
1769
|
+
request_text: str | None = None,
|
|
1770
|
+
recovery_attempt_session_id: str | None = None,
|
|
1771
|
+
recovery_attempt_owner_epoch: int | None = None,
|
|
1772
|
+
recovery_attempt_fingerprint: str | None = None,
|
|
1773
|
+
recovery_attempt_consumed: bool = False,
|
|
1774
|
+
) -> DurableBridgeOperationSnapshot | None:
|
|
1775
|
+
"""Create a fenced operation identity, or return the existing one."""
|
|
1776
|
+
async with sqlite_writer_section():
|
|
1777
|
+
owner_exists = await self._session.scalar(
|
|
1778
|
+
select(HttpBridgeSessionRecord.id)
|
|
1779
|
+
.where(
|
|
1780
|
+
HttpBridgeSessionRecord.id == session_id,
|
|
1781
|
+
HttpBridgeSessionRecord.owner_instance_id == instance_id,
|
|
1782
|
+
HttpBridgeSessionRecord.owner_epoch == owner_epoch,
|
|
1783
|
+
)
|
|
1784
|
+
.with_for_update()
|
|
1785
|
+
)
|
|
1786
|
+
if owner_exists is None:
|
|
1787
|
+
await self._session.rollback()
|
|
1788
|
+
return None
|
|
1789
|
+
operation = await self._session.scalar(
|
|
1790
|
+
select(HttpBridgeOperationRecord)
|
|
1791
|
+
.where(HttpBridgeOperationRecord.operation_id == operation_id)
|
|
1792
|
+
.with_for_update()
|
|
1793
|
+
)
|
|
1794
|
+
if operation is None:
|
|
1795
|
+
fingerprint_statement = select(HttpBridgeOperationRecord).where(
|
|
1796
|
+
HttpBridgeOperationRecord.request_fingerprint == request_fingerprint
|
|
1797
|
+
)
|
|
1798
|
+
if api_key_scope is not None:
|
|
1799
|
+
fingerprint_statement = fingerprint_statement.join(
|
|
1800
|
+
HttpBridgeSessionRecord,
|
|
1801
|
+
HttpBridgeSessionRecord.id == HttpBridgeOperationRecord.session_id,
|
|
1802
|
+
).where(HttpBridgeSessionRecord.api_key_scope == api_key_scope)
|
|
1803
|
+
operation = await self._session.scalar(fingerprint_statement.with_for_update())
|
|
1804
|
+
if operation is not None:
|
|
1805
|
+
rebound_from_session_id = operation.session_id if operation.state == "failed" else None
|
|
1806
|
+
rebound_from_account_id = operation.account_id if operation.state == "failed" else None
|
|
1807
|
+
rebound_from_model = operation.model if operation.state == "failed" else None
|
|
1808
|
+
rebound_from_parent_response_id = operation.parent_response_id if operation.state == "failed" else None
|
|
1809
|
+
if operation.state == "abandoned":
|
|
1810
|
+
# Abandonment is a terminal duplicate-suppression fence.
|
|
1811
|
+
# A later continuation must receive the public recovery
|
|
1812
|
+
# signal rather than rebinding or resetting this row.
|
|
1813
|
+
snapshot = _to_operation_snapshot(operation)
|
|
1814
|
+
await self._session.rollback()
|
|
1815
|
+
return snapshot
|
|
1816
|
+
if recovery_attempt_consumed:
|
|
1817
|
+
# A REPLAYED recovery checkpoint is immutable. Return the
|
|
1818
|
+
# existing row for safe transcript replay or fail-closed
|
|
1819
|
+
# handling; never rebind a failed row and clear its spool.
|
|
1820
|
+
snapshot = _to_operation_snapshot(operation)
|
|
1821
|
+
await self._session.rollback()
|
|
1822
|
+
return snapshot
|
|
1823
|
+
rebound = False
|
|
1824
|
+
handoff_allowed = True
|
|
1825
|
+
if operation.session_id != session_id and operation.state not in {"completed", "incomplete"}:
|
|
1826
|
+
# A global fingerprint can outlive the durable session
|
|
1827
|
+
# that first recorded it. Do not steal an operation from
|
|
1828
|
+
# a still-live owner: its stream may still be dispatching
|
|
1829
|
+
# the turn, and rebinding would fence its writes while a
|
|
1830
|
+
# second owner sends a duplicate upstream request.
|
|
1831
|
+
previous_session = await self._session.scalar(
|
|
1832
|
+
select(HttpBridgeSessionRecord)
|
|
1833
|
+
.where(HttpBridgeSessionRecord.id == operation.session_id)
|
|
1834
|
+
.with_for_update()
|
|
1835
|
+
)
|
|
1836
|
+
now = utcnow()
|
|
1837
|
+
recovery_handoff_allowed = False
|
|
1838
|
+
if (
|
|
1839
|
+
previous_session is not None
|
|
1840
|
+
and recovery_attempt_session_id == operation.session_id
|
|
1841
|
+
and recovery_attempt_owner_epoch is not None
|
|
1842
|
+
and recovery_attempt_fingerprint is not None
|
|
1843
|
+
and previous_session.owner_instance_id == instance_id
|
|
1844
|
+
and previous_session.owner_epoch == recovery_attempt_owner_epoch
|
|
1845
|
+
):
|
|
1846
|
+
# A fresh account-neutral replay has already fenced
|
|
1847
|
+
# the one-shot journal on the origin session. That
|
|
1848
|
+
# journal owner must remain fenced until settlement,
|
|
1849
|
+
# but the operation itself must move to the
|
|
1850
|
+
# replacement owner so its transcript and outcome
|
|
1851
|
+
# writes are accepted there. This is the only
|
|
1852
|
+
# cross-session handoff allowed while the origin
|
|
1853
|
+
# lease is still active.
|
|
1854
|
+
recovery_attempt = await self._session.scalar(
|
|
1855
|
+
select(HttpBridgeRecoveryAttemptRecord)
|
|
1856
|
+
.where(
|
|
1857
|
+
HttpBridgeRecoveryAttemptRecord.session_id == recovery_attempt_session_id,
|
|
1858
|
+
HttpBridgeRecoveryAttemptRecord.request_fingerprint == recovery_attempt_fingerprint,
|
|
1859
|
+
HttpBridgeRecoveryAttemptRecord.state == HttpBridgeRecoveryAttemptState.REPLAYED,
|
|
1860
|
+
HttpBridgeRecoveryAttemptRecord.response_id.is_(None),
|
|
1861
|
+
)
|
|
1862
|
+
.with_for_update()
|
|
1863
|
+
)
|
|
1864
|
+
recovery_handoff_allowed = recovery_attempt is not None
|
|
1865
|
+
handoff_allowed = (
|
|
1866
|
+
recovery_handoff_allowed
|
|
1867
|
+
or previous_session is None
|
|
1868
|
+
or not (
|
|
1869
|
+
previous_session.owner_instance_id is not None
|
|
1870
|
+
and previous_session.lease_expires_at is not None
|
|
1871
|
+
# PostgreSQL returns timestamptz values with an
|
|
1872
|
+
# attached UTC offset, while ``utcnow`` is a
|
|
1873
|
+
# naive UTC value used by the durable layer.
|
|
1874
|
+
# Normalize before comparing so cross-session
|
|
1875
|
+
# recovery remains database-backend agnostic.
|
|
1876
|
+
and to_utc_naive(previous_session.lease_expires_at) > now
|
|
1877
|
+
)
|
|
1878
|
+
)
|
|
1879
|
+
if handoff_allowed:
|
|
1880
|
+
# Transfer only nonterminal operations to the currently
|
|
1881
|
+
# fenced owner before the caller resets the attempt
|
|
1882
|
+
# spool; completed transcripts remain attached to
|
|
1883
|
+
# their original session for replay.
|
|
1884
|
+
operation.session_id = session_id
|
|
1885
|
+
operation.account_id = account_id
|
|
1886
|
+
operation.model = model
|
|
1887
|
+
operation.parent_response_id = parent_response_id
|
|
1888
|
+
if request_text is not None and operation.request_text is None:
|
|
1889
|
+
operation.request_text = request_text
|
|
1890
|
+
operation.updated_at = now
|
|
1891
|
+
if operation.state == "failed" and handoff_allowed:
|
|
1892
|
+
# An explicit upstream failure is retryable. Rebind the
|
|
1893
|
+
# durable operation to the current owner while preserving
|
|
1894
|
+
# its global identity; concurrent reconnects will see the
|
|
1895
|
+
# submitted state and remain fenced.
|
|
1896
|
+
operation.session_id = session_id
|
|
1897
|
+
operation.account_id = account_id
|
|
1898
|
+
operation.model = model
|
|
1899
|
+
operation.parent_response_id = parent_response_id
|
|
1900
|
+
if request_text is not None and operation.request_text is None:
|
|
1901
|
+
operation.request_text = request_text
|
|
1902
|
+
operation.state = "submitted"
|
|
1903
|
+
operation.response_id = None
|
|
1904
|
+
# A failed attempt is a new replay attempt. Remove the
|
|
1905
|
+
# previous attempt's SSE spool atomically so a later
|
|
1906
|
+
# successful retry cannot replay a stale response.failed
|
|
1907
|
+
# event before its fresh response.created sequence.
|
|
1908
|
+
await self._delete_operation_spool_material((operation.operation_id,))
|
|
1909
|
+
operation.event_bytes = 0
|
|
1910
|
+
operation.event_spool_complete = False
|
|
1911
|
+
operation.terminal_append_phase = HTTP_BRIDGE_TERMINAL_APPEND_PHASE_PENDING
|
|
1912
|
+
operation.spool_format = HTTP_BRIDGE_SPOOL_FORMAT_ROWS_V1
|
|
1913
|
+
operation.updated_at = utcnow()
|
|
1914
|
+
rebound = True
|
|
1915
|
+
if request_text is not None and operation.request_text is None:
|
|
1916
|
+
operation.request_text = request_text
|
|
1917
|
+
operation.updated_at = utcnow()
|
|
1918
|
+
snapshot = _to_operation_snapshot(
|
|
1919
|
+
operation,
|
|
1920
|
+
rebound=rebound,
|
|
1921
|
+
rebound_from_session_id=rebound_from_session_id,
|
|
1922
|
+
rebound_from_account_id=rebound_from_account_id,
|
|
1923
|
+
rebound_from_model=rebound_from_model,
|
|
1924
|
+
rebound_from_parent_response_id=rebound_from_parent_response_id,
|
|
1925
|
+
)
|
|
1926
|
+
await self._session.commit()
|
|
1927
|
+
return snapshot
|
|
1928
|
+
operation = HttpBridgeOperationRecord(
|
|
1929
|
+
operation_id=operation_id,
|
|
1930
|
+
session_id=session_id,
|
|
1931
|
+
request_fingerprint=request_fingerprint,
|
|
1932
|
+
account_id=account_id,
|
|
1933
|
+
model=model,
|
|
1934
|
+
parent_response_id=parent_response_id,
|
|
1935
|
+
request_text=request_text,
|
|
1936
|
+
state="submitted",
|
|
1937
|
+
# A transcript is replayable only after the event batcher has
|
|
1938
|
+
# drained and finalized it. Set this explicitly rather than
|
|
1939
|
+
# relying on a backend-specific schema default (notably the
|
|
1940
|
+
# pre-existing SQLite default on migrated databases).
|
|
1941
|
+
event_spool_complete=False,
|
|
1942
|
+
spool_format=HTTP_BRIDGE_SPOOL_FORMAT_ROWS_V1,
|
|
1943
|
+
)
|
|
1944
|
+
self._session.add(operation)
|
|
1945
|
+
try:
|
|
1946
|
+
await self._session.commit()
|
|
1947
|
+
except IntegrityError:
|
|
1948
|
+
await self._session.rollback()
|
|
1949
|
+
operation = await self._session.scalar(
|
|
1950
|
+
select(HttpBridgeOperationRecord).where(HttpBridgeOperationRecord.operation_id == operation_id)
|
|
1951
|
+
)
|
|
1952
|
+
if operation is None:
|
|
1953
|
+
# A reconnect may derive a different session-scoped
|
|
1954
|
+
# operation ID for the same anchored request. The global
|
|
1955
|
+
# fingerprint fence makes that race resolve to the
|
|
1956
|
+
# already-recorded operation instead of dispatching a
|
|
1957
|
+
# duplicate.
|
|
1958
|
+
fingerprint_statement = select(HttpBridgeOperationRecord).where(
|
|
1959
|
+
HttpBridgeOperationRecord.request_fingerprint == request_fingerprint
|
|
1960
|
+
)
|
|
1961
|
+
if api_key_scope is not None:
|
|
1962
|
+
fingerprint_statement = fingerprint_statement.join(
|
|
1963
|
+
HttpBridgeSessionRecord,
|
|
1964
|
+
HttpBridgeSessionRecord.id == HttpBridgeOperationRecord.session_id,
|
|
1965
|
+
).where(HttpBridgeSessionRecord.api_key_scope == api_key_scope)
|
|
1966
|
+
operation = await self._session.scalar(fingerprint_statement)
|
|
1967
|
+
if operation is None:
|
|
1968
|
+
raise
|
|
1969
|
+
return _to_operation_snapshot(operation)
|
|
1970
|
+
await self._session.refresh(operation)
|
|
1971
|
+
return _to_operation_snapshot(operation, created=True)
|
|
1972
|
+
|
|
1973
|
+
async def get_operation(self, *, operation_id: str) -> DurableBridgeOperationSnapshot | None:
|
|
1974
|
+
operation = await self._session.scalar(
|
|
1975
|
+
select(HttpBridgeOperationRecord).where(HttpBridgeOperationRecord.operation_id == operation_id)
|
|
1976
|
+
)
|
|
1977
|
+
return _to_operation_snapshot(operation) if operation is not None else None
|
|
1978
|
+
|
|
1979
|
+
async def abandon_stale_operations(
|
|
1980
|
+
self,
|
|
1981
|
+
*,
|
|
1982
|
+
cutoff: datetime,
|
|
1983
|
+
lease_expired_before: datetime,
|
|
1984
|
+
protected_operation_ids: Collection[str] = (),
|
|
1985
|
+
batch_size: int = _PURGE_CLOSED_BATCH_SIZE,
|
|
1986
|
+
scan_cursor: DurableBridgeOperationAbandonmentScanCursor | None = None,
|
|
1987
|
+
) -> DurableBridgeOperationAbandonmentSweep:
|
|
1988
|
+
"""Fence stale ownerless ambiguous operations without replaying them.
|
|
1989
|
+
|
|
1990
|
+
The candidate read and the conditional update run in one writer
|
|
1991
|
+
transaction. PostgreSQL locks the operation and owning session on the
|
|
1992
|
+
normal bounded-predicate path while SQLite is serialized by
|
|
1993
|
+
``sqlite_writer_section``. An oversized protection snapshot is read in
|
|
1994
|
+
finite pages without an expanding predicate; the caller resumes from
|
|
1995
|
+
the returned keyset cursor on the next sweep. Its final updates still
|
|
1996
|
+
compare every value that can change the abandonment decision, including
|
|
1997
|
+
durable event-spool progress, so a recovery claim, owner renewal, or
|
|
1998
|
+
status event that wins the race leaves the row untouched.
|
|
1999
|
+
"""
|
|
2000
|
+
now = utcnow()
|
|
2001
|
+
if batch_size <= 0:
|
|
2002
|
+
return DurableBridgeOperationAbandonmentSweep(abandonments=(), next_cursor=None)
|
|
2003
|
+
protected_ids = tuple(
|
|
2004
|
+
dict.fromkeys(str(operation_id) for operation_id in protected_operation_ids if operation_id)
|
|
2005
|
+
)
|
|
2006
|
+
protected_id_set = frozenset(protected_ids)
|
|
2007
|
+
# A large snapshot cannot be represented by one expanding ``NOT IN``
|
|
2008
|
+
# predicate safely; the bounded-page path below filters it in memory.
|
|
2009
|
+
use_bounded_protection = len(protected_ids) > _PROTECTED_OPERATION_ID_SAFE_LIMIT
|
|
2010
|
+
ambiguous_states = ("unknown", "acknowledged")
|
|
2011
|
+
stale_owner = or_(
|
|
2012
|
+
HttpBridgeSessionRecord.lease_expires_at.is_(None),
|
|
2013
|
+
HttpBridgeSessionRecord.lease_expires_at <= lease_expired_before,
|
|
2014
|
+
)
|
|
2015
|
+
candidate_filter = [
|
|
2016
|
+
HttpBridgeOperationRecord.state.in_(ambiguous_states),
|
|
2017
|
+
HttpBridgeOperationRecord.updated_at < cutoff,
|
|
2018
|
+
stale_owner,
|
|
2019
|
+
]
|
|
2020
|
+
if protected_ids and not use_bounded_protection:
|
|
2021
|
+
candidate_filter.append(~HttpBridgeOperationRecord.operation_id.in_(protected_ids))
|
|
2022
|
+
|
|
2023
|
+
abandoned: list[DurableBridgeOperationAbandonment] = []
|
|
2024
|
+
next_cursor: DurableBridgeOperationAbandonmentScanCursor | None = None
|
|
2025
|
+
async with sqlite_writer_section():
|
|
2026
|
+
candidates = []
|
|
2027
|
+
cursor = scan_cursor if use_bounded_protection else None
|
|
2028
|
+
scanned_rows = 0
|
|
2029
|
+
while len(candidates) < batch_size and (
|
|
2030
|
+
not use_bounded_protection or scanned_rows < _PROTECTED_OPERATION_SCAN_BUDGET
|
|
2031
|
+
):
|
|
2032
|
+
page_filter = list(candidate_filter)
|
|
2033
|
+
if cursor is not None:
|
|
2034
|
+
# Best-effort keyset resume: on SQLite a row whose
|
|
2035
|
+
# ``updated_at`` was stamped by ``onupdate`` shares the
|
|
2036
|
+
# cursor row's second but not its text form, so it is
|
|
2037
|
+
# neither ``>`` nor ``==`` the bound cursor value and is
|
|
2038
|
+
# skipped until the cursor wraps to ``None``. The sweep
|
|
2039
|
+
# still converges because every pass restarts from the
|
|
2040
|
+
# oldest unprotected row once the scan reaches the end.
|
|
2041
|
+
page_filter.append(
|
|
2042
|
+
or_(
|
|
2043
|
+
HttpBridgeOperationRecord.updated_at > cursor.updated_at,
|
|
2044
|
+
and_(
|
|
2045
|
+
HttpBridgeOperationRecord.updated_at == cursor.updated_at,
|
|
2046
|
+
HttpBridgeOperationRecord.operation_id > cursor.operation_id,
|
|
2047
|
+
),
|
|
2048
|
+
)
|
|
2049
|
+
)
|
|
2050
|
+
page_size = batch_size
|
|
2051
|
+
if use_bounded_protection:
|
|
2052
|
+
page_size = min(
|
|
2053
|
+
_PROTECTED_OPERATION_ID_SAFE_LIMIT,
|
|
2054
|
+
_PROTECTED_OPERATION_SCAN_BUDGET - scanned_rows,
|
|
2055
|
+
)
|
|
2056
|
+
statement = (
|
|
2057
|
+
select(
|
|
2058
|
+
HttpBridgeOperationRecord,
|
|
2059
|
+
HttpBridgeSessionRecord.owner_instance_id,
|
|
2060
|
+
HttpBridgeSessionRecord.owner_epoch,
|
|
2061
|
+
HttpBridgeOperationRecord.event_bytes,
|
|
2062
|
+
)
|
|
2063
|
+
.join(
|
|
2064
|
+
HttpBridgeSessionRecord,
|
|
2065
|
+
HttpBridgeSessionRecord.id == HttpBridgeOperationRecord.session_id,
|
|
2066
|
+
)
|
|
2067
|
+
.where(*page_filter)
|
|
2068
|
+
.order_by(
|
|
2069
|
+
HttpBridgeOperationRecord.updated_at.asc(),
|
|
2070
|
+
HttpBridgeOperationRecord.operation_id.asc(),
|
|
2071
|
+
)
|
|
2072
|
+
.limit(page_size)
|
|
2073
|
+
)
|
|
2074
|
+
statement = statement.with_for_update()
|
|
2075
|
+
selected = await self._session.execute(statement)
|
|
2076
|
+
page = list(selected.all())
|
|
2077
|
+
if not page:
|
|
2078
|
+
# Reaching the end after a prior cursor wraps the next
|
|
2079
|
+
# heartbeat to the beginning of the keyset.
|
|
2080
|
+
next_cursor = None
|
|
2081
|
+
break
|
|
2082
|
+
if use_bounded_protection:
|
|
2083
|
+
scanned_rows += len(page)
|
|
2084
|
+
candidates.extend(row for row in page if str(row[0].operation_id) not in protected_id_set)
|
|
2085
|
+
last_operation = page[-1][0]
|
|
2086
|
+
next_cursor = DurableBridgeOperationAbandonmentScanCursor(
|
|
2087
|
+
updated_at=last_operation.updated_at,
|
|
2088
|
+
operation_id=str(last_operation.operation_id),
|
|
2089
|
+
)
|
|
2090
|
+
if len(candidates) >= batch_size:
|
|
2091
|
+
candidates = candidates[:batch_size]
|
|
2092
|
+
break
|
|
2093
|
+
cursor = next_cursor
|
|
2094
|
+
if len(page) < page_size:
|
|
2095
|
+
next_cursor = None
|
|
2096
|
+
break
|
|
2097
|
+
if scanned_rows >= _PROTECTED_OPERATION_SCAN_BUDGET:
|
|
2098
|
+
break
|
|
2099
|
+
else:
|
|
2100
|
+
candidates = page
|
|
2101
|
+
break
|
|
2102
|
+
for operation, owner_instance_id, owner_epoch, candidate_event_bytes in candidates:
|
|
2103
|
+
source_state = str(operation.state)
|
|
2104
|
+
candidate_updated_at = operation.updated_at
|
|
2105
|
+
candidate_event_bytes = int(candidate_event_bytes or 0)
|
|
2106
|
+
owner_predicates = [
|
|
2107
|
+
HttpBridgeSessionRecord.id == operation.session_id,
|
|
2108
|
+
HttpBridgeSessionRecord.owner_epoch == owner_epoch,
|
|
2109
|
+
]
|
|
2110
|
+
if owner_instance_id is None:
|
|
2111
|
+
owner_predicates.extend(
|
|
2112
|
+
(
|
|
2113
|
+
HttpBridgeSessionRecord.owner_instance_id.is_(None),
|
|
2114
|
+
or_(
|
|
2115
|
+
HttpBridgeSessionRecord.lease_expires_at.is_(None),
|
|
2116
|
+
HttpBridgeSessionRecord.lease_expires_at <= lease_expired_before,
|
|
2117
|
+
),
|
|
2118
|
+
)
|
|
2119
|
+
)
|
|
2120
|
+
owner_lease_outcome = "ownerless"
|
|
2121
|
+
else:
|
|
2122
|
+
owner_predicates.extend(
|
|
2123
|
+
(
|
|
2124
|
+
HttpBridgeSessionRecord.owner_instance_id == owner_instance_id,
|
|
2125
|
+
or_(
|
|
2126
|
+
HttpBridgeSessionRecord.lease_expires_at.is_(None),
|
|
2127
|
+
HttpBridgeSessionRecord.lease_expires_at <= lease_expired_before,
|
|
2128
|
+
),
|
|
2129
|
+
)
|
|
2130
|
+
)
|
|
2131
|
+
owner_lease_outcome = "expired"
|
|
2132
|
+
# The inactivity clock is compared against ``cutoff`` rather
|
|
2133
|
+
# than for equality with the loaded candidate value. SQLite
|
|
2134
|
+
# stores the ``onupdate=func.now()`` timestamp written by the
|
|
2135
|
+
# event appenders as second-precision text while the ORM
|
|
2136
|
+
# binds the loaded datetime back with microseconds, so an
|
|
2137
|
+
# equality predicate never matches the rows this sweep
|
|
2138
|
+
# exists for. Any competing progress, claim, or renewal write
|
|
2139
|
+
# stamps ``updated_at`` at roughly ``now``, which is never
|
|
2140
|
+
# older than ``cutoff``, so the race fence is preserved.
|
|
2141
|
+
compare_and_set = await self._session.execute(
|
|
2142
|
+
update(HttpBridgeOperationRecord)
|
|
2143
|
+
.where(
|
|
2144
|
+
HttpBridgeOperationRecord.operation_id == operation.operation_id,
|
|
2145
|
+
HttpBridgeOperationRecord.state == source_state,
|
|
2146
|
+
HttpBridgeOperationRecord.updated_at < cutoff,
|
|
2147
|
+
HttpBridgeOperationRecord.event_bytes == candidate_event_bytes,
|
|
2148
|
+
exists(select(HttpBridgeSessionRecord.id).where(*owner_predicates)),
|
|
2149
|
+
)
|
|
2150
|
+
.values(state="abandoned", updated_at=now)
|
|
2151
|
+
)
|
|
2152
|
+
if not getattr(compare_and_set, "rowcount", 0):
|
|
2153
|
+
continue
|
|
2154
|
+
age_seconds = max(0.0, (to_utc_naive(now) - to_utc_naive(candidate_updated_at)).total_seconds())
|
|
2155
|
+
abandoned.append(
|
|
2156
|
+
DurableBridgeOperationAbandonment(
|
|
2157
|
+
source_state=source_state,
|
|
2158
|
+
age_seconds=min(age_seconds, float(_ABANDONMENT_LOG_AGE_CAP_SECONDS)),
|
|
2159
|
+
owner_lease_outcome=owner_lease_outcome,
|
|
2160
|
+
session_hash=durable_bridge_hash(operation.session_id)[:16],
|
|
2161
|
+
)
|
|
2162
|
+
)
|
|
2163
|
+
await self._session.commit()
|
|
2164
|
+
return DurableBridgeOperationAbandonmentSweep(
|
|
2165
|
+
abandonments=tuple(abandoned),
|
|
2166
|
+
next_cursor=next_cursor if use_bounded_protection else None,
|
|
2167
|
+
)
|
|
2168
|
+
|
|
2169
|
+
async def reset_operation_event_spool(
|
|
2170
|
+
self,
|
|
2171
|
+
*,
|
|
2172
|
+
operation_id: str,
|
|
2173
|
+
session_id: str,
|
|
2174
|
+
instance_id: str,
|
|
2175
|
+
owner_epoch: int,
|
|
2176
|
+
) -> bool:
|
|
2177
|
+
"""Start a fresh transcript for a server-owned ambiguous retry."""
|
|
2178
|
+
async with sqlite_writer_section():
|
|
2179
|
+
owner_exists = await self._session.scalar(
|
|
2180
|
+
select(HttpBridgeSessionRecord.id)
|
|
2181
|
+
.where(
|
|
2182
|
+
HttpBridgeSessionRecord.id == session_id,
|
|
2183
|
+
HttpBridgeSessionRecord.owner_instance_id == instance_id,
|
|
2184
|
+
HttpBridgeSessionRecord.owner_epoch == owner_epoch,
|
|
2185
|
+
)
|
|
2186
|
+
.with_for_update()
|
|
2187
|
+
)
|
|
2188
|
+
operation = await self._session.scalar(
|
|
2189
|
+
select(HttpBridgeOperationRecord)
|
|
2190
|
+
.where(
|
|
2191
|
+
HttpBridgeOperationRecord.operation_id == operation_id,
|
|
2192
|
+
HttpBridgeOperationRecord.session_id == session_id,
|
|
2193
|
+
HttpBridgeOperationRecord.state.not_in(("completed", "incomplete", "abandoned")),
|
|
2194
|
+
)
|
|
2195
|
+
.with_for_update()
|
|
2196
|
+
)
|
|
2197
|
+
if owner_exists is None or operation is None or operation.state == "abandoned":
|
|
2198
|
+
await self._session.rollback()
|
|
2199
|
+
return False
|
|
2200
|
+
await self._delete_operation_spool_material((operation_id,))
|
|
2201
|
+
operation.event_bytes = 0
|
|
2202
|
+
operation.event_spool_complete = False
|
|
2203
|
+
# The phase fences the terminal write of one attempt only. A fresh
|
|
2204
|
+
# transcript is a fresh attempt, so it must not inherit a previous
|
|
2205
|
+
# attempt's recorded outcome (``failed`` rows are resettable).
|
|
2206
|
+
operation.terminal_append_phase = HTTP_BRIDGE_TERMINAL_APPEND_PHASE_PENDING
|
|
2207
|
+
operation.spool_format = HTTP_BRIDGE_SPOOL_FORMAT_ROWS_V1
|
|
2208
|
+
operation.updated_at = utcnow()
|
|
2209
|
+
await self._session.commit()
|
|
2210
|
+
return True
|
|
2211
|
+
|
|
2212
|
+
async def claim_unknown_operation_for_recovery(
|
|
2213
|
+
self,
|
|
2214
|
+
*,
|
|
2215
|
+
operation_id: str,
|
|
2216
|
+
session_id: str,
|
|
2217
|
+
instance_id: str,
|
|
2218
|
+
owner_epoch: int,
|
|
2219
|
+
max_recovery_dispatches: int | None = None,
|
|
2220
|
+
) -> bool:
|
|
2221
|
+
"""Atomically claim an UNKNOWN operation for one recovery attempt.
|
|
2222
|
+
|
|
2223
|
+
Recovery admission can be reached by multiple reconnects at once. A
|
|
2224
|
+
reset followed by a later state transition leaves a window where each
|
|
2225
|
+
reconnect can observe UNKNOWN and submit the same operation. Keep the
|
|
2226
|
+
owner fence, state transition, and transcript reset in one serialized
|
|
2227
|
+
write so exactly one caller can move UNKNOWN back to SUBMITTED.
|
|
2228
|
+
"""
|
|
2229
|
+
async with sqlite_writer_section():
|
|
2230
|
+
owner_exists = await self._session.scalar(
|
|
2231
|
+
select(HttpBridgeSessionRecord.id)
|
|
2232
|
+
.where(
|
|
2233
|
+
HttpBridgeSessionRecord.id == session_id,
|
|
2234
|
+
HttpBridgeSessionRecord.owner_instance_id == instance_id,
|
|
2235
|
+
HttpBridgeSessionRecord.owner_epoch == owner_epoch,
|
|
2236
|
+
)
|
|
2237
|
+
.with_for_update()
|
|
2238
|
+
)
|
|
2239
|
+
operation = await self._session.scalar(
|
|
2240
|
+
select(HttpBridgeOperationRecord)
|
|
2241
|
+
.where(
|
|
2242
|
+
HttpBridgeOperationRecord.operation_id == operation_id,
|
|
2243
|
+
HttpBridgeOperationRecord.session_id == session_id,
|
|
2244
|
+
HttpBridgeOperationRecord.state == "unknown",
|
|
2245
|
+
)
|
|
2246
|
+
.with_for_update()
|
|
2247
|
+
)
|
|
2248
|
+
if owner_exists is None or operation is None or operation.state == "abandoned":
|
|
2249
|
+
await self._session.rollback()
|
|
2250
|
+
return False
|
|
2251
|
+
if max_recovery_dispatches is not None and operation.recovery_dispatch_count >= max_recovery_dispatches:
|
|
2252
|
+
await self._session.rollback()
|
|
2253
|
+
return False
|
|
2254
|
+
await self._delete_operation_spool_material((operation_id,))
|
|
2255
|
+
operation.state = "submitted"
|
|
2256
|
+
operation.response_id = None
|
|
2257
|
+
operation.recovery_dispatch_count += 1
|
|
2258
|
+
operation.event_bytes = 0
|
|
2259
|
+
operation.event_spool_complete = False
|
|
2260
|
+
operation.spool_format = HTTP_BRIDGE_SPOOL_FORMAT_ROWS_V1
|
|
2261
|
+
operation.updated_at = utcnow()
|
|
2262
|
+
await self._session.commit()
|
|
2263
|
+
return True
|
|
2264
|
+
|
|
2265
|
+
async def mark_operation_unknown(
|
|
2266
|
+
self,
|
|
2267
|
+
*,
|
|
2268
|
+
operation_id: str,
|
|
2269
|
+
session_id: str,
|
|
2270
|
+
instance_id: str,
|
|
2271
|
+
owner_epoch: int,
|
|
2272
|
+
restore_recovery_dispatch_claim: bool = False,
|
|
2273
|
+
) -> bool:
|
|
2274
|
+
"""Fence an ambiguously dispatched SUBMITTED operation as UNKNOWN.
|
|
2275
|
+
|
|
2276
|
+
The operation event reader can race the send-failure cleanup. Lock the
|
|
2277
|
+
row before changing it and leave an already acknowledged or terminal
|
|
2278
|
+
operation untouched; those states carry stronger evidence than the
|
|
2279
|
+
transport exception and must never be downgraded to UNKNOWN.
|
|
2280
|
+
"""
|
|
2281
|
+
async with sqlite_writer_section():
|
|
2282
|
+
owner_exists = await self._session.scalar(
|
|
2283
|
+
select(HttpBridgeSessionRecord.id)
|
|
2284
|
+
.where(
|
|
2285
|
+
HttpBridgeSessionRecord.id == session_id,
|
|
2286
|
+
HttpBridgeSessionRecord.owner_instance_id == instance_id,
|
|
2287
|
+
HttpBridgeSessionRecord.owner_epoch == owner_epoch,
|
|
2288
|
+
)
|
|
2289
|
+
.with_for_update()
|
|
2290
|
+
)
|
|
2291
|
+
operation = await self._session.scalar(
|
|
2292
|
+
select(HttpBridgeOperationRecord)
|
|
2293
|
+
.where(
|
|
2294
|
+
HttpBridgeOperationRecord.operation_id == operation_id,
|
|
2295
|
+
HttpBridgeOperationRecord.session_id == session_id,
|
|
2296
|
+
)
|
|
2297
|
+
.with_for_update()
|
|
2298
|
+
)
|
|
2299
|
+
if owner_exists is None or operation is None or operation.state == "abandoned":
|
|
2300
|
+
await self._session.rollback()
|
|
2301
|
+
return False
|
|
2302
|
+
if operation.state == "submitted":
|
|
2303
|
+
operation.state = "unknown"
|
|
2304
|
+
if restore_recovery_dispatch_claim and operation.recovery_dispatch_count > 0:
|
|
2305
|
+
operation.recovery_dispatch_count -= 1
|
|
2306
|
+
operation.updated_at = utcnow()
|
|
2307
|
+
elif (
|
|
2308
|
+
restore_recovery_dispatch_claim
|
|
2309
|
+
and operation.state == "unknown"
|
|
2310
|
+
and operation.recovery_dispatch_count > 0
|
|
2311
|
+
):
|
|
2312
|
+
# A concurrent cleanup may have fenced the row first. The
|
|
2313
|
+
# caller still owns a proven pre-dispatch recovery claim, so
|
|
2314
|
+
# refund exactly that claim while retaining UNKNOWN.
|
|
2315
|
+
operation.recovery_dispatch_count -= 1
|
|
2316
|
+
operation.updated_at = utcnow()
|
|
2317
|
+
await self._session.commit()
|
|
2318
|
+
return True
|
|
2319
|
+
|
|
2320
|
+
async def rollback_operation_before_dispatch(
|
|
2321
|
+
self,
|
|
2322
|
+
*,
|
|
2323
|
+
operation_id: str,
|
|
2324
|
+
session_id: str,
|
|
2325
|
+
instance_id: str,
|
|
2326
|
+
owner_epoch: int,
|
|
2327
|
+
restore_rebound: bool = False,
|
|
2328
|
+
rebound_from_session_id: str | None = None,
|
|
2329
|
+
rebound_from_account_id: str | None = None,
|
|
2330
|
+
rebound_from_model: str | None = None,
|
|
2331
|
+
rebound_from_parent_response_id: str | None = None,
|
|
2332
|
+
) -> bool:
|
|
2333
|
+
"""Undo an operation transition that never reached upstream."""
|
|
2334
|
+
async with sqlite_writer_section():
|
|
2335
|
+
owner_exists = await self._session.scalar(
|
|
2336
|
+
select(HttpBridgeSessionRecord.id)
|
|
2337
|
+
.where(
|
|
2338
|
+
HttpBridgeSessionRecord.id == session_id,
|
|
2339
|
+
HttpBridgeSessionRecord.owner_instance_id == instance_id,
|
|
2340
|
+
HttpBridgeSessionRecord.owner_epoch == owner_epoch,
|
|
2341
|
+
)
|
|
2342
|
+
.with_for_update()
|
|
2343
|
+
)
|
|
2344
|
+
operation = await self._session.scalar(
|
|
2345
|
+
select(HttpBridgeOperationRecord)
|
|
2346
|
+
.where(
|
|
2347
|
+
HttpBridgeOperationRecord.operation_id == operation_id,
|
|
2348
|
+
HttpBridgeOperationRecord.session_id == session_id,
|
|
2349
|
+
HttpBridgeOperationRecord.state == "submitted",
|
|
2350
|
+
HttpBridgeOperationRecord.response_id.is_(None),
|
|
2351
|
+
HttpBridgeOperationRecord.event_bytes == 0,
|
|
2352
|
+
)
|
|
2353
|
+
.with_for_update()
|
|
2354
|
+
)
|
|
2355
|
+
if owner_exists is None or operation is None:
|
|
2356
|
+
await self._session.rollback()
|
|
2357
|
+
return False
|
|
2358
|
+
if await self._operation_has_spool_material(operation_id):
|
|
2359
|
+
await self._session.rollback()
|
|
2360
|
+
return False
|
|
2361
|
+
if restore_rebound:
|
|
2362
|
+
operation.state = "failed"
|
|
2363
|
+
operation.event_spool_complete = False
|
|
2364
|
+
if rebound_from_session_id is not None:
|
|
2365
|
+
operation.session_id = rebound_from_session_id
|
|
2366
|
+
operation.account_id = rebound_from_account_id
|
|
2367
|
+
operation.model = rebound_from_model
|
|
2368
|
+
operation.parent_response_id = rebound_from_parent_response_id
|
|
2369
|
+
operation.updated_at = utcnow()
|
|
2370
|
+
else:
|
|
2371
|
+
await self._session.delete(operation)
|
|
2372
|
+
await self._session.commit()
|
|
2373
|
+
return True
|
|
2374
|
+
|
|
2375
|
+
async def get_operation_by_fingerprint(
|
|
2376
|
+
self,
|
|
2377
|
+
*,
|
|
2378
|
+
request_fingerprint: str,
|
|
2379
|
+
api_key_scope: str | None = None,
|
|
2380
|
+
) -> DurableBridgeOperationSnapshot | None:
|
|
2381
|
+
statement = select(HttpBridgeOperationRecord).where(
|
|
2382
|
+
HttpBridgeOperationRecord.request_fingerprint == request_fingerprint
|
|
2383
|
+
)
|
|
2384
|
+
if api_key_scope is not None:
|
|
2385
|
+
statement = statement.join(
|
|
2386
|
+
HttpBridgeSessionRecord,
|
|
2387
|
+
HttpBridgeSessionRecord.id == HttpBridgeOperationRecord.session_id,
|
|
2388
|
+
).where(HttpBridgeSessionRecord.api_key_scope == api_key_scope)
|
|
2389
|
+
operation = await self._session.scalar(statement)
|
|
2390
|
+
return _to_operation_snapshot(operation) if operation is not None else None
|
|
2391
|
+
|
|
2392
|
+
async def get_operation_events(self, *, operation_id: str, max_bytes: int = 8 * 1024 * 1024) -> list[str]:
|
|
2393
|
+
operation = (
|
|
2394
|
+
await self._session.execute(
|
|
2395
|
+
select(
|
|
2396
|
+
HttpBridgeOperationRecord.spool_format,
|
|
2397
|
+
HttpBridgeOperationRecord.event_bytes,
|
|
2398
|
+
).where(HttpBridgeOperationRecord.operation_id == operation_id)
|
|
2399
|
+
)
|
|
2400
|
+
).one_or_none()
|
|
2401
|
+
if operation is None:
|
|
2402
|
+
return []
|
|
2403
|
+
spool_format = str(operation.spool_format)
|
|
2404
|
+
if type(operation.event_bytes) is not int:
|
|
2405
|
+
return []
|
|
2406
|
+
expected_event_bytes = operation.event_bytes
|
|
2407
|
+
if expected_event_bytes < 0 or expected_event_bytes > max_bytes:
|
|
2408
|
+
return []
|
|
2409
|
+
if spool_format == HTTP_BRIDGE_SPOOL_FORMAT_ROWS_V1:
|
|
2410
|
+
result = await self._session.execute(
|
|
2411
|
+
select(HttpBridgeOperationEvent.event_text)
|
|
2412
|
+
.where(HttpBridgeOperationEvent.operation_id == operation_id)
|
|
2413
|
+
.order_by(HttpBridgeOperationEvent.sequence_number.asc())
|
|
2414
|
+
)
|
|
2415
|
+
events = [str(value) for value in result.scalars().all()]
|
|
2416
|
+
if sum(len(event.encode("utf-8")) for event in events) != expected_event_bytes:
|
|
2417
|
+
return []
|
|
2418
|
+
return events
|
|
2419
|
+
if spool_format != HTTP_BRIDGE_SPOOL_FORMAT_CHUNKS_V2:
|
|
2420
|
+
return []
|
|
2421
|
+
|
|
2422
|
+
chunks = (
|
|
2423
|
+
await self._session.execute(
|
|
2424
|
+
select(HttpBridgeOperationEventChunk)
|
|
2425
|
+
.where(HttpBridgeOperationEventChunk.operation_id == operation_id)
|
|
2426
|
+
.order_by(HttpBridgeOperationEventChunk.first_sequence_number.asc())
|
|
2427
|
+
)
|
|
2428
|
+
).scalars()
|
|
2429
|
+
expected_sequence = 1
|
|
2430
|
+
total_event_count = 0
|
|
2431
|
+
remaining_bytes = expected_event_bytes
|
|
2432
|
+
events: list[str] = []
|
|
2433
|
+
for chunk in chunks:
|
|
2434
|
+
if type(chunk.event_count) is not int or not isinstance(chunk.payload, bytes):
|
|
2435
|
+
return []
|
|
2436
|
+
if chunk.first_sequence_number != expected_sequence:
|
|
2437
|
+
return []
|
|
2438
|
+
total_event_count += chunk.event_count
|
|
2439
|
+
if total_event_count > DURABLE_BRIDGE_TRANSCRIPT_MAX_EVENTS:
|
|
2440
|
+
return []
|
|
2441
|
+
try:
|
|
2442
|
+
decoded = decode_durable_bridge_transcript_chunk(
|
|
2443
|
+
codec=chunk.codec,
|
|
2444
|
+
payload=bytes(chunk.payload),
|
|
2445
|
+
event_count=chunk.event_count,
|
|
2446
|
+
uncompressed_bytes=chunk.uncompressed_bytes,
|
|
2447
|
+
payload_sha256=chunk.payload_sha256,
|
|
2448
|
+
max_uncompressed_bytes=remaining_bytes + 4 * chunk.event_count,
|
|
2449
|
+
)
|
|
2450
|
+
except DurableBridgeTranscriptDecodeError:
|
|
2451
|
+
return []
|
|
2452
|
+
decoded_bytes = sum(len(event.encode("utf-8")) for event in decoded)
|
|
2453
|
+
if decoded_bytes > remaining_bytes:
|
|
2454
|
+
return []
|
|
2455
|
+
events.extend(decoded)
|
|
2456
|
+
remaining_bytes -= decoded_bytes
|
|
2457
|
+
expected_sequence += chunk.event_count
|
|
2458
|
+
return events if remaining_bytes == 0 else []
|
|
2459
|
+
|
|
2460
|
+
async def get_operation_by_response_id(self, *, response_id: str) -> DurableBridgeOperationSnapshot | None:
|
|
2461
|
+
operation = await self._session.scalar(
|
|
2462
|
+
select(HttpBridgeOperationRecord).where(
|
|
2463
|
+
HttpBridgeOperationRecord.response_id == response_id,
|
|
2464
|
+
HttpBridgeOperationRecord.state.in_(("completed", "incomplete")),
|
|
2465
|
+
)
|
|
2466
|
+
)
|
|
2467
|
+
return _to_operation_snapshot(operation) if operation is not None else None
|
|
2468
|
+
|
|
2469
|
+
async def get_replayable_transcript(
|
|
2470
|
+
self,
|
|
2471
|
+
*,
|
|
2472
|
+
response_id: str,
|
|
2473
|
+
max_turns: int = 128,
|
|
2474
|
+
max_bytes: int = 8 * 1024 * 1024,
|
|
2475
|
+
) -> list[DurableBridgeTranscriptTurn] | None:
|
|
2476
|
+
"""Return a complete parent-response chain, newest turn last.
|
|
2477
|
+
|
|
2478
|
+
Missing request bodies, truncated event spools, or a broken parent
|
|
2479
|
+
chain make the transcript ineligible for reconstruction.
|
|
2480
|
+
"""
|
|
2481
|
+
turns: list[DurableBridgeTranscriptTurn] = []
|
|
2482
|
+
visited: set[str] = set()
|
|
2483
|
+
total_bytes = 0
|
|
2484
|
+
current_response_id: str | None = response_id
|
|
2485
|
+
while current_response_id is not None:
|
|
2486
|
+
if current_response_id in visited or len(turns) >= max_turns:
|
|
2487
|
+
return None
|
|
2488
|
+
visited.add(current_response_id)
|
|
2489
|
+
operation = await self.get_operation_by_response_id(response_id=current_response_id)
|
|
2490
|
+
if operation is None or operation.request_text is None or not operation.event_spool_complete:
|
|
2491
|
+
return None
|
|
2492
|
+
remaining_event_bytes = max_bytes - total_bytes - len(operation.request_text.encode("utf-8"))
|
|
2493
|
+
if remaining_event_bytes < 0:
|
|
2494
|
+
return None
|
|
2495
|
+
events = await self.get_operation_events(
|
|
2496
|
+
operation_id=operation.operation_id,
|
|
2497
|
+
max_bytes=remaining_event_bytes,
|
|
2498
|
+
)
|
|
2499
|
+
if not events or not any(
|
|
2500
|
+
"response.completed" in event or "response.incomplete" in event for event in events
|
|
2501
|
+
):
|
|
2502
|
+
return None
|
|
2503
|
+
turn_bytes = len(operation.request_text.encode("utf-8")) + sum(
|
|
2504
|
+
len(event.encode("utf-8")) for event in events
|
|
2505
|
+
)
|
|
2506
|
+
total_bytes += turn_bytes
|
|
2507
|
+
if total_bytes > max_bytes:
|
|
2508
|
+
return None
|
|
2509
|
+
turns.append(DurableBridgeTranscriptTurn(operation=operation, events=tuple(events)))
|
|
2510
|
+
current_response_id = operation.parent_response_id
|
|
2511
|
+
turns.reverse()
|
|
2512
|
+
return turns
|
|
2513
|
+
|
|
2514
|
+
async def purge_operation_spool_batch(
|
|
2515
|
+
self,
|
|
2516
|
+
*,
|
|
2517
|
+
cutoff: datetime,
|
|
2518
|
+
batch_size: int = DURABLE_BRIDGE_OPERATION_SPOOL_PURGE_BATCH_SIZE,
|
|
2519
|
+
) -> DurableBridgeOperationPurgeBatchResult:
|
|
2520
|
+
"""Delete eligible transcript material past retention.
|
|
2521
|
+
|
|
2522
|
+
Nonterminal rows are purgeable only after their owning session is
|
|
2523
|
+
ownerless or its lease has expired. Recheck that predicate in the
|
|
2524
|
+
delete transaction so an in-flight operation cannot lose its
|
|
2525
|
+
duplicate-suppression ledger between selection and deletion.
|
|
2526
|
+
"""
|
|
2527
|
+
terminal_states = ("completed", "incomplete", "failed", "abandoned")
|
|
2528
|
+
# UNKNOWN is an ambiguous, still-live operation while its owner lease
|
|
2529
|
+
# is active. Treat it like the other nonterminal states so retention
|
|
2530
|
+
# cannot delete the duplicate-suppression fence during a long-running
|
|
2531
|
+
# server-indefinite recovery attempt.
|
|
2532
|
+
nonterminal_states = ("submitted", "acknowledged", "unknown")
|
|
2533
|
+
stale_owner = or_(
|
|
2534
|
+
HttpBridgeSessionRecord.owner_instance_id.is_(None),
|
|
2535
|
+
HttpBridgeSessionRecord.lease_expires_at.is_(None),
|
|
2536
|
+
HttpBridgeSessionRecord.lease_expires_at < utcnow(),
|
|
2537
|
+
)
|
|
2538
|
+
stale_nonterminal = and_(
|
|
2539
|
+
HttpBridgeOperationRecord.state.in_(nonterminal_states),
|
|
2540
|
+
exists(
|
|
2541
|
+
select(HttpBridgeSessionRecord.id)
|
|
2542
|
+
.where(
|
|
2543
|
+
HttpBridgeSessionRecord.id == HttpBridgeOperationRecord.session_id,
|
|
2544
|
+
stale_owner,
|
|
2545
|
+
)
|
|
2546
|
+
.correlate(HttpBridgeOperationRecord)
|
|
2547
|
+
),
|
|
2548
|
+
)
|
|
2549
|
+
purgeable = or_(HttpBridgeOperationRecord.state.in_(terminal_states), stale_nonterminal)
|
|
2550
|
+
async with sqlite_writer_section():
|
|
2551
|
+
selected = await self._session.execute(
|
|
2552
|
+
select(HttpBridgeOperationRecord)
|
|
2553
|
+
.join(
|
|
2554
|
+
HttpBridgeSessionRecord,
|
|
2555
|
+
HttpBridgeSessionRecord.id == HttpBridgeOperationRecord.session_id,
|
|
2556
|
+
)
|
|
2557
|
+
.where(HttpBridgeOperationRecord.updated_at < cutoff, purgeable)
|
|
2558
|
+
.order_by(HttpBridgeOperationRecord.updated_at.asc())
|
|
2559
|
+
.limit(batch_size)
|
|
2560
|
+
.with_for_update()
|
|
2561
|
+
)
|
|
2562
|
+
# The joined FOR UPDATE locks both the operation and owning
|
|
2563
|
+
# session on PostgreSQL, serializing retention deletion with
|
|
2564
|
+
# claim_session() on the same continuity row.
|
|
2565
|
+
operation_ids = [str(operation.operation_id) for operation in selected.scalars().all()]
|
|
2566
|
+
if not operation_ids:
|
|
2567
|
+
await self._session.commit()
|
|
2568
|
+
return DurableBridgeOperationPurgeBatchResult(
|
|
2569
|
+
selected_operations=0,
|
|
2570
|
+
deleted_operations=0,
|
|
2571
|
+
)
|
|
2572
|
+
deleted = await self._session.execute(
|
|
2573
|
+
delete(HttpBridgeOperationRecord)
|
|
2574
|
+
.where(
|
|
2575
|
+
HttpBridgeOperationRecord.operation_id.in_(operation_ids),
|
|
2576
|
+
HttpBridgeOperationRecord.updated_at < cutoff,
|
|
2577
|
+
purgeable,
|
|
2578
|
+
)
|
|
2579
|
+
.returning(HttpBridgeOperationRecord.operation_id)
|
|
2580
|
+
)
|
|
2581
|
+
deleted_ids = [str(value) for value in deleted.scalars().all()]
|
|
2582
|
+
if deleted_ids:
|
|
2583
|
+
await self._delete_operation_spool_material(deleted_ids)
|
|
2584
|
+
await self._session.commit()
|
|
2585
|
+
return DurableBridgeOperationPurgeBatchResult(
|
|
2586
|
+
selected_operations=len(operation_ids),
|
|
2587
|
+
deleted_operations=len(deleted_ids),
|
|
2588
|
+
)
|
|
2589
|
+
|
|
2590
|
+
async def purge_operation_spool(
|
|
2591
|
+
self,
|
|
2592
|
+
*,
|
|
2593
|
+
cutoff: datetime,
|
|
2594
|
+
batch_size: int = DURABLE_BRIDGE_OPERATION_SPOOL_PURGE_BATCH_SIZE,
|
|
2595
|
+
) -> int:
|
|
2596
|
+
"""Delete one eligible transcript batch and return actual deletes."""
|
|
2597
|
+
result = await self.purge_operation_spool_batch(cutoff=cutoff, batch_size=batch_size)
|
|
2598
|
+
return result.deleted_operations
|
|
2599
|
+
|
|
2600
|
+
async def append_operation_event_chunk(
|
|
2601
|
+
self,
|
|
2602
|
+
*,
|
|
2603
|
+
events: Sequence[DurableBridgeOperationEventInput],
|
|
2604
|
+
max_bytes: int,
|
|
2605
|
+
) -> bool:
|
|
2606
|
+
"""Append one compressed v2 chunk under the durable owner fence."""
|
|
2607
|
+
if not events:
|
|
2608
|
+
return True
|
|
2609
|
+
first = events[0]
|
|
2610
|
+
if any(
|
|
2611
|
+
event.operation_id != first.operation_id
|
|
2612
|
+
or event.session_id != first.session_id
|
|
2613
|
+
or event.instance_id != first.instance_id
|
|
2614
|
+
or event.owner_epoch != first.owner_epoch
|
|
2615
|
+
for event in events
|
|
2616
|
+
):
|
|
2617
|
+
return False
|
|
2618
|
+
event_texts = [event.event_text for event in events]
|
|
2619
|
+
event_bytes = sum(len(event_text.encode("utf-8")) for event_text in event_texts)
|
|
2620
|
+
if event_bytes > max_bytes:
|
|
2621
|
+
return False
|
|
2622
|
+
if not await self._chunk_append_preflight_allows_encoding(
|
|
2623
|
+
operation_id=first.operation_id,
|
|
2624
|
+
session_id=first.session_id,
|
|
2625
|
+
instance_id=first.instance_id,
|
|
2626
|
+
owner_epoch=first.owner_epoch,
|
|
2627
|
+
event_bytes=event_bytes,
|
|
2628
|
+
event_count=len(event_texts),
|
|
2629
|
+
max_bytes=max_bytes,
|
|
2630
|
+
):
|
|
2631
|
+
return False
|
|
2632
|
+
try:
|
|
2633
|
+
encoded = await asyncio.to_thread(encode_durable_bridge_transcript_chunk, event_texts)
|
|
2634
|
+
except ValueError:
|
|
2635
|
+
return False
|
|
2636
|
+
async with sqlite_writer_section():
|
|
2637
|
+
locked_operation = await self._lock_operation_for_chunk_append(
|
|
2638
|
+
operation_id=first.operation_id,
|
|
2639
|
+
session_id=first.session_id,
|
|
2640
|
+
instance_id=first.instance_id,
|
|
2641
|
+
owner_epoch=first.owner_epoch,
|
|
2642
|
+
)
|
|
2643
|
+
if locked_operation is None:
|
|
2644
|
+
return False
|
|
2645
|
+
operation, append_allowed = locked_operation
|
|
2646
|
+
if not append_allowed:
|
|
2647
|
+
await self._session.rollback()
|
|
2648
|
+
return False
|
|
2649
|
+
if int(operation.event_bytes or 0) + event_bytes > max_bytes:
|
|
2650
|
+
operation.event_spool_complete = False
|
|
2651
|
+
await self._session.commit()
|
|
2652
|
+
return False
|
|
2653
|
+
first_sequence_number = await self._next_operation_chunk_sequence(first.operation_id)
|
|
2654
|
+
if first_sequence_number - 1 + len(event_texts) > DURABLE_BRIDGE_TRANSCRIPT_MAX_EVENTS:
|
|
2655
|
+
operation.event_spool_complete = False
|
|
2656
|
+
await self._session.commit()
|
|
2657
|
+
return False
|
|
2658
|
+
self._session.add(
|
|
2659
|
+
HttpBridgeOperationEventChunk(
|
|
2660
|
+
operation_id=first.operation_id,
|
|
2661
|
+
first_sequence_number=first_sequence_number,
|
|
2662
|
+
event_count=encoded.event_count,
|
|
2663
|
+
codec=encoded.codec,
|
|
2664
|
+
uncompressed_bytes=encoded.uncompressed_bytes,
|
|
2665
|
+
payload=encoded.payload,
|
|
2666
|
+
payload_sha256=encoded.payload_sha256,
|
|
2667
|
+
)
|
|
2668
|
+
)
|
|
2669
|
+
operation.spool_format = HTTP_BRIDGE_SPOOL_FORMAT_CHUNKS_V2
|
|
2670
|
+
operation.event_bytes = int(operation.event_bytes or 0) + event_bytes
|
|
2671
|
+
await self._session.commit()
|
|
2672
|
+
return True
|
|
2673
|
+
|
|
2674
|
+
async def append_terminal_operation_chunk(
|
|
2675
|
+
self,
|
|
2676
|
+
*,
|
|
2677
|
+
operation_id: str,
|
|
2678
|
+
session_id: str,
|
|
2679
|
+
instance_id: str,
|
|
2680
|
+
owner_epoch: int,
|
|
2681
|
+
event_text: str,
|
|
2682
|
+
max_bytes: int,
|
|
2683
|
+
state: str,
|
|
2684
|
+
expected_recovery_dispatch_count: int | None = None,
|
|
2685
|
+
response_id: str | None = None,
|
|
2686
|
+
complete_spool: bool = True,
|
|
2687
|
+
) -> bool:
|
|
2688
|
+
"""Append a terminal v2 chunk and expose its outcome atomically."""
|
|
2689
|
+
event_bytes = len(event_text.encode("utf-8"))
|
|
2690
|
+
preflight_allows_encoding = await self._chunk_append_preflight_allows_encoding(
|
|
2691
|
+
operation_id=operation_id,
|
|
2692
|
+
session_id=session_id,
|
|
2693
|
+
instance_id=instance_id,
|
|
2694
|
+
owner_epoch=owner_epoch,
|
|
2695
|
+
event_bytes=event_bytes,
|
|
2696
|
+
event_count=1,
|
|
2697
|
+
max_bytes=max_bytes,
|
|
2698
|
+
)
|
|
2699
|
+
try:
|
|
2700
|
+
encoded = (
|
|
2701
|
+
await asyncio.to_thread(encode_durable_bridge_transcript_chunk, (event_text,))
|
|
2702
|
+
if preflight_allows_encoding
|
|
2703
|
+
else None
|
|
2704
|
+
)
|
|
2705
|
+
except ValueError:
|
|
2706
|
+
encoded = None
|
|
2707
|
+
async with sqlite_writer_section():
|
|
2708
|
+
locked_operation = await self._lock_operation_for_chunk_append(
|
|
2709
|
+
operation_id=operation_id,
|
|
2710
|
+
session_id=session_id,
|
|
2711
|
+
instance_id=instance_id,
|
|
2712
|
+
owner_epoch=owner_epoch,
|
|
2713
|
+
expected_recovery_dispatch_count=expected_recovery_dispatch_count,
|
|
2714
|
+
)
|
|
2715
|
+
if locked_operation is None:
|
|
2716
|
+
return False
|
|
2717
|
+
operation, append_allowed = locked_operation
|
|
2718
|
+
if not append_allowed:
|
|
2719
|
+
if operation.terminal_append_phase in _HTTP_BRIDGE_TERMINAL_APPEND_RECORDED_PHASES:
|
|
2720
|
+
# A terminal outcome is already recorded for this dispatch:
|
|
2721
|
+
# keep it verbatim instead of restamping ``state`` from a
|
|
2722
|
+
# duplicate or late terminal write.
|
|
2723
|
+
await self._session.rollback()
|
|
2724
|
+
return False
|
|
2725
|
+
operation.event_spool_complete = False
|
|
2726
|
+
operation.state = state
|
|
2727
|
+
if response_id is not None:
|
|
2728
|
+
operation.response_id = response_id
|
|
2729
|
+
operation.updated_at = utcnow()
|
|
2730
|
+
await self._session.commit()
|
|
2731
|
+
return False
|
|
2732
|
+
if int(operation.event_bytes or 0) + event_bytes > max_bytes or encoded is None:
|
|
2733
|
+
operation.event_spool_complete = False
|
|
2734
|
+
operation.state = state
|
|
2735
|
+
if response_id is not None:
|
|
2736
|
+
operation.response_id = response_id
|
|
2737
|
+
operation.updated_at = utcnow()
|
|
2738
|
+
await self._session.commit()
|
|
2739
|
+
return False
|
|
2740
|
+
first_sequence_number = await self._next_operation_chunk_sequence(operation_id)
|
|
2741
|
+
if first_sequence_number > DURABLE_BRIDGE_TRANSCRIPT_MAX_EVENTS:
|
|
2742
|
+
operation.event_spool_complete = False
|
|
2743
|
+
operation.state = state
|
|
2744
|
+
if response_id is not None:
|
|
2745
|
+
operation.response_id = response_id
|
|
2746
|
+
operation.updated_at = utcnow()
|
|
2747
|
+
await self._session.commit()
|
|
2748
|
+
return False
|
|
2749
|
+
self._session.add(
|
|
2750
|
+
HttpBridgeOperationEventChunk(
|
|
2751
|
+
operation_id=operation_id,
|
|
2752
|
+
first_sequence_number=first_sequence_number,
|
|
2753
|
+
event_count=encoded.event_count,
|
|
2754
|
+
codec=encoded.codec,
|
|
2755
|
+
uncompressed_bytes=encoded.uncompressed_bytes,
|
|
2756
|
+
payload=encoded.payload,
|
|
2757
|
+
payload_sha256=encoded.payload_sha256,
|
|
2758
|
+
)
|
|
2759
|
+
)
|
|
2760
|
+
operation.spool_format = HTTP_BRIDGE_SPOOL_FORMAT_CHUNKS_V2
|
|
2761
|
+
operation.event_bytes = int(operation.event_bytes or 0) + event_bytes
|
|
2762
|
+
operation.state = state
|
|
2763
|
+
if response_id is not None:
|
|
2764
|
+
operation.response_id = response_id
|
|
2765
|
+
operation.event_spool_complete = complete_spool
|
|
2766
|
+
# The terminal transcript block is committed with the outcome.
|
|
2767
|
+
# Recording the phase here is what makes a duplicate or late
|
|
2768
|
+
# terminal write observe a finished dispatch and refuse, while a
|
|
2769
|
+
# deferred ``complete_spool=False`` finalization is still allowed.
|
|
2770
|
+
operation.terminal_append_phase = HTTP_BRIDGE_TERMINAL_APPEND_PHASE_APPENDED
|
|
2771
|
+
operation.updated_at = utcnow()
|
|
2772
|
+
await self._session.commit()
|
|
2773
|
+
return True
|
|
2774
|
+
|
|
2775
|
+
async def append_operation_event(
|
|
2776
|
+
self,
|
|
2777
|
+
*,
|
|
2778
|
+
operation_id: str,
|
|
2779
|
+
session_id: str,
|
|
2780
|
+
instance_id: str,
|
|
2781
|
+
owner_epoch: int,
|
|
2782
|
+
event_text: str,
|
|
2783
|
+
max_bytes: int,
|
|
2784
|
+
) -> bool:
|
|
2785
|
+
"""Append one replayable SSE block under the durable owner fence."""
|
|
2786
|
+
async with sqlite_writer_section():
|
|
2787
|
+
owner_exists = await self._session.scalar(
|
|
2788
|
+
select(HttpBridgeSessionRecord.id)
|
|
2789
|
+
.where(
|
|
2790
|
+
HttpBridgeSessionRecord.id == session_id,
|
|
2791
|
+
HttpBridgeSessionRecord.owner_instance_id == instance_id,
|
|
2792
|
+
HttpBridgeSessionRecord.owner_epoch == owner_epoch,
|
|
2793
|
+
)
|
|
2794
|
+
.with_for_update()
|
|
2795
|
+
)
|
|
2796
|
+
operation = await self._session.scalar(
|
|
2797
|
+
select(HttpBridgeOperationRecord)
|
|
2798
|
+
.where(
|
|
2799
|
+
HttpBridgeOperationRecord.operation_id == operation_id,
|
|
2800
|
+
HttpBridgeOperationRecord.session_id == session_id,
|
|
2801
|
+
HttpBridgeOperationRecord.spool_format == HTTP_BRIDGE_SPOOL_FORMAT_ROWS_V1,
|
|
2802
|
+
)
|
|
2803
|
+
.with_for_update()
|
|
2804
|
+
)
|
|
2805
|
+
if owner_exists is None or operation is None or operation.state == "abandoned":
|
|
2806
|
+
await self._session.rollback()
|
|
2807
|
+
return False
|
|
2808
|
+
event_size = len(event_text.encode("utf-8"))
|
|
2809
|
+
if event_size > max_bytes or int(operation.event_bytes or 0) + event_size > max_bytes:
|
|
2810
|
+
operation.event_spool_complete = False
|
|
2811
|
+
await self._session.commit()
|
|
2812
|
+
return False
|
|
2813
|
+
next_sequence = await self._session.scalar(
|
|
2814
|
+
select(func.coalesce(func.max(HttpBridgeOperationEvent.sequence_number), 0) + 1).where(
|
|
2815
|
+
HttpBridgeOperationEvent.operation_id == operation_id,
|
|
2816
|
+
)
|
|
2817
|
+
)
|
|
2818
|
+
sequence = int(next_sequence or 1)
|
|
2819
|
+
self._session.add(
|
|
2820
|
+
HttpBridgeOperationEvent(
|
|
2821
|
+
operation_id=operation_id,
|
|
2822
|
+
sequence_number=sequence,
|
|
2823
|
+
# Include occurrence position so identical downstream
|
|
2824
|
+
# blocks remain distinct in replay transcripts.
|
|
2825
|
+
event_fingerprint=durable_bridge_hash(f"{sequence}:{event_text}"),
|
|
2826
|
+
event_text=event_text,
|
|
2827
|
+
)
|
|
2828
|
+
)
|
|
2829
|
+
operation.event_bytes = int(operation.event_bytes or 0) + event_size
|
|
2830
|
+
await self._session.commit()
|
|
2831
|
+
return True
|
|
2832
|
+
|
|
2833
|
+
async def append_terminal_operation_event(
|
|
2834
|
+
self,
|
|
2835
|
+
*,
|
|
2836
|
+
operation_id: str,
|
|
2837
|
+
session_id: str,
|
|
2838
|
+
instance_id: str,
|
|
2839
|
+
owner_epoch: int,
|
|
2840
|
+
event_text: str,
|
|
2841
|
+
max_bytes: int,
|
|
2842
|
+
state: str,
|
|
2843
|
+
expected_recovery_dispatch_count: int | None = None,
|
|
2844
|
+
response_id: str | None = None,
|
|
2845
|
+
complete_spool: bool = True,
|
|
2846
|
+
) -> bool:
|
|
2847
|
+
"""Append a terminal event and expose its operation state atomically."""
|
|
2848
|
+
async with sqlite_writer_section():
|
|
2849
|
+
owner_exists = await self._session.scalar(
|
|
2850
|
+
select(HttpBridgeSessionRecord.id)
|
|
2851
|
+
.where(
|
|
2852
|
+
HttpBridgeSessionRecord.id == session_id,
|
|
2853
|
+
HttpBridgeSessionRecord.owner_instance_id == instance_id,
|
|
2854
|
+
HttpBridgeSessionRecord.owner_epoch == owner_epoch,
|
|
2855
|
+
)
|
|
2856
|
+
.with_for_update()
|
|
2857
|
+
)
|
|
2858
|
+
# ``expected_recovery_dispatch_count`` is opt-in: a caller that
|
|
2859
|
+
# claimed a recovery dispatch pins the generation it observed, and
|
|
2860
|
+
# a caller that never claimed one passes nothing rather than a
|
|
2861
|
+
# literal 0, so an operation retained from an older release with a
|
|
2862
|
+
# non-zero counter still settles.
|
|
2863
|
+
terminal_event_statement = select(HttpBridgeOperationRecord).where(
|
|
2864
|
+
HttpBridgeOperationRecord.operation_id == operation_id,
|
|
2865
|
+
HttpBridgeOperationRecord.session_id == session_id,
|
|
2866
|
+
HttpBridgeOperationRecord.spool_format == HTTP_BRIDGE_SPOOL_FORMAT_ROWS_V1,
|
|
2867
|
+
)
|
|
2868
|
+
if expected_recovery_dispatch_count is not None:
|
|
2869
|
+
terminal_event_statement = terminal_event_statement.where(
|
|
2870
|
+
HttpBridgeOperationRecord.recovery_dispatch_count == expected_recovery_dispatch_count
|
|
2871
|
+
)
|
|
2872
|
+
operation = await self._session.scalar(terminal_event_statement.with_for_update())
|
|
2873
|
+
if owner_exists is None or operation is None or operation.state == "abandoned":
|
|
2874
|
+
await self._session.rollback()
|
|
2875
|
+
return False
|
|
2876
|
+
# Scoped to rows whose terminal outcome is already recorded: an
|
|
2877
|
+
# ordinary terminal append always observes its own row as
|
|
2878
|
+
# ``state = completed`` with an incomplete spool, because the relay
|
|
2879
|
+
# updates the operation state before appending.
|
|
2880
|
+
if operation.terminal_append_phase in _HTTP_BRIDGE_TERMINAL_APPEND_RECORDED_PHASES:
|
|
2881
|
+
await self._session.rollback()
|
|
2882
|
+
return False
|
|
2883
|
+
event_size = len(event_text.encode("utf-8"))
|
|
2884
|
+
persisted = event_size <= max_bytes and int(operation.event_bytes or 0) + event_size <= max_bytes
|
|
2885
|
+
if persisted:
|
|
2886
|
+
next_sequence = await self._session.scalar(
|
|
2887
|
+
select(func.coalesce(func.max(HttpBridgeOperationEvent.sequence_number), 0) + 1).where(
|
|
2888
|
+
HttpBridgeOperationEvent.operation_id == operation_id,
|
|
2889
|
+
)
|
|
2890
|
+
)
|
|
2891
|
+
sequence = int(next_sequence or 1)
|
|
2892
|
+
self._session.add(
|
|
2893
|
+
HttpBridgeOperationEvent(
|
|
2894
|
+
operation_id=operation_id,
|
|
2895
|
+
sequence_number=sequence,
|
|
2896
|
+
event_fingerprint=durable_bridge_hash(f"{sequence}:{event_text}"),
|
|
2897
|
+
event_text=event_text,
|
|
2898
|
+
)
|
|
2899
|
+
)
|
|
2900
|
+
operation.event_bytes = int(operation.event_bytes or 0) + event_size
|
|
2901
|
+
else:
|
|
2902
|
+
operation.event_spool_complete = False
|
|
2903
|
+
# The terminal outcome is still authoritative even when the
|
|
2904
|
+
# transcript block cannot fit in the bounded spool. Expose
|
|
2905
|
+
# the failed state so an identical retry does not remain
|
|
2906
|
+
# fenced as an in-flight operation until retention expires.
|
|
2907
|
+
operation.state = state
|
|
2908
|
+
if response_id is not None:
|
|
2909
|
+
operation.response_id = response_id
|
|
2910
|
+
operation.updated_at = utcnow()
|
|
2911
|
+
await self._session.commit()
|
|
2912
|
+
return False
|
|
2913
|
+
operation.state = state
|
|
2914
|
+
if response_id is not None:
|
|
2915
|
+
operation.response_id = response_id
|
|
2916
|
+
operation.event_spool_complete = complete_spool
|
|
2917
|
+
# Reached only when the terminal block was actually spooled (the
|
|
2918
|
+
# over-cap branch above returns early), so the dispatch's terminal
|
|
2919
|
+
# transcript outcome is now recorded.
|
|
2920
|
+
operation.terminal_append_phase = HTTP_BRIDGE_TERMINAL_APPEND_PHASE_APPENDED
|
|
2921
|
+
operation.updated_at = utcnow()
|
|
2922
|
+
await self._session.commit()
|
|
2923
|
+
return persisted
|
|
2924
|
+
|
|
2925
|
+
async def append_operation_events(
|
|
2926
|
+
self,
|
|
2927
|
+
*,
|
|
2928
|
+
events: Sequence[DurableBridgeOperationEventInput],
|
|
2929
|
+
max_bytes: int,
|
|
2930
|
+
) -> bool:
|
|
2931
|
+
"""Append a batch of SSE blocks with one fenced transaction."""
|
|
2932
|
+
if not events:
|
|
2933
|
+
return True
|
|
2934
|
+
first = events[0]
|
|
2935
|
+
if any(
|
|
2936
|
+
event.operation_id != first.operation_id
|
|
2937
|
+
or event.session_id != first.session_id
|
|
2938
|
+
or event.instance_id != first.instance_id
|
|
2939
|
+
or event.owner_epoch != first.owner_epoch
|
|
2940
|
+
for event in events
|
|
2941
|
+
):
|
|
2942
|
+
return False
|
|
2943
|
+
async with sqlite_writer_section():
|
|
2944
|
+
owner_exists = await self._session.scalar(
|
|
2945
|
+
select(HttpBridgeSessionRecord.id)
|
|
2946
|
+
.where(
|
|
2947
|
+
HttpBridgeSessionRecord.id == first.session_id,
|
|
2948
|
+
HttpBridgeSessionRecord.owner_instance_id == first.instance_id,
|
|
2949
|
+
HttpBridgeSessionRecord.owner_epoch == first.owner_epoch,
|
|
2950
|
+
)
|
|
2951
|
+
.with_for_update()
|
|
2952
|
+
)
|
|
2953
|
+
operation = await self._session.scalar(
|
|
2954
|
+
select(HttpBridgeOperationRecord)
|
|
2955
|
+
.where(
|
|
2956
|
+
HttpBridgeOperationRecord.operation_id == first.operation_id,
|
|
2957
|
+
HttpBridgeOperationRecord.session_id == first.session_id,
|
|
2958
|
+
HttpBridgeOperationRecord.spool_format == HTTP_BRIDGE_SPOOL_FORMAT_ROWS_V1,
|
|
2959
|
+
)
|
|
2960
|
+
.with_for_update()
|
|
2961
|
+
)
|
|
2962
|
+
if owner_exists is None or operation is None or operation.state == "abandoned":
|
|
2963
|
+
await self._session.rollback()
|
|
2964
|
+
return False
|
|
2965
|
+
next_sequence = await self._session.scalar(
|
|
2966
|
+
select(func.coalesce(func.max(HttpBridgeOperationEvent.sequence_number), 0) + 1).where(
|
|
2967
|
+
HttpBridgeOperationEvent.operation_id == first.operation_id,
|
|
2968
|
+
)
|
|
2969
|
+
)
|
|
2970
|
+
sequence = int(next_sequence or 1)
|
|
2971
|
+
pending: list[tuple[str, int, str, int]] = []
|
|
2972
|
+
total_bytes = int(operation.event_bytes or 0)
|
|
2973
|
+
for event in events:
|
|
2974
|
+
event_size = len(event.event_text.encode("utf-8"))
|
|
2975
|
+
if total_bytes + event_size > max_bytes:
|
|
2976
|
+
operation.event_spool_complete = False
|
|
2977
|
+
await self._session.commit()
|
|
2978
|
+
return False
|
|
2979
|
+
total_bytes += event_size
|
|
2980
|
+
pending.append(
|
|
2981
|
+
(
|
|
2982
|
+
event.event_text,
|
|
2983
|
+
sequence,
|
|
2984
|
+
durable_bridge_hash(f"{sequence}:{event.event_text}"),
|
|
2985
|
+
event_size,
|
|
2986
|
+
)
|
|
2987
|
+
)
|
|
2988
|
+
sequence += 1
|
|
2989
|
+
if pending:
|
|
2990
|
+
for event_text, sequence_number, fingerprint, event_size in pending:
|
|
2991
|
+
self._session.add(
|
|
2992
|
+
HttpBridgeOperationEvent(
|
|
2993
|
+
operation_id=first.operation_id,
|
|
2994
|
+
sequence_number=sequence_number,
|
|
2995
|
+
event_fingerprint=fingerprint,
|
|
2996
|
+
event_text=event_text,
|
|
2997
|
+
)
|
|
2998
|
+
)
|
|
2999
|
+
operation.event_bytes = total_bytes
|
|
3000
|
+
await self._session.commit()
|
|
3001
|
+
return True
|
|
3002
|
+
|
|
3003
|
+
async def finalize_operation_event_spool(
|
|
3004
|
+
self,
|
|
3005
|
+
*,
|
|
3006
|
+
operation_id: str,
|
|
3007
|
+
session_id: str,
|
|
3008
|
+
instance_id: str,
|
|
3009
|
+
owner_epoch: int,
|
|
3010
|
+
expected_state: str | None = None,
|
|
3011
|
+
) -> bool:
|
|
3012
|
+
"""Mark a terminal operation replay-complete after its queue drained."""
|
|
3013
|
+
if expected_state is not None and expected_state not in {"completed", "incomplete", "failed"}:
|
|
3014
|
+
return False
|
|
3015
|
+
async with sqlite_writer_section():
|
|
3016
|
+
owner_exists = await self._session.scalar(
|
|
3017
|
+
select(HttpBridgeSessionRecord.id)
|
|
3018
|
+
.where(
|
|
3019
|
+
HttpBridgeSessionRecord.id == session_id,
|
|
3020
|
+
HttpBridgeSessionRecord.owner_instance_id == instance_id,
|
|
3021
|
+
HttpBridgeSessionRecord.owner_epoch == owner_epoch,
|
|
3022
|
+
)
|
|
3023
|
+
.with_for_update()
|
|
3024
|
+
)
|
|
3025
|
+
predicates = [
|
|
3026
|
+
HttpBridgeOperationRecord.operation_id == operation_id,
|
|
3027
|
+
HttpBridgeOperationRecord.session_id == session_id,
|
|
3028
|
+
HttpBridgeOperationRecord.event_spool_complete.is_(False),
|
|
3029
|
+
# A settled row was published without a confirmed append, so it
|
|
3030
|
+
# is not replayable no matter which attempt reaches finalization
|
|
3031
|
+
# (a terminal append can commit just as its bound expires, which
|
|
3032
|
+
# runs settlement and leaves a finalize in flight).
|
|
3033
|
+
HttpBridgeOperationRecord.terminal_append_phase != HTTP_BRIDGE_TERMINAL_APPEND_PHASE_SETTLED,
|
|
3034
|
+
]
|
|
3035
|
+
predicates.append(
|
|
3036
|
+
HttpBridgeOperationRecord.state == expected_state
|
|
3037
|
+
if expected_state is not None
|
|
3038
|
+
else HttpBridgeOperationRecord.state.in_(("completed", "incomplete"))
|
|
3039
|
+
)
|
|
3040
|
+
result = await self._session.execute(
|
|
3041
|
+
update(HttpBridgeOperationRecord)
|
|
3042
|
+
.where(*predicates)
|
|
3043
|
+
.values(event_spool_complete=True, updated_at=utcnow())
|
|
3044
|
+
)
|
|
3045
|
+
if owner_exists is None:
|
|
3046
|
+
await self._session.rollback()
|
|
3047
|
+
return False
|
|
3048
|
+
await self._session.commit()
|
|
3049
|
+
return bool(getattr(result, "rowcount", 0))
|
|
3050
|
+
|
|
3051
|
+
async def get_latest_completed_operation(
|
|
3052
|
+
self,
|
|
3053
|
+
*,
|
|
3054
|
+
session_id: str,
|
|
3055
|
+
parent_response_id: str,
|
|
3056
|
+
request_fingerprint: str | None = None,
|
|
3057
|
+
) -> DurableBridgeOperationSnapshot | None:
|
|
3058
|
+
predicates = [
|
|
3059
|
+
HttpBridgeOperationRecord.session_id == session_id,
|
|
3060
|
+
HttpBridgeOperationRecord.parent_response_id == parent_response_id,
|
|
3061
|
+
HttpBridgeOperationRecord.state == "completed",
|
|
3062
|
+
HttpBridgeOperationRecord.response_id.is_not(None),
|
|
3063
|
+
]
|
|
3064
|
+
if request_fingerprint is not None:
|
|
3065
|
+
predicates.append(HttpBridgeOperationRecord.request_fingerprint == request_fingerprint)
|
|
3066
|
+
operation = await self._session.scalar(
|
|
3067
|
+
select(HttpBridgeOperationRecord)
|
|
3068
|
+
.where(*predicates)
|
|
3069
|
+
.order_by(HttpBridgeOperationRecord.updated_at.desc())
|
|
3070
|
+
.limit(1)
|
|
3071
|
+
)
|
|
3072
|
+
return _to_operation_snapshot(operation) if operation is not None else None
|
|
3073
|
+
|
|
3074
|
+
async def get_latest_completed_operation_any_session(
|
|
3075
|
+
self,
|
|
3076
|
+
*,
|
|
3077
|
+
parent_response_id: str,
|
|
3078
|
+
api_key_scope: str | None = None,
|
|
3079
|
+
request_fingerprint: str | None = None,
|
|
3080
|
+
) -> DurableBridgeOperationSnapshot | None:
|
|
3081
|
+
statement = select(HttpBridgeOperationRecord)
|
|
3082
|
+
if api_key_scope is not None:
|
|
3083
|
+
statement = statement.join(
|
|
3084
|
+
HttpBridgeSessionRecord,
|
|
3085
|
+
HttpBridgeSessionRecord.id == HttpBridgeOperationRecord.session_id,
|
|
3086
|
+
).where(HttpBridgeSessionRecord.api_key_scope == api_key_scope)
|
|
3087
|
+
operation = await self._session.scalar(
|
|
3088
|
+
statement.where(
|
|
3089
|
+
HttpBridgeOperationRecord.parent_response_id == parent_response_id,
|
|
3090
|
+
HttpBridgeOperationRecord.state == "completed",
|
|
3091
|
+
HttpBridgeOperationRecord.response_id.is_not(None),
|
|
3092
|
+
*(
|
|
3093
|
+
[HttpBridgeOperationRecord.request_fingerprint == request_fingerprint]
|
|
3094
|
+
if request_fingerprint is not None
|
|
3095
|
+
else []
|
|
3096
|
+
),
|
|
3097
|
+
)
|
|
3098
|
+
.order_by(HttpBridgeOperationRecord.updated_at.desc())
|
|
3099
|
+
.limit(1)
|
|
3100
|
+
)
|
|
3101
|
+
return _to_operation_snapshot(operation) if operation is not None else None
|
|
3102
|
+
|
|
3103
|
+
async def settle_terminal_append_failure(
|
|
3104
|
+
self,
|
|
3105
|
+
*,
|
|
3106
|
+
operation_id: str,
|
|
3107
|
+
session_id: str,
|
|
3108
|
+
instance_id: str,
|
|
3109
|
+
owner_epoch: int,
|
|
3110
|
+
state: str,
|
|
3111
|
+
expected_response_id: str | None,
|
|
3112
|
+
expected_recovery_dispatch_count: int | None = None,
|
|
3113
|
+
alternate_expected_response_id: str | None = None,
|
|
3114
|
+
response_id: str | None = None,
|
|
3115
|
+
) -> bool:
|
|
3116
|
+
"""Settle only the terminal attempt whose append outcome was ambiguous."""
|
|
3117
|
+
async with sqlite_writer_section():
|
|
3118
|
+
owner_exists = await self._session.scalar(
|
|
3119
|
+
select(HttpBridgeSessionRecord.id)
|
|
3120
|
+
.where(
|
|
3121
|
+
HttpBridgeSessionRecord.id == session_id,
|
|
3122
|
+
HttpBridgeSessionRecord.owner_instance_id == instance_id,
|
|
3123
|
+
HttpBridgeSessionRecord.owner_epoch == owner_epoch,
|
|
3124
|
+
)
|
|
3125
|
+
.with_for_update()
|
|
3126
|
+
)
|
|
3127
|
+
if owner_exists is None:
|
|
3128
|
+
await self._session.rollback()
|
|
3129
|
+
return False
|
|
3130
|
+
acknowledged_response_matches = (
|
|
3131
|
+
HttpBridgeOperationRecord.response_id == expected_response_id
|
|
3132
|
+
if expected_response_id is not None
|
|
3133
|
+
else HttpBridgeOperationRecord.response_id.is_(None)
|
|
3134
|
+
)
|
|
3135
|
+
if alternate_expected_response_id is not None:
|
|
3136
|
+
acknowledged_response_matches = or_(
|
|
3137
|
+
acknowledged_response_matches,
|
|
3138
|
+
HttpBridgeOperationRecord.response_id == alternate_expected_response_id,
|
|
3139
|
+
)
|
|
3140
|
+
terminal_response_matches = (
|
|
3141
|
+
HttpBridgeOperationRecord.response_id == response_id
|
|
3142
|
+
if response_id is not None
|
|
3143
|
+
else HttpBridgeOperationRecord.response_id.is_(None)
|
|
3144
|
+
)
|
|
3145
|
+
values: dict[str, object] = {
|
|
3146
|
+
"state": state,
|
|
3147
|
+
"event_spool_complete": False,
|
|
3148
|
+
"updated_at": utcnow(),
|
|
3149
|
+
}
|
|
3150
|
+
if state in _HTTP_BRIDGE_TERMINAL_OPERATION_STATES:
|
|
3151
|
+
# A terminal outcome is now published without a confirmed
|
|
3152
|
+
# transcript append. SETTLED makes the row final for this
|
|
3153
|
+
# dispatch so the append that lost the race can neither rewrite
|
|
3154
|
+
# the state nor flip the spool back to replayable.
|
|
3155
|
+
#
|
|
3156
|
+
# Only genuinely terminal settlements set it. A continuity
|
|
3157
|
+
# failure after acknowledgement settles back to
|
|
3158
|
+
# ``acknowledged``, where the operation is still live and its
|
|
3159
|
+
# later appends must keep working.
|
|
3160
|
+
values["terminal_append_phase"] = HTTP_BRIDGE_TERMINAL_APPEND_PHASE_SETTLED
|
|
3161
|
+
if response_id is not None:
|
|
3162
|
+
values["response_id"] = response_id
|
|
3163
|
+
# Opt-in recovery-dispatch fence, as in
|
|
3164
|
+
# ``append_terminal_operation_event``. The newer-attempt rejection
|
|
3165
|
+
# below does not depend on it: a retry that reset the row to
|
|
3166
|
+
# ``submitted`` matches neither ``acknowledged`` nor the terminal
|
|
3167
|
+
# state being settled, so the update touches no row.
|
|
3168
|
+
settlement_statement = update(HttpBridgeOperationRecord).where(
|
|
3169
|
+
HttpBridgeOperationRecord.operation_id == operation_id,
|
|
3170
|
+
HttpBridgeOperationRecord.session_id == session_id,
|
|
3171
|
+
HttpBridgeOperationRecord.state != "abandoned",
|
|
3172
|
+
or_(
|
|
3173
|
+
and_(HttpBridgeOperationRecord.state == "acknowledged", acknowledged_response_matches),
|
|
3174
|
+
and_(
|
|
3175
|
+
HttpBridgeOperationRecord.state == state,
|
|
3176
|
+
or_(acknowledged_response_matches, terminal_response_matches),
|
|
3177
|
+
),
|
|
3178
|
+
),
|
|
3179
|
+
)
|
|
3180
|
+
if expected_recovery_dispatch_count is not None:
|
|
3181
|
+
settlement_statement = settlement_statement.where(
|
|
3182
|
+
HttpBridgeOperationRecord.recovery_dispatch_count == expected_recovery_dispatch_count
|
|
3183
|
+
)
|
|
3184
|
+
result = await self._session.execute(settlement_statement.values(**values))
|
|
3185
|
+
await self._session.commit()
|
|
3186
|
+
return bool(getattr(result, "rowcount", 0))
|
|
3187
|
+
|
|
3188
|
+
async def update_operation(
|
|
3189
|
+
self,
|
|
3190
|
+
*,
|
|
3191
|
+
operation_id: str,
|
|
3192
|
+
session_id: str,
|
|
3193
|
+
instance_id: str,
|
|
3194
|
+
owner_epoch: int,
|
|
3195
|
+
state: str,
|
|
3196
|
+
response_id: str | None = None,
|
|
3197
|
+
) -> bool:
|
|
3198
|
+
async with sqlite_writer_section():
|
|
3199
|
+
owner_exists = await self._session.scalar(
|
|
3200
|
+
select(HttpBridgeSessionRecord.id)
|
|
3201
|
+
.where(
|
|
3202
|
+
HttpBridgeSessionRecord.id == session_id,
|
|
3203
|
+
HttpBridgeSessionRecord.owner_instance_id == instance_id,
|
|
3204
|
+
HttpBridgeSessionRecord.owner_epoch == owner_epoch,
|
|
3205
|
+
)
|
|
3206
|
+
.with_for_update()
|
|
3207
|
+
)
|
|
3208
|
+
if owner_exists is None:
|
|
3209
|
+
await self._session.rollback()
|
|
3210
|
+
return False
|
|
3211
|
+
values: dict[str, object] = {"state": state, "updated_at": utcnow()}
|
|
3212
|
+
if response_id is not None:
|
|
3213
|
+
values["response_id"] = response_id
|
|
3214
|
+
result = await self._session.execute(
|
|
3215
|
+
update(HttpBridgeOperationRecord)
|
|
3216
|
+
.where(
|
|
3217
|
+
HttpBridgeOperationRecord.operation_id == operation_id,
|
|
3218
|
+
HttpBridgeOperationRecord.session_id == session_id,
|
|
3219
|
+
HttpBridgeOperationRecord.state != "abandoned",
|
|
3220
|
+
)
|
|
3221
|
+
.values(**values)
|
|
3222
|
+
)
|
|
3223
|
+
await self._session.commit()
|
|
3224
|
+
return bool(getattr(result, "rowcount", 0))
|
|
3225
|
+
|
|
3226
|
+
async def _execute_fenced_session_update(
|
|
3227
|
+
self,
|
|
3228
|
+
*,
|
|
3229
|
+
session_id: str,
|
|
3230
|
+
instance_id: str,
|
|
3231
|
+
owner_epoch: int,
|
|
3232
|
+
values: dict[str, object],
|
|
3233
|
+
) -> DurableBridgeSessionSnapshot | None:
|
|
3234
|
+
async with sqlite_writer_section():
|
|
3235
|
+
result = await self._session.execute(
|
|
3236
|
+
update(HttpBridgeSessionRecord)
|
|
3237
|
+
.where(
|
|
3238
|
+
HttpBridgeSessionRecord.id == session_id,
|
|
3239
|
+
HttpBridgeSessionRecord.owner_instance_id == instance_id,
|
|
3240
|
+
HttpBridgeSessionRecord.owner_epoch == owner_epoch,
|
|
3241
|
+
)
|
|
3242
|
+
.values(**values)
|
|
3243
|
+
.returning(*_SNAPSHOT_COLUMNS)
|
|
3244
|
+
)
|
|
3245
|
+
updated_row = result.one_or_none()
|
|
3246
|
+
await self._session.commit()
|
|
3247
|
+
if updated_row is not None:
|
|
3248
|
+
return _returned_row_to_snapshot(updated_row)
|
|
3249
|
+
current = await self._session.get(HttpBridgeSessionRecord, session_id, populate_existing=True)
|
|
3250
|
+
return _to_snapshot(current)
|
|
3251
|
+
|
|
3252
|
+
async def get_sessions_by_ids(
|
|
3253
|
+
self,
|
|
3254
|
+
session_ids: Sequence[str],
|
|
3255
|
+
*,
|
|
3256
|
+
chunk_size: int = _SESSION_ID_LOOKUP_CHUNK_SIZE,
|
|
3257
|
+
) -> list[DurableBridgeSessionSnapshot]:
|
|
3258
|
+
unique_ids = list(dict.fromkeys(session_ids))
|
|
3259
|
+
if not unique_ids:
|
|
3260
|
+
return []
|
|
3261
|
+
snapshots: list[DurableBridgeSessionSnapshot] = []
|
|
3262
|
+
for start in range(0, len(unique_ids), chunk_size):
|
|
3263
|
+
chunk = unique_ids[start : start + chunk_size]
|
|
3264
|
+
result = await self._session.execute(
|
|
3265
|
+
select(HttpBridgeSessionRecord).where(HttpBridgeSessionRecord.id.in_(chunk))
|
|
3266
|
+
)
|
|
3267
|
+
snapshots.extend(_to_snapshot_required(row) for row in result.scalars().all())
|
|
3268
|
+
return snapshots
|
|
3269
|
+
|
|
3270
|
+
async def mark_owner_draining(self, *, instance_id: str) -> int:
|
|
3271
|
+
result = await self._session.execute(
|
|
3272
|
+
select(HttpBridgeSessionRecord).where(
|
|
3273
|
+
HttpBridgeSessionRecord.owner_instance_id == instance_id,
|
|
3274
|
+
HttpBridgeSessionRecord.state == HttpBridgeSessionState.ACTIVE,
|
|
3275
|
+
)
|
|
3276
|
+
)
|
|
3277
|
+
rows = list(result.scalars().all())
|
|
3278
|
+
now = utcnow()
|
|
3279
|
+
for row in rows:
|
|
3280
|
+
row.state = HttpBridgeSessionState.DRAINING
|
|
3281
|
+
row.last_seen_at = now
|
|
3282
|
+
await self._commit_writer_section()
|
|
3283
|
+
return len(rows)
|
|
3284
|
+
|
|
3285
|
+
async def purge_owned_sessions_on_startup(
|
|
3286
|
+
self,
|
|
3287
|
+
*,
|
|
3288
|
+
instance_id: str,
|
|
3289
|
+
owner_process_epoch: str | None = None,
|
|
3290
|
+
ownerless_cutoff: datetime | None = None,
|
|
3291
|
+
batch_size: int = _PURGE_CLOSED_BATCH_SIZE,
|
|
3292
|
+
) -> int:
|
|
3293
|
+
"""Remove durable bridge rows left by the previous process instance.
|
|
3294
|
+
|
|
3295
|
+
Ownerless ACTIVE/DRAINING rows are preserved by default: a graceful
|
|
3296
|
+
drain release intentionally clears ownership while keeping continuity
|
|
3297
|
+
aliases reusable until the full bridge idle-retention window. Callers
|
|
3298
|
+
that already computed that retention cutoff may pass ``ownerless_cutoff``
|
|
3299
|
+
to piggyback that abandoned-row cleanup onto startup.
|
|
3300
|
+
"""
|
|
3301
|
+
|
|
3302
|
+
deleted_count = 0
|
|
3303
|
+
while True:
|
|
3304
|
+
now = utcnow()
|
|
3305
|
+
if owner_process_epoch is None:
|
|
3306
|
+
owned_restart_filter = HttpBridgeSessionRecord.owner_instance_id == instance_id
|
|
3307
|
+
else:
|
|
3308
|
+
owned_restart_filter = and_(
|
|
3309
|
+
HttpBridgeSessionRecord.owner_instance_id == instance_id,
|
|
3310
|
+
or_(
|
|
3311
|
+
HttpBridgeSessionRecord.owner_process_epoch.is_(None),
|
|
3312
|
+
HttpBridgeSessionRecord.owner_process_epoch != owner_process_epoch,
|
|
3313
|
+
),
|
|
3314
|
+
)
|
|
3315
|
+
purge_predicates = [owned_restart_filter]
|
|
3316
|
+
if ownerless_cutoff is not None:
|
|
3317
|
+
purge_predicates.append(
|
|
3318
|
+
and_(
|
|
3319
|
+
HttpBridgeSessionRecord.owner_instance_id.is_(None),
|
|
3320
|
+
HttpBridgeSessionRecord.state.in_(
|
|
3321
|
+
(HttpBridgeSessionState.ACTIVE, HttpBridgeSessionState.DRAINING),
|
|
3322
|
+
),
|
|
3323
|
+
or_(
|
|
3324
|
+
HttpBridgeSessionRecord.lease_expires_at.is_(None),
|
|
3325
|
+
HttpBridgeSessionRecord.lease_expires_at < now,
|
|
3326
|
+
),
|
|
3327
|
+
HttpBridgeSessionRecord.last_seen_at < ownerless_cutoff,
|
|
3328
|
+
)
|
|
3329
|
+
)
|
|
3330
|
+
startup_purge_filter = or_(*purge_predicates)
|
|
3331
|
+
result = await self._session.execute(
|
|
3332
|
+
select(
|
|
3333
|
+
HttpBridgeSessionRecord.id,
|
|
3334
|
+
HttpBridgeSessionRecord.session_key_kind,
|
|
3335
|
+
HttpBridgeSessionRecord.session_key_value,
|
|
3336
|
+
HttpBridgeSessionRecord.owner_instance_id,
|
|
3337
|
+
HttpBridgeSessionRecord.owner_process_epoch,
|
|
3338
|
+
HttpBridgeSessionRecord.last_seen_at,
|
|
3339
|
+
)
|
|
3340
|
+
.where(startup_purge_filter)
|
|
3341
|
+
.order_by(HttpBridgeSessionRecord.last_seen_at.asc())
|
|
3342
|
+
.limit(batch_size)
|
|
3343
|
+
)
|
|
3344
|
+
candidates = list(result.all())
|
|
3345
|
+
session_ids = [candidate.id for candidate in candidates]
|
|
3346
|
+
if not session_ids:
|
|
3347
|
+
return deleted_count
|
|
3348
|
+
# Operation rows are the durable recovery ledger. Never cascade
|
|
3349
|
+
# delete a session that still owns a retained operation, including
|
|
3350
|
+
# completed replayable transcripts; detach it so the next instance
|
|
3351
|
+
# can inspect and take over without losing continuity history.
|
|
3352
|
+
operation_session_ids = set(
|
|
3353
|
+
await self._session.scalars(
|
|
3354
|
+
select(HttpBridgeOperationRecord.session_id).where(
|
|
3355
|
+
HttpBridgeOperationRecord.session_id.in_(session_ids),
|
|
3356
|
+
)
|
|
3357
|
+
)
|
|
3358
|
+
)
|
|
3359
|
+
retained_recovery_ids = {
|
|
3360
|
+
candidate.id
|
|
3361
|
+
for candidate in candidates
|
|
3362
|
+
if candidate.id in operation_session_ids
|
|
3363
|
+
or (
|
|
3364
|
+
candidate.owner_instance_id == instance_id
|
|
3365
|
+
and getattr(candidate, "owner_process_epoch", None) == owner_process_epoch
|
|
3366
|
+
and (
|
|
3367
|
+
ownerless_cutoff is None
|
|
3368
|
+
or to_utc_naive(candidate.last_seen_at) >= to_utc_naive(ownerless_cutoff)
|
|
3369
|
+
)
|
|
3370
|
+
and is_http_bridge_account_neutral_replay(
|
|
3371
|
+
kind=candidate.session_key_kind,
|
|
3372
|
+
key=candidate.session_key_value,
|
|
3373
|
+
)
|
|
3374
|
+
)
|
|
3375
|
+
}
|
|
3376
|
+
async with sqlite_writer_section():
|
|
3377
|
+
ownerless_operation_ids = {
|
|
3378
|
+
candidate.id
|
|
3379
|
+
for candidate in candidates
|
|
3380
|
+
if candidate.id in retained_recovery_ids
|
|
3381
|
+
and candidate.id in operation_session_ids
|
|
3382
|
+
and candidate.owner_instance_id is None
|
|
3383
|
+
}
|
|
3384
|
+
if ownerless_operation_ids:
|
|
3385
|
+
# The ownerless-cutoff predicate is part of the same
|
|
3386
|
+
# startup query. Refresh retained rows so the bounded
|
|
3387
|
+
# loop cannot select them forever while their operation
|
|
3388
|
+
# transcript is awaiting normal retention cleanup.
|
|
3389
|
+
await self._session.execute(
|
|
3390
|
+
update(HttpBridgeSessionRecord)
|
|
3391
|
+
.where(
|
|
3392
|
+
HttpBridgeSessionRecord.id.in_(ownerless_operation_ids),
|
|
3393
|
+
HttpBridgeSessionRecord.owner_instance_id.is_(None),
|
|
3394
|
+
)
|
|
3395
|
+
.values(last_seen_at=now, lease_expires_at=now)
|
|
3396
|
+
)
|
|
3397
|
+
if retained_recovery_ids:
|
|
3398
|
+
# A process can die after recording a submitted
|
|
3399
|
+
# operation but before upstream acknowledges it. Once
|
|
3400
|
+
# startup has fenced and detached that owner's session,
|
|
3401
|
+
# classify those rows as UNKNOWN so the replacement can
|
|
3402
|
+
# enter the normal proof-gated recovery path.
|
|
3403
|
+
operation_retained_session_ids = retained_recovery_ids & operation_session_ids
|
|
3404
|
+
if operation_retained_session_ids:
|
|
3405
|
+
eligible_operation_sessions = set(
|
|
3406
|
+
await self._session.scalars(
|
|
3407
|
+
select(HttpBridgeSessionRecord.id)
|
|
3408
|
+
.where(
|
|
3409
|
+
HttpBridgeSessionRecord.id.in_(operation_retained_session_ids),
|
|
3410
|
+
startup_purge_filter,
|
|
3411
|
+
)
|
|
3412
|
+
.with_for_update()
|
|
3413
|
+
)
|
|
3414
|
+
)
|
|
3415
|
+
await self._session.execute(
|
|
3416
|
+
update(HttpBridgeOperationRecord)
|
|
3417
|
+
.where(
|
|
3418
|
+
HttpBridgeOperationRecord.session_id.in_(eligible_operation_sessions),
|
|
3419
|
+
HttpBridgeOperationRecord.state == "submitted",
|
|
3420
|
+
)
|
|
3421
|
+
.values(state="unknown", updated_at=now)
|
|
3422
|
+
)
|
|
3423
|
+
await self._session.execute(
|
|
3424
|
+
update(HttpBridgeSessionRecord)
|
|
3425
|
+
.where(
|
|
3426
|
+
HttpBridgeSessionRecord.id.in_(retained_recovery_ids),
|
|
3427
|
+
# Detach the rows selected as belonging to the
|
|
3428
|
+
# previous process. With an explicit new epoch,
|
|
3429
|
+
# matching the new epoch here would leave old
|
|
3430
|
+
# retained rows selected forever on every loop.
|
|
3431
|
+
startup_purge_filter,
|
|
3432
|
+
)
|
|
3433
|
+
.values(
|
|
3434
|
+
owner_instance_id=None,
|
|
3435
|
+
lease_expires_at=now,
|
|
3436
|
+
state=HttpBridgeSessionState.DRAINING,
|
|
3437
|
+
closed_at=None,
|
|
3438
|
+
)
|
|
3439
|
+
)
|
|
3440
|
+
deletable_ids = [session_id for session_id in session_ids if session_id not in retained_recovery_ids]
|
|
3441
|
+
if deletable_ids:
|
|
3442
|
+
if owner_process_epoch is None:
|
|
3443
|
+
deleted = await self._session.execute(
|
|
3444
|
+
delete(HttpBridgeSessionRecord)
|
|
3445
|
+
.where(HttpBridgeSessionRecord.id.in_(deletable_ids))
|
|
3446
|
+
.where(startup_purge_filter)
|
|
3447
|
+
.returning(HttpBridgeSessionRecord.id)
|
|
3448
|
+
)
|
|
3449
|
+
deleted_ids = list(deleted.scalars().all())
|
|
3450
|
+
else:
|
|
3451
|
+
previous_process_ids = [
|
|
3452
|
+
candidate.id for candidate in candidates if candidate.owner_instance_id == instance_id
|
|
3453
|
+
]
|
|
3454
|
+
ownerless_ids = [
|
|
3455
|
+
candidate.id
|
|
3456
|
+
for candidate in candidates
|
|
3457
|
+
if candidate.owner_instance_id is None and candidate.id not in retained_recovery_ids
|
|
3458
|
+
]
|
|
3459
|
+
retired_ids: list[str] = []
|
|
3460
|
+
if previous_process_ids:
|
|
3461
|
+
retired = await self._session.execute(
|
|
3462
|
+
update(HttpBridgeSessionRecord)
|
|
3463
|
+
.where(HttpBridgeSessionRecord.id.in_(previous_process_ids))
|
|
3464
|
+
.where(
|
|
3465
|
+
HttpBridgeSessionRecord.owner_instance_id == instance_id,
|
|
3466
|
+
or_(
|
|
3467
|
+
HttpBridgeSessionRecord.owner_process_epoch.is_(None),
|
|
3468
|
+
HttpBridgeSessionRecord.owner_process_epoch != owner_process_epoch,
|
|
3469
|
+
),
|
|
3470
|
+
)
|
|
3471
|
+
.values(
|
|
3472
|
+
owner_instance_id=None,
|
|
3473
|
+
lease_expires_at=None,
|
|
3474
|
+
state=HttpBridgeSessionState.CLOSED,
|
|
3475
|
+
closed_at=now,
|
|
3476
|
+
last_seen_at=now,
|
|
3477
|
+
latest_turn_state=None,
|
|
3478
|
+
latest_response_id=None,
|
|
3479
|
+
latest_input_item_count=None,
|
|
3480
|
+
latest_input_full_fingerprint=None,
|
|
3481
|
+
latest_pending_tool_calls_json=None,
|
|
3482
|
+
)
|
|
3483
|
+
.returning(HttpBridgeSessionRecord.id)
|
|
3484
|
+
)
|
|
3485
|
+
retired_ids = list(retired.scalars().all())
|
|
3486
|
+
deleted_ownerless_ids: list[str] = []
|
|
3487
|
+
if ownerless_ids:
|
|
3488
|
+
deleted_ownerless = await self._session.execute(
|
|
3489
|
+
delete(HttpBridgeSessionRecord)
|
|
3490
|
+
.where(HttpBridgeSessionRecord.id.in_(ownerless_ids))
|
|
3491
|
+
.where(
|
|
3492
|
+
HttpBridgeSessionRecord.owner_instance_id.is_(None),
|
|
3493
|
+
HttpBridgeSessionRecord.state.in_(
|
|
3494
|
+
(HttpBridgeSessionState.ACTIVE, HttpBridgeSessionState.DRAINING),
|
|
3495
|
+
),
|
|
3496
|
+
or_(
|
|
3497
|
+
HttpBridgeSessionRecord.lease_expires_at.is_(None),
|
|
3498
|
+
HttpBridgeSessionRecord.lease_expires_at < now,
|
|
3499
|
+
),
|
|
3500
|
+
HttpBridgeSessionRecord.last_seen_at < ownerless_cutoff
|
|
3501
|
+
if ownerless_cutoff is not None
|
|
3502
|
+
else true(),
|
|
3503
|
+
)
|
|
3504
|
+
.returning(HttpBridgeSessionRecord.id)
|
|
3505
|
+
)
|
|
3506
|
+
deleted_ownerless_ids = list(deleted_ownerless.scalars().all())
|
|
3507
|
+
deleted_ids = retired_ids + deleted_ownerless_ids
|
|
3508
|
+
else:
|
|
3509
|
+
deleted_ids = []
|
|
3510
|
+
if deleted_ids:
|
|
3511
|
+
await self._session.execute(
|
|
3512
|
+
delete(HttpBridgeSessionAlias).where(HttpBridgeSessionAlias.session_id.in_(deleted_ids))
|
|
3513
|
+
)
|
|
3514
|
+
await self._session.commit()
|
|
3515
|
+
deleted_count += len(deleted_ids)
|
|
3516
|
+
|
|
3517
|
+
async def purge_closed_before(self, cutoff: datetime, *, batch_size: int = _PURGE_CLOSED_BATCH_SIZE) -> int:
|
|
3518
|
+
deleted_count = 0
|
|
3519
|
+
while True:
|
|
3520
|
+
result = await self._session.execute(
|
|
3521
|
+
select(HttpBridgeSessionRecord.id)
|
|
3522
|
+
.where(
|
|
3523
|
+
HttpBridgeSessionRecord.state == HttpBridgeSessionState.CLOSED,
|
|
3524
|
+
HttpBridgeSessionRecord.last_seen_at < cutoff,
|
|
3525
|
+
~exists(
|
|
3526
|
+
select(HttpBridgeOperationRecord.operation_id).where(
|
|
3527
|
+
HttpBridgeOperationRecord.session_id == HttpBridgeSessionRecord.id,
|
|
3528
|
+
)
|
|
3529
|
+
),
|
|
3530
|
+
)
|
|
3531
|
+
.order_by(HttpBridgeSessionRecord.last_seen_at.asc())
|
|
3532
|
+
.limit(batch_size)
|
|
3533
|
+
)
|
|
3534
|
+
session_ids = list(result.scalars().all())
|
|
3535
|
+
if not session_ids:
|
|
3536
|
+
return deleted_count
|
|
3537
|
+
async with sqlite_writer_section():
|
|
3538
|
+
await self._session.execute(
|
|
3539
|
+
delete(HttpBridgeSessionAlias).where(
|
|
3540
|
+
HttpBridgeSessionAlias.session_id.in_(
|
|
3541
|
+
select(HttpBridgeSessionRecord.id).where(
|
|
3542
|
+
HttpBridgeSessionRecord.id.in_(session_ids),
|
|
3543
|
+
HttpBridgeSessionRecord.state == HttpBridgeSessionState.CLOSED,
|
|
3544
|
+
HttpBridgeSessionRecord.last_seen_at < cutoff,
|
|
3545
|
+
~exists(
|
|
3546
|
+
select(HttpBridgeOperationRecord.operation_id).where(
|
|
3547
|
+
HttpBridgeOperationRecord.session_id == HttpBridgeSessionRecord.id,
|
|
3548
|
+
)
|
|
3549
|
+
),
|
|
3550
|
+
)
|
|
3551
|
+
)
|
|
3552
|
+
)
|
|
3553
|
+
)
|
|
3554
|
+
deleted = await self._session.execute(
|
|
3555
|
+
delete(HttpBridgeSessionRecord)
|
|
3556
|
+
.where(HttpBridgeSessionRecord.id.in_(session_ids))
|
|
3557
|
+
.where(HttpBridgeSessionRecord.state == HttpBridgeSessionState.CLOSED)
|
|
3558
|
+
.where(HttpBridgeSessionRecord.last_seen_at < cutoff)
|
|
3559
|
+
.where(
|
|
3560
|
+
~exists(
|
|
3561
|
+
select(HttpBridgeOperationRecord.operation_id).where(
|
|
3562
|
+
HttpBridgeOperationRecord.session_id == HttpBridgeSessionRecord.id,
|
|
3563
|
+
)
|
|
3564
|
+
)
|
|
3565
|
+
)
|
|
3566
|
+
.returning(HttpBridgeSessionRecord.id)
|
|
3567
|
+
)
|
|
3568
|
+
await self._session.commit()
|
|
3569
|
+
deleted_count += len(deleted.scalars().all())
|
|
3570
|
+
|
|
3571
|
+
async def retire_continuity_owner_if_unavailable(
|
|
3572
|
+
self,
|
|
3573
|
+
session_id: str,
|
|
3574
|
+
*,
|
|
3575
|
+
expected_account_id: str,
|
|
3576
|
+
recovery_deadline_epoch: int,
|
|
3577
|
+
) -> bool:
|
|
3578
|
+
"""Retire one row's owner now, when it cannot return before the deadline.
|
|
3579
|
+
|
|
3580
|
+
The scheduled sweep frees a thread six hours after its last turn. This
|
|
3581
|
+
is the request-path counterpart, and it asks a narrower question: can
|
|
3582
|
+
this owner come back before the request that is waiting on it gives up?
|
|
3583
|
+
``reset_at`` is the answer for a rate or quota limit; ``paused``,
|
|
3584
|
+
``reauth_required`` and ``deactivated`` carry no horizon at all, so the
|
|
3585
|
+
answer for them is always no.
|
|
3586
|
+
|
|
3587
|
+
Mirrors ``StickySessionsRepository.abandon_legacy_session_header_owner_if_unavailable``,
|
|
3588
|
+
including why the account row is locked first: PostgreSQL evaluates the
|
|
3589
|
+
status subquery from the UPDATE's snapshot, so without the lock a
|
|
3590
|
+
concurrent recovery can commit while this statement waits on the
|
|
3591
|
+
session row and the stale snapshot would still authorize a retirement.
|
|
3592
|
+
The status predicate stays inside the UPDATE as a second, database-level
|
|
3593
|
+
invariant so a later refactor cannot turn a prior observation into an
|
|
3594
|
+
unconditional write.
|
|
3595
|
+
|
|
3596
|
+
Writes the scope marker alone and leaves the timestamp NULL, so a
|
|
3597
|
+
replica running the previous build keeps treating ``account_id`` as
|
|
3598
|
+
hard ownership for the rest of a rolling deploy.
|
|
3599
|
+
"""
|
|
3600
|
+
if not session_id or not expected_account_id:
|
|
3601
|
+
return False
|
|
3602
|
+
owner_status_lock = (
|
|
3603
|
+
select(Account.status, Account.reset_at).where(Account.id == expected_account_id).with_for_update()
|
|
3604
|
+
)
|
|
3605
|
+
unavailable_owner = select(Account.id).where(
|
|
3606
|
+
Account.id == expected_account_id,
|
|
3607
|
+
Account.status.in_(HARD_OWNER_UNAVAILABLE_STATUSES),
|
|
3608
|
+
or_(Account.reset_at.is_(None), Account.reset_at >= recovery_deadline_epoch),
|
|
3609
|
+
)
|
|
3610
|
+
statement = (
|
|
3611
|
+
update(HttpBridgeSessionRecord)
|
|
3612
|
+
.where(
|
|
3613
|
+
HttpBridgeSessionRecord.id == session_id,
|
|
3614
|
+
HttpBridgeSessionRecord.account_id == expected_account_id,
|
|
3615
|
+
HttpBridgeSessionRecord.continuity_abandoned_at.is_(None),
|
|
3616
|
+
HttpBridgeSessionRecord.continuity_abandonment_scope.is_(None),
|
|
3617
|
+
HttpBridgeSessionRecord.account_id.in_(unavailable_owner),
|
|
3618
|
+
)
|
|
3619
|
+
.values(continuity_abandonment_scope=_REQUEST_PATH_ABANDONMENT_SCOPE)
|
|
3620
|
+
.returning(HttpBridgeSessionRecord.id)
|
|
3621
|
+
)
|
|
3622
|
+
async with sqlite_writer_section():
|
|
3623
|
+
locked = (await self._session.execute(owner_status_lock)).one_or_none()
|
|
3624
|
+
if locked is None or locked[0] not in HARD_OWNER_UNAVAILABLE_STATUSES:
|
|
3625
|
+
await self._session.commit()
|
|
3626
|
+
return False
|
|
3627
|
+
owner_reset_at = locked[1]
|
|
3628
|
+
if owner_reset_at is not None and owner_reset_at < recovery_deadline_epoch:
|
|
3629
|
+
# The owner is expected back inside the window the caller is
|
|
3630
|
+
# willing to wait, so waiting keeps the upstream prompt cache
|
|
3631
|
+
# instead of forcing a full resend onto another account.
|
|
3632
|
+
await self._session.commit()
|
|
3633
|
+
return False
|
|
3634
|
+
result = await self._session.execute(statement)
|
|
3635
|
+
await self._session.commit()
|
|
3636
|
+
return result.scalar_one_or_none() is not None
|
|
3637
|
+
|
|
3638
|
+
async def retire_stale_unavailable_bridge_owners(self, cutoff: datetime, *, now: datetime) -> int:
|
|
3639
|
+
"""Retire continuity owners that have been unroutable since ``cutoff``.
|
|
3640
|
+
|
|
3641
|
+
The durable-bridge counterpart of
|
|
3642
|
+
``StickySessionsRepository.purge_stale_hard_codex_session_mappings``,
|
|
3643
|
+
and it exists for the same reason. Deleting the row outright would be
|
|
3644
|
+
indistinguishable from "this key was never seen", which leaves the
|
|
3645
|
+
anchored lookup failing closed forever; a tombstone instead says the
|
|
3646
|
+
owner was deliberately abandoned, so a later request may pick a fresh
|
|
3647
|
+
one. Two phases:
|
|
3648
|
+
|
|
3649
|
+
1. Tombstone a row whose owner has sat in
|
|
3650
|
+
:data:`HARD_OWNER_UNAVAILABLE_STATUSES` past its reset horizon while
|
|
3651
|
+
the row itself went untouched for the whole grace window.
|
|
3652
|
+
2. Delete a tombstone that then sat another full window with nobody
|
|
3653
|
+
claiming it, as long as it owns no operation rows.
|
|
3654
|
+
|
|
3655
|
+
Unlike the row deletes elsewhere in this module, phase 1 is **not**
|
|
3656
|
+
gated on ``~exists(operation)``. That guard protects the durable
|
|
3657
|
+
recovery ledger from a ``CASCADE``, which retirement does not trigger:
|
|
3658
|
+
it writes two columns and keeps every operation row intact. Gating it
|
|
3659
|
+
would reproduce the 2026-09-04 outage, where every poisoned row
|
|
3660
|
+
happened to own operations and so was never reachable by cleanup.
|
|
3661
|
+
"""
|
|
3662
|
+
now_naive = to_utc_naive(now)
|
|
3663
|
+
cutoff_naive = to_utc_naive(cutoff)
|
|
3664
|
+
now_epoch = naive_utc_to_epoch(now_naive)
|
|
3665
|
+
unavailable_account_ids = select(Account.id).where(
|
|
3666
|
+
Account.status.in_(HARD_OWNER_UNAVAILABLE_STATUSES),
|
|
3667
|
+
or_(Account.reset_at.is_(None), Account.reset_at < now_epoch),
|
|
3668
|
+
)
|
|
3669
|
+
tombstone_stmt = (
|
|
3670
|
+
update(HttpBridgeSessionRecord)
|
|
3671
|
+
.where(
|
|
3672
|
+
HttpBridgeSessionRecord.continuity_abandoned_at.is_(None),
|
|
3673
|
+
HttpBridgeSessionRecord.last_seen_at < cutoff_naive,
|
|
3674
|
+
or_(
|
|
3675
|
+
# A request-path retirement wrote the scope marker alone.
|
|
3676
|
+
# Promote it once the row goes stale, exactly as the sticky
|
|
3677
|
+
# sweep promotes its own younger scoped marker: without
|
|
3678
|
+
# this, such a row satisfies neither phase — phase 1 wants
|
|
3679
|
+
# both columns NULL and phase 2 wants a timestamp — and
|
|
3680
|
+
# would sit in the table forever.
|
|
3681
|
+
HttpBridgeSessionRecord.continuity_abandonment_scope.is_not(None),
|
|
3682
|
+
and_(
|
|
3683
|
+
HttpBridgeSessionRecord.continuity_abandonment_scope.is_(None),
|
|
3684
|
+
HttpBridgeSessionRecord.account_id.is_not(None),
|
|
3685
|
+
HttpBridgeSessionRecord.account_id.in_(unavailable_account_ids),
|
|
3686
|
+
),
|
|
3687
|
+
),
|
|
3688
|
+
)
|
|
3689
|
+
# Timestamp with NULL scope is the global form: this sweep has no
|
|
3690
|
+
# per-source question to answer, and a replica that predates the
|
|
3691
|
+
# scope column still understands the timestamp.
|
|
3692
|
+
.values(continuity_abandoned_at=now_naive, continuity_abandonment_scope=None)
|
|
3693
|
+
.returning(HttpBridgeSessionRecord.id)
|
|
3694
|
+
)
|
|
3695
|
+
expired_tombstone_filter = (
|
|
3696
|
+
HttpBridgeSessionRecord.continuity_abandoned_at.is_not(None),
|
|
3697
|
+
HttpBridgeSessionRecord.continuity_abandonment_scope.is_(None),
|
|
3698
|
+
HttpBridgeSessionRecord.continuity_abandoned_at < cutoff_naive,
|
|
3699
|
+
~exists(
|
|
3700
|
+
select(HttpBridgeOperationRecord.operation_id).where(
|
|
3701
|
+
HttpBridgeOperationRecord.session_id == HttpBridgeSessionRecord.id,
|
|
3702
|
+
)
|
|
3703
|
+
),
|
|
3704
|
+
)
|
|
3705
|
+
async with sqlite_writer_section():
|
|
3706
|
+
tombstoned = len((await self._session.execute(tombstone_stmt)).scalars().all())
|
|
3707
|
+
# Aliases first, matching ``purge_closed_before``: the FK cascade
|
|
3708
|
+
# would cover it, but SQLite builds without foreign-key enforcement
|
|
3709
|
+
# would leave orphans that later resolve to a deleted session.
|
|
3710
|
+
await self._session.execute(
|
|
3711
|
+
delete(HttpBridgeSessionAlias).where(
|
|
3712
|
+
HttpBridgeSessionAlias.session_id.in_(
|
|
3713
|
+
select(HttpBridgeSessionRecord.id).where(*expired_tombstone_filter)
|
|
3714
|
+
)
|
|
3715
|
+
)
|
|
3716
|
+
)
|
|
3717
|
+
deleted = len(
|
|
3718
|
+
(
|
|
3719
|
+
await self._session.execute(
|
|
3720
|
+
delete(HttpBridgeSessionRecord)
|
|
3721
|
+
.where(*expired_tombstone_filter)
|
|
3722
|
+
.returning(HttpBridgeSessionRecord.id)
|
|
3723
|
+
)
|
|
3724
|
+
)
|
|
3725
|
+
.scalars()
|
|
3726
|
+
.all()
|
|
3727
|
+
)
|
|
3728
|
+
await self._session.commit()
|
|
3729
|
+
return tombstoned + deleted
|
|
3730
|
+
|
|
3731
|
+
async def purge_abandoned_before(self, cutoff: datetime, *, batch_size: int = _PURGE_CLOSED_BATCH_SIZE) -> int:
|
|
3732
|
+
"""Purge ACTIVE/DRAINING rows whose lease expired and whose activity predates the cutoff."""
|
|
3733
|
+
|
|
3734
|
+
deleted_count = 0
|
|
3735
|
+
while True:
|
|
3736
|
+
now = utcnow()
|
|
3737
|
+
abandoned_filter = (
|
|
3738
|
+
HttpBridgeSessionRecord.state.in_((HttpBridgeSessionState.ACTIVE, HttpBridgeSessionState.DRAINING)),
|
|
3739
|
+
or_(
|
|
3740
|
+
HttpBridgeSessionRecord.lease_expires_at.is_(None),
|
|
3741
|
+
HttpBridgeSessionRecord.lease_expires_at < now,
|
|
3742
|
+
),
|
|
3743
|
+
HttpBridgeSessionRecord.last_seen_at < cutoff,
|
|
3744
|
+
~exists(
|
|
3745
|
+
select(HttpBridgeOperationRecord.operation_id).where(
|
|
3746
|
+
HttpBridgeOperationRecord.session_id == HttpBridgeSessionRecord.id,
|
|
3747
|
+
)
|
|
3748
|
+
),
|
|
3749
|
+
)
|
|
3750
|
+
result = await self._session.execute(
|
|
3751
|
+
select(HttpBridgeSessionRecord.id)
|
|
3752
|
+
.where(*abandoned_filter)
|
|
3753
|
+
.order_by(HttpBridgeSessionRecord.last_seen_at.asc())
|
|
3754
|
+
.limit(batch_size)
|
|
3755
|
+
)
|
|
3756
|
+
session_ids = list(result.scalars().all())
|
|
3757
|
+
if not session_ids:
|
|
3758
|
+
return deleted_count
|
|
3759
|
+
async with sqlite_writer_section():
|
|
3760
|
+
await self._session.execute(
|
|
3761
|
+
delete(HttpBridgeSessionAlias).where(
|
|
3762
|
+
HttpBridgeSessionAlias.session_id.in_(
|
|
3763
|
+
select(HttpBridgeSessionRecord.id).where(
|
|
3764
|
+
HttpBridgeSessionRecord.id.in_(session_ids),
|
|
3765
|
+
*abandoned_filter,
|
|
3766
|
+
)
|
|
3767
|
+
)
|
|
3768
|
+
)
|
|
3769
|
+
)
|
|
3770
|
+
deleted = await self._session.execute(
|
|
3771
|
+
delete(HttpBridgeSessionRecord)
|
|
3772
|
+
.where(HttpBridgeSessionRecord.id.in_(session_ids))
|
|
3773
|
+
.where(*abandoned_filter)
|
|
3774
|
+
.returning(HttpBridgeSessionRecord.id)
|
|
3775
|
+
)
|
|
3776
|
+
await self._session.commit()
|
|
3777
|
+
deleted_count += len(deleted.scalars().all())
|
|
3778
|
+
|
|
3779
|
+
async def purge_retry_circuits_before(
|
|
3780
|
+
self,
|
|
3781
|
+
cutoff_epoch: float,
|
|
3782
|
+
*,
|
|
3783
|
+
tombstone_cutoff_epoch: float | None = None,
|
|
3784
|
+
batch_size: int = _PURGE_CLOSED_BATCH_SIZE,
|
|
3785
|
+
) -> int:
|
|
3786
|
+
# An anchor_abandoned tombstone guards continuity that outlives the
|
|
3787
|
+
# circuit TTL: reaping it on the circuit schedule would let the
|
|
3788
|
+
# unanchored-delta gate dispatch a context-free request. Tombstones
|
|
3789
|
+
# are purged only past the caller's bridge-retention cutoff, and
|
|
3790
|
+
# preserved entirely when no cutoff is given.
|
|
3791
|
+
non_tombstone = HttpBridgeRetryCircuit.last_detail.is_(None) | (
|
|
3792
|
+
HttpBridgeRetryCircuit.last_detail != _RETRY_CIRCUIT_ABANDONED_TOMBSTONE_DETAIL
|
|
3793
|
+
)
|
|
3794
|
+
# A replay claim advances only the admission generation and
|
|
3795
|
+
# deliberately leaves the timestamp unchanged, so a claimed
|
|
3796
|
+
# generation near the TTL could be reaped while its replay
|
|
3797
|
+
# is still in flight. Ever-claimed rows get one extra TTL of
|
|
3798
|
+
# grace before the scheduled purge takes them.
|
|
3799
|
+
stale_predicate = (
|
|
3800
|
+
non_tombstone
|
|
3801
|
+
& (HttpBridgeRetryCircuit.updated_at_epoch < cutoff_epoch)
|
|
3802
|
+
& (
|
|
3803
|
+
(HttpBridgeRetryCircuit.admission_generation == 0)
|
|
3804
|
+
| (
|
|
3805
|
+
HttpBridgeRetryCircuit.updated_at_epoch
|
|
3806
|
+
< cutoff_epoch - DURABLE_BRIDGE_RETRY_CIRCUIT_STATE_TTL_SECONDS
|
|
3807
|
+
)
|
|
3808
|
+
)
|
|
3809
|
+
)
|
|
3810
|
+
if tombstone_cutoff_epoch is not None:
|
|
3811
|
+
# A crash between a poison settle and its registration leaves a
|
|
3812
|
+
# live session storing the poisoned continuity while delta-only
|
|
3813
|
+
# requests keep refreshing its lease; the tombstone's own
|
|
3814
|
+
# updated_at_epoch stays fixed, so an age cutoff alone would
|
|
3815
|
+
# reap the fail-closed fence out from under that live session.
|
|
3816
|
+
# The tombstone falls only when no session still resolves the
|
|
3817
|
+
# key with continuity to protect.
|
|
3818
|
+
session_continuity_exists = (
|
|
3819
|
+
select(HttpBridgeSessionRecord.id)
|
|
3820
|
+
.where(
|
|
3821
|
+
HttpBridgeSessionRecord.session_key_kind == HttpBridgeRetryCircuit.session_key_kind,
|
|
3822
|
+
HttpBridgeSessionRecord.session_key_hash == HttpBridgeRetryCircuit.session_key_hash,
|
|
3823
|
+
HttpBridgeSessionRecord.api_key_scope == HttpBridgeRetryCircuit.api_key_scope,
|
|
3824
|
+
(HttpBridgeSessionRecord.latest_response_id.is_not(None))
|
|
3825
|
+
| (HttpBridgeSessionRecord.latest_turn_state.is_not(None)),
|
|
3826
|
+
)
|
|
3827
|
+
.exists()
|
|
3828
|
+
)
|
|
3829
|
+
alias_continuity_exists = (
|
|
3830
|
+
select(HttpBridgeSessionAlias.id)
|
|
3831
|
+
.join(
|
|
3832
|
+
HttpBridgeSessionRecord,
|
|
3833
|
+
HttpBridgeSessionAlias.session_id == HttpBridgeSessionRecord.id,
|
|
3834
|
+
)
|
|
3835
|
+
.where(
|
|
3836
|
+
HttpBridgeSessionAlias.alias_kind == HttpBridgeRetryCircuit.session_key_kind,
|
|
3837
|
+
HttpBridgeSessionAlias.alias_hash == HttpBridgeRetryCircuit.session_key_hash,
|
|
3838
|
+
HttpBridgeSessionAlias.api_key_scope == HttpBridgeRetryCircuit.api_key_scope,
|
|
3839
|
+
(HttpBridgeSessionRecord.latest_response_id.is_not(None))
|
|
3840
|
+
| (HttpBridgeSessionRecord.latest_turn_state.is_not(None)),
|
|
3841
|
+
)
|
|
3842
|
+
.exists()
|
|
3843
|
+
)
|
|
3844
|
+
live_continuity_exists = session_continuity_exists | alias_continuity_exists
|
|
3845
|
+
stale_predicate = stale_predicate | (
|
|
3846
|
+
(HttpBridgeRetryCircuit.last_detail == _RETRY_CIRCUIT_ABANDONED_TOMBSTONE_DETAIL)
|
|
3847
|
+
& (HttpBridgeRetryCircuit.updated_at_epoch < tombstone_cutoff_epoch)
|
|
3848
|
+
& ~live_continuity_exists
|
|
3849
|
+
)
|
|
3850
|
+
deleted_count = 0
|
|
3851
|
+
while True:
|
|
3852
|
+
result = await self._session.execute(
|
|
3853
|
+
select(
|
|
3854
|
+
HttpBridgeRetryCircuit.session_key_kind,
|
|
3855
|
+
HttpBridgeRetryCircuit.session_key_hash,
|
|
3856
|
+
HttpBridgeRetryCircuit.api_key_scope,
|
|
3857
|
+
)
|
|
3858
|
+
.where(stale_predicate)
|
|
3859
|
+
.limit(batch_size)
|
|
3860
|
+
)
|
|
3861
|
+
keys = [tuple(row) for row in result.fetchall()]
|
|
3862
|
+
if not keys:
|
|
3863
|
+
return deleted_count
|
|
3864
|
+
batch_deleted_count = 0
|
|
3865
|
+
async with sqlite_writer_section():
|
|
3866
|
+
for session_key_kind, session_key_hash, api_key_scope in keys:
|
|
3867
|
+
deleted = await self._session.execute(
|
|
3868
|
+
delete(HttpBridgeRetryCircuit)
|
|
3869
|
+
.where(HttpBridgeRetryCircuit.session_key_kind == session_key_kind)
|
|
3870
|
+
.where(HttpBridgeRetryCircuit.session_key_hash == session_key_hash)
|
|
3871
|
+
.where(HttpBridgeRetryCircuit.api_key_scope == api_key_scope)
|
|
3872
|
+
.where(stale_predicate)
|
|
3873
|
+
.returning(HttpBridgeRetryCircuit.session_key_hash)
|
|
3874
|
+
)
|
|
3875
|
+
batch_deleted_count += len(deleted.scalars().all())
|
|
3876
|
+
await self._session.commit()
|
|
3877
|
+
if batch_deleted_count == 0:
|
|
3878
|
+
return deleted_count
|
|
3879
|
+
deleted_count += batch_deleted_count
|
|
3880
|
+
|
|
3881
|
+
async def upsert_alias(
|
|
3882
|
+
self,
|
|
3883
|
+
*,
|
|
3884
|
+
session_id: str,
|
|
3885
|
+
alias_kind: str,
|
|
3886
|
+
alias_value: str,
|
|
3887
|
+
api_key_scope: str,
|
|
3888
|
+
) -> None:
|
|
3889
|
+
async with sqlite_writer_section():
|
|
3890
|
+
await self._execute_alias_upsert(
|
|
3891
|
+
session_id=session_id,
|
|
3892
|
+
alias_kind=alias_kind,
|
|
3893
|
+
alias_value=alias_value,
|
|
3894
|
+
api_key_scope=api_key_scope,
|
|
3895
|
+
)
|
|
3896
|
+
await self._session.commit()
|
|
3897
|
+
|
|
3898
|
+
async def register_owned_alias(
|
|
3899
|
+
self,
|
|
3900
|
+
*,
|
|
3901
|
+
session_id: str,
|
|
3902
|
+
api_key_scope: str,
|
|
3903
|
+
instance_id: str,
|
|
3904
|
+
owner_epoch: int,
|
|
3905
|
+
alias_kind: str,
|
|
3906
|
+
alias_value: str,
|
|
3907
|
+
lease_ttl_seconds: float,
|
|
3908
|
+
latest_turn_state: str | None = None,
|
|
3909
|
+
latest_response_id: str | None = None,
|
|
3910
|
+
latest_input_item_count: int | None = None,
|
|
3911
|
+
latest_input_full_fingerprint: str | None = None,
|
|
3912
|
+
latest_pending_tool_calls: Mapping[str, str] | None = None,
|
|
3913
|
+
) -> DurableBridgeAliasRegistration:
|
|
3914
|
+
"""Register continuity only while the caller still owns the durable row."""
|
|
3915
|
+
|
|
3916
|
+
async with sqlite_writer_section():
|
|
3917
|
+
now = utcnow()
|
|
3918
|
+
session_values: dict[str, object] = {
|
|
3919
|
+
"lease_expires_at": now + timedelta(seconds=max(1.0, lease_ttl_seconds)),
|
|
3920
|
+
"last_seen_at": now,
|
|
3921
|
+
}
|
|
3922
|
+
if latest_turn_state is not None:
|
|
3923
|
+
session_values["latest_turn_state"] = latest_turn_state
|
|
3924
|
+
if latest_response_id is not None:
|
|
3925
|
+
session_values["latest_response_id"] = latest_response_id
|
|
3926
|
+
session_values["latest_input_item_count"] = latest_input_item_count
|
|
3927
|
+
session_values["latest_input_full_fingerprint"] = latest_input_full_fingerprint
|
|
3928
|
+
session_values["latest_pending_tool_calls_json"] = _encode_pending_tool_calls(
|
|
3929
|
+
latest_response_id,
|
|
3930
|
+
latest_pending_tool_calls,
|
|
3931
|
+
)
|
|
3932
|
+
elif latest_input_item_count is not None and latest_input_full_fingerprint is not None:
|
|
3933
|
+
session_values["latest_input_item_count"] = latest_input_item_count
|
|
3934
|
+
session_values["latest_input_full_fingerprint"] = latest_input_full_fingerprint
|
|
3935
|
+
|
|
3936
|
+
fenced_update = await self._session.execute(
|
|
3937
|
+
update(HttpBridgeSessionRecord)
|
|
3938
|
+
.where(
|
|
3939
|
+
HttpBridgeSessionRecord.id == session_id,
|
|
3940
|
+
HttpBridgeSessionRecord.api_key_scope == api_key_scope,
|
|
3941
|
+
HttpBridgeSessionRecord.owner_instance_id == instance_id,
|
|
3942
|
+
HttpBridgeSessionRecord.owner_epoch == owner_epoch,
|
|
3943
|
+
)
|
|
3944
|
+
.values(**session_values)
|
|
3945
|
+
.returning(
|
|
3946
|
+
HttpBridgeSessionRecord.id,
|
|
3947
|
+
HttpBridgeSessionRecord.session_key_kind,
|
|
3948
|
+
HttpBridgeSessionRecord.session_key_value,
|
|
3949
|
+
)
|
|
3950
|
+
)
|
|
3951
|
+
target = fenced_update.one_or_none()
|
|
3952
|
+
if target is None:
|
|
3953
|
+
return DurableBridgeAliasRegistration.OWNER_FENCED
|
|
3954
|
+
|
|
3955
|
+
registered = await self._execute_alias_upsert(
|
|
3956
|
+
session_id=session_id,
|
|
3957
|
+
alias_kind=alias_kind,
|
|
3958
|
+
alias_value=alias_value,
|
|
3959
|
+
api_key_scope=api_key_scope,
|
|
3960
|
+
target_account_neutral_replay=is_http_bridge_account_neutral_replay(
|
|
3961
|
+
kind=target.session_key_kind,
|
|
3962
|
+
key=target.session_key_value,
|
|
3963
|
+
),
|
|
3964
|
+
)
|
|
3965
|
+
if not registered:
|
|
3966
|
+
await self._session.rollback()
|
|
3967
|
+
return DurableBridgeAliasRegistration.ALIAS_PROTECTED
|
|
3968
|
+
await self._session.commit()
|
|
3969
|
+
return DurableBridgeAliasRegistration.REGISTERED
|
|
3970
|
+
|
|
3971
|
+
async def register_reversible_turn_state_alias(
|
|
3972
|
+
self,
|
|
3973
|
+
*,
|
|
3974
|
+
session_id: str,
|
|
3975
|
+
api_key_scope: str,
|
|
3976
|
+
instance_id: str,
|
|
3977
|
+
owner_epoch: int,
|
|
3978
|
+
turn_state: str,
|
|
3979
|
+
lease_ttl_seconds: float,
|
|
3980
|
+
) -> DurableBridgeAliasRegistrationReceipt:
|
|
3981
|
+
"""Publish a pre-dispatch turn alias with enough state for exact rollback."""
|
|
3982
|
+
|
|
3983
|
+
alias_kind = "turn_state"
|
|
3984
|
+
async with sqlite_writer_section():
|
|
3985
|
+
now = utcnow()
|
|
3986
|
+
# The first UPDATE both fences ownership and acquires the target-row
|
|
3987
|
+
# lock (and SQLite's writer lock) before prior alias state is read.
|
|
3988
|
+
fenced_lock = await self._session.execute(
|
|
3989
|
+
update(HttpBridgeSessionRecord)
|
|
3990
|
+
.where(
|
|
3991
|
+
HttpBridgeSessionRecord.id == session_id,
|
|
3992
|
+
HttpBridgeSessionRecord.api_key_scope == api_key_scope,
|
|
3993
|
+
HttpBridgeSessionRecord.owner_instance_id == instance_id,
|
|
3994
|
+
HttpBridgeSessionRecord.owner_epoch == owner_epoch,
|
|
3995
|
+
)
|
|
3996
|
+
.values(
|
|
3997
|
+
lease_expires_at=now + timedelta(seconds=max(1.0, lease_ttl_seconds)),
|
|
3998
|
+
last_seen_at=now,
|
|
3999
|
+
)
|
|
4000
|
+
.returning(
|
|
4001
|
+
HttpBridgeSessionRecord.id,
|
|
4002
|
+
HttpBridgeSessionRecord.session_key_kind,
|
|
4003
|
+
HttpBridgeSessionRecord.session_key_value,
|
|
4004
|
+
HttpBridgeSessionRecord.latest_turn_state,
|
|
4005
|
+
)
|
|
4006
|
+
)
|
|
4007
|
+
target = fenced_lock.one_or_none()
|
|
4008
|
+
if target is None:
|
|
4009
|
+
await self._session.rollback()
|
|
4010
|
+
return DurableBridgeAliasRegistrationReceipt(
|
|
4011
|
+
status=DurableBridgeAliasRegistration.OWNER_FENCED,
|
|
4012
|
+
session_id=session_id,
|
|
4013
|
+
api_key_scope=api_key_scope,
|
|
4014
|
+
alias_kind=alias_kind,
|
|
4015
|
+
alias_value=turn_state,
|
|
4016
|
+
instance_id=instance_id,
|
|
4017
|
+
owner_epoch=owner_epoch,
|
|
4018
|
+
previous_alias_session_id=None,
|
|
4019
|
+
previous_alias_owner_epoch=None,
|
|
4020
|
+
previous_alias_account_id=None,
|
|
4021
|
+
previous_latest_turn_state=None,
|
|
4022
|
+
)
|
|
4023
|
+
|
|
4024
|
+
previous_latest_turn_state = target.latest_turn_state
|
|
4025
|
+
previous_alias_session_id = await self._session.scalar(
|
|
4026
|
+
select(HttpBridgeSessionAlias.session_id)
|
|
4027
|
+
.where(
|
|
4028
|
+
HttpBridgeSessionAlias.alias_kind == alias_kind,
|
|
4029
|
+
HttpBridgeSessionAlias.alias_hash == durable_bridge_hash(turn_state),
|
|
4030
|
+
HttpBridgeSessionAlias.alias_value == turn_state,
|
|
4031
|
+
HttpBridgeSessionAlias.api_key_scope == api_key_scope,
|
|
4032
|
+
)
|
|
4033
|
+
.with_for_update()
|
|
4034
|
+
)
|
|
4035
|
+
previous_alias_owner_epoch = None
|
|
4036
|
+
previous_alias_account_id = None
|
|
4037
|
+
if previous_alias_session_id is not None:
|
|
4038
|
+
previous_alias_owner = (
|
|
4039
|
+
await self._session.execute(
|
|
4040
|
+
select(
|
|
4041
|
+
HttpBridgeSessionRecord.owner_epoch,
|
|
4042
|
+
HttpBridgeSessionRecord.account_id,
|
|
4043
|
+
).where(HttpBridgeSessionRecord.id == previous_alias_session_id)
|
|
4044
|
+
)
|
|
4045
|
+
).one_or_none()
|
|
4046
|
+
if previous_alias_owner is not None:
|
|
4047
|
+
previous_alias_owner_epoch = previous_alias_owner.owner_epoch
|
|
4048
|
+
previous_alias_account_id = previous_alias_owner.account_id
|
|
4049
|
+
await self._session.execute(
|
|
4050
|
+
update(HttpBridgeSessionRecord)
|
|
4051
|
+
.where(HttpBridgeSessionRecord.id == session_id)
|
|
4052
|
+
.values(latest_turn_state=turn_state)
|
|
4053
|
+
)
|
|
4054
|
+
registered = await self._execute_alias_upsert(
|
|
4055
|
+
session_id=session_id,
|
|
4056
|
+
alias_kind=alias_kind,
|
|
4057
|
+
alias_value=turn_state,
|
|
4058
|
+
api_key_scope=api_key_scope,
|
|
4059
|
+
target_account_neutral_replay=is_http_bridge_account_neutral_replay(
|
|
4060
|
+
kind=target.session_key_kind,
|
|
4061
|
+
key=target.session_key_value,
|
|
4062
|
+
),
|
|
4063
|
+
)
|
|
4064
|
+
if not registered:
|
|
4065
|
+
await self._session.rollback()
|
|
4066
|
+
return DurableBridgeAliasRegistrationReceipt(
|
|
4067
|
+
status=DurableBridgeAliasRegistration.ALIAS_PROTECTED,
|
|
4068
|
+
session_id=session_id,
|
|
4069
|
+
api_key_scope=api_key_scope,
|
|
4070
|
+
alias_kind=alias_kind,
|
|
4071
|
+
alias_value=turn_state,
|
|
4072
|
+
instance_id=instance_id,
|
|
4073
|
+
owner_epoch=owner_epoch,
|
|
4074
|
+
previous_alias_session_id=previous_alias_session_id,
|
|
4075
|
+
previous_alias_owner_epoch=previous_alias_owner_epoch,
|
|
4076
|
+
previous_alias_account_id=previous_alias_account_id,
|
|
4077
|
+
previous_latest_turn_state=previous_latest_turn_state,
|
|
4078
|
+
)
|
|
4079
|
+
await self._session.commit()
|
|
4080
|
+
|
|
4081
|
+
return DurableBridgeAliasRegistrationReceipt(
|
|
4082
|
+
status=DurableBridgeAliasRegistration.REGISTERED,
|
|
4083
|
+
session_id=session_id,
|
|
4084
|
+
api_key_scope=api_key_scope,
|
|
4085
|
+
alias_kind=alias_kind,
|
|
4086
|
+
alias_value=turn_state,
|
|
4087
|
+
instance_id=instance_id,
|
|
4088
|
+
owner_epoch=owner_epoch,
|
|
4089
|
+
previous_alias_session_id=previous_alias_session_id,
|
|
4090
|
+
previous_alias_owner_epoch=previous_alias_owner_epoch,
|
|
4091
|
+
previous_alias_account_id=previous_alias_account_id,
|
|
4092
|
+
previous_latest_turn_state=previous_latest_turn_state,
|
|
4093
|
+
)
|
|
4094
|
+
|
|
4095
|
+
async def rollback_reversible_turn_state_alias(
|
|
4096
|
+
self,
|
|
4097
|
+
*,
|
|
4098
|
+
receipt: DurableBridgeAliasRegistrationReceipt,
|
|
4099
|
+
) -> bool:
|
|
4100
|
+
"""Undo a registered pre-dispatch alias while the same owner is fenced in."""
|
|
4101
|
+
|
|
4102
|
+
if receipt.status != DurableBridgeAliasRegistration.REGISTERED:
|
|
4103
|
+
return False
|
|
4104
|
+
|
|
4105
|
+
async with sqlite_writer_section():
|
|
4106
|
+
previous_session_valid = False
|
|
4107
|
+
dialect = self._session.get_bind().dialect.name
|
|
4108
|
+
if dialect == "postgresql":
|
|
4109
|
+
session_ids = {receipt.session_id}
|
|
4110
|
+
if receipt.previous_alias_session_id is not None:
|
|
4111
|
+
session_ids.add(receipt.previous_alias_session_id)
|
|
4112
|
+
locked_records = (
|
|
4113
|
+
(
|
|
4114
|
+
await self._session.execute(
|
|
4115
|
+
select(HttpBridgeSessionRecord)
|
|
4116
|
+
.where(HttpBridgeSessionRecord.id.in_(session_ids))
|
|
4117
|
+
.order_by(HttpBridgeSessionRecord.id)
|
|
4118
|
+
.with_for_update()
|
|
4119
|
+
)
|
|
4120
|
+
)
|
|
4121
|
+
.scalars()
|
|
4122
|
+
.all()
|
|
4123
|
+
)
|
|
4124
|
+
records_by_id = {record.id: record for record in locked_records}
|
|
4125
|
+
target_record = records_by_id.get(receipt.session_id)
|
|
4126
|
+
if (
|
|
4127
|
+
target_record is None
|
|
4128
|
+
or target_record.api_key_scope != receipt.api_key_scope
|
|
4129
|
+
or target_record.owner_instance_id != receipt.instance_id
|
|
4130
|
+
or target_record.owner_epoch != receipt.owner_epoch
|
|
4131
|
+
):
|
|
4132
|
+
await self._session.rollback()
|
|
4133
|
+
return False
|
|
4134
|
+
previous_record = (
|
|
4135
|
+
records_by_id.get(receipt.previous_alias_session_id)
|
|
4136
|
+
if receipt.previous_alias_session_id is not None
|
|
4137
|
+
else None
|
|
4138
|
+
)
|
|
4139
|
+
previous_session_valid = previous_record is not None and (
|
|
4140
|
+
previous_record.owner_epoch == receipt.previous_alias_owner_epoch
|
|
4141
|
+
and previous_record.account_id == receipt.previous_alias_account_id
|
|
4142
|
+
)
|
|
4143
|
+
|
|
4144
|
+
fenced_restore = await self._session.execute(
|
|
4145
|
+
update(HttpBridgeSessionRecord)
|
|
4146
|
+
.where(
|
|
4147
|
+
HttpBridgeSessionRecord.id == receipt.session_id,
|
|
4148
|
+
HttpBridgeSessionRecord.api_key_scope == receipt.api_key_scope,
|
|
4149
|
+
HttpBridgeSessionRecord.owner_instance_id == receipt.instance_id,
|
|
4150
|
+
HttpBridgeSessionRecord.owner_epoch == receipt.owner_epoch,
|
|
4151
|
+
)
|
|
4152
|
+
.values(
|
|
4153
|
+
latest_turn_state=case(
|
|
4154
|
+
(
|
|
4155
|
+
HttpBridgeSessionRecord.latest_turn_state == receipt.alias_value,
|
|
4156
|
+
receipt.previous_latest_turn_state,
|
|
4157
|
+
),
|
|
4158
|
+
else_=HttpBridgeSessionRecord.latest_turn_state,
|
|
4159
|
+
)
|
|
4160
|
+
)
|
|
4161
|
+
.returning(HttpBridgeSessionRecord.id)
|
|
4162
|
+
)
|
|
4163
|
+
if fenced_restore.scalar_one_or_none() is None:
|
|
4164
|
+
await self._session.rollback()
|
|
4165
|
+
return False
|
|
4166
|
+
|
|
4167
|
+
if dialect != "postgresql" and receipt.previous_alias_session_id is not None:
|
|
4168
|
+
previous_record = (
|
|
4169
|
+
await self._session.execute(
|
|
4170
|
+
select(
|
|
4171
|
+
HttpBridgeSessionRecord.owner_epoch,
|
|
4172
|
+
HttpBridgeSessionRecord.account_id,
|
|
4173
|
+
).where(HttpBridgeSessionRecord.id == receipt.previous_alias_session_id)
|
|
4174
|
+
)
|
|
4175
|
+
).one_or_none()
|
|
4176
|
+
previous_session_valid = previous_record is not None and (
|
|
4177
|
+
previous_record.owner_epoch == receipt.previous_alias_owner_epoch
|
|
4178
|
+
and previous_record.account_id == receipt.previous_alias_account_id
|
|
4179
|
+
)
|
|
4180
|
+
|
|
4181
|
+
alias_predicate = (
|
|
4182
|
+
HttpBridgeSessionAlias.session_id == receipt.session_id,
|
|
4183
|
+
HttpBridgeSessionAlias.alias_kind == receipt.alias_kind,
|
|
4184
|
+
HttpBridgeSessionAlias.alias_hash == durable_bridge_hash(receipt.alias_value),
|
|
4185
|
+
HttpBridgeSessionAlias.alias_value == receipt.alias_value,
|
|
4186
|
+
HttpBridgeSessionAlias.api_key_scope == receipt.api_key_scope,
|
|
4187
|
+
)
|
|
4188
|
+
current_alias_session_id = await self._session.scalar(
|
|
4189
|
+
select(HttpBridgeSessionAlias.session_id).where(*alias_predicate).with_for_update()
|
|
4190
|
+
)
|
|
4191
|
+
if current_alias_session_id == receipt.session_id:
|
|
4192
|
+
previous_session_id = receipt.previous_alias_session_id
|
|
4193
|
+
if previous_session_id is None:
|
|
4194
|
+
await self._session.execute(delete(HttpBridgeSessionAlias).where(*alias_predicate))
|
|
4195
|
+
elif previous_session_id != receipt.session_id:
|
|
4196
|
+
if not previous_session_valid:
|
|
4197
|
+
await self._session.execute(delete(HttpBridgeSessionAlias).where(*alias_predicate))
|
|
4198
|
+
else:
|
|
4199
|
+
await self._session.execute(
|
|
4200
|
+
update(HttpBridgeSessionAlias)
|
|
4201
|
+
.where(*alias_predicate)
|
|
4202
|
+
.values(session_id=previous_session_id, updated_at=utcnow())
|
|
4203
|
+
)
|
|
4204
|
+
await self._session.commit()
|
|
4205
|
+
return True
|
|
4206
|
+
|
|
4207
|
+
async def _execute_alias_upsert(
|
|
4208
|
+
self,
|
|
4209
|
+
*,
|
|
4210
|
+
session_id: str,
|
|
4211
|
+
alias_kind: str,
|
|
4212
|
+
alias_value: str,
|
|
4213
|
+
api_key_scope: str,
|
|
4214
|
+
target_account_neutral_replay: bool | None = None,
|
|
4215
|
+
) -> bool:
|
|
4216
|
+
dialect = self._session.get_bind().dialect.name
|
|
4217
|
+
now = utcnow()
|
|
4218
|
+
values = {
|
|
4219
|
+
"session_id": session_id,
|
|
4220
|
+
"alias_kind": alias_kind,
|
|
4221
|
+
"alias_value": alias_value,
|
|
4222
|
+
"alias_hash": durable_bridge_hash(alias_value),
|
|
4223
|
+
"api_key_scope": api_key_scope,
|
|
4224
|
+
}
|
|
4225
|
+
existing_target_is_account_neutral_replay = HttpBridgeSessionAlias.session_id.in_(
|
|
4226
|
+
select(HttpBridgeSessionRecord.id).where(
|
|
4227
|
+
HttpBridgeSessionRecord.session_key_kind == HTTP_BRIDGE_ACCOUNT_NEUTRAL_REPLAY_KIND,
|
|
4228
|
+
HttpBridgeSessionRecord.session_key_value.like(f"{HTTP_BRIDGE_ACCOUNT_NEUTRAL_REPLAY_KEY_PREFIX}%"),
|
|
4229
|
+
HttpBridgeSessionRecord.session_key_value != HTTP_BRIDGE_ACCOUNT_NEUTRAL_REPLAY_KEY_PREFIX,
|
|
4230
|
+
)
|
|
4231
|
+
)
|
|
4232
|
+
existing_target_is_rebindable = HttpBridgeSessionAlias.session_id.in_(
|
|
4233
|
+
select(HttpBridgeSessionRecord.id).where(
|
|
4234
|
+
HttpBridgeSessionRecord.session_key_kind.in_(HTTP_BRIDGE_ACCOUNT_NEUTRAL_REPLAY_REBINDABLE_KINDS),
|
|
4235
|
+
)
|
|
4236
|
+
)
|
|
4237
|
+
existing_target_is_replaceable_recovery = HttpBridgeSessionAlias.session_id.in_(
|
|
4238
|
+
select(HttpBridgeSessionRecord.id).where(
|
|
4239
|
+
HttpBridgeSessionRecord.session_key_kind == HTTP_BRIDGE_ACCOUNT_NEUTRAL_REPLAY_KIND,
|
|
4240
|
+
HttpBridgeSessionRecord.session_key_value.like(f"{HTTP_BRIDGE_ACCOUNT_NEUTRAL_REPLAY_KEY_PREFIX}%"),
|
|
4241
|
+
HttpBridgeSessionRecord.session_key_value != HTTP_BRIDGE_ACCOUNT_NEUTRAL_REPLAY_KEY_PREFIX,
|
|
4242
|
+
or_(
|
|
4243
|
+
HttpBridgeSessionRecord.owner_instance_id.is_(None),
|
|
4244
|
+
HttpBridgeSessionRecord.lease_expires_at.is_(None),
|
|
4245
|
+
HttpBridgeSessionRecord.lease_expires_at <= now,
|
|
4246
|
+
),
|
|
4247
|
+
)
|
|
4248
|
+
)
|
|
4249
|
+
conflict_where = None
|
|
4250
|
+
if target_account_neutral_replay is True:
|
|
4251
|
+
conflict_where = or_(
|
|
4252
|
+
HttpBridgeSessionAlias.session_id == session_id,
|
|
4253
|
+
existing_target_is_replaceable_recovery,
|
|
4254
|
+
existing_target_is_rebindable,
|
|
4255
|
+
)
|
|
4256
|
+
elif target_account_neutral_replay is False:
|
|
4257
|
+
conflict_where = or_(
|
|
4258
|
+
HttpBridgeSessionAlias.session_id == session_id,
|
|
4259
|
+
~existing_target_is_account_neutral_replay,
|
|
4260
|
+
)
|
|
4261
|
+
if dialect == "postgresql":
|
|
4262
|
+
statement = (
|
|
4263
|
+
pg_insert(HttpBridgeSessionAlias)
|
|
4264
|
+
.values(**values)
|
|
4265
|
+
.on_conflict_do_update(
|
|
4266
|
+
index_elements=[
|
|
4267
|
+
HttpBridgeSessionAlias.alias_kind,
|
|
4268
|
+
HttpBridgeSessionAlias.alias_hash,
|
|
4269
|
+
HttpBridgeSessionAlias.api_key_scope,
|
|
4270
|
+
],
|
|
4271
|
+
set_={
|
|
4272
|
+
"session_id": session_id,
|
|
4273
|
+
"alias_value": alias_value,
|
|
4274
|
+
"updated_at": now,
|
|
4275
|
+
},
|
|
4276
|
+
where=conflict_where,
|
|
4277
|
+
)
|
|
4278
|
+
.returning(HttpBridgeSessionAlias.session_id)
|
|
4279
|
+
)
|
|
4280
|
+
elif dialect == "sqlite":
|
|
4281
|
+
statement = (
|
|
4282
|
+
sqlite_insert(HttpBridgeSessionAlias)
|
|
4283
|
+
.values(**values)
|
|
4284
|
+
.on_conflict_do_update(
|
|
4285
|
+
index_elements=[
|
|
4286
|
+
HttpBridgeSessionAlias.alias_kind,
|
|
4287
|
+
HttpBridgeSessionAlias.alias_hash,
|
|
4288
|
+
HttpBridgeSessionAlias.api_key_scope,
|
|
4289
|
+
],
|
|
4290
|
+
set_={
|
|
4291
|
+
"session_id": session_id,
|
|
4292
|
+
"alias_value": alias_value,
|
|
4293
|
+
"updated_at": now,
|
|
4294
|
+
},
|
|
4295
|
+
where=conflict_where,
|
|
4296
|
+
)
|
|
4297
|
+
.returning(HttpBridgeSessionAlias.session_id)
|
|
4298
|
+
)
|
|
4299
|
+
else:
|
|
4300
|
+
raise RuntimeError(f"DurableBridgeRepository alias upsert unsupported for dialect={dialect!r}")
|
|
4301
|
+
result = await self._session.execute(statement)
|
|
4302
|
+
return result.scalar_one_or_none() is not None
|
|
4303
|
+
|
|
4304
|
+
async def _clear_aliases_for_session(self, session_id: str) -> None:
|
|
4305
|
+
await self._session.execute(
|
|
4306
|
+
delete(HttpBridgeSessionAlias).where(HttpBridgeSessionAlias.session_id == session_id)
|
|
4307
|
+
)
|
|
4308
|
+
|
|
4309
|
+
|
|
4310
|
+
async def missing_durable_bridge_tables(session: AsyncSession) -> tuple[str, ...]:
|
|
4311
|
+
dialect = session.get_bind().dialect.name
|
|
4312
|
+
expected = set(REQUIRED_DURABLE_BRIDGE_TABLES)
|
|
4313
|
+
if dialect == "sqlite":
|
|
4314
|
+
result = await session.execute(
|
|
4315
|
+
text(
|
|
4316
|
+
"SELECT name FROM sqlite_master "
|
|
4317
|
+
"WHERE type = 'table' "
|
|
4318
|
+
"AND name IN ('http_bridge_sessions', 'http_bridge_session_aliases', 'http_bridge_retry_circuits', "
|
|
4319
|
+
"'http_bridge_recovery_attempts', 'http_bridge_operations', 'http_bridge_operation_events', "
|
|
4320
|
+
"'http_bridge_operation_event_chunks')"
|
|
4321
|
+
)
|
|
4322
|
+
)
|
|
4323
|
+
else:
|
|
4324
|
+
result = await session.execute(
|
|
4325
|
+
text(
|
|
4326
|
+
"SELECT table_name FROM information_schema.tables "
|
|
4327
|
+
"WHERE table_schema = 'public' "
|
|
4328
|
+
"AND table_name IN ("
|
|
4329
|
+
"'http_bridge_sessions', 'http_bridge_session_aliases', 'http_bridge_retry_circuits', "
|
|
4330
|
+
"'http_bridge_recovery_attempts', 'http_bridge_operations', 'http_bridge_operation_events', "
|
|
4331
|
+
"'http_bridge_operation_event_chunks'"
|
|
4332
|
+
")"
|
|
4333
|
+
)
|
|
4334
|
+
)
|
|
4335
|
+
present = {str(row[0]) for row in result.fetchall()}
|
|
4336
|
+
return tuple(sorted(expected - present))
|
|
4337
|
+
|
|
4338
|
+
|
|
4339
|
+
_SNAPSHOT_COLUMNS = (
|
|
4340
|
+
HttpBridgeSessionRecord.id,
|
|
4341
|
+
HttpBridgeSessionRecord.session_key_kind,
|
|
4342
|
+
HttpBridgeSessionRecord.session_key_value,
|
|
4343
|
+
HttpBridgeSessionRecord.session_key_hash,
|
|
4344
|
+
HttpBridgeSessionRecord.api_key_scope,
|
|
4345
|
+
HttpBridgeSessionRecord.owner_instance_id,
|
|
4346
|
+
HttpBridgeSessionRecord.owner_process_epoch,
|
|
4347
|
+
HttpBridgeSessionRecord.owner_epoch,
|
|
4348
|
+
HttpBridgeSessionRecord.lease_expires_at,
|
|
4349
|
+
HttpBridgeSessionRecord.state,
|
|
4350
|
+
HttpBridgeSessionRecord.account_id,
|
|
4351
|
+
HttpBridgeSessionRecord.model,
|
|
4352
|
+
HttpBridgeSessionRecord.service_tier,
|
|
4353
|
+
HttpBridgeSessionRecord.latest_turn_state,
|
|
4354
|
+
HttpBridgeSessionRecord.latest_response_id,
|
|
4355
|
+
HttpBridgeSessionRecord.latest_input_item_count,
|
|
4356
|
+
HttpBridgeSessionRecord.latest_input_full_fingerprint,
|
|
4357
|
+
HttpBridgeSessionRecord.latest_pending_tool_calls_json,
|
|
4358
|
+
HttpBridgeSessionRecord.last_seen_at,
|
|
4359
|
+
HttpBridgeSessionRecord.closed_at,
|
|
4360
|
+
HttpBridgeSessionRecord.continuity_abandoned_at,
|
|
4361
|
+
HttpBridgeSessionRecord.continuity_abandonment_scope,
|
|
4362
|
+
)
|
|
4363
|
+
|
|
4364
|
+
|
|
4365
|
+
def _returned_row_to_snapshot(row: Row[tuple[object, ...]]) -> DurableBridgeSessionSnapshot:
|
|
4366
|
+
mapping = row._mapping
|
|
4367
|
+
continuity_abandoned = _bridge_continuity_is_abandoned(
|
|
4368
|
+
mapping[HttpBridgeSessionRecord.continuity_abandoned_at],
|
|
4369
|
+
mapping[HttpBridgeSessionRecord.continuity_abandonment_scope],
|
|
4370
|
+
)
|
|
4371
|
+
return DurableBridgeSessionSnapshot(
|
|
4372
|
+
id=mapping[HttpBridgeSessionRecord.id],
|
|
4373
|
+
session_key_kind=mapping[HttpBridgeSessionRecord.session_key_kind],
|
|
4374
|
+
session_key_value=mapping[HttpBridgeSessionRecord.session_key_value],
|
|
4375
|
+
session_key_hash=mapping[HttpBridgeSessionRecord.session_key_hash],
|
|
4376
|
+
api_key_scope=mapping[HttpBridgeSessionRecord.api_key_scope],
|
|
4377
|
+
owner_instance_id=mapping[HttpBridgeSessionRecord.owner_instance_id],
|
|
4378
|
+
owner_process_epoch=mapping[HttpBridgeSessionRecord.owner_process_epoch],
|
|
4379
|
+
owner_epoch=mapping[HttpBridgeSessionRecord.owner_epoch],
|
|
4380
|
+
lease_expires_at=mapping[HttpBridgeSessionRecord.lease_expires_at],
|
|
4381
|
+
state=mapping[HttpBridgeSessionRecord.state],
|
|
4382
|
+
account_id=mapping[HttpBridgeSessionRecord.account_id],
|
|
4383
|
+
model=mapping[HttpBridgeSessionRecord.model],
|
|
4384
|
+
service_tier=mapping[HttpBridgeSessionRecord.service_tier],
|
|
4385
|
+
latest_turn_state=mapping[HttpBridgeSessionRecord.latest_turn_state],
|
|
4386
|
+
latest_response_id=mapping[HttpBridgeSessionRecord.latest_response_id],
|
|
4387
|
+
latest_input_item_count=mapping[HttpBridgeSessionRecord.latest_input_item_count],
|
|
4388
|
+
latest_input_full_fingerprint=mapping[HttpBridgeSessionRecord.latest_input_full_fingerprint],
|
|
4389
|
+
latest_pending_tool_calls=_decode_pending_tool_calls(
|
|
4390
|
+
mapping[HttpBridgeSessionRecord.latest_response_id],
|
|
4391
|
+
mapping[HttpBridgeSessionRecord.latest_pending_tool_calls_json],
|
|
4392
|
+
),
|
|
4393
|
+
last_seen_at=mapping[HttpBridgeSessionRecord.last_seen_at],
|
|
4394
|
+
closed_at=mapping[HttpBridgeSessionRecord.closed_at],
|
|
4395
|
+
continuity_abandoned=continuity_abandoned,
|
|
4396
|
+
abandoned_account_id=(mapping[HttpBridgeSessionRecord.account_id] if continuity_abandoned else None),
|
|
4397
|
+
)
|
|
4398
|
+
|
|
4399
|
+
|
|
4400
|
+
def _bridge_continuity_is_abandoned(
|
|
4401
|
+
abandoned_at: datetime | None,
|
|
4402
|
+
abandonment_scope: str | None,
|
|
4403
|
+
) -> bool:
|
|
4404
|
+
"""Return whether a writer retired this row's continuity owner.
|
|
4405
|
+
|
|
4406
|
+
Mirrors ``sticky_repository._continuity_is_abandoned_for_source``. The
|
|
4407
|
+
bridge has one continuity source, so any scope marker retires the owner;
|
|
4408
|
+
the legacy-timestamp form retires it globally. Scope is written alone so a
|
|
4409
|
+
pre-migration replica, which knows neither column, keeps treating
|
|
4410
|
+
``account_id`` as hard ownership through a rolling deploy.
|
|
4411
|
+
"""
|
|
4412
|
+
return abandonment_scope is not None or abandoned_at is not None
|
|
4413
|
+
|
|
4414
|
+
|
|
4415
|
+
def _to_snapshot(row: HttpBridgeSessionRecord | None) -> DurableBridgeSessionSnapshot | None:
|
|
4416
|
+
if row is None:
|
|
4417
|
+
return None
|
|
4418
|
+
continuity_abandoned = _bridge_continuity_is_abandoned(
|
|
4419
|
+
row.continuity_abandoned_at,
|
|
4420
|
+
row.continuity_abandonment_scope,
|
|
4421
|
+
)
|
|
4422
|
+
return DurableBridgeSessionSnapshot(
|
|
4423
|
+
id=row.id,
|
|
4424
|
+
session_key_kind=row.session_key_kind,
|
|
4425
|
+
session_key_value=row.session_key_value,
|
|
4426
|
+
session_key_hash=row.session_key_hash,
|
|
4427
|
+
api_key_scope=row.api_key_scope,
|
|
4428
|
+
owner_instance_id=row.owner_instance_id,
|
|
4429
|
+
owner_process_epoch=row.owner_process_epoch,
|
|
4430
|
+
owner_epoch=row.owner_epoch,
|
|
4431
|
+
lease_expires_at=row.lease_expires_at,
|
|
4432
|
+
state=row.state,
|
|
4433
|
+
account_id=row.account_id,
|
|
4434
|
+
model=row.model,
|
|
4435
|
+
service_tier=row.service_tier,
|
|
4436
|
+
latest_turn_state=row.latest_turn_state,
|
|
4437
|
+
latest_response_id=row.latest_response_id,
|
|
4438
|
+
latest_input_item_count=row.latest_input_item_count,
|
|
4439
|
+
latest_input_full_fingerprint=row.latest_input_full_fingerprint,
|
|
4440
|
+
latest_pending_tool_calls=_decode_pending_tool_calls(
|
|
4441
|
+
row.latest_response_id,
|
|
4442
|
+
row.latest_pending_tool_calls_json,
|
|
4443
|
+
),
|
|
4444
|
+
last_seen_at=row.last_seen_at,
|
|
4445
|
+
closed_at=row.closed_at,
|
|
4446
|
+
continuity_abandoned=continuity_abandoned,
|
|
4447
|
+
abandoned_account_id=row.account_id if continuity_abandoned else None,
|
|
4448
|
+
)
|
|
4449
|
+
|
|
4450
|
+
|
|
4451
|
+
def _to_snapshot_required(row: HttpBridgeSessionRecord) -> DurableBridgeSessionSnapshot:
|
|
4452
|
+
snapshot = _to_snapshot(row)
|
|
4453
|
+
if snapshot is None:
|
|
4454
|
+
raise RuntimeError("Expected durable bridge session snapshot")
|
|
4455
|
+
return snapshot
|
|
4456
|
+
|
|
4457
|
+
|
|
4458
|
+
def _to_recovery_attempt_snapshot(
|
|
4459
|
+
row: HttpBridgeRecoveryAttemptRecord,
|
|
4460
|
+
) -> DurableBridgeRecoveryAttemptSnapshot:
|
|
4461
|
+
return DurableBridgeRecoveryAttemptSnapshot(
|
|
4462
|
+
session_id=row.session_id,
|
|
4463
|
+
request_fingerprint=row.request_fingerprint,
|
|
4464
|
+
request_id=row.request_id,
|
|
4465
|
+
account_id=row.account_id,
|
|
4466
|
+
model=row.model,
|
|
4467
|
+
replay_safe=bool(row.replay_safe),
|
|
4468
|
+
state=row.state,
|
|
4469
|
+
response_id=row.response_id,
|
|
4470
|
+
)
|
|
4471
|
+
|
|
4472
|
+
|
|
4473
|
+
def _to_operation_snapshot(
|
|
4474
|
+
row: HttpBridgeOperationRecord,
|
|
4475
|
+
*,
|
|
4476
|
+
created: bool = False,
|
|
4477
|
+
rebound: bool = False,
|
|
4478
|
+
rebound_from_session_id: str | None = None,
|
|
4479
|
+
rebound_from_account_id: str | None = None,
|
|
4480
|
+
rebound_from_model: str | None = None,
|
|
4481
|
+
rebound_from_parent_response_id: str | None = None,
|
|
4482
|
+
) -> DurableBridgeOperationSnapshot:
|
|
4483
|
+
return DurableBridgeOperationSnapshot(
|
|
4484
|
+
operation_id=row.operation_id,
|
|
4485
|
+
session_id=row.session_id,
|
|
4486
|
+
request_fingerprint=row.request_fingerprint,
|
|
4487
|
+
account_id=row.account_id,
|
|
4488
|
+
model=row.model,
|
|
4489
|
+
parent_response_id=row.parent_response_id,
|
|
4490
|
+
state=row.state,
|
|
4491
|
+
response_id=row.response_id,
|
|
4492
|
+
recovery_dispatch_count=row.recovery_dispatch_count,
|
|
4493
|
+
request_text=row.request_text,
|
|
4494
|
+
event_spool_complete=bool(row.event_spool_complete),
|
|
4495
|
+
created=created,
|
|
4496
|
+
rebound=rebound,
|
|
4497
|
+
rebound_from_session_id=rebound_from_session_id,
|
|
4498
|
+
rebound_from_account_id=rebound_from_account_id,
|
|
4499
|
+
rebound_from_model=rebound_from_model,
|
|
4500
|
+
rebound_from_parent_response_id=rebound_from_parent_response_id,
|
|
4501
|
+
)
|
|
4502
|
+
|
|
4503
|
+
|
|
4504
|
+
def _to_retry_circuit_snapshot(row: HttpBridgeRetryCircuit | None) -> DurableBridgeRetryCircuitSnapshot | None:
|
|
4505
|
+
if row is None:
|
|
4506
|
+
return None
|
|
4507
|
+
return DurableBridgeRetryCircuitSnapshot(
|
|
4508
|
+
session_key_kind=row.session_key_kind,
|
|
4509
|
+
session_key_hash=row.session_key_hash,
|
|
4510
|
+
api_key_scope=row.api_key_scope,
|
|
4511
|
+
consecutive_failures=row.consecutive_failures,
|
|
4512
|
+
cooldown_until_epoch=row.cooldown_until_epoch,
|
|
4513
|
+
last_detail=row.last_detail,
|
|
4514
|
+
updated_at_epoch=row.updated_at_epoch,
|
|
4515
|
+
admission_generation=row.admission_generation,
|
|
4516
|
+
)
|