codexLB-up 1.25.0b10__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- app/__init__.py +10 -0
- app/admin_cli.py +494 -0
- app/cli.py +339 -0
- app/codex_sessions_retag.py +487 -0
- app/core/__init__.py +0 -0
- app/core/audit/__init__.py +0 -0
- app/core/audit/service.py +216 -0
- app/core/audit/types.py +94 -0
- app/core/auth/__init__.py +236 -0
- app/core/auth/api_key_cache.py +65 -0
- app/core/auth/dashboard_access.py +429 -0
- app/core/auth/dashboard_mode.py +171 -0
- app/core/auth/dashboard_session_ttl.py +74 -0
- app/core/auth/dashboard_users_cache.py +120 -0
- app/core/auth/dependencies.py +710 -0
- app/core/auth/external_identity.py +78 -0
- app/core/auth/guardian.py +374 -0
- app/core/auth/models.py +49 -0
- app/core/auth/providers/__init__.py +129 -0
- app/core/auth/providers/oidc.py +838 -0
- app/core/auth/providers/registry.py +118 -0
- app/core/auth/refresh.py +394 -0
- app/core/auth/step_up.py +117 -0
- app/core/auth/totp.py +81 -0
- app/core/balancer/__init__.py +71 -0
- app/core/balancer/logic.py +1612 -0
- app/core/balancer/rendezvous_hash.py +21 -0
- app/core/balancer/types.py +21 -0
- app/core/bootstrap.py +172 -0
- app/core/cache/invalidation.py +426 -0
- app/core/clients/__init__.py +0 -0
- app/core/clients/codex.py +996 -0
- app/core/clients/codex_version.py +187 -0
- app/core/clients/codex_version_snapshot.py +2 -0
- app/core/clients/files.py +534 -0
- app/core/clients/headers.py +29 -0
- app/core/clients/http.py +511 -0
- app/core/clients/model_fetcher.py +276 -0
- app/core/clients/native_egress.py +1304 -0
- app/core/clients/oauth.py +477 -0
- app/core/clients/proxy.py +6089 -0
- app/core/clients/proxy_websocket.py +1411 -0
- app/core/clients/rate_limit_reset_credits.py +412 -0
- app/core/clients/stream_errors.py +12 -0
- app/core/clients/thread_cache_identity.py +347 -0
- app/core/clients/usage.py +467 -0
- app/core/clock.py +170 -0
- app/core/config/__init__.py +0 -0
- app/core/config/background_jobs.py +73 -0
- app/core/config/context_window_overrides.py +166 -0
- app/core/config/dashboard_overrides.py +145 -0
- app/core/config/inheritable.py +47 -0
- app/core/config/key_fingerprint.py +115 -0
- app/core/config/settings.py +872 -0
- app/core/config/settings_cache.py +100 -0
- app/core/config/spool_retention.py +217 -0
- app/core/config/tiers.py +174 -0
- app/core/conversation_archive.py +568 -0
- app/core/crypto.py +37 -0
- app/core/draining.py +11 -0
- app/core/errors.py +365 -0
- app/core/exceptions.py +155 -0
- app/core/handlers/__init__.py +3 -0
- app/core/handlers/exceptions.py +478 -0
- app/core/http_protocol.py +176 -0
- app/core/http_protocol_httptools.py +204 -0
- app/core/ingress_limits.py +17 -0
- app/core/metadata_store.py +18 -0
- app/core/metrics/__init__.py +0 -0
- app/core/metrics/middleware.py +74 -0
- app/core/metrics/prometheus.py +604 -0
- app/core/middleware/__init__.py +25 -0
- app/core/middleware/api_firewall.py +111 -0
- app/core/middleware/app_version.py +37 -0
- app/core/middleware/dashboard_auth_proxy.py +55 -0
- app/core/middleware/dashboard_csrf.py +177 -0
- app/core/middleware/dashboard_gzip.py +41 -0
- app/core/middleware/dashboard_overrides.py +59 -0
- app/core/middleware/firewall_cache.py +91 -0
- app/core/middleware/inflight.py +126 -0
- app/core/middleware/multipart_content_encoding.py +129 -0
- app/core/middleware/path_rewrite.py +148 -0
- app/core/middleware/request_body_limit.py +196 -0
- app/core/middleware/request_decompression.py +209 -0
- app/core/middleware/request_id.py +72 -0
- app/core/middleware/required_capability_http.py +116 -0
- app/core/middleware/trusted_proxy_headers.py +36 -0
- app/core/multipart.py +237 -0
- app/core/multipart_fields.py +91 -0
- app/core/openai/__init__.py +0 -0
- app/core/openai/chat_requests.py +493 -0
- app/core/openai/chat_responses.py +931 -0
- app/core/openai/contracts.py +46 -0
- app/core/openai/exceptions.py +24 -0
- app/core/openai/host_models.py +14 -0
- app/core/openai/images.py +413 -0
- app/core/openai/message_coercion.py +376 -0
- app/core/openai/model_refresh_scheduler.py +423 -0
- app/core/openai/model_registry.py +997 -0
- app/core/openai/model_registry_store.py +548 -0
- app/core/openai/models.py +172 -0
- app/core/openai/parsing.py +97 -0
- app/core/openai/requests.py +1980 -0
- app/core/openai/strict_schema.py +199 -0
- app/core/openai/tool_call_safety.py +99 -0
- app/core/openai/v1_requests.py +187 -0
- app/core/plan_types.py +90 -0
- app/core/prestop.py +202 -0
- app/core/rate_limiter/__init__.py +0 -0
- app/core/rate_limiter/db_rate_limiter.py +215 -0
- app/core/request_locality.py +473 -0
- app/core/resilience/__init__.py +0 -0
- app/core/resilience/backpressure.py +57 -0
- app/core/resilience/bulkhead.py +202 -0
- app/core/resilience/circuit_breaker.py +150 -0
- app/core/resilience/degradation.py +55 -0
- app/core/resilience/loop_lag_monitor.py +60 -0
- app/core/resilience/memory_monitor.py +137 -0
- app/core/resilience/network_recovery.py +256 -0
- app/core/resilience/overload.py +98 -0
- app/core/resilience/toggles.py +114 -0
- app/core/retention/__init__.py +0 -0
- app/core/retention/job.py +240 -0
- app/core/retention/scheduler.py +69 -0
- app/core/runtime_logging.py +601 -0
- app/core/scheduling/__init__.py +0 -0
- app/core/scheduling/leader_election.py +993 -0
- app/core/scheduling/leader_election_handle.py +25 -0
- app/core/server.py +140 -0
- app/core/shutdown.py +326 -0
- app/core/socket_peer.py +22 -0
- app/core/startup.py +41 -0
- app/core/timeout_invariants.py +347 -0
- app/core/tracing/__init__.py +0 -0
- app/core/tracing/otel.py +207 -0
- app/core/types.py +6 -0
- app/core/upstream_proxy/__init__.py +13 -0
- app/core/upstream_proxy/cache.py +130 -0
- app/core/upstream_proxy/resolver.py +189 -0
- app/core/upstream_proxy/types.py +104 -0
- app/core/usage/__init__.py +407 -0
- app/core/usage/depletion.py +109 -0
- app/core/usage/live_hub.py +41 -0
- app/core/usage/live_snapshots.py +174 -0
- app/core/usage/logs.py +212 -0
- app/core/usage/metadata_scheduler.py +118 -0
- app/core/usage/models.py +54 -0
- app/core/usage/pricing.py +576 -0
- app/core/usage/pricing_catalog.py +241 -0
- app/core/usage/pricing_snapshot.json +821 -0
- app/core/usage/quota.py +133 -0
- app/core/usage/refresh_policy.py +21 -0
- app/core/usage/refresh_scheduler.py +713 -0
- app/core/usage/reset_credits_refresh_scheduler.py +435 -0
- app/core/usage/types.py +168 -0
- app/core/utils/__init__.py +0 -0
- app/core/utils/json_guards.py +18 -0
- app/core/utils/locks.py +33 -0
- app/core/utils/masking.py +20 -0
- app/core/utils/proxy_env.py +104 -0
- app/core/utils/request_id.py +65 -0
- app/core/utils/retry.py +77 -0
- app/core/utils/shared_future.py +172 -0
- app/core/utils/sse.py +286 -0
- app/core/utils/time.py +24 -0
- app/db/__init__.py +0 -0
- app/db/account_identity_lock.py +56 -0
- app/db/alembic/__init__.py +1 -0
- app/db/alembic/env.py +70 -0
- app/db/alembic/revision_ids.py +43 -0
- app/db/alembic/versions/20260213_000000_base_schema.py +248 -0
- app/db/alembic/versions/20260213_000100_normalize_account_plan_types.py +45 -0
- app/db/alembic/versions/20260213_000200_add_request_logs_reasoning_effort.py +45 -0
- app/db/alembic/versions/20260213_000300_add_accounts_reset_at.py +45 -0
- app/db/alembic/versions/20260213_000400_add_accounts_chatgpt_account_id.py +45 -0
- app/db/alembic/versions/20260213_000500_add_dashboard_settings.py +70 -0
- app/db/alembic/versions/20260213_000600_add_dashboard_settings_totp.py +64 -0
- app/db/alembic/versions/20260213_000700_add_dashboard_settings_password.py +59 -0
- app/db/alembic/versions/20260213_000800_add_api_keys.py +87 -0
- app/db/alembic/versions/20260214_000000_add_api_key_limits.py +143 -0
- app/db/alembic/versions/20260215_000000_add_idx_logs_requested_at.py +38 -0
- app/db/alembic/versions/20260218_000000_add_api_key_usage_reservations.py +117 -0
- app/db/alembic/versions/20260218_000100_add_import_without_overwrite_and_drop_accounts_email_unique.py +198 -0
- app/db/alembic/versions/20260225_000000_add_dashboard_settings_routing_strategy.py +53 -0
- app/db/alembic/versions/20260228_020000_align_api_key_limit_enum_types.py +104 -0
- app/db/alembic/versions/20260228_030000_add_api_firewall_allowlist.py +42 -0
- app/db/alembic/versions/20260306_000000_add_request_logs_service_tier.py +45 -0
- app/db/alembic/versions/20260307_000000_add_api_key_enforcement_fields.py +56 -0
- app/db/alembic/versions/20260308_000000_add_sqlite_performance_indexes.py +41 -0
- app/db/alembic/versions/20260309_000000_add_additional_usage_history.py +71 -0
- app/db/alembic/versions/20260309_000000_request_logs_nullable_account_id.py +28 -0
- app/db/alembic/versions/20260310_000000_add_request_logs_transport.py +45 -0
- app/db/alembic/versions/20260310_000000_fix_postgresql_enum_value_casing.py +79 -0
- app/db/alembic/versions/20260310_120000_add_sticky_session_kinds_and_affinity_ttl.py +110 -0
- app/db/alembic/versions/20260312_000000_add_additional_usage_quota_key.py +193 -0
- app/db/alembic/versions/20260312_000000_split_sticky_sessions_primary_key_by_kind.py +96 -0
- app/db/alembic/versions/20260312_010000_merge_additional_usage_and_sticky_session_heads.py +25 -0
- app/db/alembic/versions/20260312_020000_merge_request_logs_transport_and_feature_heads.py +25 -0
- app/db/alembic/versions/20260312_020000_merge_request_logs_transport_and_sticky_heads.py +25 -0
- app/db/alembic/versions/20260312_120000_add_dashboard_upstream_stream_transport.py +62 -0
- app/db/alembic/versions/20260313_024500_merge_request_log_transport_heads.py +27 -0
- app/db/alembic/versions/20260319_100937_increase_prompt_cache_ttl_to_1800s.py +73 -0
- app/db/alembic/versions/20260319_111500_merge_prompt_cache_ttl_and_upstream_transport_heads.py +25 -0
- app/db/alembic/versions/20260319_183000_normalize_sqlite_account_status_casing.py +48 -0
- app/db/alembic/versions/20260320_000000_add_request_log_requested_actual_tiers.py +51 -0
- app/db/alembic/versions/20260321_190000_tighten_dashboard_db_indexes.py +64 -0
- app/db/alembic/versions/20260321_210000_merge_request_log_tiers_and_dashboard_index_heads.py +25 -0
- app/db/alembic/versions/20260325_000000_add_request_log_cost.py +193 -0
- app/db/alembic/versions/20260328_000000_add_rate_limit_attempts.py +48 -0
- app/db/alembic/versions/20260328_120000_add_request_log_ttft.py +37 -0
- app/db/alembic/versions/20260328_130000_add_audit_logs_table.py +45 -0
- app/db/alembic/versions/20260328_140000_add_scheduler_leader_table.py +39 -0
- app/db/alembic/versions/20260330_000000_add_cache_locality_settings.py +69 -0
- app/db/alembic/versions/20260330_010000_merge_scheduler_leader_and_cache_locality_heads.py +25 -0
- app/db/alembic/versions/20260330_020000_add_bridge_ring_members.py +46 -0
- app/db/alembic/versions/20260401_000000_add_api_key_enforced_service_tier.py +52 -0
- app/db/alembic/versions/20260401_000000_add_cache_invalidation.py +41 -0
- app/db/alembic/versions/20260401_020000_merge_cache_invalidation_and_api_key_service_tier_heads.py +22 -0
- app/db/alembic/versions/20260402_000000_switch_dashboard_routing_default_to_capacity_weighted.py +66 -0
- app/db/alembic/versions/20260403_000000_add_credit_api_key_limit_values.py +44 -0
- app/db/alembic/versions/20260406_000000_add_api_key_account_assignments.py +57 -0
- app/db/alembic/versions/20260406_010000_add_api_key_assignment_scope_flag.py +75 -0
- app/db/alembic/versions/20260407_000000_add_bridge_gateway_safe_mode.py +51 -0
- app/db/alembic/versions/20260407_010000_merge_api_key_assignment_and_bridge_gateway_heads.py +25 -0
- app/db/alembic/versions/20260408_000000_switch_import_without_overwrite_default_to_true.py +73 -0
- app/db/alembic/versions/20260408_010000_merge_import_without_overwrite_and_assignment_heads.py +25 -0
- app/db/alembic/versions/20260409_000000_add_http_bridge_sessions.py +130 -0
- app/db/alembic/versions/20260409_000000_switch_sticky_threads_and_prefer_earlier_reset_defaults_to_true.py +89 -0
- app/db/alembic/versions/20260409_010000_add_dashboard_settings_bootstrap_token.py +57 -0
- app/db/alembic/versions/20260409_010000_merge_http_bridge_and_import_without_overwrite_heads.py +25 -0
- app/db/alembic/versions/20260409_020000_fix_http_bridge_last_seen_index.py +46 -0
- app/db/alembic/versions/20260410_000000_backfill_pristine_dashboard_settings_defaults.py +77 -0
- app/db/alembic/versions/20260410_030000_restore_import_without_overwrite_default_true.py +73 -0
- app/db/alembic/versions/20260410_040000_merge_dashboard_defaults_and_import_default_heads.py +25 -0
- app/db/alembic/versions/20260413_000000_add_accounts_blocked_at.py +44 -0
- app/db/alembic/versions/20260415_160000_add_automations_tables.py +189 -0
- app/db/alembic/versions/20260415_160000_add_request_logs_response_lookup_index.py +67 -0
- app/db/alembic/versions/20260417_000000_add_request_log_plan_type.py +44 -0
- app/db/alembic/versions/20260418_190000_add_automation_runs_cycle_key.py +87 -0
- app/db/alembic/versions/20260419_000000_add_automation_run_cycle_metadata.py +56 -0
- app/db/alembic/versions/20260419_020000_add_automation_run_cycles_snapshot_tables.py +342 -0
- app/db/alembic/versions/20260421_000000_add_dashboard_session_ttl_setting.py +51 -0
- app/db/alembic/versions/20260421_120000_merge_request_log_lookup_and_plan_type_heads.py +25 -0
- app/db/alembic/versions/20260421_130000_merge_automation_and_request_log_heads.py +25 -0
- app/db/alembic/versions/20260422_103000_normalize_legacy_automation_run_cycle_keys.py +648 -0
- app/db/alembic/versions/20260423_120000_add_api_key_limit_reset_at_index.py +43 -0
- app/db/alembic/versions/20260424_000000_merge_dashboard_session_ttl_and_request_log_heads.py +25 -0
- app/db/alembic/versions/20260426_000000_add_dashboard_relative_availability_settings.py +68 -0
- app/db/alembic/versions/20260427_130000_add_warmup_model_and_request_kind.py +75 -0
- app/db/alembic/versions/20260506_000000_add_account_routing_policy.py +37 -0
- app/db/alembic/versions/20260509_000000_add_api_key_traffic_class.py +37 -0
- app/db/alembic/versions/20260509_010000_add_additional_quota_routing_policies.py +37 -0
- app/db/alembic/versions/20260513_000000_add_accounts_alias.py +44 -0
- app/db/alembic/versions/20260513_000000_add_api_key_apply_to_codex_model.py +59 -0
- app/db/alembic/versions/20260514_000000_add_request_logs_api_key_time_index.py +43 -0
- app/db/alembic/versions/20260515_000000_add_prefer_earlier_reset_window.py +47 -0
- app/db/alembic/versions/20260515_000000_soft_delete_request_logs_on_account_delete.py +83 -0
- app/db/alembic/versions/20260515_020000_add_split_sticky_budget_thresholds.py +62 -0
- app/db/alembic/versions/20260516_000000_add_sqlite_hot_path_indexes.py +75 -0
- app/db/alembic/versions/20260517_000000_merge_sqlite_hot_path_and_request_log_delete_heads.py +24 -0
- app/db/alembic/versions/20260518_000000_add_http_bridge_durable_input_prefix.py +44 -0
- app/db/alembic/versions/20260518_010000_merge_http_bridge_and_sqlite_recovery_heads.py +25 -0
- app/db/alembic/versions/20260518_120000_add_dashboard_guest_access.py +57 -0
- app/db/alembic/versions/20260520_000000_merge_api_key_and_http_bridge_heads.py +26 -0
- app/db/alembic/versions/20260520_010000_add_request_logs_api_key_account_index.py +45 -0
- app/db/alembic/versions/20260520_010000_merge_automation_and_api_key_heads.py +26 -0
- app/db/alembic/versions/20260520_030000_add_quota_planner.py +194 -0
- app/db/alembic/versions/20260520_040000_merge_quota_planner_and_request_log_index_heads.py +24 -0
- app/db/alembic/versions/20260521_000000_add_account_security_work_authorized.py +46 -0
- app/db/alembic/versions/20260521_000000_merge_dashboard_guest_and_api_key_heads.py +26 -0
- app/db/alembic/versions/20260521_000000_merge_prefer_reset_window_head.py +26 -0
- app/db/alembic/versions/20260521_000000_merge_warmup_and_request_log_index_heads.py +25 -0
- app/db/alembic/versions/20260522_000000_add_limit_warmup_trigger.py +195 -0
- app/db/alembic/versions/20260524_000000_merge_accounts_alias_and_reset_window_heads.py +24 -0
- app/db/alembic/versions/20260525_000000_add_usage_raw_window_latest_index.py +31 -0
- app/db/alembic/versions/20260526_000000_add_request_log_failure_metadata.py +100 -0
- app/db/alembic/versions/20260531_000000_add_account_workspace_identity.py +39 -0
- app/db/alembic/versions/20260531_000000_add_single_account_routing.py +42 -0
- app/db/alembic/versions/20260601_000000_merge_relative_availability_and_usage_raw_heads.py +24 -0
- app/db/alembic/versions/20260601_010000_add_api_key_usage_sections.py +58 -0
- app/db/alembic/versions/20260601_010000_add_routing_policy_persistence.py +67 -0
- app/db/alembic/versions/20260601_010000_merge_security_work_and_relative_availability_heads.py +25 -0
- app/db/alembic/versions/20260601_010000_merge_split_sticky_and_relative_usage_heads.py +25 -0
- app/db/alembic/versions/20260601_010000_merge_warmup_and_relative_availability_heads.py +25 -0
- app/db/alembic/versions/20260601_020000_merge_additional_quota_routing_and_relative_availability_heads.py +25 -0
- app/db/alembic/versions/20260601_020000_merge_quota_planner_and_relative_availability_heads.py +25 -0
- app/db/alembic/versions/20260601_020000_merge_reset_window_and_relative_availability_heads.py +25 -0
- app/db/alembic/versions/20260601_020000_merge_routing_policy_and_split_threshold_heads.py +24 -0
- app/db/alembic/versions/20260601_020000_merge_warmup_and_request_log_failure_heads.py +25 -0
- app/db/alembic/versions/20260601_030000_merge_single_account_and_routing_quota_heads.py +29 -0
- app/db/alembic/versions/20260601_040000_merge_security_work_and_routing_consolidation_heads.py +25 -0
- app/db/alembic/versions/20260601_050000_merge_quota_planner_and_routing_consolidation_heads.py +25 -0
- app/db/alembic/versions/20260602_050000_add_upstream_proxy_routing.py +164 -0
- app/db/alembic/versions/20260602_060000_merge_account_workspace_and_failure_heads.py +25 -0
- app/db/alembic/versions/20260602_070000_add_request_log_useragent_fields.py +86 -0
- app/db/alembic/versions/20260602_070000_merge_quota_planner_and_account_workspace_heads.py +25 -0
- app/db/alembic/versions/20260602_080000_merge_upstream_proxy_and_quota_planner_heads.py +25 -0
- app/db/alembic/versions/20260602_080000_merge_useragent_and_upstream_proxy_heads.py +24 -0
- app/db/alembic/versions/20260603_000000_add_weekly_pace_working_days.py +52 -0
- app/db/alembic/versions/20260603_000000_free_account_monthly_window.py +60 -0
- app/db/alembic/versions/20260604_000000_add_reauth_required_account_status.py +159 -0
- app/db/alembic/versions/20260604_010000_merge_automations_and_reauth_status_heads.py +25 -0
- app/db/alembic/versions/20260604_010000_merge_dashboard_guest_and_reauth_status_heads.py +25 -0
- app/db/alembic/versions/20260604_010000_merge_useragent_and_reauth_heads.py +24 -0
- app/db/alembic/versions/20260607_000000_merge_weekly_monthly_useragent_heads.py +28 -0
- app/db/alembic/versions/20260608_000000_add_hide_upstream_quota_from_api_keys.py +50 -0
- app/db/alembic/versions/20260611_000000_add_request_log_archive_request_id.py +47 -0
- app/db/alembic/versions/20260611_000000_merge_automations_and_weekly_monthly_useragent_heads.py +26 -0
- app/db/alembic/versions/20260611_000000_merge_dashboard_guest_and_weekly_useragent_heads.py +26 -0
- app/db/alembic/versions/20260611_010000_add_request_log_client_ip.py +57 -0
- app/db/alembic/versions/20260611_020000_merge_request_log_archive_and_client_ip_heads.py +26 -0
- app/db/alembic/versions/20260611_030000_merge_dashboard_guest_and_request_log_client_ip_heads.py +26 -0
- app/db/alembic/versions/20260613_000000_add_accounts_codex_installation_id.py +57 -0
- app/db/alembic/versions/20260613_010000_merge_installation_id_and_dashboard_weekly_heads.py +26 -0
- app/db/alembic/versions/20260615_000000_merge_automations_and_dashboard_guest_heads.py +26 -0
- app/db/alembic/versions/20260623_000000_add_limit_warmup_exhausted_threshold.py +51 -0
- app/db/alembic/versions/20260626_000000_add_http_downstream_transport_policy.py +46 -0
- app/db/alembic/versions/20260626_010000_add_request_logs_upstream_transport.py +31 -0
- app/db/alembic/versions/20260629_000000_add_dashboard_query_hot_path_indexes.py +53 -0
- app/db/alembic/versions/20260630_000000_add_limit_warmup_staggered_idle.py +47 -0
- app/db/alembic/versions/20260630_000000_merge_request_log_client_ip_and_dashboard_hot_path_heads.py +26 -0
- app/db/alembic/versions/20260630_010000_merge_warmup_and_request_log_dashboard_heads.py +26 -0
- app/db/alembic/versions/20260630_020000_merge_automations_and_main_heads.py +26 -0
- app/db/alembic/versions/20260630_020000_merge_warmup_threshold_and_main_heads.py +26 -0
- app/db/alembic/versions/20260630_030000_add_automation_run_model_snapshot.py +59 -0
- app/db/alembic/versions/20260630_040000_merge_automation_snapshot_and_warmup_threshold_heads.py +26 -0
- app/db/alembic/versions/20260630_050000_add_automation_run_prompt_snapshot.py +52 -0
- app/db/alembic/versions/20260701_000000_add_weekly_pace_smoothing_minutes.py +51 -0
- app/db/alembic/versions/20260702_000000_add_model_sources.py +176 -0
- app/db/alembic/versions/20260705_000000_add_model_source_audio_transcriptions.py +50 -0
- app/db/alembic/versions/20260705_000000_harden_dashboard_session_ttl.py +62 -0
- app/db/alembic/versions/20260706_000000_add_model_source_audio_per_minute.py +43 -0
- app/db/alembic/versions/20260707_000000_merge_model_sources_and_session_ttl.py +24 -0
- app/db/alembic/versions/20260707_010000_merge_automations_and_model_sources_heads.py +24 -0
- app/db/alembic/versions/20260707_020000_add_automation_job_account_scope.py +125 -0
- app/db/alembic/versions/20260709_000000_add_ttft_phase_observability.py +66 -0
- app/db/alembic/versions/20260711_000000_add_dashboard_prohibit_fast_mode.py +42 -0
- app/db/alembic/versions/20260711_010000_add_account_chatgpt_user_id.py +40 -0
- app/db/alembic/versions/20260711_020000_add_dashboard_account_concurrency_caps.py +71 -0
- app/db/alembic/versions/20260711_030000_add_limit_warmup_idle_threshold.py +51 -0
- app/db/alembic/versions/20260712_010000_add_account_usage_rollups.py +56 -0
- app/db/alembic/versions/20260712_020000_add_api_key_usage_rollups.py +53 -0
- app/db/alembic/versions/20260713_020000_add_model_registry_snapshot.py +42 -0
- app/db/alembic/versions/20260713_040000_add_account_refresh_claims.py +42 -0
- app/db/alembic/versions/20260713_040000_add_replica_guardrails.py +64 -0
- app/db/alembic/versions/20260713_070000_add_reset_credit_redeem_tables.py +58 -0
- app/db/alembic/versions/20260714_000000_add_oauth_flow_states.py +63 -0
- app/db/alembic/versions/20260715_000000_add_request_log_queue_latency.py +44 -0
- app/db/alembic/versions/20260716_000000_add_oauth_device_flow_slots.py +41 -0
- app/db/alembic/versions/20260716_010000_add_dashboard_retention_settings.py +48 -0
- app/db/alembic/versions/20260716_010000_add_reset_credit_display_settings.py +73 -0
- app/db/alembic/versions/20260717_000000_add_http_bridge_retry_circuits.py +47 -0
- app/db/alembic/versions/20260717_000000_merge_retention_and_reset_credit_display_heads.py +26 -0
- app/db/alembic/versions/20260717_000000_optimize_dashboard_hot_path_indexes.py +162 -0
- app/db/alembic/versions/20260717_000001_merge_retry_circuits_and_dashboard_indexes.py +22 -0
- app/db/alembic/versions/20260720_000000_add_request_log_conversation_id.py +57 -0
- app/db/alembic/versions/20260722_000000_backfill_request_log_useragent_families.py +40 -0
- app/db/alembic/versions/20260724_000000_add_request_usage_time_rollups.py +176 -0
- app/db/alembic/versions/20260724_000000_merge_request_log_schema_heads.py +33 -0
- app/db/alembic/versions/20260725_000000_add_http_bridge_pending_tool_calls.py +43 -0
- app/db/alembic/versions/20260726_000000_add_account_plan_downgrade_observations.py +44 -0
- app/db/alembic/versions/20260726_000000_repair_request_usage_rollups_after_merge.py +37 -0
- app/db/alembic/versions/20260727_000000_add_sticky_session_continuity_abandoned_at.py +45 -0
- app/db/alembic/versions/20260728_000000_merge_pending_tool_calls_and_rollup_repair_heads.py +31 -0
- app/db/alembic/versions/20260730_000000_add_api_key_fair_share_threshold.py +44 -0
- app/db/alembic/versions/20260730_000000_add_http_bridge_recovery_attempts.py +60 -0
- app/db/alembic/versions/20260731_000000_add_capability_lineage_markers.py +38 -0
- app/db/alembic/versions/20260803_000000_merge_http_bridge_recovery_and_capability_lineage_heads.py +30 -0
- app/db/alembic/versions/20260804_000000_add_http_bridge_operations.py +63 -0
- app/db/alembic/versions/20260804_000001_add_global_http_bridge_operation_fingerprint.py +53 -0
- app/db/alembic/versions/20260804_230000_add_request_log_connection_request_kind.py +43 -0
- app/db/alembic/versions/20260805_000000_add_http_bridge_operation_spool.py +75 -0
- app/db/alembic/versions/20260805_000001_finalize_http_bridge_operation_spool.py +65 -0
- app/db/alembic/versions/20260806_000000_add_additional_usage_alias_probe_indexes.py +84 -0
- app/db/alembic/versions/20260806_000000_add_anonymous_telemetry.py +49 -0
- app/db/alembic/versions/20260806_010000_add_conversation_presence_rollup.py +72 -0
- app/db/alembic/versions/20260806_020000_add_usage_history_bulk_covering_indexes.py +99 -0
- app/db/alembic/versions/20260806_030000_add_api_key_allowed_reasoning_efforts.py +65 -0
- app/db/alembic/versions/20260806_120000_add_http_bridge_owner_process_epoch.py +57 -0
- app/db/alembic/versions/20260807_000000_merge_http_bridge_operation_spool_and_latest_main.py +24 -0
- app/db/alembic/versions/20260808_000000_tune_usage_history_autovacuum.py +54 -0
- app/db/alembic/versions/20260810_000000_add_http_bridge_recovery_dispatch_count.py +44 -0
- app/db/alembic/versions/20260811_000000_add_hourly_rollup_cancelled_count.py +92 -0
- app/db/alembic/versions/20260812_000000_merge_recovery_dispatch_and_hourly_cancelled_heads.py +25 -0
- app/db/alembic/versions/20260812_120000_add_sticky_abandonment_scope.py +43 -0
- app/db/alembic/versions/20260813_000000_add_file_account_pins.py +40 -0
- app/db/alembic/versions/20260816_000000_add_account_pending_deletion.py +81 -0
- app/db/alembic/versions/20260816_000000_add_model_source_embeddings.py +50 -0
- app/db/alembic/versions/20260821_000000_add_retry_circuit_admission_generation.py +49 -0
- app/db/alembic/versions/20260826_000000_add_http_bridge_event_chunks.py +107 -0
- app/db/alembic/versions/20260828_000000_add_accounts_chatgpt_identity_index.py +55 -0
- app/db/alembic/versions/20260830_000000_add_quota_warmup_claim_expiry.py +83 -0
- app/db/alembic/versions/20260908_000000_add_guest_session_generation.py +37 -0
- app/db/alembic/versions/20260908_000000_add_subscription_overflow.py +140 -0
- app/db/alembic/versions/20260908_000000_replace_upstream_stream_transport_default_sentinel.py +50 -0
- app/db/alembic/versions/20260908_020000_merge_overflow_transport_heads.py +27 -0
- app/db/alembic/versions/20260909_000000_add_dashboard_roles.py +72 -0
- app/db/alembic/versions/20260909_010000_add_dashboard_users.py +183 -0
- app/db/alembic/versions/20260909_020000_reproject_compat_admin_credentials.py +105 -0
- app/db/alembic/versions/20260909_030000_add_audit_actor_columns.py +103 -0
- app/db/alembic/versions/20260909_030000_dashboard_resilience_toggle_settings.py +63 -0
- app/db/alembic/versions/20260909_040000_add_dashboard_user_invites.py +54 -0
- app/db/alembic/versions/20260909_040000_dashboard_timeout_settings.py +68 -0
- app/db/alembic/versions/20260909_050000_dashboard_routing_overload_settings.py +65 -0
- app/db/alembic/versions/20260909_060000_add_report_rollup.py +74 -0
- app/db/alembic/versions/20260909_070000_automation_run_claim_budget.py +50 -0
- app/db/alembic/versions/20260909_080000_dashboard_stream_bridge_budgets.py +62 -0
- app/db/alembic/versions/20260909_090000_dashboard_background_job_toggles.py +63 -0
- app/db/alembic/versions/20260909_100000_dashboard_codex_prewarm.py +49 -0
- app/db/alembic/versions/20260909_110000_model_context_window_overrides.py +45 -0
- app/db/alembic/versions/20260909_120000_dashboard_conversation_archive.py +48 -0
- app/db/alembic/versions/20260909_130000_add_request_logs_live_facet_indexes.py +96 -0
- app/db/alembic/versions/20260910_000000_add_totp_required_for_admin_role.py +44 -0
- app/db/alembic/versions/20260910_000000_request_logs_missing_cost_index.py +52 -0
- app/db/alembic/versions/20260910_010000_add_dashboard_auth_providers.py +134 -0
- app/db/alembic/versions/20260910_010000_dashboard_spool_retention.py +52 -0
- app/db/alembic/versions/20260910_020000_add_dashboard_role_mappings.py +74 -0
- app/db/alembic/versions/20260910_143000_request_log_affinity.py +28 -0
- app/db/alembic/versions/20260910_180000_merge_affinity_guest_heads.py +17 -0
- app/db/alembic/versions/20260910_200000_merge_affinity_identity_heads.py +17 -0
- app/db/alembic/versions/20260910_220000_merge_affinity_invite_heads.py +17 -0
- app/db/alembic/versions/20260911_000000_model_source_pins_kind_expires_index.py +43 -0
- app/db/alembic/versions/20260911_010000_merge_pin_index_and_affinity_heads.py +17 -0
- app/db/alembic/versions/20260911_020000_add_http_bridge_terminal_append_phase.py +54 -0
- app/db/alembic/versions/20260911_030000_add_local_login_policy.py +48 -0
- app/db/alembic/versions/20260911_040000_add_thread_cache_identity_mode.py +48 -0
- app/db/alembic/versions/20260911_060000_add_bridge_session_continuity_abandonment.py +40 -0
- app/db/alembic/versions/20260912_000000_merge_thread_cache_and_bridge_retirement_heads.py +22 -0
- app/db/alembic/versions/20260912_010000_drop_legacy_dashboard_credentials.py +140 -0
- app/db/alembic/versions/20260913_000000_add_oidc_provider_flow.py +134 -0
- app/db/alembic/versions/20260914_000000_add_scim_tokens.py +78 -0
- app/db/alembic/versions/20260914_000000_drop_subscription_overflow_schema.py +132 -0
- app/db/alembic/versions/20260918_000000_merge_scim_and_overflow_heads.py +17 -0
- app/db/alembic/versions/__init__.py +1 -0
- app/db/backup.py +64 -0
- app/db/migrate.py +1249 -0
- app/db/migration_lock.py +160 -0
- app/db/migration_url.py +18 -0
- app/db/models.py +3100 -0
- app/db/recover.py +244 -0
- app/db/session.py +1295 -0
- app/db/snapshot.py +46 -0
- app/db/sqlite_lock_retry.py +190 -0
- app/db/sqlite_utils.py +549 -0
- app/dependencies.py +462 -0
- app/main.py +1240 -0
- app/modules/__init__.py +0 -0
- app/modules/accounts/__init__.py +0 -0
- app/modules/accounts/api.py +426 -0
- app/modules/accounts/auth_manager.py +1439 -0
- app/modules/accounts/background_repository.py +160 -0
- app/modules/accounts/deletion.py +540 -0
- app/modules/accounts/mappers.py +654 -0
- app/modules/accounts/refresh_claims.py +352 -0
- app/modules/accounts/repository.py +1679 -0
- app/modules/accounts/schemas.py +308 -0
- app/modules/accounts/service.py +809 -0
- app/modules/accounts/usage_rollup.py +486 -0
- app/modules/accounts/usage_rollup_scheduler.py +75 -0
- app/modules/accounts/usage_time_rollup.py +1037 -0
- app/modules/accounts/usage_time_rollup_read.py +522 -0
- app/modules/api_keys/__init__.py +1 -0
- app/modules/api_keys/api.py +344 -0
- app/modules/api_keys/last_used_coalescer.py +216 -0
- app/modules/api_keys/limit_windows.py +41 -0
- app/modules/api_keys/repository.py +1200 -0
- app/modules/api_keys/reset_scheduler.py +85 -0
- app/modules/api_keys/schemas.py +135 -0
- app/modules/api_keys/service.py +2123 -0
- app/modules/audit/__init__.py +0 -0
- app/modules/audit/api.py +88 -0
- app/modules/audit/repository.py +74 -0
- app/modules/audit/schemas.py +33 -0
- app/modules/audit/service.py +71 -0
- app/modules/auth_providers/__init__.py +0 -0
- app/modules/auth_providers/api.py +127 -0
- app/modules/auth_providers/config.py +343 -0
- app/modules/auth_providers/repository.py +101 -0
- app/modules/auth_providers/schemas.py +88 -0
- app/modules/auth_providers/seed.py +105 -0
- app/modules/auth_providers/service.py +289 -0
- app/modules/automations/__init__.py +1 -0
- app/modules/automations/api.py +369 -0
- app/modules/automations/repository.py +1896 -0
- app/modules/automations/scheduler.py +107 -0
- app/modules/automations/schemas.py +158 -0
- app/modules/automations/service.py +2548 -0
- app/modules/cache_isolation_probe/__init__.py +0 -0
- app/modules/cache_isolation_probe/api.py +188 -0
- app/modules/cache_isolation_probe/prefix.py +144 -0
- app/modules/cache_isolation_probe/schemas.py +94 -0
- app/modules/cache_isolation_probe/sender.py +229 -0
- app/modules/cache_isolation_probe/service.py +525 -0
- app/modules/conversation_archive/__init__.py +1 -0
- app/modules/conversation_archive/api.py +101 -0
- app/modules/conversation_archive/schemas.py +28 -0
- app/modules/conversation_archive/service.py +165 -0
- app/modules/dashboard/__init__.py +1 -0
- app/modules/dashboard/api.py +106 -0
- app/modules/dashboard/builders.py +107 -0
- app/modules/dashboard/repository.py +304 -0
- app/modules/dashboard/schemas.py +148 -0
- app/modules/dashboard/service.py +534 -0
- app/modules/dashboard/timeframes.py +19 -0
- app/modules/dashboard/weekly_pace.py +617 -0
- app/modules/dashboard_auth/__init__.py +1 -0
- app/modules/dashboard_auth/api.py +1344 -0
- app/modules/dashboard_auth/oidc_api.py +753 -0
- app/modules/dashboard_auth/oidc_flows.py +330 -0
- app/modules/dashboard_auth/repository.py +452 -0
- app/modules/dashboard_auth/schemas.py +232 -0
- app/modules/dashboard_auth/service.py +1384 -0
- app/modules/dashboard_roles/__init__.py +0 -0
- app/modules/dashboard_roles/api.py +64 -0
- app/modules/dashboard_roles/repository.py +54 -0
- app/modules/dashboard_roles/schemas.py +29 -0
- app/modules/dashboard_roles/seed.py +80 -0
- app/modules/dashboard_roles/service.py +142 -0
- app/modules/dashboard_users/__init__.py +0 -0
- app/modules/dashboard_users/api.py +278 -0
- app/modules/dashboard_users/break_glass.py +169 -0
- app/modules/dashboard_users/credentials.py +18 -0
- app/modules/dashboard_users/identity_resolver.py +665 -0
- app/modules/dashboard_users/repository.py +725 -0
- app/modules/dashboard_users/schemas.py +117 -0
- app/modules/dashboard_users/service.py +1054 -0
- app/modules/firewall/__init__.py +1 -0
- app/modules/firewall/api.py +78 -0
- app/modules/firewall/repository.py +51 -0
- app/modules/firewall/schemas.py +28 -0
- app/modules/firewall/service.py +93 -0
- app/modules/fleet/__init__.py +1 -0
- app/modules/fleet/api.py +177 -0
- app/modules/fleet/mappers.py +56 -0
- app/modules/fleet/observability.py +372 -0
- app/modules/fleet/schemas.py +119 -0
- app/modules/health/__init__.py +0 -0
- app/modules/health/api.py +246 -0
- app/modules/health/schemas.py +27 -0
- app/modules/limit_warmup/__init__.py +1 -0
- app/modules/limit_warmup/repository.py +159 -0
- app/modules/limit_warmup/service.py +1001 -0
- app/modules/model_sources/__init__.py +1 -0
- app/modules/model_sources/api.py +99 -0
- app/modules/model_sources/catalog.py +307 -0
- app/modules/model_sources/forwarding.py +1435 -0
- app/modules/model_sources/projection.py +55 -0
- app/modules/model_sources/repository.py +201 -0
- app/modules/model_sources/schemas.py +79 -0
- app/modules/model_sources/selection.py +166 -0
- app/modules/model_sources/service.py +236 -0
- app/modules/oauth/__init__.py +0 -0
- app/modules/oauth/api.py +101 -0
- app/modules/oauth/repository.py +238 -0
- app/modules/oauth/schemas.py +49 -0
- app/modules/oauth/service.py +1196 -0
- app/modules/oauth/templates/oauth_success.html +122 -0
- app/modules/proxy/__init__.py +0 -0
- app/modules/proxy/_load_balancer/__init__.py +1 -0
- app/modules/proxy/_load_balancer/error_rate.py +116 -0
- app/modules/proxy/_load_balancer/latency_cohort.py +154 -0
- app/modules/proxy/_load_balancer/model_eligibility.py +250 -0
- app/modules/proxy/_load_balancer/opportunistic_admission.py +350 -0
- app/modules/proxy/_load_balancer/overload_backoff.py +394 -0
- app/modules/proxy/_load_balancer/sticky_selection.py +2417 -0
- app/modules/proxy/_load_balancer/throughput_cohort.py +233 -0
- app/modules/proxy/_load_balancer/ttft_cohort.py +214 -0
- app/modules/proxy/_load_balancer/tunables.py +97 -0
- app/modules/proxy/_load_balancer/types.py +96 -0
- app/modules/proxy/_load_balancer/unbound_selection.py +534 -0
- app/modules/proxy/_service/__init__.py +6 -0
- app/modules/proxy/_service/api_key_usage.py +745 -0
- app/modules/proxy/_service/clock_budget.py +38 -0
- app/modules/proxy/_service/codex_control.py +611 -0
- app/modules/proxy/_service/compact.py +2211 -0
- app/modules/proxy/_service/file_ops.py +749 -0
- app/modules/proxy/_service/http_bridge/__init__.py +3 -0
- app/modules/proxy/_service/http_bridge/accepted_replay.py +331 -0
- app/modules/proxy/_service/http_bridge/account_sessions.py +57 -0
- app/modules/proxy/_service/http_bridge/activity.py +178 -0
- app/modules/proxy/_service/http_bridge/helpers.py +3959 -0
- app/modules/proxy/_service/http_bridge/mixin.py +2435 -0
- app/modules/proxy/_service/http_bridge/owner_forwarding.py +609 -0
- app/modules/proxy/_service/http_bridge/protocol.py +167 -0
- app/modules/proxy/_service/http_bridge/proxy_failover.py +58 -0
- app/modules/proxy/_service/http_bridge/quarantine.py +443 -0
- app/modules/proxy/_service/http_bridge/request_submit.py +4582 -0
- app/modules/proxy/_service/http_bridge/retry_circuit.py +2442 -0
- app/modules/proxy/_service/http_bridge/service_stubs.py +492 -0
- app/modules/proxy/_service/http_bridge/session_registry.py +778 -0
- app/modules/proxy/_service/http_bridge/streaming.py +5328 -0
- app/modules/proxy/_service/http_bridge/upstream_events.py +4538 -0
- app/modules/proxy/_service/observability.py +410 -0
- app/modules/proxy/_service/rate_limit.py +361 -0
- app/modules/proxy/_service/realtime_live.py +655 -0
- app/modules/proxy/_service/refresh.py +154 -0
- app/modules/proxy/_service/request_log.py +585 -0
- app/modules/proxy/_service/response_create.py +930 -0
- app/modules/proxy/_service/streaming/__init__.py +3 -0
- app/modules/proxy/_service/streaming/helpers.py +1317 -0
- app/modules/proxy/_service/streaming/mixin.py +1100 -0
- app/modules/proxy/_service/streaming/protocol.py +32 -0
- app/modules/proxy/_service/streaming/retry.py +3762 -0
- app/modules/proxy/_service/support.py +2122 -0
- app/modules/proxy/_service/transcribe.py +434 -0
- app/modules/proxy/_service/warmup.py +483 -0
- app/modules/proxy/_service/websocket/__init__.py +3 -0
- app/modules/proxy/_service/websocket/helpers.py +2360 -0
- app/modules/proxy/_service/websocket/mixin.py +7285 -0
- app/modules/proxy/_service/websocket/protocol.py +74 -0
- app/modules/proxy/account_cache.py +229 -0
- app/modules/proxy/account_eligibility.py +87 -0
- app/modules/proxy/additional_model_limits.py +40 -0
- app/modules/proxy/affinity.py +830 -0
- app/modules/proxy/affinity_observation.py +101 -0
- app/modules/proxy/api.py +10254 -0
- app/modules/proxy/api_key_usage.py +113 -0
- app/modules/proxy/cap_partitioning.py +280 -0
- app/modules/proxy/capability_lineage.py +65 -0
- app/modules/proxy/capability_lineage_repository.py +82 -0
- app/modules/proxy/capability_routing.py +253 -0
- app/modules/proxy/continuity.py +112 -0
- app/modules/proxy/downstream_delivery.py +277 -0
- app/modules/proxy/durable_bridge_coordinator.py +1217 -0
- app/modules/proxy/durable_bridge_repository.py +4516 -0
- app/modules/proxy/durable_bridge_runtime.py +9 -0
- app/modules/proxy/durable_bridge_transcript_codec.py +122 -0
- app/modules/proxy/fair_share.py +131 -0
- app/modules/proxy/file_pin_repository.py +210 -0
- app/modules/proxy/helpers.py +439 -0
- app/modules/proxy/http_bridge_event_batcher.py +613 -0
- app/modules/proxy/http_bridge_forwarding.py +797 -0
- app/modules/proxy/http_continuation.py +76 -0
- app/modules/proxy/images_observability.py +62 -0
- app/modules/proxy/images_service.py +1054 -0
- app/modules/proxy/load_balancer.py +3016 -0
- app/modules/proxy/rate_limit_cache.py +46 -0
- app/modules/proxy/replay_safety.py +1063 -0
- app/modules/proxy/repo_bundle.py +31 -0
- app/modules/proxy/request_policy.py +1054 -0
- app/modules/proxy/ring_membership.py +251 -0
- app/modules/proxy/schemas.py +345 -0
- app/modules/proxy/selection_errors.py +41 -0
- app/modules/proxy/service.py +2600 -0
- app/modules/proxy/source_admission.py +125 -0
- app/modules/proxy/source_dispatch.py +999 -0
- app/modules/proxy/sticky_repository.py +831 -0
- app/modules/proxy/thread_anchors.py +491 -0
- app/modules/proxy/tool_call_dedupe.py +802 -0
- app/modules/proxy/types.py +54 -0
- app/modules/proxy/work_admission.py +121 -0
- app/modules/quota_planner/__init__.py +1 -0
- app/modules/quota_planner/api.py +276 -0
- app/modules/quota_planner/logic.py +743 -0
- app/modules/quota_planner/repository.py +722 -0
- app/modules/quota_planner/scheduler.py +212 -0
- app/modules/quota_planner/schemas.py +102 -0
- app/modules/quota_planner/warmup.py +693 -0
- app/modules/rate_limit_reset_credits/__init__.py +0 -0
- app/modules/rate_limit_reset_credits/api.py +601 -0
- app/modules/rate_limit_reset_credits/redeem_coordination.py +274 -0
- app/modules/rate_limit_reset_credits/store.py +114 -0
- app/modules/reports/__init__.py +1 -0
- app/modules/reports/api.py +134 -0
- app/modules/reports/cache.py +74 -0
- app/modules/reports/filters.py +27 -0
- app/modules/reports/repository.py +525 -0
- app/modules/reports/rollup.py +150 -0
- app/modules/reports/rollup_read.py +148 -0
- app/modules/reports/schemas.py +123 -0
- app/modules/reports/service.py +287 -0
- app/modules/reports/thread_identity.py +256 -0
- app/modules/request_logs/__init__.py +0 -0
- app/modules/request_logs/api.py +224 -0
- app/modules/request_logs/cost_backfill.py +154 -0
- app/modules/request_logs/mappers.py +97 -0
- app/modules/request_logs/repository.py +1765 -0
- app/modules/request_logs/schemas.py +145 -0
- app/modules/request_logs/service.py +348 -0
- app/modules/role_mappings/__init__.py +0 -0
- app/modules/role_mappings/api.py +206 -0
- app/modules/role_mappings/matching.py +55 -0
- app/modules/role_mappings/repository.py +106 -0
- app/modules/role_mappings/schemas.py +67 -0
- app/modules/role_mappings/service.py +334 -0
- app/modules/runtime/__init__.py +1 -0
- app/modules/runtime/api.py +18 -0
- app/modules/runtime/schemas.py +14 -0
- app/modules/runtime/service.py +218 -0
- app/modules/scim/__init__.py +1 -0
- app/modules/scim/api.py +194 -0
- app/modules/scim/dependencies.py +112 -0
- app/modules/scim/errors.py +77 -0
- app/modules/scim/management_api.py +156 -0
- app/modules/scim/repository.py +212 -0
- app/modules/scim/schemas.py +145 -0
- app/modules/scim/service.py +786 -0
- app/modules/scim/tokens.py +43 -0
- app/modules/settings/__init__.py +1 -0
- app/modules/settings/api.py +1677 -0
- app/modules/settings/repository.py +606 -0
- app/modules/settings/schemas.py +496 -0
- app/modules/settings/service.py +769 -0
- app/modules/shared/__init__.py +0 -0
- app/modules/shared/schemas.py +22 -0
- app/modules/sticky_sessions/__init__.py +1 -0
- app/modules/sticky_sessions/api.py +118 -0
- app/modules/sticky_sessions/cleanup_scheduler.py +487 -0
- app/modules/sticky_sessions/schemas.py +78 -0
- app/modules/sticky_sessions/service.py +196 -0
- app/modules/telemetry/__init__.py +5 -0
- app/modules/telemetry/api.py +115 -0
- app/modules/telemetry/clients.py +62 -0
- app/modules/telemetry/consent.py +115 -0
- app/modules/telemetry/scheduler.py +85 -0
- app/modules/telemetry/schemas.py +163 -0
- app/modules/telemetry/sender.py +221 -0
- app/modules/telemetry/snapshot.py +490 -0
- app/modules/usage/__init__.py +0 -0
- app/modules/usage/additional_quota_keys.py +379 -0
- app/modules/usage/api.py +45 -0
- app/modules/usage/background_repository.py +147 -0
- app/modules/usage/builders.py +559 -0
- app/modules/usage/depletion_service.py +277 -0
- app/modules/usage/live_ingest.py +356 -0
- app/modules/usage/mappers.py +29 -0
- app/modules/usage/plan_downgrade_observations.py +488 -0
- app/modules/usage/repository.py +1890 -0
- app/modules/usage/schemas.py +68 -0
- app/modules/usage/service.py +102 -0
- app/modules/usage/updater.py +1488 -0
- app/static/assets/Trans-Cj-J9mVA.js +1 -0
- app/static/assets/access-page-DHzZiXZi.js +1 -0
- app/static/assets/account-cost-donut-DGhfA4U8.js +1 -0
- app/static/assets/account-identifiers-CxbdLshZ.js +1 -0
- app/static/assets/account-status-DIsU5sUS.js +1 -0
- app/static/assets/account-trend-chart-BptlfwwQ.js +1 -0
- app/static/assets/accounts-page-Bbai8ift.js +3 -0
- app/static/assets/api-Dwa7XxPe.js +1 -0
- app/static/assets/api-client-dugX1Pti.js +55 -0
- app/static/assets/api-key-create-dialog-DnRm0oYN.js +1 -0
- app/static/assets/api-key-created-dialog-B6jkDG9f.js +1 -0
- app/static/assets/api-key-edit-dialog-Ck3Bnc3V.js +1 -0
- app/static/assets/api-trend-chart-BchCw7Hj.js +1 -0
- app/static/assets/apis-page-BvXz9c6i.js +2 -0
- app/static/assets/arrow-up-Dqh_KzJo.js +1 -0
- app/static/assets/arrow-up-down-LBRgVDGY.js +1 -0
- app/static/assets/automations-page-DQgNAJOR.js +3 -0
- app/static/assets/badge-DqMb9es0.js +1 -0
- app/static/assets/blur-email-HEvKagG2.js +1 -0
- app/static/assets/button-BjVmmiGW.js +1 -0
- app/static/assets/checkbox-apsl9vzk.js +1 -0
- app/static/assets/chevron-left-1d_pHVa5.js +1 -0
- app/static/assets/circle-alert-BS0xd_o1.js +1 -0
- app/static/assets/clsx-DB0hHKMi.js +1 -0
- app/static/assets/colors-BBy8GEtr.js +1 -0
- app/static/assets/copy-button-D7nAvOKR.js +1 -0
- app/static/assets/cost-per-day-chart-YdqlkZVM.js +1 -0
- app/static/assets/createLucideIcon-DBLzLNh2.js +1 -0
- app/static/assets/dashboard-page-CSdhBy0f.js +2 -0
- app/static/assets/distribution-metric-format-yZTy1ZEu.js +1 -0
- app/static/assets/donut-chart-Bxc69cra.js +1 -0
- app/static/assets/download-kQteIcsB.js +1 -0
- app/static/assets/ellipsis-B_5OgIre.js +1 -0
- app/static/assets/empty-state-JcA7tc8m.js +1 -0
- app/static/assets/es2015-bMH8YKZT.js +41 -0
- app/static/assets/es6-B264ni6W.js +69 -0
- app/static/assets/external-link-UVAOgMQ0.js +1 -0
- app/static/assets/i18n-BWqZ9Ua2.js +1 -0
- app/static/assets/inbox-B_RM2s1J.js +1 -0
- app/static/assets/index-DIpd264Z.js +23 -0
- app/static/assets/index-DMeLKRJx.css +2 -0
- app/static/assets/inherit-badge-QhTYVaU6.js +1 -0
- app/static/assets/invite-dialog-0CYh0GiQ.js +1 -0
- app/static/assets/iso-Bpe5Ihbb.js +1 -0
- app/static/assets/label-CGaLmLNf.js +1 -0
- app/static/assets/lazy-recharts-Pv8p9ArM.js +2 -0
- app/static/assets/loading-overlay-CzKQfHLI.js +1 -0
- app/static/assets/mini-quota-bar-cGT_t8Ae.js +1 -0
- app/static/assets/model-distribution-donut-BKjAfT48.js +1 -0
- app/static/assets/model-source-multi-select-D2d06Vc7.js +1 -0
- app/static/assets/multi-select-filter-KIA5qU6Z.js +1 -0
- app/static/assets/oauth-dialog-XeaGGDjS.js +1 -0
- app/static/assets/pagination-controls-D1Ypd0RU.js +1 -0
- app/static/assets/payload-CIYnGZz9.js +1 -0
- app/static/assets/queue-wait-chart-VoV7Yg9L.js +1 -0
- app/static/assets/report-chart-card-BXDliwcc.js +1 -0
- app/static/assets/reports-page-CGCJeMHG.js +3 -0
- app/static/assets/reset-credit-confirm-dialog-QOifTnVB.js +1 -0
- app/static/assets/rolldown-runtime-hePW80VL.js +1 -0
- app/static/assets/search-HfhqwERt.js +1 -0
- app/static/assets/select-Cl33bz-g.js +1 -0
- app/static/assets/settings-page-mZj01HYl.js +2 -0
- app/static/assets/shim-CZwyM4Q7.js +1 -0
- app/static/assets/skeleton-C_EUc4Rp.js +1 -0
- app/static/assets/sparkline-chart-DdNNsddM.js +1 -0
- app/static/assets/status-badge-CHHA5X6c.js +1 -0
- app/static/assets/switch-CzxgokcM.js +1 -0
- app/static/assets/table-DXaNSQ0G.js +1 -0
- app/static/assets/time-to-first-token-chart--68L0u_v.js +1 -0
- app/static/assets/tokens-per-day-chart-Cs82BFuE.js +1 -0
- app/static/assets/tokens-per-second-chart-AD2dfOn6.js +1 -0
- app/static/assets/totp-settings-CRHtxQ7j.js +1 -0
- app/static/assets/trash-D3KyWr1h.js +1 -0
- app/static/assets/use-account-quota-display-BPmrTBTC.js +1 -0
- app/static/assets/use-accounts-PWlVGw3z.js +1 -0
- app/static/assets/use-auth-CuDlW3iC.js +1 -0
- app/static/assets/use-chart-colors-DIu6tClI.js +1 -0
- app/static/assets/use-model-sources-Bul24JBu.js +1 -0
- app/static/assets/use-models-DUCPUuZg.js +1 -0
- app/static/assets/use-reduced-motion-hAm5r6_d.js +1 -0
- app/static/assets/useTranslation--56nHhlH.js +1 -0
- app/static/assets/useragent-distribution-donut-Dup6sf_n.js +1 -0
- app/static/assets/users-uIC3TIlE.js +1 -0
- app/static/assets/vendor-query-_bHZuxEx.js +1 -0
- app/static/assets/vendor-react-BVL0YZPW.js +9 -0
- app/static/favicon.svg +10 -0
- app/static/fonts/GeistSans-Variable.woff2 +0 -0
- app/static/fonts/jetbrains-mono-latin-ext.woff2 +0 -0
- app/static/fonts/jetbrains-mono-latin.woff2 +0 -0
- app/static/index.html +29 -0
- codexlb_up-1.25.0b10.dist-info/METADATA +124 -0
- codexlb_up-1.25.0b10.dist-info/RECORD +827 -0
- codexlb_up-1.25.0b10.dist-info/WHEEL +4 -0
- codexlb_up-1.25.0b10.dist-info/entry_points.txt +3 -0
- codexlb_up-1.25.0b10.dist-info/licenses/LICENSE +21 -0
- config/additional_quota_registry.json +11 -0
|
@@ -0,0 +1,1896 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
from collections.abc import Sequence
|
|
4
|
+
from dataclasses import dataclass
|
|
5
|
+
from datetime import datetime, timedelta
|
|
6
|
+
from hashlib import sha1
|
|
7
|
+
from uuid import uuid4
|
|
8
|
+
|
|
9
|
+
from sqlalchemy import and_, case, delete, exists, func, insert, literal, or_, select, union_all, update
|
|
10
|
+
from sqlalchemy.exc import IntegrityError
|
|
11
|
+
from sqlalchemy.ext.asyncio import AsyncSession
|
|
12
|
+
from sqlalchemy.orm import selectinload
|
|
13
|
+
from sqlalchemy.sql.selectable import CTE
|
|
14
|
+
|
|
15
|
+
from app.core.config.dashboard_overrides import with_dashboard_overrides
|
|
16
|
+
from app.core.config.settings import get_settings
|
|
17
|
+
from app.core.utils.time import utcnow
|
|
18
|
+
from app.db.models import (
|
|
19
|
+
Account,
|
|
20
|
+
AccountStatus,
|
|
21
|
+
AutomationJob,
|
|
22
|
+
AutomationJobAccount,
|
|
23
|
+
AutomationRun,
|
|
24
|
+
AutomationRunCycle,
|
|
25
|
+
AutomationRunCycleAccount,
|
|
26
|
+
)
|
|
27
|
+
|
|
28
|
+
DEFAULT_AUTOMATION_SCHEDULE_DAYS = ["mon", "tue", "wed", "thu", "fri", "sat", "sun"]
|
|
29
|
+
|
|
30
|
+
# A running claim is reclaimable once it has been held for the larger of the
|
|
31
|
+
# compact request budget it was claimed under and the current budget, plus this
|
|
32
|
+
# grace, never sooner than the floor. A dashboard change can therefore only
|
|
33
|
+
# widen a live claim's window, never narrow it.
|
|
34
|
+
RUN_CLAIM_GRACE_SECONDS = 30.0
|
|
35
|
+
RUN_CLAIM_MIN_TIMEOUT_SECONDS = 30.0
|
|
36
|
+
|
|
37
|
+
|
|
38
|
+
def effective_compact_request_budget_seconds() -> float:
|
|
39
|
+
"""The compact request budget currently in effect (dashboard value when bound)."""
|
|
40
|
+
return with_dashboard_overrides(get_settings()).compact_request_budget_seconds
|
|
41
|
+
|
|
42
|
+
|
|
43
|
+
def run_claim_timeout_seconds(claim_budget_seconds: float | None, *, fallback_budget_seconds: float) -> float:
|
|
44
|
+
"""Reclaim window for a claim made under ``claim_budget_seconds``.
|
|
45
|
+
|
|
46
|
+
The window covers the larger of the pinned budget and
|
|
47
|
+
``fallback_budget_seconds`` (the current effective budget): lowering the
|
|
48
|
+
dashboard budget cannot shrink the window under an in-flight run, and a
|
|
49
|
+
raised budget still covers an attempt started by a writer that advanced
|
|
50
|
+
``started_at`` without refreshing the pin (pre-pin replica during a rolling
|
|
51
|
+
deploy). ``None`` marks a row that never had a pin; it uses the current
|
|
52
|
+
budget alone.
|
|
53
|
+
"""
|
|
54
|
+
budget = (
|
|
55
|
+
fallback_budget_seconds if claim_budget_seconds is None else max(claim_budget_seconds, fallback_budget_seconds)
|
|
56
|
+
)
|
|
57
|
+
return max(RUN_CLAIM_MIN_TIMEOUT_SECONDS, budget + RUN_CLAIM_GRACE_SECONDS)
|
|
58
|
+
|
|
59
|
+
|
|
60
|
+
def run_stale_started_before(
|
|
61
|
+
*,
|
|
62
|
+
now_utc: datetime,
|
|
63
|
+
claim_budget_seconds: float | None,
|
|
64
|
+
fallback_budget_seconds: float,
|
|
65
|
+
) -> datetime:
|
|
66
|
+
"""Latest ``started_at`` at which a claim under ``claim_budget_seconds`` counts as stale at ``now_utc``."""
|
|
67
|
+
return now_utc - timedelta(
|
|
68
|
+
seconds=run_claim_timeout_seconds(claim_budget_seconds, fallback_budget_seconds=fallback_budget_seconds)
|
|
69
|
+
)
|
|
70
|
+
|
|
71
|
+
|
|
72
|
+
@dataclass(frozen=True, slots=True)
|
|
73
|
+
class AutomationRunRecord:
|
|
74
|
+
id: str
|
|
75
|
+
job_id: str
|
|
76
|
+
job_name: str | None
|
|
77
|
+
model: str | None
|
|
78
|
+
reasoning_effort: str | None
|
|
79
|
+
prompt: str | None
|
|
80
|
+
trigger: str
|
|
81
|
+
status: str
|
|
82
|
+
slot_key: str
|
|
83
|
+
cycle_key: str
|
|
84
|
+
cycle_expected_accounts: int | None
|
|
85
|
+
cycle_window_end: datetime | None
|
|
86
|
+
scheduled_for: datetime
|
|
87
|
+
started_at: datetime
|
|
88
|
+
finished_at: datetime | None
|
|
89
|
+
account_id: str | None
|
|
90
|
+
error_code: str | None
|
|
91
|
+
error_message: str | None
|
|
92
|
+
attempt_count: int
|
|
93
|
+
claim_budget_seconds: float | None
|
|
94
|
+
|
|
95
|
+
|
|
96
|
+
@dataclass(frozen=True, slots=True)
|
|
97
|
+
class AutomationScheduledRunClaimRecord:
|
|
98
|
+
run: AutomationRunRecord | None
|
|
99
|
+
snapshot_account_exists: bool
|
|
100
|
+
|
|
101
|
+
|
|
102
|
+
@dataclass(frozen=True, slots=True)
|
|
103
|
+
class AutomationJobRecord:
|
|
104
|
+
id: str
|
|
105
|
+
name: str
|
|
106
|
+
enabled: bool
|
|
107
|
+
schedule_type: str
|
|
108
|
+
schedule_time: str
|
|
109
|
+
schedule_timezone: str
|
|
110
|
+
schedule_days: list[str]
|
|
111
|
+
schedule_threshold_minutes: int
|
|
112
|
+
include_paused_accounts: bool
|
|
113
|
+
account_scope_all: bool
|
|
114
|
+
model: str
|
|
115
|
+
reasoning_effort: str | None
|
|
116
|
+
prompt: str
|
|
117
|
+
account_ids: list[str]
|
|
118
|
+
created_at: datetime
|
|
119
|
+
updated_at: datetime
|
|
120
|
+
|
|
121
|
+
|
|
122
|
+
@dataclass(frozen=True, slots=True)
|
|
123
|
+
class AutomationRunCycleAccountRecord:
|
|
124
|
+
account_id: str
|
|
125
|
+
slot_key: str | None
|
|
126
|
+
position: int
|
|
127
|
+
scheduled_for: datetime
|
|
128
|
+
|
|
129
|
+
|
|
130
|
+
@dataclass(frozen=True, slots=True)
|
|
131
|
+
class AutomationRunCycleRecord:
|
|
132
|
+
cycle_key: str
|
|
133
|
+
job_id: str
|
|
134
|
+
trigger: str
|
|
135
|
+
cycle_expected_accounts: int
|
|
136
|
+
cycle_window_end: datetime | None
|
|
137
|
+
include_paused_accounts: bool
|
|
138
|
+
accounts: list[AutomationRunCycleAccountRecord]
|
|
139
|
+
created_at: datetime
|
|
140
|
+
|
|
141
|
+
|
|
142
|
+
@dataclass(frozen=True, slots=True)
|
|
143
|
+
class AutomationJobsFilterOptionsRecord:
|
|
144
|
+
account_ids: list[str]
|
|
145
|
+
models: list[str]
|
|
146
|
+
statuses: list[str]
|
|
147
|
+
schedule_types: list[str]
|
|
148
|
+
|
|
149
|
+
|
|
150
|
+
@dataclass(frozen=True, slots=True)
|
|
151
|
+
class AutomationRunsFilterOptionsRecord:
|
|
152
|
+
account_ids: list[str]
|
|
153
|
+
models: list[str]
|
|
154
|
+
|
|
155
|
+
|
|
156
|
+
class AutomationsRepository:
|
|
157
|
+
def __init__(self, session: AsyncSession) -> None:
|
|
158
|
+
self._session = session
|
|
159
|
+
|
|
160
|
+
async def list_jobs(self) -> list[AutomationJobRecord]:
|
|
161
|
+
result = await self._session.execute(
|
|
162
|
+
select(AutomationJob)
|
|
163
|
+
.options(selectinload(AutomationJob.account_links))
|
|
164
|
+
.order_by(AutomationJob.created_at.desc(), AutomationJob.id.asc())
|
|
165
|
+
)
|
|
166
|
+
jobs = list(result.scalars().all())
|
|
167
|
+
return [self._job_from_model(job) for job in jobs]
|
|
168
|
+
|
|
169
|
+
async def list_jobs_page(
|
|
170
|
+
self,
|
|
171
|
+
*,
|
|
172
|
+
limit: int,
|
|
173
|
+
offset: int,
|
|
174
|
+
search: str | None = None,
|
|
175
|
+
account_ids: Sequence[str] | None = None,
|
|
176
|
+
models: Sequence[str] | None = None,
|
|
177
|
+
statuses: Sequence[str] | None = None,
|
|
178
|
+
schedule_types: Sequence[str] | None = None,
|
|
179
|
+
) -> tuple[list[AutomationJobRecord], int]:
|
|
180
|
+
conditions = self._build_job_conditions(
|
|
181
|
+
search=search,
|
|
182
|
+
account_ids=account_ids,
|
|
183
|
+
models=models,
|
|
184
|
+
statuses=statuses,
|
|
185
|
+
schedule_types=schedule_types,
|
|
186
|
+
)
|
|
187
|
+
stmt = (
|
|
188
|
+
select(AutomationJob)
|
|
189
|
+
.options(selectinload(AutomationJob.account_links))
|
|
190
|
+
.order_by(AutomationJob.created_at.desc(), AutomationJob.id.asc())
|
|
191
|
+
.offset(offset)
|
|
192
|
+
.limit(limit)
|
|
193
|
+
)
|
|
194
|
+
if conditions:
|
|
195
|
+
stmt = stmt.where(and_(*conditions))
|
|
196
|
+
result = await self._session.execute(stmt)
|
|
197
|
+
jobs = [self._job_from_model(job) for job in result.scalars().all()]
|
|
198
|
+
|
|
199
|
+
count_stmt = select(func.count(AutomationJob.id))
|
|
200
|
+
if conditions:
|
|
201
|
+
count_stmt = count_stmt.where(and_(*conditions))
|
|
202
|
+
total = int((await self._session.execute(count_stmt)).scalar_one() or 0)
|
|
203
|
+
return jobs, total
|
|
204
|
+
|
|
205
|
+
async def list_job_filter_options(
|
|
206
|
+
self,
|
|
207
|
+
*,
|
|
208
|
+
search: str | None = None,
|
|
209
|
+
account_ids: Sequence[str] | None = None,
|
|
210
|
+
models: Sequence[str] | None = None,
|
|
211
|
+
statuses: Sequence[str] | None = None,
|
|
212
|
+
schedule_types: Sequence[str] | None = None,
|
|
213
|
+
) -> AutomationJobsFilterOptionsRecord:
|
|
214
|
+
conditions = self._build_job_conditions(
|
|
215
|
+
search=search,
|
|
216
|
+
account_ids=account_ids,
|
|
217
|
+
models=models,
|
|
218
|
+
statuses=statuses,
|
|
219
|
+
schedule_types=schedule_types,
|
|
220
|
+
)
|
|
221
|
+
|
|
222
|
+
account_stmt = (
|
|
223
|
+
select(AutomationJobAccount.account_id)
|
|
224
|
+
.distinct()
|
|
225
|
+
.join(AutomationJob, AutomationJob.id == AutomationJobAccount.job_id)
|
|
226
|
+
.order_by(AutomationJobAccount.account_id.asc())
|
|
227
|
+
)
|
|
228
|
+
model_stmt = select(AutomationJob.model).distinct().order_by(AutomationJob.model.asc())
|
|
229
|
+
type_stmt = select(AutomationJob.schedule_type).distinct().order_by(AutomationJob.schedule_type.asc())
|
|
230
|
+
status_stmt = select(AutomationJob.enabled).distinct()
|
|
231
|
+
if conditions:
|
|
232
|
+
clause = and_(*conditions)
|
|
233
|
+
account_stmt = account_stmt.where(clause)
|
|
234
|
+
model_stmt = model_stmt.where(clause)
|
|
235
|
+
type_stmt = type_stmt.where(clause)
|
|
236
|
+
status_stmt = status_stmt.where(clause)
|
|
237
|
+
|
|
238
|
+
account_ids_rows = await self._session.execute(account_stmt)
|
|
239
|
+
model_rows = await self._session.execute(model_stmt)
|
|
240
|
+
type_rows = await self._session.execute(type_stmt)
|
|
241
|
+
status_rows = await self._session.execute(status_stmt)
|
|
242
|
+
|
|
243
|
+
statuses = sorted({"enabled" if bool(value) else "disabled" for (value,) in status_rows.all()})
|
|
244
|
+
return AutomationJobsFilterOptionsRecord(
|
|
245
|
+
account_ids=[value for (value,) in account_ids_rows.all() if value],
|
|
246
|
+
models=[value for (value,) in model_rows.all() if value],
|
|
247
|
+
statuses=statuses,
|
|
248
|
+
schedule_types=[value for (value,) in type_rows.all() if value],
|
|
249
|
+
)
|
|
250
|
+
|
|
251
|
+
async def list_enabled_jobs(self) -> list[AutomationJobRecord]:
|
|
252
|
+
result = await self._session.execute(
|
|
253
|
+
select(AutomationJob)
|
|
254
|
+
.where(AutomationJob.enabled.is_(True))
|
|
255
|
+
.options(selectinload(AutomationJob.account_links))
|
|
256
|
+
.order_by(AutomationJob.created_at.asc(), AutomationJob.id.asc())
|
|
257
|
+
)
|
|
258
|
+
jobs = list(result.scalars().all())
|
|
259
|
+
return [self._job_from_model(job) for job in jobs]
|
|
260
|
+
|
|
261
|
+
async def list_due_scheduled_run_cycle_job_ids(
|
|
262
|
+
self,
|
|
263
|
+
*,
|
|
264
|
+
now_utc: datetime,
|
|
265
|
+
) -> list[str]:
|
|
266
|
+
batch_size = 500
|
|
267
|
+
offset = 0
|
|
268
|
+
due_job_ids: list[str] = []
|
|
269
|
+
known_job_ids: set[str] = set()
|
|
270
|
+
while True:
|
|
271
|
+
candidates = await self._list_due_scheduled_run_cycle_candidates(
|
|
272
|
+
job_id=None,
|
|
273
|
+
limit=batch_size,
|
|
274
|
+
offset=offset,
|
|
275
|
+
)
|
|
276
|
+
if not candidates:
|
|
277
|
+
break
|
|
278
|
+
|
|
279
|
+
due_cycles = await self._filter_due_scheduled_run_cycles(candidates, now_utc=now_utc)
|
|
280
|
+
for cycle in due_cycles:
|
|
281
|
+
if cycle.job_id and cycle.job_id not in known_job_ids:
|
|
282
|
+
known_job_ids.add(cycle.job_id)
|
|
283
|
+
due_job_ids.append(cycle.job_id)
|
|
284
|
+
|
|
285
|
+
if len(candidates) < batch_size:
|
|
286
|
+
break
|
|
287
|
+
offset += batch_size
|
|
288
|
+
return due_job_ids
|
|
289
|
+
|
|
290
|
+
async def get_job(self, job_id: str) -> AutomationJobRecord | None:
|
|
291
|
+
result = await self._session.execute(
|
|
292
|
+
select(AutomationJob).where(AutomationJob.id == job_id).options(selectinload(AutomationJob.account_links))
|
|
293
|
+
)
|
|
294
|
+
job = result.scalar_one_or_none()
|
|
295
|
+
if job is None:
|
|
296
|
+
return None
|
|
297
|
+
return self._job_from_model(job)
|
|
298
|
+
|
|
299
|
+
async def get_jobs_by_ids(self, job_ids: Sequence[str]) -> dict[str, AutomationJobRecord]:
|
|
300
|
+
normalized_job_ids = [job_id for job_id in job_ids if job_id]
|
|
301
|
+
if not normalized_job_ids:
|
|
302
|
+
return {}
|
|
303
|
+
result = await self._session.execute(
|
|
304
|
+
select(AutomationJob)
|
|
305
|
+
.where(AutomationJob.id.in_(list(dict.fromkeys(normalized_job_ids))))
|
|
306
|
+
.options(selectinload(AutomationJob.account_links))
|
|
307
|
+
)
|
|
308
|
+
jobs = [self._job_from_model(job) for job in result.scalars().all()]
|
|
309
|
+
return {job.id: job for job in jobs}
|
|
310
|
+
|
|
311
|
+
async def create_job(
|
|
312
|
+
self,
|
|
313
|
+
*,
|
|
314
|
+
name: str,
|
|
315
|
+
enabled: bool,
|
|
316
|
+
schedule_type: str,
|
|
317
|
+
schedule_time: str,
|
|
318
|
+
schedule_timezone: str,
|
|
319
|
+
schedule_days: Sequence[str],
|
|
320
|
+
schedule_threshold_minutes: int,
|
|
321
|
+
include_paused_accounts: bool,
|
|
322
|
+
model: str,
|
|
323
|
+
reasoning_effort: str | None,
|
|
324
|
+
prompt: str,
|
|
325
|
+
account_ids: Sequence[str],
|
|
326
|
+
) -> AutomationJobRecord:
|
|
327
|
+
job = AutomationJob(
|
|
328
|
+
id=f"job_{uuid4().hex}",
|
|
329
|
+
name=name,
|
|
330
|
+
enabled=enabled,
|
|
331
|
+
schedule_type=schedule_type,
|
|
332
|
+
schedule_time=schedule_time,
|
|
333
|
+
schedule_timezone=schedule_timezone,
|
|
334
|
+
schedule_days=_serialize_schedule_days(schedule_days),
|
|
335
|
+
schedule_threshold_minutes=schedule_threshold_minutes,
|
|
336
|
+
include_paused_accounts=include_paused_accounts,
|
|
337
|
+
account_scope_all=not bool(account_ids),
|
|
338
|
+
model=model,
|
|
339
|
+
reasoning_effort=reasoning_effort,
|
|
340
|
+
prompt=prompt,
|
|
341
|
+
)
|
|
342
|
+
job.account_links = [
|
|
343
|
+
AutomationJobAccount(job_id=job.id, account_id=account_id, position=index)
|
|
344
|
+
for index, account_id in enumerate(account_ids)
|
|
345
|
+
]
|
|
346
|
+
self._session.add(job)
|
|
347
|
+
await self._session.commit()
|
|
348
|
+
await self._session.refresh(job)
|
|
349
|
+
await self._session.refresh(job, attribute_names=["account_links"])
|
|
350
|
+
return self._job_from_model(job)
|
|
351
|
+
|
|
352
|
+
async def update_job(
|
|
353
|
+
self,
|
|
354
|
+
job_id: str,
|
|
355
|
+
*,
|
|
356
|
+
name: str | None = None,
|
|
357
|
+
enabled: bool | None = None,
|
|
358
|
+
schedule_type: str | None = None,
|
|
359
|
+
schedule_time: str | None = None,
|
|
360
|
+
schedule_timezone: str | None = None,
|
|
361
|
+
schedule_days: Sequence[str] | None = None,
|
|
362
|
+
schedule_threshold_minutes: int | None = None,
|
|
363
|
+
include_paused_accounts: bool | None = None,
|
|
364
|
+
model: str | None = None,
|
|
365
|
+
reasoning_effort: str | None = None,
|
|
366
|
+
reasoning_effort_set: bool = False,
|
|
367
|
+
prompt: str | None = None,
|
|
368
|
+
account_ids: Sequence[str] | None = None,
|
|
369
|
+
) -> AutomationJobRecord | None:
|
|
370
|
+
result = await self._session.execute(
|
|
371
|
+
select(AutomationJob).where(AutomationJob.id == job_id).options(selectinload(AutomationJob.account_links))
|
|
372
|
+
)
|
|
373
|
+
job = result.scalar_one_or_none()
|
|
374
|
+
if job is None:
|
|
375
|
+
return None
|
|
376
|
+
|
|
377
|
+
if name is not None:
|
|
378
|
+
job.name = name
|
|
379
|
+
if enabled is not None:
|
|
380
|
+
job.enabled = enabled
|
|
381
|
+
if schedule_type is not None:
|
|
382
|
+
job.schedule_type = schedule_type
|
|
383
|
+
if schedule_time is not None:
|
|
384
|
+
job.schedule_time = schedule_time
|
|
385
|
+
if schedule_timezone is not None:
|
|
386
|
+
job.schedule_timezone = schedule_timezone
|
|
387
|
+
if schedule_days is not None:
|
|
388
|
+
job.schedule_days = _serialize_schedule_days(schedule_days)
|
|
389
|
+
if schedule_threshold_minutes is not None:
|
|
390
|
+
job.schedule_threshold_minutes = schedule_threshold_minutes
|
|
391
|
+
if include_paused_accounts is not None:
|
|
392
|
+
job.include_paused_accounts = include_paused_accounts
|
|
393
|
+
if model is not None:
|
|
394
|
+
job.model = model
|
|
395
|
+
if reasoning_effort_set:
|
|
396
|
+
job.reasoning_effort = reasoning_effort
|
|
397
|
+
if prompt is not None:
|
|
398
|
+
job.prompt = prompt
|
|
399
|
+
if account_ids is not None:
|
|
400
|
+
current_account_ids = [
|
|
401
|
+
link.account_id for link in sorted(job.account_links, key=lambda link: link.position)
|
|
402
|
+
]
|
|
403
|
+
next_account_ids = list(account_ids)
|
|
404
|
+
job.account_scope_all = not bool(next_account_ids)
|
|
405
|
+
if current_account_ids != next_account_ids:
|
|
406
|
+
job.updated_at = utcnow()
|
|
407
|
+
await self._session.execute(delete(AutomationJobAccount).where(AutomationJobAccount.job_id == job.id))
|
|
408
|
+
await self._session.flush()
|
|
409
|
+
self._session.add_all(
|
|
410
|
+
[
|
|
411
|
+
AutomationJobAccount(job_id=job.id, account_id=account_id, position=index)
|
|
412
|
+
for index, account_id in enumerate(next_account_ids)
|
|
413
|
+
]
|
|
414
|
+
)
|
|
415
|
+
|
|
416
|
+
await self._session.commit()
|
|
417
|
+
await self._session.refresh(job)
|
|
418
|
+
await self._session.refresh(job, attribute_names=["account_links"])
|
|
419
|
+
return self._job_from_model(job)
|
|
420
|
+
|
|
421
|
+
async def delete_job(self, job_id: str) -> bool:
|
|
422
|
+
result = await self._session.execute(
|
|
423
|
+
delete(AutomationJob).where(AutomationJob.id == job_id).returning(AutomationJob.id)
|
|
424
|
+
)
|
|
425
|
+
await self._session.commit()
|
|
426
|
+
return result.scalar_one_or_none() is not None
|
|
427
|
+
|
|
428
|
+
async def list_existing_account_ids(self, account_ids: Sequence[str]) -> set[str]:
|
|
429
|
+
if not account_ids:
|
|
430
|
+
return set()
|
|
431
|
+
result = await self._session.execute(select(Account.id).where(Account.id.in_(list(account_ids))))
|
|
432
|
+
return set(result.scalars().all())
|
|
433
|
+
|
|
434
|
+
async def claim_run(
|
|
435
|
+
self,
|
|
436
|
+
*,
|
|
437
|
+
job_id: str,
|
|
438
|
+
trigger: str,
|
|
439
|
+
slot_key: str,
|
|
440
|
+
cycle_key: str,
|
|
441
|
+
cycle_expected_accounts: int | None,
|
|
442
|
+
cycle_window_end: datetime | None,
|
|
443
|
+
scheduled_for: datetime,
|
|
444
|
+
started_at: datetime,
|
|
445
|
+
account_id: str | None = None,
|
|
446
|
+
) -> AutomationRunRecord | None:
|
|
447
|
+
model, reasoning_effort, prompt = await self._get_job_snapshot(job_id)
|
|
448
|
+
run = AutomationRun(
|
|
449
|
+
id=f"run_{uuid4().hex}",
|
|
450
|
+
job_id=job_id,
|
|
451
|
+
trigger=trigger,
|
|
452
|
+
slot_key=slot_key,
|
|
453
|
+
cycle_key=cycle_key,
|
|
454
|
+
cycle_expected_accounts=cycle_expected_accounts,
|
|
455
|
+
cycle_window_end=cycle_window_end,
|
|
456
|
+
model=model,
|
|
457
|
+
reasoning_effort=reasoning_effort,
|
|
458
|
+
prompt=prompt,
|
|
459
|
+
scheduled_for=scheduled_for,
|
|
460
|
+
started_at=started_at,
|
|
461
|
+
status="running",
|
|
462
|
+
account_id=account_id,
|
|
463
|
+
attempt_count=0,
|
|
464
|
+
claim_budget_seconds=effective_compact_request_budget_seconds(),
|
|
465
|
+
)
|
|
466
|
+
self._session.add(run)
|
|
467
|
+
try:
|
|
468
|
+
await self._session.commit()
|
|
469
|
+
except IntegrityError:
|
|
470
|
+
await self._session.rollback()
|
|
471
|
+
return None
|
|
472
|
+
await self._session.refresh(run)
|
|
473
|
+
return self._run_from_model(run)
|
|
474
|
+
|
|
475
|
+
async def claim_scheduled_cycle_account_run(
|
|
476
|
+
self,
|
|
477
|
+
*,
|
|
478
|
+
job_id: str,
|
|
479
|
+
trigger: str,
|
|
480
|
+
slot_key: str,
|
|
481
|
+
cycle_key: str,
|
|
482
|
+
cycle_expected_accounts: int | None,
|
|
483
|
+
cycle_window_end: datetime | None,
|
|
484
|
+
scheduled_for: datetime,
|
|
485
|
+
started_at: datetime,
|
|
486
|
+
account_id: str,
|
|
487
|
+
) -> AutomationScheduledRunClaimRecord:
|
|
488
|
+
model, reasoning_effort, prompt = await self._get_job_snapshot(job_id)
|
|
489
|
+
run_id = f"run_{uuid4().hex}"
|
|
490
|
+
stmt = (
|
|
491
|
+
insert(AutomationRun)
|
|
492
|
+
.from_select(
|
|
493
|
+
[
|
|
494
|
+
"id",
|
|
495
|
+
"job_id",
|
|
496
|
+
"trigger",
|
|
497
|
+
"slot_key",
|
|
498
|
+
"cycle_key",
|
|
499
|
+
"cycle_expected_accounts",
|
|
500
|
+
"cycle_window_end",
|
|
501
|
+
"model",
|
|
502
|
+
"reasoning_effort",
|
|
503
|
+
"prompt",
|
|
504
|
+
"scheduled_for",
|
|
505
|
+
"started_at",
|
|
506
|
+
"status",
|
|
507
|
+
"account_id",
|
|
508
|
+
"attempt_count",
|
|
509
|
+
"claim_budget_seconds",
|
|
510
|
+
],
|
|
511
|
+
select(
|
|
512
|
+
literal(run_id),
|
|
513
|
+
literal(job_id),
|
|
514
|
+
literal(trigger),
|
|
515
|
+
literal(slot_key),
|
|
516
|
+
literal(cycle_key),
|
|
517
|
+
literal(cycle_expected_accounts),
|
|
518
|
+
literal(cycle_window_end),
|
|
519
|
+
literal(model),
|
|
520
|
+
literal(reasoning_effort),
|
|
521
|
+
literal(prompt),
|
|
522
|
+
literal(scheduled_for),
|
|
523
|
+
literal(started_at),
|
|
524
|
+
literal("running"),
|
|
525
|
+
literal(account_id),
|
|
526
|
+
literal(0),
|
|
527
|
+
literal(effective_compact_request_budget_seconds()),
|
|
528
|
+
).where(
|
|
529
|
+
exists(
|
|
530
|
+
select(AutomationRunCycleAccount.account_id)
|
|
531
|
+
.where(AutomationRunCycleAccount.cycle_key == cycle_key)
|
|
532
|
+
.where(AutomationRunCycleAccount.account_id == account_id)
|
|
533
|
+
)
|
|
534
|
+
),
|
|
535
|
+
)
|
|
536
|
+
.returning(AutomationRun)
|
|
537
|
+
)
|
|
538
|
+
try:
|
|
539
|
+
result = await self._session.execute(stmt)
|
|
540
|
+
run = result.scalar_one_or_none()
|
|
541
|
+
await self._session.commit()
|
|
542
|
+
except IntegrityError:
|
|
543
|
+
await self._session.rollback()
|
|
544
|
+
snapshot_account_exists = await self.run_cycle_account_exists(cycle_key=cycle_key, account_id=account_id)
|
|
545
|
+
return AutomationScheduledRunClaimRecord(run=None, snapshot_account_exists=snapshot_account_exists)
|
|
546
|
+
if run is None:
|
|
547
|
+
return AutomationScheduledRunClaimRecord(run=None, snapshot_account_exists=False)
|
|
548
|
+
return AutomationScheduledRunClaimRecord(run=self._run_from_model(run), snapshot_account_exists=True)
|
|
549
|
+
|
|
550
|
+
async def get_run_cycle(self, *, cycle_key: str) -> AutomationRunCycleRecord | None:
|
|
551
|
+
result = await self._session.execute(
|
|
552
|
+
select(AutomationRunCycle)
|
|
553
|
+
.where(AutomationRunCycle.cycle_key == cycle_key)
|
|
554
|
+
.options(selectinload(AutomationRunCycle.cycle_accounts))
|
|
555
|
+
.execution_options(populate_existing=True)
|
|
556
|
+
.limit(1)
|
|
557
|
+
)
|
|
558
|
+
cycle = result.scalar_one_or_none()
|
|
559
|
+
if cycle is None:
|
|
560
|
+
return None
|
|
561
|
+
return self._run_cycle_from_model(cycle)
|
|
562
|
+
|
|
563
|
+
async def list_due_scheduled_run_cycles(
|
|
564
|
+
self,
|
|
565
|
+
*,
|
|
566
|
+
job_id: str,
|
|
567
|
+
now_utc: datetime,
|
|
568
|
+
limit: int = 500,
|
|
569
|
+
) -> list[AutomationRunCycleRecord]:
|
|
570
|
+
batch_size = max(limit, 500)
|
|
571
|
+
offset = 0
|
|
572
|
+
due_cycles: list[AutomationRunCycleRecord] = []
|
|
573
|
+
while len(due_cycles) < limit:
|
|
574
|
+
candidates = await self._list_due_scheduled_run_cycle_candidates(
|
|
575
|
+
job_id=job_id,
|
|
576
|
+
limit=batch_size,
|
|
577
|
+
offset=offset,
|
|
578
|
+
)
|
|
579
|
+
if not candidates:
|
|
580
|
+
break
|
|
581
|
+
due_cycles.extend(await self._filter_due_scheduled_run_cycles(candidates, now_utc=now_utc))
|
|
582
|
+
if len(candidates) < batch_size:
|
|
583
|
+
break
|
|
584
|
+
offset += batch_size
|
|
585
|
+
return due_cycles[:limit]
|
|
586
|
+
|
|
587
|
+
async def _list_due_scheduled_run_cycle_candidates(
|
|
588
|
+
self,
|
|
589
|
+
*,
|
|
590
|
+
job_id: str | None,
|
|
591
|
+
limit: int,
|
|
592
|
+
offset: int,
|
|
593
|
+
) -> list[AutomationRunCycleRecord]:
|
|
594
|
+
stmt = (
|
|
595
|
+
select(AutomationRunCycle)
|
|
596
|
+
.where(AutomationRunCycle.trigger == "scheduled")
|
|
597
|
+
.options(selectinload(AutomationRunCycle.cycle_accounts))
|
|
598
|
+
.execution_options(populate_existing=True)
|
|
599
|
+
.order_by(AutomationRunCycle.created_at.asc(), AutomationRunCycle.cycle_key.asc())
|
|
600
|
+
.offset(offset)
|
|
601
|
+
.limit(limit)
|
|
602
|
+
)
|
|
603
|
+
if job_id is not None:
|
|
604
|
+
stmt = stmt.where(AutomationRunCycle.job_id == job_id)
|
|
605
|
+
result = await self._session.execute(stmt)
|
|
606
|
+
return [self._run_cycle_from_model(cycle) for cycle in result.scalars().all()]
|
|
607
|
+
|
|
608
|
+
async def _filter_due_scheduled_run_cycles(
|
|
609
|
+
self,
|
|
610
|
+
candidates: list[AutomationRunCycleRecord],
|
|
611
|
+
*,
|
|
612
|
+
now_utc: datetime,
|
|
613
|
+
) -> list[AutomationRunCycleRecord]:
|
|
614
|
+
if not candidates:
|
|
615
|
+
return []
|
|
616
|
+
cycle_keys = [cycle.cycle_key for cycle in candidates]
|
|
617
|
+
fallback_budget_seconds = effective_compact_request_budget_seconds()
|
|
618
|
+
result = await self._session.execute(
|
|
619
|
+
select(
|
|
620
|
+
AutomationRun.cycle_key,
|
|
621
|
+
AutomationRun.slot_key,
|
|
622
|
+
AutomationRun.status,
|
|
623
|
+
AutomationRun.finished_at,
|
|
624
|
+
AutomationRun.started_at,
|
|
625
|
+
AutomationRun.scheduled_for,
|
|
626
|
+
AutomationRun.claim_budget_seconds,
|
|
627
|
+
).where(AutomationRun.cycle_key.in_(cycle_keys))
|
|
628
|
+
)
|
|
629
|
+
occupied_slot_keys_by_cycle_key: dict[str, set[str]] = {}
|
|
630
|
+
for cycle_key, slot_key, status, finished_at, started_at, scheduled_for, claim_budget_seconds in result.all():
|
|
631
|
+
is_stale_running = (
|
|
632
|
+
status == "running"
|
|
633
|
+
and finished_at is None
|
|
634
|
+
and (
|
|
635
|
+
started_at <= scheduled_for
|
|
636
|
+
or started_at
|
|
637
|
+
< run_stale_started_before(
|
|
638
|
+
now_utc=now_utc,
|
|
639
|
+
claim_budget_seconds=claim_budget_seconds,
|
|
640
|
+
fallback_budget_seconds=fallback_budget_seconds,
|
|
641
|
+
)
|
|
642
|
+
)
|
|
643
|
+
)
|
|
644
|
+
if is_stale_running:
|
|
645
|
+
continue
|
|
646
|
+
occupied_slot_keys_by_cycle_key.setdefault(cycle_key, set()).add(slot_key)
|
|
647
|
+
|
|
648
|
+
due_cycles: list[AutomationRunCycleRecord] = []
|
|
649
|
+
for cycle in candidates:
|
|
650
|
+
due_slot = _parse_scheduled_cycle_due_slot(cycle.cycle_key, job_id=cycle.job_id)
|
|
651
|
+
if due_slot is None:
|
|
652
|
+
due_cycles.append(cycle)
|
|
653
|
+
continue
|
|
654
|
+
occupied_slot_keys = occupied_slot_keys_by_cycle_key.get(cycle.cycle_key, set())
|
|
655
|
+
if not cycle.accounts:
|
|
656
|
+
if not occupied_slot_keys and due_slot <= now_utc:
|
|
657
|
+
due_cycles.append(cycle)
|
|
658
|
+
continue
|
|
659
|
+
has_due_account = False
|
|
660
|
+
for cycle_account in cycle.accounts:
|
|
661
|
+
if cycle_account.scheduled_for > now_utc:
|
|
662
|
+
continue
|
|
663
|
+
slot_key = _scheduled_slot_key(
|
|
664
|
+
cycle.job_id,
|
|
665
|
+
account_id=cycle_account.account_id,
|
|
666
|
+
due_slot=due_slot,
|
|
667
|
+
)
|
|
668
|
+
if slot_key in occupied_slot_keys:
|
|
669
|
+
continue
|
|
670
|
+
has_due_account = True
|
|
671
|
+
break
|
|
672
|
+
if has_due_account:
|
|
673
|
+
due_cycles.append(cycle)
|
|
674
|
+
return due_cycles
|
|
675
|
+
|
|
676
|
+
async def create_run_cycle(
|
|
677
|
+
self,
|
|
678
|
+
*,
|
|
679
|
+
cycle_key: str,
|
|
680
|
+
job_id: str,
|
|
681
|
+
trigger: str,
|
|
682
|
+
cycle_expected_accounts: int,
|
|
683
|
+
cycle_window_end: datetime | None,
|
|
684
|
+
accounts: Sequence[tuple[str, datetime]],
|
|
685
|
+
include_paused_accounts: bool = False,
|
|
686
|
+
) -> AutomationRunCycleRecord:
|
|
687
|
+
cycle = AutomationRunCycle(
|
|
688
|
+
cycle_key=cycle_key,
|
|
689
|
+
job_id=job_id,
|
|
690
|
+
trigger=trigger,
|
|
691
|
+
cycle_expected_accounts=cycle_expected_accounts,
|
|
692
|
+
cycle_window_end=cycle_window_end,
|
|
693
|
+
include_paused_accounts=include_paused_accounts,
|
|
694
|
+
)
|
|
695
|
+
due_slot = _parse_scheduled_cycle_due_slot(cycle_key, job_id=job_id) if trigger == "scheduled" else None
|
|
696
|
+
cycle.cycle_accounts = [
|
|
697
|
+
AutomationRunCycleAccount(
|
|
698
|
+
cycle_key=cycle_key,
|
|
699
|
+
account_id=account_id,
|
|
700
|
+
slot_key=_cycle_account_slot_key(
|
|
701
|
+
job_id=job_id,
|
|
702
|
+
trigger=trigger,
|
|
703
|
+
account_id=account_id,
|
|
704
|
+
due_slot=due_slot,
|
|
705
|
+
),
|
|
706
|
+
position=index,
|
|
707
|
+
scheduled_for=scheduled_for,
|
|
708
|
+
)
|
|
709
|
+
for index, (account_id, scheduled_for) in enumerate(accounts)
|
|
710
|
+
]
|
|
711
|
+
self._session.add(cycle)
|
|
712
|
+
try:
|
|
713
|
+
await self._session.commit()
|
|
714
|
+
except IntegrityError:
|
|
715
|
+
await self._session.rollback()
|
|
716
|
+
stored_cycle = await self.get_run_cycle(cycle_key=cycle_key)
|
|
717
|
+
if stored_cycle is None:
|
|
718
|
+
raise LookupError(f"Automation run cycle not found: {cycle_key}")
|
|
719
|
+
return stored_cycle
|
|
720
|
+
|
|
721
|
+
async def create_run_cycle_with_runs(
|
|
722
|
+
self,
|
|
723
|
+
*,
|
|
724
|
+
cycle_key: str,
|
|
725
|
+
job_id: str,
|
|
726
|
+
trigger: str,
|
|
727
|
+
cycle_expected_accounts: int,
|
|
728
|
+
cycle_window_end: datetime | None,
|
|
729
|
+
accounts: Sequence[tuple[str, datetime]],
|
|
730
|
+
runs: Sequence[tuple[str, datetime, str | None]],
|
|
731
|
+
started_at: datetime,
|
|
732
|
+
include_paused_accounts: bool = False,
|
|
733
|
+
) -> tuple[AutomationRunCycleRecord, list[AutomationRunRecord]]:
|
|
734
|
+
model, reasoning_effort, prompt = await self._get_job_snapshot(job_id)
|
|
735
|
+
cycle = AutomationRunCycle(
|
|
736
|
+
cycle_key=cycle_key,
|
|
737
|
+
job_id=job_id,
|
|
738
|
+
trigger=trigger,
|
|
739
|
+
cycle_expected_accounts=cycle_expected_accounts,
|
|
740
|
+
cycle_window_end=cycle_window_end,
|
|
741
|
+
include_paused_accounts=include_paused_accounts,
|
|
742
|
+
)
|
|
743
|
+
due_slot = _parse_scheduled_cycle_due_slot(cycle_key, job_id=job_id) if trigger == "scheduled" else None
|
|
744
|
+
cycle.cycle_accounts = [
|
|
745
|
+
AutomationRunCycleAccount(
|
|
746
|
+
cycle_key=cycle_key,
|
|
747
|
+
account_id=account_id,
|
|
748
|
+
slot_key=_cycle_account_slot_key(
|
|
749
|
+
job_id=job_id,
|
|
750
|
+
trigger=trigger,
|
|
751
|
+
account_id=account_id,
|
|
752
|
+
due_slot=due_slot,
|
|
753
|
+
),
|
|
754
|
+
position=index,
|
|
755
|
+
scheduled_for=scheduled_for,
|
|
756
|
+
)
|
|
757
|
+
for index, (account_id, scheduled_for) in enumerate(accounts)
|
|
758
|
+
]
|
|
759
|
+
claim_budget_seconds = effective_compact_request_budget_seconds()
|
|
760
|
+
claimed_runs = [
|
|
761
|
+
AutomationRun(
|
|
762
|
+
id=f"run_{uuid4().hex}",
|
|
763
|
+
job_id=job_id,
|
|
764
|
+
trigger=trigger,
|
|
765
|
+
slot_key=slot_key,
|
|
766
|
+
cycle_key=cycle_key,
|
|
767
|
+
cycle_expected_accounts=cycle_expected_accounts,
|
|
768
|
+
cycle_window_end=cycle_window_end,
|
|
769
|
+
model=model,
|
|
770
|
+
reasoning_effort=reasoning_effort,
|
|
771
|
+
prompt=prompt,
|
|
772
|
+
scheduled_for=scheduled_for,
|
|
773
|
+
started_at=started_at,
|
|
774
|
+
status="running",
|
|
775
|
+
account_id=account_id,
|
|
776
|
+
attempt_count=0,
|
|
777
|
+
claim_budget_seconds=claim_budget_seconds,
|
|
778
|
+
)
|
|
779
|
+
for slot_key, scheduled_for, account_id in runs
|
|
780
|
+
]
|
|
781
|
+
self._session.add(cycle)
|
|
782
|
+
self._session.add_all(claimed_runs)
|
|
783
|
+
try:
|
|
784
|
+
await self._session.commit()
|
|
785
|
+
except IntegrityError:
|
|
786
|
+
await self._session.rollback()
|
|
787
|
+
raise
|
|
788
|
+
stored_cycle = await self.get_run_cycle(cycle_key=cycle_key)
|
|
789
|
+
if stored_cycle is None:
|
|
790
|
+
raise LookupError(f"Automation run cycle not found: {cycle_key}")
|
|
791
|
+
return stored_cycle, [self._run_from_model(run) for run in claimed_runs]
|
|
792
|
+
|
|
793
|
+
async def complete_run(
|
|
794
|
+
self,
|
|
795
|
+
run_id: str,
|
|
796
|
+
*,
|
|
797
|
+
status: str,
|
|
798
|
+
finished_at: datetime,
|
|
799
|
+
account_id: str | None,
|
|
800
|
+
error_code: str | None,
|
|
801
|
+
error_message: str | None,
|
|
802
|
+
attempt_count: int,
|
|
803
|
+
) -> AutomationRunRecord:
|
|
804
|
+
run = await self._session.get(AutomationRun, run_id)
|
|
805
|
+
if run is None:
|
|
806
|
+
raise LookupError(f"Automation run not found: {run_id}")
|
|
807
|
+
run.status = status
|
|
808
|
+
run.finished_at = finished_at
|
|
809
|
+
run.account_id = account_id
|
|
810
|
+
run.error_code = error_code
|
|
811
|
+
run.error_message = error_message
|
|
812
|
+
run.attempt_count = attempt_count
|
|
813
|
+
await self._session.commit()
|
|
814
|
+
await self._session.refresh(run)
|
|
815
|
+
return self._run_from_model(run)
|
|
816
|
+
|
|
817
|
+
async def list_runs(self, job_id: str, *, limit: int) -> list[AutomationRunRecord]:
|
|
818
|
+
result = await self._session.execute(
|
|
819
|
+
select(AutomationRun, AutomationJob.name, AutomationJob.model, AutomationJob.reasoning_effort)
|
|
820
|
+
.join(AutomationJob, AutomationJob.id == AutomationRun.job_id)
|
|
821
|
+
.where(AutomationRun.job_id == job_id)
|
|
822
|
+
.order_by(AutomationRun.started_at.desc(), AutomationRun.id.desc())
|
|
823
|
+
.limit(limit)
|
|
824
|
+
)
|
|
825
|
+
return [
|
|
826
|
+
self._run_from_model(run, job_name=job_name, model=model, reasoning_effort=reasoning_effort)
|
|
827
|
+
for run, job_name, model, reasoning_effort in result.all()
|
|
828
|
+
]
|
|
829
|
+
|
|
830
|
+
async def _get_job_snapshot(self, job_id: str) -> tuple[str | None, str | None, str | None]:
|
|
831
|
+
result = await self._session.execute(
|
|
832
|
+
select(AutomationJob.model, AutomationJob.reasoning_effort, AutomationJob.prompt)
|
|
833
|
+
.where(AutomationJob.id == job_id)
|
|
834
|
+
.limit(1)
|
|
835
|
+
)
|
|
836
|
+
row = result.one_or_none()
|
|
837
|
+
if row is None:
|
|
838
|
+
return None, None, None
|
|
839
|
+
return row[0], row[1], row[2]
|
|
840
|
+
|
|
841
|
+
async def get_run(self, run_id: str) -> AutomationRunRecord | None:
|
|
842
|
+
result = await self._session.execute(
|
|
843
|
+
select(AutomationRun, AutomationJob.name, AutomationJob.model, AutomationJob.reasoning_effort)
|
|
844
|
+
.join(AutomationJob, AutomationJob.id == AutomationRun.job_id)
|
|
845
|
+
.where(AutomationRun.id == run_id)
|
|
846
|
+
.limit(1)
|
|
847
|
+
)
|
|
848
|
+
row = result.one_or_none()
|
|
849
|
+
if row is None:
|
|
850
|
+
return None
|
|
851
|
+
run, job_name, model, reasoning_effort = row
|
|
852
|
+
return self._run_from_model(run, job_name=job_name, model=model, reasoning_effort=reasoning_effort)
|
|
853
|
+
|
|
854
|
+
async def list_runs_for_cycle_key(self, *, cycle_key: str) -> list[AutomationRunRecord]:
|
|
855
|
+
result = await self._session.execute(
|
|
856
|
+
select(AutomationRun, AutomationJob.name, AutomationJob.model, AutomationJob.reasoning_effort)
|
|
857
|
+
.join(AutomationJob, AutomationJob.id == AutomationRun.job_id)
|
|
858
|
+
.where(AutomationRun.cycle_key == cycle_key)
|
|
859
|
+
.order_by(AutomationRun.started_at.desc(), AutomationRun.id.desc())
|
|
860
|
+
)
|
|
861
|
+
return [
|
|
862
|
+
self._run_from_model(run, job_name=job_name, model=model, reasoning_effort=reasoning_effort)
|
|
863
|
+
for run, job_name, model, reasoning_effort in result.all()
|
|
864
|
+
]
|
|
865
|
+
|
|
866
|
+
async def list_due_manual_runs(
|
|
867
|
+
self,
|
|
868
|
+
*,
|
|
869
|
+
now_utc: datetime,
|
|
870
|
+
cycle_key: str | None = None,
|
|
871
|
+
limit: int = 500,
|
|
872
|
+
) -> list[AutomationRunRecord]:
|
|
873
|
+
# The SQL bound is the shortest reclaim window any row can have; the
|
|
874
|
+
# exact per-row window (pinned at claim time, current budget for
|
|
875
|
+
# legacy NULL rows) is applied to the fetched candidates below.
|
|
876
|
+
earliest_stale_started_before = now_utc - timedelta(seconds=RUN_CLAIM_MIN_TIMEOUT_SECONDS)
|
|
877
|
+
fallback_budget_seconds = effective_compact_request_budget_seconds()
|
|
878
|
+
stmt = (
|
|
879
|
+
select(AutomationRun, AutomationJob.name, AutomationJob.model, AutomationJob.reasoning_effort)
|
|
880
|
+
.join(AutomationJob, AutomationJob.id == AutomationRun.job_id)
|
|
881
|
+
.outerjoin(
|
|
882
|
+
AutomationRunCycleAccount,
|
|
883
|
+
and_(
|
|
884
|
+
AutomationRunCycleAccount.cycle_key == AutomationRun.cycle_key,
|
|
885
|
+
AutomationRunCycleAccount.account_id == AutomationRun.account_id,
|
|
886
|
+
),
|
|
887
|
+
)
|
|
888
|
+
.where(AutomationRun.trigger == "manual")
|
|
889
|
+
.where(AutomationRun.status == "running")
|
|
890
|
+
.where(AutomationRun.finished_at.is_(None))
|
|
891
|
+
.where(AutomationRun.scheduled_for <= now_utc)
|
|
892
|
+
.where(
|
|
893
|
+
or_(
|
|
894
|
+
AutomationRun.started_at <= AutomationRun.scheduled_for,
|
|
895
|
+
AutomationRun.started_at < earliest_stale_started_before,
|
|
896
|
+
)
|
|
897
|
+
)
|
|
898
|
+
.order_by(
|
|
899
|
+
AutomationRun.scheduled_for.asc(),
|
|
900
|
+
func.coalesce(AutomationRunCycleAccount.position, 2147483647).asc(),
|
|
901
|
+
AutomationRun.started_at.asc(),
|
|
902
|
+
AutomationRun.id.asc(),
|
|
903
|
+
)
|
|
904
|
+
)
|
|
905
|
+
if cycle_key is not None:
|
|
906
|
+
stmt = stmt.where(AutomationRun.cycle_key == cycle_key)
|
|
907
|
+
# Page through the candidates so claims still inside their own window
|
|
908
|
+
# never consume ``limit`` slots owed to eligible rows behind them.
|
|
909
|
+
due_runs: list[AutomationRunRecord] = []
|
|
910
|
+
offset = 0
|
|
911
|
+
while len(due_runs) < limit:
|
|
912
|
+
result = await self._session.execute(stmt.offset(offset).limit(limit))
|
|
913
|
+
rows = result.all()
|
|
914
|
+
due_runs.extend(
|
|
915
|
+
run
|
|
916
|
+
for run in (
|
|
917
|
+
self._run_from_model(run, job_name=job_name, model=model, reasoning_effort=reasoning_effort)
|
|
918
|
+
for run, job_name, model, reasoning_effort in rows
|
|
919
|
+
)
|
|
920
|
+
if run.started_at <= run.scheduled_for
|
|
921
|
+
or run.started_at
|
|
922
|
+
< run_stale_started_before(
|
|
923
|
+
now_utc=now_utc,
|
|
924
|
+
claim_budget_seconds=run.claim_budget_seconds,
|
|
925
|
+
fallback_budget_seconds=fallback_budget_seconds,
|
|
926
|
+
)
|
|
927
|
+
)
|
|
928
|
+
if len(rows) < limit:
|
|
929
|
+
break
|
|
930
|
+
offset += limit
|
|
931
|
+
return due_runs[:limit]
|
|
932
|
+
|
|
933
|
+
async def claim_manual_run_execution(
|
|
934
|
+
self,
|
|
935
|
+
run_id: str,
|
|
936
|
+
*,
|
|
937
|
+
observed_started_at: datetime,
|
|
938
|
+
claimed_started_at: datetime,
|
|
939
|
+
stale_started_before: datetime,
|
|
940
|
+
) -> AutomationRunRecord | None:
|
|
941
|
+
result = await self._session.execute(
|
|
942
|
+
update(AutomationRun)
|
|
943
|
+
.where(AutomationRun.id == run_id)
|
|
944
|
+
.where(AutomationRun.trigger == "manual")
|
|
945
|
+
.where(AutomationRun.status == "running")
|
|
946
|
+
.where(AutomationRun.finished_at.is_(None))
|
|
947
|
+
.where(AutomationRun.account_id.is_not(None))
|
|
948
|
+
.where(AutomationRun.started_at == observed_started_at)
|
|
949
|
+
.where(
|
|
950
|
+
or_(
|
|
951
|
+
AutomationRun.started_at <= AutomationRun.scheduled_for,
|
|
952
|
+
AutomationRun.started_at < stale_started_before,
|
|
953
|
+
)
|
|
954
|
+
)
|
|
955
|
+
.values(
|
|
956
|
+
started_at=claimed_started_at,
|
|
957
|
+
claim_budget_seconds=effective_compact_request_budget_seconds(),
|
|
958
|
+
)
|
|
959
|
+
.returning(AutomationRun)
|
|
960
|
+
)
|
|
961
|
+
run = result.scalar_one_or_none()
|
|
962
|
+
await self._session.commit()
|
|
963
|
+
if run is None:
|
|
964
|
+
return None
|
|
965
|
+
return self._run_from_model(run)
|
|
966
|
+
|
|
967
|
+
async def claim_scheduled_cycle_run_execution(
|
|
968
|
+
self,
|
|
969
|
+
run_id: str,
|
|
970
|
+
*,
|
|
971
|
+
observed_started_at: datetime,
|
|
972
|
+
claimed_started_at: datetime,
|
|
973
|
+
stale_started_before: datetime,
|
|
974
|
+
) -> AutomationRunRecord | None:
|
|
975
|
+
result = await self._session.execute(
|
|
976
|
+
update(AutomationRun)
|
|
977
|
+
.where(AutomationRun.id == run_id)
|
|
978
|
+
.where(AutomationRun.trigger == "scheduled")
|
|
979
|
+
.where(AutomationRun.status == "running")
|
|
980
|
+
.where(AutomationRun.finished_at.is_(None))
|
|
981
|
+
.where(AutomationRun.started_at == observed_started_at)
|
|
982
|
+
.where(
|
|
983
|
+
or_(
|
|
984
|
+
AutomationRun.started_at <= AutomationRun.scheduled_for,
|
|
985
|
+
AutomationRun.started_at < stale_started_before,
|
|
986
|
+
)
|
|
987
|
+
)
|
|
988
|
+
.values(
|
|
989
|
+
started_at=claimed_started_at,
|
|
990
|
+
claim_budget_seconds=effective_compact_request_budget_seconds(),
|
|
991
|
+
)
|
|
992
|
+
.returning(AutomationRun)
|
|
993
|
+
)
|
|
994
|
+
run = result.scalar_one_or_none()
|
|
995
|
+
await self._session.commit()
|
|
996
|
+
if run is None:
|
|
997
|
+
return None
|
|
998
|
+
return self._run_from_model(run)
|
|
999
|
+
|
|
1000
|
+
async def skip_unclaimed_manual_run_placeholder(
|
|
1001
|
+
self,
|
|
1002
|
+
run_id: str,
|
|
1003
|
+
*,
|
|
1004
|
+
cycle_key: str,
|
|
1005
|
+
account_id: str,
|
|
1006
|
+
observed_started_at: datetime,
|
|
1007
|
+
skipped_at: datetime,
|
|
1008
|
+
) -> bool:
|
|
1009
|
+
result = await self._session.execute(
|
|
1010
|
+
update(AutomationRun)
|
|
1011
|
+
.where(AutomationRun.id == run_id)
|
|
1012
|
+
.where(AutomationRun.trigger == "manual")
|
|
1013
|
+
.where(AutomationRun.status == "running")
|
|
1014
|
+
.where(AutomationRun.finished_at.is_(None))
|
|
1015
|
+
.where(or_(AutomationRun.account_id == account_id, AutomationRun.account_id.is_(None)))
|
|
1016
|
+
.where(AutomationRun.started_at == observed_started_at)
|
|
1017
|
+
.where(AutomationRun.attempt_count == 0)
|
|
1018
|
+
.where(AutomationRun.started_at <= AutomationRun.scheduled_for)
|
|
1019
|
+
.values(
|
|
1020
|
+
status="partial",
|
|
1021
|
+
finished_at=skipped_at,
|
|
1022
|
+
account_id=None,
|
|
1023
|
+
error_code=None,
|
|
1024
|
+
error_message=None,
|
|
1025
|
+
)
|
|
1026
|
+
.returning(AutomationRun.id)
|
|
1027
|
+
)
|
|
1028
|
+
skipped_run_id = result.scalar_one_or_none()
|
|
1029
|
+
if skipped_run_id is None:
|
|
1030
|
+
await self._session.commit()
|
|
1031
|
+
return False
|
|
1032
|
+
|
|
1033
|
+
await self._session.execute(
|
|
1034
|
+
delete(AutomationRunCycleAccount)
|
|
1035
|
+
.where(AutomationRunCycleAccount.cycle_key == cycle_key)
|
|
1036
|
+
.where(AutomationRunCycleAccount.account_id == account_id)
|
|
1037
|
+
)
|
|
1038
|
+
await self._sync_run_cycle_expected_accounts(cycle_key=cycle_key)
|
|
1039
|
+
await self._session.commit()
|
|
1040
|
+
self._session.expire_all()
|
|
1041
|
+
return True
|
|
1042
|
+
|
|
1043
|
+
async def delete_run_cycle_account(self, *, cycle_key: str, account_id: str) -> bool:
|
|
1044
|
+
result = await self._session.execute(
|
|
1045
|
+
delete(AutomationRunCycleAccount)
|
|
1046
|
+
.where(AutomationRunCycleAccount.cycle_key == cycle_key)
|
|
1047
|
+
.where(AutomationRunCycleAccount.account_id == account_id)
|
|
1048
|
+
.returning(AutomationRunCycleAccount.account_id)
|
|
1049
|
+
)
|
|
1050
|
+
deleted_account_id = result.scalar_one_or_none()
|
|
1051
|
+
if deleted_account_id is not None:
|
|
1052
|
+
await self._sync_run_cycle_expected_accounts(cycle_key=cycle_key)
|
|
1053
|
+
await self._session.commit()
|
|
1054
|
+
self._session.expire_all()
|
|
1055
|
+
return deleted_account_id is not None
|
|
1056
|
+
|
|
1057
|
+
async def run_cycle_account_exists(self, *, cycle_key: str, account_id: str) -> bool:
|
|
1058
|
+
result = await self._session.execute(
|
|
1059
|
+
select(
|
|
1060
|
+
exists(
|
|
1061
|
+
select(AutomationRunCycleAccount.account_id)
|
|
1062
|
+
.where(AutomationRunCycleAccount.cycle_key == cycle_key)
|
|
1063
|
+
.where(AutomationRunCycleAccount.account_id == account_id)
|
|
1064
|
+
)
|
|
1065
|
+
)
|
|
1066
|
+
)
|
|
1067
|
+
return bool(result.scalar_one())
|
|
1068
|
+
|
|
1069
|
+
async def _sync_run_cycle_expected_accounts(self, *, cycle_key: str) -> None:
|
|
1070
|
+
remaining_count = int(
|
|
1071
|
+
(
|
|
1072
|
+
await self._session.execute(
|
|
1073
|
+
select(func.count(AutomationRunCycleAccount.account_id)).where(
|
|
1074
|
+
AutomationRunCycleAccount.cycle_key == cycle_key
|
|
1075
|
+
)
|
|
1076
|
+
)
|
|
1077
|
+
).scalar_one()
|
|
1078
|
+
or 0
|
|
1079
|
+
)
|
|
1080
|
+
await self._session.execute(
|
|
1081
|
+
update(AutomationRun)
|
|
1082
|
+
.where(AutomationRun.cycle_key == cycle_key)
|
|
1083
|
+
.values(cycle_expected_accounts=remaining_count)
|
|
1084
|
+
)
|
|
1085
|
+
await self._session.execute(
|
|
1086
|
+
update(AutomationRunCycle)
|
|
1087
|
+
.where(AutomationRunCycle.cycle_key == cycle_key)
|
|
1088
|
+
.values(cycle_expected_accounts=remaining_count)
|
|
1089
|
+
)
|
|
1090
|
+
|
|
1091
|
+
async def list_runs_page(
|
|
1092
|
+
self,
|
|
1093
|
+
*,
|
|
1094
|
+
limit: int,
|
|
1095
|
+
offset: int,
|
|
1096
|
+
search: str | None = None,
|
|
1097
|
+
account_ids: Sequence[str] | None = None,
|
|
1098
|
+
models: Sequence[str] | None = None,
|
|
1099
|
+
statuses: Sequence[str] | None = None,
|
|
1100
|
+
triggers: Sequence[str] | None = None,
|
|
1101
|
+
job_ids: Sequence[str] | None = None,
|
|
1102
|
+
) -> tuple[list[AutomationRunRecord], int]:
|
|
1103
|
+
conditions = self._build_run_conditions(
|
|
1104
|
+
search=search,
|
|
1105
|
+
account_ids=account_ids,
|
|
1106
|
+
models=models,
|
|
1107
|
+
statuses=statuses,
|
|
1108
|
+
triggers=triggers,
|
|
1109
|
+
job_ids=job_ids,
|
|
1110
|
+
)
|
|
1111
|
+
stmt = (
|
|
1112
|
+
select(AutomationRun, AutomationJob.name, AutomationJob.model, AutomationJob.reasoning_effort)
|
|
1113
|
+
.join(AutomationJob, AutomationJob.id == AutomationRun.job_id)
|
|
1114
|
+
.order_by(AutomationRun.started_at.desc(), AutomationRun.id.desc())
|
|
1115
|
+
.offset(offset)
|
|
1116
|
+
.limit(limit)
|
|
1117
|
+
)
|
|
1118
|
+
if conditions:
|
|
1119
|
+
stmt = stmt.where(and_(*conditions))
|
|
1120
|
+
result = await self._session.execute(stmt)
|
|
1121
|
+
runs = [
|
|
1122
|
+
self._run_from_model(run, job_name=job_name, model=model, reasoning_effort=reasoning_effort)
|
|
1123
|
+
for run, job_name, model, reasoning_effort in result.all()
|
|
1124
|
+
]
|
|
1125
|
+
|
|
1126
|
+
count_stmt = (
|
|
1127
|
+
select(func.count(AutomationRun.id))
|
|
1128
|
+
.select_from(AutomationRun)
|
|
1129
|
+
.join(AutomationJob, AutomationJob.id == AutomationRun.job_id)
|
|
1130
|
+
)
|
|
1131
|
+
if conditions:
|
|
1132
|
+
count_stmt = count_stmt.where(and_(*conditions))
|
|
1133
|
+
total = int((await self._session.execute(count_stmt)).scalar_one() or 0)
|
|
1134
|
+
return runs, total
|
|
1135
|
+
|
|
1136
|
+
def _build_grouped_run_candidates(
|
|
1137
|
+
self,
|
|
1138
|
+
*,
|
|
1139
|
+
search: str | None,
|
|
1140
|
+
account_ids: Sequence[str] | None,
|
|
1141
|
+
models: Sequence[str] | None,
|
|
1142
|
+
triggers: Sequence[str] | None,
|
|
1143
|
+
job_ids: Sequence[str] | None,
|
|
1144
|
+
) -> tuple[CTE, CTE]:
|
|
1145
|
+
conditions = self._build_run_conditions(
|
|
1146
|
+
search=search,
|
|
1147
|
+
account_ids=None,
|
|
1148
|
+
models=models,
|
|
1149
|
+
statuses=None,
|
|
1150
|
+
triggers=triggers,
|
|
1151
|
+
job_ids=job_ids,
|
|
1152
|
+
)
|
|
1153
|
+
filtered_runs_stmt = select(
|
|
1154
|
+
AutomationRun.id.label("run_id"),
|
|
1155
|
+
AutomationRun.cycle_key.label("cycle_key"),
|
|
1156
|
+
AutomationRun.status.label("status"),
|
|
1157
|
+
AutomationRun.started_at.label("started_at"),
|
|
1158
|
+
).join(AutomationJob, AutomationJob.id == AutomationRun.job_id)
|
|
1159
|
+
if conditions:
|
|
1160
|
+
filtered_runs_stmt = filtered_runs_stmt.where(and_(*conditions))
|
|
1161
|
+
grouped_account_match = self._build_grouped_run_account_match(account_ids=account_ids)
|
|
1162
|
+
if grouped_account_match is not None:
|
|
1163
|
+
filtered_runs_stmt = filtered_runs_stmt.where(grouped_account_match)
|
|
1164
|
+
filtered_runs = filtered_runs_stmt.cte("filtered_automation_runs")
|
|
1165
|
+
candidate_cycles = select(filtered_runs.c.cycle_key).distinct().cte("candidate_automation_run_cycles")
|
|
1166
|
+
ranked_representatives = select(
|
|
1167
|
+
filtered_runs.c.run_id,
|
|
1168
|
+
filtered_runs.c.cycle_key,
|
|
1169
|
+
filtered_runs.c.status.label("fallback_status"),
|
|
1170
|
+
func.row_number()
|
|
1171
|
+
.over(
|
|
1172
|
+
partition_by=filtered_runs.c.cycle_key,
|
|
1173
|
+
order_by=(filtered_runs.c.started_at.desc(), filtered_runs.c.run_id.desc()),
|
|
1174
|
+
)
|
|
1175
|
+
.label("cycle_rank"),
|
|
1176
|
+
).cte("ranked_automation_run_representatives")
|
|
1177
|
+
return candidate_cycles, ranked_representatives
|
|
1178
|
+
|
|
1179
|
+
def _build_lightweight_run_cycle_scope(
|
|
1180
|
+
self,
|
|
1181
|
+
*,
|
|
1182
|
+
search: str | None,
|
|
1183
|
+
account_ids: Sequence[str] | None,
|
|
1184
|
+
models: Sequence[str] | None,
|
|
1185
|
+
triggers: Sequence[str] | None,
|
|
1186
|
+
job_ids: Sequence[str] | None,
|
|
1187
|
+
) -> CTE:
|
|
1188
|
+
candidate_cycles, ranked_representatives = self._build_grouped_run_candidates(
|
|
1189
|
+
search=search,
|
|
1190
|
+
account_ids=account_ids,
|
|
1191
|
+
models=models,
|
|
1192
|
+
triggers=triggers,
|
|
1193
|
+
job_ids=job_ids,
|
|
1194
|
+
)
|
|
1195
|
+
cycle_order = (
|
|
1196
|
+
select(
|
|
1197
|
+
AutomationRun.cycle_key.label("cycle_key"),
|
|
1198
|
+
func.max(case((AutomationRun.trigger == "manual", 1), else_=0)).label("has_manual_trigger"),
|
|
1199
|
+
func.min(case((AutomationRun.trigger == "manual", AutomationRun.scheduled_for), else_=None)).label(
|
|
1200
|
+
"manual_cycle_started_at"
|
|
1201
|
+
),
|
|
1202
|
+
func.min(case((AutomationRun.trigger != "manual", AutomationRun.started_at), else_=None)).label(
|
|
1203
|
+
"non_manual_cycle_started_at"
|
|
1204
|
+
),
|
|
1205
|
+
)
|
|
1206
|
+
.join(candidate_cycles, candidate_cycles.c.cycle_key == AutomationRun.cycle_key)
|
|
1207
|
+
.group_by(AutomationRun.cycle_key)
|
|
1208
|
+
.cte("automation_run_cycle_order")
|
|
1209
|
+
)
|
|
1210
|
+
return (
|
|
1211
|
+
select(
|
|
1212
|
+
ranked_representatives.c.run_id,
|
|
1213
|
+
ranked_representatives.c.cycle_key,
|
|
1214
|
+
case(
|
|
1215
|
+
(cycle_order.c.has_manual_trigger == 1, cycle_order.c.manual_cycle_started_at),
|
|
1216
|
+
else_=cycle_order.c.non_manual_cycle_started_at,
|
|
1217
|
+
).label("cycle_started_at"),
|
|
1218
|
+
)
|
|
1219
|
+
.join(cycle_order, cycle_order.c.cycle_key == ranked_representatives.c.cycle_key)
|
|
1220
|
+
.where(ranked_representatives.c.cycle_rank == 1)
|
|
1221
|
+
.cte("automation_run_cycle_scope")
|
|
1222
|
+
)
|
|
1223
|
+
|
|
1224
|
+
def _build_full_run_cycle_scope(
|
|
1225
|
+
self,
|
|
1226
|
+
*,
|
|
1227
|
+
now_utc: datetime,
|
|
1228
|
+
search: str | None,
|
|
1229
|
+
account_ids: Sequence[str] | None,
|
|
1230
|
+
models: Sequence[str] | None,
|
|
1231
|
+
triggers: Sequence[str] | None,
|
|
1232
|
+
job_ids: Sequence[str] | None,
|
|
1233
|
+
) -> CTE:
|
|
1234
|
+
candidate_cycles, ranked_representatives = self._build_grouped_run_candidates(
|
|
1235
|
+
search=search,
|
|
1236
|
+
account_ids=account_ids,
|
|
1237
|
+
models=models,
|
|
1238
|
+
triggers=triggers,
|
|
1239
|
+
job_ids=job_ids,
|
|
1240
|
+
)
|
|
1241
|
+
cycle_runs = (
|
|
1242
|
+
select(
|
|
1243
|
+
AutomationRun.cycle_key.label("cycle_key"),
|
|
1244
|
+
AutomationRun.trigger.label("trigger"),
|
|
1245
|
+
AutomationRun.status.label("status"),
|
|
1246
|
+
AutomationRun.account_id.label("account_id"),
|
|
1247
|
+
AutomationRun.started_at.label("started_at"),
|
|
1248
|
+
AutomationRun.finished_at.label("finished_at"),
|
|
1249
|
+
AutomationRun.scheduled_for.label("scheduled_for"),
|
|
1250
|
+
AutomationRun.cycle_window_end.label("cycle_window_end"),
|
|
1251
|
+
AutomationRun.cycle_expected_accounts.label("cycle_expected_accounts"),
|
|
1252
|
+
AutomationRun.attempt_count.label("attempt_count"),
|
|
1253
|
+
AutomationJob.include_paused_accounts.label("include_paused_accounts"),
|
|
1254
|
+
Account.status.label("account_status"),
|
|
1255
|
+
)
|
|
1256
|
+
.join(AutomationJob, AutomationJob.id == AutomationRun.job_id)
|
|
1257
|
+
.join(candidate_cycles, candidate_cycles.c.cycle_key == AutomationRun.cycle_key)
|
|
1258
|
+
.outerjoin(Account, Account.id == AutomationRun.account_id)
|
|
1259
|
+
.cte("automation_runs_for_candidate_cycles")
|
|
1260
|
+
)
|
|
1261
|
+
snapshot_accounts = (
|
|
1262
|
+
select(
|
|
1263
|
+
AutomationRunCycleAccount.cycle_key.label("cycle_key"),
|
|
1264
|
+
func.count(func.distinct(AutomationRunCycleAccount.account_id)).label("included_accounts"),
|
|
1265
|
+
)
|
|
1266
|
+
.join(candidate_cycles, candidate_cycles.c.cycle_key == AutomationRunCycleAccount.cycle_key)
|
|
1267
|
+
.join(AutomationRunCycle, AutomationRunCycle.cycle_key == AutomationRunCycleAccount.cycle_key)
|
|
1268
|
+
.outerjoin(Account, Account.id == AutomationRunCycleAccount.account_id)
|
|
1269
|
+
.outerjoin(
|
|
1270
|
+
AutomationRun,
|
|
1271
|
+
and_(
|
|
1272
|
+
AutomationRun.cycle_key == AutomationRunCycleAccount.cycle_key,
|
|
1273
|
+
or_(
|
|
1274
|
+
and_(
|
|
1275
|
+
AutomationRunCycleAccount.slot_key.is_not(None),
|
|
1276
|
+
AutomationRun.slot_key == AutomationRunCycleAccount.slot_key,
|
|
1277
|
+
),
|
|
1278
|
+
and_(
|
|
1279
|
+
AutomationRunCycleAccount.slot_key.is_(None),
|
|
1280
|
+
AutomationRun.account_id == AutomationRunCycleAccount.account_id,
|
|
1281
|
+
),
|
|
1282
|
+
),
|
|
1283
|
+
),
|
|
1284
|
+
)
|
|
1285
|
+
.where(
|
|
1286
|
+
or_(
|
|
1287
|
+
AutomationRun.id.is_not(None),
|
|
1288
|
+
Account.status.in_((AccountStatus.ACTIVE, AccountStatus.REAUTH_REQUIRED)),
|
|
1289
|
+
and_(
|
|
1290
|
+
Account.status == AccountStatus.PAUSED,
|
|
1291
|
+
AutomationRunCycle.include_paused_accounts.is_(True),
|
|
1292
|
+
),
|
|
1293
|
+
)
|
|
1294
|
+
)
|
|
1295
|
+
.group_by(AutomationRunCycleAccount.cycle_key)
|
|
1296
|
+
.cte("eligible_automation_run_cycle_accounts")
|
|
1297
|
+
)
|
|
1298
|
+
|
|
1299
|
+
countable_outcome = or_(
|
|
1300
|
+
cycle_runs.c.account_id.is_not(None),
|
|
1301
|
+
cycle_runs.c.trigger == "scheduled",
|
|
1302
|
+
cycle_runs.c.attempt_count > 0,
|
|
1303
|
+
)
|
|
1304
|
+
account_is_eligible = and_(
|
|
1305
|
+
cycle_runs.c.account_id.is_not(None),
|
|
1306
|
+
or_(
|
|
1307
|
+
cycle_runs.c.status != "running",
|
|
1308
|
+
cycle_runs.c.account_status.in_((AccountStatus.ACTIVE, AccountStatus.REAUTH_REQUIRED)),
|
|
1309
|
+
and_(
|
|
1310
|
+
cycle_runs.c.account_status == AccountStatus.PAUSED,
|
|
1311
|
+
cycle_runs.c.include_paused_accounts.is_(True),
|
|
1312
|
+
),
|
|
1313
|
+
),
|
|
1314
|
+
)
|
|
1315
|
+
hidden_manual_placeholder = and_(
|
|
1316
|
+
cycle_runs.c.trigger == "manual",
|
|
1317
|
+
cycle_runs.c.status == "running",
|
|
1318
|
+
cycle_runs.c.finished_at.is_(None),
|
|
1319
|
+
cycle_runs.c.attempt_count == 0,
|
|
1320
|
+
cycle_runs.c.started_at <= cycle_runs.c.scheduled_for,
|
|
1321
|
+
~account_is_eligible,
|
|
1322
|
+
)
|
|
1323
|
+
visible_countable_outcome = and_(countable_outcome, ~hidden_manual_placeholder)
|
|
1324
|
+
completed_countable_run = case(
|
|
1325
|
+
(and_(cycle_runs.c.status != "running", visible_countable_outcome), 1),
|
|
1326
|
+
else_=0,
|
|
1327
|
+
)
|
|
1328
|
+
cycle_aggregate = (
|
|
1329
|
+
select(
|
|
1330
|
+
cycle_runs.c.cycle_key.label("cycle_key"),
|
|
1331
|
+
func.max(case((cycle_runs.c.trigger == "manual", 1), else_=0)).label("has_manual_trigger"),
|
|
1332
|
+
func.min(case((cycle_runs.c.trigger == "manual", cycle_runs.c.scheduled_for), else_=None)).label(
|
|
1333
|
+
"manual_cycle_started_at"
|
|
1334
|
+
),
|
|
1335
|
+
func.min(case((cycle_runs.c.trigger != "manual", cycle_runs.c.started_at), else_=None)).label(
|
|
1336
|
+
"non_manual_cycle_started_at"
|
|
1337
|
+
),
|
|
1338
|
+
func.sum(completed_countable_run).label("completed_accounts"),
|
|
1339
|
+
func.sum(case((and_(visible_countable_outcome, cycle_runs.c.status == "success"), 1), else_=0)).label(
|
|
1340
|
+
"success_count"
|
|
1341
|
+
),
|
|
1342
|
+
func.sum(case((and_(visible_countable_outcome, cycle_runs.c.status == "failed"), 1), else_=0)).label(
|
|
1343
|
+
"failed_count"
|
|
1344
|
+
),
|
|
1345
|
+
func.sum(case((and_(visible_countable_outcome, cycle_runs.c.status == "partial"), 1), else_=0)).label(
|
|
1346
|
+
"partial_count"
|
|
1347
|
+
),
|
|
1348
|
+
func.sum(case((and_(~hidden_manual_placeholder, cycle_runs.c.status == "running"), 1), else_=0)).label(
|
|
1349
|
+
"running_count"
|
|
1350
|
+
),
|
|
1351
|
+
func.sum(case((visible_countable_outcome, 1), else_=0)).label("visible_accounts"),
|
|
1352
|
+
func.max(func.coalesce(cycle_runs.c.cycle_expected_accounts, 0)).label("snapshot_expected_accounts"),
|
|
1353
|
+
func.max(func.coalesce(cycle_runs.c.cycle_window_end, cycle_runs.c.scheduled_for)).label("window_end"),
|
|
1354
|
+
)
|
|
1355
|
+
.group_by(cycle_runs.c.cycle_key)
|
|
1356
|
+
.cte("automation_run_cycle_aggregate")
|
|
1357
|
+
)
|
|
1358
|
+
cycle_rows = (
|
|
1359
|
+
select(
|
|
1360
|
+
ranked_representatives.c.run_id,
|
|
1361
|
+
ranked_representatives.c.cycle_key,
|
|
1362
|
+
case(
|
|
1363
|
+
(cycle_aggregate.c.has_manual_trigger == 1, cycle_aggregate.c.manual_cycle_started_at),
|
|
1364
|
+
else_=cycle_aggregate.c.non_manual_cycle_started_at,
|
|
1365
|
+
).label("cycle_started_at"),
|
|
1366
|
+
cycle_aggregate.c.has_manual_trigger,
|
|
1367
|
+
ranked_representatives.c.fallback_status,
|
|
1368
|
+
cycle_aggregate.c.completed_accounts,
|
|
1369
|
+
cycle_aggregate.c.success_count,
|
|
1370
|
+
cycle_aggregate.c.failed_count,
|
|
1371
|
+
cycle_aggregate.c.partial_count,
|
|
1372
|
+
cycle_aggregate.c.running_count,
|
|
1373
|
+
cycle_aggregate.c.visible_accounts.label("expected_accounts"),
|
|
1374
|
+
cycle_aggregate.c.window_end,
|
|
1375
|
+
snapshot_accounts.c.included_accounts,
|
|
1376
|
+
)
|
|
1377
|
+
.join(cycle_aggregate, cycle_aggregate.c.cycle_key == ranked_representatives.c.cycle_key)
|
|
1378
|
+
.outerjoin(snapshot_accounts, snapshot_accounts.c.cycle_key == ranked_representatives.c.cycle_key)
|
|
1379
|
+
.where(ranked_representatives.c.cycle_rank == 1)
|
|
1380
|
+
.cte("automation_run_cycle_status_inputs")
|
|
1381
|
+
)
|
|
1382
|
+
expected_accounts = case(
|
|
1383
|
+
(cycle_rows.c.has_manual_trigger == 1, cycle_rows.c.expected_accounts),
|
|
1384
|
+
(
|
|
1385
|
+
func.coalesce(cycle_rows.c.included_accounts, cycle_rows.c.expected_accounts)
|
|
1386
|
+
> cycle_rows.c.expected_accounts,
|
|
1387
|
+
func.coalesce(cycle_rows.c.included_accounts, cycle_rows.c.expected_accounts),
|
|
1388
|
+
),
|
|
1389
|
+
else_=cycle_rows.c.expected_accounts,
|
|
1390
|
+
)
|
|
1391
|
+
effective_total = case(
|
|
1392
|
+
(expected_accounts > cycle_rows.c.completed_accounts, expected_accounts),
|
|
1393
|
+
else_=cycle_rows.c.completed_accounts,
|
|
1394
|
+
)
|
|
1395
|
+
pending = effective_total - cycle_rows.c.completed_accounts
|
|
1396
|
+
effective_status = case(
|
|
1397
|
+
(cycle_rows.c.running_count > 0, "running"),
|
|
1398
|
+
(and_(pending > 0, now_utc <= cycle_rows.c.window_end), "running"),
|
|
1399
|
+
(pending > 0, case((cycle_rows.c.completed_accounts > 0, "partial"), else_="failed")),
|
|
1400
|
+
(
|
|
1401
|
+
and_(
|
|
1402
|
+
cycle_rows.c.success_count > 0,
|
|
1403
|
+
cycle_rows.c.failed_count == 0,
|
|
1404
|
+
cycle_rows.c.partial_count == 0,
|
|
1405
|
+
),
|
|
1406
|
+
"success",
|
|
1407
|
+
),
|
|
1408
|
+
(
|
|
1409
|
+
and_(
|
|
1410
|
+
cycle_rows.c.success_count > 0,
|
|
1411
|
+
or_(cycle_rows.c.failed_count > 0, cycle_rows.c.partial_count > 0),
|
|
1412
|
+
),
|
|
1413
|
+
"partial",
|
|
1414
|
+
),
|
|
1415
|
+
(
|
|
1416
|
+
and_(
|
|
1417
|
+
cycle_rows.c.failed_count > 0,
|
|
1418
|
+
cycle_rows.c.success_count == 0,
|
|
1419
|
+
cycle_rows.c.partial_count == 0,
|
|
1420
|
+
),
|
|
1421
|
+
"failed",
|
|
1422
|
+
),
|
|
1423
|
+
(cycle_rows.c.partial_count > 0, "partial"),
|
|
1424
|
+
else_=cycle_rows.c.fallback_status,
|
|
1425
|
+
).label("effective_status")
|
|
1426
|
+
return select(
|
|
1427
|
+
cycle_rows.c.run_id,
|
|
1428
|
+
cycle_rows.c.cycle_key,
|
|
1429
|
+
cycle_rows.c.cycle_started_at,
|
|
1430
|
+
effective_status,
|
|
1431
|
+
).cte("automation_run_cycle_scope")
|
|
1432
|
+
|
|
1433
|
+
async def list_run_cycles_page(
|
|
1434
|
+
self,
|
|
1435
|
+
*,
|
|
1436
|
+
limit: int,
|
|
1437
|
+
offset: int,
|
|
1438
|
+
now_utc: datetime,
|
|
1439
|
+
search: str | None = None,
|
|
1440
|
+
account_ids: Sequence[str] | None = None,
|
|
1441
|
+
models: Sequence[str] | None = None,
|
|
1442
|
+
statuses: Sequence[str] | None = None,
|
|
1443
|
+
triggers: Sequence[str] | None = None,
|
|
1444
|
+
job_ids: Sequence[str] | None = None,
|
|
1445
|
+
) -> tuple[list[AutomationRunRecord], int]:
|
|
1446
|
+
if statuses:
|
|
1447
|
+
full_cycle_scope = self._build_full_run_cycle_scope(
|
|
1448
|
+
now_utc=now_utc,
|
|
1449
|
+
search=search,
|
|
1450
|
+
account_ids=account_ids,
|
|
1451
|
+
models=models,
|
|
1452
|
+
triggers=triggers,
|
|
1453
|
+
job_ids=job_ids,
|
|
1454
|
+
)
|
|
1455
|
+
cycle_scope = (
|
|
1456
|
+
select(
|
|
1457
|
+
full_cycle_scope.c.run_id,
|
|
1458
|
+
full_cycle_scope.c.cycle_key,
|
|
1459
|
+
full_cycle_scope.c.cycle_started_at,
|
|
1460
|
+
)
|
|
1461
|
+
.where(full_cycle_scope.c.effective_status.in_(list(statuses)))
|
|
1462
|
+
.cte("matching_automation_run_cycles")
|
|
1463
|
+
)
|
|
1464
|
+
else:
|
|
1465
|
+
cycle_scope = self._build_lightweight_run_cycle_scope(
|
|
1466
|
+
search=search,
|
|
1467
|
+
account_ids=account_ids,
|
|
1468
|
+
models=models,
|
|
1469
|
+
triggers=triggers,
|
|
1470
|
+
job_ids=job_ids,
|
|
1471
|
+
)
|
|
1472
|
+
|
|
1473
|
+
page_scope = (
|
|
1474
|
+
select(
|
|
1475
|
+
cycle_scope.c.run_id,
|
|
1476
|
+
cycle_scope.c.cycle_started_at,
|
|
1477
|
+
func.count().over().label("total_count"),
|
|
1478
|
+
)
|
|
1479
|
+
.order_by(cycle_scope.c.cycle_started_at.desc(), cycle_scope.c.run_id.desc())
|
|
1480
|
+
.offset(offset)
|
|
1481
|
+
.limit(limit)
|
|
1482
|
+
.cte("automation_run_cycle_page")
|
|
1483
|
+
)
|
|
1484
|
+
rows = (
|
|
1485
|
+
await self._session.execute(
|
|
1486
|
+
select(
|
|
1487
|
+
AutomationRun,
|
|
1488
|
+
AutomationJob.name,
|
|
1489
|
+
AutomationJob.model,
|
|
1490
|
+
AutomationJob.reasoning_effort,
|
|
1491
|
+
page_scope.c.total_count,
|
|
1492
|
+
)
|
|
1493
|
+
.join(page_scope, page_scope.c.run_id == AutomationRun.id)
|
|
1494
|
+
.join(AutomationJob, AutomationJob.id == AutomationRun.job_id)
|
|
1495
|
+
.order_by(page_scope.c.cycle_started_at.desc(), page_scope.c.run_id.desc())
|
|
1496
|
+
)
|
|
1497
|
+
).all()
|
|
1498
|
+
if rows:
|
|
1499
|
+
total = int(rows[0].total_count or 0)
|
|
1500
|
+
runs = [
|
|
1501
|
+
self._run_from_model(
|
|
1502
|
+
run,
|
|
1503
|
+
job_name=job_name,
|
|
1504
|
+
model=model,
|
|
1505
|
+
reasoning_effort=reasoning_effort,
|
|
1506
|
+
)
|
|
1507
|
+
for run, job_name, model, reasoning_effort, _total_count in rows
|
|
1508
|
+
]
|
|
1509
|
+
return runs, total
|
|
1510
|
+
|
|
1511
|
+
if offset > 0:
|
|
1512
|
+
total = int((await self._session.execute(select(func.count()).select_from(cycle_scope))).scalar_one() or 0)
|
|
1513
|
+
return [], total
|
|
1514
|
+
return [], 0
|
|
1515
|
+
|
|
1516
|
+
async def list_run_filter_options(
|
|
1517
|
+
self,
|
|
1518
|
+
*,
|
|
1519
|
+
now_utc: datetime | None = None,
|
|
1520
|
+
search: str | None = None,
|
|
1521
|
+
account_ids: Sequence[str] | None = None,
|
|
1522
|
+
models: Sequence[str] | None = None,
|
|
1523
|
+
statuses: Sequence[str] | None = None,
|
|
1524
|
+
triggers: Sequence[str] | None = None,
|
|
1525
|
+
job_ids: Sequence[str] | None = None,
|
|
1526
|
+
) -> AutomationRunsFilterOptionsRecord:
|
|
1527
|
+
if statuses:
|
|
1528
|
+
cycle_scope = self._build_full_run_cycle_scope(
|
|
1529
|
+
now_utc=now_utc or utcnow(),
|
|
1530
|
+
search=search,
|
|
1531
|
+
account_ids=account_ids,
|
|
1532
|
+
models=models,
|
|
1533
|
+
triggers=triggers,
|
|
1534
|
+
job_ids=job_ids,
|
|
1535
|
+
)
|
|
1536
|
+
matching_cycles = (
|
|
1537
|
+
select(cycle_scope.c.cycle_key)
|
|
1538
|
+
.where(cycle_scope.c.effective_status.in_(list(statuses)))
|
|
1539
|
+
.cte("matching_automation_run_cycles")
|
|
1540
|
+
)
|
|
1541
|
+
facet_source = (
|
|
1542
|
+
select(
|
|
1543
|
+
AutomationRun.account_id.label("account_id"),
|
|
1544
|
+
func.coalesce(AutomationRun.model, AutomationJob.model).label("model"),
|
|
1545
|
+
)
|
|
1546
|
+
.join(AutomationJob, AutomationJob.id == AutomationRun.job_id)
|
|
1547
|
+
.join(matching_cycles, matching_cycles.c.cycle_key == AutomationRun.cycle_key)
|
|
1548
|
+
.cte("automation_run_option_source")
|
|
1549
|
+
)
|
|
1550
|
+
else:
|
|
1551
|
+
conditions = self._build_run_conditions(
|
|
1552
|
+
search=search,
|
|
1553
|
+
account_ids=account_ids,
|
|
1554
|
+
models=models,
|
|
1555
|
+
statuses=None,
|
|
1556
|
+
triggers=triggers,
|
|
1557
|
+
job_ids=job_ids,
|
|
1558
|
+
)
|
|
1559
|
+
facet_source_stmt = select(
|
|
1560
|
+
AutomationRun.account_id.label("account_id"),
|
|
1561
|
+
func.coalesce(AutomationRun.model, AutomationJob.model).label("model"),
|
|
1562
|
+
).join(AutomationJob, AutomationJob.id == AutomationRun.job_id)
|
|
1563
|
+
if conditions:
|
|
1564
|
+
facet_source_stmt = facet_source_stmt.where(and_(*conditions))
|
|
1565
|
+
facet_source = facet_source_stmt.cte("automation_run_option_source")
|
|
1566
|
+
|
|
1567
|
+
account_facet = (
|
|
1568
|
+
select(
|
|
1569
|
+
literal("account").label("facet"),
|
|
1570
|
+
facet_source.c.account_id.label("value"),
|
|
1571
|
+
)
|
|
1572
|
+
.where(facet_source.c.account_id.is_not(None))
|
|
1573
|
+
.distinct()
|
|
1574
|
+
)
|
|
1575
|
+
model_facet = (
|
|
1576
|
+
select(
|
|
1577
|
+
literal("model").label("facet"),
|
|
1578
|
+
facet_source.c.model.label("value"),
|
|
1579
|
+
)
|
|
1580
|
+
.where(facet_source.c.model.is_not(None))
|
|
1581
|
+
.distinct()
|
|
1582
|
+
)
|
|
1583
|
+
facets = union_all(account_facet, model_facet).subquery("automation_run_option_facets")
|
|
1584
|
+
rows = (
|
|
1585
|
+
await self._session.execute(
|
|
1586
|
+
select(facets.c.facet, facets.c.value).order_by(facets.c.facet.asc(), facets.c.value.asc())
|
|
1587
|
+
)
|
|
1588
|
+
).all()
|
|
1589
|
+
return AutomationRunsFilterOptionsRecord(
|
|
1590
|
+
account_ids=[value for facet, value in rows if facet == "account" and value],
|
|
1591
|
+
models=[value for facet, value in rows if facet == "model" and value],
|
|
1592
|
+
)
|
|
1593
|
+
|
|
1594
|
+
async def get_latest_runs_by_job_ids(self, job_ids: Sequence[str]) -> dict[str, AutomationRunRecord]:
|
|
1595
|
+
if not job_ids:
|
|
1596
|
+
return {}
|
|
1597
|
+
normalized_job_ids = list(dict.fromkeys(job_id for job_id in job_ids if job_id))
|
|
1598
|
+
if not normalized_job_ids:
|
|
1599
|
+
return {}
|
|
1600
|
+
|
|
1601
|
+
cycle_agg = (
|
|
1602
|
+
select(
|
|
1603
|
+
AutomationRun.job_id.label("job_id"),
|
|
1604
|
+
AutomationRun.cycle_key.label("cycle_key"),
|
|
1605
|
+
func.max(case((AutomationRun.trigger == "manual", 1), else_=0)).label("has_manual_trigger"),
|
|
1606
|
+
func.min(case((AutomationRun.trigger == "manual", AutomationRun.scheduled_for), else_=None)).label(
|
|
1607
|
+
"manual_cycle_started_at"
|
|
1608
|
+
),
|
|
1609
|
+
func.min(case((AutomationRun.trigger != "manual", AutomationRun.started_at), else_=None)).label(
|
|
1610
|
+
"non_manual_cycle_started_at"
|
|
1611
|
+
),
|
|
1612
|
+
)
|
|
1613
|
+
.where(AutomationRun.job_id.in_(normalized_job_ids))
|
|
1614
|
+
.group_by(AutomationRun.job_id, AutomationRun.cycle_key)
|
|
1615
|
+
.subquery()
|
|
1616
|
+
)
|
|
1617
|
+
cycle_started_at = case(
|
|
1618
|
+
(cycle_agg.c.has_manual_trigger == 1, cycle_agg.c.manual_cycle_started_at),
|
|
1619
|
+
else_=cycle_agg.c.non_manual_cycle_started_at,
|
|
1620
|
+
)
|
|
1621
|
+
ranked_cycles = select(
|
|
1622
|
+
cycle_agg.c.job_id,
|
|
1623
|
+
cycle_agg.c.cycle_key,
|
|
1624
|
+
func.row_number()
|
|
1625
|
+
.over(
|
|
1626
|
+
partition_by=cycle_agg.c.job_id,
|
|
1627
|
+
order_by=(cycle_started_at.desc(), cycle_agg.c.cycle_key.desc()),
|
|
1628
|
+
)
|
|
1629
|
+
.label("cycle_rank"),
|
|
1630
|
+
).subquery()
|
|
1631
|
+
ranked_runs = (
|
|
1632
|
+
select(
|
|
1633
|
+
AutomationRun.id.label("run_id"),
|
|
1634
|
+
AutomationRun.job_id.label("job_id"),
|
|
1635
|
+
AutomationRun.cycle_key.label("cycle_key"),
|
|
1636
|
+
func.row_number()
|
|
1637
|
+
.over(
|
|
1638
|
+
partition_by=(AutomationRun.job_id, AutomationRun.cycle_key),
|
|
1639
|
+
order_by=(AutomationRun.started_at.desc(), AutomationRun.id.desc()),
|
|
1640
|
+
)
|
|
1641
|
+
.label("run_rank"),
|
|
1642
|
+
)
|
|
1643
|
+
.where(AutomationRun.job_id.in_(normalized_job_ids))
|
|
1644
|
+
.subquery()
|
|
1645
|
+
)
|
|
1646
|
+
result = await self._session.execute(
|
|
1647
|
+
select(AutomationRun)
|
|
1648
|
+
.join(ranked_runs, ranked_runs.c.run_id == AutomationRun.id)
|
|
1649
|
+
.join(
|
|
1650
|
+
ranked_cycles,
|
|
1651
|
+
and_(
|
|
1652
|
+
ranked_cycles.c.job_id == ranked_runs.c.job_id,
|
|
1653
|
+
ranked_cycles.c.cycle_key == ranked_runs.c.cycle_key,
|
|
1654
|
+
),
|
|
1655
|
+
)
|
|
1656
|
+
.where(ranked_runs.c.run_rank == 1)
|
|
1657
|
+
.where(ranked_cycles.c.cycle_rank == 1)
|
|
1658
|
+
.order_by(AutomationRun.job_id.asc())
|
|
1659
|
+
)
|
|
1660
|
+
latest: dict[str, AutomationRunRecord] = {}
|
|
1661
|
+
for run in result.scalars().all():
|
|
1662
|
+
if run.job_id in latest:
|
|
1663
|
+
continue
|
|
1664
|
+
latest[run.job_id] = self._run_from_model(run)
|
|
1665
|
+
return latest
|
|
1666
|
+
|
|
1667
|
+
@staticmethod
|
|
1668
|
+
def _job_from_model(job: AutomationJob) -> AutomationJobRecord:
|
|
1669
|
+
sorted_accounts = sorted(job.account_links, key=lambda link: link.position)
|
|
1670
|
+
return AutomationJobRecord(
|
|
1671
|
+
id=job.id,
|
|
1672
|
+
name=job.name,
|
|
1673
|
+
enabled=job.enabled,
|
|
1674
|
+
schedule_type=job.schedule_type,
|
|
1675
|
+
schedule_time=job.schedule_time,
|
|
1676
|
+
schedule_timezone=job.schedule_timezone,
|
|
1677
|
+
schedule_days=_parse_schedule_days(job.schedule_days),
|
|
1678
|
+
schedule_threshold_minutes=job.schedule_threshold_minutes,
|
|
1679
|
+
include_paused_accounts=job.include_paused_accounts,
|
|
1680
|
+
account_scope_all=job.account_scope_all,
|
|
1681
|
+
model=job.model,
|
|
1682
|
+
reasoning_effort=job.reasoning_effort,
|
|
1683
|
+
prompt=job.prompt,
|
|
1684
|
+
account_ids=[link.account_id for link in sorted_accounts],
|
|
1685
|
+
created_at=job.created_at,
|
|
1686
|
+
updated_at=job.updated_at,
|
|
1687
|
+
)
|
|
1688
|
+
|
|
1689
|
+
@staticmethod
|
|
1690
|
+
def _run_from_model(
|
|
1691
|
+
run: AutomationRun,
|
|
1692
|
+
*,
|
|
1693
|
+
job_name: str | None = None,
|
|
1694
|
+
model: str | None = None,
|
|
1695
|
+
reasoning_effort: str | None = None,
|
|
1696
|
+
) -> AutomationRunRecord:
|
|
1697
|
+
return AutomationRunRecord(
|
|
1698
|
+
id=run.id,
|
|
1699
|
+
job_id=run.job_id,
|
|
1700
|
+
job_name=job_name,
|
|
1701
|
+
model=run.model or model,
|
|
1702
|
+
reasoning_effort=run.reasoning_effort if run.model is not None else reasoning_effort,
|
|
1703
|
+
prompt=run.prompt,
|
|
1704
|
+
trigger=run.trigger,
|
|
1705
|
+
status=run.status,
|
|
1706
|
+
slot_key=run.slot_key,
|
|
1707
|
+
cycle_key=run.cycle_key,
|
|
1708
|
+
cycle_expected_accounts=run.cycle_expected_accounts,
|
|
1709
|
+
cycle_window_end=run.cycle_window_end,
|
|
1710
|
+
scheduled_for=run.scheduled_for,
|
|
1711
|
+
started_at=run.started_at,
|
|
1712
|
+
finished_at=run.finished_at,
|
|
1713
|
+
account_id=run.account_id,
|
|
1714
|
+
error_code=run.error_code,
|
|
1715
|
+
error_message=run.error_message,
|
|
1716
|
+
attempt_count=run.attempt_count,
|
|
1717
|
+
claim_budget_seconds=run.claim_budget_seconds,
|
|
1718
|
+
)
|
|
1719
|
+
|
|
1720
|
+
@staticmethod
|
|
1721
|
+
def _run_cycle_from_model(cycle: AutomationRunCycle) -> AutomationRunCycleRecord:
|
|
1722
|
+
cycle_accounts = sorted(cycle.cycle_accounts, key=lambda entry: (entry.position, entry.account_id))
|
|
1723
|
+
return AutomationRunCycleRecord(
|
|
1724
|
+
cycle_key=cycle.cycle_key,
|
|
1725
|
+
job_id=cycle.job_id,
|
|
1726
|
+
trigger=cycle.trigger,
|
|
1727
|
+
cycle_expected_accounts=cycle.cycle_expected_accounts,
|
|
1728
|
+
cycle_window_end=cycle.cycle_window_end,
|
|
1729
|
+
include_paused_accounts=cycle.include_paused_accounts,
|
|
1730
|
+
accounts=[
|
|
1731
|
+
AutomationRunCycleAccountRecord(
|
|
1732
|
+
account_id=entry.account_id,
|
|
1733
|
+
slot_key=entry.slot_key,
|
|
1734
|
+
position=entry.position,
|
|
1735
|
+
scheduled_for=entry.scheduled_for,
|
|
1736
|
+
)
|
|
1737
|
+
for entry in cycle_accounts
|
|
1738
|
+
],
|
|
1739
|
+
created_at=cycle.created_at,
|
|
1740
|
+
)
|
|
1741
|
+
|
|
1742
|
+
@staticmethod
|
|
1743
|
+
def _build_job_conditions(
|
|
1744
|
+
*,
|
|
1745
|
+
search: str | None,
|
|
1746
|
+
account_ids: Sequence[str] | None,
|
|
1747
|
+
models: Sequence[str] | None,
|
|
1748
|
+
statuses: Sequence[str] | None,
|
|
1749
|
+
schedule_types: Sequence[str] | None,
|
|
1750
|
+
) -> list:
|
|
1751
|
+
conditions = []
|
|
1752
|
+
normalized_search = (search or "").strip()
|
|
1753
|
+
if normalized_search:
|
|
1754
|
+
like = f"%{normalized_search}%"
|
|
1755
|
+
conditions.append(
|
|
1756
|
+
or_(
|
|
1757
|
+
AutomationJob.id.ilike(like),
|
|
1758
|
+
AutomationJob.name.ilike(like),
|
|
1759
|
+
AutomationJob.prompt.ilike(like),
|
|
1760
|
+
AutomationJob.model.ilike(like),
|
|
1761
|
+
AutomationJob.reasoning_effort.ilike(like),
|
|
1762
|
+
)
|
|
1763
|
+
)
|
|
1764
|
+
normalized_accounts = [value.strip() for value in (account_ids or []) if value and value.strip()]
|
|
1765
|
+
if normalized_accounts:
|
|
1766
|
+
matching_account_links = select(AutomationJobAccount.job_id).where(
|
|
1767
|
+
AutomationJobAccount.account_id.in_(normalized_accounts)
|
|
1768
|
+
)
|
|
1769
|
+
job_has_all_account_scope = AutomationJob.account_scope_all.is_(True)
|
|
1770
|
+
conditions.append(
|
|
1771
|
+
or_(
|
|
1772
|
+
AutomationJob.id.in_(matching_account_links),
|
|
1773
|
+
job_has_all_account_scope,
|
|
1774
|
+
)
|
|
1775
|
+
)
|
|
1776
|
+
normalized_models = [value.strip() for value in (models or []) if value and value.strip()]
|
|
1777
|
+
if normalized_models:
|
|
1778
|
+
conditions.append(AutomationJob.model.in_(normalized_models))
|
|
1779
|
+
normalized_types = [value.strip() for value in (schedule_types or []) if value and value.strip()]
|
|
1780
|
+
if normalized_types:
|
|
1781
|
+
conditions.append(AutomationJob.schedule_type.in_(normalized_types))
|
|
1782
|
+
normalized_statuses = {value.strip().lower() for value in (statuses or []) if value and value.strip()}
|
|
1783
|
+
if normalized_statuses and "all" not in normalized_statuses:
|
|
1784
|
+
enabled_values: list[bool] = []
|
|
1785
|
+
if "enabled" in normalized_statuses:
|
|
1786
|
+
enabled_values.append(True)
|
|
1787
|
+
if "disabled" in normalized_statuses:
|
|
1788
|
+
enabled_values.append(False)
|
|
1789
|
+
if enabled_values:
|
|
1790
|
+
conditions.append(AutomationJob.enabled.in_(enabled_values))
|
|
1791
|
+
else:
|
|
1792
|
+
conditions.append(AutomationJob.id == "__none__")
|
|
1793
|
+
return conditions
|
|
1794
|
+
|
|
1795
|
+
@staticmethod
|
|
1796
|
+
def _build_run_conditions(
|
|
1797
|
+
*,
|
|
1798
|
+
search: str | None,
|
|
1799
|
+
account_ids: Sequence[str] | None,
|
|
1800
|
+
models: Sequence[str] | None,
|
|
1801
|
+
statuses: Sequence[str] | None,
|
|
1802
|
+
triggers: Sequence[str] | None,
|
|
1803
|
+
job_ids: Sequence[str] | None,
|
|
1804
|
+
) -> list:
|
|
1805
|
+
conditions = []
|
|
1806
|
+
normalized_search = (search or "").strip()
|
|
1807
|
+
run_model = func.coalesce(AutomationRun.model, AutomationJob.model)
|
|
1808
|
+
run_reasoning_effort = func.coalesce(AutomationRun.reasoning_effort, AutomationJob.reasoning_effort)
|
|
1809
|
+
if normalized_search:
|
|
1810
|
+
like = f"%{normalized_search}%"
|
|
1811
|
+
conditions.append(
|
|
1812
|
+
or_(
|
|
1813
|
+
AutomationRun.id.ilike(like),
|
|
1814
|
+
AutomationRun.job_id.ilike(like),
|
|
1815
|
+
AutomationRun.account_id.ilike(like),
|
|
1816
|
+
AutomationRun.error_code.ilike(like),
|
|
1817
|
+
AutomationRun.error_message.ilike(like),
|
|
1818
|
+
AutomationJob.name.ilike(like),
|
|
1819
|
+
run_model.ilike(like),
|
|
1820
|
+
run_reasoning_effort.ilike(like),
|
|
1821
|
+
)
|
|
1822
|
+
)
|
|
1823
|
+
normalized_accounts = [value.strip() for value in (account_ids or []) if value and value.strip()]
|
|
1824
|
+
if normalized_accounts:
|
|
1825
|
+
conditions.append(AutomationRun.account_id.in_(normalized_accounts))
|
|
1826
|
+
normalized_models = [value.strip() for value in (models or []) if value and value.strip()]
|
|
1827
|
+
if normalized_models:
|
|
1828
|
+
conditions.append(run_model.in_(normalized_models))
|
|
1829
|
+
normalized_statuses = [value.strip().lower() for value in (statuses or []) if value and value.strip()]
|
|
1830
|
+
if normalized_statuses:
|
|
1831
|
+
conditions.append(AutomationRun.status.in_(normalized_statuses))
|
|
1832
|
+
normalized_triggers = [value.strip().lower() for value in (triggers or []) if value and value.strip()]
|
|
1833
|
+
if normalized_triggers:
|
|
1834
|
+
conditions.append(AutomationRun.trigger.in_(normalized_triggers))
|
|
1835
|
+
normalized_job_ids = [value.strip() for value in (job_ids or []) if value and value.strip()]
|
|
1836
|
+
if normalized_job_ids:
|
|
1837
|
+
conditions.append(AutomationRun.job_id.in_(normalized_job_ids))
|
|
1838
|
+
return conditions
|
|
1839
|
+
|
|
1840
|
+
@staticmethod
|
|
1841
|
+
def _build_grouped_run_account_match(
|
|
1842
|
+
*,
|
|
1843
|
+
account_ids: Sequence[str] | None,
|
|
1844
|
+
):
|
|
1845
|
+
normalized_accounts = [value.strip() for value in (account_ids or []) if value and value.strip()]
|
|
1846
|
+
if not normalized_accounts:
|
|
1847
|
+
return None
|
|
1848
|
+
snapshot_cycle_keys = select(AutomationRunCycleAccount.cycle_key).where(
|
|
1849
|
+
AutomationRunCycleAccount.account_id.in_(normalized_accounts)
|
|
1850
|
+
)
|
|
1851
|
+
return or_(
|
|
1852
|
+
AutomationRun.account_id.in_(normalized_accounts),
|
|
1853
|
+
AutomationRun.cycle_key.in_(snapshot_cycle_keys),
|
|
1854
|
+
)
|
|
1855
|
+
|
|
1856
|
+
|
|
1857
|
+
def _parse_schedule_days(value: str | None) -> list[str]:
|
|
1858
|
+
if not value:
|
|
1859
|
+
return list(DEFAULT_AUTOMATION_SCHEDULE_DAYS)
|
|
1860
|
+
parsed = [part.strip().lower() for part in value.split(",") if part.strip()]
|
|
1861
|
+
if not parsed:
|
|
1862
|
+
return list(DEFAULT_AUTOMATION_SCHEDULE_DAYS)
|
|
1863
|
+
return parsed
|
|
1864
|
+
|
|
1865
|
+
|
|
1866
|
+
def _serialize_schedule_days(days: Sequence[str]) -> str:
|
|
1867
|
+
if not days:
|
|
1868
|
+
return ",".join(DEFAULT_AUTOMATION_SCHEDULE_DAYS)
|
|
1869
|
+
normalized = [day.strip().lower() for day in days if day.strip()]
|
|
1870
|
+
if not normalized:
|
|
1871
|
+
return ",".join(DEFAULT_AUTOMATION_SCHEDULE_DAYS)
|
|
1872
|
+
return ",".join(normalized)
|
|
1873
|
+
|
|
1874
|
+
|
|
1875
|
+
def _scheduled_slot_key(job_id: str, *, account_id: str, due_slot: datetime) -> str:
|
|
1876
|
+
seed = f"{job_id}:{due_slot.isoformat()}:{account_id}"
|
|
1877
|
+
digest = sha1(seed.encode("utf-8")).hexdigest()[:20]
|
|
1878
|
+
return f"scheduled:{job_id}:{digest}"
|
|
1879
|
+
|
|
1880
|
+
|
|
1881
|
+
def _cycle_account_slot_key(*, job_id: str, trigger: str, account_id: str, due_slot: datetime | None) -> str | None:
|
|
1882
|
+
if trigger != "scheduled":
|
|
1883
|
+
return None
|
|
1884
|
+
if due_slot is None:
|
|
1885
|
+
return None
|
|
1886
|
+
return _scheduled_slot_key(job_id, account_id=account_id, due_slot=due_slot)
|
|
1887
|
+
|
|
1888
|
+
|
|
1889
|
+
def _parse_scheduled_cycle_due_slot(cycle_key: str, *, job_id: str) -> datetime | None:
|
|
1890
|
+
parts = cycle_key.split(":", maxsplit=2)
|
|
1891
|
+
if len(parts) != 3 or parts[0] != "scheduled" or parts[1] != job_id:
|
|
1892
|
+
return None
|
|
1893
|
+
try:
|
|
1894
|
+
return datetime.fromisoformat(parts[2].removesuffix("Z"))
|
|
1895
|
+
except ValueError:
|
|
1896
|
+
return None
|