loopx 0.4.8__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- loopx/__init__.py +5 -0
- loopx/agent_onboarding.py +654 -0
- loopx/agent_registry.py +112 -0
- loopx/ark_managed_agent_host.py +59 -0
- loopx/authority.py +805 -0
- loopx/benchmark.py +2875 -0
- loopx/benchmark_adapters/__init__.py +1 -0
- loopx/benchmark_adapters/agentissue.py +2644 -0
- loopx/benchmark_adapters/agents_last_exam.py +3998 -0
- loopx/benchmark_adapters/edgebench.py +322 -0
- loopx/benchmark_adapters/skillsbench.py +5978 -0
- loopx/benchmark_adapters/skillsbench_acp_failure_policy.py +143 -0
- loopx/benchmark_adapters/skillsbench_acp_process.py +31 -0
- loopx/benchmark_adapters/skillsbench_acp_relay.py +4832 -0
- loopx/benchmark_adapters/skillsbench_batch.py +124 -0
- loopx/benchmark_adapters/skillsbench_bridge_guard.py +209 -0
- loopx/benchmark_adapters/skillsbench_bridge_summary.py +203 -0
- loopx/benchmark_adapters/skillsbench_codex_goal_recovery.py +271 -0
- loopx/benchmark_adapters/skillsbench_codex_goal_trace.py +81 -0
- loopx/benchmark_adapters/skillsbench_codex_runtime.py +339 -0
- loopx/benchmark_adapters/skillsbench_dockerfile_runtime.py +467 -0
- loopx/benchmark_adapters/skillsbench_failure_signals.py +652 -0
- loopx/benchmark_adapters/skillsbench_proxy_runtime.py +327 -0
- loopx/benchmark_adapters/skillsbench_remote_bridge.py +402 -0
- loopx/benchmark_adapters/skillsbench_result_discovery.py +143 -0
- loopx/benchmark_adapters/skillsbench_runner_profile.py +436 -0
- loopx/benchmark_adapters/skillsbench_runner_source.py +99 -0
- loopx/benchmark_adapters/skillsbench_setup_preflight.py +771 -0
- loopx/benchmark_adapters/skillsbench_signals.py +15 -0
- loopx/benchmark_adapters/skillsbench_task_source.py +141 -0
- loopx/benchmark_adapters/skillsbench_turn_route.py +723 -0
- loopx/benchmark_adapters/skillsbench_turn_runtime.py +1069 -0
- loopx/benchmark_adapters/skillsbench_typed_repair.py +689 -0
- loopx/benchmark_adapters/skillsbench_uv_cache.py +111 -0
- loopx/benchmark_adapters/skillsbench_verifier_bootstrap.py +227 -0
- loopx/benchmark_adapters/skillsbench_verifier_cache.py +138 -0
- loopx/benchmark_adapters/terminal_bench.py +10078 -0
- loopx/benchmark_case_analysis.py +1276 -0
- loopx/benchmark_case_state.py +1079 -0
- loopx/benchmark_core/__init__.py +239 -0
- loopx/benchmark_core/adapter.py +84 -0
- loopx/benchmark_core/artifacts.py +517 -0
- loopx/benchmark_core/attempts.py +199 -0
- loopx/benchmark_core/container_exec.py +216 -0
- loopx/benchmark_core/io.py +68 -0
- loopx/benchmark_core/lifecycle.py +211 -0
- loopx/benchmark_core/loop_protocol.py +689 -0
- loopx/benchmark_core/observable_handles.py +348 -0
- loopx/benchmark_core/parity.py +256 -0
- loopx/benchmark_core/remote_closeout.py +482 -0
- loopx/benchmark_core/rounds.py +215 -0
- loopx/benchmark_core/route_profile.py +509 -0
- loopx/benchmark_core/run_permissions.py +206 -0
- loopx/benchmark_core/split_control.py +925 -0
- loopx/benchmark_core/turn_fidelity.py +326 -0
- loopx/benchmark_ledger.py +3793 -0
- loopx/benchmark_ledger_countability.py +372 -0
- loopx/benchmark_ledger_current.py +724 -0
- loopx/benchmark_trajectory.py +405 -0
- loopx/benchmarks/__init__.py +1 -0
- loopx/benchmarks/qualification/__init__.py +1 -0
- loopx/benchmarks/qualification/release_outcome_baseline.py +360 -0
- loopx/benchmarks/read_models/__init__.py +1 -0
- loopx/benchmarks/read_models/benchmark_attempt_accounting.py +53 -0
- loopx/benchmarks/read_models/benchmark_comparison.py +414 -0
- loopx/benchmarks/read_models/benchmark_event_timeline.py +113 -0
- loopx/benchmarks/read_models/benchmark_experiment_report.py +475 -0
- loopx/benchmarks/read_models/benchmark_learning_ledger.py +137 -0
- loopx/benchmarks/read_models/benchmark_lifecycle_contracts.py +228 -0
- loopx/benchmarks/read_models/benchmark_projection.py +723 -0
- loopx/benchmarks/read_models/benchmark_result.py +146 -0
- loopx/benchmarks/read_models/benchmark_run_execution_contract.py +116 -0
- loopx/benchmarks/read_models/benchmark_run_failure.py +157 -0
- loopx/benchmarks/read_models/benchmark_run_metrics.py +213 -0
- loopx/benchmarks/read_models/benchmark_run_post_execution.py +635 -0
- loopx/benchmarks/read_models/benchmark_run_pre_execution.py +541 -0
- loopx/benchmarks/read_models/benchmark_status_compaction.py +1255 -0
- loopx/benchmarks/read_models/benchmark_status_runner.py +780 -0
- loopx/benchmarks/read_models/goal_start_control_score.py +857 -0
- loopx/benchmarks/read_models/skillsbench_post_run_debug.py +746 -0
- loopx/benchmarks/read_models/skillsbench_verifier_attribution.py +269 -0
- loopx/bootstrap.py +1116 -0
- loopx/bootstrap_command_pack.py +2167 -0
- loopx/boundary_authority.py +199 -0
- loopx/canary/__init__.py +1 -0
- loopx/canary/maintainability_ratchet.py +800 -0
- loopx/canary/planner.py +1984 -0
- loopx/canary/premerge.py +1130 -0
- loopx/canary/qualification_profiles.py +309 -0
- loopx/canary/quality_surface_catalog.py +838 -0
- loopx/canary/release_profiles.py +51 -0
- loopx/canary/runner.py +1107 -0
- loopx/canary/smoke_health.py +581 -0
- loopx/canary/smoke_profiles.py +212 -0
- loopx/capabilities/__init__.py +0 -0
- loopx/capabilities/agent_turn_recall/__init__.py +17 -0
- loopx/capabilities/agent_turn_recall/cli.py +369 -0
- loopx/capabilities/agent_turn_recall/core.py +296 -0
- loopx/capabilities/auto_research/__init__.py +16 -0
- loopx/capabilities/auto_research/bootstrap_contract.py +157 -0
- loopx/capabilities/auto_research/cli.py +1468 -0
- loopx/capabilities/auto_research/core.py +11 -0
- loopx/capabilities/auto_research/defaults.py +79 -0
- loopx/capabilities/auto_research/demo_e2e.py +1848 -0
- loopx/capabilities/auto_research/demo_supervisor.py +186 -0
- loopx/capabilities/auto_research/evidence_packet.py +767 -0
- loopx/capabilities/auto_research/human_view.py +794 -0
- loopx/capabilities/auto_research/kernel.py +191 -0
- loopx/capabilities/auto_research/knn_demo_workspace.py +322 -0
- loopx/capabilities/auto_research/live_evidence.py +248 -0
- loopx/capabilities/auto_research/preset.py +176 -0
- loopx/capabilities/auto_research/research_state.py +1085 -0
- loopx/capabilities/auto_research/role_profiles.py +394 -0
- loopx/capabilities/auto_research/rollout_append.py +97 -0
- loopx/capabilities/auto_research/terminal_result_contract.py +422 -0
- loopx/capabilities/auto_research/terminal_result_projection.py +171 -0
- loopx/capabilities/auto_research/terminal_result_query.py +233 -0
- loopx/capabilities/auto_research/terminal_results.py +349 -0
- loopx/capabilities/auto_research/user_contract.py +190 -0
- loopx/capabilities/auto_research/worker_loop.py +163 -0
- loopx/capabilities/auto_research/worker_runtime.py +777 -0
- loopx/capabilities/auto_research/worker_skill/SKILL.md +343 -0
- loopx/capabilities/benchmark_toolkit/__init__.py +19 -0
- loopx/capabilities/benchmark_toolkit/integrity.py +387 -0
- loopx/capabilities/catalog.py +1875 -0
- loopx/capabilities/change_quality/__init__.py +19 -0
- loopx/capabilities/change_quality/cli.py +171 -0
- loopx/capabilities/change_quality/context.py +156 -0
- loopx/capabilities/change_quality/oracles.py +269 -0
- loopx/capabilities/change_quality/policy.py +34 -0
- loopx/capabilities/change_quality/receipt.py +482 -0
- loopx/capabilities/change_quality/result.py +493 -0
- loopx/capabilities/change_quality/scope.py +171 -0
- loopx/capabilities/change_quality/shadow.py +680 -0
- loopx/capabilities/content_ops/__init__.py +0 -0
- loopx/capabilities/content_ops/cli.py +649 -0
- loopx/capabilities/content_ops/connector_packets.py +164 -0
- loopx/capabilities/content_ops/item_lifecycle.py +1000 -0
- loopx/capabilities/content_ops/layout.py +451 -0
- loopx/capabilities/content_ops/markdown.py +456 -0
- loopx/capabilities/content_ops/schemas.py +51 -0
- loopx/capabilities/content_ops/social_browser_x.py +107 -0
- loopx/capabilities/content_ops/surface.py +1956 -0
- loopx/capabilities/content_ops/templates/layout-catalog-v0.json +72 -0
- loopx/capabilities/context_providers/__init__.py +36 -0
- loopx/capabilities/context_providers/base.py +189 -0
- loopx/capabilities/context_providers/factory.py +32 -0
- loopx/capabilities/context_providers/openviking.py +702 -0
- loopx/capabilities/context_providers/service_ownership.py +185 -0
- loopx/capabilities/decision_context/__init__.py +129 -0
- loopx/capabilities/decision_context/architecture.py +83 -0
- loopx/capabilities/decision_context/assembler.py +849 -0
- loopx/capabilities/decision_context/catalog_entry.py +195 -0
- loopx/capabilities/decision_context/cli.py +310 -0
- loopx/capabilities/decision_context/cursor_commit.py +535 -0
- loopx/capabilities/decision_context/outcome_feedback.py +352 -0
- loopx/capabilities/decision_context/packets.py +654 -0
- loopx/capabilities/decision_context/private_state.py +189 -0
- loopx/capabilities/decision_context/profile.py +453 -0
- loopx/capabilities/decision_context/providers.py +228 -0
- loopx/capabilities/decision_context/review_settlement.py +136 -0
- loopx/capabilities/decision_context/runtime.py +273 -0
- loopx/capabilities/decision_context/sources.py +415 -0
- loopx/capabilities/explore/__init__.py +1 -0
- loopx/capabilities/explore/activation.py +198 -0
- loopx/capabilities/explore/adaptive_replay_planner.py +221 -0
- loopx/capabilities/explore/child_replay_runtime.py +463 -0
- loopx/capabilities/explore/composition_frontier.py +291 -0
- loopx/capabilities/explore/counterfactual_runtime.py +578 -0
- loopx/capabilities/explore/episode_runtime.py +647 -0
- loopx/capabilities/explore/harness_checkpoint.py +171 -0
- loopx/capabilities/explore/harness_gate.py +115 -0
- loopx/capabilities/explore/harness_runtime.py +1124 -0
- loopx/capabilities/explore/replay_metrics.py +206 -0
- loopx/capabilities/explore/replay_runtime.py +1271 -0
- loopx/capabilities/explore/resource_portfolio.py +173 -0
- loopx/capabilities/explore/result_log.py +974 -0
- loopx/capabilities/explore/router_state.py +432 -0
- loopx/capabilities/explore/source_history_reconcile.py +255 -0
- loopx/capabilities/explore/speculative_scheduler.py +498 -0
- loopx/capabilities/explore/todo_branch_plan.py +650 -0
- loopx/capabilities/explore/todo_evidence.py +141 -0
- loopx/capabilities/explore/trace_runtime.py +284 -0
- loopx/capabilities/explore/worker_branch_plan.py +1257 -0
- loopx/capabilities/integration_branch/__init__.py +13 -0
- loopx/capabilities/integration_branch/cli.py +148 -0
- loopx/capabilities/integration_branch/core.py +916 -0
- loopx/capabilities/issue_fix/__init__.py +19 -0
- loopx/capabilities/issue_fix/acceptance_loop.py +1050 -0
- loopx/capabilities/issue_fix/candidate_evidence.py +503 -0
- loopx/capabilities/issue_fix/candidate_preflight.py +676 -0
- loopx/capabilities/issue_fix/cli.py +1822 -0
- loopx/capabilities/issue_fix/cli_input.py +87 -0
- loopx/capabilities/issue_fix/content_ops_cli.py +148 -0
- loopx/capabilities/issue_fix/discovered_issue_promotion.py +947 -0
- loopx/capabilities/issue_fix/explore_projection.py +710 -0
- loopx/capabilities/issue_fix/feasibility.py +542 -0
- loopx/capabilities/issue_fix/github_public.py +661 -0
- loopx/capabilities/issue_fix/intake_surface.py +832 -0
- loopx/capabilities/issue_fix/metadata_preview.py +218 -0
- loopx/capabilities/issue_fix/metrics_projection.py +1340 -0
- loopx/capabilities/issue_fix/metrics_supplement.py +634 -0
- loopx/capabilities/issue_fix/metrics_supplement_cli.py +127 -0
- loopx/capabilities/issue_fix/outcome_projection.py +1235 -0
- loopx/capabilities/issue_fix/periodic_report.py +189 -0
- loopx/capabilities/issue_fix/pr_description.py +418 -0
- loopx/capabilities/issue_fix/pr_gate_reconcile.py +496 -0
- loopx/capabilities/issue_fix/pr_gate_reconcile_cli.py +464 -0
- loopx/capabilities/issue_fix/pr_lifecycle.py +1327 -0
- loopx/capabilities/issue_fix/pr_lifecycle_rollout.py +85 -0
- loopx/capabilities/issue_fix/pr_monitor_materialization.py +257 -0
- loopx/capabilities/issue_fix/pr_review_ack.py +439 -0
- loopx/capabilities/issue_fix/provider_hooks.py +24 -0
- loopx/capabilities/issue_fix/repository_commit_evidence.py +186 -0
- loopx/capabilities/issue_fix/repository_context.py +457 -0
- loopx/capabilities/issue_fix/repository_memory.py +459 -0
- loopx/capabilities/issue_fix/repository_memory_provider.py +1454 -0
- loopx/capabilities/issue_fix/repository_snapshot.py +454 -0
- loopx/capabilities/issue_fix/reviewer_cli.py +917 -0
- loopx/capabilities/issue_fix/reviewer_notification.py +882 -0
- loopx/capabilities/issue_fix/reviewer_notification_drain.py +942 -0
- loopx/capabilities/issue_fix/reviewer_recommendation.py +1057 -0
- loopx/capabilities/issue_fix/reviewer_request.py +1282 -0
- loopx/capabilities/issue_fix/reward_memory.py +879 -0
- loopx/capabilities/issue_fix/workflow_plan.py +1286 -0
- loopx/capabilities/material_lifecycle/__init__.py +161 -0
- loopx/capabilities/material_lifecycle/_validation.py +183 -0
- loopx/capabilities/material_lifecycle/apply.py +672 -0
- loopx/capabilities/material_lifecycle/architecture.py +122 -0
- loopx/capabilities/material_lifecycle/cli.py +161 -0
- loopx/capabilities/material_lifecycle/decision_planning.py +470 -0
- loopx/capabilities/material_lifecycle/explore_execution.py +306 -0
- loopx/capabilities/material_lifecycle/intake.py +869 -0
- loopx/capabilities/material_lifecycle/inventory.py +147 -0
- loopx/capabilities/material_lifecycle/lifecycle.py +98 -0
- loopx/capabilities/material_lifecycle/preparation.py +147 -0
- loopx/capabilities/material_lifecycle/project_skill.py +83 -0
- loopx/capabilities/material_lifecycle/ranking.py +267 -0
- loopx/capabilities/material_lifecycle/readable_projection.py +500 -0
- loopx/capabilities/material_lifecycle/rebuild.py +480 -0
- loopx/capabilities/material_lifecycle/settlement.py +238 -0
- loopx/capabilities/periodic_report/__init__.py +71 -0
- loopx/capabilities/periodic_report/adapters.py +939 -0
- loopx/capabilities/periodic_report/archive.py +422 -0
- loopx/capabilities/periodic_report/bindings.py +705 -0
- loopx/capabilities/periodic_report/cli.py +277 -0
- loopx/capabilities/periodic_report/core.py +691 -0
- loopx/capabilities/periodic_report/extension_envelope.py +66 -0
- loopx/capabilities/periodic_report/presets.py +103 -0
- loopx/capabilities/periodic_report/profile.py +235 -0
- loopx/capabilities/periodic_report/project_progress.py +179 -0
- loopx/capabilities/periodic_report/triggers.py +452 -0
- loopx/capabilities/pr_review_queue/__init__.py +17 -0
- loopx/capabilities/pr_review_queue/core.py +506 -0
- loopx/capabilities/pr_review_queue/review_contract.py +506 -0
- loopx/capabilities/registry.py +192 -0
- loopx/capabilities/reward_memory/__init__.py +75 -0
- loopx/capabilities/reward_memory/application.py +819 -0
- loopx/capabilities/reward_memory/architecture.py +572 -0
- loopx/capabilities/reward_memory/candidate_review.py +511 -0
- loopx/capabilities/reward_memory/cli.py +469 -0
- loopx/capabilities/reward_memory/dogfood.py +574 -0
- loopx/capabilities/reward_memory/evaluation.py +296 -0
- loopx/capabilities/reward_memory/evaluation_fixtures.py +362 -0
- loopx/capabilities/reward_memory/experiment.py +567 -0
- loopx/capabilities/reward_memory/health.py +222 -0
- loopx/capabilities/reward_memory/ingestion.py +519 -0
- loopx/capabilities/reward_memory/registry.py +600 -0
- loopx/capabilities/reward_memory/runtime_hooks.py +312 -0
- loopx/capabilities/reward_memory/scoped_feedback.py +173 -0
- loopx/capabilities/semantic_preference/__init__.py +12 -0
- loopx/capabilities/semantic_preference/cli.py +189 -0
- loopx/capabilities/semantic_preference/contract.py +592 -0
- loopx/capabilities/semantic_preference/reward_memory.py +62 -0
- loopx/capabilities/value_connectors/__init__.py +1 -0
- loopx/capabilities/value_connectors/cli.py +401 -0
- loopx/capabilities/value_connectors/finance_extension_migration.py +108 -0
- loopx/capabilities/value_connectors/install_check.py +147 -0
- loopx/capabilities/value_connectors/planner.py +733 -0
- loopx/capabilities/value_connectors/source_map.py +446 -0
- loopx/claude_goal_baseline.py +138 -0
- loopx/claude_goal_mode/__init__.py +23 -0
- loopx/claude_goal_mode/hooks/goal_policy.py +212 -0
- loopx/claude_goal_mode/hooks/goal_state.py +139 -0
- loopx/claude_goal_mode/mcp/loopx_mcp.py +167 -0
- loopx/claude_goal_mode/scripts/connect.py +103 -0
- loopx/claude_goal_mode/scripts/goalmode_cmd.py +241 -0
- loopx/claude_goal_mode/scripts/install.py +328 -0
- loopx/claude_goal_mode/statusline/goal_status.py +97 -0
- loopx/cli.py +836 -0
- loopx/cli_commands/__init__.py +334 -0
- loopx/cli_commands/_host_thread.py +13 -0
- loopx/cli_commands/agentissue_runner_flow.py +447 -0
- loopx/cli_commands/agents_last_exam.py +160 -0
- loopx/cli_commands/agents_last_exam_baked_input.py +302 -0
- loopx/cli_commands/agents_last_exam_host_codex.py +374 -0
- loopx/cli_commands/agents_last_exam_launch_dry_run.py +372 -0
- loopx/cli_commands/agents_last_exam_local_plan.py +322 -0
- loopx/cli_commands/agents_last_exam_runner_source.py +352 -0
- loopx/cli_commands/agents_last_exam_task_material.py +335 -0
- loopx/cli_commands/agents_last_exam_validation_gate.py +236 -0
- loopx/cli_commands/benchmark_boundary.py +499 -0
- loopx/cli_commands/benchmark_dispatch.py +161 -0
- loopx/cli_commands/benchmark_release_outcome.py +123 -0
- loopx/cli_commands/benchmark_review_lifecycle.py +1275 -0
- loopx/cli_commands/benchmark_run_ledger.py +763 -0
- loopx/cli_commands/benchmark_run_ledger_case_analysis.py +249 -0
- loopx/cli_commands/benchmark_run_ledger_classification.py +45 -0
- loopx/cli_commands/benchmark_run_ledger_maintenance.py +486 -0
- loopx/cli_commands/benchmark_run_ledger_maintenance_registration.py +342 -0
- loopx/cli_commands/benchmark_run_ledger_maintenance_rendering.py +233 -0
- loopx/cli_commands/benchmark_run_ledger_parity.py +92 -0
- loopx/cli_commands/bootstrap_connect.py +238 -0
- loopx/cli_commands/canary.py +707 -0
- loopx/cli_commands/canary_release_qualification.py +79 -0
- loopx/cli_commands/capability.py +96 -0
- loopx/cli_commands/doctor.py +43 -0
- loopx/cli_commands/dreaming.py +143 -0
- loopx/cli_commands/edgebench.py +205 -0
- loopx/cli_commands/evidence_log.py +275 -0
- loopx/cli_commands/explore.py +989 -0
- loopx/cli_commands/explore_planning_commands.py +157 -0
- loopx/cli_commands/extension.py +271 -0
- loopx/cli_commands/first_run_report.py +73 -0
- loopx/cli_commands/goal_channel.py +656 -0
- loopx/cli_commands/handoff_mode.py +158 -0
- loopx/cli_commands/history.py +622 -0
- loopx/cli_commands/host_mode_plan.py +113 -0
- loopx/cli_commands/lark_inbox.py +431 -0
- loopx/cli_commands/lark_kanban.py +629 -0
- loopx/cli_commands/ml_experiment.py +321 -0
- loopx/cli_commands/multi_agent.py +211 -0
- loopx/cli_commands/opencode2_goal_worker.py +217 -0
- loopx/cli_commands/pr_review.py +167 -0
- loopx/cli_commands/presentation.py +218 -0
- loopx/cli_commands/preset.py +96 -0
- loopx/cli_commands/project.py +150 -0
- loopx/cli_commands/project_lifecycle.py +915 -0
- loopx/cli_commands/quota.py +859 -0
- loopx/cli_commands/quota_registration.py +241 -0
- loopx/cli_commands/quota_request.py +113 -0
- loopx/cli_commands/ready_score.py +110 -0
- loopx/cli_commands/registry_admin.py +975 -0
- loopx/cli_commands/registry_admin_configure.py +344 -0
- loopx/cli_commands/registry_admin_peer.py +84 -0
- loopx/cli_commands/registry_authority.py +218 -0
- loopx/cli_commands/review_batch.py +146 -0
- loopx/cli_commands/slash_commands.py +145 -0
- loopx/cli_commands/start_goal.py +251 -0
- loopx/cli_commands/starter.py +175 -0
- loopx/cli_commands/starter_bootstrap.py +179 -0
- loopx/cli_commands/starter_bootstrap_registration.py +198 -0
- loopx/cli_commands/starter_runtime_idle.py +107 -0
- loopx/cli_commands/starter_scheduler.py +207 -0
- loopx/cli_commands/starter_session_runtime.py +152 -0
- loopx/cli_commands/starter_visible_common.py +54 -0
- loopx/cli_commands/starter_visible_driver.py +161 -0
- loopx/cli_commands/starter_visible_pilot.py +278 -0
- loopx/cli_commands/status.py +867 -0
- loopx/cli_commands/status_registration.py +239 -0
- loopx/cli_commands/summary_all.py +222 -0
- loopx/cli_commands/support_control.py +809 -0
- loopx/cli_commands/support_control_registry.py +68 -0
- loopx/cli_commands/support_control_supervisor.py +289 -0
- loopx/cli_commands/task_lease.py +306 -0
- loopx/cli_commands/terminal_bench_adapter.py +717 -0
- loopx/cli_commands/terminal_bench_environment_result.py +1246 -0
- loopx/cli_commands/todo.py +940 -0
- loopx/cli_commands/todo_argument_validation.py +572 -0
- loopx/cli_commands/todo_event.py +114 -0
- loopx/cli_commands/turn.py +804 -0
- loopx/cli_commands/version.py +46 -0
- loopx/cli_commands/worker_bridge.py +659 -0
- loopx/cli_rollout.py +314 -0
- loopx/codex_cli_goal_tui.py +672 -0
- loopx/codex_cli_probe.py +1530 -0
- loopx/codex_cli_probe_markdown.py +935 -0
- loopx/codex_cli_runtime_probe.py +733 -0
- loopx/codex_cli_scheduler.py +564 -0
- loopx/codex_goal_baseline.py +620 -0
- loopx/configuration_catalog.py +617 -0
- loopx/configure_goal.py +1375 -0
- loopx/contract.py +996 -0
- loopx/control_plane/__init__.py +71 -0
- loopx/control_plane/agents/__init__.py +1 -0
- loopx/control_plane/agents/agent_lane_recommendation.py +516 -0
- loopx/control_plane/agents/agent_scope.py +1578 -0
- loopx/control_plane/agents/agent_scope_frontier.py +60 -0
- loopx/control_plane/agents/capability_gate.py +531 -0
- loopx/control_plane/agents/identity.py +140 -0
- loopx/control_plane/agents/legacy_migration.py +169 -0
- loopx/control_plane/agents/management_projection.py +658 -0
- loopx/control_plane/agents/material_frontier.py +608 -0
- loopx/control_plane/agents/material_handoff.py +156 -0
- loopx/control_plane/agents/multi_agent/__init__.py +1 -0
- loopx/control_plane/agents/multi_agent/codex_executable.py +207 -0
- loopx/control_plane/agents/multi_agent/collective_round_ledger.py +387 -0
- loopx/control_plane/agents/multi_agent/contract.py +474 -0
- loopx/control_plane/agents/multi_agent/recipe.py +110 -0
- loopx/control_plane/agents/multi_agent/role_successor.py +297 -0
- loopx/control_plane/agents/multi_agent/runtime_scripts.py +426 -0
- loopx/control_plane/agents/multi_agent/visible_launch_policy.py +149 -0
- loopx/control_plane/agents/multi_agent/visible_wake_scheduler.py +392 -0
- loopx/control_plane/agents/profile.py +216 -0
- loopx/control_plane/agents/runtime_model.py +73 -0
- loopx/control_plane/agents/subagent_activity.py +164 -0
- loopx/control_plane/agents/supervisor.py +544 -0
- loopx/control_plane/agents/supervisor_events.py +462 -0
- loopx/control_plane/agents/supervisor_inject.py +204 -0
- loopx/control_plane/agents/work_mode.py +56 -0
- loopx/control_plane/agents/workspace_guard.py +364 -0
- loopx/control_plane/effect_program.py +644 -0
- loopx/control_plane/goals/__init__.py +1 -0
- loopx/control_plane/goals/active_state_event_projection.py +103 -0
- loopx/control_plane/goals/active_state_metadata.py +47 -0
- loopx/control_plane/goals/active_state_sections.py +58 -0
- loopx/control_plane/goals/configure_goal_service.py +354 -0
- loopx/control_plane/goals/contract_health.py +132 -0
- loopx/control_plane/goals/dreaming.py +152 -0
- loopx/control_plane/goals/global_registry_health.py +199 -0
- loopx/control_plane/goals/global_registry_shadow.py +33 -0
- loopx/control_plane/goals/goal_channel.py +34 -0
- loopx/control_plane/goals/goal_channel_projection.py +560 -0
- loopx/control_plane/goals/goal_frontier/__init__.py +1917 -0
- loopx/control_plane/goals/goal_frontier/ack_policy.py +149 -0
- loopx/control_plane/goals/goal_frontier/outcome_continuity.py +437 -0
- loopx/control_plane/goals/goal_frontier/replan_rules.py +210 -0
- loopx/control_plane/goals/goal_frontier/semantic_history.py +314 -0
- loopx/control_plane/goals/goal_frontier/terminal.py +180 -0
- loopx/control_plane/goals/goal_vision.py +443 -0
- loopx/control_plane/goals/goal_vision_policy.py +36 -0
- loopx/control_plane/goals/goal_vision_state.py +62 -0
- loopx/control_plane/goals/goal_vision_wait.py +290 -0
- loopx/control_plane/goals/path_resolution.py +20 -0
- loopx/control_plane/goals/start_contract.py +206 -0
- loopx/control_plane/goals/vision_checkpoint.py +92 -0
- loopx/control_plane/handoff/__init__.py +1 -0
- loopx/control_plane/handoff/cross_runtime_impl_review.py +311 -0
- loopx/control_plane/handoff/delivery_contract.py +161 -0
- loopx/control_plane/handoff/handoff_runs.py +71 -0
- loopx/control_plane/handoff/project_handoff.py +155 -0
- loopx/control_plane/handoff/review_batch.py +463 -0
- loopx/control_plane/handoff/review_packet_context.py +216 -0
- loopx/control_plane/heartbeat/agent.py +173 -0
- loopx/control_plane/heartbeat/budget.py +66 -0
- loopx/control_plane/heartbeat/builder.py +501 -0
- loopx/control_plane/heartbeat/host.py +64 -0
- loopx/control_plane/heartbeat/rules.py +68 -0
- loopx/control_plane/heartbeat/task_body.py +759 -0
- loopx/control_plane/heartbeat/visible_goal.py +86 -0
- loopx/control_plane/projects/__init__.py +1 -0
- loopx/control_plane/projects/contract.py +25 -0
- loopx/control_plane/projects/registry.py +663 -0
- loopx/control_plane/quota/__init__.py +1 -0
- loopx/control_plane/quota/cli_projection.py +704 -0
- loopx/control_plane/quota/decision_summary.py +431 -0
- loopx/control_plane/quota/effect_program.py +152 -0
- loopx/control_plane/quota/error_codes.py +19 -0
- loopx/control_plane/quota/goal_boundary.py +464 -0
- loopx/control_plane/quota/heartbeat_receipt.py +277 -0
- loopx/control_plane/quota/heartbeat_recommendation.py +718 -0
- loopx/control_plane/quota/host_poll_receipts.py +162 -0
- loopx/control_plane/quota/live_decision.py +142 -0
- loopx/control_plane/quota/monitor_poll.py +786 -0
- loopx/control_plane/quota/policy_constants.py +40 -0
- loopx/control_plane/quota/projection_repair.py +262 -0
- loopx/control_plane/quota/recent_runs.py +210 -0
- loopx/control_plane/quota/scheduler_ack.py +490 -0
- loopx/control_plane/quota/selected_todo_projection.py +139 -0
- loopx/control_plane/quota/settlement.py +437 -0
- loopx/control_plane/quota/settlement_cli.py +246 -0
- loopx/control_plane/quota/settlement_validation.py +64 -0
- loopx/control_plane/quota/settlement_workspace_causality.py +180 -0
- loopx/control_plane/quota/should_run.py +249 -0
- loopx/control_plane/quota/should_run_packet.py +1165 -0
- loopx/control_plane/quota/should_run_prepare.py +675 -0
- loopx/control_plane/quota/slot_accounting.py +1123 -0
- loopx/control_plane/quota/spend_sources.py +11 -0
- loopx/control_plane/quota/stall_repair.py +397 -0
- loopx/control_plane/quota/states.py +29 -0
- loopx/control_plane/quota/task_orchestration.py +448 -0
- loopx/control_plane/quota/task_orchestration_admission.py +497 -0
- loopx/control_plane/quota/turn_envelope.py +889 -0
- loopx/control_plane/quota/usage_summary.py +140 -0
- loopx/control_plane/reward_memory.py +43 -0
- loopx/control_plane/runtime/__init__.py +2 -0
- loopx/control_plane/runtime/active_user_assisted_pilot.py +275 -0
- loopx/control_plane/runtime/agent_scoped_evidence_log.py +435 -0
- loopx/control_plane/runtime/decision_freshness.py +203 -0
- loopx/control_plane/runtime/event_ledger.py +197 -0
- loopx/control_plane/runtime/event_store_migration_bridge.py +196 -0
- loopx/control_plane/runtime/goal_project_route.py +70 -0
- loopx/control_plane/runtime/local_state_write_correctness.py +242 -0
- loopx/control_plane/runtime/promotion_readiness.py +152 -0
- loopx/control_plane/runtime/public_safety.py +120 -0
- loopx/control_plane/runtime/run_artifacts.py +78 -0
- loopx/control_plane/runtime/run_compaction.py +397 -0
- loopx/control_plane/runtime/run_context_retention.py +241 -0
- loopx/control_plane/runtime/run_history.py +132 -0
- loopx/control_plane/runtime/run_index_duplicates.py +205 -0
- loopx/control_plane/runtime/run_index_rebuild.py +263 -0
- loopx/control_plane/runtime/run_ingest_health.py +336 -0
- loopx/control_plane/runtime/runtime_projection_route.py +624 -0
- loopx/control_plane/runtime/runtime_projection_writer.py +98 -0
- loopx/control_plane/runtime/session_runtime.py +339 -0
- loopx/control_plane/runtime/shared_runtime_material_projection.py +332 -0
- loopx/control_plane/runtime/shared_runtime_refresh_projection.py +183 -0
- loopx/control_plane/runtime/stale_latest_run.py +90 -0
- loopx/control_plane/runtime/status_classifications.py +49 -0
- loopx/control_plane/runtime/status_projection_cache.py +235 -0
- loopx/control_plane/runtime/stride_observation.py +144 -0
- loopx/control_plane/runtime/time.py +39 -0
- loopx/control_plane/runtime/trajectory_hygiene.py +149 -0
- loopx/control_plane/runtime/validation_command.py +69 -0
- loopx/control_plane/scheduler/__init__.py +1 -0
- loopx/control_plane/scheduler/ack.py +329 -0
- loopx/control_plane/scheduler/arbitration.py +188 -0
- loopx/control_plane/scheduler/automation_liveness.py +183 -0
- loopx/control_plane/scheduler/execution_context.py +555 -0
- loopx/control_plane/scheduler/external_evidence_observation.py +428 -0
- loopx/control_plane/scheduler/monitor_display.py +143 -0
- loopx/control_plane/scheduler/monitor_poll_policy.py +161 -0
- loopx/control_plane/scheduler/monitor_poll_writeback.py +351 -0
- loopx/control_plane/scheduler/monitor_target.py +64 -0
- loopx/control_plane/scheduler/monitor_todo.py +146 -0
- loopx/control_plane/scheduler/monitor_wait.py +237 -0
- loopx/control_plane/scheduler/scheduler_hint.py +1284 -0
- loopx/control_plane/scheduler/state.py +354 -0
- loopx/control_plane/scheduler/state_transition_rules.py +179 -0
- loopx/control_plane/scheduler/time.py +10 -0
- loopx/control_plane/settlement_driver.py +293 -0
- loopx/control_plane/status/__init__.py +6 -0
- loopx/control_plane/status/active_state_projection.py +105 -0
- loopx/control_plane/status/agent_lane_projection.py +375 -0
- loopx/control_plane/status/attention_projection.py +74 -0
- loopx/control_plane/status/autonomous_replan_projection.py +103 -0
- loopx/control_plane/status/collection.py +140 -0
- loopx/control_plane/status/contract_projection.py +31 -0
- loopx/control_plane/status/dreaming_projection.py +52 -0
- loopx/control_plane/status/goal_attention_projection.py +157 -0
- loopx/control_plane/status/lifecycle_projection.py +110 -0
- loopx/control_plane/status/monitor_display_projection.py +69 -0
- loopx/control_plane/status/registry_health_projection.py +75 -0
- loopx/control_plane/status/run_projection.py +70 -0
- loopx/control_plane/status/runtime_summaries.py +161 -0
- loopx/control_plane/testing/__init__.py +1 -0
- loopx/control_plane/testing/actual_default_model_behavior_portfolio.py +1371 -0
- loopx/control_plane/testing/canary_harness.py +182 -0
- loopx/control_plane/testing/capability_monitor_repair_tool_behavior.py +674 -0
- loopx/control_plane/testing/cli_output_budget.py +807 -0
- loopx/control_plane/testing/cli_output_differential.py +250 -0
- loopx/control_plane/testing/cli_output_semantics.py +87 -0
- loopx/control_plane/testing/control_plane_composition_scenarios.py +225 -0
- loopx/control_plane/testing/decision_replay.py +268 -0
- loopx/control_plane/testing/doubao_model_behavior_actor.py +559 -0
- loopx/control_plane/testing/model_behavior_corpus.py +344 -0
- loopx/control_plane/testing/model_behavior_qualification.py +769 -0
- loopx/control_plane/testing/model_behavior_retained_cases.py +235 -0
- loopx/control_plane/testing/model_tool_behavior.py +536 -0
- loopx/control_plane/testing/onboarding_model_behavior_qualification.py +642 -0
- loopx/control_plane/testing/quota_fixtures.py +208 -0
- loopx/control_plane/testing/quota_should_run_parity.py +57 -0
- loopx/control_plane/testing/release_commit_qualification.py +671 -0
- loopx/control_plane/testing/replan_semantic_action_behavior.py +1302 -0
- loopx/control_plane/testing/scoped_gate_successor_tool_behavior.py +527 -0
- loopx/control_plane/testing/selected_todo_tool_behavior.py +1002 -0
- loopx/control_plane/testing/terminal_settlement_tool_behavior.py +656 -0
- loopx/control_plane/todos/__init__.py +1 -0
- loopx/control_plane/todos/active_state_editing.py +296 -0
- loopx/control_plane/todos/active_state_todo_parser.py +138 -0
- loopx/control_plane/todos/active_state_todos.py +175 -0
- loopx/control_plane/todos/addition.py +103 -0
- loopx/control_plane/todos/claim_visibility.py +253 -0
- loopx/control_plane/todos/completed_archive.py +139 -0
- loopx/control_plane/todos/completion_fence.py +49 -0
- loopx/control_plane/todos/completion_policy.py +153 -0
- loopx/control_plane/todos/completion_validation.py +248 -0
- loopx/control_plane/todos/completion_validation_accountability.py +27 -0
- loopx/control_plane/todos/completion_validation_projection.py +57 -0
- loopx/control_plane/todos/contract.py +1476 -0
- loopx/control_plane/todos/decision_scope.py +554 -0
- loopx/control_plane/todos/deferred_resume.py +546 -0
- loopx/control_plane/todos/durable_completion.py +201 -0
- loopx/control_plane/todos/event_writeback.py +484 -0
- loopx/control_plane/todos/frontier_deadline.py +132 -0
- loopx/control_plane/todos/handoff_gate.py +283 -0
- loopx/control_plane/todos/handoff_mode.py +444 -0
- loopx/control_plane/todos/handoff_note.py +202 -0
- loopx/control_plane/todos/line_update.py +361 -0
- loopx/control_plane/todos/list_projection.py +205 -0
- loopx/control_plane/todos/markdown.py +199 -0
- loopx/control_plane/todos/monitor_metadata.py +88 -0
- loopx/control_plane/todos/mutation_authority.py +299 -0
- loopx/control_plane/todos/projection.py +655 -0
- loopx/control_plane/todos/quota_summary.py +1138 -0
- loopx/control_plane/todos/route_continuation.py +267 -0
- loopx/control_plane/todos/succession_warning.py +174 -0
- loopx/control_plane/todos/summary_item.py +223 -0
- loopx/control_plane/todos/text.py +30 -0
- loopx/control_plane/todos/todo_index.py +226 -0
- loopx/control_plane/todos/todo_summary.py +1458 -0
- loopx/control_plane/todos/unblock_resume.py +326 -0
- loopx/control_plane/todos/user_gate.py +263 -0
- loopx/control_plane/todos/write_hint.py +63 -0
- loopx/control_plane/todos/write_policy.py +135 -0
- loopx/control_plane/turn_driver/__init__.py +85 -0
- loopx/control_plane/turn_driver/codex_cli.py +502 -0
- loopx/control_plane/turn_driver/driver.py +355 -0
- loopx/control_plane/turn_driver/executor.py +1468 -0
- loopx/control_plane/turn_driver/loop_controller.py +669 -0
- loopx/control_plane/turn_driver/settlement.py +318 -0
- loopx/control_plane/turn_driver/transaction.py +375 -0
- loopx/control_plane/work_items/__init__.py +1 -0
- loopx/control_plane/work_items/attention_fields.py +56 -0
- loopx/control_plane/work_items/attention_item.py +77 -0
- loopx/control_plane/work_items/attention_queue.py +322 -0
- loopx/control_plane/work_items/attention_routing.py +213 -0
- loopx/control_plane/work_items/autonomous_candidates.py +135 -0
- loopx/control_plane/work_items/autonomous_replan_ack.py +276 -0
- loopx/control_plane/work_items/autonomous_replan_obligation.py +786 -0
- loopx/control_plane/work_items/backlog_hygiene.py +59 -0
- loopx/control_plane/work_items/capability_monitor_fallback.py +221 -0
- loopx/control_plane/work_items/delivery_batch_scale.py +66 -0
- loopx/control_plane/work_items/delivery_outcome.py +152 -0
- loopx/control_plane/work_items/delivery_signals.py +113 -0
- loopx/control_plane/work_items/execution_obligation.py +235 -0
- loopx/control_plane/work_items/goal_route_hint.py +320 -0
- loopx/control_plane/work_items/interaction_contract.py +1540 -0
- loopx/control_plane/work_items/issue_meta_surface.py +159 -0
- loopx/control_plane/work_items/lifecycle.py +139 -0
- loopx/control_plane/work_items/operator_inbox.py +266 -0
- loopx/control_plane/work_items/outcome_followthrough.py +69 -0
- loopx/control_plane/work_items/primary_action.py +326 -0
- loopx/control_plane/work_items/progress_observation.py +630 -0
- loopx/control_plane/work_items/project_asset.py +675 -0
- loopx/control_plane/work_items/repair_delta.py +693 -0
- loopx/control_plane/work_items/runtime_capability_reentry.py +168 -0
- loopx/control_plane/work_items/semantic_replan_writeback.py +177 -0
- loopx/control_plane/work_items/status_contract.py +49 -0
- loopx/control_plane/work_items/task_graph.py +1046 -0
- loopx/control_plane/work_items/task_lease.py +1254 -0
- loopx/control_plane/work_items/task_lease_settlement.py +422 -0
- loopx/control_plane/work_items/work_lane.py +510 -0
- loopx/control_plane/work_items/work_lane_context.py +161 -0
- loopx/demo.py +247 -0
- loopx/diagnose.py +633 -0
- loopx/doctor.py +1251 -0
- loopx/domain_packs/__init__.py +1 -0
- loopx/domain_packs/issue_fix.py +571 -0
- loopx/domain_packs/ml_experiment.py +854 -0
- loopx/domain_state.py +137 -0
- loopx/dreaming.py +706 -0
- loopx/entrypoint.py +16 -0
- loopx/event_sourced_state.py +981 -0
- loopx/execution_profile.py +286 -0
- loopx/experiments/__init__.py +1 -0
- loopx/experiments/planner_worker/__init__.py +1 -0
- loopx/experiments/planner_worker/contract.py +523 -0
- loopx/experiments/planner_worker/runtime.py +391 -0
- loopx/experiments/planner_worker/traex.py +461 -0
- loopx/explore_graph.py +11 -0
- loopx/extensions/__init__.py +1 -0
- loopx/extensions/bundled.py +28 -0
- loopx/extensions/execution_envelope.py +126 -0
- loopx/extensions/lark/__init__.py +11 -0
- loopx/extensions/lark/event_collector.py +478 -0
- loopx/extensions/lark/event_collector_runtime.py +506 -0
- loopx/extensions/lark/event_inbox.py +454 -0
- loopx/extensions/lark/extension.toml +88 -0
- loopx/extensions/lark/goal_channel.py +44 -0
- loopx/extensions/lark/goal_channel_contracts.py +388 -0
- loopx/extensions/lark/goal_channel_lifecycle.py +218 -0
- loopx/extensions/lark/goal_channel_runtime.py +792 -0
- loopx/extensions/lark/goal_channel_setup.py +805 -0
- loopx/extensions/lark/goal_channel_targets.py +215 -0
- loopx/extensions/lark/goal_channel_transport.py +281 -0
- loopx/extensions/lark/inbox_reactions.py +650 -0
- loopx/extensions/lark/inbox_reply.py +430 -0
- loopx/extensions/lark/presentation/__init__.py +11 -0
- loopx/extensions/lark/presentation/explore_results.py +2276 -0
- loopx/extensions/lark/presentation/explore_singleflight.py +127 -0
- loopx/extensions/lark/presentation/explore_source_guard.py +121 -0
- loopx/extensions/lark/presentation/explore_stage_document.py +703 -0
- loopx/extensions/lark/presentation/explore_visual_integrity.py +122 -0
- loopx/extensions/lark/presentation/explore_visual_readback.py +452 -0
- loopx/extensions/lark/presentation/explore_visual_styles.py +156 -0
- loopx/extensions/lark/presentation/issue_fix_surface.py +612 -0
- loopx/extensions/lark/presentation/kanban.py +2791 -0
- loopx/extensions/lark/presentation/message_card.py +112 -0
- loopx/extensions/lark/presentation/periodic_report.py +261 -0
- loopx/extensions/lark/presentation/projection_rows.py +600 -0
- loopx/extensions/lark/presentation/record_io.py +95 -0
- loopx/extensions/lark/presentation/sync_receipt.py +145 -0
- loopx/extensions/lark/private_json.py +40 -0
- loopx/extensions/lark/provider.py +86 -0
- loopx/extensions/lark/reviewer_notification.py +604 -0
- loopx/extensions/manifest.py +385 -0
- loopx/extensions/openviking_periodic_report/__init__.py +17 -0
- loopx/extensions/openviking_periodic_report/activation.py +173 -0
- loopx/extensions/openviking_periodic_report/extension.toml +17 -0
- loopx/extensions/openviking_periodic_report/provider.py +355 -0
- loopx/extensions/openviking_periodic_report/sink.py +117 -0
- loopx/extensions/openviking_semantic_preference/__init__.py +5 -0
- loopx/extensions/openviking_semantic_preference/extension.toml +16 -0
- loopx/extensions/openviking_semantic_preference/history_export.py +484 -0
- loopx/extensions/openviking_semantic_preference/project_peer.py +68 -0
- loopx/extensions/openviking_semantic_preference/provider.py +312 -0
- loopx/extensions/presentation.py +979 -0
- loopx/extensions/process_runtime.py +204 -0
- loopx/extensions/readiness.py +168 -0
- loopx/extensions/runtime.py +931 -0
- loopx/extensions/scaffold.py +335 -0
- loopx/feedback.py +581 -0
- loopx/file_lock.py +382 -0
- loopx/global_registry.py +842 -0
- loopx/global_risks.py +970 -0
- loopx/global_todos.py +568 -0
- loopx/handoff_budget.py +28 -0
- loopx/heartbeat_prequota.py +80 -0
- loopx/heartbeat_prompt.py +159 -0
- loopx/help_surface.py +516 -0
- loopx/history.py +1507 -0
- loopx/host_loop_activation.py +1311 -0
- loopx/host_mode_planner.py +991 -0
- loopx/install_contract.py +1 -0
- loopx/interface_budget.py +196 -0
- loopx/long_task_cadence.py +208 -0
- loopx/materials.py +185 -0
- loopx/ml_experiment.py +3 -0
- loopx/onboarding.py +214 -0
- loopx/opencode2_goal_mode/README.md +81 -0
- loopx/opencode2_goal_mode/__init__.py +9 -0
- loopx/opencode2_goal_mode/opencode2-goal-worker.mjs +1018 -0
- loopx/opencode_goal_mode/README.md +99 -0
- loopx/opencode_goal_mode/__init__.py +13 -0
- loopx/opencode_goal_mode/goal-bridge-runtime.mjs +858 -0
- loopx/opencode_goal_mode/loopx-goal.js +8 -0
- loopx/operator_gate.py +420 -0
- loopx/orchestration.py +127 -0
- loopx/paths.py +59 -0
- loopx/pi_goal_mode/README.md +67 -0
- loopx/pi_goal_mode/__init__.py +13 -0
- loopx/pi_goal_mode/loopx-goal.ts +254 -0
- loopx/pi_goal_mode/pi-goal-loop-runtime.mjs +574 -0
- loopx/pr_review.py +1206 -0
- loopx/presentation/__init__.py +1 -0
- loopx/presentation/explore_views.py +1334 -0
- loopx/presentation/markdown.py +61 -0
- loopx/presentation/projection_source_reconcile.py +140 -0
- loopx/presentation/public_safety.py +42 -0
- loopx/presentation/renderers/__init__.py +17 -0
- loopx/presentation/renderers/goal_channel_html.py +269 -0
- loopx/presentation/renderers/periodic_report_html.py +786 -0
- loopx/presentation/renderers/periodic_report_markdown.py +184 -0
- loopx/presentation/renderers/quota_event_markdown.py +116 -0
- loopx/presentation/renderers/quota_markdown.py +1112 -0
- loopx/presentation/renderers/status_markdown.py +1570 -0
- loopx/presentation/renderers/trajectory_hygiene_markdown.py +39 -0
- loopx/presentation/renderers/turn_envelope_markdown.py +33 -0
- loopx/presentation/sinks/__init__.py +5 -0
- loopx/presentation/sinks/openviking_periodic_report.py +7 -0
- loopx/presentation/static_site.py +691 -0
- loopx/presets.py +369 -0
- loopx/project_alias.py +217 -0
- loopx/project_map.py +589 -0
- loopx/project_prompt.py +1153 -0
- loopx/project_skill_cli.py +125 -0
- loopx/project_skill_delivery.py +470 -0
- loopx/project_uninstall.py +462 -0
- loopx/promotion_gate.py +197 -0
- loopx/quota.py +1197 -0
- loopx/ready_score.py +413 -0
- loopx/registry.py +621 -0
- loopx/registry_writability.py +64 -0
- loopx/release_candidate.py +148 -0
- loopx/release_manifest.py +316 -0
- loopx/repository_identity.py +100 -0
- loopx/review_packet.py +1024 -0
- loopx/rollout_event_log.py +505 -0
- loopx/runtime.py +112 -0
- loopx/self_update.py +750 -0
- loopx/session_runtime.py +418 -0
- loopx/skill_install_readback.py +500 -0
- loopx/slash_command_install.py +1393 -0
- loopx/slash_commands.py +264 -0
- loopx/state_backup.py +573 -0
- loopx/state_migration.py +350 -0
- loopx/state_projection.py +809 -0
- loopx/state_refresh.py +1416 -0
- loopx/status.py +1383 -0
- loopx/status_server.py +935 -0
- loopx/summary_all.py +725 -0
- loopx/terminal_bench_agent.py +2056 -0
- loopx/thread_agent_binding.py +408 -0
- loopx/todo_followups.py +168 -0
- loopx/todo_suggestion_prompt.py +204 -0
- loopx/todos.py +2229 -0
- loopx/turn_identity.py +17 -0
- loopx/upgrade.py +1083 -0
- loopx/visible_governance.py +667 -0
- loopx/visible_multi_agent_launcher.py +1253 -0
- loopx/visible_multi_agent_tmux.py +429 -0
- loopx/worker_bridge.py +1574 -0
- loopx-0.4.8.dist-info/METADATA +708 -0
- loopx-0.4.8.dist-info/RECORD +811 -0
- loopx-0.4.8.dist-info/WHEEL +5 -0
- loopx-0.4.8.dist-info/entry_points.txt +5 -0
- loopx-0.4.8.dist-info/licenses/LICENSE +202 -0
- loopx-0.4.8.dist-info/licenses/LICENSE-MIT +21 -0
- loopx-0.4.8.dist-info/licenses/NOTICE +6 -0
- loopx-0.4.8.dist-info/top_level.txt +1 -0
|
@@ -0,0 +1,4832 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
import asyncio
|
|
4
|
+
import contextlib
|
|
5
|
+
import hashlib
|
|
6
|
+
import json
|
|
7
|
+
import os
|
|
8
|
+
import re
|
|
9
|
+
import selectors
|
|
10
|
+
import shlex
|
|
11
|
+
import shutil
|
|
12
|
+
import signal
|
|
13
|
+
import subprocess
|
|
14
|
+
import sys
|
|
15
|
+
import tempfile
|
|
16
|
+
import time
|
|
17
|
+
import uuid
|
|
18
|
+
from dataclasses import dataclass
|
|
19
|
+
from pathlib import Path
|
|
20
|
+
from typing import Any, TextIO
|
|
21
|
+
|
|
22
|
+
from loopx.benchmark_case_state import (
|
|
23
|
+
BENCHMARK_CASE_LOOPX_AGENT_ID,
|
|
24
|
+
BENCHMARK_CASE_LOOPX_CLI_PATH,
|
|
25
|
+
BENCHMARK_CASE_LOOPX_REGISTRY_PATH,
|
|
26
|
+
BENCHMARK_CASE_LOOPX_RUNTIME_ROOT,
|
|
27
|
+
BENCHMARK_CASE_LOOPX_TODO_ID,
|
|
28
|
+
benchmark_case_goal_id,
|
|
29
|
+
benchmark_case_loopx_command_prefix,
|
|
30
|
+
)
|
|
31
|
+
from loopx.benchmark_adapters.skillsbench_remote_bridge import (
|
|
32
|
+
run_skillsbench_remote_command_file_bridge_probe,
|
|
33
|
+
)
|
|
34
|
+
from loopx.benchmark_adapters.skillsbench_turn_runtime import (
|
|
35
|
+
SKILLSBENCH_TURN_AGENT_VALIDATION_HANDOFF_RESPONSE,
|
|
36
|
+
SkillsBenchTurnAgentResult,
|
|
37
|
+
run_skillsbench_loopx_turn_relay,
|
|
38
|
+
)
|
|
39
|
+
from loopx.benchmark_adapters.skillsbench_bridge_summary import (
|
|
40
|
+
bridge_summary_has_inflight_operation as _bridge_summary_has_inflight_operation,
|
|
41
|
+
bridge_summary_has_meaningful_agent_progress as _bridge_summary_has_meaningful_agent_progress,
|
|
42
|
+
bridge_summary_has_successful_task_operation as _bridge_summary_has_successful_task_operation,
|
|
43
|
+
bridge_summary_task_progress_receipt as _bridge_summary_task_progress_receipt,
|
|
44
|
+
bridge_operation_record_interrupted as _bridge_operation_record_interrupted,
|
|
45
|
+
prompt_requires_meaningful_bridge_progress as _prompt_requires_meaningful_bridge_progress,
|
|
46
|
+
)
|
|
47
|
+
from loopx.benchmark_adapters.skillsbench_bridge_guard import (
|
|
48
|
+
LOOPX_COMMAND_INSTRUMENTATION_SOURCE,
|
|
49
|
+
LOOPX_TODO_OUTPUT_INSTRUMENTATION_SOURCE,
|
|
50
|
+
)
|
|
51
|
+
from loopx.benchmark_adapters.skillsbench_acp_failure_policy import (
|
|
52
|
+
RECOVERABLE_CODEX_TURN_FAILURE_CATEGORIES,
|
|
53
|
+
RECOVERABLE_CODEX_TURN_FAILURE_PREFIX,
|
|
54
|
+
recoverable_codex_turn_failure_message as _recoverable_codex_turn_failure_message,
|
|
55
|
+
)
|
|
56
|
+
from loopx.benchmark_adapters.skillsbench_acp_process import (
|
|
57
|
+
write_process_stdin_async as _write_process_stdin_async,
|
|
58
|
+
)
|
|
59
|
+
from loopx.benchmark_adapters.skillsbench_codex_goal_recovery import (
|
|
60
|
+
CODEX_CLI_GOAL_POST_BRIDGE_CONTINUE_PROMPT,
|
|
61
|
+
POST_BRIDGE_RECOVERY_ATTEMPT_LIMIT,
|
|
62
|
+
PRE_BRIDGE_RECOVERY_ATTEMPT_LIMIT,
|
|
63
|
+
codex_cli_tui_post_bridge_blocker_stage,
|
|
64
|
+
codex_cli_tui_post_bridge_recovery_action,
|
|
65
|
+
codex_cli_tui_post_bridge_recovery_skip_reason,
|
|
66
|
+
codex_cli_tui_pre_bridge_blocker_stage,
|
|
67
|
+
codex_cli_tui_pre_bridge_recovery_action,
|
|
68
|
+
codex_cli_tui_pre_bridge_recovery_skip_reason,
|
|
69
|
+
codex_cli_tui_pre_bridge_terminal_stage,
|
|
70
|
+
codex_cli_tui_pre_bridge_terminal_skip_reason,
|
|
71
|
+
write_private_codex_cli_goal_tui_tail,
|
|
72
|
+
)
|
|
73
|
+
from loopx.codex_cli_goal_tui import (
|
|
74
|
+
CODEX_CLI_GOAL_KICKOFF_PROMPT,
|
|
75
|
+
CODEX_CLI_GOAL_TASK_PROMPT_FILENAME,
|
|
76
|
+
build_codex_cli_goal_file_objective,
|
|
77
|
+
build_codex_cli_goal_tui_input,
|
|
78
|
+
build_codex_cli_tui_command,
|
|
79
|
+
CodexCliGoalLifecycleGeneration,
|
|
80
|
+
codex_cli_goal_watchdog_expired,
|
|
81
|
+
codex_cli_goal_reset_pre_bridge_deadlines,
|
|
82
|
+
codex_cli_goal_should_ignore_stale_terminal,
|
|
83
|
+
codex_cli_goal_should_submit_kickoff,
|
|
84
|
+
codex_cli_tui_environment,
|
|
85
|
+
codex_cli_tui_shell_command,
|
|
86
|
+
codex_cli_tui_input_prompt_visible,
|
|
87
|
+
codex_cli_tui_retryable_startup_blocker_stage,
|
|
88
|
+
codex_cli_tui_turn_active,
|
|
89
|
+
resolve_codex_cli_binary,
|
|
90
|
+
start_codex_cli_goal_tui_session,
|
|
91
|
+
tmux_capture,
|
|
92
|
+
tmux_kill_session,
|
|
93
|
+
tmux_paste_file_and_submit,
|
|
94
|
+
tmux_submit_enter,
|
|
95
|
+
tmux_type_text_and_submit,
|
|
96
|
+
)
|
|
97
|
+
from loopx.control_plane.turn_driver import codex_cli_session_id_from_jsonl
|
|
98
|
+
|
|
99
|
+
SAFE_LOOPX_TODO_ID_RE = re.compile(r"^todo_[A-Za-z0-9_-]{6,80}$")
|
|
100
|
+
SAFE_LOOPX_GOAL_ID_RE = re.compile(r"^[A-Za-z0-9][A-Za-z0-9_.:-]{0,120}$")
|
|
101
|
+
CODEX_EXEC_TRANSPORT_RETRY_LIMIT = 1
|
|
102
|
+
CODEX_EXEC_SESSION_ROLLOVER_LIMIT = 1
|
|
103
|
+
CODEX_EXEC_SAME_SESSION_CONTINUATION_LIMIT = 1
|
|
104
|
+
CODEX_EXEC_PROGRESS_HANDOFF_SETTLE_TIMEOUT_SEC = 1.0
|
|
105
|
+
CODEX_EXEC_PROGRESS_VALIDATION_HANDOFF_CATEGORIES = frozenset(
|
|
106
|
+
{
|
|
107
|
+
"codex_exec_timeout",
|
|
108
|
+
"codex_exec_bridge_idle_timeout",
|
|
109
|
+
}
|
|
110
|
+
)
|
|
111
|
+
|
|
112
|
+
|
|
113
|
+
def _loopx_turn_local_session_root(
|
|
114
|
+
worker_public_trace_dir: str | None,
|
|
115
|
+
session_id: str,
|
|
116
|
+
) -> Path | None:
|
|
117
|
+
if not worker_public_trace_dir:
|
|
118
|
+
return None
|
|
119
|
+
session_digest = hashlib.sha256(session_id.encode("utf-8")).hexdigest()[:24]
|
|
120
|
+
return (
|
|
121
|
+
Path(worker_public_trace_dir).parent
|
|
122
|
+
/ ".loopx-turn-codex-sessions"
|
|
123
|
+
/ session_digest
|
|
124
|
+
)
|
|
125
|
+
|
|
126
|
+
|
|
127
|
+
def _safe_loopx_todo_id(value: object) -> str:
|
|
128
|
+
text = str(value or "")
|
|
129
|
+
return text if SAFE_LOOPX_TODO_ID_RE.match(text) else ""
|
|
130
|
+
|
|
131
|
+
|
|
132
|
+
def _safe_loopx_goal_id(value: object) -> str:
|
|
133
|
+
text = str(value or "")
|
|
134
|
+
return text if SAFE_LOOPX_GOAL_ID_RE.match(text) else ""
|
|
135
|
+
|
|
136
|
+
|
|
137
|
+
REPO_ROOT = Path(__file__).resolve().parents[2]
|
|
138
|
+
REVERSE_CHANNEL_BRIDGE_SCRIPT = (
|
|
139
|
+
REPO_ROOT / "scripts" / "skillsbench_reverse_channel_bridge.py"
|
|
140
|
+
)
|
|
141
|
+
SKILLSBENCH_LOCAL_ACP_RELAY_SCHEMA_VERSION = "skillsbench_local_acp_relay_v0"
|
|
142
|
+
SKILLSBENCH_LOCAL_ACP_RELAY_PROBE_SCHEMA_VERSION = (
|
|
143
|
+
"skillsbench_local_acp_relay_probe_v0"
|
|
144
|
+
)
|
|
145
|
+
SKILLSBENCH_HOST_LOCAL_ACP_TRANSPORT_PROBE_SCHEMA_VERSION = (
|
|
146
|
+
"skillsbench_host_local_acp_transport_probe_v0"
|
|
147
|
+
)
|
|
148
|
+
SKILLSBENCH_LOCAL_ACP_RELAY_READY_MARKER = "LOOPX_SKILLSBENCH_LOCAL_ACP_RELAY_READY"
|
|
149
|
+
SKILLSBENCH_LOCAL_ACP_RELAY_HEALTH_PROMPT = (
|
|
150
|
+
"LoopX relay health check. Reply exactly "
|
|
151
|
+
f"{SKILLSBENCH_LOCAL_ACP_RELAY_READY_MARKER} and end the turn."
|
|
152
|
+
)
|
|
153
|
+
SKILLSBENCH_LOCAL_ACP_RELAY_BRIDGE_PREFLIGHT_MARKER = "LOOPX_SKILLSBENCH_LOCAL_ACP_RELAY_BRIDGE_READY"
|
|
154
|
+
SKILLSBENCH_LOCAL_ACP_RELAY_BRIDGE_PREFLIGHT_PROMPT = (
|
|
155
|
+
"LoopX bridge action preflight. First use the private bridge command from "
|
|
156
|
+
"the relay packet to run one JSON preflight request that does not require "
|
|
157
|
+
"scored sandbox target environment variables. Do not plan, explain, inspect "
|
|
158
|
+
"files, or reply before that bridge request returns. Your first tool action "
|
|
159
|
+
"should be a shell "
|
|
160
|
+
"pipeline that sends the JSON request to the private bridge command shown "
|
|
161
|
+
"in the packet, for example: "
|
|
162
|
+
"`printf '%s\\n' '{\"operation\":\"preflight\"}' | <private bridge command>`. "
|
|
163
|
+
"After the bridge response returns, reply exactly "
|
|
164
|
+
f"{SKILLSBENCH_LOCAL_ACP_RELAY_BRIDGE_PREFLIGHT_MARKER} and end the turn."
|
|
165
|
+
)
|
|
166
|
+
CODEX_CLI_GOAL_THREAD_PREWARM_TIMEOUT_SEC = 120
|
|
167
|
+
SKILLSBENCH_OUTPUT_PATH_CONTRACT = (
|
|
168
|
+
"- Honor every task input and output path exactly as explicit task "
|
|
169
|
+
"instructions or visible task metadata declare it. Keep absolute paths "
|
|
170
|
+
"unchanged. Resolve a relative path from the directory containing the "
|
|
171
|
+
"metadata file that declares it, or otherwise from the sandbox task "
|
|
172
|
+
"working directory. Do not force relative paths into `/root` or `/app` "
|
|
173
|
+
"merely because those roots are available."
|
|
174
|
+
)
|
|
175
|
+
|
|
176
|
+
|
|
177
|
+
@contextlib.contextmanager
|
|
178
|
+
def _temporary_directory_ignore_cleanup_errors(*, prefix: str):
|
|
179
|
+
path = tempfile.mkdtemp(prefix=prefix)
|
|
180
|
+
try:
|
|
181
|
+
yield path
|
|
182
|
+
finally:
|
|
183
|
+
shutil.rmtree(path, ignore_errors=True)
|
|
184
|
+
|
|
185
|
+
def _prompt_requires_bridge_first_action(prompt: str) -> bool:
|
|
186
|
+
text = prompt or ""
|
|
187
|
+
if "Private bridge command:" not in text:
|
|
188
|
+
return False
|
|
189
|
+
lowered = text.lower()
|
|
190
|
+
required_markers = (
|
|
191
|
+
SKILLSBENCH_LOCAL_ACP_RELAY_BRIDGE_PREFLIGHT_MARKER.lower(),
|
|
192
|
+
"mandatory product-mode solver checkpoint",
|
|
193
|
+
"mandatory product-mode closeout checkpoint",
|
|
194
|
+
"must start with either a task-facing sandbox bridge operation",
|
|
195
|
+
"first action required",
|
|
196
|
+
"your first tool action should be a shell",
|
|
197
|
+
"your first agent action must be a shell/tool call",
|
|
198
|
+
"your first agent action must be a task-facing shell/tool call",
|
|
199
|
+
"first run the case-local quota/todo commands",
|
|
200
|
+
"first loopx cli action must invoke `start-goal",
|
|
201
|
+
"this benchmark treatment executes the actual agent contract for `/loopx",
|
|
202
|
+
"compact ranked",
|
|
203
|
+
"selected runnable p0",
|
|
204
|
+
)
|
|
205
|
+
return any(marker in lowered for marker in required_markers)
|
|
206
|
+
|
|
207
|
+
|
|
208
|
+
def _is_bridge_action_preflight_prompt(prompt: str) -> bool:
|
|
209
|
+
text = prompt or ""
|
|
210
|
+
return (
|
|
211
|
+
SKILLSBENCH_LOCAL_ACP_RELAY_BRIDGE_PREFLIGHT_MARKER in text
|
|
212
|
+
and "LoopX bridge action preflight" in text
|
|
213
|
+
)
|
|
214
|
+
|
|
215
|
+
|
|
216
|
+
def _json_rpc_result(message_id: Any, result: dict[str, Any]) -> dict[str, Any]:
|
|
217
|
+
return {"jsonrpc": "2.0", "id": message_id, "result": result}
|
|
218
|
+
|
|
219
|
+
|
|
220
|
+
def _json_rpc_error(message_id: Any, code: int, message: str) -> dict[str, Any]:
|
|
221
|
+
return {
|
|
222
|
+
"jsonrpc": "2.0",
|
|
223
|
+
"id": message_id,
|
|
224
|
+
"error": {"code": code, "message": message},
|
|
225
|
+
}
|
|
226
|
+
|
|
227
|
+
|
|
228
|
+
def _text_blocks_to_prompt(prompt: Any) -> str:
|
|
229
|
+
if not isinstance(prompt, list):
|
|
230
|
+
return ""
|
|
231
|
+
parts: list[str] = []
|
|
232
|
+
for block in prompt:
|
|
233
|
+
if isinstance(block, dict) and block.get("type") == "text":
|
|
234
|
+
text = str(block.get("text") or "").strip()
|
|
235
|
+
if text:
|
|
236
|
+
parts.append(text)
|
|
237
|
+
return "\n\n".join(parts)
|
|
238
|
+
|
|
239
|
+
|
|
240
|
+
def _safe_cwd(value: Any, *, default: str) -> str:
|
|
241
|
+
text = str(value or "").strip()
|
|
242
|
+
return text or default
|
|
243
|
+
|
|
244
|
+
|
|
245
|
+
def _public_bridge_label(value: Any, *, limit: int = 120) -> str:
|
|
246
|
+
text = str(value or "").strip()
|
|
247
|
+
if not text:
|
|
248
|
+
return ""
|
|
249
|
+
text = text.replace("/", "_").replace("\\", "_")
|
|
250
|
+
text = re.sub(r"[^A-Za-z0-9_.:-]+", "_", text).strip("._:-")
|
|
251
|
+
return text[:limit]
|
|
252
|
+
|
|
253
|
+
|
|
254
|
+
def _public_bridge_label_list(value: Any, *, limit: int = 80) -> list[str]:
|
|
255
|
+
if not isinstance(value, list):
|
|
256
|
+
return []
|
|
257
|
+
labels: list[str] = []
|
|
258
|
+
for item in value[:12]:
|
|
259
|
+
label = _public_bridge_label(item, limit=limit)
|
|
260
|
+
if label:
|
|
261
|
+
labels.append(label)
|
|
262
|
+
return labels
|
|
263
|
+
|
|
264
|
+
|
|
265
|
+
def _public_bridge_operations(value: Any) -> list[dict[str, Any]]:
|
|
266
|
+
if not isinstance(value, list):
|
|
267
|
+
return []
|
|
268
|
+
operations: list[dict[str, Any]] = []
|
|
269
|
+
for item in value[:8]:
|
|
270
|
+
if not isinstance(item, dict):
|
|
271
|
+
continue
|
|
272
|
+
operation: dict[str, Any] = {}
|
|
273
|
+
for field in ("kind", "label", "status"):
|
|
274
|
+
label = _public_bridge_label(item.get(field), limit=80)
|
|
275
|
+
if label:
|
|
276
|
+
operation[field] = label
|
|
277
|
+
for field in ("exit_code_zero", "content_match"):
|
|
278
|
+
flag = item.get(field)
|
|
279
|
+
if isinstance(flag, bool):
|
|
280
|
+
operation[field] = flag
|
|
281
|
+
if operation:
|
|
282
|
+
operations.append(operation)
|
|
283
|
+
return operations
|
|
284
|
+
|
|
285
|
+
|
|
286
|
+
def _codex_exec_failure_category(
|
|
287
|
+
*,
|
|
288
|
+
returncode: int | None,
|
|
289
|
+
stderr_text: str,
|
|
290
|
+
stdout_text: str = "",
|
|
291
|
+
) -> str:
|
|
292
|
+
text = "\n".join(
|
|
293
|
+
value
|
|
294
|
+
for value in (
|
|
295
|
+
stderr_text or "",
|
|
296
|
+
_codex_exec_jsonl_error_text(stdout_text),
|
|
297
|
+
)
|
|
298
|
+
if value
|
|
299
|
+
).lower()
|
|
300
|
+
if any(
|
|
301
|
+
token in text
|
|
302
|
+
for token in (
|
|
303
|
+
"not authenticated",
|
|
304
|
+
"authentication",
|
|
305
|
+
"unauthorized",
|
|
306
|
+
"api key",
|
|
307
|
+
"login required",
|
|
308
|
+
"please login",
|
|
309
|
+
"401",
|
|
310
|
+
)
|
|
311
|
+
):
|
|
312
|
+
return "codex_auth_or_login_required"
|
|
313
|
+
if "model" in text and any(
|
|
314
|
+
token in text
|
|
315
|
+
for token in (
|
|
316
|
+
"not found",
|
|
317
|
+
"unknown",
|
|
318
|
+
"unsupported",
|
|
319
|
+
"invalid",
|
|
320
|
+
"does not exist",
|
|
321
|
+
"unavailable",
|
|
322
|
+
"not supported",
|
|
323
|
+
)
|
|
324
|
+
):
|
|
325
|
+
return "codex_model_unavailable"
|
|
326
|
+
if "codex/responses" in text and any(
|
|
327
|
+
token in text
|
|
328
|
+
for token in (
|
|
329
|
+
"failed to connect to websocket",
|
|
330
|
+
"stream disconnected before completion",
|
|
331
|
+
"error sending request",
|
|
332
|
+
"connection refused",
|
|
333
|
+
)
|
|
334
|
+
):
|
|
335
|
+
return "codex_responses_stream_unavailable"
|
|
336
|
+
if any(
|
|
337
|
+
token in text
|
|
338
|
+
for token in (
|
|
339
|
+
"failed to refresh available models",
|
|
340
|
+
"stream disconnected before completion",
|
|
341
|
+
"error sending request",
|
|
342
|
+
"request error",
|
|
343
|
+
"connection refused",
|
|
344
|
+
"connection timed out",
|
|
345
|
+
)
|
|
346
|
+
) and any(
|
|
347
|
+
token in text
|
|
348
|
+
for token in (
|
|
349
|
+
"chatgpt.com",
|
|
350
|
+
"api.openai.com",
|
|
351
|
+
"backend-api",
|
|
352
|
+
"available models",
|
|
353
|
+
)
|
|
354
|
+
):
|
|
355
|
+
return "codex_network_or_api_unreachable"
|
|
356
|
+
if (
|
|
357
|
+
"connectionrefusederror" in text
|
|
358
|
+
or "connection refused" in text
|
|
359
|
+
or "failed to establish a new connection" in text
|
|
360
|
+
or ("create_connection" in text and "socket" in text)
|
|
361
|
+
):
|
|
362
|
+
return "codex_reverse_channel_unavailable"
|
|
363
|
+
if "hit your usage limit" in text or "usage limit" in text:
|
|
364
|
+
return "codex_usage_limit"
|
|
365
|
+
if "codex_exec_first_action_timeout" in text:
|
|
366
|
+
return "codex_exec_first_action_timeout"
|
|
367
|
+
if "codex_exec_task_output_quiet_timeout" in text:
|
|
368
|
+
return "codex_exec_task_output_quiet_timeout"
|
|
369
|
+
if "codex_exec_bridge_idle_timeout" in text:
|
|
370
|
+
return "codex_exec_bridge_idle_timeout"
|
|
371
|
+
if "unexpected argument" in text or "unrecognized option" in text:
|
|
372
|
+
return "codex_cli_argument_incompatible"
|
|
373
|
+
if any(
|
|
374
|
+
token in text
|
|
375
|
+
for token in (
|
|
376
|
+
"command not found",
|
|
377
|
+
"no such file or directory",
|
|
378
|
+
"modulenotfounderror",
|
|
379
|
+
"importerror",
|
|
380
|
+
"failed to spawn",
|
|
381
|
+
)
|
|
382
|
+
):
|
|
383
|
+
return "codex_cli_or_environment_missing"
|
|
384
|
+
if "api.openai.com" in text or "chatgpt.com" in text:
|
|
385
|
+
if any(token in text for token in ("timed out", "timeout", "connection")):
|
|
386
|
+
return "codex_network_or_api_unreachable"
|
|
387
|
+
if returncode == 124:
|
|
388
|
+
return "codex_exec_timeout"
|
|
389
|
+
if returncode is not None and 0 < returncode <= 255:
|
|
390
|
+
return f"codex_exec_exit_{returncode}"
|
|
391
|
+
if returncode is not None:
|
|
392
|
+
return "codex_exec_exit_nonzero"
|
|
393
|
+
return "codex_exec_failed"
|
|
394
|
+
|
|
395
|
+
|
|
396
|
+
def _codex_exec_jsonl_error_text(stdout_text: str) -> str:
|
|
397
|
+
"""Extract only typed Codex error messages from private JSONL output."""
|
|
398
|
+
|
|
399
|
+
messages: list[str] = []
|
|
400
|
+
for line in (stdout_text or "").splitlines():
|
|
401
|
+
try:
|
|
402
|
+
event = json.loads(line)
|
|
403
|
+
except (json.JSONDecodeError, TypeError):
|
|
404
|
+
continue
|
|
405
|
+
if not isinstance(event, dict):
|
|
406
|
+
continue
|
|
407
|
+
event_type = str(event.get("type") or "").strip().lower()
|
|
408
|
+
item = event.get("item") if isinstance(event.get("item"), dict) else {}
|
|
409
|
+
item_type = str(item.get("type") or "").strip().lower()
|
|
410
|
+
if event_type not in {"error", "turn.failed"} and item_type != "error":
|
|
411
|
+
continue
|
|
412
|
+
error = event.get("error")
|
|
413
|
+
candidates = [event.get("message")]
|
|
414
|
+
if isinstance(error, dict):
|
|
415
|
+
candidates.append(error.get("message"))
|
|
416
|
+
elif isinstance(error, str):
|
|
417
|
+
candidates.append(error)
|
|
418
|
+
if item_type == "error":
|
|
419
|
+
candidates.extend((item.get("message"), item.get("text")))
|
|
420
|
+
for candidate in candidates:
|
|
421
|
+
if not isinstance(candidate, str):
|
|
422
|
+
continue
|
|
423
|
+
message = candidate.strip()
|
|
424
|
+
if message:
|
|
425
|
+
messages.append(message[:4_000])
|
|
426
|
+
if sum(len(message) for message in messages) >= 16_000:
|
|
427
|
+
break
|
|
428
|
+
return "\n".join(messages)[:16_000]
|
|
429
|
+
|
|
430
|
+
|
|
431
|
+
def _codex_exec_side_effect_free_recovery_allowed(
|
|
432
|
+
*,
|
|
433
|
+
category: str,
|
|
434
|
+
bridge_summary_path: Path | None,
|
|
435
|
+
final_message_present: bool,
|
|
436
|
+
turn_deadline: float | None,
|
|
437
|
+
) -> bool:
|
|
438
|
+
if (
|
|
439
|
+
category not in RECOVERABLE_CODEX_TURN_FAILURE_CATEGORIES
|
|
440
|
+
or final_message_present
|
|
441
|
+
or bridge_summary_path is None
|
|
442
|
+
or turn_deadline is not None
|
|
443
|
+
and time.monotonic() >= turn_deadline
|
|
444
|
+
or _bridge_summary_has_inflight_operation(bridge_summary_path)
|
|
445
|
+
or _bridge_summary_has_meaningful_agent_progress(
|
|
446
|
+
bridge_summary_path,
|
|
447
|
+
allow_loopx_closeout=True,
|
|
448
|
+
)
|
|
449
|
+
):
|
|
450
|
+
return False
|
|
451
|
+
receipt = _bridge_summary_task_progress_receipt(bridge_summary_path)
|
|
452
|
+
return bool(
|
|
453
|
+
receipt.get("task_facing_operation_count") == 0
|
|
454
|
+
and receipt.get("task_facing_success_count") == 0
|
|
455
|
+
and receipt.get("raw_material_recorded") is False
|
|
456
|
+
)
|
|
457
|
+
|
|
458
|
+
|
|
459
|
+
def _codex_exec_transport_retry_allowed(
|
|
460
|
+
*,
|
|
461
|
+
category: str,
|
|
462
|
+
bridge_summary_path: Path | None,
|
|
463
|
+
retry_count: int,
|
|
464
|
+
final_message_present: bool,
|
|
465
|
+
turn_deadline: float | None,
|
|
466
|
+
) -> bool:
|
|
467
|
+
return bool(
|
|
468
|
+
retry_count < CODEX_EXEC_TRANSPORT_RETRY_LIMIT
|
|
469
|
+
and _codex_exec_side_effect_free_recovery_allowed(
|
|
470
|
+
category=category,
|
|
471
|
+
bridge_summary_path=bridge_summary_path,
|
|
472
|
+
final_message_present=final_message_present,
|
|
473
|
+
turn_deadline=turn_deadline,
|
|
474
|
+
)
|
|
475
|
+
)
|
|
476
|
+
|
|
477
|
+
|
|
478
|
+
def _codex_exec_same_session_continuation_allowed(
|
|
479
|
+
*,
|
|
480
|
+
category: str,
|
|
481
|
+
bridge_summary_path: Path | None,
|
|
482
|
+
continuation_count: int,
|
|
483
|
+
thread_present: bool,
|
|
484
|
+
final_message_present: bool,
|
|
485
|
+
turn_deadline: float | None,
|
|
486
|
+
) -> bool:
|
|
487
|
+
if (
|
|
488
|
+
continuation_count >= CODEX_EXEC_SAME_SESSION_CONTINUATION_LIMIT
|
|
489
|
+
or category not in RECOVERABLE_CODEX_TURN_FAILURE_CATEGORIES
|
|
490
|
+
or not thread_present
|
|
491
|
+
or final_message_present
|
|
492
|
+
or bridge_summary_path is None
|
|
493
|
+
or turn_deadline is not None
|
|
494
|
+
and time.monotonic() >= turn_deadline
|
|
495
|
+
or _bridge_summary_has_inflight_operation(bridge_summary_path)
|
|
496
|
+
):
|
|
497
|
+
return False
|
|
498
|
+
receipt = _bridge_summary_task_progress_receipt(bridge_summary_path)
|
|
499
|
+
return bool(
|
|
500
|
+
receipt.get("task_facing_operation_count", 0) > 0
|
|
501
|
+
and receipt.get("task_facing_success_count", 0) > 0
|
|
502
|
+
and receipt.get("raw_material_recorded") is False
|
|
503
|
+
)
|
|
504
|
+
|
|
505
|
+
|
|
506
|
+
def _codex_exec_progress_validation_handoff_allowed(
|
|
507
|
+
*,
|
|
508
|
+
category: str,
|
|
509
|
+
bridge_summary_path: Path | None,
|
|
510
|
+
same_session_continuation_scheduled: bool,
|
|
511
|
+
final_message_present: bool,
|
|
512
|
+
turn_deadline: float | None,
|
|
513
|
+
) -> bool:
|
|
514
|
+
turn_deadline_expired = bool(
|
|
515
|
+
turn_deadline is not None and time.monotonic() >= turn_deadline
|
|
516
|
+
)
|
|
517
|
+
if (
|
|
518
|
+
category not in CODEX_EXEC_PROGRESS_VALIDATION_HANDOFF_CATEGORIES
|
|
519
|
+
or same_session_continuation_scheduled
|
|
520
|
+
or final_message_present
|
|
521
|
+
or bridge_summary_path is None
|
|
522
|
+
or (
|
|
523
|
+
turn_deadline_expired
|
|
524
|
+
and category != "codex_exec_timeout"
|
|
525
|
+
)
|
|
526
|
+
or _bridge_summary_has_inflight_operation(bridge_summary_path)
|
|
527
|
+
):
|
|
528
|
+
return False
|
|
529
|
+
receipt = _bridge_summary_task_progress_receipt(bridge_summary_path)
|
|
530
|
+
return bool(
|
|
531
|
+
receipt.get("task_facing_operation_count", 0) > 0
|
|
532
|
+
and receipt.get("task_facing_success_count", 0) > 0
|
|
533
|
+
and receipt.get("raw_material_recorded") is False
|
|
534
|
+
)
|
|
535
|
+
|
|
536
|
+
|
|
537
|
+
def _wait_for_bridge_summary_quiescence(
|
|
538
|
+
bridge_summary_path: Path | None,
|
|
539
|
+
*,
|
|
540
|
+
timeout_sec: float = CODEX_EXEC_PROGRESS_HANDOFF_SETTLE_TIMEOUT_SEC,
|
|
541
|
+
poll_interval_sec: float = 0.05,
|
|
542
|
+
) -> bool:
|
|
543
|
+
"""Let a terminating bridge wrapper finish its final compact record."""
|
|
544
|
+
|
|
545
|
+
if bridge_summary_path is None:
|
|
546
|
+
return False
|
|
547
|
+
deadline = time.monotonic() + max(0.0, timeout_sec)
|
|
548
|
+
while _bridge_summary_has_inflight_operation(bridge_summary_path):
|
|
549
|
+
remaining = deadline - time.monotonic()
|
|
550
|
+
if remaining <= 0:
|
|
551
|
+
return False
|
|
552
|
+
time.sleep(min(max(0.001, poll_interval_sec), remaining))
|
|
553
|
+
return True
|
|
554
|
+
|
|
555
|
+
|
|
556
|
+
def _codex_exec_session_rollover_allowed(
|
|
557
|
+
*,
|
|
558
|
+
category: str,
|
|
559
|
+
bridge_summary_path: Path | None,
|
|
560
|
+
retry_count: int,
|
|
561
|
+
rollover_count: int,
|
|
562
|
+
final_message_present: bool,
|
|
563
|
+
turn_deadline: float | None,
|
|
564
|
+
) -> bool:
|
|
565
|
+
return bool(
|
|
566
|
+
retry_count == CODEX_EXEC_TRANSPORT_RETRY_LIMIT
|
|
567
|
+
and rollover_count < CODEX_EXEC_SESSION_ROLLOVER_LIMIT
|
|
568
|
+
and _codex_exec_side_effect_free_recovery_allowed(
|
|
569
|
+
category=category,
|
|
570
|
+
bridge_summary_path=bridge_summary_path,
|
|
571
|
+
final_message_present=final_message_present,
|
|
572
|
+
turn_deadline=turn_deadline,
|
|
573
|
+
)
|
|
574
|
+
)
|
|
575
|
+
|
|
576
|
+
|
|
577
|
+
def _codex_exec_same_session_continuation_prompt() -> str:
|
|
578
|
+
return (
|
|
579
|
+
"Continue the same bounded task in this thread after the recoverable "
|
|
580
|
+
"host interruption.\n"
|
|
581
|
+
"- Preserve and inspect the progress already made in this thread and "
|
|
582
|
+
"the current workspace.\n"
|
|
583
|
+
"- Do not replay completed operations or restart the task from its "
|
|
584
|
+
"original prompt.\n"
|
|
585
|
+
"- Complete the next necessary task-facing work, validate it from "
|
|
586
|
+
"visible task context, then finish this Turn.\n"
|
|
587
|
+
"- Do not use or request official verifier, reward, pass/fail, hidden-test, "
|
|
588
|
+
"or gold-answer feedback."
|
|
589
|
+
)
|
|
590
|
+
|
|
591
|
+
|
|
592
|
+
def _codex_exec_session_rollover_prompt(
|
|
593
|
+
*, original_prompt: str, continuation_prompt: str
|
|
594
|
+
) -> str:
|
|
595
|
+
return (
|
|
596
|
+
original_prompt.rstrip()
|
|
597
|
+
+ "\n\n"
|
|
598
|
+
+ "LoopX Turn fresh-session recovery contract:\n"
|
|
599
|
+
+ "- Continue the same task after a native resume transport failure.\n"
|
|
600
|
+
+ "- Rebuild context from the current durable workspace and case-local "
|
|
601
|
+
+ "LoopX state; prior task-facing progress is already present there.\n"
|
|
602
|
+
+ "- Perform only the next bounded task-facing step, then stop.\n"
|
|
603
|
+
+ "- Do not use or request official verifier, reward, pass/fail, hidden-test, "
|
|
604
|
+
+ "or gold-answer feedback.\n\n"
|
|
605
|
+
+ continuation_prompt.strip()
|
|
606
|
+
)
|
|
607
|
+
|
|
608
|
+
|
|
609
|
+
def _prompt_with_app_server_closeout_instruction(prompt_text: str) -> str:
|
|
610
|
+
"""Ask native Goal workers to end promptly after scored output exists."""
|
|
611
|
+
|
|
612
|
+
return (
|
|
613
|
+
prompt_text.rstrip()
|
|
614
|
+
+ "\n\n"
|
|
615
|
+
+ "Native Codex Goal worker closeout contract:\n"
|
|
616
|
+
+ "- Solve the task using only the available benchmark workspace or the "
|
|
617
|
+
+ "private bridge packet above.\n"
|
|
618
|
+
+ SKILLSBENCH_OUTPUT_PATH_CONTRACT
|
|
619
|
+
+ "\n"
|
|
620
|
+
+ "- Before writing the final scored output for optimization, "
|
|
621
|
+
+ "scheduling, allocation, routing, planning, or data-processing "
|
|
622
|
+
+ "tasks, run a task-derived quality self-check using only visible "
|
|
623
|
+
+ "task instructions and workspace data: validate hard constraints, "
|
|
624
|
+
+ "compute or estimate the visible objective when the task defines "
|
|
625
|
+
+ "one, compare at least one simple alternative or repair pass when "
|
|
626
|
+
+ "feasible, and only then write the final task-specified output. Do not "
|
|
627
|
+
+ "use official verifier/reward/pass-fail output, hidden tests, "
|
|
628
|
+
+ "gold answers, or external benchmark feedback for this self-check.\n"
|
|
629
|
+
+ "- After the task-required scored output file is written, immediately "
|
|
630
|
+
+ "end the turn with one short confirmation.\n"
|
|
631
|
+
+ "- Do not keep optimizing, narrating, or rechecking after the scored "
|
|
632
|
+
+ "output exists unless the task explicitly requires more work.\n"
|
|
633
|
+
)
|
|
634
|
+
|
|
635
|
+
|
|
636
|
+
def _normalized_app_server_goal_prompt_style(style: str | None) -> str:
|
|
637
|
+
text = str(style or "").strip().lower()
|
|
638
|
+
if text in {"bridge-only", "native-goal", "cli-exec-like"}:
|
|
639
|
+
return text
|
|
640
|
+
return "bridge-only"
|
|
641
|
+
|
|
642
|
+
|
|
643
|
+
@dataclass(frozen=True)
|
|
644
|
+
class CodexExecConfig:
|
|
645
|
+
codex_bin: str = "codex"
|
|
646
|
+
sandbox: str = "workspace-write"
|
|
647
|
+
model: str | None = None
|
|
648
|
+
route: str = "unknown"
|
|
649
|
+
timeout_sec: int = 7200
|
|
650
|
+
dry_run_response: str | None = None
|
|
651
|
+
app_server_goal_worker: bool = False
|
|
652
|
+
codex_cli_goal_worker: bool = False
|
|
653
|
+
dataset: str = "skillsbench-v1.1"
|
|
654
|
+
task_id: str = "llm-prefix-cache-replay"
|
|
655
|
+
run_group_id: str = ""
|
|
656
|
+
job_name: str = ""
|
|
657
|
+
rollout_name: str = ""
|
|
658
|
+
approval_policy: str = "never"
|
|
659
|
+
response_timeout_sec: float = 30.0
|
|
660
|
+
worker_script: str | None = None
|
|
661
|
+
stream_heartbeat_interval_sec: float = 120.0
|
|
662
|
+
first_action_timeout_sec: float = 0.0
|
|
663
|
+
goal_active_timeout_sec: float = 180.0
|
|
664
|
+
app_server_goal_followup_max: int = 0
|
|
665
|
+
app_server_goal_prompt_style: str = "bridge-only"
|
|
666
|
+
bridge_idle_timeout_sec: float = 0.0
|
|
667
|
+
task_output_quiet_timeout_sec: float = 0.0
|
|
668
|
+
reasoning_effort: str | None = None
|
|
669
|
+
codex_api_proxy: str | None = None
|
|
670
|
+
codex_cli_goal_thread_prewarm: bool = False
|
|
671
|
+
worker_public_trace_dir: str | None = None
|
|
672
|
+
remote_command_file_bridge_command: str | None = None
|
|
673
|
+
remote_command_file_bridge_agent_command: str | None = None
|
|
674
|
+
remote_command_file_bridge_timeout_sec: float = 10.0
|
|
675
|
+
loopx_workflow_lifecycle_checkpoint: bool = False
|
|
676
|
+
loopx_turn_agent_cli: bool = False
|
|
677
|
+
loopx_turn_validation_command: str | None = None
|
|
678
|
+
loopx_turn_max_turns: int = 1
|
|
679
|
+
loopx_turn_progress_exit_code: int = 10
|
|
680
|
+
loopx_turn_terminal_policy: str = "validator"
|
|
681
|
+
loopx_case_goal_id: str = "skillsbench-case"
|
|
682
|
+
loopx_case_agent_id: str = BENCHMARK_CASE_LOOPX_AGENT_ID
|
|
683
|
+
loopx_case_todo_id: str = BENCHMARK_CASE_LOOPX_TODO_ID
|
|
684
|
+
loopx_case_cli_path: str = BENCHMARK_CASE_LOOPX_CLI_PATH
|
|
685
|
+
loopx_case_registry_path: str = BENCHMARK_CASE_LOOPX_REGISTRY_PATH
|
|
686
|
+
loopx_case_runtime_root: str = BENCHMARK_CASE_LOOPX_RUNTIME_ROOT
|
|
687
|
+
|
|
688
|
+
|
|
689
|
+
class SkillsBenchLocalAcpRelay:
|
|
690
|
+
"""Minimal ACP stdio server for the SkillsBench local-driver route.
|
|
691
|
+
|
|
692
|
+
The relay is intentionally a local ACP adapter, not a remote Codex runtime.
|
|
693
|
+
In dry-run mode it proves the BenchFlow JSON-RPC handshake without invoking
|
|
694
|
+
Codex. In normal mode it delegates each prompt to the local Codex CLI and
|
|
695
|
+
sends only the final assistant message back over ACP stdout.
|
|
696
|
+
"""
|
|
697
|
+
|
|
698
|
+
def __init__(self, config: CodexExecConfig):
|
|
699
|
+
self._config = config
|
|
700
|
+
self._sessions: dict[str, dict[str, Any]] = {}
|
|
701
|
+
self._published_lifecycle_stages: set[str] = set()
|
|
702
|
+
self._workflow_checkpoint_count = 0
|
|
703
|
+
self._bridge_summary_snapshot_ids: dict[str, str] = {}
|
|
704
|
+
self._bridge_summary_snapshot_indexes: dict[str, int] = {}
|
|
705
|
+
self._latest_loopx_turn_agent_progress_receipt: dict[str, Any] = {}
|
|
706
|
+
|
|
707
|
+
def serve(self, stdin: TextIO = sys.stdin, stdout: TextIO = sys.stdout) -> int:
|
|
708
|
+
for line in stdin:
|
|
709
|
+
line = line.strip()
|
|
710
|
+
if not line:
|
|
711
|
+
continue
|
|
712
|
+
try:
|
|
713
|
+
message = json.loads(line)
|
|
714
|
+
except json.JSONDecodeError:
|
|
715
|
+
continue
|
|
716
|
+
if not isinstance(message, dict) or message.get("jsonrpc") != "2.0":
|
|
717
|
+
continue
|
|
718
|
+
response = self.handle_message(message, stdout=stdout)
|
|
719
|
+
if response is not None:
|
|
720
|
+
self._write(stdout, response)
|
|
721
|
+
return 0
|
|
722
|
+
|
|
723
|
+
def handle_message(
|
|
724
|
+
self, message: dict[str, Any], *, stdout: TextIO
|
|
725
|
+
) -> dict[str, Any] | None:
|
|
726
|
+
method = str(message.get("method") or "")
|
|
727
|
+
message_id = message.get("id")
|
|
728
|
+
params = message.get("params") if isinstance(message.get("params"), dict) else {}
|
|
729
|
+
if "id" not in message:
|
|
730
|
+
if method == "session/cancel":
|
|
731
|
+
return None
|
|
732
|
+
return None
|
|
733
|
+
if method == "initialize":
|
|
734
|
+
self._publish_worker_lifecycle_trace("initialize")
|
|
735
|
+
return _json_rpc_result(
|
|
736
|
+
message_id,
|
|
737
|
+
{
|
|
738
|
+
"protocolVersion": int(params.get("protocolVersion") or 0),
|
|
739
|
+
"agentCapabilities": {
|
|
740
|
+
"promptCapabilities": {
|
|
741
|
+
"image": False,
|
|
742
|
+
"audio": False,
|
|
743
|
+
"embeddedContext": False,
|
|
744
|
+
},
|
|
745
|
+
"mcpCapabilities": {"sse": False, "http": False},
|
|
746
|
+
"loadSession": True,
|
|
747
|
+
},
|
|
748
|
+
"agentInfo": {
|
|
749
|
+
"name": "loopx-skillsbench-local-acp-relay",
|
|
750
|
+
"version": SKILLSBENCH_LOCAL_ACP_RELAY_SCHEMA_VERSION,
|
|
751
|
+
},
|
|
752
|
+
},
|
|
753
|
+
)
|
|
754
|
+
if method in {"session/new", "session/load"}:
|
|
755
|
+
session_id = str(params.get("sessionId") or f"gh-sb-{uuid.uuid4().hex[:12]}")
|
|
756
|
+
self._sessions[session_id] = {
|
|
757
|
+
"cwd": _safe_cwd(params.get("cwd"), default=os.getcwd()),
|
|
758
|
+
"model": None,
|
|
759
|
+
"cancelled": False,
|
|
760
|
+
}
|
|
761
|
+
self._publish_worker_lifecycle_trace("session_new")
|
|
762
|
+
return _json_rpc_result(message_id, {"sessionId": session_id})
|
|
763
|
+
if method == "session/set_model":
|
|
764
|
+
session = self._sessions.get(str(params.get("sessionId") or ""))
|
|
765
|
+
if session is None:
|
|
766
|
+
return _json_rpc_error(message_id, -32001, "unknown session")
|
|
767
|
+
session["model"] = str(params.get("modelId") or "")
|
|
768
|
+
return _json_rpc_result(message_id, {})
|
|
769
|
+
if method == "session/prompt":
|
|
770
|
+
return self._handle_prompt(message_id, params, stdout=stdout)
|
|
771
|
+
return _json_rpc_error(message_id, -32601, f"method not found: {method}")
|
|
772
|
+
|
|
773
|
+
def _handle_prompt(
|
|
774
|
+
self, message_id: Any, params: dict[str, Any], *, stdout: TextIO
|
|
775
|
+
) -> dict[str, Any]:
|
|
776
|
+
session_id = str(params.get("sessionId") or "")
|
|
777
|
+
session = self._sessions.get(session_id)
|
|
778
|
+
if session is None:
|
|
779
|
+
return _json_rpc_error(message_id, -32001, "unknown session")
|
|
780
|
+
prompt_text = _text_blocks_to_prompt(params.get("prompt"))
|
|
781
|
+
if not prompt_text:
|
|
782
|
+
return _json_rpc_error(message_id, -32602, "prompt text missing")
|
|
783
|
+
try:
|
|
784
|
+
response_text = self._run_codex(
|
|
785
|
+
prompt_text,
|
|
786
|
+
session=session,
|
|
787
|
+
session_id=session_id,
|
|
788
|
+
stdout=stdout,
|
|
789
|
+
)
|
|
790
|
+
except TimeoutError:
|
|
791
|
+
return _json_rpc_error(message_id, -32002, "local codex execution timeout")
|
|
792
|
+
except RuntimeError as exc:
|
|
793
|
+
return _json_rpc_error(message_id, -32003, str(exc))
|
|
794
|
+
self._write(
|
|
795
|
+
stdout,
|
|
796
|
+
{
|
|
797
|
+
"jsonrpc": "2.0",
|
|
798
|
+
"method": "session/update",
|
|
799
|
+
"params": {
|
|
800
|
+
"sessionId": session_id,
|
|
801
|
+
"update": {
|
|
802
|
+
"sessionUpdate": "agent_message_chunk",
|
|
803
|
+
"content": {"type": "text", "text": response_text},
|
|
804
|
+
},
|
|
805
|
+
},
|
|
806
|
+
},
|
|
807
|
+
)
|
|
808
|
+
output_tokens = max(1, len(response_text.encode("utf-8")) // 4)
|
|
809
|
+
input_tokens = max(1, len(prompt_text.encode("utf-8")) // 4)
|
|
810
|
+
return _json_rpc_result(
|
|
811
|
+
message_id,
|
|
812
|
+
{
|
|
813
|
+
"stopReason": "end_turn",
|
|
814
|
+
"usage": {
|
|
815
|
+
"inputTokens": input_tokens,
|
|
816
|
+
"outputTokens": output_tokens,
|
|
817
|
+
"totalTokens": input_tokens + output_tokens,
|
|
818
|
+
},
|
|
819
|
+
},
|
|
820
|
+
)
|
|
821
|
+
|
|
822
|
+
def _run_codex(
|
|
823
|
+
self,
|
|
824
|
+
prompt_text: str,
|
|
825
|
+
*,
|
|
826
|
+
session: dict[str, Any],
|
|
827
|
+
session_id: str,
|
|
828
|
+
stdout: TextIO,
|
|
829
|
+
_bypass_loopx_turn: bool = False,
|
|
830
|
+
_turn_deadline: float | None = None,
|
|
831
|
+
_transport_retry_count: int = 0,
|
|
832
|
+
_session_rollover_index: int = 0,
|
|
833
|
+
_same_session_continuation_count: int = 0,
|
|
834
|
+
) -> str:
|
|
835
|
+
if self._config.dry_run_response is not None:
|
|
836
|
+
return self._config.dry_run_response
|
|
837
|
+
if self._config.loopx_turn_agent_cli and not _bypass_loopx_turn:
|
|
838
|
+
sequence_deadline = time.monotonic() + max(
|
|
839
|
+
1.0, float(self._config.timeout_sec)
|
|
840
|
+
)
|
|
841
|
+
session["_loopx_turn_original_prompt"] = prompt_text
|
|
842
|
+
session["_loopx_turn_codex_rollover_count"] = 0
|
|
843
|
+
try:
|
|
844
|
+
return run_skillsbench_loopx_turn_relay(
|
|
845
|
+
prompt=prompt_text,
|
|
846
|
+
session_id=session_id,
|
|
847
|
+
relay_config=self._config,
|
|
848
|
+
agent_runner=lambda turn_prompt: self._run_loopx_turn_agent_prompt(
|
|
849
|
+
turn_prompt,
|
|
850
|
+
session=session,
|
|
851
|
+
session_id=session_id,
|
|
852
|
+
stdout=stdout,
|
|
853
|
+
turn_deadline=sequence_deadline,
|
|
854
|
+
),
|
|
855
|
+
trace_writer=self._write_worker_public_trace,
|
|
856
|
+
)
|
|
857
|
+
finally:
|
|
858
|
+
session.pop("_loopx_turn_codex_thread_id", None)
|
|
859
|
+
session.pop("_loopx_turn_codex_rollover_count", None)
|
|
860
|
+
session.pop("_loopx_turn_original_prompt", None)
|
|
861
|
+
session_root = _loopx_turn_local_session_root(
|
|
862
|
+
self._config.worker_public_trace_dir,
|
|
863
|
+
session_id,
|
|
864
|
+
)
|
|
865
|
+
if session_root is not None:
|
|
866
|
+
shutil.rmtree(session_root, ignore_errors=True)
|
|
867
|
+
with contextlib.suppress(OSError):
|
|
868
|
+
session_root.parent.rmdir()
|
|
869
|
+
if self._config.app_server_goal_worker:
|
|
870
|
+
return self._run_app_server_goal_worker(
|
|
871
|
+
prompt_text,
|
|
872
|
+
session=session,
|
|
873
|
+
session_id=session_id,
|
|
874
|
+
stdout=stdout,
|
|
875
|
+
)
|
|
876
|
+
if self._config.codex_cli_goal_worker:
|
|
877
|
+
return self._run_codex_cli_goal_worker(
|
|
878
|
+
prompt_text,
|
|
879
|
+
session=session,
|
|
880
|
+
session_id=session_id,
|
|
881
|
+
stdout=stdout,
|
|
882
|
+
)
|
|
883
|
+
with tempfile.TemporaryDirectory(prefix="gh-skillsbench-acp-") as tmp:
|
|
884
|
+
tmp_path = Path(tmp)
|
|
885
|
+
output_path = tmp_path / "last-message.txt"
|
|
886
|
+
stdout_path = tmp_path / "codex-stdout.txt"
|
|
887
|
+
stderr_path = tmp_path / "codex-stderr.txt"
|
|
888
|
+
prompt_for_codex = prompt_text
|
|
889
|
+
cwd = _safe_cwd(session.get("cwd"), default=os.getcwd())
|
|
890
|
+
resumable_loopx_turn = bool(
|
|
891
|
+
_bypass_loopx_turn
|
|
892
|
+
and self._config.loopx_turn_agent_cli
|
|
893
|
+
)
|
|
894
|
+
resume_session_id = (
|
|
895
|
+
str(session.get("_loopx_turn_codex_thread_id") or "")
|
|
896
|
+
if resumable_loopx_turn
|
|
897
|
+
else ""
|
|
898
|
+
)
|
|
899
|
+
bridge_server_proc: subprocess.Popen[str] | None = None
|
|
900
|
+
if self._config.remote_command_file_bridge_command:
|
|
901
|
+
if _is_bridge_action_preflight_prompt(prompt_text):
|
|
902
|
+
bridge_probe = self._reverse_channel_json_preflight_probe()
|
|
903
|
+
else:
|
|
904
|
+
bridge_probe = self._consume_remote_bridge_for_solver()
|
|
905
|
+
self._publish_remote_bridge_consumption_trace(bridge_probe)
|
|
906
|
+
if bridge_probe.get("ready") is not True:
|
|
907
|
+
raise RuntimeError("remote command/file bridge probe failed")
|
|
908
|
+
if resumable_loopx_turn:
|
|
909
|
+
session_root = _loopx_turn_local_session_root(
|
|
910
|
+
self._config.worker_public_trace_dir,
|
|
911
|
+
session_id,
|
|
912
|
+
)
|
|
913
|
+
if session_root is None:
|
|
914
|
+
raise RuntimeError(
|
|
915
|
+
"resumable LoopX Turn requires a public trace directory"
|
|
916
|
+
)
|
|
917
|
+
local_cwd = session_root / "cwd"
|
|
918
|
+
else:
|
|
919
|
+
local_cwd = tmp_path / "local-codex-cwd"
|
|
920
|
+
local_cwd.mkdir(parents=True, exist_ok=True)
|
|
921
|
+
cwd = str(local_cwd)
|
|
922
|
+
bridge_summary_path = tmp_path / "remote-bridge-agent-ops.jsonl"
|
|
923
|
+
agent_bridge_command = (
|
|
924
|
+
self._config.remote_command_file_bridge_agent_command
|
|
925
|
+
or self._config.remote_command_file_bridge_command
|
|
926
|
+
or ""
|
|
927
|
+
)
|
|
928
|
+
agent_bridge_command, bridge_server_proc = (
|
|
929
|
+
self._start_json_file_bridge_server(
|
|
930
|
+
tmp_path=tmp_path,
|
|
931
|
+
local_cwd=local_cwd,
|
|
932
|
+
bridge_command=agent_bridge_command,
|
|
933
|
+
)
|
|
934
|
+
)
|
|
935
|
+
instrumented_bridge = self._write_instrumented_bridge_wrapper(
|
|
936
|
+
tmp_path=tmp_path,
|
|
937
|
+
summary_path=bridge_summary_path,
|
|
938
|
+
bridge_command=agent_bridge_command,
|
|
939
|
+
)
|
|
940
|
+
bridge_command_for_agent = str(instrumented_bridge)
|
|
941
|
+
if self._config.loopx_workflow_lifecycle_checkpoint:
|
|
942
|
+
self._workflow_checkpoint_count += 1
|
|
943
|
+
self._run_loopx_workflow_lifecycle_checkpoint(
|
|
944
|
+
checkpoint_index=self._workflow_checkpoint_count,
|
|
945
|
+
)
|
|
946
|
+
prompt_for_codex = self._prompt_with_remote_bridge_packet(
|
|
947
|
+
prompt_text,
|
|
948
|
+
bridge_probe=bridge_probe,
|
|
949
|
+
bridge_command_for_agent=bridge_command_for_agent,
|
|
950
|
+
)
|
|
951
|
+
else:
|
|
952
|
+
bridge_summary_path = None
|
|
953
|
+
if resume_session_id:
|
|
954
|
+
cmd = [
|
|
955
|
+
self._config.codex_bin,
|
|
956
|
+
"exec",
|
|
957
|
+
"resume",
|
|
958
|
+
"--skip-git-repo-check",
|
|
959
|
+
"--output-last-message",
|
|
960
|
+
str(output_path),
|
|
961
|
+
"--json",
|
|
962
|
+
]
|
|
963
|
+
else:
|
|
964
|
+
cmd = [self._config.codex_bin, "exec"]
|
|
965
|
+
if not resumable_loopx_turn:
|
|
966
|
+
cmd.append("--ephemeral")
|
|
967
|
+
cmd.extend(
|
|
968
|
+
[
|
|
969
|
+
"--skip-git-repo-check",
|
|
970
|
+
"--sandbox",
|
|
971
|
+
self._config.sandbox,
|
|
972
|
+
"-C",
|
|
973
|
+
cwd,
|
|
974
|
+
"--output-last-message",
|
|
975
|
+
str(output_path),
|
|
976
|
+
"--json",
|
|
977
|
+
]
|
|
978
|
+
)
|
|
979
|
+
if self._config.reasoning_effort:
|
|
980
|
+
cmd.extend(
|
|
981
|
+
[
|
|
982
|
+
"-c",
|
|
983
|
+
"model_reasoning_effort="
|
|
984
|
+
+ json.dumps(str(self._config.reasoning_effort)),
|
|
985
|
+
]
|
|
986
|
+
)
|
|
987
|
+
model = self._config.model or session.get("model")
|
|
988
|
+
if model:
|
|
989
|
+
cmd.extend(["--model", str(model)])
|
|
990
|
+
if resume_session_id:
|
|
991
|
+
cmd.extend([resume_session_id, "-"])
|
|
992
|
+
elif resumable_loopx_turn:
|
|
993
|
+
cmd.append("-")
|
|
994
|
+
codex_stdin_prompt = prompt_for_codex
|
|
995
|
+
stdout_text = ""
|
|
996
|
+
stderr_text = ""
|
|
997
|
+
try:
|
|
998
|
+
with stdout_path.open("w", encoding="utf-8") as stdout_file, stderr_path.open(
|
|
999
|
+
"w", encoding="utf-8"
|
|
1000
|
+
) as stderr_file:
|
|
1001
|
+
codex_env = os.environ.copy()
|
|
1002
|
+
if bridge_summary_path is not None:
|
|
1003
|
+
codex_env["LOOPX_REMOTE_AGENT_OPS_SUMMARY_PATH"] = str(
|
|
1004
|
+
bridge_summary_path
|
|
1005
|
+
)
|
|
1006
|
+
proc = subprocess.Popen(
|
|
1007
|
+
cmd,
|
|
1008
|
+
stdin=subprocess.PIPE,
|
|
1009
|
+
stdout=stdout_file,
|
|
1010
|
+
stderr=stderr_file,
|
|
1011
|
+
text=True,
|
|
1012
|
+
env=codex_env,
|
|
1013
|
+
start_new_session=True,
|
|
1014
|
+
)
|
|
1015
|
+
_write_process_stdin_async(proc, codex_stdin_prompt)
|
|
1016
|
+
remaining_turn_seconds = (
|
|
1017
|
+
_turn_deadline - time.monotonic()
|
|
1018
|
+
if _turn_deadline is not None
|
|
1019
|
+
else float(self._config.timeout_sec)
|
|
1020
|
+
)
|
|
1021
|
+
if remaining_turn_seconds <= 0:
|
|
1022
|
+
self._terminate_codex_process(proc)
|
|
1023
|
+
return _recoverable_codex_turn_failure_message(
|
|
1024
|
+
"codex_exec_turn_time_budget_exhausted"
|
|
1025
|
+
)
|
|
1026
|
+
effective_timeout_sec = remaining_turn_seconds
|
|
1027
|
+
deadline = time.monotonic() + effective_timeout_sec
|
|
1028
|
+
first_action_deadline = 0.0
|
|
1029
|
+
if (
|
|
1030
|
+
bridge_summary_path is not None
|
|
1031
|
+
and self._config.first_action_timeout_sec > 0
|
|
1032
|
+
and _prompt_requires_bridge_first_action(prompt_for_codex)
|
|
1033
|
+
):
|
|
1034
|
+
first_action_deadline = (
|
|
1035
|
+
time.monotonic()
|
|
1036
|
+
+ max(1.0, self._config.first_action_timeout_sec)
|
|
1037
|
+
)
|
|
1038
|
+
meaningful_progress_deadline = 0.0
|
|
1039
|
+
meaningful_progress_seen = False
|
|
1040
|
+
meaningful_progress_required = (
|
|
1041
|
+
bridge_summary_path is not None
|
|
1042
|
+
and self._config.first_action_timeout_sec > 0
|
|
1043
|
+
and _prompt_requires_meaningful_bridge_progress(
|
|
1044
|
+
prompt_for_codex,
|
|
1045
|
+
route=self._config.route,
|
|
1046
|
+
)
|
|
1047
|
+
)
|
|
1048
|
+
allow_loopx_closeout_progress = self._config.route.startswith(
|
|
1049
|
+
"loopx-"
|
|
1050
|
+
)
|
|
1051
|
+
if meaningful_progress_required:
|
|
1052
|
+
meaningful_progress_deadline = (
|
|
1053
|
+
time.monotonic()
|
|
1054
|
+
+ max(1.0, self._config.first_action_timeout_sec)
|
|
1055
|
+
)
|
|
1056
|
+
first_action_seen = not bool(first_action_deadline)
|
|
1057
|
+
bridge_idle_timeout_sec = max(
|
|
1058
|
+
0.0,
|
|
1059
|
+
float(self._config.bridge_idle_timeout_sec or 0.0),
|
|
1060
|
+
)
|
|
1061
|
+
bridge_activity_seen = False
|
|
1062
|
+
last_bridge_summary_size = 0
|
|
1063
|
+
last_bridge_activity_at = time.monotonic()
|
|
1064
|
+
next_heartbeat = (
|
|
1065
|
+
time.monotonic()
|
|
1066
|
+
+ max(1.0, self._config.stream_heartbeat_interval_sec)
|
|
1067
|
+
)
|
|
1068
|
+
while proc.poll() is None:
|
|
1069
|
+
now = time.monotonic()
|
|
1070
|
+
if bridge_summary_path is not None:
|
|
1071
|
+
try:
|
|
1072
|
+
current_bridge_summary_size = (
|
|
1073
|
+
bridge_summary_path.stat().st_size
|
|
1074
|
+
)
|
|
1075
|
+
except OSError:
|
|
1076
|
+
current_bridge_summary_size = 0
|
|
1077
|
+
if current_bridge_summary_size > last_bridge_summary_size:
|
|
1078
|
+
last_bridge_summary_size = current_bridge_summary_size
|
|
1079
|
+
last_bridge_activity_at = now
|
|
1080
|
+
bridge_activity_seen = True
|
|
1081
|
+
first_action_seen = True
|
|
1082
|
+
elif (
|
|
1083
|
+
not first_action_seen
|
|
1084
|
+
and current_bridge_summary_size > 0
|
|
1085
|
+
):
|
|
1086
|
+
first_action_seen = True
|
|
1087
|
+
if not meaningful_progress_seen:
|
|
1088
|
+
meaningful_progress_seen = (
|
|
1089
|
+
_bridge_summary_has_meaningful_agent_progress(
|
|
1090
|
+
bridge_summary_path,
|
|
1091
|
+
allow_loopx_closeout=(
|
|
1092
|
+
allow_loopx_closeout_progress
|
|
1093
|
+
),
|
|
1094
|
+
)
|
|
1095
|
+
)
|
|
1096
|
+
if (
|
|
1097
|
+
not first_action_seen
|
|
1098
|
+
and first_action_deadline
|
|
1099
|
+
and now >= first_action_deadline
|
|
1100
|
+
):
|
|
1101
|
+
self._terminate_codex_process(proc)
|
|
1102
|
+
if bridge_summary_path is not None:
|
|
1103
|
+
self._publish_remote_bridge_agent_operations_trace(
|
|
1104
|
+
bridge_summary_path=bridge_summary_path,
|
|
1105
|
+
)
|
|
1106
|
+
self._publish_codex_exec_failure_trace(
|
|
1107
|
+
stage="first_action_timeout",
|
|
1108
|
+
returncode=124,
|
|
1109
|
+
stdout_text="",
|
|
1110
|
+
stderr_text="codex_exec_first_action_timeout\n",
|
|
1111
|
+
final_message_present=output_path.exists(),
|
|
1112
|
+
final_message_bytes=(
|
|
1113
|
+
output_path.stat().st_size
|
|
1114
|
+
if output_path.exists()
|
|
1115
|
+
else 0
|
|
1116
|
+
),
|
|
1117
|
+
failure_category="codex_exec_first_action_timeout",
|
|
1118
|
+
)
|
|
1119
|
+
return _recoverable_codex_turn_failure_message(
|
|
1120
|
+
"codex_exec_first_action_timeout"
|
|
1121
|
+
)
|
|
1122
|
+
if (
|
|
1123
|
+
meaningful_progress_required
|
|
1124
|
+
and not meaningful_progress_seen
|
|
1125
|
+
and meaningful_progress_deadline
|
|
1126
|
+
and now >= meaningful_progress_deadline
|
|
1127
|
+
):
|
|
1128
|
+
self._terminate_codex_process(proc)
|
|
1129
|
+
if bridge_summary_path is not None:
|
|
1130
|
+
self._publish_remote_bridge_agent_operations_trace(
|
|
1131
|
+
bridge_summary_path=bridge_summary_path,
|
|
1132
|
+
)
|
|
1133
|
+
self._publish_codex_exec_failure_trace(
|
|
1134
|
+
stage="meaningful_bridge_progress_timeout",
|
|
1135
|
+
returncode=124,
|
|
1136
|
+
stdout_text="",
|
|
1137
|
+
stderr_text="codex_exec_first_action_timeout\n",
|
|
1138
|
+
final_message_present=output_path.exists(),
|
|
1139
|
+
final_message_bytes=(
|
|
1140
|
+
output_path.stat().st_size
|
|
1141
|
+
if output_path.exists()
|
|
1142
|
+
else 0
|
|
1143
|
+
),
|
|
1144
|
+
failure_category="codex_exec_first_action_timeout",
|
|
1145
|
+
)
|
|
1146
|
+
return _recoverable_codex_turn_failure_message(
|
|
1147
|
+
"codex_exec_first_action_timeout"
|
|
1148
|
+
)
|
|
1149
|
+
if (
|
|
1150
|
+
bridge_activity_seen
|
|
1151
|
+
and bridge_summary_path is not None
|
|
1152
|
+
and bridge_idle_timeout_sec > 0
|
|
1153
|
+
and not _bridge_summary_has_inflight_operation(
|
|
1154
|
+
bridge_summary_path
|
|
1155
|
+
)
|
|
1156
|
+
and now - last_bridge_activity_at >= bridge_idle_timeout_sec
|
|
1157
|
+
):
|
|
1158
|
+
self._terminate_codex_process(proc)
|
|
1159
|
+
stdout_text = (
|
|
1160
|
+
stdout_path.read_text(
|
|
1161
|
+
encoding="utf-8", errors="replace"
|
|
1162
|
+
)
|
|
1163
|
+
if stdout_path.exists()
|
|
1164
|
+
else ""
|
|
1165
|
+
)
|
|
1166
|
+
stderr_text = (
|
|
1167
|
+
stderr_path.read_text(
|
|
1168
|
+
encoding="utf-8", errors="replace"
|
|
1169
|
+
)
|
|
1170
|
+
if stderr_path.exists()
|
|
1171
|
+
else ""
|
|
1172
|
+
)
|
|
1173
|
+
if resumable_loopx_turn and not resume_session_id:
|
|
1174
|
+
observed_session_id = (
|
|
1175
|
+
codex_cli_session_id_from_jsonl(stdout_text)
|
|
1176
|
+
)
|
|
1177
|
+
if observed_session_id:
|
|
1178
|
+
session["_loopx_turn_codex_thread_id"] = (
|
|
1179
|
+
observed_session_id
|
|
1180
|
+
)
|
|
1181
|
+
continuation_scheduled = (
|
|
1182
|
+
_codex_exec_same_session_continuation_allowed(
|
|
1183
|
+
category="codex_exec_bridge_idle_timeout",
|
|
1184
|
+
bridge_summary_path=bridge_summary_path,
|
|
1185
|
+
continuation_count=(
|
|
1186
|
+
_same_session_continuation_count
|
|
1187
|
+
),
|
|
1188
|
+
thread_present=bool(
|
|
1189
|
+
session.get("_loopx_turn_codex_thread_id")
|
|
1190
|
+
),
|
|
1191
|
+
final_message_present=output_path.exists(),
|
|
1192
|
+
turn_deadline=_turn_deadline,
|
|
1193
|
+
)
|
|
1194
|
+
)
|
|
1195
|
+
validation_handoff_scheduled = (
|
|
1196
|
+
_bypass_loopx_turn
|
|
1197
|
+
and self._config.loopx_turn_agent_cli
|
|
1198
|
+
and _codex_exec_progress_validation_handoff_allowed(
|
|
1199
|
+
category="codex_exec_bridge_idle_timeout",
|
|
1200
|
+
bridge_summary_path=bridge_summary_path,
|
|
1201
|
+
same_session_continuation_scheduled=(
|
|
1202
|
+
continuation_scheduled
|
|
1203
|
+
),
|
|
1204
|
+
final_message_present=output_path.exists(),
|
|
1205
|
+
turn_deadline=_turn_deadline,
|
|
1206
|
+
)
|
|
1207
|
+
)
|
|
1208
|
+
self._publish_remote_bridge_agent_operations_trace(
|
|
1209
|
+
bridge_summary_path=bridge_summary_path,
|
|
1210
|
+
)
|
|
1211
|
+
self._publish_codex_exec_failure_trace(
|
|
1212
|
+
stage="bridge_idle_timeout",
|
|
1213
|
+
returncode=124,
|
|
1214
|
+
stdout_text=stdout_text,
|
|
1215
|
+
stderr_text=stderr_text,
|
|
1216
|
+
final_message_present=output_path.exists(),
|
|
1217
|
+
final_message_bytes=(
|
|
1218
|
+
output_path.stat().st_size
|
|
1219
|
+
if output_path.exists()
|
|
1220
|
+
else 0
|
|
1221
|
+
),
|
|
1222
|
+
failure_category="codex_exec_bridge_idle_timeout",
|
|
1223
|
+
same_session_continuation_index=(
|
|
1224
|
+
_same_session_continuation_count
|
|
1225
|
+
),
|
|
1226
|
+
same_session_continuation_scheduled=(
|
|
1227
|
+
continuation_scheduled
|
|
1228
|
+
),
|
|
1229
|
+
independent_validation_handoff_scheduled=(
|
|
1230
|
+
validation_handoff_scheduled
|
|
1231
|
+
),
|
|
1232
|
+
)
|
|
1233
|
+
if continuation_scheduled:
|
|
1234
|
+
self._terminate_bridge_server_process(
|
|
1235
|
+
bridge_server_proc
|
|
1236
|
+
)
|
|
1237
|
+
bridge_server_proc = None
|
|
1238
|
+
next_continuation_index = (
|
|
1239
|
+
_same_session_continuation_count + 1
|
|
1240
|
+
)
|
|
1241
|
+
try:
|
|
1242
|
+
response = self._run_codex(
|
|
1243
|
+
_codex_exec_same_session_continuation_prompt(),
|
|
1244
|
+
session=session,
|
|
1245
|
+
session_id=session_id,
|
|
1246
|
+
stdout=stdout,
|
|
1247
|
+
_bypass_loopx_turn=_bypass_loopx_turn,
|
|
1248
|
+
_turn_deadline=_turn_deadline,
|
|
1249
|
+
_transport_retry_count=0,
|
|
1250
|
+
_session_rollover_index=(
|
|
1251
|
+
_session_rollover_index
|
|
1252
|
+
),
|
|
1253
|
+
_same_session_continuation_count=(
|
|
1254
|
+
next_continuation_index
|
|
1255
|
+
),
|
|
1256
|
+
)
|
|
1257
|
+
except (RuntimeError, TimeoutError):
|
|
1258
|
+
self._publish_codex_exec_same_session_continuation_trace(
|
|
1259
|
+
stage="failed",
|
|
1260
|
+
continuation_index=(
|
|
1261
|
+
next_continuation_index
|
|
1262
|
+
),
|
|
1263
|
+
thread_present=bool(
|
|
1264
|
+
session.get(
|
|
1265
|
+
"_loopx_turn_codex_thread_id"
|
|
1266
|
+
)
|
|
1267
|
+
),
|
|
1268
|
+
)
|
|
1269
|
+
raise
|
|
1270
|
+
continuation_succeeded = not response.startswith(
|
|
1271
|
+
RECOVERABLE_CODEX_TURN_FAILURE_PREFIX
|
|
1272
|
+
)
|
|
1273
|
+
self._publish_codex_exec_same_session_continuation_trace(
|
|
1274
|
+
stage=(
|
|
1275
|
+
"completed"
|
|
1276
|
+
if continuation_succeeded
|
|
1277
|
+
else "failed"
|
|
1278
|
+
),
|
|
1279
|
+
continuation_index=next_continuation_index,
|
|
1280
|
+
thread_present=bool(
|
|
1281
|
+
session.get("_loopx_turn_codex_thread_id")
|
|
1282
|
+
),
|
|
1283
|
+
)
|
|
1284
|
+
return response
|
|
1285
|
+
if validation_handoff_scheduled:
|
|
1286
|
+
self._latest_loopx_turn_agent_progress_receipt = (
|
|
1287
|
+
_bridge_summary_task_progress_receipt(
|
|
1288
|
+
bridge_summary_path
|
|
1289
|
+
)
|
|
1290
|
+
)
|
|
1291
|
+
return (
|
|
1292
|
+
SKILLSBENCH_TURN_AGENT_VALIDATION_HANDOFF_RESPONSE
|
|
1293
|
+
)
|
|
1294
|
+
return _recoverable_codex_turn_failure_message(
|
|
1295
|
+
"codex_exec_bridge_idle_timeout"
|
|
1296
|
+
)
|
|
1297
|
+
if now >= deadline:
|
|
1298
|
+
self._terminate_codex_process(proc)
|
|
1299
|
+
raise subprocess.TimeoutExpired(
|
|
1300
|
+
cmd,
|
|
1301
|
+
effective_timeout_sec,
|
|
1302
|
+
)
|
|
1303
|
+
if now >= next_heartbeat:
|
|
1304
|
+
self._write_worker_heartbeat(
|
|
1305
|
+
stdout,
|
|
1306
|
+
session_id=session_id,
|
|
1307
|
+
text="local codex exec still running",
|
|
1308
|
+
)
|
|
1309
|
+
next_heartbeat = (
|
|
1310
|
+
now
|
|
1311
|
+
+ max(
|
|
1312
|
+
1.0,
|
|
1313
|
+
self._config.stream_heartbeat_interval_sec,
|
|
1314
|
+
)
|
|
1315
|
+
)
|
|
1316
|
+
time.sleep(0.2)
|
|
1317
|
+
except subprocess.TimeoutExpired:
|
|
1318
|
+
failure_category = "codex_exec_timeout"
|
|
1319
|
+
stdout_text = (
|
|
1320
|
+
stdout_path.read_text(encoding="utf-8", errors="replace")
|
|
1321
|
+
if stdout_path.exists()
|
|
1322
|
+
else ""
|
|
1323
|
+
)
|
|
1324
|
+
stderr_text = (
|
|
1325
|
+
stderr_path.read_text(encoding="utf-8", errors="replace")
|
|
1326
|
+
if stderr_path.exists()
|
|
1327
|
+
else ""
|
|
1328
|
+
)
|
|
1329
|
+
validation_handoff_eligible = bool(
|
|
1330
|
+
_bypass_loopx_turn and self._config.loopx_turn_agent_cli
|
|
1331
|
+
)
|
|
1332
|
+
if validation_handoff_eligible:
|
|
1333
|
+
_wait_for_bridge_summary_quiescence(bridge_summary_path)
|
|
1334
|
+
validation_handoff_scheduled = bool(
|
|
1335
|
+
validation_handoff_eligible
|
|
1336
|
+
and _codex_exec_progress_validation_handoff_allowed(
|
|
1337
|
+
category=failure_category,
|
|
1338
|
+
bridge_summary_path=bridge_summary_path,
|
|
1339
|
+
same_session_continuation_scheduled=False,
|
|
1340
|
+
final_message_present=output_path.exists(),
|
|
1341
|
+
turn_deadline=_turn_deadline,
|
|
1342
|
+
)
|
|
1343
|
+
)
|
|
1344
|
+
if bridge_summary_path is not None:
|
|
1345
|
+
self._publish_remote_bridge_agent_operations_trace(
|
|
1346
|
+
bridge_summary_path=bridge_summary_path,
|
|
1347
|
+
)
|
|
1348
|
+
self._publish_codex_exec_failure_trace(
|
|
1349
|
+
stage="timeout",
|
|
1350
|
+
returncode=124,
|
|
1351
|
+
stdout_text=stdout_text,
|
|
1352
|
+
stderr_text=stderr_text,
|
|
1353
|
+
final_message_present=output_path.exists(),
|
|
1354
|
+
final_message_bytes=(
|
|
1355
|
+
output_path.stat().st_size if output_path.exists() else 0
|
|
1356
|
+
),
|
|
1357
|
+
failure_category=failure_category,
|
|
1358
|
+
independent_validation_handoff_scheduled=(
|
|
1359
|
+
validation_handoff_scheduled
|
|
1360
|
+
),
|
|
1361
|
+
)
|
|
1362
|
+
if validation_handoff_scheduled:
|
|
1363
|
+
self._latest_loopx_turn_agent_progress_receipt = (
|
|
1364
|
+
_bridge_summary_task_progress_receipt(bridge_summary_path)
|
|
1365
|
+
)
|
|
1366
|
+
return SKILLSBENCH_TURN_AGENT_VALIDATION_HANDOFF_RESPONSE
|
|
1367
|
+
return _recoverable_codex_turn_failure_message(failure_category)
|
|
1368
|
+
finally:
|
|
1369
|
+
self._terminate_bridge_server_process(bridge_server_proc)
|
|
1370
|
+
stdout_text = (
|
|
1371
|
+
stdout_path.read_text(encoding="utf-8", errors="replace")
|
|
1372
|
+
if stdout_path.exists()
|
|
1373
|
+
else ""
|
|
1374
|
+
)
|
|
1375
|
+
stderr_text = (
|
|
1376
|
+
stderr_path.read_text(encoding="utf-8", errors="replace")
|
|
1377
|
+
if stderr_path.exists()
|
|
1378
|
+
else ""
|
|
1379
|
+
)
|
|
1380
|
+
if proc.returncode != 0:
|
|
1381
|
+
category = _codex_exec_failure_category(
|
|
1382
|
+
returncode=proc.returncode,
|
|
1383
|
+
stderr_text=stderr_text,
|
|
1384
|
+
stdout_text=stdout_text,
|
|
1385
|
+
)
|
|
1386
|
+
recoverable_turn_failure = bool(
|
|
1387
|
+
category in RECOVERABLE_CODEX_TURN_FAILURE_CATEGORIES
|
|
1388
|
+
and prompt_text.strip()
|
|
1389
|
+
!= SKILLSBENCH_LOCAL_ACP_RELAY_HEALTH_PROMPT
|
|
1390
|
+
and not _is_bridge_action_preflight_prompt(prompt_text)
|
|
1391
|
+
)
|
|
1392
|
+
retry_scheduled = bool(
|
|
1393
|
+
recoverable_turn_failure
|
|
1394
|
+
and resumable_loopx_turn
|
|
1395
|
+
and _codex_exec_transport_retry_allowed(
|
|
1396
|
+
category=category,
|
|
1397
|
+
bridge_summary_path=bridge_summary_path,
|
|
1398
|
+
retry_count=_transport_retry_count,
|
|
1399
|
+
final_message_present=output_path.exists(),
|
|
1400
|
+
turn_deadline=_turn_deadline,
|
|
1401
|
+
)
|
|
1402
|
+
)
|
|
1403
|
+
rollover_count = int(
|
|
1404
|
+
session.get("_loopx_turn_codex_rollover_count") or 0
|
|
1405
|
+
)
|
|
1406
|
+
session_rollover_scheduled = bool(
|
|
1407
|
+
recoverable_turn_failure
|
|
1408
|
+
and resumable_loopx_turn
|
|
1409
|
+
and bool(resume_session_id)
|
|
1410
|
+
and _codex_exec_session_rollover_allowed(
|
|
1411
|
+
category=category,
|
|
1412
|
+
bridge_summary_path=bridge_summary_path,
|
|
1413
|
+
retry_count=_transport_retry_count,
|
|
1414
|
+
rollover_count=rollover_count,
|
|
1415
|
+
final_message_present=output_path.exists(),
|
|
1416
|
+
turn_deadline=_turn_deadline,
|
|
1417
|
+
)
|
|
1418
|
+
)
|
|
1419
|
+
self._publish_codex_exec_failure_trace(
|
|
1420
|
+
stage="exit_nonzero",
|
|
1421
|
+
returncode=proc.returncode,
|
|
1422
|
+
stdout_text=stdout_text,
|
|
1423
|
+
stderr_text=stderr_text,
|
|
1424
|
+
final_message_present=output_path.exists(),
|
|
1425
|
+
final_message_bytes=(
|
|
1426
|
+
output_path.stat().st_size if output_path.exists() else 0
|
|
1427
|
+
),
|
|
1428
|
+
failure_category=category,
|
|
1429
|
+
recoverable_turn_failure=recoverable_turn_failure,
|
|
1430
|
+
transport_retry_index=_transport_retry_count,
|
|
1431
|
+
retry_scheduled=retry_scheduled,
|
|
1432
|
+
session_rollover_index=_session_rollover_index,
|
|
1433
|
+
session_rollover_scheduled=session_rollover_scheduled,
|
|
1434
|
+
)
|
|
1435
|
+
if bridge_summary_path is not None:
|
|
1436
|
+
self._publish_remote_bridge_agent_operations_trace(
|
|
1437
|
+
bridge_summary_path=bridge_summary_path,
|
|
1438
|
+
)
|
|
1439
|
+
if retry_scheduled:
|
|
1440
|
+
return self._run_codex(
|
|
1441
|
+
prompt_text,
|
|
1442
|
+
session=session,
|
|
1443
|
+
session_id=session_id,
|
|
1444
|
+
stdout=stdout,
|
|
1445
|
+
_bypass_loopx_turn=_bypass_loopx_turn,
|
|
1446
|
+
_turn_deadline=_turn_deadline,
|
|
1447
|
+
_transport_retry_count=_transport_retry_count + 1,
|
|
1448
|
+
_session_rollover_index=_session_rollover_index,
|
|
1449
|
+
_same_session_continuation_count=(
|
|
1450
|
+
_same_session_continuation_count
|
|
1451
|
+
),
|
|
1452
|
+
)
|
|
1453
|
+
if session_rollover_scheduled:
|
|
1454
|
+
original_prompt = str(
|
|
1455
|
+
session.get("_loopx_turn_original_prompt") or ""
|
|
1456
|
+
).strip()
|
|
1457
|
+
if not original_prompt:
|
|
1458
|
+
return _recoverable_codex_turn_failure_message(category)
|
|
1459
|
+
next_rollover_index = rollover_count + 1
|
|
1460
|
+
session["_loopx_turn_codex_rollover_count"] = next_rollover_index
|
|
1461
|
+
previous_thread_present = bool(
|
|
1462
|
+
session.pop("_loopx_turn_codex_thread_id", None)
|
|
1463
|
+
)
|
|
1464
|
+
rollover_prompt = _codex_exec_session_rollover_prompt(
|
|
1465
|
+
original_prompt=original_prompt,
|
|
1466
|
+
continuation_prompt=prompt_text,
|
|
1467
|
+
)
|
|
1468
|
+
try:
|
|
1469
|
+
response = self._run_codex(
|
|
1470
|
+
rollover_prompt,
|
|
1471
|
+
session=session,
|
|
1472
|
+
session_id=session_id,
|
|
1473
|
+
stdout=stdout,
|
|
1474
|
+
_bypass_loopx_turn=_bypass_loopx_turn,
|
|
1475
|
+
_turn_deadline=_turn_deadline,
|
|
1476
|
+
_transport_retry_count=(
|
|
1477
|
+
CODEX_EXEC_TRANSPORT_RETRY_LIMIT
|
|
1478
|
+
),
|
|
1479
|
+
_session_rollover_index=next_rollover_index,
|
|
1480
|
+
_same_session_continuation_count=(
|
|
1481
|
+
_same_session_continuation_count
|
|
1482
|
+
),
|
|
1483
|
+
)
|
|
1484
|
+
except (RuntimeError, TimeoutError):
|
|
1485
|
+
self._publish_codex_exec_session_rollover_trace(
|
|
1486
|
+
stage="failed",
|
|
1487
|
+
rollover_index=next_rollover_index,
|
|
1488
|
+
resume_failure_count=_transport_retry_count + 1,
|
|
1489
|
+
previous_thread_present=previous_thread_present,
|
|
1490
|
+
fresh_thread_present=bool(
|
|
1491
|
+
session.get("_loopx_turn_codex_thread_id")
|
|
1492
|
+
),
|
|
1493
|
+
)
|
|
1494
|
+
raise
|
|
1495
|
+
rollover_succeeded = not response.startswith(
|
|
1496
|
+
RECOVERABLE_CODEX_TURN_FAILURE_PREFIX
|
|
1497
|
+
)
|
|
1498
|
+
self._publish_codex_exec_session_rollover_trace(
|
|
1499
|
+
stage="completed" if rollover_succeeded else "failed",
|
|
1500
|
+
rollover_index=next_rollover_index,
|
|
1501
|
+
resume_failure_count=_transport_retry_count + 1,
|
|
1502
|
+
previous_thread_present=previous_thread_present,
|
|
1503
|
+
fresh_thread_present=bool(
|
|
1504
|
+
session.get("_loopx_turn_codex_thread_id")
|
|
1505
|
+
),
|
|
1506
|
+
)
|
|
1507
|
+
return response
|
|
1508
|
+
if recoverable_turn_failure:
|
|
1509
|
+
return _recoverable_codex_turn_failure_message(category)
|
|
1510
|
+
raise RuntimeError(f"local codex execution failed: {category}")
|
|
1511
|
+
if resumable_loopx_turn and not resume_session_id:
|
|
1512
|
+
observed_session_id = codex_cli_session_id_from_jsonl(stdout_text)
|
|
1513
|
+
if not observed_session_id:
|
|
1514
|
+
self._publish_codex_exec_failure_trace(
|
|
1515
|
+
stage="session_start_missing",
|
|
1516
|
+
returncode=0,
|
|
1517
|
+
stdout_text=stdout_text,
|
|
1518
|
+
stderr_text=stderr_text,
|
|
1519
|
+
final_message_present=output_path.exists(),
|
|
1520
|
+
final_message_bytes=(
|
|
1521
|
+
output_path.stat().st_size if output_path.exists() else 0
|
|
1522
|
+
),
|
|
1523
|
+
failure_category="codex_exec_session_missing",
|
|
1524
|
+
recoverable_turn_failure=True,
|
|
1525
|
+
)
|
|
1526
|
+
return _recoverable_codex_turn_failure_message(
|
|
1527
|
+
"codex_exec_session_missing"
|
|
1528
|
+
)
|
|
1529
|
+
session["_loopx_turn_codex_thread_id"] = observed_session_id
|
|
1530
|
+
try:
|
|
1531
|
+
response = output_path.read_text(encoding="utf-8").strip()
|
|
1532
|
+
except OSError as exc:
|
|
1533
|
+
self._publish_codex_exec_failure_trace(
|
|
1534
|
+
stage="final_message_missing",
|
|
1535
|
+
returncode=proc.returncode,
|
|
1536
|
+
stdout_text=stdout_text,
|
|
1537
|
+
stderr_text=stderr_text,
|
|
1538
|
+
final_message_present=False,
|
|
1539
|
+
final_message_bytes=0,
|
|
1540
|
+
failure_category="codex_final_message_missing",
|
|
1541
|
+
)
|
|
1542
|
+
raise RuntimeError("local codex final message missing") from exc
|
|
1543
|
+
if bridge_summary_path is not None:
|
|
1544
|
+
self._publish_remote_bridge_agent_operations_trace(
|
|
1545
|
+
bridge_summary_path=bridge_summary_path,
|
|
1546
|
+
)
|
|
1547
|
+
if _bypass_loopx_turn and self._config.loopx_turn_agent_cli:
|
|
1548
|
+
self._latest_loopx_turn_agent_progress_receipt = (
|
|
1549
|
+
_bridge_summary_task_progress_receipt(bridge_summary_path)
|
|
1550
|
+
)
|
|
1551
|
+
return response or "local codex returned an empty final message"
|
|
1552
|
+
|
|
1553
|
+
def _run_loopx_turn_agent_prompt(
|
|
1554
|
+
self,
|
|
1555
|
+
prompt_text: str,
|
|
1556
|
+
*,
|
|
1557
|
+
session: dict[str, Any],
|
|
1558
|
+
session_id: str,
|
|
1559
|
+
stdout: TextIO,
|
|
1560
|
+
turn_deadline: float,
|
|
1561
|
+
) -> SkillsBenchTurnAgentResult:
|
|
1562
|
+
self._latest_loopx_turn_agent_progress_receipt = {}
|
|
1563
|
+
response = self._run_codex(
|
|
1564
|
+
prompt_text,
|
|
1565
|
+
session=session,
|
|
1566
|
+
session_id=session_id,
|
|
1567
|
+
stdout=stdout,
|
|
1568
|
+
_bypass_loopx_turn=True,
|
|
1569
|
+
_turn_deadline=turn_deadline,
|
|
1570
|
+
)
|
|
1571
|
+
return SkillsBenchTurnAgentResult(
|
|
1572
|
+
response_text=response,
|
|
1573
|
+
progress_evidence=dict(self._latest_loopx_turn_agent_progress_receipt),
|
|
1574
|
+
)
|
|
1575
|
+
|
|
1576
|
+
def _run_codex_cli_goal_worker(
|
|
1577
|
+
self,
|
|
1578
|
+
prompt_text: str,
|
|
1579
|
+
*,
|
|
1580
|
+
session: dict[str, Any],
|
|
1581
|
+
session_id: str,
|
|
1582
|
+
stdout: TextIO,
|
|
1583
|
+
) -> str:
|
|
1584
|
+
"""Run one ACP prompt through the real Codex CLI TUI /goal surface."""
|
|
1585
|
+
|
|
1586
|
+
resolved_codex_bin = resolve_codex_cli_binary(self._config.codex_bin)
|
|
1587
|
+
if not resolved_codex_bin:
|
|
1588
|
+
self._publish_codex_cli_goal_trace(
|
|
1589
|
+
ok=False,
|
|
1590
|
+
stage="codex_cli_unavailable",
|
|
1591
|
+
goal_active_observed=False,
|
|
1592
|
+
goal_terminal_observed=False,
|
|
1593
|
+
first_action_observed=False,
|
|
1594
|
+
bridge_summary_path=None,
|
|
1595
|
+
goal_slash_command_submitted=False,
|
|
1596
|
+
)
|
|
1597
|
+
raise RuntimeError("codex cli goal worker requires Codex CLI")
|
|
1598
|
+
if shutil.which("tmux") is None:
|
|
1599
|
+
self._publish_codex_cli_goal_trace(
|
|
1600
|
+
ok=False,
|
|
1601
|
+
stage="tmux_missing",
|
|
1602
|
+
goal_active_observed=False,
|
|
1603
|
+
goal_terminal_observed=False,
|
|
1604
|
+
first_action_observed=False,
|
|
1605
|
+
bridge_summary_path=None,
|
|
1606
|
+
goal_slash_command_submitted=False,
|
|
1607
|
+
)
|
|
1608
|
+
raise RuntimeError("codex cli goal worker requires tmux")
|
|
1609
|
+
with tempfile.TemporaryDirectory(prefix="gh-skillsbench-cli-goal-") as tmp:
|
|
1610
|
+
tmp_path = Path(tmp)
|
|
1611
|
+
prompt_for_codex = prompt_text
|
|
1612
|
+
cwd = _safe_cwd(session.get("cwd"), default=os.getcwd())
|
|
1613
|
+
bridge_server_proc: subprocess.Popen[str] | None = None
|
|
1614
|
+
bridge_summary_path: Path | None = None
|
|
1615
|
+
goal_prompt_file_used = False
|
|
1616
|
+
goal_command_submission_method = "paste-buffer"
|
|
1617
|
+
if self._config.remote_command_file_bridge_command:
|
|
1618
|
+
if _is_bridge_action_preflight_prompt(prompt_text):
|
|
1619
|
+
bridge_probe = self._reverse_channel_json_preflight_probe()
|
|
1620
|
+
else:
|
|
1621
|
+
bridge_probe = self._consume_remote_bridge_for_solver()
|
|
1622
|
+
self._publish_remote_bridge_consumption_trace(bridge_probe)
|
|
1623
|
+
if bridge_probe.get("ready") is not True:
|
|
1624
|
+
self._publish_codex_cli_goal_trace(
|
|
1625
|
+
ok=False,
|
|
1626
|
+
stage="bridge_probe_failed",
|
|
1627
|
+
goal_active_observed=False,
|
|
1628
|
+
goal_terminal_observed=False,
|
|
1629
|
+
first_action_observed=False,
|
|
1630
|
+
bridge_summary_path=None,
|
|
1631
|
+
goal_slash_command_submitted=False,
|
|
1632
|
+
)
|
|
1633
|
+
raise RuntimeError("remote command/file bridge probe failed")
|
|
1634
|
+
local_cwd = tmp_path / "local-codex-cli-goal-cwd"
|
|
1635
|
+
local_cwd.mkdir(parents=True, exist_ok=True)
|
|
1636
|
+
cwd = str(local_cwd)
|
|
1637
|
+
bridge_summary_path = tmp_path / "remote-bridge-agent-ops.jsonl"
|
|
1638
|
+
agent_bridge_command = (
|
|
1639
|
+
self._config.remote_command_file_bridge_agent_command
|
|
1640
|
+
or self._config.remote_command_file_bridge_command
|
|
1641
|
+
or ""
|
|
1642
|
+
)
|
|
1643
|
+
agent_bridge_command, bridge_server_proc = (
|
|
1644
|
+
self._start_json_file_bridge_server(
|
|
1645
|
+
tmp_path=tmp_path,
|
|
1646
|
+
local_cwd=local_cwd,
|
|
1647
|
+
bridge_command=agent_bridge_command,
|
|
1648
|
+
)
|
|
1649
|
+
)
|
|
1650
|
+
instrumented_bridge = self._write_instrumented_bridge_wrapper(
|
|
1651
|
+
tmp_path=tmp_path,
|
|
1652
|
+
summary_path=bridge_summary_path,
|
|
1653
|
+
bridge_command=agent_bridge_command,
|
|
1654
|
+
)
|
|
1655
|
+
prompt_for_codex = self._prompt_with_remote_bridge_packet(
|
|
1656
|
+
prompt_text,
|
|
1657
|
+
bridge_probe=bridge_probe,
|
|
1658
|
+
bridge_command_for_agent=str(instrumented_bridge),
|
|
1659
|
+
)
|
|
1660
|
+
prompt_instruction_path = (
|
|
1661
|
+
Path(cwd) / CODEX_CLI_GOAL_TASK_PROMPT_FILENAME
|
|
1662
|
+
)
|
|
1663
|
+
prompt_instruction_path.write_text(
|
|
1664
|
+
prompt_for_codex,
|
|
1665
|
+
encoding="utf-8",
|
|
1666
|
+
)
|
|
1667
|
+
prompt_for_goal = build_codex_cli_goal_file_objective(
|
|
1668
|
+
CODEX_CLI_GOAL_TASK_PROMPT_FILENAME
|
|
1669
|
+
)
|
|
1670
|
+
goal_prompt_file_used = True
|
|
1671
|
+
goal_command_submission_method = "typed"
|
|
1672
|
+
else:
|
|
1673
|
+
prompt_for_goal = prompt_for_codex
|
|
1674
|
+
goal_command_text = build_codex_cli_goal_tui_input(prompt_for_goal)
|
|
1675
|
+
prompt_path = tmp_path / "goal-prompt.txt"
|
|
1676
|
+
prompt_path.write_text(
|
|
1677
|
+
goal_command_text,
|
|
1678
|
+
encoding="utf-8",
|
|
1679
|
+
)
|
|
1680
|
+
tmux_name = f"gh-sb-cli-goal-{uuid.uuid4().hex[:10]}"
|
|
1681
|
+
cmd = build_codex_cli_tui_command(
|
|
1682
|
+
codex_bin=resolved_codex_bin,
|
|
1683
|
+
sandbox=self._config.sandbox,
|
|
1684
|
+
approval_policy=self._config.approval_policy,
|
|
1685
|
+
cwd=cwd,
|
|
1686
|
+
reasoning_effort=self._config.reasoning_effort,
|
|
1687
|
+
model=self._config.model or session.get("model"),
|
|
1688
|
+
)
|
|
1689
|
+
shell_command = codex_cli_tui_shell_command(
|
|
1690
|
+
cmd,
|
|
1691
|
+
env=codex_cli_tui_environment(self._config.codex_api_proxy),
|
|
1692
|
+
)
|
|
1693
|
+
goal_active_observed = False
|
|
1694
|
+
goal_terminal_observed = False
|
|
1695
|
+
goal_failed_observed = False
|
|
1696
|
+
goal_slash_command_submitted = False
|
|
1697
|
+
goal_kickoff_prompt_submitted = False
|
|
1698
|
+
post_bridge_terminal_stage = ""
|
|
1699
|
+
first_action_seen = False
|
|
1700
|
+
bridge_activity_seen = False
|
|
1701
|
+
last_bridge_summary_size = 0
|
|
1702
|
+
last_bridge_activity_at = time.monotonic()
|
|
1703
|
+
meaningful_progress_seen = False
|
|
1704
|
+
pre_bridge_recovery_attempt_count = 0
|
|
1705
|
+
pre_bridge_recovery_action = ""
|
|
1706
|
+
pre_bridge_recovery_skip_reason = ""
|
|
1707
|
+
post_bridge_recovery_attempt_count = 0
|
|
1708
|
+
post_bridge_recovery_action = ""
|
|
1709
|
+
post_bridge_recovery_skip_reason = ""
|
|
1710
|
+
pre_bridge_terminal_stage = ""
|
|
1711
|
+
thread_prewarm = self._config.codex_cli_goal_thread_prewarm
|
|
1712
|
+
self._last_codex_cli_goal_lifecycle = goal_lifecycle = CodexCliGoalLifecycleGeneration()
|
|
1713
|
+
try:
|
|
1714
|
+
startup_stage, thread_prewarm_observed = start_codex_cli_goal_tui_session(
|
|
1715
|
+
tmux_name=tmux_name,
|
|
1716
|
+
cwd=cwd,
|
|
1717
|
+
shell_command=shell_command,
|
|
1718
|
+
thread_prewarm=thread_prewarm,
|
|
1719
|
+
thread_prewarm_timeout_sec=CODEX_CLI_GOAL_THREAD_PREWARM_TIMEOUT_SEC,
|
|
1720
|
+
)
|
|
1721
|
+
if startup_stage:
|
|
1722
|
+
self._publish_codex_cli_goal_trace(
|
|
1723
|
+
ok=False,
|
|
1724
|
+
stage=startup_stage,
|
|
1725
|
+
goal_active_observed=False,
|
|
1726
|
+
goal_terminal_observed=False,
|
|
1727
|
+
first_action_observed=False,
|
|
1728
|
+
bridge_summary_path=bridge_summary_path,
|
|
1729
|
+
thread_prewarm_observed=thread_prewarm_observed,
|
|
1730
|
+
goal_prompt_file_used=goal_prompt_file_used,
|
|
1731
|
+
goal_command_submission_method=goal_command_submission_method,
|
|
1732
|
+
goal_slash_command_submitted=False,
|
|
1733
|
+
)
|
|
1734
|
+
return _recoverable_codex_turn_failure_message(
|
|
1735
|
+
"codex_exec_first_action_timeout"
|
|
1736
|
+
)
|
|
1737
|
+
goal_lifecycle.begin(tmux_capture(tmux_name))
|
|
1738
|
+
if goal_prompt_file_used:
|
|
1739
|
+
tmux_type_text_and_submit(
|
|
1740
|
+
tmux_name=tmux_name,
|
|
1741
|
+
text=goal_command_text,
|
|
1742
|
+
)
|
|
1743
|
+
else:
|
|
1744
|
+
tmux_paste_file_and_submit(
|
|
1745
|
+
tmux_name=tmux_name,
|
|
1746
|
+
prompt_path=prompt_path,
|
|
1747
|
+
buffer_suffix="prompt",
|
|
1748
|
+
)
|
|
1749
|
+
goal_slash_command_submitted = True
|
|
1750
|
+
deadline = time.monotonic() + self._config.timeout_sec
|
|
1751
|
+
goal_active_deadline = 0.0
|
|
1752
|
+
if (
|
|
1753
|
+
bridge_summary_path is not None
|
|
1754
|
+
and self._config.goal_active_timeout_sec > 0
|
|
1755
|
+
and _prompt_requires_bridge_first_action(prompt_for_codex)
|
|
1756
|
+
):
|
|
1757
|
+
goal_active_timeout_sec = max(
|
|
1758
|
+
1.0,
|
|
1759
|
+
float(self._config.goal_active_timeout_sec or 0.0),
|
|
1760
|
+
)
|
|
1761
|
+
goal_active_deadline = (
|
|
1762
|
+
time.monotonic() + goal_active_timeout_sec
|
|
1763
|
+
)
|
|
1764
|
+
first_action_deadline = 0.0
|
|
1765
|
+
if (
|
|
1766
|
+
bridge_summary_path is not None
|
|
1767
|
+
and self._config.first_action_timeout_sec > 0
|
|
1768
|
+
and _prompt_requires_bridge_first_action(prompt_for_codex)
|
|
1769
|
+
):
|
|
1770
|
+
first_action_deadline = (
|
|
1771
|
+
time.monotonic()
|
|
1772
|
+
+ max(1.0, self._config.first_action_timeout_sec)
|
|
1773
|
+
)
|
|
1774
|
+
meaningful_progress_required = (
|
|
1775
|
+
bridge_summary_path is not None
|
|
1776
|
+
and self._config.first_action_timeout_sec > 0
|
|
1777
|
+
and _prompt_requires_meaningful_bridge_progress(
|
|
1778
|
+
prompt_for_codex,
|
|
1779
|
+
route=self._config.route,
|
|
1780
|
+
)
|
|
1781
|
+
)
|
|
1782
|
+
meaningful_progress_deadline = (
|
|
1783
|
+
time.monotonic()
|
|
1784
|
+
+ max(1.0, self._config.first_action_timeout_sec)
|
|
1785
|
+
if meaningful_progress_required
|
|
1786
|
+
else 0.0
|
|
1787
|
+
)
|
|
1788
|
+
first_action_seen = not bool(first_action_deadline)
|
|
1789
|
+
bridge_idle_timeout_sec = max(
|
|
1790
|
+
0.0,
|
|
1791
|
+
float(self._config.bridge_idle_timeout_sec or 0.0),
|
|
1792
|
+
)
|
|
1793
|
+
task_output_quiet_timeout_sec = max(
|
|
1794
|
+
0.0,
|
|
1795
|
+
float(self._config.task_output_quiet_timeout_sec or 0.0),
|
|
1796
|
+
)
|
|
1797
|
+
next_heartbeat = (
|
|
1798
|
+
time.monotonic()
|
|
1799
|
+
+ max(1.0, self._config.stream_heartbeat_interval_sec)
|
|
1800
|
+
)
|
|
1801
|
+
task_output_progress_seen = False
|
|
1802
|
+
last_task_output_activity_at = last_bridge_activity_at
|
|
1803
|
+
while time.monotonic() < deadline:
|
|
1804
|
+
now = time.monotonic()
|
|
1805
|
+
capture = self._last_codex_cli_goal_tui_capture = tmux_capture(tmux_name)
|
|
1806
|
+
turn_active = codex_cli_tui_turn_active(capture)
|
|
1807
|
+
goal_lifecycle.observe(capture, turn_active=turn_active)
|
|
1808
|
+
if turn_active:
|
|
1809
|
+
last_task_output_activity_at = now
|
|
1810
|
+
if goal_lifecycle.active_observed:
|
|
1811
|
+
goal_active_observed = True
|
|
1812
|
+
goal_failed_now = goal_lifecycle.failed_advanced
|
|
1813
|
+
if bridge_summary_path is not None:
|
|
1814
|
+
try:
|
|
1815
|
+
current_bridge_summary_size = bridge_summary_path.stat().st_size
|
|
1816
|
+
except OSError:
|
|
1817
|
+
current_bridge_summary_size = 0
|
|
1818
|
+
if current_bridge_summary_size > last_bridge_summary_size:
|
|
1819
|
+
last_bridge_summary_size = current_bridge_summary_size
|
|
1820
|
+
last_bridge_activity_at = now
|
|
1821
|
+
last_task_output_activity_at = now
|
|
1822
|
+
bridge_activity_seen = True
|
|
1823
|
+
first_action_seen = True
|
|
1824
|
+
elif (
|
|
1825
|
+
not first_action_seen
|
|
1826
|
+
and current_bridge_summary_size > 0
|
|
1827
|
+
):
|
|
1828
|
+
first_action_seen = True
|
|
1829
|
+
if not meaningful_progress_seen:
|
|
1830
|
+
meaningful_progress_seen = (
|
|
1831
|
+
_bridge_summary_has_meaningful_agent_progress(
|
|
1832
|
+
bridge_summary_path,
|
|
1833
|
+
allow_loopx_closeout=False,
|
|
1834
|
+
)
|
|
1835
|
+
)
|
|
1836
|
+
if (
|
|
1837
|
+
not task_output_progress_seen
|
|
1838
|
+
and task_output_quiet_timeout_sec > 0
|
|
1839
|
+
):
|
|
1840
|
+
task_output_progress_seen = (
|
|
1841
|
+
_bridge_summary_has_successful_task_operation(
|
|
1842
|
+
bridge_summary_path
|
|
1843
|
+
)
|
|
1844
|
+
)
|
|
1845
|
+
if codex_cli_goal_should_submit_kickoff(
|
|
1846
|
+
bridge_enabled=bridge_summary_path is not None,
|
|
1847
|
+
goal_active_observed=goal_active_observed,
|
|
1848
|
+
kickoff_submitted=goal_kickoff_prompt_submitted,
|
|
1849
|
+
turn_active=turn_active,
|
|
1850
|
+
first_action_seen=first_action_seen,
|
|
1851
|
+
capture=capture,
|
|
1852
|
+
):
|
|
1853
|
+
goal_kickoff_prompt_submitted = True
|
|
1854
|
+
tmux_type_text_and_submit(
|
|
1855
|
+
tmux_name=tmux_name,
|
|
1856
|
+
text=CODEX_CLI_GOAL_KICKOFF_PROMPT,
|
|
1857
|
+
)
|
|
1858
|
+
(
|
|
1859
|
+
goal_active_deadline,
|
|
1860
|
+
first_action_deadline,
|
|
1861
|
+
meaningful_progress_deadline,
|
|
1862
|
+
) = codex_cli_goal_reset_pre_bridge_deadlines(
|
|
1863
|
+
now=now,
|
|
1864
|
+
first_action_timeout_sec=self._config.first_action_timeout_sec,
|
|
1865
|
+
goal_active_deadline=goal_active_deadline,
|
|
1866
|
+
goal_active_timeout_sec=self._config.goal_active_timeout_sec,
|
|
1867
|
+
first_action_deadline=first_action_deadline,
|
|
1868
|
+
meaningful_progress_deadline=meaningful_progress_deadline,
|
|
1869
|
+
)
|
|
1870
|
+
next_heartbeat = now + max(
|
|
1871
|
+
1.0,
|
|
1872
|
+
self._config.stream_heartbeat_interval_sec,
|
|
1873
|
+
)
|
|
1874
|
+
continue
|
|
1875
|
+
if codex_cli_goal_should_ignore_stale_terminal(
|
|
1876
|
+
goal_failed_now=goal_failed_now,
|
|
1877
|
+
kickoff_submitted=goal_kickoff_prompt_submitted,
|
|
1878
|
+
first_action_seen=first_action_seen,
|
|
1879
|
+
turn_active=turn_active,
|
|
1880
|
+
first_action_deadline=first_action_deadline,
|
|
1881
|
+
now=now,
|
|
1882
|
+
):
|
|
1883
|
+
goal_failed_now = False
|
|
1884
|
+
if goal_lifecycle.achieved_advanced:
|
|
1885
|
+
goal_terminal_observed = True
|
|
1886
|
+
break
|
|
1887
|
+
retryable_startup_blocker_stage = ""
|
|
1888
|
+
if not goal_active_observed and not first_action_seen:
|
|
1889
|
+
retryable_startup_blocker_stage = codex_cli_tui_retryable_startup_blocker_stage(capture)
|
|
1890
|
+
if retryable_startup_blocker_stage:
|
|
1891
|
+
tmux_kill_session(tmux_name)
|
|
1892
|
+
if bridge_summary_path is not None:
|
|
1893
|
+
self._publish_remote_bridge_agent_operations_trace(
|
|
1894
|
+
bridge_summary_path=bridge_summary_path,
|
|
1895
|
+
)
|
|
1896
|
+
self._publish_codex_cli_goal_trace(
|
|
1897
|
+
ok=False,
|
|
1898
|
+
stage=retryable_startup_blocker_stage,
|
|
1899
|
+
goal_active_observed=goal_active_observed,
|
|
1900
|
+
goal_terminal_observed=goal_terminal_observed,
|
|
1901
|
+
first_action_observed=first_action_seen,
|
|
1902
|
+
bridge_summary_path=bridge_summary_path,
|
|
1903
|
+
thread_prewarm_observed=thread_prewarm_observed,
|
|
1904
|
+
goal_prompt_file_used=goal_prompt_file_used,
|
|
1905
|
+
goal_command_submission_method=goal_command_submission_method,
|
|
1906
|
+
goal_kickoff_prompt_submitted=goal_kickoff_prompt_submitted,
|
|
1907
|
+
)
|
|
1908
|
+
return _recoverable_codex_turn_failure_message(
|
|
1909
|
+
"codex_cli_goal_" + retryable_startup_blocker_stage
|
|
1910
|
+
)
|
|
1911
|
+
pre_bridge_blocker_stage = ""
|
|
1912
|
+
if bridge_summary_path is not None and not first_action_seen:
|
|
1913
|
+
pre_bridge_blocker_stage = (
|
|
1914
|
+
codex_cli_tui_pre_bridge_blocker_stage(
|
|
1915
|
+
capture,
|
|
1916
|
+
prompt_visible=(
|
|
1917
|
+
codex_cli_tui_input_prompt_visible(capture)
|
|
1918
|
+
),
|
|
1919
|
+
terminal_observed=goal_failed_now,
|
|
1920
|
+
)
|
|
1921
|
+
)
|
|
1922
|
+
if pre_bridge_blocker_stage:
|
|
1923
|
+
recovery_action = codex_cli_tui_pre_bridge_recovery_action(
|
|
1924
|
+
capture,
|
|
1925
|
+
stage=pre_bridge_blocker_stage,
|
|
1926
|
+
)
|
|
1927
|
+
if recovery_action in {
|
|
1928
|
+
"press_enter",
|
|
1929
|
+
"restart_tui_goal",
|
|
1930
|
+
"typed_goal_resubmit",
|
|
1931
|
+
} and (
|
|
1932
|
+
pre_bridge_recovery_attempt_count
|
|
1933
|
+
< PRE_BRIDGE_RECOVERY_ATTEMPT_LIMIT
|
|
1934
|
+
):
|
|
1935
|
+
pre_bridge_recovery_attempt_count += 1
|
|
1936
|
+
pre_bridge_recovery_action = recovery_action
|
|
1937
|
+
restart_stage = ""
|
|
1938
|
+
if recovery_action == "press_enter":
|
|
1939
|
+
tmux_submit_enter(tmux_name)
|
|
1940
|
+
else:
|
|
1941
|
+
if recovery_action == "restart_tui_goal":
|
|
1942
|
+
tmux_kill_session(tmux_name)
|
|
1943
|
+
(
|
|
1944
|
+
restart_stage,
|
|
1945
|
+
thread_prewarm_observed,
|
|
1946
|
+
) = start_codex_cli_goal_tui_session(
|
|
1947
|
+
tmux_name=tmux_name,
|
|
1948
|
+
cwd=cwd,
|
|
1949
|
+
shell_command=shell_command,
|
|
1950
|
+
thread_prewarm=thread_prewarm,
|
|
1951
|
+
thread_prewarm_timeout_sec=CODEX_CLI_GOAL_THREAD_PREWARM_TIMEOUT_SEC,
|
|
1952
|
+
)
|
|
1953
|
+
if not restart_stage:
|
|
1954
|
+
capture = tmux_capture(tmux_name)
|
|
1955
|
+
goal_terminal_observed = False
|
|
1956
|
+
goal_failed_observed = False
|
|
1957
|
+
if not restart_stage:
|
|
1958
|
+
goal_lifecycle.begin(capture)
|
|
1959
|
+
goal_active_observed = False
|
|
1960
|
+
goal_kickoff_prompt_submitted = False
|
|
1961
|
+
tmux_type_text_and_submit(
|
|
1962
|
+
tmux_name=tmux_name,
|
|
1963
|
+
text=goal_command_text,
|
|
1964
|
+
)
|
|
1965
|
+
if restart_stage:
|
|
1966
|
+
pre_bridge_blocker_stage = restart_stage
|
|
1967
|
+
pre_bridge_recovery_skip_reason = "restart_failed"
|
|
1968
|
+
else:
|
|
1969
|
+
deadlines = codex_cli_goal_reset_pre_bridge_deadlines(
|
|
1970
|
+
now=now,
|
|
1971
|
+
first_action_timeout_sec=self._config.first_action_timeout_sec,
|
|
1972
|
+
goal_active_deadline=goal_active_deadline,
|
|
1973
|
+
goal_active_timeout_sec=self._config.goal_active_timeout_sec,
|
|
1974
|
+
first_action_deadline=first_action_deadline,
|
|
1975
|
+
meaningful_progress_deadline=meaningful_progress_deadline,
|
|
1976
|
+
)
|
|
1977
|
+
goal_active_deadline, first_action_deadline, meaningful_progress_deadline = deadlines
|
|
1978
|
+
if recovery_action in {
|
|
1979
|
+
"restart_tui_goal",
|
|
1980
|
+
"typed_goal_resubmit",
|
|
1981
|
+
}:
|
|
1982
|
+
next_heartbeat = now + max(
|
|
1983
|
+
1.0,
|
|
1984
|
+
self._config.stream_heartbeat_interval_sec,
|
|
1985
|
+
)
|
|
1986
|
+
else:
|
|
1987
|
+
time.sleep(1.0)
|
|
1988
|
+
continue
|
|
1989
|
+
pre_bridge_recovery_skip_reason = pre_bridge_recovery_skip_reason or (
|
|
1990
|
+
codex_cli_tui_pre_bridge_recovery_skip_reason(
|
|
1991
|
+
capture,
|
|
1992
|
+
stage=pre_bridge_blocker_stage,
|
|
1993
|
+
recovery_action=recovery_action,
|
|
1994
|
+
)
|
|
1995
|
+
)
|
|
1996
|
+
if (
|
|
1997
|
+
not pre_bridge_recovery_skip_reason
|
|
1998
|
+
and recovery_action
|
|
1999
|
+
in {
|
|
2000
|
+
"press_enter",
|
|
2001
|
+
"restart_tui_goal",
|
|
2002
|
+
"typed_goal_resubmit",
|
|
2003
|
+
}
|
|
2004
|
+
):
|
|
2005
|
+
pre_bridge_recovery_skip_reason = "retry_limit_reached"
|
|
2006
|
+
tmux_kill_session(tmux_name)
|
|
2007
|
+
if bridge_summary_path is not None:
|
|
2008
|
+
self._publish_remote_bridge_agent_operations_trace(
|
|
2009
|
+
bridge_summary_path=bridge_summary_path,
|
|
2010
|
+
)
|
|
2011
|
+
self._publish_codex_cli_goal_trace(
|
|
2012
|
+
ok=False,
|
|
2013
|
+
stage=pre_bridge_blocker_stage,
|
|
2014
|
+
goal_active_observed=goal_active_observed,
|
|
2015
|
+
goal_terminal_observed=goal_terminal_observed,
|
|
2016
|
+
first_action_observed=first_action_seen,
|
|
2017
|
+
bridge_summary_path=bridge_summary_path,
|
|
2018
|
+
thread_prewarm_observed=thread_prewarm_observed,
|
|
2019
|
+
goal_prompt_file_used=goal_prompt_file_used,
|
|
2020
|
+
goal_command_submission_method=(
|
|
2021
|
+
goal_command_submission_method
|
|
2022
|
+
),
|
|
2023
|
+
goal_kickoff_prompt_submitted=goal_kickoff_prompt_submitted,
|
|
2024
|
+
post_bridge_recovery_attempt_count=pre_bridge_recovery_attempt_count,
|
|
2025
|
+
post_bridge_recovery_action=pre_bridge_recovery_action,
|
|
2026
|
+
post_bridge_recovery_skip_reason=pre_bridge_recovery_skip_reason,
|
|
2027
|
+
)
|
|
2028
|
+
return _recoverable_codex_turn_failure_message(
|
|
2029
|
+
"codex_cli_goal_" + pre_bridge_blocker_stage
|
|
2030
|
+
)
|
|
2031
|
+
if goal_failed_now:
|
|
2032
|
+
if bridge_summary_path is not None and not first_action_seen:
|
|
2033
|
+
prompt_visible = codex_cli_tui_input_prompt_visible(capture)
|
|
2034
|
+
pre_bridge_recovery_skip_reason = codex_cli_tui_pre_bridge_terminal_skip_reason(capture, prompt_visible=prompt_visible)
|
|
2035
|
+
pre_bridge_terminal_stage = codex_cli_tui_pre_bridge_terminal_stage(capture, prompt_visible=prompt_visible)
|
|
2036
|
+
elif bridge_summary_path is not None:
|
|
2037
|
+
post_bridge_terminal_stage = (
|
|
2038
|
+
codex_cli_tui_post_bridge_blocker_stage(
|
|
2039
|
+
capture,
|
|
2040
|
+
prompt_visible=(
|
|
2041
|
+
codex_cli_tui_input_prompt_visible(capture)
|
|
2042
|
+
),
|
|
2043
|
+
)
|
|
2044
|
+
)
|
|
2045
|
+
goal_terminal_observed = True
|
|
2046
|
+
goal_failed_observed = True
|
|
2047
|
+
break
|
|
2048
|
+
if (
|
|
2049
|
+
not goal_active_observed
|
|
2050
|
+
and not first_action_seen
|
|
2051
|
+
and goal_active_deadline
|
|
2052
|
+
and now >= goal_active_deadline
|
|
2053
|
+
):
|
|
2054
|
+
tmux_kill_session(tmux_name)
|
|
2055
|
+
if bridge_summary_path is not None:
|
|
2056
|
+
self._publish_remote_bridge_agent_operations_trace(
|
|
2057
|
+
bridge_summary_path=bridge_summary_path,
|
|
2058
|
+
)
|
|
2059
|
+
self._publish_codex_cli_goal_trace(
|
|
2060
|
+
ok=False,
|
|
2061
|
+
stage="goal_active_timeout",
|
|
2062
|
+
goal_active_observed=False,
|
|
2063
|
+
goal_terminal_observed=goal_terminal_observed,
|
|
2064
|
+
first_action_observed=False,
|
|
2065
|
+
bridge_summary_path=bridge_summary_path,
|
|
2066
|
+
thread_prewarm_observed=thread_prewarm_observed,
|
|
2067
|
+
goal_prompt_file_used=goal_prompt_file_used,
|
|
2068
|
+
goal_command_submission_method=(
|
|
2069
|
+
goal_command_submission_method
|
|
2070
|
+
),
|
|
2071
|
+
post_bridge_recovery_attempt_count=pre_bridge_recovery_attempt_count,
|
|
2072
|
+
post_bridge_recovery_action=pre_bridge_recovery_action,
|
|
2073
|
+
post_bridge_recovery_skip_reason=pre_bridge_recovery_skip_reason,
|
|
2074
|
+
)
|
|
2075
|
+
return _recoverable_codex_turn_failure_message(
|
|
2076
|
+
"codex_cli_goal_goal_active_timeout"
|
|
2077
|
+
)
|
|
2078
|
+
if (
|
|
2079
|
+
not first_action_seen
|
|
2080
|
+
and codex_cli_goal_watchdog_expired(
|
|
2081
|
+
deadline=first_action_deadline,
|
|
2082
|
+
now=now,
|
|
2083
|
+
turn_active=turn_active,
|
|
2084
|
+
)
|
|
2085
|
+
):
|
|
2086
|
+
tmux_kill_session(tmux_name)
|
|
2087
|
+
if bridge_summary_path is not None:
|
|
2088
|
+
self._publish_remote_bridge_agent_operations_trace(
|
|
2089
|
+
bridge_summary_path=bridge_summary_path,
|
|
2090
|
+
)
|
|
2091
|
+
self._publish_codex_cli_goal_trace(
|
|
2092
|
+
ok=False,
|
|
2093
|
+
stage="first_action_timeout",
|
|
2094
|
+
goal_active_observed=goal_active_observed,
|
|
2095
|
+
goal_terminal_observed=goal_terminal_observed,
|
|
2096
|
+
first_action_observed=False,
|
|
2097
|
+
bridge_summary_path=bridge_summary_path,
|
|
2098
|
+
thread_prewarm_observed=thread_prewarm_observed,
|
|
2099
|
+
goal_prompt_file_used=goal_prompt_file_used,
|
|
2100
|
+
goal_command_submission_method=(
|
|
2101
|
+
goal_command_submission_method
|
|
2102
|
+
),
|
|
2103
|
+
goal_kickoff_prompt_submitted=(
|
|
2104
|
+
goal_kickoff_prompt_submitted
|
|
2105
|
+
),
|
|
2106
|
+
)
|
|
2107
|
+
return _recoverable_codex_turn_failure_message(
|
|
2108
|
+
"codex_exec_first_action_timeout"
|
|
2109
|
+
)
|
|
2110
|
+
if (
|
|
2111
|
+
meaningful_progress_required
|
|
2112
|
+
and not meaningful_progress_seen
|
|
2113
|
+
and codex_cli_goal_watchdog_expired(
|
|
2114
|
+
deadline=meaningful_progress_deadline,
|
|
2115
|
+
now=now,
|
|
2116
|
+
turn_active=turn_active,
|
|
2117
|
+
)
|
|
2118
|
+
):
|
|
2119
|
+
tmux_kill_session(tmux_name)
|
|
2120
|
+
if bridge_summary_path is not None:
|
|
2121
|
+
self._publish_remote_bridge_agent_operations_trace(
|
|
2122
|
+
bridge_summary_path=bridge_summary_path,
|
|
2123
|
+
)
|
|
2124
|
+
self._publish_codex_cli_goal_trace(
|
|
2125
|
+
ok=False,
|
|
2126
|
+
stage="meaningful_bridge_progress_timeout",
|
|
2127
|
+
goal_active_observed=goal_active_observed,
|
|
2128
|
+
goal_terminal_observed=goal_terminal_observed,
|
|
2129
|
+
first_action_observed=first_action_seen,
|
|
2130
|
+
bridge_summary_path=bridge_summary_path,
|
|
2131
|
+
thread_prewarm_observed=thread_prewarm_observed,
|
|
2132
|
+
goal_prompt_file_used=goal_prompt_file_used,
|
|
2133
|
+
goal_command_submission_method=(
|
|
2134
|
+
goal_command_submission_method
|
|
2135
|
+
),
|
|
2136
|
+
)
|
|
2137
|
+
return _recoverable_codex_turn_failure_message(
|
|
2138
|
+
"codex_exec_first_action_timeout"
|
|
2139
|
+
)
|
|
2140
|
+
post_bridge_blocker_stage = ""
|
|
2141
|
+
if (
|
|
2142
|
+
bridge_activity_seen
|
|
2143
|
+
and bridge_summary_path is not None
|
|
2144
|
+
and not _bridge_summary_has_inflight_operation(
|
|
2145
|
+
bridge_summary_path
|
|
2146
|
+
) and (not post_bridge_recovery_attempt_count or now - last_bridge_activity_at >= 30.0)
|
|
2147
|
+
):
|
|
2148
|
+
post_bridge_blocker_stage = (
|
|
2149
|
+
codex_cli_tui_post_bridge_blocker_stage(
|
|
2150
|
+
capture,
|
|
2151
|
+
prompt_visible=(
|
|
2152
|
+
codex_cli_tui_input_prompt_visible(capture)
|
|
2153
|
+
),
|
|
2154
|
+
)
|
|
2155
|
+
)
|
|
2156
|
+
if post_bridge_blocker_stage:
|
|
2157
|
+
recovery_action = codex_cli_tui_post_bridge_recovery_action(
|
|
2158
|
+
capture,
|
|
2159
|
+
stage=post_bridge_blocker_stage,
|
|
2160
|
+
)
|
|
2161
|
+
if recovery_action == "press_enter" and post_bridge_recovery_attempt_count < POST_BRIDGE_RECOVERY_ATTEMPT_LIMIT:
|
|
2162
|
+
post_bridge_recovery_attempt_count += 1
|
|
2163
|
+
post_bridge_recovery_action = recovery_action
|
|
2164
|
+
tmux_submit_enter(tmux_name)
|
|
2165
|
+
last_bridge_activity_at = now
|
|
2166
|
+
next_heartbeat = (
|
|
2167
|
+
now
|
|
2168
|
+
+ max(1.0, self._config.stream_heartbeat_interval_sec)
|
|
2169
|
+
)
|
|
2170
|
+
time.sleep(1.0)
|
|
2171
|
+
continue
|
|
2172
|
+
if recovery_action == "typed_continue" and post_bridge_recovery_attempt_count < POST_BRIDGE_RECOVERY_ATTEMPT_LIMIT:
|
|
2173
|
+
post_bridge_recovery_attempt_count += 1
|
|
2174
|
+
post_bridge_recovery_action = recovery_action
|
|
2175
|
+
tmux_type_text_and_submit(
|
|
2176
|
+
tmux_name=tmux_name,
|
|
2177
|
+
text=CODEX_CLI_GOAL_POST_BRIDGE_CONTINUE_PROMPT,
|
|
2178
|
+
)
|
|
2179
|
+
last_bridge_activity_at = now
|
|
2180
|
+
next_heartbeat = (
|
|
2181
|
+
now
|
|
2182
|
+
+ max(1.0, self._config.stream_heartbeat_interval_sec)
|
|
2183
|
+
)
|
|
2184
|
+
continue
|
|
2185
|
+
post_bridge_recovery_skip_reason = (
|
|
2186
|
+
codex_cli_tui_post_bridge_recovery_skip_reason(
|
|
2187
|
+
capture,
|
|
2188
|
+
stage=post_bridge_blocker_stage,
|
|
2189
|
+
recovery_action=recovery_action,
|
|
2190
|
+
)
|
|
2191
|
+
)
|
|
2192
|
+
if (
|
|
2193
|
+
not post_bridge_recovery_skip_reason
|
|
2194
|
+
and recovery_action in {"press_enter", "typed_continue"}
|
|
2195
|
+
):
|
|
2196
|
+
post_bridge_recovery_skip_reason = "retry_limit_reached"
|
|
2197
|
+
tmux_kill_session(tmux_name)
|
|
2198
|
+
self._publish_remote_bridge_agent_operations_trace(
|
|
2199
|
+
bridge_summary_path=bridge_summary_path,
|
|
2200
|
+
)
|
|
2201
|
+
self._publish_codex_cli_goal_trace(
|
|
2202
|
+
ok=False,
|
|
2203
|
+
stage=post_bridge_blocker_stage,
|
|
2204
|
+
goal_active_observed=goal_active_observed,
|
|
2205
|
+
goal_terminal_observed=goal_terminal_observed,
|
|
2206
|
+
first_action_observed=first_action_seen,
|
|
2207
|
+
bridge_summary_path=bridge_summary_path,
|
|
2208
|
+
thread_prewarm_observed=thread_prewarm_observed,
|
|
2209
|
+
goal_prompt_file_used=goal_prompt_file_used,
|
|
2210
|
+
goal_command_submission_method=(
|
|
2211
|
+
goal_command_submission_method
|
|
2212
|
+
),
|
|
2213
|
+
post_bridge_recovery_attempt_count=(
|
|
2214
|
+
pre_bridge_recovery_attempt_count
|
|
2215
|
+
+ post_bridge_recovery_attempt_count
|
|
2216
|
+
),
|
|
2217
|
+
post_bridge_recovery_action=(
|
|
2218
|
+
post_bridge_recovery_action
|
|
2219
|
+
or pre_bridge_recovery_action
|
|
2220
|
+
),
|
|
2221
|
+
post_bridge_recovery_skip_reason=(
|
|
2222
|
+
post_bridge_recovery_skip_reason
|
|
2223
|
+
or pre_bridge_recovery_skip_reason
|
|
2224
|
+
),
|
|
2225
|
+
)
|
|
2226
|
+
return _recoverable_codex_turn_failure_message(
|
|
2227
|
+
"codex_cli_goal_" + post_bridge_blocker_stage
|
|
2228
|
+
)
|
|
2229
|
+
if (
|
|
2230
|
+
task_output_progress_seen
|
|
2231
|
+
and bridge_summary_path is not None
|
|
2232
|
+
and task_output_quiet_timeout_sec > 0
|
|
2233
|
+
and not turn_active
|
|
2234
|
+
and not _bridge_summary_has_inflight_operation(
|
|
2235
|
+
bridge_summary_path
|
|
2236
|
+
)
|
|
2237
|
+
and now - last_task_output_activity_at
|
|
2238
|
+
>= task_output_quiet_timeout_sec
|
|
2239
|
+
):
|
|
2240
|
+
tmux_kill_session(tmux_name)
|
|
2241
|
+
self._publish_remote_bridge_agent_operations_trace(
|
|
2242
|
+
bridge_summary_path=bridge_summary_path,
|
|
2243
|
+
)
|
|
2244
|
+
self._publish_codex_cli_goal_trace(
|
|
2245
|
+
ok=False,
|
|
2246
|
+
stage="task_output_quiet_timeout",
|
|
2247
|
+
goal_active_observed=goal_active_observed,
|
|
2248
|
+
goal_terminal_observed=goal_terminal_observed,
|
|
2249
|
+
first_action_observed=first_action_seen,
|
|
2250
|
+
bridge_summary_path=bridge_summary_path,
|
|
2251
|
+
thread_prewarm_observed=thread_prewarm_observed,
|
|
2252
|
+
goal_prompt_file_used=goal_prompt_file_used,
|
|
2253
|
+
goal_command_submission_method=(
|
|
2254
|
+
goal_command_submission_method
|
|
2255
|
+
),
|
|
2256
|
+
post_bridge_recovery_attempt_count=(
|
|
2257
|
+
pre_bridge_recovery_attempt_count
|
|
2258
|
+
+ post_bridge_recovery_attempt_count
|
|
2259
|
+
),
|
|
2260
|
+
post_bridge_recovery_action=(
|
|
2261
|
+
post_bridge_recovery_action
|
|
2262
|
+
or pre_bridge_recovery_action
|
|
2263
|
+
),
|
|
2264
|
+
post_bridge_recovery_skip_reason=(
|
|
2265
|
+
post_bridge_recovery_skip_reason
|
|
2266
|
+
or pre_bridge_recovery_skip_reason
|
|
2267
|
+
),
|
|
2268
|
+
)
|
|
2269
|
+
return _recoverable_codex_turn_failure_message(
|
|
2270
|
+
"codex_exec_task_output_quiet_timeout"
|
|
2271
|
+
)
|
|
2272
|
+
if (
|
|
2273
|
+
bridge_activity_seen
|
|
2274
|
+
and bridge_summary_path is not None
|
|
2275
|
+
and bridge_idle_timeout_sec > 0
|
|
2276
|
+
and not _bridge_summary_has_inflight_operation(
|
|
2277
|
+
bridge_summary_path
|
|
2278
|
+
)
|
|
2279
|
+
and now - last_bridge_activity_at >= bridge_idle_timeout_sec
|
|
2280
|
+
):
|
|
2281
|
+
tmux_kill_session(tmux_name)
|
|
2282
|
+
self._publish_remote_bridge_agent_operations_trace(
|
|
2283
|
+
bridge_summary_path=bridge_summary_path,
|
|
2284
|
+
)
|
|
2285
|
+
self._publish_codex_cli_goal_trace(
|
|
2286
|
+
ok=False,
|
|
2287
|
+
stage="bridge_idle_timeout",
|
|
2288
|
+
goal_active_observed=goal_active_observed,
|
|
2289
|
+
goal_terminal_observed=goal_terminal_observed,
|
|
2290
|
+
first_action_observed=first_action_seen,
|
|
2291
|
+
bridge_summary_path=bridge_summary_path,
|
|
2292
|
+
thread_prewarm_observed=thread_prewarm_observed,
|
|
2293
|
+
goal_prompt_file_used=goal_prompt_file_used,
|
|
2294
|
+
goal_command_submission_method=(
|
|
2295
|
+
goal_command_submission_method
|
|
2296
|
+
),
|
|
2297
|
+
post_bridge_recovery_attempt_count=(
|
|
2298
|
+
pre_bridge_recovery_attempt_count
|
|
2299
|
+
+ post_bridge_recovery_attempt_count
|
|
2300
|
+
),
|
|
2301
|
+
post_bridge_recovery_action=(
|
|
2302
|
+
post_bridge_recovery_action
|
|
2303
|
+
or pre_bridge_recovery_action
|
|
2304
|
+
),
|
|
2305
|
+
post_bridge_recovery_skip_reason=(
|
|
2306
|
+
post_bridge_recovery_skip_reason
|
|
2307
|
+
or pre_bridge_recovery_skip_reason
|
|
2308
|
+
),
|
|
2309
|
+
)
|
|
2310
|
+
return _recoverable_codex_turn_failure_message(
|
|
2311
|
+
"codex_exec_bridge_idle_timeout"
|
|
2312
|
+
)
|
|
2313
|
+
if now >= next_heartbeat:
|
|
2314
|
+
self._write_worker_heartbeat(
|
|
2315
|
+
stdout,
|
|
2316
|
+
session_id=session_id,
|
|
2317
|
+
text="local codex cli /goal still running",
|
|
2318
|
+
)
|
|
2319
|
+
next_heartbeat = (
|
|
2320
|
+
now
|
|
2321
|
+
+ max(1.0, self._config.stream_heartbeat_interval_sec)
|
|
2322
|
+
)
|
|
2323
|
+
time.sleep(0.5)
|
|
2324
|
+
if bridge_summary_path is not None:
|
|
2325
|
+
self._publish_remote_bridge_agent_operations_trace(
|
|
2326
|
+
bridge_summary_path=bridge_summary_path,
|
|
2327
|
+
)
|
|
2328
|
+
self._publish_codex_cli_goal_trace(
|
|
2329
|
+
ok=bool(goal_terminal_observed and not goal_failed_observed),
|
|
2330
|
+
stage=(
|
|
2331
|
+
pre_bridge_terminal_stage
|
|
2332
|
+
or post_bridge_terminal_stage
|
|
2333
|
+
or (
|
|
2334
|
+
"goal_achieved"
|
|
2335
|
+
if goal_terminal_observed and not goal_failed_observed
|
|
2336
|
+
else "goal_failed"
|
|
2337
|
+
if goal_failed_observed
|
|
2338
|
+
else "timeout"
|
|
2339
|
+
)
|
|
2340
|
+
),
|
|
2341
|
+
goal_active_observed=goal_active_observed,
|
|
2342
|
+
goal_terminal_observed=goal_terminal_observed,
|
|
2343
|
+
first_action_observed=first_action_seen,
|
|
2344
|
+
bridge_summary_path=bridge_summary_path,
|
|
2345
|
+
thread_prewarm_observed=thread_prewarm_observed,
|
|
2346
|
+
goal_prompt_file_used=goal_prompt_file_used,
|
|
2347
|
+
goal_command_submission_method=goal_command_submission_method,
|
|
2348
|
+
goal_kickoff_prompt_submitted=goal_kickoff_prompt_submitted,
|
|
2349
|
+
post_bridge_recovery_attempt_count=(
|
|
2350
|
+
pre_bridge_recovery_attempt_count
|
|
2351
|
+
+ post_bridge_recovery_attempt_count
|
|
2352
|
+
),
|
|
2353
|
+
post_bridge_recovery_action=(
|
|
2354
|
+
post_bridge_recovery_action or pre_bridge_recovery_action
|
|
2355
|
+
),
|
|
2356
|
+
post_bridge_recovery_skip_reason=(
|
|
2357
|
+
post_bridge_recovery_skip_reason
|
|
2358
|
+
or pre_bridge_recovery_skip_reason
|
|
2359
|
+
),
|
|
2360
|
+
)
|
|
2361
|
+
if goal_terminal_observed and not goal_failed_observed:
|
|
2362
|
+
return "codex cli /goal completed"
|
|
2363
|
+
if pre_bridge_terminal_stage:
|
|
2364
|
+
return _recoverable_codex_turn_failure_message("codex_cli_goal_" + pre_bridge_terminal_stage)
|
|
2365
|
+
if post_bridge_terminal_stage:
|
|
2366
|
+
return _recoverable_codex_turn_failure_message(
|
|
2367
|
+
"codex_cli_goal_" + post_bridge_terminal_stage
|
|
2368
|
+
)
|
|
2369
|
+
if goal_failed_observed:
|
|
2370
|
+
return _recoverable_codex_turn_failure_message(
|
|
2371
|
+
"codex_exec_failed"
|
|
2372
|
+
)
|
|
2373
|
+
return _recoverable_codex_turn_failure_message("codex_exec_timeout")
|
|
2374
|
+
except subprocess.SubprocessError as exc:
|
|
2375
|
+
if bridge_summary_path is not None:
|
|
2376
|
+
self._publish_remote_bridge_agent_operations_trace(
|
|
2377
|
+
bridge_summary_path=bridge_summary_path,
|
|
2378
|
+
)
|
|
2379
|
+
self._publish_codex_cli_goal_trace(
|
|
2380
|
+
ok=False,
|
|
2381
|
+
stage="tmux_or_input_failed",
|
|
2382
|
+
goal_active_observed=goal_active_observed,
|
|
2383
|
+
goal_terminal_observed=goal_terminal_observed,
|
|
2384
|
+
first_action_observed=first_action_seen,
|
|
2385
|
+
bridge_summary_path=bridge_summary_path,
|
|
2386
|
+
thread_prewarm_observed=False,
|
|
2387
|
+
goal_prompt_file_used=goal_prompt_file_used,
|
|
2388
|
+
goal_command_submission_method=goal_command_submission_method,
|
|
2389
|
+
goal_slash_command_submitted=goal_slash_command_submitted,
|
|
2390
|
+
)
|
|
2391
|
+
raise RuntimeError("codex cli goal worker failed before run") from exc
|
|
2392
|
+
finally:
|
|
2393
|
+
tmux_kill_session(tmux_name)
|
|
2394
|
+
self._terminate_bridge_server_process(bridge_server_proc)
|
|
2395
|
+
|
|
2396
|
+
def _publish_codex_cli_goal_trace(
|
|
2397
|
+
self,
|
|
2398
|
+
*,
|
|
2399
|
+
ok: bool,
|
|
2400
|
+
stage: str,
|
|
2401
|
+
goal_active_observed: bool,
|
|
2402
|
+
goal_terminal_observed: bool,
|
|
2403
|
+
first_action_observed: bool,
|
|
2404
|
+
bridge_summary_path: Path | None,
|
|
2405
|
+
thread_prewarm_observed: bool = False,
|
|
2406
|
+
goal_prompt_file_used: bool = False,
|
|
2407
|
+
goal_command_submission_method: str = "",
|
|
2408
|
+
goal_slash_command_submitted: bool = True,
|
|
2409
|
+
goal_kickoff_prompt_submitted: bool = False,
|
|
2410
|
+
post_bridge_recovery_attempt_count: int = 0,
|
|
2411
|
+
post_bridge_recovery_action: str = "",
|
|
2412
|
+
post_bridge_recovery_skip_reason: str = "",
|
|
2413
|
+
) -> None:
|
|
2414
|
+
if not self._config.worker_public_trace_dir:
|
|
2415
|
+
return
|
|
2416
|
+
safe_stage = "".join(
|
|
2417
|
+
ch if ch.isalnum() or ch == "_" else "_"
|
|
2418
|
+
for ch in str(stage or "").strip().lower()
|
|
2419
|
+
) or "unknown"
|
|
2420
|
+
private_tui_tail = write_private_codex_cli_goal_tui_tail(self._config.worker_public_trace_dir, safe_stage, getattr(self, "_last_codex_cli_goal_tui_capture", ""))
|
|
2421
|
+
bridge_request_count = 0
|
|
2422
|
+
task_facing_success_count = 0
|
|
2423
|
+
if bridge_summary_path is not None and bridge_summary_path.exists():
|
|
2424
|
+
try:
|
|
2425
|
+
lines = bridge_summary_path.read_text(
|
|
2426
|
+
encoding="utf-8", errors="replace"
|
|
2427
|
+
).splitlines()
|
|
2428
|
+
except OSError:
|
|
2429
|
+
lines = []
|
|
2430
|
+
for line in lines:
|
|
2431
|
+
try:
|
|
2432
|
+
record = json.loads(line)
|
|
2433
|
+
except json.JSONDecodeError:
|
|
2434
|
+
continue
|
|
2435
|
+
if not isinstance(record, dict):
|
|
2436
|
+
continue
|
|
2437
|
+
if str(record.get("record_phase") or "").lower() == "start":
|
|
2438
|
+
bridge_request_count += 1
|
|
2439
|
+
if (
|
|
2440
|
+
str(record.get("record_phase") or "").lower() == "complete"
|
|
2441
|
+
and record.get("task_facing_operation") is True
|
|
2442
|
+
and (record.get("success") is True or record.get("returncode") == 0)
|
|
2443
|
+
):
|
|
2444
|
+
task_facing_success_count += 1
|
|
2445
|
+
trace = {
|
|
2446
|
+
"schema_version": "skillsbench_host_local_acp_relay_public_trace_v0",
|
|
2447
|
+
"ok": bool(ok),
|
|
2448
|
+
"route": self._config.route,
|
|
2449
|
+
"trace_kind": "codex_cli_goal_tui",
|
|
2450
|
+
"benchmark_id": self._config.dataset,
|
|
2451
|
+
"task_id": self._config.task_id,
|
|
2452
|
+
"codex_cli_goal": {
|
|
2453
|
+
"schema_version": "skillsbench_codex_cli_goal_tui_v0",
|
|
2454
|
+
"stage": safe_stage,
|
|
2455
|
+
"goal_slash_command_submitted": bool(goal_slash_command_submitted),
|
|
2456
|
+
"goal_thread_prewarm_observed": bool(thread_prewarm_observed),
|
|
2457
|
+
"goal_thread_prewarm_timeout_sec": CODEX_CLI_GOAL_THREAD_PREWARM_TIMEOUT_SEC if self._config.codex_cli_goal_thread_prewarm else 0,
|
|
2458
|
+
"goal_prompt_file_used": bool(goal_prompt_file_used),
|
|
2459
|
+
"goal_prompt_file_raw_path_recorded": False,
|
|
2460
|
+
"goal_command_submission_method": str(
|
|
2461
|
+
goal_command_submission_method or ""
|
|
2462
|
+
)[:40],
|
|
2463
|
+
"goal_active_observed": bool(goal_active_observed),
|
|
2464
|
+
"goal_terminal_observed": bool(goal_terminal_observed),
|
|
2465
|
+
"goal_kickoff_prompt_submitted": bool(
|
|
2466
|
+
goal_kickoff_prompt_submitted
|
|
2467
|
+
),
|
|
2468
|
+
"goal_kickoff_prompt_raw_text_recorded": False,
|
|
2469
|
+
**getattr(self, "_last_codex_cli_goal_lifecycle", CodexCliGoalLifecycleGeneration()).trace_fields(),
|
|
2470
|
+
"first_action_observed": bool(first_action_observed),
|
|
2471
|
+
"bridge_request_count": bridge_request_count,
|
|
2472
|
+
"task_facing_success_count": task_facing_success_count,
|
|
2473
|
+
"post_bridge_recovery_attempt_count": max(
|
|
2474
|
+
0,
|
|
2475
|
+
int(post_bridge_recovery_attempt_count or 0),
|
|
2476
|
+
),
|
|
2477
|
+
"post_bridge_recovery_action": str(
|
|
2478
|
+
post_bridge_recovery_action or ""
|
|
2479
|
+
)[:40],
|
|
2480
|
+
"post_bridge_recovery_skip_reason": str(
|
|
2481
|
+
post_bridge_recovery_skip_reason or ""
|
|
2482
|
+
)[:80],
|
|
2483
|
+
"reasoning_effort": str(self._config.reasoning_effort or "")[:40],
|
|
2484
|
+
"codex_api_proxy_env_injected": bool(
|
|
2485
|
+
codex_cli_tui_environment(self._config.codex_api_proxy)
|
|
2486
|
+
),
|
|
2487
|
+
"codex_api_proxy_raw_url_recorded": False,
|
|
2488
|
+
**private_tui_tail,
|
|
2489
|
+
"raw_tui_capture_recorded": False,
|
|
2490
|
+
"raw_task_text_recorded": False,
|
|
2491
|
+
"raw_stdout_recorded": False,
|
|
2492
|
+
"raw_stderr_recorded": False,
|
|
2493
|
+
"credential_values_recorded": False,
|
|
2494
|
+
},
|
|
2495
|
+
"boundary": {
|
|
2496
|
+
"raw_command_recorded": False,
|
|
2497
|
+
"raw_stdout_recorded": False,
|
|
2498
|
+
"raw_stderr_recorded": False,
|
|
2499
|
+
"raw_task_text_recorded": False,
|
|
2500
|
+
"raw_logs_recorded": False,
|
|
2501
|
+
"raw_trajectory_recorded": False,
|
|
2502
|
+
"credential_values_recorded": False,
|
|
2503
|
+
"host_paths_recorded": False,
|
|
2504
|
+
"remote_paths_recorded": False,
|
|
2505
|
+
"upload_performed": False,
|
|
2506
|
+
"submit_performed": False,
|
|
2507
|
+
},
|
|
2508
|
+
}
|
|
2509
|
+
self._write_worker_public_trace(trace)
|
|
2510
|
+
|
|
2511
|
+
def _terminate_codex_process(
|
|
2512
|
+
self,
|
|
2513
|
+
proc: subprocess.Popen[str],
|
|
2514
|
+
*,
|
|
2515
|
+
grace_sec: float = 5.0,
|
|
2516
|
+
) -> None:
|
|
2517
|
+
if proc.poll() is not None:
|
|
2518
|
+
return
|
|
2519
|
+
try:
|
|
2520
|
+
os.killpg(proc.pid, signal.SIGTERM)
|
|
2521
|
+
except (AttributeError, ProcessLookupError, PermissionError, OSError):
|
|
2522
|
+
try:
|
|
2523
|
+
proc.terminate()
|
|
2524
|
+
except (ProcessLookupError, PermissionError, OSError):
|
|
2525
|
+
pass
|
|
2526
|
+
try:
|
|
2527
|
+
proc.wait(timeout=grace_sec)
|
|
2528
|
+
return
|
|
2529
|
+
except subprocess.TimeoutExpired:
|
|
2530
|
+
pass
|
|
2531
|
+
try:
|
|
2532
|
+
os.killpg(proc.pid, signal.SIGKILL)
|
|
2533
|
+
except (AttributeError, ProcessLookupError, PermissionError, OSError):
|
|
2534
|
+
try:
|
|
2535
|
+
proc.kill()
|
|
2536
|
+
except (ProcessLookupError, PermissionError, OSError):
|
|
2537
|
+
pass
|
|
2538
|
+
try:
|
|
2539
|
+
proc.wait(timeout=grace_sec)
|
|
2540
|
+
except subprocess.TimeoutExpired:
|
|
2541
|
+
pass
|
|
2542
|
+
|
|
2543
|
+
def _terminate_bridge_server_process(
|
|
2544
|
+
self,
|
|
2545
|
+
proc: subprocess.Popen[str] | None,
|
|
2546
|
+
*,
|
|
2547
|
+
grace_sec: float = 2.0,
|
|
2548
|
+
) -> None:
|
|
2549
|
+
if proc is None or proc.poll() is not None:
|
|
2550
|
+
return
|
|
2551
|
+
try:
|
|
2552
|
+
proc.terminate()
|
|
2553
|
+
except (ProcessLookupError, PermissionError, OSError):
|
|
2554
|
+
return
|
|
2555
|
+
try:
|
|
2556
|
+
proc.wait(timeout=grace_sec)
|
|
2557
|
+
return
|
|
2558
|
+
except subprocess.TimeoutExpired:
|
|
2559
|
+
pass
|
|
2560
|
+
try:
|
|
2561
|
+
proc.kill()
|
|
2562
|
+
except (ProcessLookupError, PermissionError, OSError):
|
|
2563
|
+
return
|
|
2564
|
+
try:
|
|
2565
|
+
proc.wait(timeout=grace_sec)
|
|
2566
|
+
except subprocess.TimeoutExpired:
|
|
2567
|
+
pass
|
|
2568
|
+
|
|
2569
|
+
def _start_json_file_bridge_server(
|
|
2570
|
+
self,
|
|
2571
|
+
*,
|
|
2572
|
+
tmp_path: Path,
|
|
2573
|
+
local_cwd: Path,
|
|
2574
|
+
bridge_command: str,
|
|
2575
|
+
) -> tuple[str, subprocess.Popen[str] | None]:
|
|
2576
|
+
if not bridge_command or not REVERSE_CHANNEL_BRIDGE_SCRIPT.exists():
|
|
2577
|
+
return bridge_command, None
|
|
2578
|
+
queue_dir = local_cwd / "loopx-reverse-channel-queue"
|
|
2579
|
+
client_path = local_cwd / "loopx-json-file-bridge"
|
|
2580
|
+
subprocess.run(
|
|
2581
|
+
[
|
|
2582
|
+
sys.executable,
|
|
2583
|
+
str(REVERSE_CHANNEL_BRIDGE_SCRIPT),
|
|
2584
|
+
"write-client",
|
|
2585
|
+
"--kind",
|
|
2586
|
+
"json-file",
|
|
2587
|
+
"--queue-dir",
|
|
2588
|
+
str(queue_dir),
|
|
2589
|
+
"--output",
|
|
2590
|
+
str(client_path),
|
|
2591
|
+
],
|
|
2592
|
+
stdout=subprocess.PIPE,
|
|
2593
|
+
stderr=subprocess.PIPE,
|
|
2594
|
+
text=True,
|
|
2595
|
+
check=True,
|
|
2596
|
+
)
|
|
2597
|
+
proc = subprocess.Popen(
|
|
2598
|
+
[
|
|
2599
|
+
sys.executable,
|
|
2600
|
+
str(REVERSE_CHANNEL_BRIDGE_SCRIPT),
|
|
2601
|
+
"serve-json-file",
|
|
2602
|
+
"--queue-dir",
|
|
2603
|
+
str(queue_dir),
|
|
2604
|
+
"--bridge-command",
|
|
2605
|
+
bridge_command,
|
|
2606
|
+
"--timeout-sec",
|
|
2607
|
+
str(max(1.0, float(self._config.remote_command_file_bridge_timeout_sec))),
|
|
2608
|
+
],
|
|
2609
|
+
stdin=subprocess.DEVNULL,
|
|
2610
|
+
stdout=subprocess.DEVNULL,
|
|
2611
|
+
stderr=subprocess.DEVNULL,
|
|
2612
|
+
text=True,
|
|
2613
|
+
cwd=str(tmp_path),
|
|
2614
|
+
)
|
|
2615
|
+
return shlex.quote(str(client_path)), proc
|
|
2616
|
+
|
|
2617
|
+
def _consume_remote_bridge_for_solver(self) -> dict[str, Any]:
|
|
2618
|
+
return run_skillsbench_remote_command_file_bridge_probe(
|
|
2619
|
+
self._config.remote_command_file_bridge_command,
|
|
2620
|
+
timeout_sec=self._config.remote_command_file_bridge_timeout_sec,
|
|
2621
|
+
)
|
|
2622
|
+
|
|
2623
|
+
def _reverse_channel_json_preflight_probe(self) -> dict[str, Any]:
|
|
2624
|
+
return {
|
|
2625
|
+
"schema_version": "skillsbench_remote_command_file_bridge_probe_v0",
|
|
2626
|
+
"ready": True,
|
|
2627
|
+
"first_blocker": "skillsbench_reverse_channel_json_preflight_ready",
|
|
2628
|
+
"stage": "pre_sandbox_reverse_channel_json",
|
|
2629
|
+
"elapsed_ms": 0,
|
|
2630
|
+
"bridge_command_invoked": False,
|
|
2631
|
+
"response_schema_version": (
|
|
2632
|
+
"skillsbench_reverse_channel_json_preflight_response_v0"
|
|
2633
|
+
),
|
|
2634
|
+
"required_operations": ["preflight"],
|
|
2635
|
+
"operation_count": 1,
|
|
2636
|
+
"operations": [
|
|
2637
|
+
{
|
|
2638
|
+
"kind": "preflight",
|
|
2639
|
+
"label": "reverse_channel_json_bridge",
|
|
2640
|
+
"status": "ok",
|
|
2641
|
+
}
|
|
2642
|
+
],
|
|
2643
|
+
"missing_operations": [],
|
|
2644
|
+
"failed_operations": [],
|
|
2645
|
+
"boundary_violations": [],
|
|
2646
|
+
"raw_command_recorded": False,
|
|
2647
|
+
"raw_stdout_recorded": False,
|
|
2648
|
+
"raw_stderr_recorded": False,
|
|
2649
|
+
"raw_task_text_recorded": False,
|
|
2650
|
+
"raw_logs_recorded": False,
|
|
2651
|
+
"raw_trajectory_recorded": False,
|
|
2652
|
+
"credential_values_recorded": False,
|
|
2653
|
+
"host_paths_recorded": False,
|
|
2654
|
+
"remote_paths_recorded": False,
|
|
2655
|
+
"upload_performed": False,
|
|
2656
|
+
"submit_performed": False,
|
|
2657
|
+
}
|
|
2658
|
+
|
|
2659
|
+
def _run_remote_bridge_exec(self, command: str) -> dict[str, Any]:
|
|
2660
|
+
bridge_command = self._config.remote_command_file_bridge_command or ""
|
|
2661
|
+
request = {
|
|
2662
|
+
"operation": "exec",
|
|
2663
|
+
"cwd": "/app",
|
|
2664
|
+
"command": command,
|
|
2665
|
+
"timeout_sec": max(10.0, self._config.remote_command_file_bridge_timeout_sec),
|
|
2666
|
+
}
|
|
2667
|
+
started = time.monotonic()
|
|
2668
|
+
proc = subprocess.run(
|
|
2669
|
+
bridge_command,
|
|
2670
|
+
input=json.dumps(request),
|
|
2671
|
+
text=True,
|
|
2672
|
+
stdout=subprocess.PIPE,
|
|
2673
|
+
stderr=subprocess.PIPE,
|
|
2674
|
+
shell=True,
|
|
2675
|
+
timeout=max(10.0, self._config.remote_command_file_bridge_timeout_sec + 5),
|
|
2676
|
+
check=False,
|
|
2677
|
+
)
|
|
2678
|
+
return {
|
|
2679
|
+
"returncode": int(proc.returncode),
|
|
2680
|
+
"stdout_bytes": len((proc.stdout or "").encode("utf-8")),
|
|
2681
|
+
"stderr_bytes": len((proc.stderr or "").encode("utf-8")),
|
|
2682
|
+
"elapsed_ms": int((time.monotonic() - started) * 1000),
|
|
2683
|
+
"raw_stdout_recorded": False,
|
|
2684
|
+
"raw_stderr_recorded": False,
|
|
2685
|
+
"raw_command_recorded": False,
|
|
2686
|
+
}
|
|
2687
|
+
|
|
2688
|
+
def _run_loopx_workflow_lifecycle_checkpoint(
|
|
2689
|
+
self,
|
|
2690
|
+
*,
|
|
2691
|
+
checkpoint_index: int,
|
|
2692
|
+
) -> None:
|
|
2693
|
+
case_goal_id = self._config.loopx_case_goal_id or benchmark_case_goal_id(
|
|
2694
|
+
self._config.task_id
|
|
2695
|
+
)
|
|
2696
|
+
case_agent_id = self._config.loopx_case_agent_id or BENCHMARK_CASE_LOOPX_AGENT_ID
|
|
2697
|
+
case_todo_id = self._config.loopx_case_todo_id or BENCHMARK_CASE_LOOPX_TODO_ID
|
|
2698
|
+
cli_prefix = benchmark_case_loopx_command_prefix(
|
|
2699
|
+
case_cli_path=self._config.loopx_case_cli_path,
|
|
2700
|
+
case_registry_path=self._config.loopx_case_registry_path,
|
|
2701
|
+
case_runtime_root=self._config.loopx_case_runtime_root,
|
|
2702
|
+
)
|
|
2703
|
+
note = shlex.quote(
|
|
2704
|
+
f"workflow driver lifecycle checkpoint {checkpoint_index}"
|
|
2705
|
+
)
|
|
2706
|
+
evidence = shlex.quote(
|
|
2707
|
+
"public-safe orchestrated checkpoint: case-local LoopX state touched"
|
|
2708
|
+
)
|
|
2709
|
+
commands = [
|
|
2710
|
+
(
|
|
2711
|
+
"quota should-run",
|
|
2712
|
+
"read",
|
|
2713
|
+
f"{cli_prefix} quota should-run --goal-id {shlex.quote(case_goal_id)} "
|
|
2714
|
+
f"--agent-id {shlex.quote(case_agent_id)} "
|
|
2715
|
+
"--runtime-profile outer_controller",
|
|
2716
|
+
),
|
|
2717
|
+
(
|
|
2718
|
+
"todo claim",
|
|
2719
|
+
"write",
|
|
2720
|
+
f"{cli_prefix} todo claim --goal-id {shlex.quote(case_goal_id)} "
|
|
2721
|
+
f"--todo-id {shlex.quote(case_todo_id)} "
|
|
2722
|
+
f"--claimed-by {shlex.quote(case_agent_id)} "
|
|
2723
|
+
f"--agent-id {shlex.quote(case_agent_id)}",
|
|
2724
|
+
),
|
|
2725
|
+
(
|
|
2726
|
+
"todo update",
|
|
2727
|
+
"write",
|
|
2728
|
+
f"{cli_prefix} todo update --goal-id {shlex.quote(case_goal_id)} "
|
|
2729
|
+
f"--todo-id {shlex.quote(case_todo_id)} --status open "
|
|
2730
|
+
f"--note {note} --evidence {evidence} "
|
|
2731
|
+
f"--claimed-by {shlex.quote(case_agent_id)} "
|
|
2732
|
+
f"--agent-id {shlex.quote(case_agent_id)}",
|
|
2733
|
+
),
|
|
2734
|
+
(
|
|
2735
|
+
"refresh-state",
|
|
2736
|
+
"write",
|
|
2737
|
+
f"{cli_prefix} refresh-state --goal-id {shlex.quote(case_goal_id)} "
|
|
2738
|
+
"--classification benchmark_case_lifecycle_checkpoint "
|
|
2739
|
+
"--delivery-batch-scale single_surface "
|
|
2740
|
+
"--delivery-outcome surface_only "
|
|
2741
|
+
f"--agent-id {shlex.quote(case_agent_id)} "
|
|
2742
|
+
"--agent-lane benchmark_case --no-global-sync",
|
|
2743
|
+
),
|
|
2744
|
+
]
|
|
2745
|
+
command_results: list[dict[str, Any]] = []
|
|
2746
|
+
failures = 0
|
|
2747
|
+
for command_name, io_kind, command in commands:
|
|
2748
|
+
try:
|
|
2749
|
+
result = self._run_remote_bridge_exec(command)
|
|
2750
|
+
except (OSError, subprocess.SubprocessError, TimeoutError) as exc:
|
|
2751
|
+
result = {
|
|
2752
|
+
"returncode": 124 if isinstance(exc, TimeoutError) else 1,
|
|
2753
|
+
"stdout_bytes": 0,
|
|
2754
|
+
"stderr_bytes": 0,
|
|
2755
|
+
"elapsed_ms": 0,
|
|
2756
|
+
"raw_stdout_recorded": False,
|
|
2757
|
+
"raw_stderr_recorded": False,
|
|
2758
|
+
"raw_command_recorded": False,
|
|
2759
|
+
}
|
|
2760
|
+
result["command_name"] = command_name
|
|
2761
|
+
result["io_kind"] = io_kind
|
|
2762
|
+
command_results.append(result)
|
|
2763
|
+
if result.get("returncode") != 0:
|
|
2764
|
+
failures += 1
|
|
2765
|
+
break
|
|
2766
|
+
self._publish_loopx_workflow_lifecycle_checkpoint_trace(
|
|
2767
|
+
checkpoint_index=checkpoint_index,
|
|
2768
|
+
command_results=command_results,
|
|
2769
|
+
failure_count=failures,
|
|
2770
|
+
)
|
|
2771
|
+
if failures:
|
|
2772
|
+
raise RuntimeError("LoopX workflow lifecycle checkpoint failed")
|
|
2773
|
+
|
|
2774
|
+
def _publish_loopx_workflow_lifecycle_checkpoint_trace(
|
|
2775
|
+
self,
|
|
2776
|
+
*,
|
|
2777
|
+
checkpoint_index: int,
|
|
2778
|
+
command_results: list[dict[str, Any]],
|
|
2779
|
+
failure_count: int,
|
|
2780
|
+
) -> None:
|
|
2781
|
+
if not self._config.worker_public_trace_dir:
|
|
2782
|
+
return
|
|
2783
|
+
command_counts: dict[str, int] = {}
|
|
2784
|
+
returncode_counts: dict[str, int] = {}
|
|
2785
|
+
state_read_count = 0
|
|
2786
|
+
state_write_count = 0
|
|
2787
|
+
for result in command_results:
|
|
2788
|
+
name = str(result.get("command_name") or "unknown")[:80]
|
|
2789
|
+
command_counts[name] = command_counts.get(name, 0) + 1
|
|
2790
|
+
rc = result.get("returncode")
|
|
2791
|
+
if isinstance(rc, int) and not isinstance(rc, bool):
|
|
2792
|
+
key = str(rc)
|
|
2793
|
+
else:
|
|
2794
|
+
key = "unknown"
|
|
2795
|
+
returncode_counts[key] = returncode_counts.get(key, 0) + 1
|
|
2796
|
+
io_kind = result.get("io_kind")
|
|
2797
|
+
if io_kind == "read":
|
|
2798
|
+
state_read_count += 1
|
|
2799
|
+
elif io_kind == "write":
|
|
2800
|
+
state_write_count += 1
|
|
2801
|
+
raw_material_recorded = any(
|
|
2802
|
+
result.get(field) is True
|
|
2803
|
+
for result in command_results
|
|
2804
|
+
for field in (
|
|
2805
|
+
"raw_command_recorded",
|
|
2806
|
+
"raw_stdout_recorded",
|
|
2807
|
+
"raw_stderr_recorded",
|
|
2808
|
+
)
|
|
2809
|
+
)
|
|
2810
|
+
trace = {
|
|
2811
|
+
"schema_version": "skillsbench_host_local_acp_relay_public_trace_v0",
|
|
2812
|
+
"ok": failure_count == 0,
|
|
2813
|
+
"route": self._config.route,
|
|
2814
|
+
"trace_kind": "remote_command_file_bridge_driver_lifecycle_checkpoint",
|
|
2815
|
+
"benchmark_id": self._config.dataset,
|
|
2816
|
+
"task_id": self._config.task_id,
|
|
2817
|
+
"remote_command_file_bridge_driver_lifecycle_checkpoint": {
|
|
2818
|
+
"schema_version": (
|
|
2819
|
+
"skillsbench_remote_command_file_bridge_driver_lifecycle_v0"
|
|
2820
|
+
),
|
|
2821
|
+
"execution_style": "orchestrated_agentloop_loopx_cli",
|
|
2822
|
+
"checkpoint_index": checkpoint_index,
|
|
2823
|
+
"checkpoint_count": 1,
|
|
2824
|
+
"request_count": len(command_results),
|
|
2825
|
+
"success_count": len(command_results) - max(0, failure_count),
|
|
2826
|
+
"failure_count": max(0, failure_count),
|
|
2827
|
+
"loopx_cli_call_count": len(command_results),
|
|
2828
|
+
"loopx_state_read_count": state_read_count,
|
|
2829
|
+
"loopx_state_write_count": state_write_count,
|
|
2830
|
+
"command_counts": dict(sorted(command_counts.items())),
|
|
2831
|
+
"returncode_counts": dict(sorted(returncode_counts.items())),
|
|
2832
|
+
"raw_material_recorded": raw_material_recorded,
|
|
2833
|
+
},
|
|
2834
|
+
"boundary": {
|
|
2835
|
+
"raw_command_recorded": False,
|
|
2836
|
+
"raw_stdout_recorded": False,
|
|
2837
|
+
"raw_stderr_recorded": False,
|
|
2838
|
+
"raw_task_text_recorded": False,
|
|
2839
|
+
"raw_logs_recorded": False,
|
|
2840
|
+
"raw_trajectory_recorded": False,
|
|
2841
|
+
"credential_values_recorded": False,
|
|
2842
|
+
"host_paths_recorded": False,
|
|
2843
|
+
"remote_paths_recorded": False,
|
|
2844
|
+
"upload_performed": False,
|
|
2845
|
+
"submit_performed": False,
|
|
2846
|
+
},
|
|
2847
|
+
}
|
|
2848
|
+
self._write_worker_public_trace(trace)
|
|
2849
|
+
|
|
2850
|
+
def _prompt_with_remote_bridge_packet(
|
|
2851
|
+
self,
|
|
2852
|
+
prompt_text: str,
|
|
2853
|
+
*,
|
|
2854
|
+
bridge_probe: dict[str, Any],
|
|
2855
|
+
bridge_command_for_agent: str | None = None,
|
|
2856
|
+
) -> str:
|
|
2857
|
+
operation_count = bridge_probe.get("operation_count")
|
|
2858
|
+
if not isinstance(operation_count, int) or isinstance(operation_count, bool):
|
|
2859
|
+
operation_count = 0
|
|
2860
|
+
bridge_command = (
|
|
2861
|
+
bridge_command_for_agent
|
|
2862
|
+
or self._config.remote_command_file_bridge_command
|
|
2863
|
+
or ""
|
|
2864
|
+
)
|
|
2865
|
+
first_exec_request = json.dumps(
|
|
2866
|
+
{
|
|
2867
|
+
"operation": "exec",
|
|
2868
|
+
"cwd": "/app",
|
|
2869
|
+
"command": "pwd && ls -la",
|
|
2870
|
+
"timeout_sec": 10,
|
|
2871
|
+
},
|
|
2872
|
+
separators=(",", ":"),
|
|
2873
|
+
)
|
|
2874
|
+
first_exec_command = (
|
|
2875
|
+
f"printf '%s\\n' {shlex.quote(first_exec_request)} | {bridge_command}"
|
|
2876
|
+
if bridge_command
|
|
2877
|
+
else (
|
|
2878
|
+
"printf '%s\\n' "
|
|
2879
|
+
f"{shlex.quote(first_exec_request)} | <private bridge command>"
|
|
2880
|
+
)
|
|
2881
|
+
)
|
|
2882
|
+
packet = f"""
|
|
2883
|
+
|
|
2884
|
+
LoopX SkillsBench remote workspace bridge:
|
|
2885
|
+
- This local Codex process is outside the scored SkillsBench sandbox.
|
|
2886
|
+
- Use the command below as a private JSON bridge for sandbox exec, file write, file read, and cleanup operations.
|
|
2887
|
+
- Send one JSON request on stdin and read one private JSON response on stdout.
|
|
2888
|
+
- FIRST ACTION REQUIRED: before prose planning or final answer, run this exact
|
|
2889
|
+
shell command without Markdown backticks; JSON must be piped on stdin:
|
|
2890
|
+
```sh
|
|
2891
|
+
{first_exec_command}
|
|
2892
|
+
```
|
|
2893
|
+
- Invoke additional operations by piping JSON to the bridge; never pass JSON as argv.
|
|
2894
|
+
- Request examples:
|
|
2895
|
+
- {{"operation":"exec","cwd":"/app","command":"pwd","timeout_sec":10}}
|
|
2896
|
+
- {{"operation":"read_file","path":"/app/path/to/file","max_bytes":20000}}
|
|
2897
|
+
- {{"operation":"write_file","path":"/app/path/to/task-specified-output","content":"..."}}
|
|
2898
|
+
- {{"operation":"read_file","path":"/root/task-input-or-data","max_bytes":20000}}
|
|
2899
|
+
- {{"operation":"cleanup","path":"/tmp/path/to/temp"}}
|
|
2900
|
+
- Allowed sandbox path roots are `/app`, `/tmp`, and `/root`; use an absolute
|
|
2901
|
+
root only when the task instruction or visible task metadata names it.
|
|
2902
|
+
{SKILLSBENCH_OUTPUT_PATH_CONTRACT}
|
|
2903
|
+
- Do not upload, submit, expose credentials, quote the bridge command in final output, or record raw stdout/stderr/task text in public artifacts.
|
|
2904
|
+
- The bridge readiness probe completed with ready=true and operation_count={operation_count}.
|
|
2905
|
+
- If a LoopX product-mode lifecycle contract is present later in this prompt,
|
|
2906
|
+
execute its case-local LoopX CLI commands through this JSON bridge
|
|
2907
|
+
(`operation=exec`, `cwd=/app`) before prose planning or final answer. A
|
|
2908
|
+
prose-only response without bridge requests is not a valid product-mode turn.
|
|
2909
|
+
|
|
2910
|
+
Private bridge command:
|
|
2911
|
+
{bridge_command}
|
|
2912
|
+
""".strip()
|
|
2913
|
+
return f"{packet}\n\n{prompt_text}"
|
|
2914
|
+
|
|
2915
|
+
def _write_instrumented_bridge_wrapper(
|
|
2916
|
+
self,
|
|
2917
|
+
*,
|
|
2918
|
+
tmp_path: Path,
|
|
2919
|
+
summary_path: Path,
|
|
2920
|
+
bridge_command: str | None = None,
|
|
2921
|
+
) -> Path:
|
|
2922
|
+
wrapper_path = tmp_path / "loopx-remote-bridge"
|
|
2923
|
+
command = bridge_command or self._config.remote_command_file_bridge_command or ""
|
|
2924
|
+
script = f"""#!/usr/bin/env python3
|
|
2925
|
+
from __future__ import annotations
|
|
2926
|
+
|
|
2927
|
+
import json
|
|
2928
|
+
import re
|
|
2929
|
+
import shlex
|
|
2930
|
+
import subprocess
|
|
2931
|
+
import sys
|
|
2932
|
+
from pathlib import Path
|
|
2933
|
+
|
|
2934
|
+
SUMMARY_PATH = Path({str(summary_path)!r})
|
|
2935
|
+
BRIDGE_COMMAND = {command!r}
|
|
2936
|
+
PROBE_REQUEST_SCHEMA_VERSION = "skillsbench_remote_command_file_bridge_probe_request_v0"
|
|
2937
|
+
PROBE_OPERATION_LABELS = {{
|
|
2938
|
+
"bounded_noop_command",
|
|
2939
|
+
"probe_marker_write",
|
|
2940
|
+
"probe_marker_read",
|
|
2941
|
+
"probe_marker_cleanup",
|
|
2942
|
+
}}
|
|
2943
|
+
PROBE_PATH_LABELS = {{"bridge_probe_marker"}}
|
|
2944
|
+
CONTENT_COMPARE_MAX_BYTES = 200_000
|
|
2945
|
+
|
|
2946
|
+
{LOOPX_COMMAND_INSTRUMENTATION_SOURCE}
|
|
2947
|
+
{LOOPX_TODO_OUTPUT_INSTRUMENTATION_SOURCE}
|
|
2948
|
+
|
|
2949
|
+
SAFE_LOOPX_TODO_ID_RE = re.compile(r"^todo_[A-Za-z0-9_-]{{6,80}}$")
|
|
2950
|
+
SAFE_LOOPX_GOAL_ID_RE = re.compile(r"^[A-Za-z0-9][A-Za-z0-9_.:-]{{0,120}}$")
|
|
2951
|
+
|
|
2952
|
+
def loopx_public_fields(command: str) -> dict[str, str]:
|
|
2953
|
+
invocations = loopx_invocation_argvs(command)
|
|
2954
|
+
tokens = invocations[0] if invocations else []
|
|
2955
|
+
fields: dict[str, str] = {{}}
|
|
2956
|
+
i = 0
|
|
2957
|
+
while i < len(tokens):
|
|
2958
|
+
token = tokens[i]
|
|
2959
|
+
name = token
|
|
2960
|
+
value = ""
|
|
2961
|
+
if token.startswith("--") and "=" in token:
|
|
2962
|
+
name, value = token.split("=", 1)
|
|
2963
|
+
elif token in {{"--goal-id", "--todo-id"}} and i + 1 < len(tokens):
|
|
2964
|
+
value = tokens[i + 1]
|
|
2965
|
+
i += 1
|
|
2966
|
+
if name == "--todo-id" and SAFE_LOOPX_TODO_ID_RE.match(value or ""):
|
|
2967
|
+
fields["loopx_todo_id"] = value
|
|
2968
|
+
elif name == "--goal-id" and SAFE_LOOPX_GOAL_ID_RE.match(value or ""):
|
|
2969
|
+
fields["loopx_goal_id"] = value
|
|
2970
|
+
i += 1
|
|
2971
|
+
return fields
|
|
2972
|
+
|
|
2973
|
+
def private_bridge_probe(request):
|
|
2974
|
+
try:
|
|
2975
|
+
proc = subprocess.run(
|
|
2976
|
+
BRIDGE_COMMAND,
|
|
2977
|
+
input=json.dumps(request, separators=(",", ":")),
|
|
2978
|
+
text=True,
|
|
2979
|
+
stdout=subprocess.PIPE,
|
|
2980
|
+
stderr=subprocess.PIPE,
|
|
2981
|
+
shell=True,
|
|
2982
|
+
)
|
|
2983
|
+
except OSError:
|
|
2984
|
+
return None
|
|
2985
|
+
if proc.returncode != 0:
|
|
2986
|
+
return None
|
|
2987
|
+
try:
|
|
2988
|
+
response = json.loads(proc.stdout or "")
|
|
2989
|
+
except Exception:
|
|
2990
|
+
return None
|
|
2991
|
+
return response if isinstance(response, dict) else None
|
|
2992
|
+
|
|
2993
|
+
def durable_write_changes_content(payload, request_path):
|
|
2994
|
+
content = payload.get("content")
|
|
2995
|
+
if not isinstance(content, str):
|
|
2996
|
+
return False
|
|
2997
|
+
existence = private_bridge_probe({{
|
|
2998
|
+
"operation": "exec",
|
|
2999
|
+
"cwd": "/app",
|
|
3000
|
+
"command": "test -e " + shlex.quote(request_path),
|
|
3001
|
+
"timeout_sec": 10,
|
|
3002
|
+
}})
|
|
3003
|
+
if existence is None:
|
|
3004
|
+
return False
|
|
3005
|
+
if existence.get("ok") is not True:
|
|
3006
|
+
return existence.get("exit_code") == 1
|
|
3007
|
+
current = private_bridge_probe({{
|
|
3008
|
+
"operation": "read_file",
|
|
3009
|
+
"path": request_path,
|
|
3010
|
+
"max_bytes": CONTENT_COMPARE_MAX_BYTES,
|
|
3011
|
+
"timeout_sec": 30,
|
|
3012
|
+
}})
|
|
3013
|
+
if current is None or current.get("ok") is not True:
|
|
3014
|
+
return False
|
|
3015
|
+
current_content = current.get("content")
|
|
3016
|
+
if not isinstance(current_content, str):
|
|
3017
|
+
return False
|
|
3018
|
+
if current.get("content_truncated") is True:
|
|
3019
|
+
return not content.startswith(current_content)
|
|
3020
|
+
return current_content != content
|
|
3021
|
+
|
|
3022
|
+
raw = sys.stdin.read()
|
|
3023
|
+
record: dict[str, object] = {{
|
|
3024
|
+
"schema_version": "skillsbench_remote_bridge_agent_operation_v0",
|
|
3025
|
+
"raw_request_recorded": False,
|
|
3026
|
+
"raw_stdout_recorded": False,
|
|
3027
|
+
"raw_stderr_recorded": False,
|
|
3028
|
+
"raw_task_text_recorded": False,
|
|
3029
|
+
"credential_values_recorded": False,
|
|
3030
|
+
"host_paths_recorded": False,
|
|
3031
|
+
"remote_paths_recorded": False,
|
|
3032
|
+
}}
|
|
3033
|
+
try:
|
|
3034
|
+
payload = json.loads(raw)
|
|
3035
|
+
except Exception:
|
|
3036
|
+
payload = {{}}
|
|
3037
|
+
operation = payload.get("operation") if isinstance(payload, dict) else ""
|
|
3038
|
+
record["operation"] = operation if isinstance(operation, str) else "unknown"
|
|
3039
|
+
operation_label = payload.get("label") if isinstance(payload, dict) else ""
|
|
3040
|
+
path_label = payload.get("path_label") if isinstance(payload, dict) else ""
|
|
3041
|
+
if isinstance(operation_label, str) and operation_label:
|
|
3042
|
+
record["operation_label"] = operation_label[:80]
|
|
3043
|
+
if isinstance(path_label, str) and path_label:
|
|
3044
|
+
record["path_label"] = path_label[:80]
|
|
3045
|
+
bridge_probe_operation = bool(
|
|
3046
|
+
isinstance(payload, dict)
|
|
3047
|
+
and (
|
|
3048
|
+
payload.get("schema_version") == PROBE_REQUEST_SCHEMA_VERSION
|
|
3049
|
+
or payload.get("probe_id") == "skillsbench_remote_command_file_bridge_probe"
|
|
3050
|
+
or (
|
|
3051
|
+
isinstance(operation_label, str)
|
|
3052
|
+
and operation_label in PROBE_OPERATION_LABELS
|
|
3053
|
+
)
|
|
3054
|
+
or (isinstance(path_label, str) and path_label in PROBE_PATH_LABELS)
|
|
3055
|
+
)
|
|
3056
|
+
)
|
|
3057
|
+
subcommands: list[str] = []
|
|
3058
|
+
loopx_fields: dict[str, str] = {{}}
|
|
3059
|
+
loopx_invocations = 0
|
|
3060
|
+
if isinstance(payload, dict) and payload.get("operation") == "exec":
|
|
3061
|
+
command_text = payload.get("command")
|
|
3062
|
+
if isinstance(command_text, str):
|
|
3063
|
+
loopx_invocations = loopx_invocation_count(command_text)
|
|
3064
|
+
subcommands = loopx_subcommands(command_text)
|
|
3065
|
+
loopx_fields = loopx_public_fields(command_text)
|
|
3066
|
+
record["loopx_invocation_count"] = loopx_invocations
|
|
3067
|
+
record["loopx_cli_call"] = loopx_invocations > 0
|
|
3068
|
+
record["loopx_subcommands"] = subcommands[:2]
|
|
3069
|
+
record.update(loopx_fields)
|
|
3070
|
+
record["loopx_state_read"] = subcommands[:2] in (["start-goal"], ["quota", "should-run"], ["status"], ["diagnose"])
|
|
3071
|
+
record["loopx_state_write"] = bool(subcommands and (
|
|
3072
|
+
subcommands[0] in {{"todo", "refresh-state"}}
|
|
3073
|
+
or subcommands[:2] == ["quota", "spend-slot"]
|
|
3074
|
+
))
|
|
3075
|
+
record["task_facing_operation"] = bool(
|
|
3076
|
+
not bridge_probe_operation
|
|
3077
|
+
and (
|
|
3078
|
+
operation in {{"read_file", "write_file", "cleanup"}}
|
|
3079
|
+
or (operation == "exec" and loopx_invocations == 0)
|
|
3080
|
+
)
|
|
3081
|
+
)
|
|
3082
|
+
request_path = payload.get("path") if isinstance(payload, dict) else ""
|
|
3083
|
+
durable_task_write = bool(
|
|
3084
|
+
operation == "write_file"
|
|
3085
|
+
and isinstance(request_path, str)
|
|
3086
|
+
and request_path.startswith(("/app/", "/root/"))
|
|
3087
|
+
and not request_path.startswith(
|
|
3088
|
+
(
|
|
3089
|
+
"/app/.loopx/",
|
|
3090
|
+
"/app/.codex/",
|
|
3091
|
+
"/app/.git/",
|
|
3092
|
+
"/app/.cache/",
|
|
3093
|
+
"/app/node_modules/",
|
|
3094
|
+
"/root/.loopx/",
|
|
3095
|
+
"/root/.codex/",
|
|
3096
|
+
"/root/.cache/",
|
|
3097
|
+
"/root/.local/",
|
|
3098
|
+
)
|
|
3099
|
+
)
|
|
3100
|
+
)
|
|
3101
|
+
record["durable_task_write"] = durable_task_write
|
|
3102
|
+
record["durable_task_content_changed"] = bool(
|
|
3103
|
+
durable_task_write
|
|
3104
|
+
and isinstance(payload, dict)
|
|
3105
|
+
and durable_write_changes_content(payload, request_path)
|
|
3106
|
+
)
|
|
3107
|
+
if durable_task_write:
|
|
3108
|
+
record["durable_task_write_root"] = (
|
|
3109
|
+
"app" if request_path.startswith("/app/") else "root"
|
|
3110
|
+
)
|
|
3111
|
+
record["bridge_probe_operation"] = bridge_probe_operation
|
|
3112
|
+
record["operation_observed"] = True
|
|
3113
|
+
|
|
3114
|
+
def append_record(item: dict[str, object]) -> None:
|
|
3115
|
+
try:
|
|
3116
|
+
SUMMARY_PATH.parent.mkdir(parents=True, exist_ok=True)
|
|
3117
|
+
with SUMMARY_PATH.open("a", encoding="utf-8") as fh:
|
|
3118
|
+
fh.write(json.dumps(item, sort_keys=True) + "\\n")
|
|
3119
|
+
except OSError:
|
|
3120
|
+
pass
|
|
3121
|
+
|
|
3122
|
+
record["record_phase"] = "start"
|
|
3123
|
+
append_record(record)
|
|
3124
|
+
enforce_single_loopx_invocation(loopx_invocations, record, append_record)
|
|
3125
|
+
proc = subprocess.run(
|
|
3126
|
+
BRIDGE_COMMAND,
|
|
3127
|
+
input=raw,
|
|
3128
|
+
text=True,
|
|
3129
|
+
stdout=subprocess.PIPE,
|
|
3130
|
+
stderr=subprocess.PIPE,
|
|
3131
|
+
shell=True,
|
|
3132
|
+
)
|
|
3133
|
+
complete_record = dict(record)
|
|
3134
|
+
complete_record["record_phase"] = "complete"
|
|
3135
|
+
try:
|
|
3136
|
+
bridge_response = json.loads(proc.stdout or "")
|
|
3137
|
+
except Exception:
|
|
3138
|
+
bridge_response = {{}}
|
|
3139
|
+
response_ok = bridge_response.get("ok") if isinstance(bridge_response, dict) else None
|
|
3140
|
+
response_exit_code = (
|
|
3141
|
+
bridge_response.get("exit_code") if isinstance(bridge_response, dict) else None
|
|
3142
|
+
)
|
|
3143
|
+
if isinstance(response_ok, bool):
|
|
3144
|
+
operation_returncode = (
|
|
3145
|
+
response_exit_code
|
|
3146
|
+
if isinstance(response_exit_code, int) and not isinstance(response_exit_code, bool)
|
|
3147
|
+
else 0 if response_ok else 1
|
|
3148
|
+
)
|
|
3149
|
+
operation_success = bool(response_ok and operation_returncode == 0)
|
|
3150
|
+
else:
|
|
3151
|
+
operation_returncode = int(proc.returncode)
|
|
3152
|
+
operation_success = proc.returncode == 0
|
|
3153
|
+
complete_record["returncode"] = operation_returncode
|
|
3154
|
+
complete_record["success"] = operation_success
|
|
3155
|
+
complete_record["stdout_bytes"] = len((proc.stdout or "").encode("utf-8"))
|
|
3156
|
+
complete_record["stderr_bytes"] = len((proc.stderr or "").encode("utf-8"))
|
|
3157
|
+
attach_successful_todo_add_id(complete_record, subcommands, proc.stdout or "")
|
|
3158
|
+
if proc.returncode != 0:
|
|
3159
|
+
stderr_text = proc.stderr or ""
|
|
3160
|
+
if int(proc.returncode) < 0:
|
|
3161
|
+
complete_record["failure_category"] = "bridge_operation_interrupted"
|
|
3162
|
+
complete_record["interrupted"] = True
|
|
3163
|
+
complete_record["controller_interrupted"] = True
|
|
3164
|
+
elif "PermissionError" in stderr_text or "Operation not permitted" in stderr_text:
|
|
3165
|
+
complete_record["failure_category"] = "bridge_client_permission_error"
|
|
3166
|
+
elif "No such file or directory" in stderr_text:
|
|
3167
|
+
complete_record["failure_category"] = "bridge_command_not_found"
|
|
3168
|
+
elif proc.returncode == 255 and BRIDGE_COMMAND.lstrip().startswith("ssh "):
|
|
3169
|
+
complete_record["failure_category"] = "bridge_ssh_unavailable"
|
|
3170
|
+
else:
|
|
3171
|
+
complete_record["failure_category"] = "bridge_command_failed"
|
|
3172
|
+
elif not operation_success:
|
|
3173
|
+
complete_record["failure_category"] = "bridge_operation_failed"
|
|
3174
|
+
append_record(complete_record)
|
|
3175
|
+
sys.stdout.write(proc.stdout)
|
|
3176
|
+
sys.stderr.write(proc.stderr)
|
|
3177
|
+
raise SystemExit(proc.returncode)
|
|
3178
|
+
"""
|
|
3179
|
+
wrapper_path.write_text(script, encoding="utf-8")
|
|
3180
|
+
wrapper_path.chmod(0o700)
|
|
3181
|
+
return wrapper_path
|
|
3182
|
+
|
|
3183
|
+
def _publish_remote_bridge_agent_operations_trace(
|
|
3184
|
+
self,
|
|
3185
|
+
*,
|
|
3186
|
+
bridge_summary_path: Path,
|
|
3187
|
+
) -> None:
|
|
3188
|
+
if not self._config.worker_public_trace_dir:
|
|
3189
|
+
return
|
|
3190
|
+
operation_counts: dict[str, int] = {}
|
|
3191
|
+
loopx_subcommand_counts: dict[str, int] = {}
|
|
3192
|
+
successful_loopx_subcommand_counts: dict[str, int] = {}
|
|
3193
|
+
successful_loopx_command_records: list[dict[str, object]] = []
|
|
3194
|
+
returncode_counts: dict[str, int] = {}
|
|
3195
|
+
failure_category_counts: dict[str, int] = {}
|
|
3196
|
+
request_count = 0
|
|
3197
|
+
success_count = 0
|
|
3198
|
+
failure_count = 0
|
|
3199
|
+
loopx_cli_call_count = 0
|
|
3200
|
+
state_read_count = 0
|
|
3201
|
+
state_write_count = 0
|
|
3202
|
+
task_facing_operation_count = 0
|
|
3203
|
+
preflight_success_count = 0
|
|
3204
|
+
preflight_failure_count = 0
|
|
3205
|
+
task_facing_success_count = 0
|
|
3206
|
+
task_facing_failure_count = 0
|
|
3207
|
+
probe_operation_count = 0
|
|
3208
|
+
inflight_operation_count = 0
|
|
3209
|
+
interrupted_operation_count = 0
|
|
3210
|
+
task_facing_interrupted_count = 0
|
|
3211
|
+
raw_material_recorded = False
|
|
3212
|
+
starts = 0
|
|
3213
|
+
completions = 0
|
|
3214
|
+
if bridge_summary_path.exists():
|
|
3215
|
+
for line in bridge_summary_path.read_text(
|
|
3216
|
+
encoding="utf-8",
|
|
3217
|
+
errors="replace",
|
|
3218
|
+
).splitlines():
|
|
3219
|
+
try:
|
|
3220
|
+
record = json.loads(line)
|
|
3221
|
+
except json.JSONDecodeError:
|
|
3222
|
+
continue
|
|
3223
|
+
if not isinstance(record, dict):
|
|
3224
|
+
continue
|
|
3225
|
+
operation = str(record.get("operation") or "unknown")[:40]
|
|
3226
|
+
phase = str(record.get("record_phase") or "").strip().lower()
|
|
3227
|
+
counts_as_request = phase != "complete"
|
|
3228
|
+
counts_as_completion = phase == "complete" or (
|
|
3229
|
+
not phase and ("returncode" in record or "success" in record)
|
|
3230
|
+
)
|
|
3231
|
+
interrupted_completion = counts_as_completion and _bridge_operation_record_interrupted(record)
|
|
3232
|
+
if phase == "complete" and not interrupted_completion:
|
|
3233
|
+
completions += 1
|
|
3234
|
+
elif phase == "start" or record.get("operation_observed") is True:
|
|
3235
|
+
starts += 1
|
|
3236
|
+
if counts_as_request:
|
|
3237
|
+
request_count += 1
|
|
3238
|
+
operation_counts[operation] = (
|
|
3239
|
+
operation_counts.get(operation, 0) + 1
|
|
3240
|
+
)
|
|
3241
|
+
rc = record.get("returncode")
|
|
3242
|
+
if counts_as_completion:
|
|
3243
|
+
if isinstance(rc, int) and not isinstance(rc, bool):
|
|
3244
|
+
returncode_counts[str(rc)] = (
|
|
3245
|
+
returncode_counts.get(str(rc), 0) + 1
|
|
3246
|
+
)
|
|
3247
|
+
if interrupted_completion:
|
|
3248
|
+
interrupted_operation_count += 1
|
|
3249
|
+
if record.get("task_facing_operation") is True:
|
|
3250
|
+
task_facing_interrupted_count += 1
|
|
3251
|
+
elif rc == 0:
|
|
3252
|
+
success_count += 1
|
|
3253
|
+
if operation == "preflight":
|
|
3254
|
+
preflight_success_count += 1
|
|
3255
|
+
if record.get("task_facing_operation") is True:
|
|
3256
|
+
task_facing_success_count += 1
|
|
3257
|
+
else:
|
|
3258
|
+
failure_count += 1
|
|
3259
|
+
if operation == "preflight":
|
|
3260
|
+
preflight_failure_count += 1
|
|
3261
|
+
if record.get("task_facing_operation") is True:
|
|
3262
|
+
task_facing_failure_count += 1
|
|
3263
|
+
category = str(
|
|
3264
|
+
record.get("failure_category")
|
|
3265
|
+
or "bridge_command_failed"
|
|
3266
|
+
)[:80]
|
|
3267
|
+
failure_category_counts[category] = (
|
|
3268
|
+
failure_category_counts.get(category, 0) + 1
|
|
3269
|
+
)
|
|
3270
|
+
elif interrupted_completion:
|
|
3271
|
+
interrupted_operation_count += 1
|
|
3272
|
+
if record.get("task_facing_operation") is True:
|
|
3273
|
+
task_facing_interrupted_count += 1
|
|
3274
|
+
elif record.get("success") is True:
|
|
3275
|
+
success_count += 1
|
|
3276
|
+
if operation == "preflight":
|
|
3277
|
+
preflight_success_count += 1
|
|
3278
|
+
if record.get("task_facing_operation") is True:
|
|
3279
|
+
task_facing_success_count += 1
|
|
3280
|
+
returncode_counts["unknown_success"] = (
|
|
3281
|
+
returncode_counts.get("unknown_success", 0) + 1
|
|
3282
|
+
)
|
|
3283
|
+
elif record.get("success") is False:
|
|
3284
|
+
failure_count += 1
|
|
3285
|
+
if operation == "preflight":
|
|
3286
|
+
preflight_failure_count += 1
|
|
3287
|
+
if record.get("task_facing_operation") is True:
|
|
3288
|
+
task_facing_failure_count += 1
|
|
3289
|
+
returncode_counts["unknown_failure"] = (
|
|
3290
|
+
returncode_counts.get("unknown_failure", 0) + 1
|
|
3291
|
+
)
|
|
3292
|
+
category = str(
|
|
3293
|
+
record.get("failure_category") or "bridge_command_failed"
|
|
3294
|
+
)[:80]
|
|
3295
|
+
failure_category_counts[category] = (
|
|
3296
|
+
failure_category_counts.get(category, 0) + 1
|
|
3297
|
+
)
|
|
3298
|
+
if counts_as_request and record.get("loopx_cli_call") is True:
|
|
3299
|
+
loopx_cli_call_count += 1
|
|
3300
|
+
if counts_as_request and record.get("loopx_state_read") is True:
|
|
3301
|
+
state_read_count += 1
|
|
3302
|
+
if counts_as_request and record.get("loopx_state_write") is True:
|
|
3303
|
+
state_write_count += 1
|
|
3304
|
+
if counts_as_request and record.get("task_facing_operation") is True:
|
|
3305
|
+
task_facing_operation_count += 1
|
|
3306
|
+
if counts_as_request and record.get("bridge_probe_operation") is True:
|
|
3307
|
+
probe_operation_count += 1
|
|
3308
|
+
subcommands = record.get("loopx_subcommands")
|
|
3309
|
+
if isinstance(subcommands, list) and subcommands:
|
|
3310
|
+
key = " ".join(
|
|
3311
|
+
str(item)
|
|
3312
|
+
for item in subcommands[:2]
|
|
3313
|
+
if re.match(r"^[A-Za-z][A-Za-z0-9_-]{0,40}$", str(item))
|
|
3314
|
+
)
|
|
3315
|
+
if key and counts_as_request:
|
|
3316
|
+
loopx_subcommand_counts[key] = (
|
|
3317
|
+
loopx_subcommand_counts.get(key, 0) + 1
|
|
3318
|
+
)
|
|
3319
|
+
if key and counts_as_completion:
|
|
3320
|
+
if record.get("success") is True or record.get("returncode") == 0:
|
|
3321
|
+
successful_loopx_subcommand_counts[key] = (
|
|
3322
|
+
successful_loopx_subcommand_counts.get(key, 0) + 1
|
|
3323
|
+
)
|
|
3324
|
+
command_record: dict[str, object] = {
|
|
3325
|
+
"subcommand": key,
|
|
3326
|
+
}
|
|
3327
|
+
todo_id = _safe_loopx_todo_id(record.get("loopx_todo_id"))
|
|
3328
|
+
if todo_id:
|
|
3329
|
+
command_record["todo_id"] = todo_id
|
|
3330
|
+
goal_id = _safe_loopx_goal_id(record.get("loopx_goal_id"))
|
|
3331
|
+
if goal_id:
|
|
3332
|
+
command_record["goal_id"] = goal_id
|
|
3333
|
+
if len(successful_loopx_command_records) < 128:
|
|
3334
|
+
successful_loopx_command_records.append(command_record)
|
|
3335
|
+
raw_material_recorded = raw_material_recorded or any(
|
|
3336
|
+
record.get(field) is True
|
|
3337
|
+
for field in (
|
|
3338
|
+
"raw_request_recorded",
|
|
3339
|
+
"raw_stdout_recorded",
|
|
3340
|
+
"raw_stderr_recorded",
|
|
3341
|
+
"raw_task_text_recorded",
|
|
3342
|
+
"credential_values_recorded",
|
|
3343
|
+
"host_paths_recorded",
|
|
3344
|
+
"remote_paths_recorded",
|
|
3345
|
+
)
|
|
3346
|
+
)
|
|
3347
|
+
inflight_operation_count = max(0, starts - completions)
|
|
3348
|
+
snapshot_key = str(bridge_summary_path)
|
|
3349
|
+
snapshot_id = self._bridge_summary_snapshot_ids.setdefault(
|
|
3350
|
+
snapshot_key,
|
|
3351
|
+
f"bridge-{uuid.uuid4().hex[:12]}",
|
|
3352
|
+
)
|
|
3353
|
+
snapshot_index = (
|
|
3354
|
+
self._bridge_summary_snapshot_indexes.get(snapshot_key, 0) + 1
|
|
3355
|
+
)
|
|
3356
|
+
self._bridge_summary_snapshot_indexes[snapshot_key] = snapshot_index
|
|
3357
|
+
trace = {
|
|
3358
|
+
"schema_version": "skillsbench_host_local_acp_relay_public_trace_v0",
|
|
3359
|
+
"ok": True,
|
|
3360
|
+
"route": self._config.route,
|
|
3361
|
+
"trace_kind": "remote_command_file_bridge_agent_operations",
|
|
3362
|
+
"benchmark_id": self._config.dataset,
|
|
3363
|
+
"task_id": self._config.task_id,
|
|
3364
|
+
"remote_command_file_bridge_agent_operations": {
|
|
3365
|
+
"schema_version": (
|
|
3366
|
+
"skillsbench_remote_command_file_bridge_agent_operations_v0"
|
|
3367
|
+
),
|
|
3368
|
+
"snapshot_id": snapshot_id,
|
|
3369
|
+
"snapshot_index": snapshot_index,
|
|
3370
|
+
"snapshot_semantics": "cumulative",
|
|
3371
|
+
"request_count": request_count,
|
|
3372
|
+
"success_count": success_count,
|
|
3373
|
+
"failure_count": failure_count,
|
|
3374
|
+
"operation_counts": dict(sorted(operation_counts.items())),
|
|
3375
|
+
"returncode_counts": dict(sorted(returncode_counts.items())),
|
|
3376
|
+
"failure_category_counts": dict(
|
|
3377
|
+
sorted(failure_category_counts.items())
|
|
3378
|
+
),
|
|
3379
|
+
"loopx_cli_call_count": loopx_cli_call_count,
|
|
3380
|
+
"loopx_cli_subcommand_counts": dict(
|
|
3381
|
+
sorted(loopx_subcommand_counts.items())
|
|
3382
|
+
),
|
|
3383
|
+
"successful_loopx_cli_subcommand_counts": dict(
|
|
3384
|
+
sorted(successful_loopx_subcommand_counts.items())
|
|
3385
|
+
),
|
|
3386
|
+
"successful_loopx_cli_command_records": (
|
|
3387
|
+
successful_loopx_command_records
|
|
3388
|
+
),
|
|
3389
|
+
"loopx_state_read_count": state_read_count,
|
|
3390
|
+
"loopx_state_write_count": state_write_count,
|
|
3391
|
+
"task_facing_operation_count": task_facing_operation_count,
|
|
3392
|
+
"preflight_success_count": preflight_success_count,
|
|
3393
|
+
"preflight_failure_count": preflight_failure_count,
|
|
3394
|
+
"task_facing_success_count": task_facing_success_count,
|
|
3395
|
+
"task_facing_failure_count": task_facing_failure_count,
|
|
3396
|
+
"probe_operation_count": probe_operation_count,
|
|
3397
|
+
"inflight_operation_count": inflight_operation_count,
|
|
3398
|
+
"interrupted_operation_count": interrupted_operation_count,
|
|
3399
|
+
"task_facing_interrupted_count": task_facing_interrupted_count,
|
|
3400
|
+
"raw_material_recorded": raw_material_recorded,
|
|
3401
|
+
},
|
|
3402
|
+
"boundary": {
|
|
3403
|
+
"raw_command_recorded": False,
|
|
3404
|
+
"raw_stdout_recorded": False,
|
|
3405
|
+
"raw_stderr_recorded": False,
|
|
3406
|
+
"raw_task_text_recorded": False,
|
|
3407
|
+
"raw_logs_recorded": False,
|
|
3408
|
+
"raw_trajectory_recorded": False,
|
|
3409
|
+
"credential_values_recorded": False,
|
|
3410
|
+
"host_paths_recorded": False,
|
|
3411
|
+
"remote_paths_recorded": False,
|
|
3412
|
+
"upload_performed": False,
|
|
3413
|
+
"submit_performed": False,
|
|
3414
|
+
},
|
|
3415
|
+
}
|
|
3416
|
+
self._write_worker_public_trace(trace)
|
|
3417
|
+
|
|
3418
|
+
def _publish_remote_bridge_consumption_trace(
|
|
3419
|
+
self,
|
|
3420
|
+
bridge_probe: dict[str, Any],
|
|
3421
|
+
) -> None:
|
|
3422
|
+
if not self._config.worker_public_trace_dir:
|
|
3423
|
+
return
|
|
3424
|
+
operation_count = bridge_probe.get("operation_count")
|
|
3425
|
+
if not isinstance(operation_count, int) or isinstance(operation_count, bool):
|
|
3426
|
+
operation_count = 0
|
|
3427
|
+
boundary = {
|
|
3428
|
+
"raw_command_recorded": bridge_probe.get("raw_command_recorded") is True,
|
|
3429
|
+
"raw_stdout_recorded": bridge_probe.get("raw_stdout_recorded") is True,
|
|
3430
|
+
"raw_stderr_recorded": bridge_probe.get("raw_stderr_recorded") is True,
|
|
3431
|
+
"raw_task_text_recorded": bridge_probe.get("raw_task_text_recorded") is True,
|
|
3432
|
+
"raw_logs_recorded": bridge_probe.get("raw_logs_recorded") is True,
|
|
3433
|
+
"raw_trajectory_recorded": bridge_probe.get("raw_trajectory_recorded") is True,
|
|
3434
|
+
"credential_values_recorded": (
|
|
3435
|
+
bridge_probe.get("credential_values_recorded") is True
|
|
3436
|
+
),
|
|
3437
|
+
"host_paths_recorded": bridge_probe.get("host_paths_recorded") is True,
|
|
3438
|
+
"remote_paths_recorded": bridge_probe.get("remote_paths_recorded") is True,
|
|
3439
|
+
"upload_performed": bridge_probe.get("upload_performed") is True,
|
|
3440
|
+
"submit_performed": bridge_probe.get("submit_performed") is True,
|
|
3441
|
+
}
|
|
3442
|
+
trace = {
|
|
3443
|
+
"schema_version": "skillsbench_host_local_acp_relay_public_trace_v0",
|
|
3444
|
+
"ok": bridge_probe.get("ready") is True,
|
|
3445
|
+
"route": self._config.route,
|
|
3446
|
+
"trace_kind": "remote_command_file_bridge_solver_consumption",
|
|
3447
|
+
"benchmark_id": self._config.dataset,
|
|
3448
|
+
"task_id": self._config.task_id,
|
|
3449
|
+
"remote_command_file_bridge": {
|
|
3450
|
+
"schema_version": "skillsbench_remote_command_file_bridge_solver_consumption_v0",
|
|
3451
|
+
"consumed_by_solver": True,
|
|
3452
|
+
"probe_ready": bridge_probe.get("ready") is True,
|
|
3453
|
+
"operation_count": operation_count,
|
|
3454
|
+
"first_blocker": _public_bridge_label(
|
|
3455
|
+
bridge_probe.get("first_blocker"), limit=120
|
|
3456
|
+
),
|
|
3457
|
+
"response_first_blocker": _public_bridge_label(
|
|
3458
|
+
bridge_probe.get("response_first_blocker"), limit=120
|
|
3459
|
+
),
|
|
3460
|
+
"stage": _public_bridge_label(bridge_probe.get("stage"), limit=80),
|
|
3461
|
+
"bridge_command_invoked": (
|
|
3462
|
+
bridge_probe.get("bridge_command_invoked") is True
|
|
3463
|
+
),
|
|
3464
|
+
"bridge_command_recorded": False,
|
|
3465
|
+
"required_operations": _public_bridge_label_list(
|
|
3466
|
+
bridge_probe.get("required_operations")
|
|
3467
|
+
),
|
|
3468
|
+
"missing_operations": _public_bridge_label_list(
|
|
3469
|
+
bridge_probe.get("missing_operations")
|
|
3470
|
+
),
|
|
3471
|
+
"failed_operations": _public_bridge_label_list(
|
|
3472
|
+
bridge_probe.get("failed_operations")
|
|
3473
|
+
),
|
|
3474
|
+
"boundary_violations": _public_bridge_label_list(
|
|
3475
|
+
bridge_probe.get("boundary_violations")
|
|
3476
|
+
),
|
|
3477
|
+
"operations": _public_bridge_operations(bridge_probe.get("operations")),
|
|
3478
|
+
},
|
|
3479
|
+
"boundary": boundary,
|
|
3480
|
+
}
|
|
3481
|
+
response_schema_version = _public_bridge_label(
|
|
3482
|
+
bridge_probe.get("response_schema_version"), limit=120
|
|
3483
|
+
)
|
|
3484
|
+
if response_schema_version:
|
|
3485
|
+
trace["remote_command_file_bridge"][
|
|
3486
|
+
"response_schema_version"
|
|
3487
|
+
] = response_schema_version
|
|
3488
|
+
elapsed_ms = bridge_probe.get("elapsed_ms")
|
|
3489
|
+
if isinstance(elapsed_ms, int) and not isinstance(elapsed_ms, bool):
|
|
3490
|
+
trace["remote_command_file_bridge"]["elapsed_ms"] = max(
|
|
3491
|
+
0, min(elapsed_ms, 600_000)
|
|
3492
|
+
)
|
|
3493
|
+
self._write_worker_public_trace(trace)
|
|
3494
|
+
|
|
3495
|
+
def _publish_codex_exec_failure_trace(
|
|
3496
|
+
self,
|
|
3497
|
+
*,
|
|
3498
|
+
stage: str,
|
|
3499
|
+
returncode: int | None,
|
|
3500
|
+
stdout_text: str,
|
|
3501
|
+
stderr_text: str,
|
|
3502
|
+
final_message_present: bool,
|
|
3503
|
+
final_message_bytes: int,
|
|
3504
|
+
failure_category: str | None = None,
|
|
3505
|
+
recoverable_turn_failure: bool | None = None,
|
|
3506
|
+
transport_retry_index: int = 0,
|
|
3507
|
+
retry_scheduled: bool = False,
|
|
3508
|
+
session_rollover_index: int = 0,
|
|
3509
|
+
session_rollover_scheduled: bool = False,
|
|
3510
|
+
same_session_continuation_index: int = 0,
|
|
3511
|
+
same_session_continuation_scheduled: bool = False,
|
|
3512
|
+
independent_validation_handoff_scheduled: bool = False,
|
|
3513
|
+
) -> None:
|
|
3514
|
+
if not self._config.worker_public_trace_dir:
|
|
3515
|
+
return
|
|
3516
|
+
safe_stage = "".join(
|
|
3517
|
+
ch if ch.isalnum() or ch == "_" else "_"
|
|
3518
|
+
for ch in str(stage or "").strip().lower()
|
|
3519
|
+
)
|
|
3520
|
+
if not safe_stage:
|
|
3521
|
+
safe_stage = "codex_exec_failed"
|
|
3522
|
+
category = failure_category or _codex_exec_failure_category(
|
|
3523
|
+
returncode=returncode,
|
|
3524
|
+
stderr_text=stderr_text,
|
|
3525
|
+
stdout_text=stdout_text,
|
|
3526
|
+
)
|
|
3527
|
+
if recoverable_turn_failure is None:
|
|
3528
|
+
recoverable_turn_failure = (
|
|
3529
|
+
category in RECOVERABLE_CODEX_TURN_FAILURE_CATEGORIES
|
|
3530
|
+
)
|
|
3531
|
+
trace = {
|
|
3532
|
+
"schema_version": "skillsbench_host_local_acp_relay_public_trace_v0",
|
|
3533
|
+
"ok": False,
|
|
3534
|
+
"route": self._config.route,
|
|
3535
|
+
"trace_kind": "codex_exec_process_failure",
|
|
3536
|
+
"benchmark_id": self._config.dataset,
|
|
3537
|
+
"task_id": self._config.task_id,
|
|
3538
|
+
"codex_exec_process": {
|
|
3539
|
+
"schema_version": "skillsbench_codex_exec_process_failure_v0",
|
|
3540
|
+
"stage": safe_stage,
|
|
3541
|
+
"failure_category": str(category or "codex_exec_failed")[:120],
|
|
3542
|
+
"recoverable_turn_failure": recoverable_turn_failure,
|
|
3543
|
+
"transport_retry_index": max(0, int(transport_retry_index or 0)),
|
|
3544
|
+
"retry_scheduled": bool(retry_scheduled),
|
|
3545
|
+
"session_rollover_index": max(
|
|
3546
|
+
0, int(session_rollover_index or 0)
|
|
3547
|
+
),
|
|
3548
|
+
"session_rollover_scheduled": bool(session_rollover_scheduled),
|
|
3549
|
+
"same_session_continuation_index": max(
|
|
3550
|
+
0, int(same_session_continuation_index or 0)
|
|
3551
|
+
),
|
|
3552
|
+
"same_session_continuation_scheduled": bool(
|
|
3553
|
+
same_session_continuation_scheduled
|
|
3554
|
+
),
|
|
3555
|
+
"independent_validation_handoff_scheduled": bool(
|
|
3556
|
+
independent_validation_handoff_scheduled
|
|
3557
|
+
),
|
|
3558
|
+
"returncode": (
|
|
3559
|
+
returncode
|
|
3560
|
+
if isinstance(returncode, int)
|
|
3561
|
+
and not isinstance(returncode, bool)
|
|
3562
|
+
else None
|
|
3563
|
+
),
|
|
3564
|
+
"stdout_bytes": len((stdout_text or "").encode("utf-8")),
|
|
3565
|
+
"stderr_bytes": len((stderr_text or "").encode("utf-8")),
|
|
3566
|
+
"final_message_present": bool(final_message_present),
|
|
3567
|
+
"final_message_bytes": max(0, int(final_message_bytes or 0)),
|
|
3568
|
+
"raw_stdout_recorded": False,
|
|
3569
|
+
"raw_stderr_recorded": False,
|
|
3570
|
+
"raw_task_text_recorded": False,
|
|
3571
|
+
"raw_trajectory_recorded": False,
|
|
3572
|
+
"credential_values_recorded": False,
|
|
3573
|
+
"host_paths_recorded": False,
|
|
3574
|
+
},
|
|
3575
|
+
"boundary": {
|
|
3576
|
+
"raw_command_recorded": False,
|
|
3577
|
+
"raw_stdout_recorded": False,
|
|
3578
|
+
"raw_stderr_recorded": False,
|
|
3579
|
+
"raw_task_text_recorded": False,
|
|
3580
|
+
"raw_logs_recorded": False,
|
|
3581
|
+
"raw_trajectory_recorded": False,
|
|
3582
|
+
"credential_values_recorded": False,
|
|
3583
|
+
"host_paths_recorded": False,
|
|
3584
|
+
"remote_paths_recorded": False,
|
|
3585
|
+
"upload_performed": False,
|
|
3586
|
+
"submit_performed": False,
|
|
3587
|
+
},
|
|
3588
|
+
}
|
|
3589
|
+
self._write_worker_public_trace(trace)
|
|
3590
|
+
|
|
3591
|
+
def _publish_codex_exec_same_session_continuation_trace(
|
|
3592
|
+
self,
|
|
3593
|
+
*,
|
|
3594
|
+
stage: str,
|
|
3595
|
+
continuation_index: int,
|
|
3596
|
+
thread_present: bool,
|
|
3597
|
+
) -> None:
|
|
3598
|
+
if not self._config.worker_public_trace_dir:
|
|
3599
|
+
return
|
|
3600
|
+
safe_stage = str(stage or "failed").strip().lower()
|
|
3601
|
+
if safe_stage not in {"completed", "failed"}:
|
|
3602
|
+
safe_stage = "failed"
|
|
3603
|
+
trace = {
|
|
3604
|
+
"schema_version": "skillsbench_host_local_acp_relay_public_trace_v0",
|
|
3605
|
+
"ok": safe_stage == "completed",
|
|
3606
|
+
"route": self._config.route,
|
|
3607
|
+
"trace_kind": "codex_exec_same_session_continuation",
|
|
3608
|
+
"benchmark_id": self._config.dataset,
|
|
3609
|
+
"task_id": self._config.task_id,
|
|
3610
|
+
"codex_exec_same_session_continuation": {
|
|
3611
|
+
"schema_version": (
|
|
3612
|
+
"skillsbench_codex_exec_same_session_continuation_v0"
|
|
3613
|
+
),
|
|
3614
|
+
"stage": safe_stage,
|
|
3615
|
+
"continuation_index": max(1, int(continuation_index or 1)),
|
|
3616
|
+
"thread_present": bool(thread_present),
|
|
3617
|
+
"shared_sequence_deadline_preserved": True,
|
|
3618
|
+
"original_prompt_replayed": False,
|
|
3619
|
+
"thread_ids_recorded": False,
|
|
3620
|
+
"raw_task_text_recorded": False,
|
|
3621
|
+
},
|
|
3622
|
+
"boundary": {
|
|
3623
|
+
"raw_command_recorded": False,
|
|
3624
|
+
"raw_stdout_recorded": False,
|
|
3625
|
+
"raw_stderr_recorded": False,
|
|
3626
|
+
"raw_task_text_recorded": False,
|
|
3627
|
+
"raw_logs_recorded": False,
|
|
3628
|
+
"raw_trajectory_recorded": False,
|
|
3629
|
+
"credential_values_recorded": False,
|
|
3630
|
+
"host_paths_recorded": False,
|
|
3631
|
+
"remote_paths_recorded": False,
|
|
3632
|
+
"upload_performed": False,
|
|
3633
|
+
"submit_performed": False,
|
|
3634
|
+
},
|
|
3635
|
+
}
|
|
3636
|
+
self._write_worker_public_trace(trace)
|
|
3637
|
+
|
|
3638
|
+
def _publish_codex_exec_session_rollover_trace(
|
|
3639
|
+
self,
|
|
3640
|
+
*,
|
|
3641
|
+
stage: str,
|
|
3642
|
+
rollover_index: int,
|
|
3643
|
+
resume_failure_count: int,
|
|
3644
|
+
previous_thread_present: bool,
|
|
3645
|
+
fresh_thread_present: bool,
|
|
3646
|
+
) -> None:
|
|
3647
|
+
if not self._config.worker_public_trace_dir:
|
|
3648
|
+
return
|
|
3649
|
+
safe_stage = str(stage or "failed").strip().lower()
|
|
3650
|
+
if safe_stage not in {"completed", "failed"}:
|
|
3651
|
+
safe_stage = "failed"
|
|
3652
|
+
trace = {
|
|
3653
|
+
"schema_version": "skillsbench_host_local_acp_relay_public_trace_v0",
|
|
3654
|
+
"ok": safe_stage == "completed",
|
|
3655
|
+
"route": self._config.route,
|
|
3656
|
+
"trace_kind": "codex_exec_session_rollover",
|
|
3657
|
+
"benchmark_id": self._config.dataset,
|
|
3658
|
+
"task_id": self._config.task_id,
|
|
3659
|
+
"codex_exec_session_rollover": {
|
|
3660
|
+
"schema_version": "skillsbench_codex_exec_session_rollover_v0",
|
|
3661
|
+
"stage": safe_stage,
|
|
3662
|
+
"rollover_index": max(1, int(rollover_index or 1)),
|
|
3663
|
+
"resume_failure_count": max(2, int(resume_failure_count or 2)),
|
|
3664
|
+
"previous_thread_present": bool(previous_thread_present),
|
|
3665
|
+
"fresh_thread_present": bool(fresh_thread_present),
|
|
3666
|
+
"shared_sequence_deadline_preserved": True,
|
|
3667
|
+
"original_prompt_available_in_private_memory": True,
|
|
3668
|
+
"original_prompt_recorded": False,
|
|
3669
|
+
"thread_ids_recorded": False,
|
|
3670
|
+
"raw_task_text_recorded": False,
|
|
3671
|
+
},
|
|
3672
|
+
"boundary": {
|
|
3673
|
+
"raw_command_recorded": False,
|
|
3674
|
+
"raw_stdout_recorded": False,
|
|
3675
|
+
"raw_stderr_recorded": False,
|
|
3676
|
+
"raw_task_text_recorded": False,
|
|
3677
|
+
"raw_logs_recorded": False,
|
|
3678
|
+
"raw_trajectory_recorded": False,
|
|
3679
|
+
"credential_values_recorded": False,
|
|
3680
|
+
"host_paths_recorded": False,
|
|
3681
|
+
"remote_paths_recorded": False,
|
|
3682
|
+
"upload_performed": False,
|
|
3683
|
+
"submit_performed": False,
|
|
3684
|
+
},
|
|
3685
|
+
}
|
|
3686
|
+
self._write_worker_public_trace(trace)
|
|
3687
|
+
|
|
3688
|
+
def _run_app_server_goal_worker(
|
|
3689
|
+
self,
|
|
3690
|
+
prompt_text: str,
|
|
3691
|
+
*,
|
|
3692
|
+
session: dict[str, Any],
|
|
3693
|
+
session_id: str,
|
|
3694
|
+
stdout: TextIO,
|
|
3695
|
+
) -> str:
|
|
3696
|
+
cwd = _safe_cwd(session.get("cwd"), default=os.getcwd())
|
|
3697
|
+
self._publish_worker_lifecycle_trace("prompt_received")
|
|
3698
|
+
with _temporary_directory_ignore_cleanup_errors(
|
|
3699
|
+
prefix="gh-skillsbench-goal-worker-"
|
|
3700
|
+
) as tmp:
|
|
3701
|
+
tmp_path = Path(tmp)
|
|
3702
|
+
prompt_path = tmp_path / "prompt.txt"
|
|
3703
|
+
output_json = tmp_path / "worker.compact.json"
|
|
3704
|
+
response_path = tmp_path / "response.txt"
|
|
3705
|
+
prompt_for_worker = prompt_text
|
|
3706
|
+
bridge_server_proc: subprocess.Popen[str] | None = None
|
|
3707
|
+
bridge_summary_path: Path | None = None
|
|
3708
|
+
if self._config.remote_command_file_bridge_command:
|
|
3709
|
+
if _is_bridge_action_preflight_prompt(prompt_text):
|
|
3710
|
+
bridge_probe = self._reverse_channel_json_preflight_probe()
|
|
3711
|
+
else:
|
|
3712
|
+
bridge_probe = self._consume_remote_bridge_for_solver()
|
|
3713
|
+
self._publish_remote_bridge_consumption_trace(bridge_probe)
|
|
3714
|
+
if bridge_probe.get("ready") is not True:
|
|
3715
|
+
raise RuntimeError("remote command/file bridge probe failed")
|
|
3716
|
+
local_cwd = tmp_path / "local-codex-cwd"
|
|
3717
|
+
local_cwd.mkdir(parents=True, exist_ok=True)
|
|
3718
|
+
cwd = str(local_cwd)
|
|
3719
|
+
bridge_summary_path = tmp_path / "remote-bridge-agent-ops.jsonl"
|
|
3720
|
+
agent_bridge_command = (
|
|
3721
|
+
self._config.remote_command_file_bridge_agent_command
|
|
3722
|
+
or self._config.remote_command_file_bridge_command
|
|
3723
|
+
or ""
|
|
3724
|
+
)
|
|
3725
|
+
agent_bridge_command, bridge_server_proc = (
|
|
3726
|
+
self._start_json_file_bridge_server(
|
|
3727
|
+
tmp_path=tmp_path,
|
|
3728
|
+
local_cwd=local_cwd,
|
|
3729
|
+
bridge_command=agent_bridge_command,
|
|
3730
|
+
)
|
|
3731
|
+
)
|
|
3732
|
+
instrumented_bridge = self._write_instrumented_bridge_wrapper(
|
|
3733
|
+
tmp_path=tmp_path,
|
|
3734
|
+
summary_path=bridge_summary_path,
|
|
3735
|
+
bridge_command=agent_bridge_command,
|
|
3736
|
+
)
|
|
3737
|
+
prompt_for_worker = self._prompt_with_remote_bridge_packet(
|
|
3738
|
+
prompt_text,
|
|
3739
|
+
bridge_probe=bridge_probe,
|
|
3740
|
+
bridge_command_for_agent=str(instrumented_bridge),
|
|
3741
|
+
)
|
|
3742
|
+
app_server_goal_prompt_style = _normalized_app_server_goal_prompt_style(
|
|
3743
|
+
self._config.app_server_goal_prompt_style
|
|
3744
|
+
)
|
|
3745
|
+
app_server_goal_closeout_injected = (
|
|
3746
|
+
app_server_goal_prompt_style == "native-goal"
|
|
3747
|
+
)
|
|
3748
|
+
if app_server_goal_closeout_injected:
|
|
3749
|
+
prompt_for_worker = _prompt_with_app_server_closeout_instruction(
|
|
3750
|
+
prompt_for_worker
|
|
3751
|
+
)
|
|
3752
|
+
prompt_path.write_text(prompt_for_worker, encoding="utf-8")
|
|
3753
|
+
worker_script = (
|
|
3754
|
+
Path(self._config.worker_script).expanduser()
|
|
3755
|
+
if self._config.worker_script
|
|
3756
|
+
else Path(__file__).resolve().parents[2]
|
|
3757
|
+
/ "scripts"
|
|
3758
|
+
/ "skillsbench_host_codex_goal_worker.py"
|
|
3759
|
+
)
|
|
3760
|
+
worker_first_action_timeout_sec = self._config.first_action_timeout_sec
|
|
3761
|
+
cmd = [
|
|
3762
|
+
sys.executable,
|
|
3763
|
+
str(worker_script),
|
|
3764
|
+
"--dataset",
|
|
3765
|
+
self._config.dataset,
|
|
3766
|
+
"--task-id",
|
|
3767
|
+
self._config.task_id,
|
|
3768
|
+
"--run-group-id",
|
|
3769
|
+
self._config.run_group_id,
|
|
3770
|
+
"--job-name",
|
|
3771
|
+
self._config.job_name,
|
|
3772
|
+
"--rollout-name",
|
|
3773
|
+
self._config.rollout_name,
|
|
3774
|
+
"--codex-bin",
|
|
3775
|
+
self._config.codex_bin,
|
|
3776
|
+
"--sandbox",
|
|
3777
|
+
self._config.sandbox,
|
|
3778
|
+
"--approval-policy",
|
|
3779
|
+
self._config.approval_policy,
|
|
3780
|
+
"--work-dir",
|
|
3781
|
+
cwd,
|
|
3782
|
+
"--prompt-file",
|
|
3783
|
+
str(prompt_path),
|
|
3784
|
+
"--output-json",
|
|
3785
|
+
str(output_json),
|
|
3786
|
+
"--response-text-file",
|
|
3787
|
+
str(response_path),
|
|
3788
|
+
"--response-timeout-sec",
|
|
3789
|
+
str(self._config.response_timeout_sec),
|
|
3790
|
+
"--turn-timeout-sec",
|
|
3791
|
+
str(self._config.timeout_sec),
|
|
3792
|
+
"--first-action-timeout-sec",
|
|
3793
|
+
str(worker_first_action_timeout_sec),
|
|
3794
|
+
"--normal-followup-max",
|
|
3795
|
+
str(max(0, int(self._config.app_server_goal_followup_max or 0))),
|
|
3796
|
+
"--app-server-goal-prompt-style",
|
|
3797
|
+
app_server_goal_prompt_style,
|
|
3798
|
+
"--reasoning-effort",
|
|
3799
|
+
str(self._config.reasoning_effort or "high"),
|
|
3800
|
+
"--runner-integration-ready",
|
|
3801
|
+
]
|
|
3802
|
+
model = self._config.model or session.get("model")
|
|
3803
|
+
if model:
|
|
3804
|
+
cmd.extend(["--model", str(model)])
|
|
3805
|
+
try:
|
|
3806
|
+
proc = subprocess.Popen(
|
|
3807
|
+
cmd,
|
|
3808
|
+
stdin=subprocess.DEVNULL,
|
|
3809
|
+
stdout=subprocess.PIPE,
|
|
3810
|
+
stderr=subprocess.PIPE,
|
|
3811
|
+
text=True,
|
|
3812
|
+
start_new_session=True,
|
|
3813
|
+
)
|
|
3814
|
+
self._write_worker_heartbeat(
|
|
3815
|
+
stdout,
|
|
3816
|
+
session_id=session_id,
|
|
3817
|
+
text="host app-server goal worker started",
|
|
3818
|
+
)
|
|
3819
|
+
deadline = time.monotonic() + self._config.timeout_sec + 60
|
|
3820
|
+
next_heartbeat = (
|
|
3821
|
+
time.monotonic()
|
|
3822
|
+
+ max(1.0, self._config.stream_heartbeat_interval_sec)
|
|
3823
|
+
)
|
|
3824
|
+
bridge_activity_seen = False
|
|
3825
|
+
last_bridge_summary_size = 0
|
|
3826
|
+
last_bridge_activity_at = time.monotonic()
|
|
3827
|
+
bridge_idle_timeout_sec = max(
|
|
3828
|
+
0.0,
|
|
3829
|
+
float(self._config.bridge_idle_timeout_sec or 0.0),
|
|
3830
|
+
)
|
|
3831
|
+
task_output_quiet_timeout_sec = max(
|
|
3832
|
+
0.0,
|
|
3833
|
+
float(self._config.task_output_quiet_timeout_sec or 0.0),
|
|
3834
|
+
)
|
|
3835
|
+
bridge_first_action_deadline = 0.0
|
|
3836
|
+
if (
|
|
3837
|
+
bridge_summary_path is not None
|
|
3838
|
+
and self._config.first_action_timeout_sec > 0
|
|
3839
|
+
):
|
|
3840
|
+
bridge_first_action_deadline = (
|
|
3841
|
+
time.monotonic()
|
|
3842
|
+
+ max(1.0, self._config.first_action_timeout_sec)
|
|
3843
|
+
)
|
|
3844
|
+
meaningful_progress_deadline = 0.0
|
|
3845
|
+
meaningful_progress_seen = False
|
|
3846
|
+
progress_route = self._config.route
|
|
3847
|
+
if self._config.app_server_goal_worker and progress_route == "unknown":
|
|
3848
|
+
progress_route = "codex-app-server-goal-baseline"
|
|
3849
|
+
meaningful_progress_required = (
|
|
3850
|
+
bridge_summary_path is not None
|
|
3851
|
+
and self._config.first_action_timeout_sec > 0
|
|
3852
|
+
and _prompt_requires_meaningful_bridge_progress(
|
|
3853
|
+
prompt_for_worker,
|
|
3854
|
+
route=progress_route,
|
|
3855
|
+
)
|
|
3856
|
+
)
|
|
3857
|
+
allow_loopx_closeout_progress = progress_route.startswith("loopx-")
|
|
3858
|
+
if meaningful_progress_required:
|
|
3859
|
+
meaningful_progress_deadline = (
|
|
3860
|
+
time.monotonic()
|
|
3861
|
+
+ max(1.0, self._config.first_action_timeout_sec)
|
|
3862
|
+
)
|
|
3863
|
+
task_output_progress_seen = False
|
|
3864
|
+
while proc.poll() is None:
|
|
3865
|
+
now = time.monotonic()
|
|
3866
|
+
if bridge_summary_path is not None:
|
|
3867
|
+
try:
|
|
3868
|
+
current_bridge_summary_size = (
|
|
3869
|
+
bridge_summary_path.stat().st_size
|
|
3870
|
+
)
|
|
3871
|
+
except OSError:
|
|
3872
|
+
current_bridge_summary_size = 0
|
|
3873
|
+
if current_bridge_summary_size > last_bridge_summary_size:
|
|
3874
|
+
last_bridge_summary_size = current_bridge_summary_size
|
|
3875
|
+
last_bridge_activity_at = now
|
|
3876
|
+
bridge_activity_seen = True
|
|
3877
|
+
if not meaningful_progress_seen:
|
|
3878
|
+
meaningful_progress_seen = (
|
|
3879
|
+
_bridge_summary_has_meaningful_agent_progress(
|
|
3880
|
+
bridge_summary_path,
|
|
3881
|
+
allow_loopx_closeout=allow_loopx_closeout_progress,
|
|
3882
|
+
)
|
|
3883
|
+
)
|
|
3884
|
+
if (
|
|
3885
|
+
not task_output_progress_seen
|
|
3886
|
+
and task_output_quiet_timeout_sec > 0
|
|
3887
|
+
):
|
|
3888
|
+
task_output_progress_seen = (
|
|
3889
|
+
_bridge_summary_has_successful_task_operation(
|
|
3890
|
+
bridge_summary_path
|
|
3891
|
+
)
|
|
3892
|
+
)
|
|
3893
|
+
if (
|
|
3894
|
+
not bridge_activity_seen
|
|
3895
|
+
and bridge_summary_path is not None
|
|
3896
|
+
and bridge_first_action_deadline
|
|
3897
|
+
and now >= bridge_first_action_deadline
|
|
3898
|
+
):
|
|
3899
|
+
self._terminate_codex_process(proc, grace_sec=2)
|
|
3900
|
+
stdout_text, stderr_text = proc.communicate(timeout=2)
|
|
3901
|
+
self._publish_remote_bridge_agent_operations_trace(
|
|
3902
|
+
bridge_summary_path=bridge_summary_path,
|
|
3903
|
+
)
|
|
3904
|
+
if not self._publish_worker_trace(output_json):
|
|
3905
|
+
self._publish_worker_failure_trace(
|
|
3906
|
+
stage="first_action_timeout",
|
|
3907
|
+
returncode=proc.returncode,
|
|
3908
|
+
stdout_text=stdout_text,
|
|
3909
|
+
stderr_text="codex_exec_first_action_timeout\n",
|
|
3910
|
+
)
|
|
3911
|
+
self._terminate_bridge_server_process(bridge_server_proc)
|
|
3912
|
+
return _recoverable_codex_turn_failure_message(
|
|
3913
|
+
"codex_exec_first_action_timeout"
|
|
3914
|
+
)
|
|
3915
|
+
if (
|
|
3916
|
+
meaningful_progress_required
|
|
3917
|
+
and not meaningful_progress_seen
|
|
3918
|
+
and bridge_summary_path is not None
|
|
3919
|
+
and meaningful_progress_deadline
|
|
3920
|
+
and now >= meaningful_progress_deadline
|
|
3921
|
+
):
|
|
3922
|
+
self._terminate_codex_process(proc, grace_sec=2)
|
|
3923
|
+
stdout_text, stderr_text = proc.communicate(timeout=2)
|
|
3924
|
+
self._publish_remote_bridge_agent_operations_trace(
|
|
3925
|
+
bridge_summary_path=bridge_summary_path,
|
|
3926
|
+
)
|
|
3927
|
+
if not self._publish_worker_trace(output_json):
|
|
3928
|
+
self._publish_worker_failure_trace(
|
|
3929
|
+
stage="meaningful_bridge_progress_timeout",
|
|
3930
|
+
returncode=proc.returncode,
|
|
3931
|
+
stdout_text=stdout_text,
|
|
3932
|
+
stderr_text="codex_exec_first_action_timeout\n",
|
|
3933
|
+
)
|
|
3934
|
+
self._terminate_bridge_server_process(bridge_server_proc)
|
|
3935
|
+
return _recoverable_codex_turn_failure_message(
|
|
3936
|
+
"codex_exec_first_action_timeout"
|
|
3937
|
+
)
|
|
3938
|
+
if (
|
|
3939
|
+
task_output_progress_seen
|
|
3940
|
+
and bridge_summary_path is not None
|
|
3941
|
+
and task_output_quiet_timeout_sec > 0
|
|
3942
|
+
and not _bridge_summary_has_inflight_operation(
|
|
3943
|
+
bridge_summary_path
|
|
3944
|
+
)
|
|
3945
|
+
and now - last_bridge_activity_at
|
|
3946
|
+
>= task_output_quiet_timeout_sec
|
|
3947
|
+
):
|
|
3948
|
+
self._terminate_codex_process(proc, grace_sec=2)
|
|
3949
|
+
stdout_text, stderr_text = proc.communicate(timeout=2)
|
|
3950
|
+
self._publish_remote_bridge_agent_operations_trace(
|
|
3951
|
+
bridge_summary_path=bridge_summary_path,
|
|
3952
|
+
)
|
|
3953
|
+
if not self._publish_worker_trace(output_json):
|
|
3954
|
+
self._publish_worker_failure_trace(
|
|
3955
|
+
stage="task_output_quiet_timeout",
|
|
3956
|
+
returncode=proc.returncode,
|
|
3957
|
+
stdout_text=stdout_text,
|
|
3958
|
+
stderr_text=(
|
|
3959
|
+
"codex_exec_task_output_quiet_timeout\n"
|
|
3960
|
+
),
|
|
3961
|
+
)
|
|
3962
|
+
self._terminate_bridge_server_process(bridge_server_proc)
|
|
3963
|
+
return _recoverable_codex_turn_failure_message(
|
|
3964
|
+
"codex_exec_task_output_quiet_timeout"
|
|
3965
|
+
)
|
|
3966
|
+
if (
|
|
3967
|
+
bridge_activity_seen
|
|
3968
|
+
and bridge_summary_path is not None
|
|
3969
|
+
and bridge_idle_timeout_sec > 0
|
|
3970
|
+
and not _bridge_summary_has_inflight_operation(
|
|
3971
|
+
bridge_summary_path
|
|
3972
|
+
)
|
|
3973
|
+
and now - last_bridge_activity_at >= bridge_idle_timeout_sec
|
|
3974
|
+
):
|
|
3975
|
+
self._terminate_codex_process(proc, grace_sec=2)
|
|
3976
|
+
stdout_text, stderr_text = proc.communicate(timeout=2)
|
|
3977
|
+
self._publish_remote_bridge_agent_operations_trace(
|
|
3978
|
+
bridge_summary_path=bridge_summary_path,
|
|
3979
|
+
)
|
|
3980
|
+
if not self._publish_worker_trace(output_json):
|
|
3981
|
+
self._publish_worker_failure_trace(
|
|
3982
|
+
stage="bridge_idle_timeout",
|
|
3983
|
+
returncode=proc.returncode,
|
|
3984
|
+
stdout_text=stdout_text,
|
|
3985
|
+
stderr_text=stderr_text,
|
|
3986
|
+
)
|
|
3987
|
+
self._terminate_bridge_server_process(bridge_server_proc)
|
|
3988
|
+
return _recoverable_codex_turn_failure_message(
|
|
3989
|
+
"codex_exec_bridge_idle_timeout"
|
|
3990
|
+
)
|
|
3991
|
+
if now >= deadline:
|
|
3992
|
+
self._terminate_codex_process(proc, grace_sec=2)
|
|
3993
|
+
stdout_text, stderr_text = proc.communicate(timeout=2)
|
|
3994
|
+
if bridge_summary_path is not None:
|
|
3995
|
+
self._publish_remote_bridge_agent_operations_trace(
|
|
3996
|
+
bridge_summary_path=bridge_summary_path,
|
|
3997
|
+
)
|
|
3998
|
+
if not self._publish_worker_trace(output_json):
|
|
3999
|
+
self._publish_worker_failure_trace(
|
|
4000
|
+
stage="timeout",
|
|
4001
|
+
returncode=proc.returncode,
|
|
4002
|
+
stdout_text=stdout_text,
|
|
4003
|
+
stderr_text=stderr_text,
|
|
4004
|
+
)
|
|
4005
|
+
self._terminate_bridge_server_process(bridge_server_proc)
|
|
4006
|
+
raise TimeoutError
|
|
4007
|
+
if now >= next_heartbeat:
|
|
4008
|
+
self._write_worker_heartbeat(
|
|
4009
|
+
stdout,
|
|
4010
|
+
session_id=session_id,
|
|
4011
|
+
text="host app-server goal worker still running",
|
|
4012
|
+
)
|
|
4013
|
+
next_heartbeat = (
|
|
4014
|
+
now + max(1.0, self._config.stream_heartbeat_interval_sec)
|
|
4015
|
+
)
|
|
4016
|
+
time.sleep(0.2)
|
|
4017
|
+
stdout_text, stderr_text = proc.communicate(timeout=5)
|
|
4018
|
+
except subprocess.TimeoutExpired as exc:
|
|
4019
|
+
self._terminate_codex_process(proc, grace_sec=2)
|
|
4020
|
+
if bridge_summary_path is not None:
|
|
4021
|
+
self._publish_remote_bridge_agent_operations_trace(
|
|
4022
|
+
bridge_summary_path=bridge_summary_path,
|
|
4023
|
+
)
|
|
4024
|
+
if not self._publish_worker_trace(output_json):
|
|
4025
|
+
self._publish_worker_failure_trace(
|
|
4026
|
+
stage="communicate_timeout",
|
|
4027
|
+
returncode=proc.returncode,
|
|
4028
|
+
stdout_text="",
|
|
4029
|
+
stderr_text="",
|
|
4030
|
+
)
|
|
4031
|
+
self._terminate_bridge_server_process(bridge_server_proc)
|
|
4032
|
+
raise TimeoutError from exc
|
|
4033
|
+
except BaseException:
|
|
4034
|
+
self._terminate_codex_process(proc, grace_sec=2)
|
|
4035
|
+
self._terminate_bridge_server_process(bridge_server_proc)
|
|
4036
|
+
raise
|
|
4037
|
+
if proc.returncode != 0:
|
|
4038
|
+
if bridge_summary_path is not None:
|
|
4039
|
+
self._publish_remote_bridge_agent_operations_trace(
|
|
4040
|
+
bridge_summary_path=bridge_summary_path,
|
|
4041
|
+
)
|
|
4042
|
+
if not self._publish_worker_trace(output_json):
|
|
4043
|
+
self._publish_worker_failure_trace(
|
|
4044
|
+
stage="worker_exit_nonzero_before_public_trace",
|
|
4045
|
+
returncode=proc.returncode,
|
|
4046
|
+
stdout_text=stdout_text,
|
|
4047
|
+
stderr_text=stderr_text,
|
|
4048
|
+
)
|
|
4049
|
+
self._terminate_bridge_server_process(bridge_server_proc)
|
|
4050
|
+
raise RuntimeError("host app-server goal worker failed")
|
|
4051
|
+
trace_required = bool(self._config.worker_public_trace_dir)
|
|
4052
|
+
trace_published = self._publish_worker_trace(output_json)
|
|
4053
|
+
if bridge_summary_path is not None:
|
|
4054
|
+
self._publish_remote_bridge_agent_operations_trace(
|
|
4055
|
+
bridge_summary_path=bridge_summary_path,
|
|
4056
|
+
)
|
|
4057
|
+
if trace_required and not trace_published:
|
|
4058
|
+
self._publish_worker_failure_trace(
|
|
4059
|
+
stage="worker_exit_zero_before_public_trace",
|
|
4060
|
+
returncode=proc.returncode,
|
|
4061
|
+
stdout_text=stdout_text,
|
|
4062
|
+
stderr_text=stderr_text,
|
|
4063
|
+
)
|
|
4064
|
+
self._terminate_bridge_server_process(bridge_server_proc)
|
|
4065
|
+
raise RuntimeError("host app-server goal worker public trace missing")
|
|
4066
|
+
try:
|
|
4067
|
+
response = response_path.read_text(encoding="utf-8").strip()
|
|
4068
|
+
except OSError as exc:
|
|
4069
|
+
self._terminate_bridge_server_process(bridge_server_proc)
|
|
4070
|
+
raise RuntimeError("host app-server goal worker response missing") from exc
|
|
4071
|
+
self._terminate_bridge_server_process(bridge_server_proc)
|
|
4072
|
+
return response or "host app-server goal worker returned an empty final message"
|
|
4073
|
+
|
|
4074
|
+
def _publish_worker_trace(self, output_json: Path) -> bool:
|
|
4075
|
+
"""Persist a public-safe app-server worker trace for the reducer.
|
|
4076
|
+
|
|
4077
|
+
The worker output is already compact, but this method still rewrites a
|
|
4078
|
+
smaller allowlisted shape so the benchmark run directory never needs the
|
|
4079
|
+
private response text or raw app-server stream.
|
|
4080
|
+
"""
|
|
4081
|
+
|
|
4082
|
+
if not self._config.worker_public_trace_dir:
|
|
4083
|
+
return False
|
|
4084
|
+
try:
|
|
4085
|
+
payload = json.loads(output_json.read_text(encoding="utf-8"))
|
|
4086
|
+
except (OSError, json.JSONDecodeError):
|
|
4087
|
+
return False
|
|
4088
|
+
if not isinstance(payload, dict):
|
|
4089
|
+
return False
|
|
4090
|
+
|
|
4091
|
+
def compact_dict(source: Any, allowed: tuple[str, ...]) -> dict[str, Any]:
|
|
4092
|
+
if not isinstance(source, dict):
|
|
4093
|
+
return {}
|
|
4094
|
+
result: dict[str, Any] = {}
|
|
4095
|
+
for key in allowed:
|
|
4096
|
+
value = source.get(key)
|
|
4097
|
+
if isinstance(value, (str, bool, int, float)) and not (
|
|
4098
|
+
isinstance(value, float) and (value != value)
|
|
4099
|
+
):
|
|
4100
|
+
result[key] = value
|
|
4101
|
+
return result
|
|
4102
|
+
|
|
4103
|
+
worker_contract = payload.get("worker_contract")
|
|
4104
|
+
if not isinstance(worker_contract, dict):
|
|
4105
|
+
worker_contract = {}
|
|
4106
|
+
worker_adapter = (
|
|
4107
|
+
worker_contract.get("worker_adapter")
|
|
4108
|
+
if isinstance(worker_contract.get("worker_adapter"), dict)
|
|
4109
|
+
else {}
|
|
4110
|
+
)
|
|
4111
|
+
trace = {
|
|
4112
|
+
"schema_version": "skillsbench_host_codex_goal_worker_public_trace_v0",
|
|
4113
|
+
"ok": payload.get("ok") is True,
|
|
4114
|
+
"route": "codex-app-server-goal-baseline",
|
|
4115
|
+
"benchmark_id": str(payload.get("benchmark_id") or ""),
|
|
4116
|
+
"task_id": str(payload.get("task_id") or ""),
|
|
4117
|
+
"worker_contract": compact_dict(
|
|
4118
|
+
worker_contract,
|
|
4119
|
+
(
|
|
4120
|
+
"schema_version",
|
|
4121
|
+
"route",
|
|
4122
|
+
"ready",
|
|
4123
|
+
"runner_integration_ready",
|
|
4124
|
+
"first_blocker",
|
|
4125
|
+
),
|
|
4126
|
+
),
|
|
4127
|
+
"worker_adapter": compact_dict(
|
|
4128
|
+
worker_adapter,
|
|
4129
|
+
(
|
|
4130
|
+
"reasoning_effort",
|
|
4131
|
+
"prompt_style",
|
|
4132
|
+
"agent_execution_mode",
|
|
4133
|
+
"worker_surface",
|
|
4134
|
+
"context_only_followup_supported",
|
|
4135
|
+
),
|
|
4136
|
+
),
|
|
4137
|
+
"prompt": compact_dict(
|
|
4138
|
+
payload.get("prompt"),
|
|
4139
|
+
("sha256", "chars", "raw_recorded", "style"),
|
|
4140
|
+
),
|
|
4141
|
+
"turn": compact_dict(
|
|
4142
|
+
payload.get("turn"),
|
|
4143
|
+
(
|
|
4144
|
+
"schema_version",
|
|
4145
|
+
"thread_id_present",
|
|
4146
|
+
"goal_get_present",
|
|
4147
|
+
"goal_status",
|
|
4148
|
+
"turn_id_present",
|
|
4149
|
+
"turn_id_source",
|
|
4150
|
+
"turn_start_response_turn_id_present",
|
|
4151
|
+
"turn_event_stream_turn_id_present",
|
|
4152
|
+
"turn_status",
|
|
4153
|
+
"turn_completed_observed",
|
|
4154
|
+
"agent_message_delta_count",
|
|
4155
|
+
"agent_message_item_count",
|
|
4156
|
+
"item_completed_count",
|
|
4157
|
+
"assistant_message_present",
|
|
4158
|
+
"assistant_message_chars",
|
|
4159
|
+
"completion_hard_gate",
|
|
4160
|
+
"completion_source_of_truth",
|
|
4161
|
+
"first_action_timeout_sec",
|
|
4162
|
+
"first_action_observed",
|
|
4163
|
+
"effective_action_observed",
|
|
4164
|
+
"assistant_message_context_only",
|
|
4165
|
+
"post_context_assistant_chars",
|
|
4166
|
+
"turn_attempt_count",
|
|
4167
|
+
"turn_completed_attempt_count",
|
|
4168
|
+
"assistant_message_attempt_count",
|
|
4169
|
+
"context_only_turn_count",
|
|
4170
|
+
"context_only_followup_max",
|
|
4171
|
+
"context_only_recovery_attempted",
|
|
4172
|
+
"context_only_recovery_succeeded",
|
|
4173
|
+
"context_only_followup_start_attempted",
|
|
4174
|
+
"context_only_followup_start_succeeded",
|
|
4175
|
+
"context_only_followup_start_error_type",
|
|
4176
|
+
"normal_followup_max",
|
|
4177
|
+
"normal_followup_attempted",
|
|
4178
|
+
"normal_followup_succeeded",
|
|
4179
|
+
"normal_followup_start_attempted_count",
|
|
4180
|
+
"normal_followup_start_succeeded_count",
|
|
4181
|
+
"normal_followup_start_error_type",
|
|
4182
|
+
"transport_reconnect_attempted",
|
|
4183
|
+
"transport_reconnect_succeeded",
|
|
4184
|
+
"transport_reconnect_reason",
|
|
4185
|
+
"goal_reactivation_attempted",
|
|
4186
|
+
"goal_reactivation_succeeded",
|
|
4187
|
+
"goal_reactivation_previous_status",
|
|
4188
|
+
"goal_reactivation_result_status",
|
|
4189
|
+
"reasoning_effort",
|
|
4190
|
+
"raw_transcript_recorded",
|
|
4191
|
+
"raw_assistant_message_recorded",
|
|
4192
|
+
),
|
|
4193
|
+
),
|
|
4194
|
+
"context_only_recovery": compact_dict(
|
|
4195
|
+
payload.get("context_only_recovery"),
|
|
4196
|
+
(
|
|
4197
|
+
"enabled",
|
|
4198
|
+
"max_followups",
|
|
4199
|
+
"attempted",
|
|
4200
|
+
"succeeded",
|
|
4201
|
+
"followup_start_attempted",
|
|
4202
|
+
"followup_start_succeeded",
|
|
4203
|
+
"followup_start_error_type",
|
|
4204
|
+
"context_only_turn_count",
|
|
4205
|
+
"turn_attempt_count",
|
|
4206
|
+
),
|
|
4207
|
+
),
|
|
4208
|
+
"normal_followup": compact_dict(
|
|
4209
|
+
payload.get("normal_followup"),
|
|
4210
|
+
(
|
|
4211
|
+
"enabled",
|
|
4212
|
+
"max_followups",
|
|
4213
|
+
"attempted",
|
|
4214
|
+
"succeeded",
|
|
4215
|
+
"followup_start_attempted_count",
|
|
4216
|
+
"followup_start_succeeded_count",
|
|
4217
|
+
"followup_start_error_type",
|
|
4218
|
+
"turn_attempt_count",
|
|
4219
|
+
"reward_feedback_provided",
|
|
4220
|
+
"verifier_feedback_provided",
|
|
4221
|
+
),
|
|
4222
|
+
),
|
|
4223
|
+
"worker_result": compact_dict(
|
|
4224
|
+
payload,
|
|
4225
|
+
(
|
|
4226
|
+
"ok",
|
|
4227
|
+
"error_type",
|
|
4228
|
+
),
|
|
4229
|
+
),
|
|
4230
|
+
"private_response_text": compact_dict(
|
|
4231
|
+
payload.get("private_response_text"),
|
|
4232
|
+
("written", "path_recorded", "raw_recorded_in_public_json"),
|
|
4233
|
+
),
|
|
4234
|
+
"boundary": compact_dict(
|
|
4235
|
+
payload.get("boundary"),
|
|
4236
|
+
(
|
|
4237
|
+
"raw_task_text_recorded",
|
|
4238
|
+
"raw_logs_recorded",
|
|
4239
|
+
"raw_trajectory_recorded",
|
|
4240
|
+
"credential_values_recorded",
|
|
4241
|
+
"host_paths_recorded",
|
|
4242
|
+
),
|
|
4243
|
+
),
|
|
4244
|
+
}
|
|
4245
|
+
self._write_worker_public_trace(trace)
|
|
4246
|
+
return True
|
|
4247
|
+
|
|
4248
|
+
def _publish_worker_failure_trace(
|
|
4249
|
+
self,
|
|
4250
|
+
*,
|
|
4251
|
+
stage: str,
|
|
4252
|
+
returncode: int | None,
|
|
4253
|
+
stdout_text: str,
|
|
4254
|
+
stderr_text: str,
|
|
4255
|
+
) -> None:
|
|
4256
|
+
"""Persist compact failure evidence when the host worker writes no trace."""
|
|
4257
|
+
|
|
4258
|
+
if not self._config.worker_public_trace_dir:
|
|
4259
|
+
return
|
|
4260
|
+
safe_stage = "".join(
|
|
4261
|
+
ch if ch.isalnum() or ch == "_" else "_"
|
|
4262
|
+
for ch in str(stage or "").strip().lower()
|
|
4263
|
+
)
|
|
4264
|
+
if not safe_stage:
|
|
4265
|
+
safe_stage = "worker_failed_before_public_trace"
|
|
4266
|
+
failure_category = _codex_exec_failure_category(
|
|
4267
|
+
returncode=returncode,
|
|
4268
|
+
stderr_text=stderr_text,
|
|
4269
|
+
)
|
|
4270
|
+
trace = {
|
|
4271
|
+
"schema_version": "skillsbench_host_codex_goal_worker_public_trace_v0",
|
|
4272
|
+
"ok": False,
|
|
4273
|
+
"route": "codex-app-server-goal-baseline",
|
|
4274
|
+
"trace_kind": "host_worker_process_failure",
|
|
4275
|
+
"benchmark_id": self._config.dataset,
|
|
4276
|
+
"task_id": self._config.task_id,
|
|
4277
|
+
"worker_process": {
|
|
4278
|
+
"schema_version": "skillsbench_host_worker_process_failure_v0",
|
|
4279
|
+
"stage": safe_stage,
|
|
4280
|
+
"failure_category": failure_category,
|
|
4281
|
+
"returncode": returncode
|
|
4282
|
+
if isinstance(returncode, int) and not isinstance(returncode, bool)
|
|
4283
|
+
else None,
|
|
4284
|
+
"stdout_bytes": len((stdout_text or "").encode("utf-8")),
|
|
4285
|
+
"stderr_bytes": len((stderr_text or "").encode("utf-8")),
|
|
4286
|
+
"raw_stdout_recorded": False,
|
|
4287
|
+
"raw_stderr_recorded": False,
|
|
4288
|
+
"host_paths_recorded": False,
|
|
4289
|
+
},
|
|
4290
|
+
"worker_contract": {
|
|
4291
|
+
"schema_version": "skillsbench_app_server_goal_worker_contract_v0",
|
|
4292
|
+
"route": "codex-app-server-goal-baseline",
|
|
4293
|
+
"ready": False,
|
|
4294
|
+
"runner_integration_ready": True,
|
|
4295
|
+
"first_blocker": safe_stage,
|
|
4296
|
+
},
|
|
4297
|
+
"prompt": {"raw_recorded": False},
|
|
4298
|
+
"turn": {
|
|
4299
|
+
"thread_id_present": False,
|
|
4300
|
+
"goal_get_present": False,
|
|
4301
|
+
"turn_id_present": False,
|
|
4302
|
+
"turn_completed_observed": False,
|
|
4303
|
+
"assistant_message_present": False,
|
|
4304
|
+
"raw_transcript_recorded": False,
|
|
4305
|
+
"raw_assistant_message_recorded": False,
|
|
4306
|
+
},
|
|
4307
|
+
"private_response_text": {
|
|
4308
|
+
"written": False,
|
|
4309
|
+
"path_recorded": False,
|
|
4310
|
+
"raw_recorded_in_public_json": False,
|
|
4311
|
+
},
|
|
4312
|
+
"boundary": {
|
|
4313
|
+
"raw_task_text_recorded": False,
|
|
4314
|
+
"raw_logs_recorded": False,
|
|
4315
|
+
"raw_trajectory_recorded": False,
|
|
4316
|
+
"credential_values_recorded": False,
|
|
4317
|
+
"host_paths_recorded": False,
|
|
4318
|
+
},
|
|
4319
|
+
}
|
|
4320
|
+
self._write_worker_public_trace(trace)
|
|
4321
|
+
|
|
4322
|
+
def _publish_worker_lifecycle_trace(self, stage: str) -> None:
|
|
4323
|
+
"""Record public-safe relay lifecycle evidence before any prompt runs."""
|
|
4324
|
+
|
|
4325
|
+
if (
|
|
4326
|
+
not self._config.app_server_goal_worker
|
|
4327
|
+
or not self._config.worker_public_trace_dir
|
|
4328
|
+
):
|
|
4329
|
+
return
|
|
4330
|
+
safe_stage = "".join(
|
|
4331
|
+
ch if ch.isalnum() or ch == "_" else "_"
|
|
4332
|
+
for ch in str(stage or "").strip().lower()
|
|
4333
|
+
)
|
|
4334
|
+
if not safe_stage:
|
|
4335
|
+
safe_stage = "unknown"
|
|
4336
|
+
if safe_stage in self._published_lifecycle_stages:
|
|
4337
|
+
return
|
|
4338
|
+
self._published_lifecycle_stages.add(safe_stage)
|
|
4339
|
+
trace = {
|
|
4340
|
+
"schema_version": "skillsbench_host_codex_goal_worker_public_trace_v0",
|
|
4341
|
+
"ok": False,
|
|
4342
|
+
"route": "codex-app-server-goal-baseline",
|
|
4343
|
+
"trace_kind": "relay_lifecycle",
|
|
4344
|
+
"benchmark_id": self._config.dataset,
|
|
4345
|
+
"task_id": self._config.task_id,
|
|
4346
|
+
"relay": {
|
|
4347
|
+
"schema_version": "skillsbench_app_server_goal_worker_lifecycle_trace_v0",
|
|
4348
|
+
"stage": safe_stage,
|
|
4349
|
+
"app_server_goal_worker": True,
|
|
4350
|
+
"worker_public_trace_configured": True,
|
|
4351
|
+
"raw_prompt_recorded": False,
|
|
4352
|
+
"raw_stdout_recorded": False,
|
|
4353
|
+
"raw_stderr_recorded": False,
|
|
4354
|
+
"host_paths_recorded": False,
|
|
4355
|
+
},
|
|
4356
|
+
"worker_contract": {
|
|
4357
|
+
"schema_version": "skillsbench_app_server_goal_worker_contract_v0",
|
|
4358
|
+
"route": "codex-app-server-goal-baseline",
|
|
4359
|
+
"ready": False,
|
|
4360
|
+
"runner_integration_ready": True,
|
|
4361
|
+
"first_blocker": "relay_lifecycle_only_no_worker_turn_yet",
|
|
4362
|
+
},
|
|
4363
|
+
"prompt": {"raw_recorded": False},
|
|
4364
|
+
"turn": {
|
|
4365
|
+
"thread_id_present": False,
|
|
4366
|
+
"goal_get_present": False,
|
|
4367
|
+
"turn_id_present": False,
|
|
4368
|
+
"turn_completed_observed": False,
|
|
4369
|
+
"assistant_message_present": False,
|
|
4370
|
+
"raw_transcript_recorded": False,
|
|
4371
|
+
"raw_assistant_message_recorded": False,
|
|
4372
|
+
},
|
|
4373
|
+
"private_response_text": {
|
|
4374
|
+
"written": False,
|
|
4375
|
+
"path_recorded": False,
|
|
4376
|
+
"raw_recorded_in_public_json": False,
|
|
4377
|
+
},
|
|
4378
|
+
"boundary": {
|
|
4379
|
+
"raw_task_text_recorded": False,
|
|
4380
|
+
"raw_logs_recorded": False,
|
|
4381
|
+
"raw_trajectory_recorded": False,
|
|
4382
|
+
"credential_values_recorded": False,
|
|
4383
|
+
"host_paths_recorded": False,
|
|
4384
|
+
},
|
|
4385
|
+
}
|
|
4386
|
+
self._write_worker_public_trace(trace)
|
|
4387
|
+
|
|
4388
|
+
def _write_worker_public_trace(self, trace: dict[str, Any]) -> None:
|
|
4389
|
+
if not self._config.worker_public_trace_dir:
|
|
4390
|
+
return
|
|
4391
|
+
try:
|
|
4392
|
+
trace_dir = Path(self._config.worker_public_trace_dir).expanduser()
|
|
4393
|
+
trace_dir.mkdir(parents=True, exist_ok=True)
|
|
4394
|
+
trace_path = trace_dir / f"worker-{uuid.uuid4().hex[:12]}.compact.json"
|
|
4395
|
+
trace_path.write_text(
|
|
4396
|
+
json.dumps(trace, indent=2, sort_keys=True) + "\n",
|
|
4397
|
+
encoding="utf-8",
|
|
4398
|
+
)
|
|
4399
|
+
except OSError:
|
|
4400
|
+
return
|
|
4401
|
+
|
|
4402
|
+
def _write_worker_heartbeat(
|
|
4403
|
+
self,
|
|
4404
|
+
stdout: TextIO,
|
|
4405
|
+
*,
|
|
4406
|
+
session_id: str,
|
|
4407
|
+
text: str,
|
|
4408
|
+
) -> None:
|
|
4409
|
+
"""Emit a public-safe ACP activity heartbeat while the host worker runs."""
|
|
4410
|
+
|
|
4411
|
+
self._write(
|
|
4412
|
+
stdout,
|
|
4413
|
+
{
|
|
4414
|
+
"jsonrpc": "2.0",
|
|
4415
|
+
"method": "session/update",
|
|
4416
|
+
"params": {
|
|
4417
|
+
"sessionId": session_id,
|
|
4418
|
+
"update": {
|
|
4419
|
+
"sessionUpdate": "agent_thought_chunk",
|
|
4420
|
+
"content": {"type": "text", "text": text},
|
|
4421
|
+
},
|
|
4422
|
+
},
|
|
4423
|
+
},
|
|
4424
|
+
)
|
|
4425
|
+
|
|
4426
|
+
@staticmethod
|
|
4427
|
+
def _write(stdout: TextIO, message: dict[str, Any]) -> None:
|
|
4428
|
+
stdout.write(json.dumps(message, separators=(",", ":")) + "\n")
|
|
4429
|
+
stdout.flush()
|
|
4430
|
+
|
|
4431
|
+
|
|
4432
|
+
def default_skillsbench_local_acp_relay_command() -> list[str]:
|
|
4433
|
+
script = (
|
|
4434
|
+
Path(__file__).resolve().parents[2]
|
|
4435
|
+
/ "scripts"
|
|
4436
|
+
/ "skillsbench_local_acp_relay.py"
|
|
4437
|
+
)
|
|
4438
|
+
return [
|
|
4439
|
+
sys.executable,
|
|
4440
|
+
str(script),
|
|
4441
|
+
"--dry-run-response",
|
|
4442
|
+
SKILLSBENCH_LOCAL_ACP_RELAY_READY_MARKER,
|
|
4443
|
+
]
|
|
4444
|
+
|
|
4445
|
+
|
|
4446
|
+
def run_skillsbench_local_acp_relay_probe(
|
|
4447
|
+
command: str | list[str] | tuple[str, ...] | None = None,
|
|
4448
|
+
*,
|
|
4449
|
+
timeout_sec: float = 10.0,
|
|
4450
|
+
prompt_text: str | None = None,
|
|
4451
|
+
required_response_marker: str | None = None,
|
|
4452
|
+
model_id: str | None = "probe-model",
|
|
4453
|
+
env: dict[str, str] | None = None,
|
|
4454
|
+
) -> dict[str, Any]:
|
|
4455
|
+
argv = (
|
|
4456
|
+
_command_to_argv(command)
|
|
4457
|
+
if command
|
|
4458
|
+
else default_skillsbench_local_acp_relay_command()
|
|
4459
|
+
)
|
|
4460
|
+
started = time.monotonic()
|
|
4461
|
+
proc: subprocess.Popen[bytes] | None = None
|
|
4462
|
+
stage = "spawn"
|
|
4463
|
+
request_count = 0
|
|
4464
|
+
try:
|
|
4465
|
+
proc = subprocess.Popen(
|
|
4466
|
+
argv,
|
|
4467
|
+
stdin=subprocess.PIPE,
|
|
4468
|
+
stdout=subprocess.PIPE,
|
|
4469
|
+
stderr=subprocess.PIPE,
|
|
4470
|
+
env=({**os.environ, **env} if env else None),
|
|
4471
|
+
)
|
|
4472
|
+
stage = "initialize"
|
|
4473
|
+
initialize = _probe_request(
|
|
4474
|
+
proc,
|
|
4475
|
+
{"jsonrpc": "2.0", "id": 1, "method": "initialize", "params": {}},
|
|
4476
|
+
timeout_at=started + timeout_sec,
|
|
4477
|
+
)
|
|
4478
|
+
request_count += 1
|
|
4479
|
+
stage = "session_new"
|
|
4480
|
+
session = _probe_request(
|
|
4481
|
+
proc,
|
|
4482
|
+
{
|
|
4483
|
+
"jsonrpc": "2.0",
|
|
4484
|
+
"id": 2,
|
|
4485
|
+
"method": "session/new",
|
|
4486
|
+
"params": {"cwd": os.getcwd(), "mcpServers": []},
|
|
4487
|
+
},
|
|
4488
|
+
timeout_at=started + timeout_sec,
|
|
4489
|
+
)
|
|
4490
|
+
request_count += 1
|
|
4491
|
+
session_id = str(session.get("result", {}).get("sessionId") or "")
|
|
4492
|
+
next_id = 3
|
|
4493
|
+
if model_id:
|
|
4494
|
+
stage = "set_model"
|
|
4495
|
+
_probe_request(
|
|
4496
|
+
proc,
|
|
4497
|
+
{
|
|
4498
|
+
"jsonrpc": "2.0",
|
|
4499
|
+
"id": next_id,
|
|
4500
|
+
"method": "session/set_model",
|
|
4501
|
+
"params": {"sessionId": session_id, "modelId": str(model_id)},
|
|
4502
|
+
},
|
|
4503
|
+
timeout_at=started + timeout_sec,
|
|
4504
|
+
)
|
|
4505
|
+
request_count += 1
|
|
4506
|
+
next_id += 1
|
|
4507
|
+
stage = "prompt"
|
|
4508
|
+
agent_message_chunks: list[str] = []
|
|
4509
|
+
prompt = _probe_request(
|
|
4510
|
+
proc,
|
|
4511
|
+
{
|
|
4512
|
+
"jsonrpc": "2.0",
|
|
4513
|
+
"id": next_id,
|
|
4514
|
+
"method": "session/prompt",
|
|
4515
|
+
"params": {
|
|
4516
|
+
"sessionId": session_id,
|
|
4517
|
+
"prompt": [
|
|
4518
|
+
{
|
|
4519
|
+
"type": "text",
|
|
4520
|
+
"text": (
|
|
4521
|
+
prompt_text
|
|
4522
|
+
or SKILLSBENCH_LOCAL_ACP_RELAY_HEALTH_PROMPT
|
|
4523
|
+
),
|
|
4524
|
+
}
|
|
4525
|
+
],
|
|
4526
|
+
},
|
|
4527
|
+
},
|
|
4528
|
+
timeout_at=started + timeout_sec,
|
|
4529
|
+
agent_message_chunks=agent_message_chunks,
|
|
4530
|
+
)
|
|
4531
|
+
request_count += 1
|
|
4532
|
+
prompt_usage = (
|
|
4533
|
+
prompt.get("result", {}).get("usage")
|
|
4534
|
+
if isinstance(prompt.get("result"), dict)
|
|
4535
|
+
else {}
|
|
4536
|
+
)
|
|
4537
|
+
usage_total = (
|
|
4538
|
+
prompt_usage.get("totalTokens") if isinstance(prompt_usage, dict) else None
|
|
4539
|
+
)
|
|
4540
|
+
usage_ready = isinstance(usage_total, int) and not isinstance(
|
|
4541
|
+
usage_total, bool
|
|
4542
|
+
) and usage_total > 0
|
|
4543
|
+
agent_message_present = bool("".join(agent_message_chunks).strip())
|
|
4544
|
+
response_marker_observed = True
|
|
4545
|
+
if required_response_marker:
|
|
4546
|
+
response_marker_observed = (
|
|
4547
|
+
required_response_marker in "".join(agent_message_chunks)
|
|
4548
|
+
)
|
|
4549
|
+
ready = (
|
|
4550
|
+
initialize.get("result", {}).get("agentInfo", {}).get("name")
|
|
4551
|
+
== "loopx-skillsbench-local-acp-relay"
|
|
4552
|
+
and bool(session_id)
|
|
4553
|
+
and prompt.get("result", {}).get("stopReason") == "end_turn"
|
|
4554
|
+
and usage_ready
|
|
4555
|
+
and response_marker_observed
|
|
4556
|
+
)
|
|
4557
|
+
first_blocker = "skillsbench_local_acp_relay_ready"
|
|
4558
|
+
if not ready:
|
|
4559
|
+
first_blocker = "skillsbench_local_acp_relay_probe_failed"
|
|
4560
|
+
if required_response_marker and not response_marker_observed:
|
|
4561
|
+
first_blocker = "skillsbench_local_acp_relay_response_marker_missing"
|
|
4562
|
+
return _relay_probe_payload(
|
|
4563
|
+
ready=ready,
|
|
4564
|
+
first_blocker=first_blocker,
|
|
4565
|
+
stage="complete",
|
|
4566
|
+
request_count=request_count,
|
|
4567
|
+
prompt_usage_total_tokens=usage_total if usage_ready else 0,
|
|
4568
|
+
response_marker_required=bool(required_response_marker),
|
|
4569
|
+
response_marker_observed=response_marker_observed,
|
|
4570
|
+
agent_message_present=agent_message_present,
|
|
4571
|
+
)
|
|
4572
|
+
except (OSError, RuntimeError, TimeoutError, json.JSONDecodeError):
|
|
4573
|
+
return _relay_probe_payload(
|
|
4574
|
+
ready=False,
|
|
4575
|
+
first_blocker=f"skillsbench_local_acp_relay_{stage}_failed",
|
|
4576
|
+
stage=stage,
|
|
4577
|
+
request_count=0,
|
|
4578
|
+
prompt_usage_total_tokens=0,
|
|
4579
|
+
response_marker_required=bool(required_response_marker),
|
|
4580
|
+
response_marker_observed=False,
|
|
4581
|
+
agent_message_present=False,
|
|
4582
|
+
)
|
|
4583
|
+
finally:
|
|
4584
|
+
if proc is not None:
|
|
4585
|
+
_terminate_probe_process(proc)
|
|
4586
|
+
|
|
4587
|
+
|
|
4588
|
+
def run_skillsbench_host_local_acp_transport_probe(
|
|
4589
|
+
command: str | list[str] | tuple[str, ...] | None = None,
|
|
4590
|
+
*,
|
|
4591
|
+
skillsbench_root: str | Path | None = None,
|
|
4592
|
+
timeout_sec: float = 10.0,
|
|
4593
|
+
cwd: str | Path | None = None,
|
|
4594
|
+
) -> dict[str, Any]:
|
|
4595
|
+
"""Probe the real BenchFlow ACPClient against a host-local ACP relay.
|
|
4596
|
+
|
|
4597
|
+
This proves the host-local stdio transport route without invoking Codex,
|
|
4598
|
+
reading task text, launching a task sandbox, or recording raw ACP traffic.
|
|
4599
|
+
"""
|
|
4600
|
+
|
|
4601
|
+
argv = (
|
|
4602
|
+
_command_to_argv(command)
|
|
4603
|
+
if command
|
|
4604
|
+
else default_skillsbench_local_acp_relay_command()
|
|
4605
|
+
)
|
|
4606
|
+
stage = {"value": "import"}
|
|
4607
|
+
try:
|
|
4608
|
+
_prepend_optional_skillsbench_site_packages(skillsbench_root)
|
|
4609
|
+
from benchflow.acp.client import ACPClient
|
|
4610
|
+
from benchflow.acp.transport import StdioTransport
|
|
4611
|
+
|
|
4612
|
+
async def probe() -> dict[str, Any]:
|
|
4613
|
+
client = ACPClient(
|
|
4614
|
+
StdioTransport(
|
|
4615
|
+
command=argv[0],
|
|
4616
|
+
args=argv[1:],
|
|
4617
|
+
cwd=str(cwd) if cwd is not None else None,
|
|
4618
|
+
)
|
|
4619
|
+
)
|
|
4620
|
+
|
|
4621
|
+
async def step(name: str, awaitable: Any) -> Any:
|
|
4622
|
+
stage["value"] = name
|
|
4623
|
+
return await asyncio.wait_for(awaitable, timeout=timeout_sec)
|
|
4624
|
+
|
|
4625
|
+
request_count = 0
|
|
4626
|
+
try:
|
|
4627
|
+
await step("connect", client.connect())
|
|
4628
|
+
initialize = await step("initialize", client.initialize())
|
|
4629
|
+
request_count += 1
|
|
4630
|
+
session = await step(
|
|
4631
|
+
"session_new",
|
|
4632
|
+
client.session_new(
|
|
4633
|
+
cwd=str(cwd) if cwd is not None else os.getcwd()
|
|
4634
|
+
),
|
|
4635
|
+
)
|
|
4636
|
+
request_count += 1
|
|
4637
|
+
await step("set_model", client.set_model("probe-model"))
|
|
4638
|
+
request_count += 1
|
|
4639
|
+
prompt = await step(
|
|
4640
|
+
"prompt",
|
|
4641
|
+
client.prompt(SKILLSBENCH_LOCAL_ACP_RELAY_HEALTH_PROMPT),
|
|
4642
|
+
)
|
|
4643
|
+
request_count += 1
|
|
4644
|
+
agent_name = str(getattr(initialize.agent_info, "name", "") or "")
|
|
4645
|
+
session_id = str(getattr(session, "session_id", "") or "")
|
|
4646
|
+
stop_reason = str(getattr(prompt, "stop_reason", "") or "")
|
|
4647
|
+
ready = (
|
|
4648
|
+
agent_name == "loopx-skillsbench-local-acp-relay"
|
|
4649
|
+
and bool(session_id)
|
|
4650
|
+
and stop_reason == "end_turn"
|
|
4651
|
+
)
|
|
4652
|
+
return _host_local_transport_probe_payload(
|
|
4653
|
+
ready=ready,
|
|
4654
|
+
first_blocker=(
|
|
4655
|
+
"skillsbench_host_local_acp_transport_ready"
|
|
4656
|
+
if ready
|
|
4657
|
+
else "skillsbench_host_local_acp_transport_probe_failed"
|
|
4658
|
+
),
|
|
4659
|
+
stage="complete",
|
|
4660
|
+
request_count=request_count,
|
|
4661
|
+
benchflow_acp_client_used=True,
|
|
4662
|
+
)
|
|
4663
|
+
finally:
|
|
4664
|
+
await client.close()
|
|
4665
|
+
|
|
4666
|
+
return asyncio.run(probe())
|
|
4667
|
+
except Exception:
|
|
4668
|
+
return _host_local_transport_probe_payload(
|
|
4669
|
+
ready=False,
|
|
4670
|
+
first_blocker=(
|
|
4671
|
+
f"skillsbench_host_local_acp_transport_{stage['value']}_failed"
|
|
4672
|
+
),
|
|
4673
|
+
stage=stage["value"],
|
|
4674
|
+
request_count=0,
|
|
4675
|
+
benchflow_acp_client_used=stage["value"] != "import",
|
|
4676
|
+
)
|
|
4677
|
+
|
|
4678
|
+
|
|
4679
|
+
def _command_to_argv(command: str | list[str] | tuple[str, ...]) -> list[str]:
|
|
4680
|
+
if isinstance(command, str):
|
|
4681
|
+
return shlex.split(command)
|
|
4682
|
+
return [str(part) for part in command]
|
|
4683
|
+
|
|
4684
|
+
|
|
4685
|
+
def _prepend_optional_skillsbench_site_packages(
|
|
4686
|
+
skillsbench_root: str | Path | None,
|
|
4687
|
+
) -> None:
|
|
4688
|
+
if skillsbench_root is None:
|
|
4689
|
+
return
|
|
4690
|
+
root = Path(skillsbench_root).expanduser()
|
|
4691
|
+
venv = root / ".venv"
|
|
4692
|
+
if not venv.exists():
|
|
4693
|
+
return
|
|
4694
|
+
for candidate in sorted((venv / "lib").glob("python*/site-packages")):
|
|
4695
|
+
if candidate.exists():
|
|
4696
|
+
candidate_text = str(candidate)
|
|
4697
|
+
if candidate_text not in sys.path:
|
|
4698
|
+
sys.path.insert(0, candidate_text)
|
|
4699
|
+
return
|
|
4700
|
+
|
|
4701
|
+
|
|
4702
|
+
def _probe_request(
|
|
4703
|
+
proc: subprocess.Popen[bytes],
|
|
4704
|
+
message: dict[str, Any],
|
|
4705
|
+
*,
|
|
4706
|
+
timeout_at: float,
|
|
4707
|
+
agent_message_chunks: list[str] | None = None,
|
|
4708
|
+
) -> dict[str, Any]:
|
|
4709
|
+
if proc.stdin is None or proc.stdout is None:
|
|
4710
|
+
raise RuntimeError("probe process pipes missing")
|
|
4711
|
+
proc.stdin.write((json.dumps(message) + "\n").encode())
|
|
4712
|
+
proc.stdin.flush()
|
|
4713
|
+
selector = selectors.DefaultSelector()
|
|
4714
|
+
selector.register(proc.stdout, selectors.EVENT_READ)
|
|
4715
|
+
pending = b""
|
|
4716
|
+
try:
|
|
4717
|
+
while True:
|
|
4718
|
+
remaining = timeout_at - time.monotonic()
|
|
4719
|
+
if remaining <= 0:
|
|
4720
|
+
raise TimeoutError("ACP relay probe timed out")
|
|
4721
|
+
events = selector.select(timeout=remaining)
|
|
4722
|
+
if not events:
|
|
4723
|
+
raise TimeoutError("ACP relay probe timed out")
|
|
4724
|
+
chunk = os.read(proc.stdout.fileno(), 65536)
|
|
4725
|
+
if not chunk:
|
|
4726
|
+
raise RuntimeError("ACP relay closed stdout")
|
|
4727
|
+
pending += chunk
|
|
4728
|
+
while b"\n" in pending:
|
|
4729
|
+
raw_line, pending = pending.split(b"\n", 1)
|
|
4730
|
+
if not raw_line.strip():
|
|
4731
|
+
continue
|
|
4732
|
+
decoded = json.loads(raw_line.decode())
|
|
4733
|
+
if (
|
|
4734
|
+
agent_message_chunks is not None
|
|
4735
|
+
and isinstance(decoded, dict)
|
|
4736
|
+
and decoded.get("jsonrpc") == "2.0"
|
|
4737
|
+
and decoded.get("method") == "session/update"
|
|
4738
|
+
):
|
|
4739
|
+
params = decoded.get("params")
|
|
4740
|
+
update = (
|
|
4741
|
+
params.get("update") if isinstance(params, dict) else None
|
|
4742
|
+
)
|
|
4743
|
+
content = (
|
|
4744
|
+
update.get("content") if isinstance(update, dict) else None
|
|
4745
|
+
)
|
|
4746
|
+
text = content.get("text") if isinstance(content, dict) else None
|
|
4747
|
+
if isinstance(text, str):
|
|
4748
|
+
agent_message_chunks.append(text)
|
|
4749
|
+
if (
|
|
4750
|
+
isinstance(decoded, dict)
|
|
4751
|
+
and decoded.get("jsonrpc") == "2.0"
|
|
4752
|
+
and decoded.get("id") == message.get("id")
|
|
4753
|
+
):
|
|
4754
|
+
if decoded.get("error"):
|
|
4755
|
+
raise RuntimeError(
|
|
4756
|
+
str(decoded["error"].get("message") or "error")
|
|
4757
|
+
)
|
|
4758
|
+
return decoded
|
|
4759
|
+
finally:
|
|
4760
|
+
selector.close()
|
|
4761
|
+
|
|
4762
|
+
|
|
4763
|
+
def _terminate_probe_process(proc: subprocess.Popen[bytes]) -> None:
|
|
4764
|
+
if proc.stdin:
|
|
4765
|
+
try:
|
|
4766
|
+
proc.stdin.close()
|
|
4767
|
+
except OSError:
|
|
4768
|
+
pass
|
|
4769
|
+
try:
|
|
4770
|
+
proc.terminate()
|
|
4771
|
+
proc.wait(timeout=2)
|
|
4772
|
+
except (OSError, subprocess.TimeoutExpired):
|
|
4773
|
+
try:
|
|
4774
|
+
proc.kill()
|
|
4775
|
+
proc.wait(timeout=2)
|
|
4776
|
+
except OSError:
|
|
4777
|
+
pass
|
|
4778
|
+
|
|
4779
|
+
|
|
4780
|
+
def _relay_probe_payload(
|
|
4781
|
+
*,
|
|
4782
|
+
ready: bool,
|
|
4783
|
+
first_blocker: str,
|
|
4784
|
+
stage: str,
|
|
4785
|
+
request_count: int,
|
|
4786
|
+
prompt_usage_total_tokens: int = 0,
|
|
4787
|
+
response_marker_required: bool = False,
|
|
4788
|
+
response_marker_observed: bool = True,
|
|
4789
|
+
agent_message_present: bool = False,
|
|
4790
|
+
) -> dict[str, Any]:
|
|
4791
|
+
return {
|
|
4792
|
+
"schema_version": SKILLSBENCH_LOCAL_ACP_RELAY_PROBE_SCHEMA_VERSION,
|
|
4793
|
+
"ready": ready,
|
|
4794
|
+
"first_blocker": first_blocker,
|
|
4795
|
+
"stage": stage,
|
|
4796
|
+
"request_count": request_count,
|
|
4797
|
+
"prompt_usage_total_tokens": max(0, int(prompt_usage_total_tokens or 0)),
|
|
4798
|
+
"response_marker_required": bool(response_marker_required),
|
|
4799
|
+
"response_marker_observed": bool(response_marker_observed),
|
|
4800
|
+
"agent_message_present": bool(agent_message_present),
|
|
4801
|
+
"worker_protocol": "acp_stdio",
|
|
4802
|
+
"codex_cli_invoked": False,
|
|
4803
|
+
"raw_output_recorded": False,
|
|
4804
|
+
"raw_event_jsonl_recorded": False,
|
|
4805
|
+
"credential_values_recorded": False,
|
|
4806
|
+
"host_paths_recorded": False,
|
|
4807
|
+
}
|
|
4808
|
+
|
|
4809
|
+
|
|
4810
|
+
def _host_local_transport_probe_payload(
|
|
4811
|
+
*,
|
|
4812
|
+
ready: bool,
|
|
4813
|
+
first_blocker: str,
|
|
4814
|
+
stage: str,
|
|
4815
|
+
request_count: int,
|
|
4816
|
+
benchflow_acp_client_used: bool,
|
|
4817
|
+
) -> dict[str, Any]:
|
|
4818
|
+
return {
|
|
4819
|
+
"schema_version": SKILLSBENCH_HOST_LOCAL_ACP_TRANSPORT_PROBE_SCHEMA_VERSION,
|
|
4820
|
+
"ready": ready,
|
|
4821
|
+
"first_blocker": first_blocker,
|
|
4822
|
+
"stage": stage,
|
|
4823
|
+
"request_count": request_count,
|
|
4824
|
+
"benchflow_acp_client_used": benchflow_acp_client_used,
|
|
4825
|
+
"transport": "host_local_stdio",
|
|
4826
|
+
"container_transport_used": False,
|
|
4827
|
+
"codex_cli_invoked": False,
|
|
4828
|
+
"raw_output_recorded": False,
|
|
4829
|
+
"raw_event_jsonl_recorded": False,
|
|
4830
|
+
"credential_values_recorded": False,
|
|
4831
|
+
"host_paths_recorded": False,
|
|
4832
|
+
}
|