loopx 0.4.8__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- loopx/__init__.py +5 -0
- loopx/agent_onboarding.py +654 -0
- loopx/agent_registry.py +112 -0
- loopx/ark_managed_agent_host.py +59 -0
- loopx/authority.py +805 -0
- loopx/benchmark.py +2875 -0
- loopx/benchmark_adapters/__init__.py +1 -0
- loopx/benchmark_adapters/agentissue.py +2644 -0
- loopx/benchmark_adapters/agents_last_exam.py +3998 -0
- loopx/benchmark_adapters/edgebench.py +322 -0
- loopx/benchmark_adapters/skillsbench.py +5978 -0
- loopx/benchmark_adapters/skillsbench_acp_failure_policy.py +143 -0
- loopx/benchmark_adapters/skillsbench_acp_process.py +31 -0
- loopx/benchmark_adapters/skillsbench_acp_relay.py +4832 -0
- loopx/benchmark_adapters/skillsbench_batch.py +124 -0
- loopx/benchmark_adapters/skillsbench_bridge_guard.py +209 -0
- loopx/benchmark_adapters/skillsbench_bridge_summary.py +203 -0
- loopx/benchmark_adapters/skillsbench_codex_goal_recovery.py +271 -0
- loopx/benchmark_adapters/skillsbench_codex_goal_trace.py +81 -0
- loopx/benchmark_adapters/skillsbench_codex_runtime.py +339 -0
- loopx/benchmark_adapters/skillsbench_dockerfile_runtime.py +467 -0
- loopx/benchmark_adapters/skillsbench_failure_signals.py +652 -0
- loopx/benchmark_adapters/skillsbench_proxy_runtime.py +327 -0
- loopx/benchmark_adapters/skillsbench_remote_bridge.py +402 -0
- loopx/benchmark_adapters/skillsbench_result_discovery.py +143 -0
- loopx/benchmark_adapters/skillsbench_runner_profile.py +436 -0
- loopx/benchmark_adapters/skillsbench_runner_source.py +99 -0
- loopx/benchmark_adapters/skillsbench_setup_preflight.py +771 -0
- loopx/benchmark_adapters/skillsbench_signals.py +15 -0
- loopx/benchmark_adapters/skillsbench_task_source.py +141 -0
- loopx/benchmark_adapters/skillsbench_turn_route.py +723 -0
- loopx/benchmark_adapters/skillsbench_turn_runtime.py +1069 -0
- loopx/benchmark_adapters/skillsbench_typed_repair.py +689 -0
- loopx/benchmark_adapters/skillsbench_uv_cache.py +111 -0
- loopx/benchmark_adapters/skillsbench_verifier_bootstrap.py +227 -0
- loopx/benchmark_adapters/skillsbench_verifier_cache.py +138 -0
- loopx/benchmark_adapters/terminal_bench.py +10078 -0
- loopx/benchmark_case_analysis.py +1276 -0
- loopx/benchmark_case_state.py +1079 -0
- loopx/benchmark_core/__init__.py +239 -0
- loopx/benchmark_core/adapter.py +84 -0
- loopx/benchmark_core/artifacts.py +517 -0
- loopx/benchmark_core/attempts.py +199 -0
- loopx/benchmark_core/container_exec.py +216 -0
- loopx/benchmark_core/io.py +68 -0
- loopx/benchmark_core/lifecycle.py +211 -0
- loopx/benchmark_core/loop_protocol.py +689 -0
- loopx/benchmark_core/observable_handles.py +348 -0
- loopx/benchmark_core/parity.py +256 -0
- loopx/benchmark_core/remote_closeout.py +482 -0
- loopx/benchmark_core/rounds.py +215 -0
- loopx/benchmark_core/route_profile.py +509 -0
- loopx/benchmark_core/run_permissions.py +206 -0
- loopx/benchmark_core/split_control.py +925 -0
- loopx/benchmark_core/turn_fidelity.py +326 -0
- loopx/benchmark_ledger.py +3793 -0
- loopx/benchmark_ledger_countability.py +372 -0
- loopx/benchmark_ledger_current.py +724 -0
- loopx/benchmark_trajectory.py +405 -0
- loopx/benchmarks/__init__.py +1 -0
- loopx/benchmarks/qualification/__init__.py +1 -0
- loopx/benchmarks/qualification/release_outcome_baseline.py +360 -0
- loopx/benchmarks/read_models/__init__.py +1 -0
- loopx/benchmarks/read_models/benchmark_attempt_accounting.py +53 -0
- loopx/benchmarks/read_models/benchmark_comparison.py +414 -0
- loopx/benchmarks/read_models/benchmark_event_timeline.py +113 -0
- loopx/benchmarks/read_models/benchmark_experiment_report.py +475 -0
- loopx/benchmarks/read_models/benchmark_learning_ledger.py +137 -0
- loopx/benchmarks/read_models/benchmark_lifecycle_contracts.py +228 -0
- loopx/benchmarks/read_models/benchmark_projection.py +723 -0
- loopx/benchmarks/read_models/benchmark_result.py +146 -0
- loopx/benchmarks/read_models/benchmark_run_execution_contract.py +116 -0
- loopx/benchmarks/read_models/benchmark_run_failure.py +157 -0
- loopx/benchmarks/read_models/benchmark_run_metrics.py +213 -0
- loopx/benchmarks/read_models/benchmark_run_post_execution.py +635 -0
- loopx/benchmarks/read_models/benchmark_run_pre_execution.py +541 -0
- loopx/benchmarks/read_models/benchmark_status_compaction.py +1255 -0
- loopx/benchmarks/read_models/benchmark_status_runner.py +780 -0
- loopx/benchmarks/read_models/goal_start_control_score.py +857 -0
- loopx/benchmarks/read_models/skillsbench_post_run_debug.py +746 -0
- loopx/benchmarks/read_models/skillsbench_verifier_attribution.py +269 -0
- loopx/bootstrap.py +1116 -0
- loopx/bootstrap_command_pack.py +2167 -0
- loopx/boundary_authority.py +199 -0
- loopx/canary/__init__.py +1 -0
- loopx/canary/maintainability_ratchet.py +800 -0
- loopx/canary/planner.py +1984 -0
- loopx/canary/premerge.py +1130 -0
- loopx/canary/qualification_profiles.py +309 -0
- loopx/canary/quality_surface_catalog.py +838 -0
- loopx/canary/release_profiles.py +51 -0
- loopx/canary/runner.py +1107 -0
- loopx/canary/smoke_health.py +581 -0
- loopx/canary/smoke_profiles.py +212 -0
- loopx/capabilities/__init__.py +0 -0
- loopx/capabilities/agent_turn_recall/__init__.py +17 -0
- loopx/capabilities/agent_turn_recall/cli.py +369 -0
- loopx/capabilities/agent_turn_recall/core.py +296 -0
- loopx/capabilities/auto_research/__init__.py +16 -0
- loopx/capabilities/auto_research/bootstrap_contract.py +157 -0
- loopx/capabilities/auto_research/cli.py +1468 -0
- loopx/capabilities/auto_research/core.py +11 -0
- loopx/capabilities/auto_research/defaults.py +79 -0
- loopx/capabilities/auto_research/demo_e2e.py +1848 -0
- loopx/capabilities/auto_research/demo_supervisor.py +186 -0
- loopx/capabilities/auto_research/evidence_packet.py +767 -0
- loopx/capabilities/auto_research/human_view.py +794 -0
- loopx/capabilities/auto_research/kernel.py +191 -0
- loopx/capabilities/auto_research/knn_demo_workspace.py +322 -0
- loopx/capabilities/auto_research/live_evidence.py +248 -0
- loopx/capabilities/auto_research/preset.py +176 -0
- loopx/capabilities/auto_research/research_state.py +1085 -0
- loopx/capabilities/auto_research/role_profiles.py +394 -0
- loopx/capabilities/auto_research/rollout_append.py +97 -0
- loopx/capabilities/auto_research/terminal_result_contract.py +422 -0
- loopx/capabilities/auto_research/terminal_result_projection.py +171 -0
- loopx/capabilities/auto_research/terminal_result_query.py +233 -0
- loopx/capabilities/auto_research/terminal_results.py +349 -0
- loopx/capabilities/auto_research/user_contract.py +190 -0
- loopx/capabilities/auto_research/worker_loop.py +163 -0
- loopx/capabilities/auto_research/worker_runtime.py +777 -0
- loopx/capabilities/auto_research/worker_skill/SKILL.md +343 -0
- loopx/capabilities/benchmark_toolkit/__init__.py +19 -0
- loopx/capabilities/benchmark_toolkit/integrity.py +387 -0
- loopx/capabilities/catalog.py +1875 -0
- loopx/capabilities/change_quality/__init__.py +19 -0
- loopx/capabilities/change_quality/cli.py +171 -0
- loopx/capabilities/change_quality/context.py +156 -0
- loopx/capabilities/change_quality/oracles.py +269 -0
- loopx/capabilities/change_quality/policy.py +34 -0
- loopx/capabilities/change_quality/receipt.py +482 -0
- loopx/capabilities/change_quality/result.py +493 -0
- loopx/capabilities/change_quality/scope.py +171 -0
- loopx/capabilities/change_quality/shadow.py +680 -0
- loopx/capabilities/content_ops/__init__.py +0 -0
- loopx/capabilities/content_ops/cli.py +649 -0
- loopx/capabilities/content_ops/connector_packets.py +164 -0
- loopx/capabilities/content_ops/item_lifecycle.py +1000 -0
- loopx/capabilities/content_ops/layout.py +451 -0
- loopx/capabilities/content_ops/markdown.py +456 -0
- loopx/capabilities/content_ops/schemas.py +51 -0
- loopx/capabilities/content_ops/social_browser_x.py +107 -0
- loopx/capabilities/content_ops/surface.py +1956 -0
- loopx/capabilities/content_ops/templates/layout-catalog-v0.json +72 -0
- loopx/capabilities/context_providers/__init__.py +36 -0
- loopx/capabilities/context_providers/base.py +189 -0
- loopx/capabilities/context_providers/factory.py +32 -0
- loopx/capabilities/context_providers/openviking.py +702 -0
- loopx/capabilities/context_providers/service_ownership.py +185 -0
- loopx/capabilities/decision_context/__init__.py +129 -0
- loopx/capabilities/decision_context/architecture.py +83 -0
- loopx/capabilities/decision_context/assembler.py +849 -0
- loopx/capabilities/decision_context/catalog_entry.py +195 -0
- loopx/capabilities/decision_context/cli.py +310 -0
- loopx/capabilities/decision_context/cursor_commit.py +535 -0
- loopx/capabilities/decision_context/outcome_feedback.py +352 -0
- loopx/capabilities/decision_context/packets.py +654 -0
- loopx/capabilities/decision_context/private_state.py +189 -0
- loopx/capabilities/decision_context/profile.py +453 -0
- loopx/capabilities/decision_context/providers.py +228 -0
- loopx/capabilities/decision_context/review_settlement.py +136 -0
- loopx/capabilities/decision_context/runtime.py +273 -0
- loopx/capabilities/decision_context/sources.py +415 -0
- loopx/capabilities/explore/__init__.py +1 -0
- loopx/capabilities/explore/activation.py +198 -0
- loopx/capabilities/explore/adaptive_replay_planner.py +221 -0
- loopx/capabilities/explore/child_replay_runtime.py +463 -0
- loopx/capabilities/explore/composition_frontier.py +291 -0
- loopx/capabilities/explore/counterfactual_runtime.py +578 -0
- loopx/capabilities/explore/episode_runtime.py +647 -0
- loopx/capabilities/explore/harness_checkpoint.py +171 -0
- loopx/capabilities/explore/harness_gate.py +115 -0
- loopx/capabilities/explore/harness_runtime.py +1124 -0
- loopx/capabilities/explore/replay_metrics.py +206 -0
- loopx/capabilities/explore/replay_runtime.py +1271 -0
- loopx/capabilities/explore/resource_portfolio.py +173 -0
- loopx/capabilities/explore/result_log.py +974 -0
- loopx/capabilities/explore/router_state.py +432 -0
- loopx/capabilities/explore/source_history_reconcile.py +255 -0
- loopx/capabilities/explore/speculative_scheduler.py +498 -0
- loopx/capabilities/explore/todo_branch_plan.py +650 -0
- loopx/capabilities/explore/todo_evidence.py +141 -0
- loopx/capabilities/explore/trace_runtime.py +284 -0
- loopx/capabilities/explore/worker_branch_plan.py +1257 -0
- loopx/capabilities/integration_branch/__init__.py +13 -0
- loopx/capabilities/integration_branch/cli.py +148 -0
- loopx/capabilities/integration_branch/core.py +916 -0
- loopx/capabilities/issue_fix/__init__.py +19 -0
- loopx/capabilities/issue_fix/acceptance_loop.py +1050 -0
- loopx/capabilities/issue_fix/candidate_evidence.py +503 -0
- loopx/capabilities/issue_fix/candidate_preflight.py +676 -0
- loopx/capabilities/issue_fix/cli.py +1822 -0
- loopx/capabilities/issue_fix/cli_input.py +87 -0
- loopx/capabilities/issue_fix/content_ops_cli.py +148 -0
- loopx/capabilities/issue_fix/discovered_issue_promotion.py +947 -0
- loopx/capabilities/issue_fix/explore_projection.py +710 -0
- loopx/capabilities/issue_fix/feasibility.py +542 -0
- loopx/capabilities/issue_fix/github_public.py +661 -0
- loopx/capabilities/issue_fix/intake_surface.py +832 -0
- loopx/capabilities/issue_fix/metadata_preview.py +218 -0
- loopx/capabilities/issue_fix/metrics_projection.py +1340 -0
- loopx/capabilities/issue_fix/metrics_supplement.py +634 -0
- loopx/capabilities/issue_fix/metrics_supplement_cli.py +127 -0
- loopx/capabilities/issue_fix/outcome_projection.py +1235 -0
- loopx/capabilities/issue_fix/periodic_report.py +189 -0
- loopx/capabilities/issue_fix/pr_description.py +418 -0
- loopx/capabilities/issue_fix/pr_gate_reconcile.py +496 -0
- loopx/capabilities/issue_fix/pr_gate_reconcile_cli.py +464 -0
- loopx/capabilities/issue_fix/pr_lifecycle.py +1327 -0
- loopx/capabilities/issue_fix/pr_lifecycle_rollout.py +85 -0
- loopx/capabilities/issue_fix/pr_monitor_materialization.py +257 -0
- loopx/capabilities/issue_fix/pr_review_ack.py +439 -0
- loopx/capabilities/issue_fix/provider_hooks.py +24 -0
- loopx/capabilities/issue_fix/repository_commit_evidence.py +186 -0
- loopx/capabilities/issue_fix/repository_context.py +457 -0
- loopx/capabilities/issue_fix/repository_memory.py +459 -0
- loopx/capabilities/issue_fix/repository_memory_provider.py +1454 -0
- loopx/capabilities/issue_fix/repository_snapshot.py +454 -0
- loopx/capabilities/issue_fix/reviewer_cli.py +917 -0
- loopx/capabilities/issue_fix/reviewer_notification.py +882 -0
- loopx/capabilities/issue_fix/reviewer_notification_drain.py +942 -0
- loopx/capabilities/issue_fix/reviewer_recommendation.py +1057 -0
- loopx/capabilities/issue_fix/reviewer_request.py +1282 -0
- loopx/capabilities/issue_fix/reward_memory.py +879 -0
- loopx/capabilities/issue_fix/workflow_plan.py +1286 -0
- loopx/capabilities/material_lifecycle/__init__.py +161 -0
- loopx/capabilities/material_lifecycle/_validation.py +183 -0
- loopx/capabilities/material_lifecycle/apply.py +672 -0
- loopx/capabilities/material_lifecycle/architecture.py +122 -0
- loopx/capabilities/material_lifecycle/cli.py +161 -0
- loopx/capabilities/material_lifecycle/decision_planning.py +470 -0
- loopx/capabilities/material_lifecycle/explore_execution.py +306 -0
- loopx/capabilities/material_lifecycle/intake.py +869 -0
- loopx/capabilities/material_lifecycle/inventory.py +147 -0
- loopx/capabilities/material_lifecycle/lifecycle.py +98 -0
- loopx/capabilities/material_lifecycle/preparation.py +147 -0
- loopx/capabilities/material_lifecycle/project_skill.py +83 -0
- loopx/capabilities/material_lifecycle/ranking.py +267 -0
- loopx/capabilities/material_lifecycle/readable_projection.py +500 -0
- loopx/capabilities/material_lifecycle/rebuild.py +480 -0
- loopx/capabilities/material_lifecycle/settlement.py +238 -0
- loopx/capabilities/periodic_report/__init__.py +71 -0
- loopx/capabilities/periodic_report/adapters.py +939 -0
- loopx/capabilities/periodic_report/archive.py +422 -0
- loopx/capabilities/periodic_report/bindings.py +705 -0
- loopx/capabilities/periodic_report/cli.py +277 -0
- loopx/capabilities/periodic_report/core.py +691 -0
- loopx/capabilities/periodic_report/extension_envelope.py +66 -0
- loopx/capabilities/periodic_report/presets.py +103 -0
- loopx/capabilities/periodic_report/profile.py +235 -0
- loopx/capabilities/periodic_report/project_progress.py +179 -0
- loopx/capabilities/periodic_report/triggers.py +452 -0
- loopx/capabilities/pr_review_queue/__init__.py +17 -0
- loopx/capabilities/pr_review_queue/core.py +506 -0
- loopx/capabilities/pr_review_queue/review_contract.py +506 -0
- loopx/capabilities/registry.py +192 -0
- loopx/capabilities/reward_memory/__init__.py +75 -0
- loopx/capabilities/reward_memory/application.py +819 -0
- loopx/capabilities/reward_memory/architecture.py +572 -0
- loopx/capabilities/reward_memory/candidate_review.py +511 -0
- loopx/capabilities/reward_memory/cli.py +469 -0
- loopx/capabilities/reward_memory/dogfood.py +574 -0
- loopx/capabilities/reward_memory/evaluation.py +296 -0
- loopx/capabilities/reward_memory/evaluation_fixtures.py +362 -0
- loopx/capabilities/reward_memory/experiment.py +567 -0
- loopx/capabilities/reward_memory/health.py +222 -0
- loopx/capabilities/reward_memory/ingestion.py +519 -0
- loopx/capabilities/reward_memory/registry.py +600 -0
- loopx/capabilities/reward_memory/runtime_hooks.py +312 -0
- loopx/capabilities/reward_memory/scoped_feedback.py +173 -0
- loopx/capabilities/semantic_preference/__init__.py +12 -0
- loopx/capabilities/semantic_preference/cli.py +189 -0
- loopx/capabilities/semantic_preference/contract.py +592 -0
- loopx/capabilities/semantic_preference/reward_memory.py +62 -0
- loopx/capabilities/value_connectors/__init__.py +1 -0
- loopx/capabilities/value_connectors/cli.py +401 -0
- loopx/capabilities/value_connectors/finance_extension_migration.py +108 -0
- loopx/capabilities/value_connectors/install_check.py +147 -0
- loopx/capabilities/value_connectors/planner.py +733 -0
- loopx/capabilities/value_connectors/source_map.py +446 -0
- loopx/claude_goal_baseline.py +138 -0
- loopx/claude_goal_mode/__init__.py +23 -0
- loopx/claude_goal_mode/hooks/goal_policy.py +212 -0
- loopx/claude_goal_mode/hooks/goal_state.py +139 -0
- loopx/claude_goal_mode/mcp/loopx_mcp.py +167 -0
- loopx/claude_goal_mode/scripts/connect.py +103 -0
- loopx/claude_goal_mode/scripts/goalmode_cmd.py +241 -0
- loopx/claude_goal_mode/scripts/install.py +328 -0
- loopx/claude_goal_mode/statusline/goal_status.py +97 -0
- loopx/cli.py +836 -0
- loopx/cli_commands/__init__.py +334 -0
- loopx/cli_commands/_host_thread.py +13 -0
- loopx/cli_commands/agentissue_runner_flow.py +447 -0
- loopx/cli_commands/agents_last_exam.py +160 -0
- loopx/cli_commands/agents_last_exam_baked_input.py +302 -0
- loopx/cli_commands/agents_last_exam_host_codex.py +374 -0
- loopx/cli_commands/agents_last_exam_launch_dry_run.py +372 -0
- loopx/cli_commands/agents_last_exam_local_plan.py +322 -0
- loopx/cli_commands/agents_last_exam_runner_source.py +352 -0
- loopx/cli_commands/agents_last_exam_task_material.py +335 -0
- loopx/cli_commands/agents_last_exam_validation_gate.py +236 -0
- loopx/cli_commands/benchmark_boundary.py +499 -0
- loopx/cli_commands/benchmark_dispatch.py +161 -0
- loopx/cli_commands/benchmark_release_outcome.py +123 -0
- loopx/cli_commands/benchmark_review_lifecycle.py +1275 -0
- loopx/cli_commands/benchmark_run_ledger.py +763 -0
- loopx/cli_commands/benchmark_run_ledger_case_analysis.py +249 -0
- loopx/cli_commands/benchmark_run_ledger_classification.py +45 -0
- loopx/cli_commands/benchmark_run_ledger_maintenance.py +486 -0
- loopx/cli_commands/benchmark_run_ledger_maintenance_registration.py +342 -0
- loopx/cli_commands/benchmark_run_ledger_maintenance_rendering.py +233 -0
- loopx/cli_commands/benchmark_run_ledger_parity.py +92 -0
- loopx/cli_commands/bootstrap_connect.py +238 -0
- loopx/cli_commands/canary.py +707 -0
- loopx/cli_commands/canary_release_qualification.py +79 -0
- loopx/cli_commands/capability.py +96 -0
- loopx/cli_commands/doctor.py +43 -0
- loopx/cli_commands/dreaming.py +143 -0
- loopx/cli_commands/edgebench.py +205 -0
- loopx/cli_commands/evidence_log.py +275 -0
- loopx/cli_commands/explore.py +989 -0
- loopx/cli_commands/explore_planning_commands.py +157 -0
- loopx/cli_commands/extension.py +271 -0
- loopx/cli_commands/first_run_report.py +73 -0
- loopx/cli_commands/goal_channel.py +656 -0
- loopx/cli_commands/handoff_mode.py +158 -0
- loopx/cli_commands/history.py +622 -0
- loopx/cli_commands/host_mode_plan.py +113 -0
- loopx/cli_commands/lark_inbox.py +431 -0
- loopx/cli_commands/lark_kanban.py +629 -0
- loopx/cli_commands/ml_experiment.py +321 -0
- loopx/cli_commands/multi_agent.py +211 -0
- loopx/cli_commands/opencode2_goal_worker.py +217 -0
- loopx/cli_commands/pr_review.py +167 -0
- loopx/cli_commands/presentation.py +218 -0
- loopx/cli_commands/preset.py +96 -0
- loopx/cli_commands/project.py +150 -0
- loopx/cli_commands/project_lifecycle.py +915 -0
- loopx/cli_commands/quota.py +859 -0
- loopx/cli_commands/quota_registration.py +241 -0
- loopx/cli_commands/quota_request.py +113 -0
- loopx/cli_commands/ready_score.py +110 -0
- loopx/cli_commands/registry_admin.py +975 -0
- loopx/cli_commands/registry_admin_configure.py +344 -0
- loopx/cli_commands/registry_admin_peer.py +84 -0
- loopx/cli_commands/registry_authority.py +218 -0
- loopx/cli_commands/review_batch.py +146 -0
- loopx/cli_commands/slash_commands.py +145 -0
- loopx/cli_commands/start_goal.py +251 -0
- loopx/cli_commands/starter.py +175 -0
- loopx/cli_commands/starter_bootstrap.py +179 -0
- loopx/cli_commands/starter_bootstrap_registration.py +198 -0
- loopx/cli_commands/starter_runtime_idle.py +107 -0
- loopx/cli_commands/starter_scheduler.py +207 -0
- loopx/cli_commands/starter_session_runtime.py +152 -0
- loopx/cli_commands/starter_visible_common.py +54 -0
- loopx/cli_commands/starter_visible_driver.py +161 -0
- loopx/cli_commands/starter_visible_pilot.py +278 -0
- loopx/cli_commands/status.py +867 -0
- loopx/cli_commands/status_registration.py +239 -0
- loopx/cli_commands/summary_all.py +222 -0
- loopx/cli_commands/support_control.py +809 -0
- loopx/cli_commands/support_control_registry.py +68 -0
- loopx/cli_commands/support_control_supervisor.py +289 -0
- loopx/cli_commands/task_lease.py +306 -0
- loopx/cli_commands/terminal_bench_adapter.py +717 -0
- loopx/cli_commands/terminal_bench_environment_result.py +1246 -0
- loopx/cli_commands/todo.py +940 -0
- loopx/cli_commands/todo_argument_validation.py +572 -0
- loopx/cli_commands/todo_event.py +114 -0
- loopx/cli_commands/turn.py +804 -0
- loopx/cli_commands/version.py +46 -0
- loopx/cli_commands/worker_bridge.py +659 -0
- loopx/cli_rollout.py +314 -0
- loopx/codex_cli_goal_tui.py +672 -0
- loopx/codex_cli_probe.py +1530 -0
- loopx/codex_cli_probe_markdown.py +935 -0
- loopx/codex_cli_runtime_probe.py +733 -0
- loopx/codex_cli_scheduler.py +564 -0
- loopx/codex_goal_baseline.py +620 -0
- loopx/configuration_catalog.py +617 -0
- loopx/configure_goal.py +1375 -0
- loopx/contract.py +996 -0
- loopx/control_plane/__init__.py +71 -0
- loopx/control_plane/agents/__init__.py +1 -0
- loopx/control_plane/agents/agent_lane_recommendation.py +516 -0
- loopx/control_plane/agents/agent_scope.py +1578 -0
- loopx/control_plane/agents/agent_scope_frontier.py +60 -0
- loopx/control_plane/agents/capability_gate.py +531 -0
- loopx/control_plane/agents/identity.py +140 -0
- loopx/control_plane/agents/legacy_migration.py +169 -0
- loopx/control_plane/agents/management_projection.py +658 -0
- loopx/control_plane/agents/material_frontier.py +608 -0
- loopx/control_plane/agents/material_handoff.py +156 -0
- loopx/control_plane/agents/multi_agent/__init__.py +1 -0
- loopx/control_plane/agents/multi_agent/codex_executable.py +207 -0
- loopx/control_plane/agents/multi_agent/collective_round_ledger.py +387 -0
- loopx/control_plane/agents/multi_agent/contract.py +474 -0
- loopx/control_plane/agents/multi_agent/recipe.py +110 -0
- loopx/control_plane/agents/multi_agent/role_successor.py +297 -0
- loopx/control_plane/agents/multi_agent/runtime_scripts.py +426 -0
- loopx/control_plane/agents/multi_agent/visible_launch_policy.py +149 -0
- loopx/control_plane/agents/multi_agent/visible_wake_scheduler.py +392 -0
- loopx/control_plane/agents/profile.py +216 -0
- loopx/control_plane/agents/runtime_model.py +73 -0
- loopx/control_plane/agents/subagent_activity.py +164 -0
- loopx/control_plane/agents/supervisor.py +544 -0
- loopx/control_plane/agents/supervisor_events.py +462 -0
- loopx/control_plane/agents/supervisor_inject.py +204 -0
- loopx/control_plane/agents/work_mode.py +56 -0
- loopx/control_plane/agents/workspace_guard.py +364 -0
- loopx/control_plane/effect_program.py +644 -0
- loopx/control_plane/goals/__init__.py +1 -0
- loopx/control_plane/goals/active_state_event_projection.py +103 -0
- loopx/control_plane/goals/active_state_metadata.py +47 -0
- loopx/control_plane/goals/active_state_sections.py +58 -0
- loopx/control_plane/goals/configure_goal_service.py +354 -0
- loopx/control_plane/goals/contract_health.py +132 -0
- loopx/control_plane/goals/dreaming.py +152 -0
- loopx/control_plane/goals/global_registry_health.py +199 -0
- loopx/control_plane/goals/global_registry_shadow.py +33 -0
- loopx/control_plane/goals/goal_channel.py +34 -0
- loopx/control_plane/goals/goal_channel_projection.py +560 -0
- loopx/control_plane/goals/goal_frontier/__init__.py +1917 -0
- loopx/control_plane/goals/goal_frontier/ack_policy.py +149 -0
- loopx/control_plane/goals/goal_frontier/outcome_continuity.py +437 -0
- loopx/control_plane/goals/goal_frontier/replan_rules.py +210 -0
- loopx/control_plane/goals/goal_frontier/semantic_history.py +314 -0
- loopx/control_plane/goals/goal_frontier/terminal.py +180 -0
- loopx/control_plane/goals/goal_vision.py +443 -0
- loopx/control_plane/goals/goal_vision_policy.py +36 -0
- loopx/control_plane/goals/goal_vision_state.py +62 -0
- loopx/control_plane/goals/goal_vision_wait.py +290 -0
- loopx/control_plane/goals/path_resolution.py +20 -0
- loopx/control_plane/goals/start_contract.py +206 -0
- loopx/control_plane/goals/vision_checkpoint.py +92 -0
- loopx/control_plane/handoff/__init__.py +1 -0
- loopx/control_plane/handoff/cross_runtime_impl_review.py +311 -0
- loopx/control_plane/handoff/delivery_contract.py +161 -0
- loopx/control_plane/handoff/handoff_runs.py +71 -0
- loopx/control_plane/handoff/project_handoff.py +155 -0
- loopx/control_plane/handoff/review_batch.py +463 -0
- loopx/control_plane/handoff/review_packet_context.py +216 -0
- loopx/control_plane/heartbeat/agent.py +173 -0
- loopx/control_plane/heartbeat/budget.py +66 -0
- loopx/control_plane/heartbeat/builder.py +501 -0
- loopx/control_plane/heartbeat/host.py +64 -0
- loopx/control_plane/heartbeat/rules.py +68 -0
- loopx/control_plane/heartbeat/task_body.py +759 -0
- loopx/control_plane/heartbeat/visible_goal.py +86 -0
- loopx/control_plane/projects/__init__.py +1 -0
- loopx/control_plane/projects/contract.py +25 -0
- loopx/control_plane/projects/registry.py +663 -0
- loopx/control_plane/quota/__init__.py +1 -0
- loopx/control_plane/quota/cli_projection.py +704 -0
- loopx/control_plane/quota/decision_summary.py +431 -0
- loopx/control_plane/quota/effect_program.py +152 -0
- loopx/control_plane/quota/error_codes.py +19 -0
- loopx/control_plane/quota/goal_boundary.py +464 -0
- loopx/control_plane/quota/heartbeat_receipt.py +277 -0
- loopx/control_plane/quota/heartbeat_recommendation.py +718 -0
- loopx/control_plane/quota/host_poll_receipts.py +162 -0
- loopx/control_plane/quota/live_decision.py +142 -0
- loopx/control_plane/quota/monitor_poll.py +786 -0
- loopx/control_plane/quota/policy_constants.py +40 -0
- loopx/control_plane/quota/projection_repair.py +262 -0
- loopx/control_plane/quota/recent_runs.py +210 -0
- loopx/control_plane/quota/scheduler_ack.py +490 -0
- loopx/control_plane/quota/selected_todo_projection.py +139 -0
- loopx/control_plane/quota/settlement.py +437 -0
- loopx/control_plane/quota/settlement_cli.py +246 -0
- loopx/control_plane/quota/settlement_validation.py +64 -0
- loopx/control_plane/quota/settlement_workspace_causality.py +180 -0
- loopx/control_plane/quota/should_run.py +249 -0
- loopx/control_plane/quota/should_run_packet.py +1165 -0
- loopx/control_plane/quota/should_run_prepare.py +675 -0
- loopx/control_plane/quota/slot_accounting.py +1123 -0
- loopx/control_plane/quota/spend_sources.py +11 -0
- loopx/control_plane/quota/stall_repair.py +397 -0
- loopx/control_plane/quota/states.py +29 -0
- loopx/control_plane/quota/task_orchestration.py +448 -0
- loopx/control_plane/quota/task_orchestration_admission.py +497 -0
- loopx/control_plane/quota/turn_envelope.py +889 -0
- loopx/control_plane/quota/usage_summary.py +140 -0
- loopx/control_plane/reward_memory.py +43 -0
- loopx/control_plane/runtime/__init__.py +2 -0
- loopx/control_plane/runtime/active_user_assisted_pilot.py +275 -0
- loopx/control_plane/runtime/agent_scoped_evidence_log.py +435 -0
- loopx/control_plane/runtime/decision_freshness.py +203 -0
- loopx/control_plane/runtime/event_ledger.py +197 -0
- loopx/control_plane/runtime/event_store_migration_bridge.py +196 -0
- loopx/control_plane/runtime/goal_project_route.py +70 -0
- loopx/control_plane/runtime/local_state_write_correctness.py +242 -0
- loopx/control_plane/runtime/promotion_readiness.py +152 -0
- loopx/control_plane/runtime/public_safety.py +120 -0
- loopx/control_plane/runtime/run_artifacts.py +78 -0
- loopx/control_plane/runtime/run_compaction.py +397 -0
- loopx/control_plane/runtime/run_context_retention.py +241 -0
- loopx/control_plane/runtime/run_history.py +132 -0
- loopx/control_plane/runtime/run_index_duplicates.py +205 -0
- loopx/control_plane/runtime/run_index_rebuild.py +263 -0
- loopx/control_plane/runtime/run_ingest_health.py +336 -0
- loopx/control_plane/runtime/runtime_projection_route.py +624 -0
- loopx/control_plane/runtime/runtime_projection_writer.py +98 -0
- loopx/control_plane/runtime/session_runtime.py +339 -0
- loopx/control_plane/runtime/shared_runtime_material_projection.py +332 -0
- loopx/control_plane/runtime/shared_runtime_refresh_projection.py +183 -0
- loopx/control_plane/runtime/stale_latest_run.py +90 -0
- loopx/control_plane/runtime/status_classifications.py +49 -0
- loopx/control_plane/runtime/status_projection_cache.py +235 -0
- loopx/control_plane/runtime/stride_observation.py +144 -0
- loopx/control_plane/runtime/time.py +39 -0
- loopx/control_plane/runtime/trajectory_hygiene.py +149 -0
- loopx/control_plane/runtime/validation_command.py +69 -0
- loopx/control_plane/scheduler/__init__.py +1 -0
- loopx/control_plane/scheduler/ack.py +329 -0
- loopx/control_plane/scheduler/arbitration.py +188 -0
- loopx/control_plane/scheduler/automation_liveness.py +183 -0
- loopx/control_plane/scheduler/execution_context.py +555 -0
- loopx/control_plane/scheduler/external_evidence_observation.py +428 -0
- loopx/control_plane/scheduler/monitor_display.py +143 -0
- loopx/control_plane/scheduler/monitor_poll_policy.py +161 -0
- loopx/control_plane/scheduler/monitor_poll_writeback.py +351 -0
- loopx/control_plane/scheduler/monitor_target.py +64 -0
- loopx/control_plane/scheduler/monitor_todo.py +146 -0
- loopx/control_plane/scheduler/monitor_wait.py +237 -0
- loopx/control_plane/scheduler/scheduler_hint.py +1284 -0
- loopx/control_plane/scheduler/state.py +354 -0
- loopx/control_plane/scheduler/state_transition_rules.py +179 -0
- loopx/control_plane/scheduler/time.py +10 -0
- loopx/control_plane/settlement_driver.py +293 -0
- loopx/control_plane/status/__init__.py +6 -0
- loopx/control_plane/status/active_state_projection.py +105 -0
- loopx/control_plane/status/agent_lane_projection.py +375 -0
- loopx/control_plane/status/attention_projection.py +74 -0
- loopx/control_plane/status/autonomous_replan_projection.py +103 -0
- loopx/control_plane/status/collection.py +140 -0
- loopx/control_plane/status/contract_projection.py +31 -0
- loopx/control_plane/status/dreaming_projection.py +52 -0
- loopx/control_plane/status/goal_attention_projection.py +157 -0
- loopx/control_plane/status/lifecycle_projection.py +110 -0
- loopx/control_plane/status/monitor_display_projection.py +69 -0
- loopx/control_plane/status/registry_health_projection.py +75 -0
- loopx/control_plane/status/run_projection.py +70 -0
- loopx/control_plane/status/runtime_summaries.py +161 -0
- loopx/control_plane/testing/__init__.py +1 -0
- loopx/control_plane/testing/actual_default_model_behavior_portfolio.py +1371 -0
- loopx/control_plane/testing/canary_harness.py +182 -0
- loopx/control_plane/testing/capability_monitor_repair_tool_behavior.py +674 -0
- loopx/control_plane/testing/cli_output_budget.py +807 -0
- loopx/control_plane/testing/cli_output_differential.py +250 -0
- loopx/control_plane/testing/cli_output_semantics.py +87 -0
- loopx/control_plane/testing/control_plane_composition_scenarios.py +225 -0
- loopx/control_plane/testing/decision_replay.py +268 -0
- loopx/control_plane/testing/doubao_model_behavior_actor.py +559 -0
- loopx/control_plane/testing/model_behavior_corpus.py +344 -0
- loopx/control_plane/testing/model_behavior_qualification.py +769 -0
- loopx/control_plane/testing/model_behavior_retained_cases.py +235 -0
- loopx/control_plane/testing/model_tool_behavior.py +536 -0
- loopx/control_plane/testing/onboarding_model_behavior_qualification.py +642 -0
- loopx/control_plane/testing/quota_fixtures.py +208 -0
- loopx/control_plane/testing/quota_should_run_parity.py +57 -0
- loopx/control_plane/testing/release_commit_qualification.py +671 -0
- loopx/control_plane/testing/replan_semantic_action_behavior.py +1302 -0
- loopx/control_plane/testing/scoped_gate_successor_tool_behavior.py +527 -0
- loopx/control_plane/testing/selected_todo_tool_behavior.py +1002 -0
- loopx/control_plane/testing/terminal_settlement_tool_behavior.py +656 -0
- loopx/control_plane/todos/__init__.py +1 -0
- loopx/control_plane/todos/active_state_editing.py +296 -0
- loopx/control_plane/todos/active_state_todo_parser.py +138 -0
- loopx/control_plane/todos/active_state_todos.py +175 -0
- loopx/control_plane/todos/addition.py +103 -0
- loopx/control_plane/todos/claim_visibility.py +253 -0
- loopx/control_plane/todos/completed_archive.py +139 -0
- loopx/control_plane/todos/completion_fence.py +49 -0
- loopx/control_plane/todos/completion_policy.py +153 -0
- loopx/control_plane/todos/completion_validation.py +248 -0
- loopx/control_plane/todos/completion_validation_accountability.py +27 -0
- loopx/control_plane/todos/completion_validation_projection.py +57 -0
- loopx/control_plane/todos/contract.py +1476 -0
- loopx/control_plane/todos/decision_scope.py +554 -0
- loopx/control_plane/todos/deferred_resume.py +546 -0
- loopx/control_plane/todos/durable_completion.py +201 -0
- loopx/control_plane/todos/event_writeback.py +484 -0
- loopx/control_plane/todos/frontier_deadline.py +132 -0
- loopx/control_plane/todos/handoff_gate.py +283 -0
- loopx/control_plane/todos/handoff_mode.py +444 -0
- loopx/control_plane/todos/handoff_note.py +202 -0
- loopx/control_plane/todos/line_update.py +361 -0
- loopx/control_plane/todos/list_projection.py +205 -0
- loopx/control_plane/todos/markdown.py +199 -0
- loopx/control_plane/todos/monitor_metadata.py +88 -0
- loopx/control_plane/todos/mutation_authority.py +299 -0
- loopx/control_plane/todos/projection.py +655 -0
- loopx/control_plane/todos/quota_summary.py +1138 -0
- loopx/control_plane/todos/route_continuation.py +267 -0
- loopx/control_plane/todos/succession_warning.py +174 -0
- loopx/control_plane/todos/summary_item.py +223 -0
- loopx/control_plane/todos/text.py +30 -0
- loopx/control_plane/todos/todo_index.py +226 -0
- loopx/control_plane/todos/todo_summary.py +1458 -0
- loopx/control_plane/todos/unblock_resume.py +326 -0
- loopx/control_plane/todos/user_gate.py +263 -0
- loopx/control_plane/todos/write_hint.py +63 -0
- loopx/control_plane/todos/write_policy.py +135 -0
- loopx/control_plane/turn_driver/__init__.py +85 -0
- loopx/control_plane/turn_driver/codex_cli.py +502 -0
- loopx/control_plane/turn_driver/driver.py +355 -0
- loopx/control_plane/turn_driver/executor.py +1468 -0
- loopx/control_plane/turn_driver/loop_controller.py +669 -0
- loopx/control_plane/turn_driver/settlement.py +318 -0
- loopx/control_plane/turn_driver/transaction.py +375 -0
- loopx/control_plane/work_items/__init__.py +1 -0
- loopx/control_plane/work_items/attention_fields.py +56 -0
- loopx/control_plane/work_items/attention_item.py +77 -0
- loopx/control_plane/work_items/attention_queue.py +322 -0
- loopx/control_plane/work_items/attention_routing.py +213 -0
- loopx/control_plane/work_items/autonomous_candidates.py +135 -0
- loopx/control_plane/work_items/autonomous_replan_ack.py +276 -0
- loopx/control_plane/work_items/autonomous_replan_obligation.py +786 -0
- loopx/control_plane/work_items/backlog_hygiene.py +59 -0
- loopx/control_plane/work_items/capability_monitor_fallback.py +221 -0
- loopx/control_plane/work_items/delivery_batch_scale.py +66 -0
- loopx/control_plane/work_items/delivery_outcome.py +152 -0
- loopx/control_plane/work_items/delivery_signals.py +113 -0
- loopx/control_plane/work_items/execution_obligation.py +235 -0
- loopx/control_plane/work_items/goal_route_hint.py +320 -0
- loopx/control_plane/work_items/interaction_contract.py +1540 -0
- loopx/control_plane/work_items/issue_meta_surface.py +159 -0
- loopx/control_plane/work_items/lifecycle.py +139 -0
- loopx/control_plane/work_items/operator_inbox.py +266 -0
- loopx/control_plane/work_items/outcome_followthrough.py +69 -0
- loopx/control_plane/work_items/primary_action.py +326 -0
- loopx/control_plane/work_items/progress_observation.py +630 -0
- loopx/control_plane/work_items/project_asset.py +675 -0
- loopx/control_plane/work_items/repair_delta.py +693 -0
- loopx/control_plane/work_items/runtime_capability_reentry.py +168 -0
- loopx/control_plane/work_items/semantic_replan_writeback.py +177 -0
- loopx/control_plane/work_items/status_contract.py +49 -0
- loopx/control_plane/work_items/task_graph.py +1046 -0
- loopx/control_plane/work_items/task_lease.py +1254 -0
- loopx/control_plane/work_items/task_lease_settlement.py +422 -0
- loopx/control_plane/work_items/work_lane.py +510 -0
- loopx/control_plane/work_items/work_lane_context.py +161 -0
- loopx/demo.py +247 -0
- loopx/diagnose.py +633 -0
- loopx/doctor.py +1251 -0
- loopx/domain_packs/__init__.py +1 -0
- loopx/domain_packs/issue_fix.py +571 -0
- loopx/domain_packs/ml_experiment.py +854 -0
- loopx/domain_state.py +137 -0
- loopx/dreaming.py +706 -0
- loopx/entrypoint.py +16 -0
- loopx/event_sourced_state.py +981 -0
- loopx/execution_profile.py +286 -0
- loopx/experiments/__init__.py +1 -0
- loopx/experiments/planner_worker/__init__.py +1 -0
- loopx/experiments/planner_worker/contract.py +523 -0
- loopx/experiments/planner_worker/runtime.py +391 -0
- loopx/experiments/planner_worker/traex.py +461 -0
- loopx/explore_graph.py +11 -0
- loopx/extensions/__init__.py +1 -0
- loopx/extensions/bundled.py +28 -0
- loopx/extensions/execution_envelope.py +126 -0
- loopx/extensions/lark/__init__.py +11 -0
- loopx/extensions/lark/event_collector.py +478 -0
- loopx/extensions/lark/event_collector_runtime.py +506 -0
- loopx/extensions/lark/event_inbox.py +454 -0
- loopx/extensions/lark/extension.toml +88 -0
- loopx/extensions/lark/goal_channel.py +44 -0
- loopx/extensions/lark/goal_channel_contracts.py +388 -0
- loopx/extensions/lark/goal_channel_lifecycle.py +218 -0
- loopx/extensions/lark/goal_channel_runtime.py +792 -0
- loopx/extensions/lark/goal_channel_setup.py +805 -0
- loopx/extensions/lark/goal_channel_targets.py +215 -0
- loopx/extensions/lark/goal_channel_transport.py +281 -0
- loopx/extensions/lark/inbox_reactions.py +650 -0
- loopx/extensions/lark/inbox_reply.py +430 -0
- loopx/extensions/lark/presentation/__init__.py +11 -0
- loopx/extensions/lark/presentation/explore_results.py +2276 -0
- loopx/extensions/lark/presentation/explore_singleflight.py +127 -0
- loopx/extensions/lark/presentation/explore_source_guard.py +121 -0
- loopx/extensions/lark/presentation/explore_stage_document.py +703 -0
- loopx/extensions/lark/presentation/explore_visual_integrity.py +122 -0
- loopx/extensions/lark/presentation/explore_visual_readback.py +452 -0
- loopx/extensions/lark/presentation/explore_visual_styles.py +156 -0
- loopx/extensions/lark/presentation/issue_fix_surface.py +612 -0
- loopx/extensions/lark/presentation/kanban.py +2791 -0
- loopx/extensions/lark/presentation/message_card.py +112 -0
- loopx/extensions/lark/presentation/periodic_report.py +261 -0
- loopx/extensions/lark/presentation/projection_rows.py +600 -0
- loopx/extensions/lark/presentation/record_io.py +95 -0
- loopx/extensions/lark/presentation/sync_receipt.py +145 -0
- loopx/extensions/lark/private_json.py +40 -0
- loopx/extensions/lark/provider.py +86 -0
- loopx/extensions/lark/reviewer_notification.py +604 -0
- loopx/extensions/manifest.py +385 -0
- loopx/extensions/openviking_periodic_report/__init__.py +17 -0
- loopx/extensions/openviking_periodic_report/activation.py +173 -0
- loopx/extensions/openviking_periodic_report/extension.toml +17 -0
- loopx/extensions/openviking_periodic_report/provider.py +355 -0
- loopx/extensions/openviking_periodic_report/sink.py +117 -0
- loopx/extensions/openviking_semantic_preference/__init__.py +5 -0
- loopx/extensions/openviking_semantic_preference/extension.toml +16 -0
- loopx/extensions/openviking_semantic_preference/history_export.py +484 -0
- loopx/extensions/openviking_semantic_preference/project_peer.py +68 -0
- loopx/extensions/openviking_semantic_preference/provider.py +312 -0
- loopx/extensions/presentation.py +979 -0
- loopx/extensions/process_runtime.py +204 -0
- loopx/extensions/readiness.py +168 -0
- loopx/extensions/runtime.py +931 -0
- loopx/extensions/scaffold.py +335 -0
- loopx/feedback.py +581 -0
- loopx/file_lock.py +382 -0
- loopx/global_registry.py +842 -0
- loopx/global_risks.py +970 -0
- loopx/global_todos.py +568 -0
- loopx/handoff_budget.py +28 -0
- loopx/heartbeat_prequota.py +80 -0
- loopx/heartbeat_prompt.py +159 -0
- loopx/help_surface.py +516 -0
- loopx/history.py +1507 -0
- loopx/host_loop_activation.py +1311 -0
- loopx/host_mode_planner.py +991 -0
- loopx/install_contract.py +1 -0
- loopx/interface_budget.py +196 -0
- loopx/long_task_cadence.py +208 -0
- loopx/materials.py +185 -0
- loopx/ml_experiment.py +3 -0
- loopx/onboarding.py +214 -0
- loopx/opencode2_goal_mode/README.md +81 -0
- loopx/opencode2_goal_mode/__init__.py +9 -0
- loopx/opencode2_goal_mode/opencode2-goal-worker.mjs +1018 -0
- loopx/opencode_goal_mode/README.md +99 -0
- loopx/opencode_goal_mode/__init__.py +13 -0
- loopx/opencode_goal_mode/goal-bridge-runtime.mjs +858 -0
- loopx/opencode_goal_mode/loopx-goal.js +8 -0
- loopx/operator_gate.py +420 -0
- loopx/orchestration.py +127 -0
- loopx/paths.py +59 -0
- loopx/pi_goal_mode/README.md +67 -0
- loopx/pi_goal_mode/__init__.py +13 -0
- loopx/pi_goal_mode/loopx-goal.ts +254 -0
- loopx/pi_goal_mode/pi-goal-loop-runtime.mjs +574 -0
- loopx/pr_review.py +1206 -0
- loopx/presentation/__init__.py +1 -0
- loopx/presentation/explore_views.py +1334 -0
- loopx/presentation/markdown.py +61 -0
- loopx/presentation/projection_source_reconcile.py +140 -0
- loopx/presentation/public_safety.py +42 -0
- loopx/presentation/renderers/__init__.py +17 -0
- loopx/presentation/renderers/goal_channel_html.py +269 -0
- loopx/presentation/renderers/periodic_report_html.py +786 -0
- loopx/presentation/renderers/periodic_report_markdown.py +184 -0
- loopx/presentation/renderers/quota_event_markdown.py +116 -0
- loopx/presentation/renderers/quota_markdown.py +1112 -0
- loopx/presentation/renderers/status_markdown.py +1570 -0
- loopx/presentation/renderers/trajectory_hygiene_markdown.py +39 -0
- loopx/presentation/renderers/turn_envelope_markdown.py +33 -0
- loopx/presentation/sinks/__init__.py +5 -0
- loopx/presentation/sinks/openviking_periodic_report.py +7 -0
- loopx/presentation/static_site.py +691 -0
- loopx/presets.py +369 -0
- loopx/project_alias.py +217 -0
- loopx/project_map.py +589 -0
- loopx/project_prompt.py +1153 -0
- loopx/project_skill_cli.py +125 -0
- loopx/project_skill_delivery.py +470 -0
- loopx/project_uninstall.py +462 -0
- loopx/promotion_gate.py +197 -0
- loopx/quota.py +1197 -0
- loopx/ready_score.py +413 -0
- loopx/registry.py +621 -0
- loopx/registry_writability.py +64 -0
- loopx/release_candidate.py +148 -0
- loopx/release_manifest.py +316 -0
- loopx/repository_identity.py +100 -0
- loopx/review_packet.py +1024 -0
- loopx/rollout_event_log.py +505 -0
- loopx/runtime.py +112 -0
- loopx/self_update.py +750 -0
- loopx/session_runtime.py +418 -0
- loopx/skill_install_readback.py +500 -0
- loopx/slash_command_install.py +1393 -0
- loopx/slash_commands.py +264 -0
- loopx/state_backup.py +573 -0
- loopx/state_migration.py +350 -0
- loopx/state_projection.py +809 -0
- loopx/state_refresh.py +1416 -0
- loopx/status.py +1383 -0
- loopx/status_server.py +935 -0
- loopx/summary_all.py +725 -0
- loopx/terminal_bench_agent.py +2056 -0
- loopx/thread_agent_binding.py +408 -0
- loopx/todo_followups.py +168 -0
- loopx/todo_suggestion_prompt.py +204 -0
- loopx/todos.py +2229 -0
- loopx/turn_identity.py +17 -0
- loopx/upgrade.py +1083 -0
- loopx/visible_governance.py +667 -0
- loopx/visible_multi_agent_launcher.py +1253 -0
- loopx/visible_multi_agent_tmux.py +429 -0
- loopx/worker_bridge.py +1574 -0
- loopx-0.4.8.dist-info/METADATA +708 -0
- loopx-0.4.8.dist-info/RECORD +811 -0
- loopx-0.4.8.dist-info/WHEEL +5 -0
- loopx-0.4.8.dist-info/entry_points.txt +5 -0
- loopx-0.4.8.dist-info/licenses/LICENSE +202 -0
- loopx-0.4.8.dist-info/licenses/LICENSE-MIT +21 -0
- loopx-0.4.8.dist-info/licenses/NOTICE +6 -0
- loopx-0.4.8.dist-info/top_level.txt +1 -0
|
@@ -0,0 +1,2644 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
import json
|
|
4
|
+
import re
|
|
5
|
+
import shlex
|
|
6
|
+
from pathlib import Path
|
|
7
|
+
from typing import Any
|
|
8
|
+
|
|
9
|
+
from ..control_plane.work_items.delivery_outcome import DeliveryOutcome
|
|
10
|
+
|
|
11
|
+
|
|
12
|
+
AGENTISSUE_BENCHMARK_ID = "agentissue-bench"
|
|
13
|
+
AGENTISSUE_CODEX_CLI_RUNNER_WRAPPER_SCHEMA_VERSION = (
|
|
14
|
+
"agentissue_bench_codex_cli_runner_dry_run_wrapper_v0"
|
|
15
|
+
)
|
|
16
|
+
AGENTISSUE_CODEX_CLI_RUNNER_BENCHMARK_RUN_MODE = (
|
|
17
|
+
"agentissue_codex_cli_runner_dry_run_wrapper"
|
|
18
|
+
)
|
|
19
|
+
AGENTISSUE_CODEX_CLI_RUNNER_SYNTHETIC_STAGING_SCHEMA_VERSION = (
|
|
20
|
+
"agentissue_bench_codex_cli_runner_synthetic_staging_v0"
|
|
21
|
+
)
|
|
22
|
+
AGENTISSUE_CODEX_CLI_RUNNER_SYNTHETIC_STAGING_MODE = (
|
|
23
|
+
"agentissue_codex_cli_runner_synthetic_staging_fixture"
|
|
24
|
+
)
|
|
25
|
+
AGENTISSUE_CODEX_CLI_RUNNER_EXECUTION_GATE_SCHEMA_VERSION = (
|
|
26
|
+
"agentissue_bench_codex_cli_runner_execution_gate_v0"
|
|
27
|
+
)
|
|
28
|
+
AGENTISSUE_CODEX_CLI_RUNNER_EXECUTION_GATE_MODE = (
|
|
29
|
+
"agentissue_codex_cli_runner_execution_gate"
|
|
30
|
+
)
|
|
31
|
+
AGENTISSUE_CODEX_CLI_RUNNER_FIRST_RUN_HANDOFF_SCHEMA_VERSION = (
|
|
32
|
+
"agentissue_bench_codex_cli_runner_first_run_handoff_v0"
|
|
33
|
+
)
|
|
34
|
+
AGENTISSUE_CODEX_CLI_RUNNER_FIRST_RUN_HANDOFF_MODE = (
|
|
35
|
+
"agentissue_codex_cli_runner_first_run_handoff_packet"
|
|
36
|
+
)
|
|
37
|
+
AGENTISSUE_CODEX_CLI_RUNNER_WORKFLOW_CHECK_SCHEMA_VERSION = (
|
|
38
|
+
"agentissue_bench_codex_cli_runner_workflow_check_v0"
|
|
39
|
+
)
|
|
40
|
+
AGENTISSUE_CODEX_CLI_RUNNER_WORKFLOW_CHECK_MODE = (
|
|
41
|
+
"agentissue_codex_cli_runner_workflow_check_packet"
|
|
42
|
+
)
|
|
43
|
+
AGENTISSUE_CODEX_CLI_RUNNER_RUN_GATE_SCHEMA_VERSION = (
|
|
44
|
+
"agentissue_bench_codex_cli_runner_run_gate_v0"
|
|
45
|
+
)
|
|
46
|
+
AGENTISSUE_CODEX_CLI_RUNNER_RUN_GATE_MODE = (
|
|
47
|
+
"agentissue_codex_cli_runner_run_gate_packet"
|
|
48
|
+
)
|
|
49
|
+
AGENTISSUE_CODEX_CLI_RUNNER_TARGET_HANDOFF_SCHEMA_VERSION = (
|
|
50
|
+
"agentissue_bench_codex_cli_runner_target_handoff_v0"
|
|
51
|
+
)
|
|
52
|
+
AGENTISSUE_CODEX_CLI_RUNNER_TARGET_HANDOFF_MODE = (
|
|
53
|
+
"agentissue_codex_cli_runner_target_handoff_packet"
|
|
54
|
+
)
|
|
55
|
+
AGENTISSUE_CODEX_CLI_RUNNER_REAL_RESULT_SCHEMA_VERSION = (
|
|
56
|
+
"agentissue_bench_codex_cli_runner_real_result_reducer_v0"
|
|
57
|
+
)
|
|
58
|
+
AGENTISSUE_CODEX_CLI_RUNNER_REAL_RESULT_MODE = (
|
|
59
|
+
"agentissue_codex_cli_runner_real_result_reducer"
|
|
60
|
+
)
|
|
61
|
+
AGENTISSUE_CODEX_CLI_RUNNER_PRIVATE_SCRIPT_SCHEMA_VERSION = (
|
|
62
|
+
"agentissue_bench_codex_cli_runner_private_script_v0"
|
|
63
|
+
)
|
|
64
|
+
AGENTISSUE_CODEX_CLI_RUNNER_PRIVATE_SCRIPT_MODE = (
|
|
65
|
+
"agentissue_codex_cli_runner_private_script"
|
|
66
|
+
)
|
|
67
|
+
AGENTISSUE_CODEX_CLI_RUNNER_SOURCE_RUNNER = (
|
|
68
|
+
"loopx_agentissue_codex_cli_runner"
|
|
69
|
+
)
|
|
70
|
+
AGENTISSUE_DEFAULT_TAG = "lagent_239"
|
|
71
|
+
AGENTISSUE_DEFAULT_IMAGE = "alfin06/agentissue-bench:lagent_239"
|
|
72
|
+
AGENTISSUE_PATCH_RELATIVE_PATH = "Patches/lagent_239/attempt.patch"
|
|
73
|
+
|
|
74
|
+
|
|
75
|
+
def _agentissue_public_label(value: Any, *, limit: int = 120) -> str:
|
|
76
|
+
text = str(value or "").strip()
|
|
77
|
+
if not text:
|
|
78
|
+
raise ValueError("agentissue label is required")
|
|
79
|
+
if not re.fullmatch(r"[A-Za-z0-9_.:-]{1,120}", text):
|
|
80
|
+
raise ValueError("agentissue label must be public-safe")
|
|
81
|
+
return text[:limit]
|
|
82
|
+
|
|
83
|
+
|
|
84
|
+
def build_agentissue_codex_cli_runner_wrapper(
|
|
85
|
+
*,
|
|
86
|
+
selected_tag: str = AGENTISSUE_DEFAULT_TAG,
|
|
87
|
+
codex_binary: str = "codex",
|
|
88
|
+
docker_binary: str = "docker",
|
|
89
|
+
job_root_placeholder: str = "<abs-private-job-root>",
|
|
90
|
+
) -> dict[str, Any]:
|
|
91
|
+
"""Build a dry-run-default AgentIssue-Bench Codex CLI runner wrapper.
|
|
92
|
+
|
|
93
|
+
The wrapper deliberately renders command and staging shapes only. It never
|
|
94
|
+
calls Codex, Docker, model APIs, or benchmark helpers; callers that append
|
|
95
|
+
the embedded benchmark_run_v0 are recording readiness, not a task score.
|
|
96
|
+
"""
|
|
97
|
+
|
|
98
|
+
tag = _agentissue_public_label(selected_tag)
|
|
99
|
+
if tag != AGENTISSUE_DEFAULT_TAG:
|
|
100
|
+
raise ValueError(
|
|
101
|
+
"agentissue Codex runner wrapper currently only supports selected tag lagent_239"
|
|
102
|
+
)
|
|
103
|
+
codex = _agentissue_public_label(codex_binary, limit=80)
|
|
104
|
+
docker = _agentissue_public_label(docker_binary, limit=80)
|
|
105
|
+
image = AGENTISSUE_DEFAULT_IMAGE
|
|
106
|
+
buggy_source = f"{job_root_placeholder}/buggy-source"
|
|
107
|
+
context_dir = f"{job_root_placeholder}/context"
|
|
108
|
+
patch_dir = f"{job_root_placeholder}/Patches/lagent_239"
|
|
109
|
+
prompt_path = f"{context_dir}/prompt.md"
|
|
110
|
+
last_message = f"{job_root_placeholder}/codex-last-message.txt"
|
|
111
|
+
compact_run_path = f"{job_root_placeholder}/benchmark_run.compact.json"
|
|
112
|
+
|
|
113
|
+
phase_order = [
|
|
114
|
+
"prepare_private_job_root",
|
|
115
|
+
"write_public_issue_context_to_private_context",
|
|
116
|
+
"pull_selected_image_opt_in",
|
|
117
|
+
"extract_buggy_source_from_selected_container_opt_in",
|
|
118
|
+
"initialize_git_baseline_in_buggy_source",
|
|
119
|
+
"run_host_local_codex_cli_patch_worker_opt_in",
|
|
120
|
+
"write_attempt_patch_from_buggy_source_git_diff",
|
|
121
|
+
"evaluate_selected_tag_container_opt_in",
|
|
122
|
+
"reduce_compact_public_evidence",
|
|
123
|
+
]
|
|
124
|
+
codex_argv = [
|
|
125
|
+
codex,
|
|
126
|
+
"exec",
|
|
127
|
+
"--ephemeral",
|
|
128
|
+
"--ignore-rules",
|
|
129
|
+
"--sandbox",
|
|
130
|
+
"workspace-write",
|
|
131
|
+
"--cd",
|
|
132
|
+
buggy_source,
|
|
133
|
+
"--add-dir",
|
|
134
|
+
job_root_placeholder,
|
|
135
|
+
"--output-last-message",
|
|
136
|
+
last_message,
|
|
137
|
+
prompt_path,
|
|
138
|
+
]
|
|
139
|
+
eval_argv = [
|
|
140
|
+
docker,
|
|
141
|
+
"run",
|
|
142
|
+
"--platform",
|
|
143
|
+
"linux/amd64",
|
|
144
|
+
"--rm",
|
|
145
|
+
"--entrypoint",
|
|
146
|
+
"bash",
|
|
147
|
+
"-v",
|
|
148
|
+
f"{patch_dir}:/patches:ro",
|
|
149
|
+
image,
|
|
150
|
+
"-c",
|
|
151
|
+
"<apply_patch_and_test_patched>",
|
|
152
|
+
]
|
|
153
|
+
wrapper = {
|
|
154
|
+
"schema_version": AGENTISSUE_CODEX_CLI_RUNNER_WRAPPER_SCHEMA_VERSION,
|
|
155
|
+
"benchmark_id": AGENTISSUE_BENCHMARK_ID,
|
|
156
|
+
"selected_tag": tag,
|
|
157
|
+
"selected_image": image,
|
|
158
|
+
"dry_run_default": True,
|
|
159
|
+
"real_execution_done": False,
|
|
160
|
+
"single_tag_only": True,
|
|
161
|
+
"staging_plan": {
|
|
162
|
+
"private_job_root_placeholder": job_root_placeholder,
|
|
163
|
+
"path_recorded": False,
|
|
164
|
+
"buggy_source_placeholder": buggy_source,
|
|
165
|
+
"context_dir_placeholder": context_dir,
|
|
166
|
+
"patch_dir_placeholder": patch_dir,
|
|
167
|
+
"prompt_path_placeholder": prompt_path,
|
|
168
|
+
"last_message_placeholder": last_message,
|
|
169
|
+
"compact_run_placeholder": compact_run_path,
|
|
170
|
+
"phase_order": phase_order,
|
|
171
|
+
},
|
|
172
|
+
"commands": {
|
|
173
|
+
"codex_patch_worker": {
|
|
174
|
+
"argv": codex_argv,
|
|
175
|
+
"runs_on_host": True,
|
|
176
|
+
"runs_after_buggy_source_extraction": True,
|
|
177
|
+
"copy_codex_home": False,
|
|
178
|
+
"auth_material_synced": False,
|
|
179
|
+
"worker_network_allowed": False,
|
|
180
|
+
"worker_docker_allowed": False,
|
|
181
|
+
"reads_fixed_diff_or_oracle": False,
|
|
182
|
+
"execute_by_default": False,
|
|
183
|
+
},
|
|
184
|
+
"patch_export": {
|
|
185
|
+
"input_source": "buggy_source_git_diff",
|
|
186
|
+
"output_relative_path": AGENTISSUE_PATCH_RELATIVE_PATH,
|
|
187
|
+
"raw_patch_public": False,
|
|
188
|
+
"patch_hash_public": True,
|
|
189
|
+
},
|
|
190
|
+
"single_tag_eval": {
|
|
191
|
+
"argv": eval_argv,
|
|
192
|
+
"official_all_tag_helper_allowed": False,
|
|
193
|
+
"docker_env_credentials": False,
|
|
194
|
+
"upload": False,
|
|
195
|
+
"submit": False,
|
|
196
|
+
"public_ranking_path": False,
|
|
197
|
+
"execute_by_default": False,
|
|
198
|
+
},
|
|
199
|
+
},
|
|
200
|
+
"execution_boundary": {
|
|
201
|
+
"codex_cli_invoked": False,
|
|
202
|
+
"model_api_invoked": False,
|
|
203
|
+
"docker_image_pulled": False,
|
|
204
|
+
"docker_container_started": False,
|
|
205
|
+
"patch_generated": False,
|
|
206
|
+
"patch_evaluated": False,
|
|
207
|
+
"raw_issue_text_read": False,
|
|
208
|
+
"raw_patch_recorded": False,
|
|
209
|
+
"raw_log_recorded": False,
|
|
210
|
+
"credential_values_recorded": False,
|
|
211
|
+
},
|
|
212
|
+
"reducer_contract": {
|
|
213
|
+
"allowed_public_fields": [
|
|
214
|
+
"tag",
|
|
215
|
+
"image_digest",
|
|
216
|
+
"patch_sha256",
|
|
217
|
+
"patch_bytes",
|
|
218
|
+
"changed_file_count",
|
|
219
|
+
"hunk_count",
|
|
220
|
+
"exit_code",
|
|
221
|
+
"resolved",
|
|
222
|
+
"duration_seconds",
|
|
223
|
+
"log_sha256",
|
|
224
|
+
"no_upload",
|
|
225
|
+
"no_submit",
|
|
226
|
+
"no_public_ranking_path",
|
|
227
|
+
],
|
|
228
|
+
"raw_issue_text_public": False,
|
|
229
|
+
"raw_patch_public": False,
|
|
230
|
+
"raw_log_public": False,
|
|
231
|
+
"absolute_paths_public": False,
|
|
232
|
+
},
|
|
233
|
+
"stop_rules": {
|
|
234
|
+
"stop_before_codex_auth_sync": True,
|
|
235
|
+
"stop_before_current_head_patch_source": True,
|
|
236
|
+
"stop_before_fixed_diff_or_oracle_read": True,
|
|
237
|
+
"stop_before_all_tag_helpers": True,
|
|
238
|
+
"stop_before_upload_submit_or_public_ranking": True,
|
|
239
|
+
"stop_before_raw_artifact_publication": True,
|
|
240
|
+
"stop_before_destructive_git_or_production": True,
|
|
241
|
+
},
|
|
242
|
+
}
|
|
243
|
+
benchmark_run = {
|
|
244
|
+
"schema_version": "benchmark_run_v0",
|
|
245
|
+
"source_runner": AGENTISSUE_CODEX_CLI_RUNNER_SOURCE_RUNNER,
|
|
246
|
+
"benchmark_id": AGENTISSUE_BENCHMARK_ID,
|
|
247
|
+
"job_name": "agentissue_lagent_239_codex_cli_runner_dry_run",
|
|
248
|
+
"mode": AGENTISSUE_CODEX_CLI_RUNNER_BENCHMARK_RUN_MODE,
|
|
249
|
+
"worker_mode": "trusted_host_codex_cli_dry_run_wrapper",
|
|
250
|
+
"trace_publicness": "compact_public_no_issue_text_no_patch_no_logs",
|
|
251
|
+
"first_blocker": "dry_run_wrapper_only_no_real_case",
|
|
252
|
+
"score_failure_attribution": "not_run_wrapper_readiness_only",
|
|
253
|
+
"real_run": False,
|
|
254
|
+
"submit_eligible": False,
|
|
255
|
+
"leaderboard_evidence": False,
|
|
256
|
+
"official_score_comparable_to_native_codex": False,
|
|
257
|
+
"official_score_claim_allowed": False,
|
|
258
|
+
"control_plane_score_applicable": True,
|
|
259
|
+
"official_task_score": {
|
|
260
|
+
"kind": "agentissue_bench_single_tag_container_eval_not_run",
|
|
261
|
+
"status": "not_run",
|
|
262
|
+
"value": None,
|
|
263
|
+
"resolved": None,
|
|
264
|
+
},
|
|
265
|
+
"progress": {
|
|
266
|
+
"n_total_trials": 1,
|
|
267
|
+
"n_completed_trials": 0,
|
|
268
|
+
"n_errored_trials": 0,
|
|
269
|
+
"n_running_trials": 0,
|
|
270
|
+
"n_pending_trials": 1,
|
|
271
|
+
"n_cancelled_trials": 0,
|
|
272
|
+
"n_retries": 0,
|
|
273
|
+
},
|
|
274
|
+
"metrics": {
|
|
275
|
+
"input_tokens": 0,
|
|
276
|
+
"cache_tokens": 0,
|
|
277
|
+
"output_tokens": 0,
|
|
278
|
+
"cost_usd": 0,
|
|
279
|
+
},
|
|
280
|
+
"validation": {
|
|
281
|
+
"runner_wrapper_built": True,
|
|
282
|
+
"dry_run_default": True,
|
|
283
|
+
"single_tag_only": True,
|
|
284
|
+
"absolute_private_job_root_placeholders": True,
|
|
285
|
+
"buggy_source_before_codex_patch": True,
|
|
286
|
+
"patch_from_buggy_source_git_diff": True,
|
|
287
|
+
"selected_tag_eval_only": True,
|
|
288
|
+
"compact_reducer_declared": True,
|
|
289
|
+
"no_codex_cli_invoked": True,
|
|
290
|
+
"no_model_api_invoked": True,
|
|
291
|
+
"no_docker_container_started": True,
|
|
292
|
+
"no_patch_generated": True,
|
|
293
|
+
"no_patch_evaluated": True,
|
|
294
|
+
"no_auth_material_sync": True,
|
|
295
|
+
"no_current_public_head_patch_source": True,
|
|
296
|
+
"no_fixed_diff_or_oracle_read": True,
|
|
297
|
+
"no_upload": True,
|
|
298
|
+
"no_submit": True,
|
|
299
|
+
"no_public_ranking_path": True,
|
|
300
|
+
},
|
|
301
|
+
"trials": [
|
|
302
|
+
{
|
|
303
|
+
"task_id": tag,
|
|
304
|
+
"trial_name": tag,
|
|
305
|
+
"source": "selected_public_tag",
|
|
306
|
+
"exception_type": "dry_run_wrapper_only_no_real_case",
|
|
307
|
+
"trajectory_present": False,
|
|
308
|
+
"artifact_manifest_present": False,
|
|
309
|
+
"trial_result_present": False,
|
|
310
|
+
}
|
|
311
|
+
],
|
|
312
|
+
"failure_attribution_labels": [
|
|
313
|
+
"no_execution_wrapper_only",
|
|
314
|
+
"ready_for_synthetic_job_root_staging",
|
|
315
|
+
],
|
|
316
|
+
"evidence_files": [
|
|
317
|
+
"benchmark_run.compact.json",
|
|
318
|
+
"runner-flow-plan.public.json",
|
|
319
|
+
],
|
|
320
|
+
"stop_conditions": [
|
|
321
|
+
"codex_auth_sync_requested",
|
|
322
|
+
"current_head_patch_source_requested",
|
|
323
|
+
"fixed_diff_or_oracle_requested",
|
|
324
|
+
"all_tag_helper_requested",
|
|
325
|
+
"upload_submit_or_public_ranking_requested",
|
|
326
|
+
"raw_artifact_publication_requested",
|
|
327
|
+
],
|
|
328
|
+
"read_boundary": {
|
|
329
|
+
"compact_only": True,
|
|
330
|
+
"raw_artifacts_read": False,
|
|
331
|
+
"task_text_read": False,
|
|
332
|
+
"trajectory_read": False,
|
|
333
|
+
"local_paths_recorded": False,
|
|
334
|
+
"docker_invoked": False,
|
|
335
|
+
"model_api_invoked": False,
|
|
336
|
+
"upload_invoked": False,
|
|
337
|
+
},
|
|
338
|
+
}
|
|
339
|
+
return {
|
|
340
|
+
**wrapper,
|
|
341
|
+
"benchmark_run": benchmark_run,
|
|
342
|
+
"recommended_next_action": (
|
|
343
|
+
"run this wrapper against a synthetic private job root, then gate any real "
|
|
344
|
+
"Codex/Docker execution behind explicit opt-in"
|
|
345
|
+
),
|
|
346
|
+
}
|
|
347
|
+
|
|
348
|
+
|
|
349
|
+
def materialize_agentissue_codex_cli_runner_synthetic_staging(
|
|
350
|
+
staging_root: str | Path,
|
|
351
|
+
*,
|
|
352
|
+
selected_tag: str = AGENTISSUE_DEFAULT_TAG,
|
|
353
|
+
codex_binary: str = "codex",
|
|
354
|
+
docker_binary: str = "docker",
|
|
355
|
+
) -> dict[str, Any]:
|
|
356
|
+
"""Create a synthetic AgentIssue runner job root without real task material."""
|
|
357
|
+
|
|
358
|
+
tag = _agentissue_public_label(selected_tag)
|
|
359
|
+
if tag != AGENTISSUE_DEFAULT_TAG:
|
|
360
|
+
raise ValueError(
|
|
361
|
+
"agentissue Codex runner synthetic staging currently only supports selected tag lagent_239"
|
|
362
|
+
)
|
|
363
|
+
root = Path(staging_root).expanduser()
|
|
364
|
+
if not str(root):
|
|
365
|
+
raise ValueError("synthetic staging root is required")
|
|
366
|
+
|
|
367
|
+
wrapper = build_agentissue_codex_cli_runner_wrapper(
|
|
368
|
+
selected_tag=tag,
|
|
369
|
+
codex_binary=codex_binary,
|
|
370
|
+
docker_binary=docker_binary,
|
|
371
|
+
)
|
|
372
|
+
context_dir = root / "context"
|
|
373
|
+
buggy_source_dir = root / "buggy-source"
|
|
374
|
+
patch_dir = root / "Patches" / tag
|
|
375
|
+
prompt_path = context_dir / "prompt.md"
|
|
376
|
+
runner_plan_path = root / "runner-flow-plan.public.json"
|
|
377
|
+
compact_run_path = root / "benchmark_run.compact.json"
|
|
378
|
+
|
|
379
|
+
prompt_text = (
|
|
380
|
+
"# Synthetic AgentIssue-Bench lagent_239 Prompt Placeholder\n\n"
|
|
381
|
+
"This fixture contains no real issue statement, source diff, test patch, "
|
|
382
|
+
"expected patch, auth value, trajectory, screenshot, or raw log.\n\n"
|
|
383
|
+
f"Expected patch output path: {AGENTISSUE_PATCH_RELATIVE_PATH}\n\n"
|
|
384
|
+
"Run boundary: do not invoke Codex, Docker, model APIs, upload, submit, "
|
|
385
|
+
"or public ranking paths from this fixture.\n"
|
|
386
|
+
)
|
|
387
|
+
benchmark_run = json.loads(json.dumps(wrapper["benchmark_run"]))
|
|
388
|
+
benchmark_run.update(
|
|
389
|
+
{
|
|
390
|
+
"job_name": "agentissue_lagent_239_codex_cli_runner_synthetic_staging",
|
|
391
|
+
"mode": AGENTISSUE_CODEX_CLI_RUNNER_SYNTHETIC_STAGING_MODE,
|
|
392
|
+
"worker_mode": "trusted_host_codex_cli_synthetic_staging_fixture",
|
|
393
|
+
"first_blocker": "synthetic_staging_only_no_real_case",
|
|
394
|
+
"score_failure_attribution": "not_run_synthetic_staging_only",
|
|
395
|
+
"failure_attribution_labels": [
|
|
396
|
+
"synthetic_staging_fixture_only",
|
|
397
|
+
"ready_for_guarded_private_source_extraction_gate",
|
|
398
|
+
],
|
|
399
|
+
"evidence_files": [
|
|
400
|
+
"benchmark_run.compact.json",
|
|
401
|
+
"runner-flow-plan.public.json",
|
|
402
|
+
],
|
|
403
|
+
}
|
|
404
|
+
)
|
|
405
|
+
benchmark_run["validation"].update(
|
|
406
|
+
{
|
|
407
|
+
"synthetic_private_job_root_materialized": True,
|
|
408
|
+
"context_dir_created": True,
|
|
409
|
+
"buggy_source_dir_created": True,
|
|
410
|
+
"patch_dir_created": True,
|
|
411
|
+
"prompt_placeholder_written": True,
|
|
412
|
+
"prompt_path_rendered": True,
|
|
413
|
+
"patch_output_parent_reserved": True,
|
|
414
|
+
"compact_run_filename_reserved": True,
|
|
415
|
+
"runner_flow_plan_public_json_written": True,
|
|
416
|
+
"no_absolute_paths_public": True,
|
|
417
|
+
}
|
|
418
|
+
)
|
|
419
|
+
for trial in benchmark_run.get("trials") or []:
|
|
420
|
+
if isinstance(trial, dict):
|
|
421
|
+
trial["exception_type"] = "synthetic_staging_only_no_real_case"
|
|
422
|
+
|
|
423
|
+
runner_plan = {
|
|
424
|
+
"schema_version": AGENTISSUE_CODEX_CLI_RUNNER_SYNTHETIC_STAGING_SCHEMA_VERSION,
|
|
425
|
+
"benchmark_id": AGENTISSUE_BENCHMARK_ID,
|
|
426
|
+
"selected_tag": tag,
|
|
427
|
+
"selected_image": AGENTISSUE_DEFAULT_IMAGE,
|
|
428
|
+
"path_recorded": False,
|
|
429
|
+
"relative_paths": {
|
|
430
|
+
"context_dir": "context",
|
|
431
|
+
"buggy_source_dir": "buggy-source",
|
|
432
|
+
"patch_dir": "Patches/lagent_239",
|
|
433
|
+
"prompt": "context/prompt.md",
|
|
434
|
+
"expected_patch": AGENTISSUE_PATCH_RELATIVE_PATH,
|
|
435
|
+
"compact_run": "benchmark_run.compact.json",
|
|
436
|
+
"runner_plan": "runner-flow-plan.public.json",
|
|
437
|
+
},
|
|
438
|
+
"command_placeholders": wrapper["commands"],
|
|
439
|
+
"execution_boundary": wrapper["execution_boundary"],
|
|
440
|
+
"stop_rules": wrapper["stop_rules"],
|
|
441
|
+
}
|
|
442
|
+
|
|
443
|
+
context_dir.mkdir(parents=True, exist_ok=True)
|
|
444
|
+
buggy_source_dir.mkdir(parents=True, exist_ok=True)
|
|
445
|
+
patch_dir.mkdir(parents=True, exist_ok=True)
|
|
446
|
+
prompt_path.write_text(prompt_text, encoding="utf-8")
|
|
447
|
+
(buggy_source_dir / ".gitkeep").write_text("", encoding="utf-8")
|
|
448
|
+
(patch_dir / ".gitkeep").write_text("", encoding="utf-8")
|
|
449
|
+
runner_plan_path.write_text(
|
|
450
|
+
json.dumps(runner_plan, ensure_ascii=False, indent=2, sort_keys=True) + "\n",
|
|
451
|
+
encoding="utf-8",
|
|
452
|
+
)
|
|
453
|
+
compact_run_path.write_text(
|
|
454
|
+
json.dumps(benchmark_run, ensure_ascii=False, indent=2, sort_keys=True) + "\n",
|
|
455
|
+
encoding="utf-8",
|
|
456
|
+
)
|
|
457
|
+
|
|
458
|
+
created_relative_paths = [
|
|
459
|
+
"context/",
|
|
460
|
+
"context/prompt.md",
|
|
461
|
+
"buggy-source/",
|
|
462
|
+
"buggy-source/.gitkeep",
|
|
463
|
+
"Patches/lagent_239/",
|
|
464
|
+
"Patches/lagent_239/.gitkeep",
|
|
465
|
+
"runner-flow-plan.public.json",
|
|
466
|
+
"benchmark_run.compact.json",
|
|
467
|
+
]
|
|
468
|
+
return {
|
|
469
|
+
"schema_version": AGENTISSUE_CODEX_CLI_RUNNER_SYNTHETIC_STAGING_SCHEMA_VERSION,
|
|
470
|
+
"benchmark_id": AGENTISSUE_BENCHMARK_ID,
|
|
471
|
+
"selected_tag": tag,
|
|
472
|
+
"selected_image": AGENTISSUE_DEFAULT_IMAGE,
|
|
473
|
+
"ready": True,
|
|
474
|
+
"materialized": True,
|
|
475
|
+
"path_recorded": False,
|
|
476
|
+
"staging_root_path_recorded": False,
|
|
477
|
+
"created_relative_paths": created_relative_paths,
|
|
478
|
+
"prompt_relative_path": "context/prompt.md",
|
|
479
|
+
"expected_patch_relative_path": AGENTISSUE_PATCH_RELATIVE_PATH,
|
|
480
|
+
"compact_run_relative_path": "benchmark_run.compact.json",
|
|
481
|
+
"runner_plan_relative_path": "runner-flow-plan.public.json",
|
|
482
|
+
"command_rendering_checks": {
|
|
483
|
+
"codex_argv_uses_prompt_placeholder": True,
|
|
484
|
+
"codex_argv_uses_buggy_source_placeholder": True,
|
|
485
|
+
"eval_argv_uses_selected_image": True,
|
|
486
|
+
"patch_output_parent_reserved": True,
|
|
487
|
+
"compact_reducer_filename_reserved": True,
|
|
488
|
+
},
|
|
489
|
+
"execution_boundary": {
|
|
490
|
+
"codex_cli_invoked": False,
|
|
491
|
+
"model_api_invoked": False,
|
|
492
|
+
"docker_image_pulled": False,
|
|
493
|
+
"docker_container_started": False,
|
|
494
|
+
"patch_generated": False,
|
|
495
|
+
"patch_evaluated": False,
|
|
496
|
+
"raw_issue_text_read": False,
|
|
497
|
+
"raw_patch_recorded": False,
|
|
498
|
+
"raw_log_recorded": False,
|
|
499
|
+
"credential_values_recorded": False,
|
|
500
|
+
},
|
|
501
|
+
"benchmark_run": benchmark_run,
|
|
502
|
+
"recommended_next_action": (
|
|
503
|
+
"add a guarded opt-in real-source extraction and host-Codex execution "
|
|
504
|
+
"gate for lagent_239, still defaulting to no-execute"
|
|
505
|
+
),
|
|
506
|
+
}
|
|
507
|
+
|
|
508
|
+
|
|
509
|
+
def materialize_agentissue_codex_cli_runner_execution_gate(
|
|
510
|
+
gate_root: str | Path,
|
|
511
|
+
*,
|
|
512
|
+
selected_tag: str = AGENTISSUE_DEFAULT_TAG,
|
|
513
|
+
codex_binary: str = "codex",
|
|
514
|
+
docker_binary: str = "docker",
|
|
515
|
+
) -> dict[str, Any]:
|
|
516
|
+
"""Create a no-execute gate packet for the first real AgentIssue runner step."""
|
|
517
|
+
|
|
518
|
+
tag = _agentissue_public_label(selected_tag)
|
|
519
|
+
if tag != AGENTISSUE_DEFAULT_TAG:
|
|
520
|
+
raise ValueError(
|
|
521
|
+
"agentissue Codex runner execution gate currently only supports selected tag lagent_239"
|
|
522
|
+
)
|
|
523
|
+
root = Path(gate_root).expanduser()
|
|
524
|
+
staging = materialize_agentissue_codex_cli_runner_synthetic_staging(
|
|
525
|
+
root,
|
|
526
|
+
selected_tag=tag,
|
|
527
|
+
codex_binary=codex_binary,
|
|
528
|
+
docker_binary=docker_binary,
|
|
529
|
+
)
|
|
530
|
+
wrapper = build_agentissue_codex_cli_runner_wrapper(
|
|
531
|
+
selected_tag=tag,
|
|
532
|
+
codex_binary=codex_binary,
|
|
533
|
+
docker_binary=docker_binary,
|
|
534
|
+
)
|
|
535
|
+
docker = _agentissue_public_label(docker_binary, limit=80)
|
|
536
|
+
image = AGENTISSUE_DEFAULT_IMAGE
|
|
537
|
+
container_label = "<tmp-agentissue-lagent-239-container>"
|
|
538
|
+
job_root = "<abs-private-job-root>"
|
|
539
|
+
buggy_source = f"{job_root}/buggy-source"
|
|
540
|
+
patch_path = f"{job_root}/{AGENTISSUE_PATCH_RELATIVE_PATH}"
|
|
541
|
+
gate_path = root / "execution-gate.public.json"
|
|
542
|
+
compact_run_path = root / "benchmark_run.compact.json"
|
|
543
|
+
|
|
544
|
+
extraction_commands = {
|
|
545
|
+
"inspect_selected_image": [docker, "image", "inspect", image],
|
|
546
|
+
"create_selected_container": [
|
|
547
|
+
docker,
|
|
548
|
+
"create",
|
|
549
|
+
"--name",
|
|
550
|
+
container_label,
|
|
551
|
+
image,
|
|
552
|
+
],
|
|
553
|
+
"copy_buggy_source": [
|
|
554
|
+
docker,
|
|
555
|
+
"cp",
|
|
556
|
+
f"{container_label}:/workspace/.",
|
|
557
|
+
buggy_source,
|
|
558
|
+
],
|
|
559
|
+
"remove_selected_container": [docker, "rm", container_label],
|
|
560
|
+
}
|
|
561
|
+
git_baseline_commands = {
|
|
562
|
+
"init": ["git", "-C", buggy_source, "init"],
|
|
563
|
+
"add": ["git", "-C", buggy_source, "add", "."],
|
|
564
|
+
"commit": [
|
|
565
|
+
"git",
|
|
566
|
+
"-C",
|
|
567
|
+
buggy_source,
|
|
568
|
+
"commit",
|
|
569
|
+
"-m",
|
|
570
|
+
"agentissue-bench-buggy-source-baseline",
|
|
571
|
+
],
|
|
572
|
+
}
|
|
573
|
+
patch_export = {
|
|
574
|
+
"input_source": "buggy_source_git_diff",
|
|
575
|
+
"command_shape": f"git -C {buggy_source} diff --binary > {patch_path}",
|
|
576
|
+
"output_relative_path": AGENTISSUE_PATCH_RELATIVE_PATH,
|
|
577
|
+
}
|
|
578
|
+
gate = {
|
|
579
|
+
"schema_version": AGENTISSUE_CODEX_CLI_RUNNER_EXECUTION_GATE_SCHEMA_VERSION,
|
|
580
|
+
"benchmark_id": AGENTISSUE_BENCHMARK_ID,
|
|
581
|
+
"selected_tag": tag,
|
|
582
|
+
"selected_image": image,
|
|
583
|
+
"path_recorded": False,
|
|
584
|
+
"default_mode": "no_execute",
|
|
585
|
+
"future_opt_in_required": True,
|
|
586
|
+
"single_tag_only": True,
|
|
587
|
+
"relative_paths": {
|
|
588
|
+
"context_prompt": "context/prompt.md",
|
|
589
|
+
"buggy_source_dir": "buggy-source",
|
|
590
|
+
"attempt_patch": AGENTISSUE_PATCH_RELATIVE_PATH,
|
|
591
|
+
"execution_gate": "execution-gate.public.json",
|
|
592
|
+
"compact_run": "benchmark_run.compact.json",
|
|
593
|
+
},
|
|
594
|
+
"source_extraction_gate": {
|
|
595
|
+
"commands": extraction_commands,
|
|
596
|
+
"selected_container_only": True,
|
|
597
|
+
"execute_by_default": False,
|
|
598
|
+
"docker_invoked": False,
|
|
599
|
+
"docker_pull_or_start_allowed": False,
|
|
600
|
+
},
|
|
601
|
+
"private_git_baseline_gate": {
|
|
602
|
+
"commands": git_baseline_commands,
|
|
603
|
+
"execute_by_default": False,
|
|
604
|
+
"destructive_git": False,
|
|
605
|
+
},
|
|
606
|
+
"host_codex_gate": {
|
|
607
|
+
"command": wrapper["commands"]["codex_patch_worker"],
|
|
608
|
+
"execute_by_default": False,
|
|
609
|
+
"codex_cli_invoked": False,
|
|
610
|
+
"auth_material_synced": False,
|
|
611
|
+
},
|
|
612
|
+
"patch_output_gate": patch_export,
|
|
613
|
+
"eval_gate": wrapper["commands"]["single_tag_eval"],
|
|
614
|
+
"stop_rules": {
|
|
615
|
+
**wrapper["stop_rules"],
|
|
616
|
+
"stop_before_real_source_extraction_without_future_gate": True,
|
|
617
|
+
"stop_before_host_codex_execution_without_future_gate": True,
|
|
618
|
+
},
|
|
619
|
+
}
|
|
620
|
+
|
|
621
|
+
benchmark_run = json.loads(json.dumps(staging["benchmark_run"]))
|
|
622
|
+
benchmark_run.update(
|
|
623
|
+
{
|
|
624
|
+
"job_name": "agentissue_lagent_239_codex_cli_runner_execution_gate",
|
|
625
|
+
"mode": AGENTISSUE_CODEX_CLI_RUNNER_EXECUTION_GATE_MODE,
|
|
626
|
+
"worker_mode": "trusted_host_codex_cli_no_execute_gate",
|
|
627
|
+
"first_blocker": "execution_gate_only_no_real_case",
|
|
628
|
+
"score_failure_attribution": "not_run_execution_gate_only",
|
|
629
|
+
"failure_attribution_labels": [
|
|
630
|
+
"execution_gate_fixture_only",
|
|
631
|
+
"ready_for_future_run_specific_opt_in",
|
|
632
|
+
],
|
|
633
|
+
"evidence_files": [
|
|
634
|
+
"execution-gate.public.json",
|
|
635
|
+
"benchmark_run.compact.json",
|
|
636
|
+
"runner-flow-plan.public.json",
|
|
637
|
+
],
|
|
638
|
+
}
|
|
639
|
+
)
|
|
640
|
+
benchmark_run["validation"].update(
|
|
641
|
+
{
|
|
642
|
+
"execution_gate_materialized": True,
|
|
643
|
+
"synthetic_staging_reused": True,
|
|
644
|
+
"selected_container_source_extraction_commands_rendered": True,
|
|
645
|
+
"private_git_baseline_commands_rendered": True,
|
|
646
|
+
"host_codex_command_readiness_rendered": True,
|
|
647
|
+
"attempt_patch_output_placement_checked": True,
|
|
648
|
+
"compact_run_filename_checked": True,
|
|
649
|
+
"future_execution_opt_in_required": True,
|
|
650
|
+
"no_real_source_extraction": True,
|
|
651
|
+
"no_real_codex_execution": True,
|
|
652
|
+
"no_docker_pull_or_start": True,
|
|
653
|
+
"no_auth_sync_to_shared_host": True,
|
|
654
|
+
"no_fixed_diff_or_oracle_read": True,
|
|
655
|
+
}
|
|
656
|
+
)
|
|
657
|
+
for trial in benchmark_run.get("trials") or []:
|
|
658
|
+
if isinstance(trial, dict):
|
|
659
|
+
trial["exception_type"] = "execution_gate_only_no_real_case"
|
|
660
|
+
|
|
661
|
+
gate_path.write_text(
|
|
662
|
+
json.dumps(gate, ensure_ascii=False, indent=2, sort_keys=True) + "\n",
|
|
663
|
+
encoding="utf-8",
|
|
664
|
+
)
|
|
665
|
+
compact_run_path.write_text(
|
|
666
|
+
json.dumps(benchmark_run, ensure_ascii=False, indent=2, sort_keys=True) + "\n",
|
|
667
|
+
encoding="utf-8",
|
|
668
|
+
)
|
|
669
|
+
return {
|
|
670
|
+
"schema_version": AGENTISSUE_CODEX_CLI_RUNNER_EXECUTION_GATE_SCHEMA_VERSION,
|
|
671
|
+
"benchmark_id": AGENTISSUE_BENCHMARK_ID,
|
|
672
|
+
"selected_tag": tag,
|
|
673
|
+
"selected_image": image,
|
|
674
|
+
"ready": True,
|
|
675
|
+
"materialized": True,
|
|
676
|
+
"path_recorded": False,
|
|
677
|
+
"gate_root_path_recorded": False,
|
|
678
|
+
"synthetic_staging": {
|
|
679
|
+
"schema_version": staging["schema_version"],
|
|
680
|
+
"ready": staging["ready"],
|
|
681
|
+
"created_relative_paths": staging["created_relative_paths"],
|
|
682
|
+
"path_recorded": False,
|
|
683
|
+
},
|
|
684
|
+
"created_relative_paths": [
|
|
685
|
+
*staging["created_relative_paths"],
|
|
686
|
+
"execution-gate.public.json",
|
|
687
|
+
],
|
|
688
|
+
"gate_relative_path": "execution-gate.public.json",
|
|
689
|
+
"compact_run_relative_path": "benchmark_run.compact.json",
|
|
690
|
+
"attempt_patch_relative_path": AGENTISSUE_PATCH_RELATIVE_PATH,
|
|
691
|
+
"gate_checks": {
|
|
692
|
+
"selected_container_source_extraction_commands_rendered": True,
|
|
693
|
+
"private_git_baseline_commands_rendered": True,
|
|
694
|
+
"host_codex_command_readiness_rendered": True,
|
|
695
|
+
"attempt_patch_output_placement_checked": True,
|
|
696
|
+
"future_execution_opt_in_required": True,
|
|
697
|
+
},
|
|
698
|
+
"execution_boundary": {
|
|
699
|
+
"codex_cli_invoked": False,
|
|
700
|
+
"model_api_invoked": False,
|
|
701
|
+
"docker_image_pulled": False,
|
|
702
|
+
"docker_container_started": False,
|
|
703
|
+
"source_extracted": False,
|
|
704
|
+
"git_baseline_created": False,
|
|
705
|
+
"patch_generated": False,
|
|
706
|
+
"patch_evaluated": False,
|
|
707
|
+
"credential_values_recorded": False,
|
|
708
|
+
"auth_material_synced": False,
|
|
709
|
+
},
|
|
710
|
+
"benchmark_run": benchmark_run,
|
|
711
|
+
"recommended_next_action": (
|
|
712
|
+
"build a no-execute first-run handoff packet for lagent_239"
|
|
713
|
+
),
|
|
714
|
+
}
|
|
715
|
+
|
|
716
|
+
|
|
717
|
+
def materialize_agentissue_codex_cli_runner_first_run_handoff(
|
|
718
|
+
handoff_root: str | Path,
|
|
719
|
+
*,
|
|
720
|
+
selected_tag: str = AGENTISSUE_DEFAULT_TAG,
|
|
721
|
+
codex_binary: str = "codex",
|
|
722
|
+
docker_binary: str = "docker",
|
|
723
|
+
) -> dict[str, Any]:
|
|
724
|
+
"""Create a no-execute first-run handoff packet for AgentIssue lagent_239."""
|
|
725
|
+
|
|
726
|
+
tag = _agentissue_public_label(selected_tag)
|
|
727
|
+
if tag != AGENTISSUE_DEFAULT_TAG:
|
|
728
|
+
raise ValueError(
|
|
729
|
+
"agentissue Codex runner first-run handoff currently only supports selected tag lagent_239"
|
|
730
|
+
)
|
|
731
|
+
root = Path(handoff_root).expanduser()
|
|
732
|
+
gate = materialize_agentissue_codex_cli_runner_execution_gate(
|
|
733
|
+
root,
|
|
734
|
+
selected_tag=tag,
|
|
735
|
+
codex_binary=codex_binary,
|
|
736
|
+
docker_binary=docker_binary,
|
|
737
|
+
)
|
|
738
|
+
handoff_path = root / "first-run-handoff.public.json"
|
|
739
|
+
handoff_markdown_path = root / "first-run-handoff.md"
|
|
740
|
+
compact_run_path = root / "benchmark_run.compact.json"
|
|
741
|
+
|
|
742
|
+
no_execute_cli_argv = [
|
|
743
|
+
"loopx",
|
|
744
|
+
"benchmark",
|
|
745
|
+
"agentissue-codex-runner-flow",
|
|
746
|
+
"--goal-id",
|
|
747
|
+
"<goal-id>",
|
|
748
|
+
"--tag",
|
|
749
|
+
tag,
|
|
750
|
+
"--execution-gate-root",
|
|
751
|
+
"<private-gate-root>",
|
|
752
|
+
"--delivery-batch-scale",
|
|
753
|
+
"multi_surface",
|
|
754
|
+
"--delivery-outcome",
|
|
755
|
+
DeliveryOutcome.OUTCOME_PROGRESS.value,
|
|
756
|
+
"--execute",
|
|
757
|
+
]
|
|
758
|
+
safety_checklist = [
|
|
759
|
+
{
|
|
760
|
+
"item": "private_job_root_selected",
|
|
761
|
+
"required_before_later_e2e": True,
|
|
762
|
+
"satisfied_by_this_packet": False,
|
|
763
|
+
},
|
|
764
|
+
{
|
|
765
|
+
"item": "codex_auth_stays_on_host",
|
|
766
|
+
"required_before_later_e2e": True,
|
|
767
|
+
"satisfied_by_this_packet": True,
|
|
768
|
+
},
|
|
769
|
+
{
|
|
770
|
+
"item": "no_codex_home_sync_to_shared_host",
|
|
771
|
+
"required_before_later_e2e": True,
|
|
772
|
+
"satisfied_by_this_packet": True,
|
|
773
|
+
},
|
|
774
|
+
{
|
|
775
|
+
"item": "selected_container_source_extraction_planned",
|
|
776
|
+
"required_before_later_e2e": True,
|
|
777
|
+
"satisfied_by_this_packet": False,
|
|
778
|
+
},
|
|
779
|
+
{
|
|
780
|
+
"item": "attempt_patch_compact_reducer_planned",
|
|
781
|
+
"required_before_later_e2e": True,
|
|
782
|
+
"satisfied_by_this_packet": True,
|
|
783
|
+
},
|
|
784
|
+
{
|
|
785
|
+
"item": "upload_submit_public_ranking_disabled",
|
|
786
|
+
"required_before_later_e2e": True,
|
|
787
|
+
"satisfied_by_this_packet": True,
|
|
788
|
+
},
|
|
789
|
+
]
|
|
790
|
+
handoff = {
|
|
791
|
+
"schema_version": AGENTISSUE_CODEX_CLI_RUNNER_FIRST_RUN_HANDOFF_SCHEMA_VERSION,
|
|
792
|
+
"benchmark_id": AGENTISSUE_BENCHMARK_ID,
|
|
793
|
+
"selected_tag": tag,
|
|
794
|
+
"selected_image": AGENTISSUE_DEFAULT_IMAGE,
|
|
795
|
+
"path_recorded": False,
|
|
796
|
+
"default_mode": "no_execute",
|
|
797
|
+
"later_operator_triggered_e2e": True,
|
|
798
|
+
"real_run_done": False,
|
|
799
|
+
"exact_command_shape": {
|
|
800
|
+
"argv": no_execute_cli_argv,
|
|
801
|
+
"runs_real_benchmark": False,
|
|
802
|
+
"appends_compact_no_run_event": True,
|
|
803
|
+
},
|
|
804
|
+
"private_artifact_boundary": {
|
|
805
|
+
"root_placeholder": "<private-gate-root>",
|
|
806
|
+
"root_path_recorded": False,
|
|
807
|
+
"public_relative_files": [
|
|
808
|
+
"runner-flow-plan.public.json",
|
|
809
|
+
"execution-gate.public.json",
|
|
810
|
+
"first-run-handoff.public.json",
|
|
811
|
+
"first-run-handoff.md",
|
|
812
|
+
"benchmark_run.compact.json",
|
|
813
|
+
],
|
|
814
|
+
"private_relative_dirs": [
|
|
815
|
+
"context/",
|
|
816
|
+
"buggy-source/",
|
|
817
|
+
"Patches/lagent_239/",
|
|
818
|
+
],
|
|
819
|
+
"raw_artifacts_public": False,
|
|
820
|
+
"absolute_paths_public": False,
|
|
821
|
+
},
|
|
822
|
+
"expected_compact_outputs": {
|
|
823
|
+
"benchmark_run_mode": AGENTISSUE_CODEX_CLI_RUNNER_FIRST_RUN_HANDOFF_MODE,
|
|
824
|
+
"compact_run": "benchmark_run.compact.json",
|
|
825
|
+
"history_event": "benchmark_run_v0",
|
|
826
|
+
"official_score_claim_allowed": False,
|
|
827
|
+
"submit_eligible": False,
|
|
828
|
+
"leaderboard_evidence": False,
|
|
829
|
+
},
|
|
830
|
+
"budget_auth_boundary": {
|
|
831
|
+
"codex_auth_values_read": False,
|
|
832
|
+
"codex_home_synced": False,
|
|
833
|
+
"model_api_invoked": False,
|
|
834
|
+
"model_budget_spent_by_packet": False,
|
|
835
|
+
"docker_invoked_by_packet": False,
|
|
836
|
+
"shared_remote_host_receives_codex_auth": False,
|
|
837
|
+
},
|
|
838
|
+
"safety_checklist": safety_checklist,
|
|
839
|
+
"no_execute_assertions": {
|
|
840
|
+
"source_extracted": False,
|
|
841
|
+
"codex_cli_invoked": False,
|
|
842
|
+
"docker_container_started": False,
|
|
843
|
+
"patch_generated": False,
|
|
844
|
+
"patch_evaluated": False,
|
|
845
|
+
"upload": False,
|
|
846
|
+
"submit": False,
|
|
847
|
+
"public_ranking_path": False,
|
|
848
|
+
"destructive_git": False,
|
|
849
|
+
"production_action": False,
|
|
850
|
+
},
|
|
851
|
+
}
|
|
852
|
+
handoff_markdown = (
|
|
853
|
+
"# AgentIssue-Bench lagent_239 First-Run Handoff\n\n"
|
|
854
|
+
"This packet is no-execute. It names the command shape, private artifact "
|
|
855
|
+
"boundary, compact outputs, budget/auth boundary, and safety checklist "
|
|
856
|
+
"for a later operator-triggered e2e run.\n\n"
|
|
857
|
+
"## Command Shape\n\n"
|
|
858
|
+
"```text\n"
|
|
859
|
+
+ " ".join(no_execute_cli_argv)
|
|
860
|
+
+ "\n```\n\n"
|
|
861
|
+
"## Boundary\n\n"
|
|
862
|
+
"- Codex auth stays on the host and is not copied to a shared machine.\n"
|
|
863
|
+
"- Public files are limited to `*.public.json`, `*.compact.json`, and this packet.\n"
|
|
864
|
+
"- No source extraction, Docker start, Codex invocation, patch generation, "
|
|
865
|
+
"evaluation, upload, submit, public ranking, destructive git, or production "
|
|
866
|
+
"action is performed by this packet.\n"
|
|
867
|
+
)
|
|
868
|
+
|
|
869
|
+
benchmark_run = json.loads(json.dumps(gate["benchmark_run"]))
|
|
870
|
+
benchmark_run.update(
|
|
871
|
+
{
|
|
872
|
+
"job_name": "agentissue_lagent_239_codex_cli_runner_first_run_handoff",
|
|
873
|
+
"mode": AGENTISSUE_CODEX_CLI_RUNNER_FIRST_RUN_HANDOFF_MODE,
|
|
874
|
+
"worker_mode": "trusted_host_codex_cli_no_execute_first_run_handoff",
|
|
875
|
+
"first_blocker": "first_run_handoff_only_no_real_case",
|
|
876
|
+
"score_failure_attribution": "not_run_first_run_handoff_only",
|
|
877
|
+
"failure_attribution_labels": [
|
|
878
|
+
"first_run_handoff_packet_only",
|
|
879
|
+
"ready_for_later_operator_triggered_e2e_run",
|
|
880
|
+
],
|
|
881
|
+
"evidence_files": [
|
|
882
|
+
"first-run-handoff.public.json",
|
|
883
|
+
"first-run-handoff.md",
|
|
884
|
+
"execution-gate.public.json",
|
|
885
|
+
"benchmark_run.compact.json",
|
|
886
|
+
"runner-flow-plan.public.json",
|
|
887
|
+
],
|
|
888
|
+
}
|
|
889
|
+
)
|
|
890
|
+
benchmark_run["validation"].update(
|
|
891
|
+
{
|
|
892
|
+
"first_run_handoff_materialized": True,
|
|
893
|
+
"exact_command_shape_rendered": True,
|
|
894
|
+
"private_artifact_boundary_declared": True,
|
|
895
|
+
"expected_compact_outputs_declared": True,
|
|
896
|
+
"budget_auth_boundary_declared": True,
|
|
897
|
+
"safety_checklist_declared": True,
|
|
898
|
+
"no_execute_packet": True,
|
|
899
|
+
"no_codex_auth_value_read": True,
|
|
900
|
+
"no_codex_home_sync": True,
|
|
901
|
+
"no_model_budget_spent_by_packet": True,
|
|
902
|
+
}
|
|
903
|
+
)
|
|
904
|
+
for trial in benchmark_run.get("trials") or []:
|
|
905
|
+
if isinstance(trial, dict):
|
|
906
|
+
trial["exception_type"] = "first_run_handoff_only_no_real_case"
|
|
907
|
+
|
|
908
|
+
handoff_path.write_text(
|
|
909
|
+
json.dumps(handoff, ensure_ascii=False, indent=2, sort_keys=True) + "\n",
|
|
910
|
+
encoding="utf-8",
|
|
911
|
+
)
|
|
912
|
+
handoff_markdown_path.write_text(handoff_markdown, encoding="utf-8")
|
|
913
|
+
compact_run_path.write_text(
|
|
914
|
+
json.dumps(benchmark_run, ensure_ascii=False, indent=2, sort_keys=True) + "\n",
|
|
915
|
+
encoding="utf-8",
|
|
916
|
+
)
|
|
917
|
+
return {
|
|
918
|
+
"schema_version": AGENTISSUE_CODEX_CLI_RUNNER_FIRST_RUN_HANDOFF_SCHEMA_VERSION,
|
|
919
|
+
"benchmark_id": AGENTISSUE_BENCHMARK_ID,
|
|
920
|
+
"selected_tag": tag,
|
|
921
|
+
"selected_image": AGENTISSUE_DEFAULT_IMAGE,
|
|
922
|
+
"ready": True,
|
|
923
|
+
"materialized": True,
|
|
924
|
+
"path_recorded": False,
|
|
925
|
+
"handoff_root_path_recorded": False,
|
|
926
|
+
"execution_gate": {
|
|
927
|
+
"schema_version": gate["schema_version"],
|
|
928
|
+
"ready": gate["ready"],
|
|
929
|
+
"gate_relative_path": gate["gate_relative_path"],
|
|
930
|
+
"path_recorded": False,
|
|
931
|
+
},
|
|
932
|
+
"created_relative_paths": [
|
|
933
|
+
*gate["created_relative_paths"],
|
|
934
|
+
"first-run-handoff.public.json",
|
|
935
|
+
"first-run-handoff.md",
|
|
936
|
+
],
|
|
937
|
+
"handoff_relative_path": "first-run-handoff.public.json",
|
|
938
|
+
"handoff_markdown_relative_path": "first-run-handoff.md",
|
|
939
|
+
"compact_run_relative_path": "benchmark_run.compact.json",
|
|
940
|
+
"handoff_checks": {
|
|
941
|
+
"exact_command_shape_rendered": True,
|
|
942
|
+
"private_artifact_boundary_declared": True,
|
|
943
|
+
"expected_compact_outputs_declared": True,
|
|
944
|
+
"budget_auth_boundary_declared": True,
|
|
945
|
+
"safety_checklist_declared": True,
|
|
946
|
+
"later_operator_triggered_e2e": True,
|
|
947
|
+
},
|
|
948
|
+
"execution_boundary": handoff["no_execute_assertions"],
|
|
949
|
+
"benchmark_run": benchmark_run,
|
|
950
|
+
"recommended_next_action": (
|
|
951
|
+
"use the no-execute first-run handoff packet as the checklist for a "
|
|
952
|
+
"later operator-triggered AgentIssue-Bench lagent_239 e2e run"
|
|
953
|
+
),
|
|
954
|
+
}
|
|
955
|
+
|
|
956
|
+
|
|
957
|
+
def materialize_agentissue_codex_cli_runner_workflow_check(
|
|
958
|
+
workflow_check_root: str | Path,
|
|
959
|
+
*,
|
|
960
|
+
selected_tag: str = AGENTISSUE_DEFAULT_TAG,
|
|
961
|
+
codex_binary: str = "codex",
|
|
962
|
+
docker_binary: str = "docker",
|
|
963
|
+
) -> dict[str, Any]:
|
|
964
|
+
"""Create a no-execute workflow check packet for AgentIssue lagent_239."""
|
|
965
|
+
|
|
966
|
+
tag = _agentissue_public_label(selected_tag)
|
|
967
|
+
if tag != AGENTISSUE_DEFAULT_TAG:
|
|
968
|
+
raise ValueError(
|
|
969
|
+
"agentissue Codex runner workflow check currently only supports selected tag lagent_239"
|
|
970
|
+
)
|
|
971
|
+
root = Path(workflow_check_root).expanduser()
|
|
972
|
+
handoff = materialize_agentissue_codex_cli_runner_first_run_handoff(
|
|
973
|
+
root,
|
|
974
|
+
selected_tag=tag,
|
|
975
|
+
codex_binary=codex_binary,
|
|
976
|
+
docker_binary=docker_binary,
|
|
977
|
+
)
|
|
978
|
+
runner_plan_path = root / "runner-flow-plan.public.json"
|
|
979
|
+
gate_path = root / "execution-gate.public.json"
|
|
980
|
+
handoff_path = root / "first-run-handoff.public.json"
|
|
981
|
+
workflow_path = root / "workflow-check.public.json"
|
|
982
|
+
compact_run_path = root / "benchmark_run.compact.json"
|
|
983
|
+
|
|
984
|
+
runner_plan = json.loads(runner_plan_path.read_text(encoding="utf-8"))
|
|
985
|
+
gate = json.loads(gate_path.read_text(encoding="utf-8"))
|
|
986
|
+
handoff_public = json.loads(handoff_path.read_text(encoding="utf-8"))
|
|
987
|
+
|
|
988
|
+
codex_command = runner_plan["command_placeholders"]["codex_patch_worker"]
|
|
989
|
+
eval_command = runner_plan["command_placeholders"]["single_tag_eval"]
|
|
990
|
+
patch_export = runner_plan["command_placeholders"]["patch_export"]
|
|
991
|
+
budget_auth = handoff_public["budget_auth_boundary"]
|
|
992
|
+
no_execute = handoff_public["no_execute_assertions"]
|
|
993
|
+
required_public_files = [
|
|
994
|
+
"runner-flow-plan.public.json",
|
|
995
|
+
"execution-gate.public.json",
|
|
996
|
+
"first-run-handoff.public.json",
|
|
997
|
+
"first-run-handoff.md",
|
|
998
|
+
"workflow-check.public.json",
|
|
999
|
+
"benchmark_run.compact.json",
|
|
1000
|
+
]
|
|
1001
|
+
required_private_dirs = [
|
|
1002
|
+
"context/",
|
|
1003
|
+
"buggy-source/",
|
|
1004
|
+
"Patches/lagent_239/",
|
|
1005
|
+
]
|
|
1006
|
+
checks = {
|
|
1007
|
+
"single_selected_tag": runner_plan["selected_tag"] == gate["selected_tag"] == handoff_public["selected_tag"] == tag,
|
|
1008
|
+
"selected_image_consistent": runner_plan["selected_image"] == gate["selected_image"] == handoff_public["selected_image"],
|
|
1009
|
+
"source_extracted_before_codex": bool(codex_command.get("runs_after_buggy_source_extraction")),
|
|
1010
|
+
"host_codex_uses_ephemeral": "--ephemeral" in codex_command.get("argv", []),
|
|
1011
|
+
"host_codex_auth_not_synced": codex_command.get("auth_material_synced") is False and budget_auth["codex_home_synced"] is False,
|
|
1012
|
+
"worker_no_network_or_docker": codex_command.get("worker_network_allowed") is False and codex_command.get("worker_docker_allowed") is False,
|
|
1013
|
+
"patch_from_buggy_source_git_diff": patch_export["input_source"] == "buggy_source_git_diff",
|
|
1014
|
+
"attempt_patch_relative_path": patch_export["output_relative_path"] == AGENTISSUE_PATCH_RELATIVE_PATH,
|
|
1015
|
+
"single_tag_eval_no_upload_submit": eval_command["upload"] is False and eval_command["submit"] is False,
|
|
1016
|
+
"single_tag_eval_no_public_ranking": eval_command["public_ranking_path"] is False,
|
|
1017
|
+
"no_execute_packet": all(value is False for value in no_execute.values()),
|
|
1018
|
+
"public_files_compact_or_public": all(
|
|
1019
|
+
path.endswith((".public.json", ".compact.json", ".md"))
|
|
1020
|
+
for path in required_public_files
|
|
1021
|
+
),
|
|
1022
|
+
"private_dirs_not_public_artifacts": all(path.endswith("/") for path in required_private_dirs),
|
|
1023
|
+
}
|
|
1024
|
+
failed_checks = [name for name, passed in checks.items() if not passed]
|
|
1025
|
+
workflow_check = {
|
|
1026
|
+
"schema_version": AGENTISSUE_CODEX_CLI_RUNNER_WORKFLOW_CHECK_SCHEMA_VERSION,
|
|
1027
|
+
"benchmark_id": AGENTISSUE_BENCHMARK_ID,
|
|
1028
|
+
"selected_tag": tag,
|
|
1029
|
+
"selected_image": AGENTISSUE_DEFAULT_IMAGE,
|
|
1030
|
+
"ready": not failed_checks,
|
|
1031
|
+
"materialized": True,
|
|
1032
|
+
"path_recorded": False,
|
|
1033
|
+
"default_mode": "no_execute",
|
|
1034
|
+
"input_packets": {
|
|
1035
|
+
"runner_plan": "runner-flow-plan.public.json",
|
|
1036
|
+
"execution_gate": "execution-gate.public.json",
|
|
1037
|
+
"first_run_handoff": "first-run-handoff.public.json",
|
|
1038
|
+
},
|
|
1039
|
+
"required_public_files": required_public_files,
|
|
1040
|
+
"required_private_dirs": required_private_dirs,
|
|
1041
|
+
"workflow_checks": checks,
|
|
1042
|
+
"failed_checks": failed_checks,
|
|
1043
|
+
"execution_boundary": {
|
|
1044
|
+
"codex_cli_invoked": False,
|
|
1045
|
+
"model_api_invoked": False,
|
|
1046
|
+
"docker_image_pulled": False,
|
|
1047
|
+
"docker_container_started": False,
|
|
1048
|
+
"source_extracted": False,
|
|
1049
|
+
"git_baseline_created": False,
|
|
1050
|
+
"patch_generated": False,
|
|
1051
|
+
"patch_evaluated": False,
|
|
1052
|
+
"credential_values_recorded": False,
|
|
1053
|
+
"auth_material_synced": False,
|
|
1054
|
+
"upload": False,
|
|
1055
|
+
"submit": False,
|
|
1056
|
+
"public_ranking_path": False,
|
|
1057
|
+
},
|
|
1058
|
+
"stop_before_later_e2e_unless": [
|
|
1059
|
+
"private_job_root_selected",
|
|
1060
|
+
"operator_explicitly_triggers_real_run",
|
|
1061
|
+
"runner_artifact_reducer_writes_compact_public_result",
|
|
1062
|
+
],
|
|
1063
|
+
}
|
|
1064
|
+
|
|
1065
|
+
benchmark_run = json.loads(json.dumps(handoff["benchmark_run"]))
|
|
1066
|
+
benchmark_run.update(
|
|
1067
|
+
{
|
|
1068
|
+
"job_name": "agentissue_lagent_239_codex_cli_runner_workflow_check",
|
|
1069
|
+
"mode": AGENTISSUE_CODEX_CLI_RUNNER_WORKFLOW_CHECK_MODE,
|
|
1070
|
+
"worker_mode": "trusted_host_codex_cli_no_execute_workflow_check",
|
|
1071
|
+
"first_blocker": "workflow_check_only_no_real_case",
|
|
1072
|
+
"score_failure_attribution": "not_run_workflow_check_only",
|
|
1073
|
+
"failure_attribution_labels": [
|
|
1074
|
+
"workflow_check_packet_only",
|
|
1075
|
+
"ready_for_later_operator_triggered_e2e_run"
|
|
1076
|
+
if not failed_checks
|
|
1077
|
+
else "workflow_check_failed_before_real_run",
|
|
1078
|
+
],
|
|
1079
|
+
"evidence_files": required_public_files,
|
|
1080
|
+
}
|
|
1081
|
+
)
|
|
1082
|
+
benchmark_run["validation"].update(
|
|
1083
|
+
{
|
|
1084
|
+
"workflow_check_materialized": True,
|
|
1085
|
+
"workflow_check_all_passed": not failed_checks,
|
|
1086
|
+
"workflow_check_failed_checks": failed_checks,
|
|
1087
|
+
"single_selected_tag": checks["single_selected_tag"],
|
|
1088
|
+
"selected_image_consistent": checks["selected_image_consistent"],
|
|
1089
|
+
"source_extracted_before_codex": checks["source_extracted_before_codex"],
|
|
1090
|
+
"host_codex_uses_ephemeral": checks["host_codex_uses_ephemeral"],
|
|
1091
|
+
"host_codex_auth_not_synced": checks["host_codex_auth_not_synced"],
|
|
1092
|
+
"worker_no_network_or_docker": checks["worker_no_network_or_docker"],
|
|
1093
|
+
"patch_from_buggy_source_git_diff": checks["patch_from_buggy_source_git_diff"],
|
|
1094
|
+
"single_tag_eval_no_upload_submit": checks["single_tag_eval_no_upload_submit"],
|
|
1095
|
+
"single_tag_eval_no_public_ranking": checks["single_tag_eval_no_public_ranking"],
|
|
1096
|
+
}
|
|
1097
|
+
)
|
|
1098
|
+
for trial in benchmark_run.get("trials") or []:
|
|
1099
|
+
if isinstance(trial, dict):
|
|
1100
|
+
trial["exception_type"] = "workflow_check_only_no_real_case"
|
|
1101
|
+
|
|
1102
|
+
workflow_path.write_text(
|
|
1103
|
+
json.dumps(workflow_check, ensure_ascii=False, indent=2, sort_keys=True) + "\n",
|
|
1104
|
+
encoding="utf-8",
|
|
1105
|
+
)
|
|
1106
|
+
compact_run_path.write_text(
|
|
1107
|
+
json.dumps(benchmark_run, ensure_ascii=False, indent=2, sort_keys=True) + "\n",
|
|
1108
|
+
encoding="utf-8",
|
|
1109
|
+
)
|
|
1110
|
+
return {
|
|
1111
|
+
"schema_version": AGENTISSUE_CODEX_CLI_RUNNER_WORKFLOW_CHECK_SCHEMA_VERSION,
|
|
1112
|
+
"benchmark_id": AGENTISSUE_BENCHMARK_ID,
|
|
1113
|
+
"selected_tag": tag,
|
|
1114
|
+
"selected_image": AGENTISSUE_DEFAULT_IMAGE,
|
|
1115
|
+
"ready": not failed_checks,
|
|
1116
|
+
"materialized": True,
|
|
1117
|
+
"path_recorded": False,
|
|
1118
|
+
"workflow_check_root_path_recorded": False,
|
|
1119
|
+
"first_run_handoff": {
|
|
1120
|
+
"schema_version": handoff["schema_version"],
|
|
1121
|
+
"ready": handoff["ready"],
|
|
1122
|
+
"handoff_relative_path": handoff["handoff_relative_path"],
|
|
1123
|
+
"path_recorded": False,
|
|
1124
|
+
},
|
|
1125
|
+
"created_relative_paths": [
|
|
1126
|
+
*handoff["created_relative_paths"],
|
|
1127
|
+
"workflow-check.public.json",
|
|
1128
|
+
],
|
|
1129
|
+
"workflow_check_relative_path": "workflow-check.public.json",
|
|
1130
|
+
"compact_run_relative_path": "benchmark_run.compact.json",
|
|
1131
|
+
"workflow_checks": checks,
|
|
1132
|
+
"failed_checks": failed_checks,
|
|
1133
|
+
"execution_boundary": workflow_check["execution_boundary"],
|
|
1134
|
+
"benchmark_run": benchmark_run,
|
|
1135
|
+
"recommended_next_action": (
|
|
1136
|
+
"use workflow-check.public.json as the pre-run invariant packet before "
|
|
1137
|
+
"any later operator-triggered AgentIssue-Bench lagent_239 e2e run"
|
|
1138
|
+
),
|
|
1139
|
+
}
|
|
1140
|
+
|
|
1141
|
+
def materialize_agentissue_codex_cli_runner_run_gate(
|
|
1142
|
+
run_gate_root: str | Path,
|
|
1143
|
+
*,
|
|
1144
|
+
selected_tag: str = AGENTISSUE_DEFAULT_TAG,
|
|
1145
|
+
codex_binary: str = "codex",
|
|
1146
|
+
docker_binary: str = "docker",
|
|
1147
|
+
) -> dict[str, Any]:
|
|
1148
|
+
"""Create a no-execute run-specific gate packet for AgentIssue lagent_239."""
|
|
1149
|
+
|
|
1150
|
+
tag = _agentissue_public_label(selected_tag)
|
|
1151
|
+
if tag != AGENTISSUE_DEFAULT_TAG:
|
|
1152
|
+
raise ValueError(
|
|
1153
|
+
"agentissue Codex runner run-specific gate currently only supports selected tag lagent_239"
|
|
1154
|
+
)
|
|
1155
|
+
root = Path(run_gate_root).expanduser()
|
|
1156
|
+
workflow = materialize_agentissue_codex_cli_runner_workflow_check(
|
|
1157
|
+
root,
|
|
1158
|
+
selected_tag=tag,
|
|
1159
|
+
codex_binary=codex_binary,
|
|
1160
|
+
docker_binary=docker_binary,
|
|
1161
|
+
)
|
|
1162
|
+
workflow_path = root / "workflow-check.public.json"
|
|
1163
|
+
gate_path = root / "execution-gate.public.json"
|
|
1164
|
+
handoff_path = root / "first-run-handoff.public.json"
|
|
1165
|
+
run_gate_path = root / "run-specific-gate.public.json"
|
|
1166
|
+
run_gate_markdown_path = root / "run-specific-gate.md"
|
|
1167
|
+
compact_run_path = root / "benchmark_run.compact.json"
|
|
1168
|
+
|
|
1169
|
+
workflow_check = json.loads(workflow_path.read_text(encoding="utf-8"))
|
|
1170
|
+
execution_gate = json.loads(gate_path.read_text(encoding="utf-8"))
|
|
1171
|
+
handoff = json.loads(handoff_path.read_text(encoding="utf-8"))
|
|
1172
|
+
|
|
1173
|
+
gate_items = [
|
|
1174
|
+
{
|
|
1175
|
+
"id": "selected_tag_and_image_locked",
|
|
1176
|
+
"owner": "agent",
|
|
1177
|
+
"required_before_real_run": True,
|
|
1178
|
+
"satisfied_by_packet": workflow_check["workflow_checks"]["single_selected_tag"]
|
|
1179
|
+
and workflow_check["workflow_checks"]["selected_image_consistent"],
|
|
1180
|
+
"public_evidence": "workflow-check.public.json",
|
|
1181
|
+
},
|
|
1182
|
+
{
|
|
1183
|
+
"id": "host_codex_auth_local_only",
|
|
1184
|
+
"owner": "agent",
|
|
1185
|
+
"required_before_real_run": True,
|
|
1186
|
+
"satisfied_by_packet": workflow_check["workflow_checks"]["host_codex_auth_not_synced"],
|
|
1187
|
+
"public_evidence": "workflow-check.public.json",
|
|
1188
|
+
},
|
|
1189
|
+
{
|
|
1190
|
+
"id": "private_job_root_selected",
|
|
1191
|
+
"owner": "agent",
|
|
1192
|
+
"required_before_real_run": True,
|
|
1193
|
+
"satisfied_by_packet": False,
|
|
1194
|
+
"stop_if_missing": True,
|
|
1195
|
+
},
|
|
1196
|
+
{
|
|
1197
|
+
"id": "operator_explicit_real_run_trigger",
|
|
1198
|
+
"owner": "owner",
|
|
1199
|
+
"required_before_real_run": True,
|
|
1200
|
+
"satisfied_by_packet": False,
|
|
1201
|
+
"stop_if_missing": True,
|
|
1202
|
+
},
|
|
1203
|
+
{
|
|
1204
|
+
"id": "selected_container_source_extracted",
|
|
1205
|
+
"owner": "agent",
|
|
1206
|
+
"required_before_real_run": True,
|
|
1207
|
+
"satisfied_by_packet": False,
|
|
1208
|
+
"public_command_shape": "execution-gate.public.json",
|
|
1209
|
+
"stop_if_missing": True,
|
|
1210
|
+
},
|
|
1211
|
+
{
|
|
1212
|
+
"id": "private_git_baseline_created_before_codex",
|
|
1213
|
+
"owner": "agent",
|
|
1214
|
+
"required_before_real_run": True,
|
|
1215
|
+
"satisfied_by_packet": False,
|
|
1216
|
+
"public_command_shape": "execution-gate.public.json",
|
|
1217
|
+
"stop_if_missing": True,
|
|
1218
|
+
},
|
|
1219
|
+
{
|
|
1220
|
+
"id": "host_codex_exec_ephemeral_from_buggy_source",
|
|
1221
|
+
"owner": "agent",
|
|
1222
|
+
"required_before_real_run": True,
|
|
1223
|
+
"satisfied_by_packet": False,
|
|
1224
|
+
"public_command_shape": "execution-gate.public.json",
|
|
1225
|
+
"stop_if_missing": True,
|
|
1226
|
+
},
|
|
1227
|
+
{
|
|
1228
|
+
"id": "attempt_patch_reducer_configured",
|
|
1229
|
+
"owner": "agent",
|
|
1230
|
+
"required_before_real_run": True,
|
|
1231
|
+
"satisfied_by_packet": workflow_check["workflow_checks"]["patch_from_buggy_source_git_diff"]
|
|
1232
|
+
and workflow_check["workflow_checks"]["attempt_patch_relative_path"],
|
|
1233
|
+
"public_evidence": AGENTISSUE_PATCH_RELATIVE_PATH,
|
|
1234
|
+
},
|
|
1235
|
+
{
|
|
1236
|
+
"id": "selected_tag_eval_no_upload_submit_ranking",
|
|
1237
|
+
"owner": "agent",
|
|
1238
|
+
"required_before_real_run": True,
|
|
1239
|
+
"satisfied_by_packet": workflow_check["workflow_checks"]["single_tag_eval_no_upload_submit"]
|
|
1240
|
+
and workflow_check["workflow_checks"]["single_tag_eval_no_public_ranking"],
|
|
1241
|
+
"public_evidence": "workflow-check.public.json",
|
|
1242
|
+
},
|
|
1243
|
+
{
|
|
1244
|
+
"id": "compact_public_reducer_enabled",
|
|
1245
|
+
"owner": "agent",
|
|
1246
|
+
"required_before_real_run": True,
|
|
1247
|
+
"satisfied_by_packet": workflow_check["workflow_checks"]["public_files_compact_or_public"],
|
|
1248
|
+
"public_evidence": "benchmark_run.compact.json",
|
|
1249
|
+
},
|
|
1250
|
+
{
|
|
1251
|
+
"id": "raw_artifact_and_auth_leak_stop_rules_enabled",
|
|
1252
|
+
"owner": "agent",
|
|
1253
|
+
"required_before_real_run": True,
|
|
1254
|
+
"satisfied_by_packet": True,
|
|
1255
|
+
"stop_if_raw_task_patch_log_trajectory_screenshot_or_auth_material_public": True,
|
|
1256
|
+
},
|
|
1257
|
+
]
|
|
1258
|
+
blocking_gate_ids = [
|
|
1259
|
+
item["id"]
|
|
1260
|
+
for item in gate_items
|
|
1261
|
+
if item["required_before_real_run"] and not item["satisfied_by_packet"]
|
|
1262
|
+
]
|
|
1263
|
+
run_gate = {
|
|
1264
|
+
"schema_version": AGENTISSUE_CODEX_CLI_RUNNER_RUN_GATE_SCHEMA_VERSION,
|
|
1265
|
+
"benchmark_id": AGENTISSUE_BENCHMARK_ID,
|
|
1266
|
+
"selected_tag": tag,
|
|
1267
|
+
"selected_image": AGENTISSUE_DEFAULT_IMAGE,
|
|
1268
|
+
"default_mode": "no_execute",
|
|
1269
|
+
"materialized": True,
|
|
1270
|
+
"path_recorded": False,
|
|
1271
|
+
"real_run_authorized": False,
|
|
1272
|
+
"ready_for_real_run": False,
|
|
1273
|
+
"ready_for_operator_review": True,
|
|
1274
|
+
"blocking_gate_ids": blocking_gate_ids,
|
|
1275
|
+
"input_packets": {
|
|
1276
|
+
"workflow_check": "workflow-check.public.json",
|
|
1277
|
+
"first_run_handoff": "first-run-handoff.public.json",
|
|
1278
|
+
"execution_gate": "execution-gate.public.json",
|
|
1279
|
+
},
|
|
1280
|
+
"owner_agent_gate_items": gate_items,
|
|
1281
|
+
"phase_order": [
|
|
1282
|
+
"select_private_job_root",
|
|
1283
|
+
"extract_selected_container_buggy_source",
|
|
1284
|
+
"create_private_git_baseline",
|
|
1285
|
+
"run_host_codex_exec_ephemeral_from_buggy_source",
|
|
1286
|
+
"export_attempt_patch_from_buggy_source_git_diff",
|
|
1287
|
+
"run_selected_tag_eval_no_upload_submit_ranking",
|
|
1288
|
+
"reduce_to_compact_public_result",
|
|
1289
|
+
],
|
|
1290
|
+
"public_artifact_policy": {
|
|
1291
|
+
"allowed_public_relative_files": [
|
|
1292
|
+
"runner-flow-plan.public.json",
|
|
1293
|
+
"execution-gate.public.json",
|
|
1294
|
+
"first-run-handoff.public.json",
|
|
1295
|
+
"workflow-check.public.json",
|
|
1296
|
+
"run-specific-gate.public.json",
|
|
1297
|
+
"run-specific-gate.md",
|
|
1298
|
+
"benchmark_run.compact.json",
|
|
1299
|
+
],
|
|
1300
|
+
"raw_task_material_public": False,
|
|
1301
|
+
"patch_content_public": False,
|
|
1302
|
+
"raw_logs_public": False,
|
|
1303
|
+
"trajectories_public": False,
|
|
1304
|
+
"screenshots_public": False,
|
|
1305
|
+
"absolute_paths_public": False,
|
|
1306
|
+
"credential_values_public": False,
|
|
1307
|
+
},
|
|
1308
|
+
"credential_boundary": {
|
|
1309
|
+
"codex_auth_values_read_by_packet": False,
|
|
1310
|
+
"codex_home_synced": False,
|
|
1311
|
+
"shared_remote_host_receives_codex_auth": False,
|
|
1312
|
+
"host_codex_auth_local_only": True,
|
|
1313
|
+
},
|
|
1314
|
+
"stop_conditions": [
|
|
1315
|
+
"private_job_root_missing",
|
|
1316
|
+
"operator_real_run_trigger_missing",
|
|
1317
|
+
"selected_container_source_not_extracted",
|
|
1318
|
+
"private_git_baseline_missing_before_codex",
|
|
1319
|
+
"host_codex_not_ephemeral_or_not_from_buggy_source",
|
|
1320
|
+
"attempt_patch_missing_or_not_from_buggy_source_git_diff",
|
|
1321
|
+
"eval_attempts_upload_submit_or_public_ranking",
|
|
1322
|
+
"public_artifact_contains_raw_task_patch_log_trajectory_screenshot_auth_or_absolute_path",
|
|
1323
|
+
],
|
|
1324
|
+
"execution_boundary": {
|
|
1325
|
+
**workflow_check["execution_boundary"],
|
|
1326
|
+
"real_run_authorized": False,
|
|
1327
|
+
"operator_trigger_recorded": False,
|
|
1328
|
+
},
|
|
1329
|
+
"rendered_command_sources": {
|
|
1330
|
+
"source_extraction_gate": execution_gate["source_extraction_gate"]["commands"],
|
|
1331
|
+
"private_git_baseline_gate": execution_gate["private_git_baseline_gate"]["commands"],
|
|
1332
|
+
"host_codex_gate": execution_gate["host_codex_gate"]["command"],
|
|
1333
|
+
"patch_output_gate": execution_gate["patch_output_gate"],
|
|
1334
|
+
"eval_gate": execution_gate["eval_gate"],
|
|
1335
|
+
},
|
|
1336
|
+
}
|
|
1337
|
+
markdown = (
|
|
1338
|
+
"# AgentIssue-Bench lagent_239 Run-Specific Gate\n\n"
|
|
1339
|
+
"This packet is no-execute. It separates the gates that are already "
|
|
1340
|
+
"covered by public/compact no-run packets from the gates that still "
|
|
1341
|
+
"block a real no-upload run.\n\n"
|
|
1342
|
+
"## Blocking Gates\n\n"
|
|
1343
|
+
+ "\n".join(f"- {gate_id}" for gate_id in blocking_gate_ids)
|
|
1344
|
+
+ "\n\n## Public Boundary\n\n"
|
|
1345
|
+
"- Codex auth stays on the host; no Codex home or auth material is synced.\n"
|
|
1346
|
+
"- Public artifacts stay compact/public and relative-path only.\n"
|
|
1347
|
+
"- Raw task material, patch content, raw logs, trajectories, screenshots, "
|
|
1348
|
+
"credentials, and absolute private paths remain private.\n"
|
|
1349
|
+
)
|
|
1350
|
+
|
|
1351
|
+
benchmark_run = json.loads(json.dumps(workflow["benchmark_run"]))
|
|
1352
|
+
benchmark_run.update(
|
|
1353
|
+
{
|
|
1354
|
+
"job_name": "agentissue_lagent_239_codex_cli_runner_run_gate",
|
|
1355
|
+
"mode": AGENTISSUE_CODEX_CLI_RUNNER_RUN_GATE_MODE,
|
|
1356
|
+
"worker_mode": "trusted_host_codex_cli_no_execute_run_gate",
|
|
1357
|
+
"first_blocker": "run_gate_packet_only_real_run_not_authorized",
|
|
1358
|
+
"score_failure_attribution": "not_run_run_gate_only",
|
|
1359
|
+
"failure_attribution_labels": [
|
|
1360
|
+
"run_specific_gate_packet_only",
|
|
1361
|
+
"real_run_blocked_until_gate_items_satisfied",
|
|
1362
|
+
],
|
|
1363
|
+
"evidence_files": run_gate["public_artifact_policy"][
|
|
1364
|
+
"allowed_public_relative_files"
|
|
1365
|
+
],
|
|
1366
|
+
}
|
|
1367
|
+
)
|
|
1368
|
+
benchmark_run["validation"].update(
|
|
1369
|
+
{
|
|
1370
|
+
"run_specific_gate_materialized": True,
|
|
1371
|
+
"owner_agent_gate_items_declared": True,
|
|
1372
|
+
"blocking_gate_ids_declared": True,
|
|
1373
|
+
"ready_for_operator_review": True,
|
|
1374
|
+
"real_run_authorized": False,
|
|
1375
|
+
"private_job_root_required": True,
|
|
1376
|
+
"operator_trigger_required": True,
|
|
1377
|
+
"phase_order_declared": True,
|
|
1378
|
+
"credential_boundary_declared": True,
|
|
1379
|
+
"public_artifact_policy_declared": True,
|
|
1380
|
+
"stop_conditions_declared": True,
|
|
1381
|
+
"no_execute_packet": True,
|
|
1382
|
+
"no_real_source_extraction": True,
|
|
1383
|
+
"no_real_codex_execution": True,
|
|
1384
|
+
"no_docker_pull_or_start": True,
|
|
1385
|
+
"no_auth_sync_to_shared_host": True,
|
|
1386
|
+
}
|
|
1387
|
+
)
|
|
1388
|
+
for trial in benchmark_run.get("trials") or []:
|
|
1389
|
+
if isinstance(trial, dict):
|
|
1390
|
+
trial["exception_type"] = "run_gate_packet_only_no_real_case"
|
|
1391
|
+
|
|
1392
|
+
run_gate_path.write_text(
|
|
1393
|
+
json.dumps(run_gate, ensure_ascii=False, indent=2, sort_keys=True) + "\n",
|
|
1394
|
+
encoding="utf-8",
|
|
1395
|
+
)
|
|
1396
|
+
run_gate_markdown_path.write_text(markdown, encoding="utf-8")
|
|
1397
|
+
compact_run_path.write_text(
|
|
1398
|
+
json.dumps(benchmark_run, ensure_ascii=False, indent=2, sort_keys=True) + "\n",
|
|
1399
|
+
encoding="utf-8",
|
|
1400
|
+
)
|
|
1401
|
+
return {
|
|
1402
|
+
"schema_version": AGENTISSUE_CODEX_CLI_RUNNER_RUN_GATE_SCHEMA_VERSION,
|
|
1403
|
+
"benchmark_id": AGENTISSUE_BENCHMARK_ID,
|
|
1404
|
+
"selected_tag": tag,
|
|
1405
|
+
"selected_image": AGENTISSUE_DEFAULT_IMAGE,
|
|
1406
|
+
"ready_for_operator_review": True,
|
|
1407
|
+
"ready_for_real_run": False,
|
|
1408
|
+
"materialized": True,
|
|
1409
|
+
"path_recorded": False,
|
|
1410
|
+
"run_gate_root_path_recorded": False,
|
|
1411
|
+
"blocking_gate_ids": blocking_gate_ids,
|
|
1412
|
+
"workflow_check": {
|
|
1413
|
+
"schema_version": workflow["schema_version"],
|
|
1414
|
+
"ready": workflow["ready"],
|
|
1415
|
+
"workflow_check_relative_path": workflow["workflow_check_relative_path"],
|
|
1416
|
+
"path_recorded": False,
|
|
1417
|
+
},
|
|
1418
|
+
"created_relative_paths": [
|
|
1419
|
+
*workflow["created_relative_paths"],
|
|
1420
|
+
"run-specific-gate.public.json",
|
|
1421
|
+
"run-specific-gate.md",
|
|
1422
|
+
],
|
|
1423
|
+
"run_gate_relative_path": "run-specific-gate.public.json",
|
|
1424
|
+
"run_gate_markdown_relative_path": "run-specific-gate.md",
|
|
1425
|
+
"compact_run_relative_path": "benchmark_run.compact.json",
|
|
1426
|
+
"gate_checks": {
|
|
1427
|
+
"owner_agent_gate_items_declared": True,
|
|
1428
|
+
"blocking_gate_ids_declared": True,
|
|
1429
|
+
"credential_boundary_declared": True,
|
|
1430
|
+
"public_artifact_policy_declared": True,
|
|
1431
|
+
"stop_conditions_declared": True,
|
|
1432
|
+
"real_run_authorized": False,
|
|
1433
|
+
},
|
|
1434
|
+
"execution_boundary": run_gate["execution_boundary"],
|
|
1435
|
+
"benchmark_run": benchmark_run,
|
|
1436
|
+
"recommended_next_action": (
|
|
1437
|
+
"review run-specific gate packet before any later real no-upload "
|
|
1438
|
+
"AgentIssue-Bench lagent_239 Docker/Codex execution"
|
|
1439
|
+
),
|
|
1440
|
+
}
|
|
1441
|
+
|
|
1442
|
+
def materialize_agentissue_codex_cli_runner_target_handoff(
|
|
1443
|
+
target_handoff_root: str | Path,
|
|
1444
|
+
*,
|
|
1445
|
+
selected_tag: str = AGENTISSUE_DEFAULT_TAG,
|
|
1446
|
+
codex_binary: str = "codex",
|
|
1447
|
+
docker_binary: str = "docker",
|
|
1448
|
+
) -> dict[str, Any]:
|
|
1449
|
+
"""Create a no-execute target-runner handoff packet for AgentIssue lagent_239."""
|
|
1450
|
+
|
|
1451
|
+
tag = _agentissue_public_label(selected_tag)
|
|
1452
|
+
if tag != AGENTISSUE_DEFAULT_TAG:
|
|
1453
|
+
raise ValueError(
|
|
1454
|
+
"agentissue Codex runner target handoff currently only supports selected tag lagent_239"
|
|
1455
|
+
)
|
|
1456
|
+
root = Path(target_handoff_root).expanduser()
|
|
1457
|
+
run_gate = materialize_agentissue_codex_cli_runner_run_gate(
|
|
1458
|
+
root,
|
|
1459
|
+
selected_tag=tag,
|
|
1460
|
+
codex_binary=codex_binary,
|
|
1461
|
+
docker_binary=docker_binary,
|
|
1462
|
+
)
|
|
1463
|
+
run_gate_path = root / "run-specific-gate.public.json"
|
|
1464
|
+
target_handoff_path = root / "target-runner-handoff.public.json"
|
|
1465
|
+
target_handoff_markdown_path = root / "target-runner-handoff.md"
|
|
1466
|
+
compact_run_path = root / "benchmark_run.compact.json"
|
|
1467
|
+
|
|
1468
|
+
run_gate_public = json.loads(run_gate_path.read_text(encoding="utf-8"))
|
|
1469
|
+
command_sources = run_gate_public["rendered_command_sources"]
|
|
1470
|
+
gate_item_ids = [
|
|
1471
|
+
item["id"] for item in run_gate_public["owner_agent_gate_items"]
|
|
1472
|
+
]
|
|
1473
|
+
required_before_execution = [
|
|
1474
|
+
"private_job_root_selected",
|
|
1475
|
+
"operator_explicit_real_run_trigger",
|
|
1476
|
+
"selected_container_source_extracted",
|
|
1477
|
+
"private_git_baseline_created_before_codex",
|
|
1478
|
+
"host_codex_exec_ephemeral_from_buggy_source",
|
|
1479
|
+
"attempt_patch_reducer_configured",
|
|
1480
|
+
"selected_tag_eval_no_upload_submit_ranking",
|
|
1481
|
+
"compact_public_reducer_enabled",
|
|
1482
|
+
"host_codex_auth_local_only",
|
|
1483
|
+
]
|
|
1484
|
+
missing_from_gate = [
|
|
1485
|
+
gate_id for gate_id in required_before_execution if gate_id not in gate_item_ids
|
|
1486
|
+
]
|
|
1487
|
+
no_execute_boundary = {
|
|
1488
|
+
**run_gate_public["execution_boundary"],
|
|
1489
|
+
"target_thread_started": False,
|
|
1490
|
+
"target_runner_executed": False,
|
|
1491
|
+
"benchmark_execution_authorized_by_packet": False,
|
|
1492
|
+
}
|
|
1493
|
+
target_handoff = {
|
|
1494
|
+
"schema_version": AGENTISSUE_CODEX_CLI_RUNNER_TARGET_HANDOFF_SCHEMA_VERSION,
|
|
1495
|
+
"benchmark_id": AGENTISSUE_BENCHMARK_ID,
|
|
1496
|
+
"selected_tag": tag,
|
|
1497
|
+
"selected_image": AGENTISSUE_DEFAULT_IMAGE,
|
|
1498
|
+
"default_mode": "no_execute",
|
|
1499
|
+
"materialized": True,
|
|
1500
|
+
"path_recorded": False,
|
|
1501
|
+
"target_handoff_root_path_recorded": False,
|
|
1502
|
+
"handoff_target": "separate_benchmark_execution_thread",
|
|
1503
|
+
"meta_heartbeat_must_not_execute": True,
|
|
1504
|
+
"real_run_authorized_by_packet": False,
|
|
1505
|
+
"ready_for_real_run": False,
|
|
1506
|
+
"ready_for_separate_execution_thread_after_gate_satisfied": (
|
|
1507
|
+
not missing_from_gate
|
|
1508
|
+
),
|
|
1509
|
+
"source_packets": {
|
|
1510
|
+
"runner_plan": "runner-flow-plan.public.json",
|
|
1511
|
+
"execution_gate": "execution-gate.public.json",
|
|
1512
|
+
"first_run_handoff": "first-run-handoff.public.json",
|
|
1513
|
+
"workflow_check": "workflow-check.public.json",
|
|
1514
|
+
"run_gate": "run-specific-gate.public.json",
|
|
1515
|
+
},
|
|
1516
|
+
"target_runner_prerequisites": required_before_execution,
|
|
1517
|
+
"missing_from_run_gate": missing_from_gate,
|
|
1518
|
+
"execution_thread_checklist": [
|
|
1519
|
+
{
|
|
1520
|
+
"phase": "select_private_job_root",
|
|
1521
|
+
"required": True,
|
|
1522
|
+
"public_packet_only": False,
|
|
1523
|
+
"private_state_allowed_in_execution_thread": True,
|
|
1524
|
+
"meta_thread_must_not_run": True,
|
|
1525
|
+
},
|
|
1526
|
+
{
|
|
1527
|
+
"phase": "extract_selected_container_buggy_source",
|
|
1528
|
+
"required": True,
|
|
1529
|
+
"command_shape_source": "run-specific-gate.public.json:rendered_command_sources.source_extraction_gate",
|
|
1530
|
+
"commands": command_sources["source_extraction_gate"],
|
|
1531
|
+
},
|
|
1532
|
+
{
|
|
1533
|
+
"phase": "create_private_git_baseline",
|
|
1534
|
+
"required": True,
|
|
1535
|
+
"command_shape_source": "run-specific-gate.public.json:rendered_command_sources.private_git_baseline_gate",
|
|
1536
|
+
"commands": command_sources["private_git_baseline_gate"],
|
|
1537
|
+
},
|
|
1538
|
+
{
|
|
1539
|
+
"phase": "run_host_codex_exec_ephemeral_from_buggy_source",
|
|
1540
|
+
"required": True,
|
|
1541
|
+
"command_shape_source": "run-specific-gate.public.json:rendered_command_sources.host_codex_gate",
|
|
1542
|
+
"command": command_sources["host_codex_gate"],
|
|
1543
|
+
"auth_boundary": "host_local_only_no_auth_sync",
|
|
1544
|
+
},
|
|
1545
|
+
{
|
|
1546
|
+
"phase": "export_attempt_patch_from_buggy_source_git_diff",
|
|
1547
|
+
"required": True,
|
|
1548
|
+
"command_shape_source": "run-specific-gate.public.json:rendered_command_sources.patch_output_gate",
|
|
1549
|
+
"output_relative_path": AGENTISSUE_PATCH_RELATIVE_PATH,
|
|
1550
|
+
"patch_content_public": False,
|
|
1551
|
+
},
|
|
1552
|
+
{
|
|
1553
|
+
"phase": "run_selected_tag_eval_no_upload_submit_ranking",
|
|
1554
|
+
"required": True,
|
|
1555
|
+
"command_shape_source": "run-specific-gate.public.json:rendered_command_sources.eval_gate",
|
|
1556
|
+
"upload": False,
|
|
1557
|
+
"submit": False,
|
|
1558
|
+
"public_ranking_path": False,
|
|
1559
|
+
},
|
|
1560
|
+
{
|
|
1561
|
+
"phase": "reduce_to_compact_public_result",
|
|
1562
|
+
"required": True,
|
|
1563
|
+
"public_outputs": [
|
|
1564
|
+
"benchmark_run.compact.json",
|
|
1565
|
+
"target-runner-handoff.public.json",
|
|
1566
|
+
],
|
|
1567
|
+
"private_outputs_not_public": [
|
|
1568
|
+
AGENTISSUE_PATCH_RELATIVE_PATH,
|
|
1569
|
+
"raw logs",
|
|
1570
|
+
"task material",
|
|
1571
|
+
"model transcript",
|
|
1572
|
+
"screenshots",
|
|
1573
|
+
"credentials",
|
|
1574
|
+
],
|
|
1575
|
+
},
|
|
1576
|
+
],
|
|
1577
|
+
"public_output_contract": {
|
|
1578
|
+
"allowed_public_relative_files": [
|
|
1579
|
+
"target-runner-handoff.public.json",
|
|
1580
|
+
"target-runner-handoff.md",
|
|
1581
|
+
*run_gate_public["public_artifact_policy"][
|
|
1582
|
+
"allowed_public_relative_files"
|
|
1583
|
+
],
|
|
1584
|
+
],
|
|
1585
|
+
"raw_task_material_public": False,
|
|
1586
|
+
"patch_content_public": False,
|
|
1587
|
+
"raw_logs_public": False,
|
|
1588
|
+
"trajectories_public": False,
|
|
1589
|
+
"screenshots_public": False,
|
|
1590
|
+
"absolute_paths_public": False,
|
|
1591
|
+
"credential_values_public": False,
|
|
1592
|
+
},
|
|
1593
|
+
"credential_boundary": {
|
|
1594
|
+
"codex_auth_values_read_by_packet": False,
|
|
1595
|
+
"codex_home_synced": False,
|
|
1596
|
+
"shared_remote_host_receives_codex_auth": False,
|
|
1597
|
+
"host_codex_auth_local_only": True,
|
|
1598
|
+
},
|
|
1599
|
+
"execution_boundary": no_execute_boundary,
|
|
1600
|
+
"stop_conditions": [
|
|
1601
|
+
"do_not_execute_in_meta_heartbeat_thread",
|
|
1602
|
+
*run_gate_public["stop_conditions"],
|
|
1603
|
+
"public_handoff_contains_raw_task_patch_log_transcript_screenshot_auth_or_absolute_path",
|
|
1604
|
+
],
|
|
1605
|
+
}
|
|
1606
|
+
markdown = (
|
|
1607
|
+
"# AgentIssue-Bench lagent_239 Target-Runner Handoff\n\n"
|
|
1608
|
+
"This packet is no-execute. It is a compact public handoff for a "
|
|
1609
|
+
"separate benchmark execution thread, not permission for the meta "
|
|
1610
|
+
"heartbeat thread to run the benchmark.\n\n"
|
|
1611
|
+
"## Target\n\n"
|
|
1612
|
+
"- handoff target: separate benchmark execution thread\n"
|
|
1613
|
+
"- meta heartbeat must not execute Codex, Docker, model APIs, source "
|
|
1614
|
+
"extraction, patch generation, eval, upload, submit, or ranking paths\n"
|
|
1615
|
+
"- real_run_authorized_by_packet=false\n\n"
|
|
1616
|
+
"## Required Gates\n\n"
|
|
1617
|
+
+ "\n".join(f"- {gate_id}" for gate_id in required_before_execution)
|
|
1618
|
+
+ "\n\n## Public Outputs\n\n"
|
|
1619
|
+
"- benchmark_run.compact.json\n"
|
|
1620
|
+
"- run-specific-gate.public.json\n"
|
|
1621
|
+
"- target-runner-handoff.public.json\n"
|
|
1622
|
+
"- target-runner-handoff.md\n\n"
|
|
1623
|
+
"Private execution artifacts stay private and must be reduced before "
|
|
1624
|
+
"any public writeback.\n"
|
|
1625
|
+
)
|
|
1626
|
+
|
|
1627
|
+
benchmark_run = json.loads(json.dumps(run_gate["benchmark_run"]))
|
|
1628
|
+
benchmark_run.update(
|
|
1629
|
+
{
|
|
1630
|
+
"job_name": "agentissue_lagent_239_codex_cli_runner_target_handoff",
|
|
1631
|
+
"mode": AGENTISSUE_CODEX_CLI_RUNNER_TARGET_HANDOFF_MODE,
|
|
1632
|
+
"worker_mode": "trusted_host_codex_cli_no_execute_target_handoff",
|
|
1633
|
+
"first_blocker": "target_handoff_packet_only_no_meta_execution",
|
|
1634
|
+
"score_failure_attribution": "not_run_target_handoff_only",
|
|
1635
|
+
"failure_attribution_labels": [
|
|
1636
|
+
"target_runner_handoff_packet_only",
|
|
1637
|
+
"ready_for_separate_execution_thread_after_gate_satisfied",
|
|
1638
|
+
],
|
|
1639
|
+
"evidence_files": target_handoff["public_output_contract"][
|
|
1640
|
+
"allowed_public_relative_files"
|
|
1641
|
+
],
|
|
1642
|
+
}
|
|
1643
|
+
)
|
|
1644
|
+
benchmark_run["validation"].update(
|
|
1645
|
+
{
|
|
1646
|
+
"target_runner_handoff_materialized": True,
|
|
1647
|
+
"handoff_target_declared": True,
|
|
1648
|
+
"meta_heartbeat_must_not_execute": True,
|
|
1649
|
+
"target_runner_prerequisites_declared": True,
|
|
1650
|
+
"real_run_authorized_by_packet": False,
|
|
1651
|
+
"target_thread_started": False,
|
|
1652
|
+
"target_runner_executed": False,
|
|
1653
|
+
"no_upload_submit_or_public_ranking": True,
|
|
1654
|
+
"no_auth_sync_to_shared_host": True,
|
|
1655
|
+
"public_output_contract_declared": True,
|
|
1656
|
+
"ready_for_separate_execution_thread_after_gate_satisfied": (
|
|
1657
|
+
not missing_from_gate
|
|
1658
|
+
),
|
|
1659
|
+
}
|
|
1660
|
+
)
|
|
1661
|
+
for trial in benchmark_run.get("trials") or []:
|
|
1662
|
+
if isinstance(trial, dict):
|
|
1663
|
+
trial["exception_type"] = "target_handoff_packet_only_no_real_case"
|
|
1664
|
+
|
|
1665
|
+
target_handoff_path.write_text(
|
|
1666
|
+
json.dumps(target_handoff, ensure_ascii=False, indent=2, sort_keys=True) + "\n",
|
|
1667
|
+
encoding="utf-8",
|
|
1668
|
+
)
|
|
1669
|
+
target_handoff_markdown_path.write_text(markdown, encoding="utf-8")
|
|
1670
|
+
compact_run_path.write_text(
|
|
1671
|
+
json.dumps(benchmark_run, ensure_ascii=False, indent=2, sort_keys=True) + "\n",
|
|
1672
|
+
encoding="utf-8",
|
|
1673
|
+
)
|
|
1674
|
+
return {
|
|
1675
|
+
"schema_version": AGENTISSUE_CODEX_CLI_RUNNER_TARGET_HANDOFF_SCHEMA_VERSION,
|
|
1676
|
+
"benchmark_id": AGENTISSUE_BENCHMARK_ID,
|
|
1677
|
+
"selected_tag": tag,
|
|
1678
|
+
"selected_image": AGENTISSUE_DEFAULT_IMAGE,
|
|
1679
|
+
"handoff_target": "separate_benchmark_execution_thread",
|
|
1680
|
+
"ready_for_real_run": False,
|
|
1681
|
+
"ready_for_separate_execution_thread_after_gate_satisfied": (
|
|
1682
|
+
not missing_from_gate
|
|
1683
|
+
),
|
|
1684
|
+
"materialized": True,
|
|
1685
|
+
"path_recorded": False,
|
|
1686
|
+
"target_handoff_root_path_recorded": False,
|
|
1687
|
+
"real_run_authorized_by_packet": False,
|
|
1688
|
+
"run_gate": {
|
|
1689
|
+
"schema_version": run_gate["schema_version"],
|
|
1690
|
+
"ready_for_operator_review": run_gate["ready_for_operator_review"],
|
|
1691
|
+
"ready_for_real_run": run_gate["ready_for_real_run"],
|
|
1692
|
+
"run_gate_relative_path": run_gate["run_gate_relative_path"],
|
|
1693
|
+
"path_recorded": False,
|
|
1694
|
+
},
|
|
1695
|
+
"created_relative_paths": [
|
|
1696
|
+
*run_gate["created_relative_paths"],
|
|
1697
|
+
"target-runner-handoff.public.json",
|
|
1698
|
+
"target-runner-handoff.md",
|
|
1699
|
+
],
|
|
1700
|
+
"target_handoff_relative_path": "target-runner-handoff.public.json",
|
|
1701
|
+
"target_handoff_markdown_relative_path": "target-runner-handoff.md",
|
|
1702
|
+
"compact_run_relative_path": "benchmark_run.compact.json",
|
|
1703
|
+
"execution_boundary": target_handoff["execution_boundary"],
|
|
1704
|
+
"target_runner_prerequisites": required_before_execution,
|
|
1705
|
+
"benchmark_run": benchmark_run,
|
|
1706
|
+
"recommended_next_action": (
|
|
1707
|
+
"hand off target-runner packet to a separate benchmark execution "
|
|
1708
|
+
"thread; keep meta heartbeat no-execute/no-upload"
|
|
1709
|
+
),
|
|
1710
|
+
}
|
|
1711
|
+
|
|
1712
|
+
def _agentissue_private_runner_script_text(
|
|
1713
|
+
*,
|
|
1714
|
+
tag: str,
|
|
1715
|
+
image: str,
|
|
1716
|
+
codex_binary: str,
|
|
1717
|
+
docker_binary: str,
|
|
1718
|
+
) -> str:
|
|
1719
|
+
codex = shlex.quote(_agentissue_public_label(codex_binary, limit=80))
|
|
1720
|
+
docker = shlex.quote(_agentissue_public_label(docker_binary, limit=80))
|
|
1721
|
+
quoted_tag = shlex.quote(tag)
|
|
1722
|
+
quoted_image = shlex.quote(image)
|
|
1723
|
+
container_buggy_source = "/app/source_code_buggy"
|
|
1724
|
+
eval_apply = "/usr/local/bin/run_test_entrypoint.sh apply_patch /patches/attempt.patch"
|
|
1725
|
+
eval_test = "/usr/local/bin/run_test_entrypoint.sh test_patched"
|
|
1726
|
+
return f"""#!/usr/bin/env bash
|
|
1727
|
+
set -euo pipefail
|
|
1728
|
+
|
|
1729
|
+
TAG="${{TAG:-{quoted_tag}}}"
|
|
1730
|
+
IMAGE="${{IMAGE:-{quoted_image}}}"
|
|
1731
|
+
CODEX_BIN="${{CODEX_BIN:-{codex}}}"
|
|
1732
|
+
DOCKER_BIN="${{DOCKER_BIN:-{docker}}}"
|
|
1733
|
+
LOOPX_BIN="${{LOOPX_BIN:-loopx}}"
|
|
1734
|
+
GOAL_ID="${{GOAL_ID:-loopx-meta}}"
|
|
1735
|
+
ALLOW_DOCKER_PULL="${{ALLOW_DOCKER_PULL:-0}}"
|
|
1736
|
+
APPEND_HISTORY="${{APPEND_HISTORY:-0}}"
|
|
1737
|
+
PRECHECK_ONLY="${{PRECHECK_ONLY:-0}}"
|
|
1738
|
+
PATCH_APPLY_SH="${{PATCH_APPLY_SH:-{eval_apply}}}"
|
|
1739
|
+
PATCH_TEST_SH="${{PATCH_TEST_SH:-{eval_test}}}"
|
|
1740
|
+
CONTAINER_BUGGY_SOURCE="${{CONTAINER_BUGGY_SOURCE:-{container_buggy_source}}}"
|
|
1741
|
+
JOB_ROOT="${{JOB_ROOT:-$(cd "$(dirname "${{BASH_SOURCE[0]}}")" && pwd)}}"
|
|
1742
|
+
CONTEXT_DIR="$JOB_ROOT/context"
|
|
1743
|
+
PROMPT_PATH="${{PROMPT_PATH:-$CONTEXT_DIR/prompt.md}}"
|
|
1744
|
+
BUGGY_SOURCE="$JOB_ROOT/buggy-source"
|
|
1745
|
+
PATCH_DIR="$JOB_ROOT/Patches/lagent_239"
|
|
1746
|
+
PATCH_PATH="$PATCH_DIR/attempt.patch"
|
|
1747
|
+
LAST_MESSAGE="$JOB_ROOT/codex-last-message.txt"
|
|
1748
|
+
MARKER_DIR="$JOB_ROOT/result-markers"
|
|
1749
|
+
BENCHMARK_RUN_JSON="$JOB_ROOT/benchmark_run.compact.json"
|
|
1750
|
+
BENCHMARK_RESULT_JSON="$JOB_ROOT/benchmark_result.compact.json"
|
|
1751
|
+
TMP_CONTAINER=""
|
|
1752
|
+
|
|
1753
|
+
fail() {{
|
|
1754
|
+
printf 'agentissue-runner: %s\\n' "$*" >&2
|
|
1755
|
+
exit 1
|
|
1756
|
+
}}
|
|
1757
|
+
|
|
1758
|
+
cleanup() {{
|
|
1759
|
+
if [ -n "$TMP_CONTAINER" ]; then
|
|
1760
|
+
"$DOCKER_BIN" rm -f "$TMP_CONTAINER" >/dev/null 2>&1 || true
|
|
1761
|
+
fi
|
|
1762
|
+
}}
|
|
1763
|
+
trap cleanup EXIT
|
|
1764
|
+
|
|
1765
|
+
require_selected_lagent239() {{
|
|
1766
|
+
[ "$TAG" = "lagent_239" ] || fail "only lagent_239 is supported"
|
|
1767
|
+
[ "$IMAGE" = "{image}" ] || fail "only the selected lagent_239 image is supported"
|
|
1768
|
+
}}
|
|
1769
|
+
|
|
1770
|
+
prepare_private_job_root() {{
|
|
1771
|
+
require_selected_lagent239
|
|
1772
|
+
mkdir -p "$CONTEXT_DIR" "$PATCH_DIR" "$MARKER_DIR"
|
|
1773
|
+
[ -s "$PROMPT_PATH" ] || fail "missing private context/prompt.md"
|
|
1774
|
+
if grep -q "Synthetic AgentIssue-Bench lagent_239 Prompt Placeholder" "$PROMPT_PATH"; then
|
|
1775
|
+
fail "replace the synthetic prompt placeholder before running Codex"
|
|
1776
|
+
fi
|
|
1777
|
+
}}
|
|
1778
|
+
|
|
1779
|
+
precheck_private_runner_environment() {{
|
|
1780
|
+
require_selected_lagent239
|
|
1781
|
+
command -v "$CODEX_BIN" >/dev/null 2>&1 || fail "Codex binary is not on PATH"
|
|
1782
|
+
command -v "$DOCKER_BIN" >/dev/null 2>&1 || fail "Docker binary is not on PATH"
|
|
1783
|
+
if ! "$DOCKER_BIN" image inspect "$IMAGE" >/dev/null 2>&1; then
|
|
1784
|
+
[ "$ALLOW_DOCKER_PULL" = "1" ] || fail "selected image is missing; set ALLOW_DOCKER_PULL=1 to pull it"
|
|
1785
|
+
"$DOCKER_BIN" pull "$IMAGE"
|
|
1786
|
+
fi
|
|
1787
|
+
"$DOCKER_BIN" run --platform linux/amd64 --rm --entrypoint bash \\
|
|
1788
|
+
-e CONTAINER_BUGGY_SOURCE="$CONTAINER_BUGGY_SOURCE" \\
|
|
1789
|
+
"$IMAGE" -lc \\
|
|
1790
|
+
'[ -d "$CONTAINER_BUGGY_SOURCE" ] && grep -q "apply_patch)" /usr/local/bin/run_test_entrypoint.sh && grep -q "test_patched)" /usr/local/bin/run_test_entrypoint.sh'
|
|
1791
|
+
}}
|
|
1792
|
+
|
|
1793
|
+
extract_buggy_source_from_selected_container() {{
|
|
1794
|
+
if [ -d "$BUGGY_SOURCE/.git" ]; then
|
|
1795
|
+
return 0
|
|
1796
|
+
fi
|
|
1797
|
+
if [ -e "$BUGGY_SOURCE" ] && [ "$(find "$BUGGY_SOURCE" -mindepth 1 -maxdepth 1 ! -name .gitkeep | wc -l | tr -d ' ')" != "0" ]; then
|
|
1798
|
+
fail "buggy-source is non-empty but has no git baseline; move it aside or set up baseline first"
|
|
1799
|
+
fi
|
|
1800
|
+
mkdir -p "$BUGGY_SOURCE"
|
|
1801
|
+
rm -f "$BUGGY_SOURCE/.gitkeep"
|
|
1802
|
+
if ! "$DOCKER_BIN" image inspect "$IMAGE" >/dev/null 2>&1; then
|
|
1803
|
+
[ "$ALLOW_DOCKER_PULL" = "1" ] || fail "selected image is missing; set ALLOW_DOCKER_PULL=1 to pull it"
|
|
1804
|
+
"$DOCKER_BIN" pull "$IMAGE"
|
|
1805
|
+
fi
|
|
1806
|
+
TMP_CONTAINER="agentissue-lagent-239-extract-$$"
|
|
1807
|
+
"$DOCKER_BIN" create --name "$TMP_CONTAINER" "$IMAGE" >/dev/null
|
|
1808
|
+
"$DOCKER_BIN" cp "$TMP_CONTAINER:$CONTAINER_BUGGY_SOURCE/." "$BUGGY_SOURCE"
|
|
1809
|
+
"$DOCKER_BIN" rm "$TMP_CONTAINER" >/dev/null
|
|
1810
|
+
TMP_CONTAINER=""
|
|
1811
|
+
[ "$(find "$BUGGY_SOURCE" -mindepth 1 -maxdepth 1 | wc -l | tr -d ' ')" != "0" ] || fail "buggy source extraction produced no files"
|
|
1812
|
+
}}
|
|
1813
|
+
|
|
1814
|
+
initialize_git_baseline_in_buggy_source() {{
|
|
1815
|
+
git -C "$BUGGY_SOURCE" rev-parse --is-inside-work-tree >/dev/null 2>&1 && return 0
|
|
1816
|
+
git -C "$BUGGY_SOURCE" init
|
|
1817
|
+
git -C "$BUGGY_SOURCE" config user.email "loopx@example.invalid"
|
|
1818
|
+
git -C "$BUGGY_SOURCE" config user.name "LoopX"
|
|
1819
|
+
git -C "$BUGGY_SOURCE" add .
|
|
1820
|
+
git -C "$BUGGY_SOURCE" commit -m "agentissue-bench-buggy-source-baseline"
|
|
1821
|
+
}}
|
|
1822
|
+
|
|
1823
|
+
run_host_local_codex_cli_patch_worker() {{
|
|
1824
|
+
"$CODEX_BIN" exec \\
|
|
1825
|
+
--ephemeral \\
|
|
1826
|
+
--ignore-rules \\
|
|
1827
|
+
--sandbox workspace-write \\
|
|
1828
|
+
--cd "$BUGGY_SOURCE" \\
|
|
1829
|
+
--add-dir "$JOB_ROOT" \\
|
|
1830
|
+
--output-last-message "$LAST_MESSAGE" \\
|
|
1831
|
+
"$PROMPT_PATH"
|
|
1832
|
+
touch "$MARKER_DIR/host_codex_cli_invoked"
|
|
1833
|
+
}}
|
|
1834
|
+
|
|
1835
|
+
write_attempt_patch_from_buggy_source_git_diff() {{
|
|
1836
|
+
git -C "$BUGGY_SOURCE" diff --binary > "$PATCH_PATH"
|
|
1837
|
+
[ -s "$PATCH_PATH" ] || fail "Codex run produced an empty git diff"
|
|
1838
|
+
}}
|
|
1839
|
+
|
|
1840
|
+
evaluate_selected_tag_container() {{
|
|
1841
|
+
rm -f "$MARKER_DIR/patch_applied" "$MARKER_DIR/test_success"
|
|
1842
|
+
set +e
|
|
1843
|
+
"$DOCKER_BIN" run \\
|
|
1844
|
+
--platform linux/amd64 \\
|
|
1845
|
+
--rm \\
|
|
1846
|
+
--entrypoint bash \\
|
|
1847
|
+
-v "$PATCH_DIR:/patches:ro" \\
|
|
1848
|
+
-v "$MARKER_DIR:/markers" \\
|
|
1849
|
+
-e PATCH_APPLY_SH="$PATCH_APPLY_SH" \\
|
|
1850
|
+
-e PATCH_TEST_SH="$PATCH_TEST_SH" \\
|
|
1851
|
+
"$IMAGE" \\
|
|
1852
|
+
-lc 'set -euo pipefail; eval "$PATCH_APPLY_SH"; touch /markers/patch_applied; eval "$PATCH_TEST_SH"; touch /markers/test_success'
|
|
1853
|
+
local exit_code=$?
|
|
1854
|
+
set -e
|
|
1855
|
+
printf '%s\\n' "$exit_code" > "$MARKER_DIR/patched_exit_code"
|
|
1856
|
+
}}
|
|
1857
|
+
|
|
1858
|
+
write_compact_public_evidence() {{
|
|
1859
|
+
export TAG IMAGE BUGGY_SOURCE PATCH_PATH MARKER_DIR
|
|
1860
|
+
python3 - "$BENCHMARK_RUN_JSON" "$BENCHMARK_RESULT_JSON" <<'PY'
|
|
1861
|
+
import hashlib
|
|
1862
|
+
import json
|
|
1863
|
+
import os
|
|
1864
|
+
import subprocess
|
|
1865
|
+
import sys
|
|
1866
|
+
from pathlib import Path
|
|
1867
|
+
|
|
1868
|
+
run_path = Path(sys.argv[1])
|
|
1869
|
+
result_path = Path(sys.argv[2])
|
|
1870
|
+
tag = os.environ["TAG"]
|
|
1871
|
+
image = os.environ["IMAGE"]
|
|
1872
|
+
source = Path(os.environ["BUGGY_SOURCE"])
|
|
1873
|
+
patch = Path(os.environ["PATCH_PATH"])
|
|
1874
|
+
markers = Path(os.environ["MARKER_DIR"])
|
|
1875
|
+
patched_exit = int((markers / "patched_exit_code").read_text().strip())
|
|
1876
|
+
patch_bytes = patch.stat().st_size if patch.exists() else 0
|
|
1877
|
+
patch_sha = hashlib.sha256(patch.read_bytes()).hexdigest() if patch.exists() else "missing"
|
|
1878
|
+
name_result = subprocess.run(
|
|
1879
|
+
["git", "-C", str(source), "diff", "--name-only"],
|
|
1880
|
+
check=False,
|
|
1881
|
+
stdout=subprocess.PIPE,
|
|
1882
|
+
stderr=subprocess.DEVNULL,
|
|
1883
|
+
text=True,
|
|
1884
|
+
)
|
|
1885
|
+
changed_files = [line for line in name_result.stdout.splitlines() if line.strip()]
|
|
1886
|
+
hunk_count = 0
|
|
1887
|
+
if patch.exists():
|
|
1888
|
+
hunk_count = sum(1 for line in patch.read_text(errors="ignore").splitlines() if line.startswith("@@ "))
|
|
1889
|
+
patch_applied = (markers / "patch_applied").exists()
|
|
1890
|
+
test_success = (markers / "test_success").exists()
|
|
1891
|
+
resolved = patched_exit == 0 and test_success
|
|
1892
|
+
score = {{
|
|
1893
|
+
"kind": "agentissue_bench_single_tag_container_eval",
|
|
1894
|
+
"resolved": resolved,
|
|
1895
|
+
"value": 1 if resolved else 0,
|
|
1896
|
+
}}
|
|
1897
|
+
validation = {{
|
|
1898
|
+
"selected_image_only": image == "alfin06/agentissue-bench:lagent_239",
|
|
1899
|
+
"single_tag_only": tag == "lagent_239",
|
|
1900
|
+
"buggy_source_extracted": source.exists(),
|
|
1901
|
+
"fixed_source_not_extracted_to_host": True,
|
|
1902
|
+
"host_codex_cli_invoked": (markers / "host_codex_cli_invoked").exists(),
|
|
1903
|
+
"patch_exported_from_buggy_source_git_diff": patch.exists() and patch_bytes > 0,
|
|
1904
|
+
"patch_applied_in_container": patch_applied,
|
|
1905
|
+
"patched_eval_exit_zero": patched_exit == 0,
|
|
1906
|
+
"patched_eval_success_marker": test_success,
|
|
1907
|
+
"no_upload": True,
|
|
1908
|
+
"no_submit": True,
|
|
1909
|
+
"no_public_ranking_path": True,
|
|
1910
|
+
"raw_logs_public": False,
|
|
1911
|
+
"patch_content_public": False,
|
|
1912
|
+
"credential_values_recorded": False,
|
|
1913
|
+
"codex_auth_synced_to_container_or_remote": False,
|
|
1914
|
+
}}
|
|
1915
|
+
benchmark_run = {{
|
|
1916
|
+
"schema_version": "benchmark_run_v0",
|
|
1917
|
+
"source_runner": "loopx_agentissue_codex_cli_runner",
|
|
1918
|
+
"benchmark_id": "agentissue-bench",
|
|
1919
|
+
"selected_tag": tag,
|
|
1920
|
+
"selected_image": image,
|
|
1921
|
+
"real_run": True,
|
|
1922
|
+
"no_upload": True,
|
|
1923
|
+
"no_submit": True,
|
|
1924
|
+
"no_public_ranking_path": True,
|
|
1925
|
+
"patch_sha256": patch_sha,
|
|
1926
|
+
"patch_bytes": patch_bytes,
|
|
1927
|
+
"changed_file_count": len(changed_files),
|
|
1928
|
+
"hunk_count": hunk_count,
|
|
1929
|
+
"patched_exit_code": patched_exit,
|
|
1930
|
+
"official_task_score": score,
|
|
1931
|
+
"validation": validation,
|
|
1932
|
+
}}
|
|
1933
|
+
benchmark_result = {{
|
|
1934
|
+
"schema_version": "benchmark_result_v0",
|
|
1935
|
+
"benchmark_id": "agentissue-bench",
|
|
1936
|
+
"selected_tag": tag,
|
|
1937
|
+
"official_task_score": score,
|
|
1938
|
+
"no_upload": True,
|
|
1939
|
+
"no_submit": True,
|
|
1940
|
+
"no_public_ranking_path": True,
|
|
1941
|
+
"patch_sha256": patch_sha,
|
|
1942
|
+
"patch_bytes": patch_bytes,
|
|
1943
|
+
"changed_file_count": len(changed_files),
|
|
1944
|
+
}}
|
|
1945
|
+
run_path.write_text(json.dumps(benchmark_run, indent=2, sort_keys=True) + "\\n")
|
|
1946
|
+
result_path.write_text(json.dumps(benchmark_result, indent=2, sort_keys=True) + "\\n")
|
|
1947
|
+
PY
|
|
1948
|
+
}}
|
|
1949
|
+
|
|
1950
|
+
reduce_compact_public_evidence() {{
|
|
1951
|
+
local args=("$LOOPX_BIN" "benchmark" "agentissue-codex-runner-flow" "--goal-id" "$GOAL_ID" "--tag" "$TAG" "--real-result-root" "$JOB_ROOT")
|
|
1952
|
+
if [ "$APPEND_HISTORY" = "1" ]; then
|
|
1953
|
+
args+=("--delivery-batch-scale" "multi_surface" "--delivery-outcome" "{DeliveryOutcome.PRIMARY_GOAL_OUTCOME.value}" "--execute")
|
|
1954
|
+
fi
|
|
1955
|
+
"${{args[@]}}"
|
|
1956
|
+
}}
|
|
1957
|
+
|
|
1958
|
+
main() {{
|
|
1959
|
+
if [ "$PRECHECK_ONLY" = "1" ]; then
|
|
1960
|
+
precheck_private_runner_environment
|
|
1961
|
+
return 0
|
|
1962
|
+
fi
|
|
1963
|
+
prepare_private_job_root
|
|
1964
|
+
extract_buggy_source_from_selected_container
|
|
1965
|
+
initialize_git_baseline_in_buggy_source
|
|
1966
|
+
run_host_local_codex_cli_patch_worker
|
|
1967
|
+
write_attempt_patch_from_buggy_source_git_diff
|
|
1968
|
+
evaluate_selected_tag_container
|
|
1969
|
+
write_compact_public_evidence
|
|
1970
|
+
reduce_compact_public_evidence
|
|
1971
|
+
}}
|
|
1972
|
+
|
|
1973
|
+
main "$@"
|
|
1974
|
+
"""
|
|
1975
|
+
|
|
1976
|
+
|
|
1977
|
+
def materialize_agentissue_codex_cli_runner_private_script(
|
|
1978
|
+
script_root: str | Path,
|
|
1979
|
+
*,
|
|
1980
|
+
selected_tag: str = AGENTISSUE_DEFAULT_TAG,
|
|
1981
|
+
codex_binary: str = "codex",
|
|
1982
|
+
docker_binary: str = "docker",
|
|
1983
|
+
) -> dict[str, Any]:
|
|
1984
|
+
"""Create a private runner script plus public manifest without executing it."""
|
|
1985
|
+
|
|
1986
|
+
tag = _agentissue_public_label(selected_tag)
|
|
1987
|
+
if tag != AGENTISSUE_DEFAULT_TAG:
|
|
1988
|
+
raise ValueError(
|
|
1989
|
+
"agentissue Codex runner private script currently only supports selected tag lagent_239"
|
|
1990
|
+
)
|
|
1991
|
+
root = Path(script_root).expanduser()
|
|
1992
|
+
handoff = materialize_agentissue_codex_cli_runner_first_run_handoff(
|
|
1993
|
+
root,
|
|
1994
|
+
selected_tag=tag,
|
|
1995
|
+
codex_binary=codex_binary,
|
|
1996
|
+
docker_binary=docker_binary,
|
|
1997
|
+
)
|
|
1998
|
+
script_path = root / "run-lagent239.private.sh"
|
|
1999
|
+
manifest_path = root / "private-runner.public.json"
|
|
2000
|
+
compact_run_path = root / "benchmark_run.compact.json"
|
|
2001
|
+
phase_order = [
|
|
2002
|
+
"prepare_private_job_root",
|
|
2003
|
+
"extract_buggy_source_from_selected_container",
|
|
2004
|
+
"initialize_git_baseline_in_buggy_source",
|
|
2005
|
+
"run_host_local_codex_cli_patch_worker",
|
|
2006
|
+
"write_attempt_patch_from_buggy_source_git_diff",
|
|
2007
|
+
"evaluate_selected_tag_container",
|
|
2008
|
+
"write_compact_public_evidence",
|
|
2009
|
+
"reduce_compact_public_evidence",
|
|
2010
|
+
]
|
|
2011
|
+
script_text = _agentissue_private_runner_script_text(
|
|
2012
|
+
tag=tag,
|
|
2013
|
+
image=AGENTISSUE_DEFAULT_IMAGE,
|
|
2014
|
+
codex_binary=codex_binary,
|
|
2015
|
+
docker_binary=docker_binary,
|
|
2016
|
+
)
|
|
2017
|
+
script_path.write_text(script_text, encoding="utf-8")
|
|
2018
|
+
script_path.chmod(0o700)
|
|
2019
|
+
|
|
2020
|
+
manifest = {
|
|
2021
|
+
"schema_version": AGENTISSUE_CODEX_CLI_RUNNER_PRIVATE_SCRIPT_SCHEMA_VERSION,
|
|
2022
|
+
"benchmark_id": AGENTISSUE_BENCHMARK_ID,
|
|
2023
|
+
"selected_tag": tag,
|
|
2024
|
+
"selected_image": AGENTISSUE_DEFAULT_IMAGE,
|
|
2025
|
+
"path_recorded": False,
|
|
2026
|
+
"root_path_recorded": False,
|
|
2027
|
+
"private_script_relative_path": "run-lagent239.private.sh",
|
|
2028
|
+
"script_content_public": False,
|
|
2029
|
+
"default_generator_mode": "no_execute",
|
|
2030
|
+
"phase_order": phase_order,
|
|
2031
|
+
"relative_outputs": {
|
|
2032
|
+
"attempt_patch": AGENTISSUE_PATCH_RELATIVE_PATH,
|
|
2033
|
+
"benchmark_run": "benchmark_run.compact.json",
|
|
2034
|
+
"benchmark_result": "benchmark_result.compact.json",
|
|
2035
|
+
"real_result": "real-result.public.json",
|
|
2036
|
+
"private_runner_manifest": "private-runner.public.json",
|
|
2037
|
+
},
|
|
2038
|
+
"operator_inputs_required": [
|
|
2039
|
+
"private context/prompt.md with public issue/task context",
|
|
2040
|
+
"host-local Codex CLI auth already present on the trusted host",
|
|
2041
|
+
"selected lagent_239 image present or ALLOW_DOCKER_PULL=1",
|
|
2042
|
+
],
|
|
2043
|
+
"script_checks": {
|
|
2044
|
+
"strict_mode": True,
|
|
2045
|
+
"precheck_only_mode": True,
|
|
2046
|
+
"selected_tag_guard": True,
|
|
2047
|
+
"selected_image_guard": True,
|
|
2048
|
+
"observed_image_source_path_default": True,
|
|
2049
|
+
"gitkeep_placeholder_safe": True,
|
|
2050
|
+
"buggy_source_extraction_phase": True,
|
|
2051
|
+
"git_baseline_phase": True,
|
|
2052
|
+
"host_codex_phase": True,
|
|
2053
|
+
"patch_export_phase": True,
|
|
2054
|
+
"selected_container_eval_phase": True,
|
|
2055
|
+
"entrypoint_eval_commands": True,
|
|
2056
|
+
"compact_reducer_phase": True,
|
|
2057
|
+
"appends_history_only_when_append_history_is_one": True,
|
|
2058
|
+
},
|
|
2059
|
+
"generator_boundary": {
|
|
2060
|
+
"codex_cli_invoked": False,
|
|
2061
|
+
"model_api_invoked": False,
|
|
2062
|
+
"docker_image_pulled": False,
|
|
2063
|
+
"docker_container_started": False,
|
|
2064
|
+
"source_extracted": False,
|
|
2065
|
+
"patch_generated": False,
|
|
2066
|
+
"patch_evaluated": False,
|
|
2067
|
+
"upload": False,
|
|
2068
|
+
"submit": False,
|
|
2069
|
+
"public_ranking_path": False,
|
|
2070
|
+
"auth_material_synced": False,
|
|
2071
|
+
"credential_values_recorded": False,
|
|
2072
|
+
"raw_logs_public": False,
|
|
2073
|
+
"patch_content_public": False,
|
|
2074
|
+
"absolute_paths_public": False,
|
|
2075
|
+
},
|
|
2076
|
+
"later_script_boundary": {
|
|
2077
|
+
"will_invoke_host_codex_cli": True,
|
|
2078
|
+
"will_start_selected_container": True,
|
|
2079
|
+
"will_write_compact_files": True,
|
|
2080
|
+
"uses_entrypoint_eval_commands": True,
|
|
2081
|
+
"upload": False,
|
|
2082
|
+
"submit": False,
|
|
2083
|
+
"public_ranking_path": False,
|
|
2084
|
+
"auth_material_sync": False,
|
|
2085
|
+
"raw_logs_public": False,
|
|
2086
|
+
"patch_content_public": False,
|
|
2087
|
+
},
|
|
2088
|
+
}
|
|
2089
|
+
_agentissue_assert_compact_public_safe(manifest, label="private-runner.public.json")
|
|
2090
|
+
|
|
2091
|
+
benchmark_run = json.loads(json.dumps(handoff["benchmark_run"]))
|
|
2092
|
+
benchmark_run.update(
|
|
2093
|
+
{
|
|
2094
|
+
"job_name": "agentissue_lagent_239_codex_cli_runner_private_script",
|
|
2095
|
+
"mode": AGENTISSUE_CODEX_CLI_RUNNER_PRIVATE_SCRIPT_MODE,
|
|
2096
|
+
"worker_mode": "trusted_host_codex_cli_private_script_generator",
|
|
2097
|
+
"first_blocker": "private_runner_script_generated_not_executed",
|
|
2098
|
+
"score_failure_attribution": "not_run_private_runner_script_generator_only",
|
|
2099
|
+
"failure_attribution_labels": [
|
|
2100
|
+
"private_runner_script_generator_only",
|
|
2101
|
+
"ready_for_controlled_script_execution_or_real_codex_regression",
|
|
2102
|
+
],
|
|
2103
|
+
"evidence_files": [
|
|
2104
|
+
"private-runner.public.json",
|
|
2105
|
+
"benchmark_run.compact.json",
|
|
2106
|
+
"first-run-handoff.public.json",
|
|
2107
|
+
"execution-gate.public.json",
|
|
2108
|
+
],
|
|
2109
|
+
}
|
|
2110
|
+
)
|
|
2111
|
+
benchmark_run["validation"].update(
|
|
2112
|
+
{
|
|
2113
|
+
"private_runner_script_materialized": True,
|
|
2114
|
+
"private_runner_manifest_materialized": True,
|
|
2115
|
+
"script_executable_bit_set": True,
|
|
2116
|
+
"script_content_not_public": True,
|
|
2117
|
+
"script_path_relative_only": True,
|
|
2118
|
+
"phase_order_rendered": True,
|
|
2119
|
+
"script_renders_source_extraction": True,
|
|
2120
|
+
"script_renders_observed_image_source_path": True,
|
|
2121
|
+
"script_renders_precheck_only": True,
|
|
2122
|
+
"script_handles_gitkeep_placeholder": True,
|
|
2123
|
+
"script_renders_git_baseline": True,
|
|
2124
|
+
"script_renders_host_codex": True,
|
|
2125
|
+
"script_renders_patch_export": True,
|
|
2126
|
+
"script_renders_selected_tag_eval": True,
|
|
2127
|
+
"script_renders_entrypoint_eval_commands": True,
|
|
2128
|
+
"script_renders_compact_evidence": True,
|
|
2129
|
+
"script_renders_real_result_reducer": True,
|
|
2130
|
+
"no_generator_codex_execution": True,
|
|
2131
|
+
"no_generator_docker_execution": True,
|
|
2132
|
+
"no_generator_model_api_invoked": True,
|
|
2133
|
+
"no_generator_upload": True,
|
|
2134
|
+
"no_generator_submit": True,
|
|
2135
|
+
"no_generator_public_ranking_path": True,
|
|
2136
|
+
"no_auth_material_sync": True,
|
|
2137
|
+
"no_raw_logs_public": True,
|
|
2138
|
+
"no_patch_content_public": True,
|
|
2139
|
+
"no_absolute_paths_public": True,
|
|
2140
|
+
}
|
|
2141
|
+
)
|
|
2142
|
+
for trial in benchmark_run.get("trials") or []:
|
|
2143
|
+
if isinstance(trial, dict):
|
|
2144
|
+
trial["exception_type"] = "private_runner_script_generated_not_executed"
|
|
2145
|
+
|
|
2146
|
+
manifest_path.write_text(
|
|
2147
|
+
json.dumps(manifest, ensure_ascii=False, indent=2, sort_keys=True) + "\n",
|
|
2148
|
+
encoding="utf-8",
|
|
2149
|
+
)
|
|
2150
|
+
compact_run_path.write_text(
|
|
2151
|
+
json.dumps(benchmark_run, ensure_ascii=False, indent=2, sort_keys=True) + "\n",
|
|
2152
|
+
encoding="utf-8",
|
|
2153
|
+
)
|
|
2154
|
+
return {
|
|
2155
|
+
"schema_version": AGENTISSUE_CODEX_CLI_RUNNER_PRIVATE_SCRIPT_SCHEMA_VERSION,
|
|
2156
|
+
"benchmark_id": AGENTISSUE_BENCHMARK_ID,
|
|
2157
|
+
"selected_tag": tag,
|
|
2158
|
+
"selected_image": AGENTISSUE_DEFAULT_IMAGE,
|
|
2159
|
+
"ready": True,
|
|
2160
|
+
"materialized": True,
|
|
2161
|
+
"path_recorded": False,
|
|
2162
|
+
"script_root_path_recorded": False,
|
|
2163
|
+
"script_relative_path": "run-lagent239.private.sh",
|
|
2164
|
+
"manifest_relative_path": "private-runner.public.json",
|
|
2165
|
+
"compact_run_relative_path": "benchmark_run.compact.json",
|
|
2166
|
+
"benchmark_result_relative_path": "benchmark_result.compact.json",
|
|
2167
|
+
"real_result_relative_path": "real-result.public.json",
|
|
2168
|
+
"created_relative_paths": [
|
|
2169
|
+
*handoff["created_relative_paths"],
|
|
2170
|
+
"run-lagent239.private.sh",
|
|
2171
|
+
"private-runner.public.json",
|
|
2172
|
+
],
|
|
2173
|
+
"phase_order": phase_order,
|
|
2174
|
+
"script_checks": manifest["script_checks"],
|
|
2175
|
+
"execution_boundary": manifest["generator_boundary"],
|
|
2176
|
+
"later_script_boundary": manifest["later_script_boundary"],
|
|
2177
|
+
"benchmark_run": benchmark_run,
|
|
2178
|
+
"recommended_next_action": (
|
|
2179
|
+
"run the private script only from a trusted local operator context, "
|
|
2180
|
+
"or add a low-frequency real Codex CLI regression that executes it "
|
|
2181
|
+
"without syncing auth material, uploading, submitting, or claiming a public ranking"
|
|
2182
|
+
),
|
|
2183
|
+
}
|
|
2184
|
+
|
|
2185
|
+
|
|
2186
|
+
AGENTISSUE_REAL_RESULT_FORBIDDEN_KEYS = {
|
|
2187
|
+
"access_token",
|
|
2188
|
+
"api_key",
|
|
2189
|
+
"authorization",
|
|
2190
|
+
"codex_auth",
|
|
2191
|
+
"credential",
|
|
2192
|
+
"environment",
|
|
2193
|
+
"file_content",
|
|
2194
|
+
"fixed_diff",
|
|
2195
|
+
"gold_material",
|
|
2196
|
+
"local_path",
|
|
2197
|
+
"password",
|
|
2198
|
+
"patch_content",
|
|
2199
|
+
"problem_statement",
|
|
2200
|
+
"raw_artifact",
|
|
2201
|
+
"raw_comment",
|
|
2202
|
+
"raw_diff",
|
|
2203
|
+
"raw_issue_body",
|
|
2204
|
+
"raw_issue_title",
|
|
2205
|
+
"raw_log",
|
|
2206
|
+
"raw_output",
|
|
2207
|
+
"raw_patch",
|
|
2208
|
+
"screenshot",
|
|
2209
|
+
"session",
|
|
2210
|
+
"solution",
|
|
2211
|
+
"source_diff",
|
|
2212
|
+
"test_body",
|
|
2213
|
+
"test_patch",
|
|
2214
|
+
"trajectory",
|
|
2215
|
+
}
|
|
2216
|
+
AGENTISSUE_REAL_RESULT_FORBIDDEN_TEXT = (
|
|
2217
|
+
"/" + "Users/",
|
|
2218
|
+
"~/.codex",
|
|
2219
|
+
".codex/auth.json",
|
|
2220
|
+
"CODEX" + "_ACCESS_TOKEN",
|
|
2221
|
+
"OPENAI" + "_API_KEY",
|
|
2222
|
+
"ANTHROPIC" + "_API_KEY",
|
|
2223
|
+
"GOOGLE" + "_API_KEY",
|
|
2224
|
+
"raw_issue_body",
|
|
2225
|
+
"raw_patch",
|
|
2226
|
+
"trajectory.json",
|
|
2227
|
+
)
|
|
2228
|
+
|
|
2229
|
+
AGENTISSUE_REAL_RESULT_REQUIRED_PHASE_CHECKS = (
|
|
2230
|
+
"selected_image_only",
|
|
2231
|
+
"single_tag_only",
|
|
2232
|
+
"buggy_source_extracted",
|
|
2233
|
+
"fixed_source_not_extracted_to_host",
|
|
2234
|
+
"host_codex_cli_invoked",
|
|
2235
|
+
"patch_exported_from_buggy_source_git_diff",
|
|
2236
|
+
)
|
|
2237
|
+
AGENTISSUE_REAL_RESULT_RESULT_PHASE_CHECKS = (
|
|
2238
|
+
"patch_applied_in_container",
|
|
2239
|
+
)
|
|
2240
|
+
|
|
2241
|
+
|
|
2242
|
+
def _agentissue_key_paths(value: Any, *, prefix: str = "") -> list[str]:
|
|
2243
|
+
if isinstance(value, dict):
|
|
2244
|
+
paths: list[str] = []
|
|
2245
|
+
for key, child in value.items():
|
|
2246
|
+
path = f"{prefix}.{key}" if prefix else str(key)
|
|
2247
|
+
paths.append(path)
|
|
2248
|
+
paths.extend(_agentissue_key_paths(child, prefix=path))
|
|
2249
|
+
return paths
|
|
2250
|
+
if isinstance(value, list):
|
|
2251
|
+
paths: list[str] = []
|
|
2252
|
+
for index, child in enumerate(value):
|
|
2253
|
+
paths.extend(_agentissue_key_paths(child, prefix=f"{prefix}[{index}]"))
|
|
2254
|
+
return paths
|
|
2255
|
+
return []
|
|
2256
|
+
|
|
2257
|
+
|
|
2258
|
+
def _agentissue_leaf(path: str) -> str:
|
|
2259
|
+
segment = path.rsplit(".", 1)[-1]
|
|
2260
|
+
if "[" in segment:
|
|
2261
|
+
segment = segment.split("[", 1)[0]
|
|
2262
|
+
return segment.lower()
|
|
2263
|
+
|
|
2264
|
+
|
|
2265
|
+
def _agentissue_public_bool(value: Any) -> bool:
|
|
2266
|
+
return bool(value) if isinstance(value, bool) else False
|
|
2267
|
+
|
|
2268
|
+
|
|
2269
|
+
def _agentissue_public_number(value: Any, *, default: int | float = 0) -> int | float:
|
|
2270
|
+
if isinstance(value, bool):
|
|
2271
|
+
return default
|
|
2272
|
+
if isinstance(value, (int, float)):
|
|
2273
|
+
return value
|
|
2274
|
+
return default
|
|
2275
|
+
|
|
2276
|
+
|
|
2277
|
+
def _agentissue_assert_compact_public_safe(payload: dict[str, Any], *, label: str) -> None:
|
|
2278
|
+
key_hits = [
|
|
2279
|
+
path
|
|
2280
|
+
for path in _agentissue_key_paths(payload)
|
|
2281
|
+
if _agentissue_leaf(path) in AGENTISSUE_REAL_RESULT_FORBIDDEN_KEYS
|
|
2282
|
+
]
|
|
2283
|
+
if key_hits:
|
|
2284
|
+
raise ValueError(f"{label} contains forbidden compact key(s): {', '.join(key_hits[:4])}")
|
|
2285
|
+
rendered = json.dumps(payload, ensure_ascii=False, sort_keys=True)
|
|
2286
|
+
leaked = [marker for marker in AGENTISSUE_REAL_RESULT_FORBIDDEN_TEXT if marker in rendered]
|
|
2287
|
+
if leaked:
|
|
2288
|
+
raise ValueError(f"{label} contains forbidden private marker(s): {', '.join(leaked[:4])}")
|
|
2289
|
+
|
|
2290
|
+
|
|
2291
|
+
def _agentissue_compact_official_score(
|
|
2292
|
+
run: dict[str, Any],
|
|
2293
|
+
result: dict[str, Any],
|
|
2294
|
+
) -> dict[str, Any]:
|
|
2295
|
+
run_score = run.get("official_task_score") if isinstance(run.get("official_task_score"), dict) else {}
|
|
2296
|
+
result_score = (
|
|
2297
|
+
result.get("official_task_score")
|
|
2298
|
+
if isinstance(result.get("official_task_score"), dict)
|
|
2299
|
+
else {}
|
|
2300
|
+
)
|
|
2301
|
+
source = result_score or run_score
|
|
2302
|
+
kind = _agentissue_public_label(
|
|
2303
|
+
source.get("kind") or "agentissue_bench_single_tag_container_eval",
|
|
2304
|
+
limit=80,
|
|
2305
|
+
)
|
|
2306
|
+
value = _agentissue_public_number(source.get("value"), default=0)
|
|
2307
|
+
resolved = source.get("resolved")
|
|
2308
|
+
if not isinstance(resolved, bool):
|
|
2309
|
+
resolved = value == 1
|
|
2310
|
+
return {
|
|
2311
|
+
"kind": kind,
|
|
2312
|
+
"value": value,
|
|
2313
|
+
"passed": bool(resolved),
|
|
2314
|
+
}
|
|
2315
|
+
|
|
2316
|
+
|
|
2317
|
+
def _agentissue_required_phase_checks(validation: dict[str, Any]) -> dict[str, bool]:
|
|
2318
|
+
checks: dict[str, bool] = {}
|
|
2319
|
+
missing: list[str] = []
|
|
2320
|
+
for key in AGENTISSUE_REAL_RESULT_REQUIRED_PHASE_CHECKS:
|
|
2321
|
+
checks[key] = validation.get(key) is True
|
|
2322
|
+
if not checks[key]:
|
|
2323
|
+
missing.append(key)
|
|
2324
|
+
for key in AGENTISSUE_REAL_RESULT_RESULT_PHASE_CHECKS:
|
|
2325
|
+
if not isinstance(validation.get(key), bool):
|
|
2326
|
+
missing.append(key)
|
|
2327
|
+
checks[key] = validation.get(key) is True
|
|
2328
|
+
if missing:
|
|
2329
|
+
raise ValueError(
|
|
2330
|
+
"real-result compact inputs are missing required runner phase proof(s): "
|
|
2331
|
+
+ ", ".join(missing)
|
|
2332
|
+
)
|
|
2333
|
+
return checks
|
|
2334
|
+
|
|
2335
|
+
|
|
2336
|
+
def materialize_agentissue_codex_cli_runner_real_result(
|
|
2337
|
+
real_result_root: str | Path,
|
|
2338
|
+
*,
|
|
2339
|
+
selected_tag: str = AGENTISSUE_DEFAULT_TAG,
|
|
2340
|
+
) -> dict[str, Any]:
|
|
2341
|
+
"""Reduce an already-completed private AgentIssue run from compact files only."""
|
|
2342
|
+
|
|
2343
|
+
tag = _agentissue_public_label(selected_tag)
|
|
2344
|
+
if tag != AGENTISSUE_DEFAULT_TAG:
|
|
2345
|
+
raise ValueError(
|
|
2346
|
+
"agentissue Codex runner real-result reducer currently only supports selected tag lagent_239"
|
|
2347
|
+
)
|
|
2348
|
+
root = Path(real_result_root).expanduser()
|
|
2349
|
+
run_path = root / "benchmark_run.compact.json"
|
|
2350
|
+
result_path = root / "benchmark_result.compact.json"
|
|
2351
|
+
public_packet_path = root / "real-result.public.json"
|
|
2352
|
+
if not run_path.exists():
|
|
2353
|
+
raise ValueError("real-result root is missing benchmark_run.compact.json")
|
|
2354
|
+
if not result_path.exists():
|
|
2355
|
+
raise ValueError("real-result root is missing benchmark_result.compact.json")
|
|
2356
|
+
run_input = json.loads(run_path.read_text(encoding="utf-8"))
|
|
2357
|
+
result_input = json.loads(result_path.read_text(encoding="utf-8"))
|
|
2358
|
+
if not isinstance(run_input, dict) or run_input.get("schema_version") != "benchmark_run_v0":
|
|
2359
|
+
raise ValueError("benchmark_run.compact.json must contain benchmark_run_v0")
|
|
2360
|
+
if not isinstance(result_input, dict) or result_input.get("schema_version") != "benchmark_result_v0":
|
|
2361
|
+
raise ValueError("benchmark_result.compact.json must contain benchmark_result_v0")
|
|
2362
|
+
_agentissue_assert_compact_public_safe(run_input, label="benchmark_run.compact.json")
|
|
2363
|
+
_agentissue_assert_compact_public_safe(result_input, label="benchmark_result.compact.json")
|
|
2364
|
+
|
|
2365
|
+
selected = _agentissue_public_label(
|
|
2366
|
+
run_input.get("selected_tag")
|
|
2367
|
+
or run_input.get("task_selector_hash")
|
|
2368
|
+
or result_input.get("selected_tag")
|
|
2369
|
+
or tag
|
|
2370
|
+
)
|
|
2371
|
+
if selected != tag:
|
|
2372
|
+
raise ValueError(f"real-result selected tag mismatch: expected {tag}, got {selected}")
|
|
2373
|
+
|
|
2374
|
+
official_score = _agentissue_compact_official_score(run_input, result_input)
|
|
2375
|
+
resolved = bool(official_score.get("passed"))
|
|
2376
|
+
patch_sha = _agentissue_public_label(
|
|
2377
|
+
run_input.get("patch_sha256") or result_input.get("patch_sha256") or "missing",
|
|
2378
|
+
limit=120,
|
|
2379
|
+
)
|
|
2380
|
+
patch_bytes = int(_agentissue_public_number(run_input.get("patch_bytes"), default=0))
|
|
2381
|
+
changed_files = int(
|
|
2382
|
+
_agentissue_public_number(
|
|
2383
|
+
run_input.get("changed_file_count") or result_input.get("changed_file_count"),
|
|
2384
|
+
default=0,
|
|
2385
|
+
)
|
|
2386
|
+
)
|
|
2387
|
+
hunk_count = int(_agentissue_public_number(run_input.get("hunk_count"), default=0))
|
|
2388
|
+
patched_exit = int(_agentissue_public_number(run_input.get("patched_exit_code"), default=0))
|
|
2389
|
+
baseline_exit = int(_agentissue_public_number(run_input.get("baseline_exit_code"), default=0))
|
|
2390
|
+
|
|
2391
|
+
validation = run_input.get("validation") if isinstance(run_input.get("validation"), dict) else {}
|
|
2392
|
+
phase_checks = _agentissue_required_phase_checks(validation)
|
|
2393
|
+
patched_eval_exit_zero = (
|
|
2394
|
+
validation.get("patched_eval_exit_zero")
|
|
2395
|
+
if isinstance(validation.get("patched_eval_exit_zero"), bool)
|
|
2396
|
+
else patched_exit == 0
|
|
2397
|
+
)
|
|
2398
|
+
patched_eval_success_marker = (
|
|
2399
|
+
validation.get("patched_eval_success_marker")
|
|
2400
|
+
if isinstance(validation.get("patched_eval_success_marker"), bool)
|
|
2401
|
+
else resolved
|
|
2402
|
+
)
|
|
2403
|
+
patch_applied = phase_checks.get("patch_applied_in_container") is True
|
|
2404
|
+
failure_label = (
|
|
2405
|
+
"resolved_single_tag_eval"
|
|
2406
|
+
if resolved
|
|
2407
|
+
else (
|
|
2408
|
+
"unresolved_patch_apply_failed_compact_result"
|
|
2409
|
+
if not patch_applied
|
|
2410
|
+
else "unresolved_single_tag_eval_compact_result"
|
|
2411
|
+
)
|
|
2412
|
+
)
|
|
2413
|
+
no_upload = _agentissue_public_bool(run_input.get("no_upload")) or _agentissue_public_bool(
|
|
2414
|
+
validation.get("no_upload")
|
|
2415
|
+
)
|
|
2416
|
+
no_submit = _agentissue_public_bool(run_input.get("no_submit")) or _agentissue_public_bool(
|
|
2417
|
+
validation.get("no_submit")
|
|
2418
|
+
)
|
|
2419
|
+
no_public_ranking = _agentissue_public_bool(
|
|
2420
|
+
run_input.get("no_public_ranking_path")
|
|
2421
|
+
) or _agentissue_public_bool(validation.get("no_public_ranking_path"))
|
|
2422
|
+
if not (no_upload and no_submit and no_public_ranking):
|
|
2423
|
+
raise ValueError(
|
|
2424
|
+
"real-result compact inputs must prove no_upload, no_submit, and no_public_ranking_path"
|
|
2425
|
+
)
|
|
2426
|
+
if validation.get("codex_auth_synced_to_container_or_remote") is True:
|
|
2427
|
+
raise ValueError("real-result compact inputs report Codex auth sync")
|
|
2428
|
+
if validation.get("credential_values_recorded") is True:
|
|
2429
|
+
raise ValueError("real-result compact inputs report credential value recording")
|
|
2430
|
+
if validation.get("raw_logs_public") is True or validation.get("patch_content_public") is True:
|
|
2431
|
+
raise ValueError("real-result compact inputs report raw logs or patch content public")
|
|
2432
|
+
|
|
2433
|
+
result_packet = {
|
|
2434
|
+
"schema_version": AGENTISSUE_CODEX_CLI_RUNNER_REAL_RESULT_SCHEMA_VERSION,
|
|
2435
|
+
"benchmark_id": AGENTISSUE_BENCHMARK_ID,
|
|
2436
|
+
"selected_tag": tag,
|
|
2437
|
+
"selected_image": AGENTISSUE_DEFAULT_IMAGE,
|
|
2438
|
+
"path_recorded": False,
|
|
2439
|
+
"real_run_done": True,
|
|
2440
|
+
"real_runner_invoked_by_reducer": False,
|
|
2441
|
+
"real_codex_invoked_by_reducer": False,
|
|
2442
|
+
"real_docker_invoked_by_reducer": False,
|
|
2443
|
+
"input_files": {
|
|
2444
|
+
"benchmark_run": {
|
|
2445
|
+
"relative_path": "benchmark_run.compact.json",
|
|
2446
|
+
"schema_version": run_input.get("schema_version"),
|
|
2447
|
+
"read": True,
|
|
2448
|
+
},
|
|
2449
|
+
"benchmark_result": {
|
|
2450
|
+
"relative_path": "benchmark_result.compact.json",
|
|
2451
|
+
"schema_version": result_input.get("schema_version"),
|
|
2452
|
+
"read": True,
|
|
2453
|
+
},
|
|
2454
|
+
},
|
|
2455
|
+
"result_summary": {
|
|
2456
|
+
"official_task_score": official_score,
|
|
2457
|
+
"resolved": resolved,
|
|
2458
|
+
"patch_sha256": patch_sha,
|
|
2459
|
+
"patch_bytes": patch_bytes,
|
|
2460
|
+
"changed_file_count": changed_files,
|
|
2461
|
+
"hunk_count": hunk_count,
|
|
2462
|
+
"patched_exit_code": patched_exit,
|
|
2463
|
+
"baseline_exit_code": baseline_exit,
|
|
2464
|
+
},
|
|
2465
|
+
"phase_checks": {
|
|
2466
|
+
**phase_checks,
|
|
2467
|
+
"patched_eval_exit_zero": patched_eval_exit_zero,
|
|
2468
|
+
"patched_eval_success_marker": patched_eval_success_marker,
|
|
2469
|
+
},
|
|
2470
|
+
"boundary": {
|
|
2471
|
+
"no_upload": no_upload,
|
|
2472
|
+
"no_submit": no_submit,
|
|
2473
|
+
"no_public_ranking_path": no_public_ranking,
|
|
2474
|
+
"codex_auth_synced": False,
|
|
2475
|
+
"credential_values_recorded": False,
|
|
2476
|
+
"raw_logs_public": False,
|
|
2477
|
+
"patch_content_public": False,
|
|
2478
|
+
"absolute_paths_public": False,
|
|
2479
|
+
},
|
|
2480
|
+
"public_outputs": [
|
|
2481
|
+
"real-result.public.json",
|
|
2482
|
+
"benchmark_run.compact.json",
|
|
2483
|
+
"benchmark_result.compact.json",
|
|
2484
|
+
],
|
|
2485
|
+
}
|
|
2486
|
+
_agentissue_assert_compact_public_safe(result_packet, label="real-result.public.json")
|
|
2487
|
+
|
|
2488
|
+
benchmark_run = {
|
|
2489
|
+
"schema_version": "benchmark_run_v0",
|
|
2490
|
+
"source_runner": AGENTISSUE_CODEX_CLI_RUNNER_SOURCE_RUNNER,
|
|
2491
|
+
"benchmark_id": AGENTISSUE_BENCHMARK_ID,
|
|
2492
|
+
"job_name": "agentissue_lagent_239_codex_cli_runner_real_result_reducer",
|
|
2493
|
+
"mode": AGENTISSUE_CODEX_CLI_RUNNER_REAL_RESULT_MODE,
|
|
2494
|
+
"worker_mode": "trusted_host_codex_cli_real_result_reducer",
|
|
2495
|
+
"trace_publicness": "compact_public_no_issue_text_no_patch_no_logs",
|
|
2496
|
+
"score_failure_attribution": failure_label,
|
|
2497
|
+
"real_run": True,
|
|
2498
|
+
"submit_eligible": False,
|
|
2499
|
+
"leaderboard_evidence": False,
|
|
2500
|
+
"official_score_comparable_to_native_codex": False,
|
|
2501
|
+
"official_score_claim_allowed": False,
|
|
2502
|
+
"control_plane_score_applicable": True,
|
|
2503
|
+
"official_task_score": official_score,
|
|
2504
|
+
"progress": {
|
|
2505
|
+
"n_total_trials": 1,
|
|
2506
|
+
"n_completed_trials": 1,
|
|
2507
|
+
"n_errored_trials": 0 if resolved else 1,
|
|
2508
|
+
"n_running_trials": 0,
|
|
2509
|
+
"n_pending_trials": 0,
|
|
2510
|
+
"n_cancelled_trials": 0,
|
|
2511
|
+
"n_retries": 0,
|
|
2512
|
+
},
|
|
2513
|
+
"metrics": {
|
|
2514
|
+
"input_tokens": 0,
|
|
2515
|
+
"cache_tokens": 0,
|
|
2516
|
+
"output_tokens": 0,
|
|
2517
|
+
"cost_usd": 0,
|
|
2518
|
+
},
|
|
2519
|
+
"validation": {
|
|
2520
|
+
"real_result_reducer_materialized": True,
|
|
2521
|
+
"compact_run_read": True,
|
|
2522
|
+
"compact_result_read": True,
|
|
2523
|
+
"selected_tag_checked": True,
|
|
2524
|
+
**phase_checks,
|
|
2525
|
+
"patch_hash_recorded": bool(patch_sha and patch_sha != "missing"),
|
|
2526
|
+
"patched_eval_exit_zero": patched_eval_exit_zero,
|
|
2527
|
+
"patched_eval_success_marker": patched_eval_success_marker,
|
|
2528
|
+
"no_upload": no_upload,
|
|
2529
|
+
"no_submit": no_submit,
|
|
2530
|
+
"no_public_ranking_path": no_public_ranking,
|
|
2531
|
+
"no_raw_logs_public": True,
|
|
2532
|
+
"no_patch_content_public": True,
|
|
2533
|
+
"no_absolute_paths_public": True,
|
|
2534
|
+
"no_codex_auth_sync": True,
|
|
2535
|
+
"no_credential_values_recorded": True,
|
|
2536
|
+
"no_reducer_codex_execution": True,
|
|
2537
|
+
"no_reducer_docker_execution": True,
|
|
2538
|
+
},
|
|
2539
|
+
"trials": [
|
|
2540
|
+
{
|
|
2541
|
+
"task_id": tag,
|
|
2542
|
+
"trial_name": tag,
|
|
2543
|
+
"source": "selected_public_tag",
|
|
2544
|
+
"exception_type": "" if resolved else failure_label,
|
|
2545
|
+
"trajectory_present": False,
|
|
2546
|
+
"artifact_manifest_present": False,
|
|
2547
|
+
"trial_result_present": True,
|
|
2548
|
+
}
|
|
2549
|
+
],
|
|
2550
|
+
"failure_attribution_labels": [failure_label],
|
|
2551
|
+
"evidence_files": [
|
|
2552
|
+
"real-result.public.json",
|
|
2553
|
+
"benchmark_run.compact.json",
|
|
2554
|
+
"benchmark_result.compact.json",
|
|
2555
|
+
],
|
|
2556
|
+
"stop_conditions": [
|
|
2557
|
+
"raw_log_requested",
|
|
2558
|
+
"patch_content_requested",
|
|
2559
|
+
"absolute_private_path_publication_requested",
|
|
2560
|
+
"upload_submit_or_public_ranking_requested",
|
|
2561
|
+
"codex_auth_sync_requested",
|
|
2562
|
+
],
|
|
2563
|
+
"read_boundary": {
|
|
2564
|
+
"compact_only": True,
|
|
2565
|
+
"raw_artifacts_read": False,
|
|
2566
|
+
"task_text_read": False,
|
|
2567
|
+
"trajectory_read": False,
|
|
2568
|
+
"local_paths_recorded": False,
|
|
2569
|
+
"docker_invoked": False,
|
|
2570
|
+
"model_api_invoked": False,
|
|
2571
|
+
"upload_invoked": False,
|
|
2572
|
+
},
|
|
2573
|
+
}
|
|
2574
|
+
benchmark_result = {
|
|
2575
|
+
"schema_version": "benchmark_result_v0",
|
|
2576
|
+
"task_id": "agentissue_bench_lagent_239",
|
|
2577
|
+
"scenario_id": AGENTISSUE_CODEX_CLI_RUNNER_REAL_RESULT_MODE,
|
|
2578
|
+
"worker_mode": "trusted_host_codex_cli_real_result_reducer",
|
|
2579
|
+
"harness_identity": "loopx",
|
|
2580
|
+
"terminal_state": "resolved" if resolved else "evaluated_unresolved",
|
|
2581
|
+
"trace_publicness": "compact_public_no_issue_text_no_patch_no_logs",
|
|
2582
|
+
"official_task_score": official_score,
|
|
2583
|
+
"validation_pass_count": 14,
|
|
2584
|
+
"validation_fail_count": 0 if resolved else 1,
|
|
2585
|
+
"changed_file_count": changed_files,
|
|
2586
|
+
"forbidden_access_count": 0,
|
|
2587
|
+
"phase_checks": {
|
|
2588
|
+
**phase_checks,
|
|
2589
|
+
"patched_eval_exit_zero": patched_eval_exit_zero,
|
|
2590
|
+
"patched_eval_success_marker": patched_eval_success_marker,
|
|
2591
|
+
},
|
|
2592
|
+
"failure_attribution_labels": benchmark_run["failure_attribution_labels"],
|
|
2593
|
+
}
|
|
2594
|
+
public_packet_path.write_text(
|
|
2595
|
+
json.dumps(result_packet, ensure_ascii=False, indent=2, sort_keys=True) + "\n",
|
|
2596
|
+
encoding="utf-8",
|
|
2597
|
+
)
|
|
2598
|
+
return {
|
|
2599
|
+
"schema_version": AGENTISSUE_CODEX_CLI_RUNNER_REAL_RESULT_SCHEMA_VERSION,
|
|
2600
|
+
"benchmark_id": AGENTISSUE_BENCHMARK_ID,
|
|
2601
|
+
"selected_tag": tag,
|
|
2602
|
+
"selected_image": AGENTISSUE_DEFAULT_IMAGE,
|
|
2603
|
+
"ready": True,
|
|
2604
|
+
"materialized": True,
|
|
2605
|
+
"path_recorded": False,
|
|
2606
|
+
"result_root_path_recorded": False,
|
|
2607
|
+
"real_run_done": True,
|
|
2608
|
+
"result_relative_path": "real-result.public.json",
|
|
2609
|
+
"compact_run_relative_path": "benchmark_run.compact.json",
|
|
2610
|
+
"compact_result_relative_path": "benchmark_result.compact.json",
|
|
2611
|
+
"result_checks": {
|
|
2612
|
+
"compact_run_read": True,
|
|
2613
|
+
"compact_result_read": True,
|
|
2614
|
+
"selected_tag_checked": True,
|
|
2615
|
+
**phase_checks,
|
|
2616
|
+
"patched_eval_exit_zero": patched_eval_exit_zero,
|
|
2617
|
+
"patched_eval_success_marker": patched_eval_success_marker,
|
|
2618
|
+
"resolved": resolved,
|
|
2619
|
+
"no_upload": no_upload,
|
|
2620
|
+
"no_submit": no_submit,
|
|
2621
|
+
"no_public_ranking_path": no_public_ranking,
|
|
2622
|
+
"raw_logs_public": False,
|
|
2623
|
+
"patch_content_public": False,
|
|
2624
|
+
"absolute_paths_public": False,
|
|
2625
|
+
},
|
|
2626
|
+
"execution_boundary": {
|
|
2627
|
+
"codex_cli_invoked_by_reducer": False,
|
|
2628
|
+
"model_api_invoked_by_reducer": False,
|
|
2629
|
+
"docker_container_started_by_reducer": False,
|
|
2630
|
+
"source_extracted_by_reducer": False,
|
|
2631
|
+
"patch_generated_by_reducer": False,
|
|
2632
|
+
"patch_evaluated_by_reducer": False,
|
|
2633
|
+
"upload": False,
|
|
2634
|
+
"submit": False,
|
|
2635
|
+
"public_ranking_path": False,
|
|
2636
|
+
},
|
|
2637
|
+
"benchmark_run": benchmark_run,
|
|
2638
|
+
"benchmark_result": benchmark_result,
|
|
2639
|
+
"public_packet": result_packet,
|
|
2640
|
+
"recommended_next_action": (
|
|
2641
|
+
"use --real-result-root for future AgentIssue-Bench lagent_239 compact "
|
|
2642
|
+
"result reductions, then compare repeat runs or extend to the next selected tag"
|
|
2643
|
+
),
|
|
2644
|
+
}
|