loopx 0.4.8__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (811) hide show
  1. loopx/__init__.py +5 -0
  2. loopx/agent_onboarding.py +654 -0
  3. loopx/agent_registry.py +112 -0
  4. loopx/ark_managed_agent_host.py +59 -0
  5. loopx/authority.py +805 -0
  6. loopx/benchmark.py +2875 -0
  7. loopx/benchmark_adapters/__init__.py +1 -0
  8. loopx/benchmark_adapters/agentissue.py +2644 -0
  9. loopx/benchmark_adapters/agents_last_exam.py +3998 -0
  10. loopx/benchmark_adapters/edgebench.py +322 -0
  11. loopx/benchmark_adapters/skillsbench.py +5978 -0
  12. loopx/benchmark_adapters/skillsbench_acp_failure_policy.py +143 -0
  13. loopx/benchmark_adapters/skillsbench_acp_process.py +31 -0
  14. loopx/benchmark_adapters/skillsbench_acp_relay.py +4832 -0
  15. loopx/benchmark_adapters/skillsbench_batch.py +124 -0
  16. loopx/benchmark_adapters/skillsbench_bridge_guard.py +209 -0
  17. loopx/benchmark_adapters/skillsbench_bridge_summary.py +203 -0
  18. loopx/benchmark_adapters/skillsbench_codex_goal_recovery.py +271 -0
  19. loopx/benchmark_adapters/skillsbench_codex_goal_trace.py +81 -0
  20. loopx/benchmark_adapters/skillsbench_codex_runtime.py +339 -0
  21. loopx/benchmark_adapters/skillsbench_dockerfile_runtime.py +467 -0
  22. loopx/benchmark_adapters/skillsbench_failure_signals.py +652 -0
  23. loopx/benchmark_adapters/skillsbench_proxy_runtime.py +327 -0
  24. loopx/benchmark_adapters/skillsbench_remote_bridge.py +402 -0
  25. loopx/benchmark_adapters/skillsbench_result_discovery.py +143 -0
  26. loopx/benchmark_adapters/skillsbench_runner_profile.py +436 -0
  27. loopx/benchmark_adapters/skillsbench_runner_source.py +99 -0
  28. loopx/benchmark_adapters/skillsbench_setup_preflight.py +771 -0
  29. loopx/benchmark_adapters/skillsbench_signals.py +15 -0
  30. loopx/benchmark_adapters/skillsbench_task_source.py +141 -0
  31. loopx/benchmark_adapters/skillsbench_turn_route.py +723 -0
  32. loopx/benchmark_adapters/skillsbench_turn_runtime.py +1069 -0
  33. loopx/benchmark_adapters/skillsbench_typed_repair.py +689 -0
  34. loopx/benchmark_adapters/skillsbench_uv_cache.py +111 -0
  35. loopx/benchmark_adapters/skillsbench_verifier_bootstrap.py +227 -0
  36. loopx/benchmark_adapters/skillsbench_verifier_cache.py +138 -0
  37. loopx/benchmark_adapters/terminal_bench.py +10078 -0
  38. loopx/benchmark_case_analysis.py +1276 -0
  39. loopx/benchmark_case_state.py +1079 -0
  40. loopx/benchmark_core/__init__.py +239 -0
  41. loopx/benchmark_core/adapter.py +84 -0
  42. loopx/benchmark_core/artifacts.py +517 -0
  43. loopx/benchmark_core/attempts.py +199 -0
  44. loopx/benchmark_core/container_exec.py +216 -0
  45. loopx/benchmark_core/io.py +68 -0
  46. loopx/benchmark_core/lifecycle.py +211 -0
  47. loopx/benchmark_core/loop_protocol.py +689 -0
  48. loopx/benchmark_core/observable_handles.py +348 -0
  49. loopx/benchmark_core/parity.py +256 -0
  50. loopx/benchmark_core/remote_closeout.py +482 -0
  51. loopx/benchmark_core/rounds.py +215 -0
  52. loopx/benchmark_core/route_profile.py +509 -0
  53. loopx/benchmark_core/run_permissions.py +206 -0
  54. loopx/benchmark_core/split_control.py +925 -0
  55. loopx/benchmark_core/turn_fidelity.py +326 -0
  56. loopx/benchmark_ledger.py +3793 -0
  57. loopx/benchmark_ledger_countability.py +372 -0
  58. loopx/benchmark_ledger_current.py +724 -0
  59. loopx/benchmark_trajectory.py +405 -0
  60. loopx/benchmarks/__init__.py +1 -0
  61. loopx/benchmarks/qualification/__init__.py +1 -0
  62. loopx/benchmarks/qualification/release_outcome_baseline.py +360 -0
  63. loopx/benchmarks/read_models/__init__.py +1 -0
  64. loopx/benchmarks/read_models/benchmark_attempt_accounting.py +53 -0
  65. loopx/benchmarks/read_models/benchmark_comparison.py +414 -0
  66. loopx/benchmarks/read_models/benchmark_event_timeline.py +113 -0
  67. loopx/benchmarks/read_models/benchmark_experiment_report.py +475 -0
  68. loopx/benchmarks/read_models/benchmark_learning_ledger.py +137 -0
  69. loopx/benchmarks/read_models/benchmark_lifecycle_contracts.py +228 -0
  70. loopx/benchmarks/read_models/benchmark_projection.py +723 -0
  71. loopx/benchmarks/read_models/benchmark_result.py +146 -0
  72. loopx/benchmarks/read_models/benchmark_run_execution_contract.py +116 -0
  73. loopx/benchmarks/read_models/benchmark_run_failure.py +157 -0
  74. loopx/benchmarks/read_models/benchmark_run_metrics.py +213 -0
  75. loopx/benchmarks/read_models/benchmark_run_post_execution.py +635 -0
  76. loopx/benchmarks/read_models/benchmark_run_pre_execution.py +541 -0
  77. loopx/benchmarks/read_models/benchmark_status_compaction.py +1255 -0
  78. loopx/benchmarks/read_models/benchmark_status_runner.py +780 -0
  79. loopx/benchmarks/read_models/goal_start_control_score.py +857 -0
  80. loopx/benchmarks/read_models/skillsbench_post_run_debug.py +746 -0
  81. loopx/benchmarks/read_models/skillsbench_verifier_attribution.py +269 -0
  82. loopx/bootstrap.py +1116 -0
  83. loopx/bootstrap_command_pack.py +2167 -0
  84. loopx/boundary_authority.py +199 -0
  85. loopx/canary/__init__.py +1 -0
  86. loopx/canary/maintainability_ratchet.py +800 -0
  87. loopx/canary/planner.py +1984 -0
  88. loopx/canary/premerge.py +1130 -0
  89. loopx/canary/qualification_profiles.py +309 -0
  90. loopx/canary/quality_surface_catalog.py +838 -0
  91. loopx/canary/release_profiles.py +51 -0
  92. loopx/canary/runner.py +1107 -0
  93. loopx/canary/smoke_health.py +581 -0
  94. loopx/canary/smoke_profiles.py +212 -0
  95. loopx/capabilities/__init__.py +0 -0
  96. loopx/capabilities/agent_turn_recall/__init__.py +17 -0
  97. loopx/capabilities/agent_turn_recall/cli.py +369 -0
  98. loopx/capabilities/agent_turn_recall/core.py +296 -0
  99. loopx/capabilities/auto_research/__init__.py +16 -0
  100. loopx/capabilities/auto_research/bootstrap_contract.py +157 -0
  101. loopx/capabilities/auto_research/cli.py +1468 -0
  102. loopx/capabilities/auto_research/core.py +11 -0
  103. loopx/capabilities/auto_research/defaults.py +79 -0
  104. loopx/capabilities/auto_research/demo_e2e.py +1848 -0
  105. loopx/capabilities/auto_research/demo_supervisor.py +186 -0
  106. loopx/capabilities/auto_research/evidence_packet.py +767 -0
  107. loopx/capabilities/auto_research/human_view.py +794 -0
  108. loopx/capabilities/auto_research/kernel.py +191 -0
  109. loopx/capabilities/auto_research/knn_demo_workspace.py +322 -0
  110. loopx/capabilities/auto_research/live_evidence.py +248 -0
  111. loopx/capabilities/auto_research/preset.py +176 -0
  112. loopx/capabilities/auto_research/research_state.py +1085 -0
  113. loopx/capabilities/auto_research/role_profiles.py +394 -0
  114. loopx/capabilities/auto_research/rollout_append.py +97 -0
  115. loopx/capabilities/auto_research/terminal_result_contract.py +422 -0
  116. loopx/capabilities/auto_research/terminal_result_projection.py +171 -0
  117. loopx/capabilities/auto_research/terminal_result_query.py +233 -0
  118. loopx/capabilities/auto_research/terminal_results.py +349 -0
  119. loopx/capabilities/auto_research/user_contract.py +190 -0
  120. loopx/capabilities/auto_research/worker_loop.py +163 -0
  121. loopx/capabilities/auto_research/worker_runtime.py +777 -0
  122. loopx/capabilities/auto_research/worker_skill/SKILL.md +343 -0
  123. loopx/capabilities/benchmark_toolkit/__init__.py +19 -0
  124. loopx/capabilities/benchmark_toolkit/integrity.py +387 -0
  125. loopx/capabilities/catalog.py +1875 -0
  126. loopx/capabilities/change_quality/__init__.py +19 -0
  127. loopx/capabilities/change_quality/cli.py +171 -0
  128. loopx/capabilities/change_quality/context.py +156 -0
  129. loopx/capabilities/change_quality/oracles.py +269 -0
  130. loopx/capabilities/change_quality/policy.py +34 -0
  131. loopx/capabilities/change_quality/receipt.py +482 -0
  132. loopx/capabilities/change_quality/result.py +493 -0
  133. loopx/capabilities/change_quality/scope.py +171 -0
  134. loopx/capabilities/change_quality/shadow.py +680 -0
  135. loopx/capabilities/content_ops/__init__.py +0 -0
  136. loopx/capabilities/content_ops/cli.py +649 -0
  137. loopx/capabilities/content_ops/connector_packets.py +164 -0
  138. loopx/capabilities/content_ops/item_lifecycle.py +1000 -0
  139. loopx/capabilities/content_ops/layout.py +451 -0
  140. loopx/capabilities/content_ops/markdown.py +456 -0
  141. loopx/capabilities/content_ops/schemas.py +51 -0
  142. loopx/capabilities/content_ops/social_browser_x.py +107 -0
  143. loopx/capabilities/content_ops/surface.py +1956 -0
  144. loopx/capabilities/content_ops/templates/layout-catalog-v0.json +72 -0
  145. loopx/capabilities/context_providers/__init__.py +36 -0
  146. loopx/capabilities/context_providers/base.py +189 -0
  147. loopx/capabilities/context_providers/factory.py +32 -0
  148. loopx/capabilities/context_providers/openviking.py +702 -0
  149. loopx/capabilities/context_providers/service_ownership.py +185 -0
  150. loopx/capabilities/decision_context/__init__.py +129 -0
  151. loopx/capabilities/decision_context/architecture.py +83 -0
  152. loopx/capabilities/decision_context/assembler.py +849 -0
  153. loopx/capabilities/decision_context/catalog_entry.py +195 -0
  154. loopx/capabilities/decision_context/cli.py +310 -0
  155. loopx/capabilities/decision_context/cursor_commit.py +535 -0
  156. loopx/capabilities/decision_context/outcome_feedback.py +352 -0
  157. loopx/capabilities/decision_context/packets.py +654 -0
  158. loopx/capabilities/decision_context/private_state.py +189 -0
  159. loopx/capabilities/decision_context/profile.py +453 -0
  160. loopx/capabilities/decision_context/providers.py +228 -0
  161. loopx/capabilities/decision_context/review_settlement.py +136 -0
  162. loopx/capabilities/decision_context/runtime.py +273 -0
  163. loopx/capabilities/decision_context/sources.py +415 -0
  164. loopx/capabilities/explore/__init__.py +1 -0
  165. loopx/capabilities/explore/activation.py +198 -0
  166. loopx/capabilities/explore/adaptive_replay_planner.py +221 -0
  167. loopx/capabilities/explore/child_replay_runtime.py +463 -0
  168. loopx/capabilities/explore/composition_frontier.py +291 -0
  169. loopx/capabilities/explore/counterfactual_runtime.py +578 -0
  170. loopx/capabilities/explore/episode_runtime.py +647 -0
  171. loopx/capabilities/explore/harness_checkpoint.py +171 -0
  172. loopx/capabilities/explore/harness_gate.py +115 -0
  173. loopx/capabilities/explore/harness_runtime.py +1124 -0
  174. loopx/capabilities/explore/replay_metrics.py +206 -0
  175. loopx/capabilities/explore/replay_runtime.py +1271 -0
  176. loopx/capabilities/explore/resource_portfolio.py +173 -0
  177. loopx/capabilities/explore/result_log.py +974 -0
  178. loopx/capabilities/explore/router_state.py +432 -0
  179. loopx/capabilities/explore/source_history_reconcile.py +255 -0
  180. loopx/capabilities/explore/speculative_scheduler.py +498 -0
  181. loopx/capabilities/explore/todo_branch_plan.py +650 -0
  182. loopx/capabilities/explore/todo_evidence.py +141 -0
  183. loopx/capabilities/explore/trace_runtime.py +284 -0
  184. loopx/capabilities/explore/worker_branch_plan.py +1257 -0
  185. loopx/capabilities/integration_branch/__init__.py +13 -0
  186. loopx/capabilities/integration_branch/cli.py +148 -0
  187. loopx/capabilities/integration_branch/core.py +916 -0
  188. loopx/capabilities/issue_fix/__init__.py +19 -0
  189. loopx/capabilities/issue_fix/acceptance_loop.py +1050 -0
  190. loopx/capabilities/issue_fix/candidate_evidence.py +503 -0
  191. loopx/capabilities/issue_fix/candidate_preflight.py +676 -0
  192. loopx/capabilities/issue_fix/cli.py +1822 -0
  193. loopx/capabilities/issue_fix/cli_input.py +87 -0
  194. loopx/capabilities/issue_fix/content_ops_cli.py +148 -0
  195. loopx/capabilities/issue_fix/discovered_issue_promotion.py +947 -0
  196. loopx/capabilities/issue_fix/explore_projection.py +710 -0
  197. loopx/capabilities/issue_fix/feasibility.py +542 -0
  198. loopx/capabilities/issue_fix/github_public.py +661 -0
  199. loopx/capabilities/issue_fix/intake_surface.py +832 -0
  200. loopx/capabilities/issue_fix/metadata_preview.py +218 -0
  201. loopx/capabilities/issue_fix/metrics_projection.py +1340 -0
  202. loopx/capabilities/issue_fix/metrics_supplement.py +634 -0
  203. loopx/capabilities/issue_fix/metrics_supplement_cli.py +127 -0
  204. loopx/capabilities/issue_fix/outcome_projection.py +1235 -0
  205. loopx/capabilities/issue_fix/periodic_report.py +189 -0
  206. loopx/capabilities/issue_fix/pr_description.py +418 -0
  207. loopx/capabilities/issue_fix/pr_gate_reconcile.py +496 -0
  208. loopx/capabilities/issue_fix/pr_gate_reconcile_cli.py +464 -0
  209. loopx/capabilities/issue_fix/pr_lifecycle.py +1327 -0
  210. loopx/capabilities/issue_fix/pr_lifecycle_rollout.py +85 -0
  211. loopx/capabilities/issue_fix/pr_monitor_materialization.py +257 -0
  212. loopx/capabilities/issue_fix/pr_review_ack.py +439 -0
  213. loopx/capabilities/issue_fix/provider_hooks.py +24 -0
  214. loopx/capabilities/issue_fix/repository_commit_evidence.py +186 -0
  215. loopx/capabilities/issue_fix/repository_context.py +457 -0
  216. loopx/capabilities/issue_fix/repository_memory.py +459 -0
  217. loopx/capabilities/issue_fix/repository_memory_provider.py +1454 -0
  218. loopx/capabilities/issue_fix/repository_snapshot.py +454 -0
  219. loopx/capabilities/issue_fix/reviewer_cli.py +917 -0
  220. loopx/capabilities/issue_fix/reviewer_notification.py +882 -0
  221. loopx/capabilities/issue_fix/reviewer_notification_drain.py +942 -0
  222. loopx/capabilities/issue_fix/reviewer_recommendation.py +1057 -0
  223. loopx/capabilities/issue_fix/reviewer_request.py +1282 -0
  224. loopx/capabilities/issue_fix/reward_memory.py +879 -0
  225. loopx/capabilities/issue_fix/workflow_plan.py +1286 -0
  226. loopx/capabilities/material_lifecycle/__init__.py +161 -0
  227. loopx/capabilities/material_lifecycle/_validation.py +183 -0
  228. loopx/capabilities/material_lifecycle/apply.py +672 -0
  229. loopx/capabilities/material_lifecycle/architecture.py +122 -0
  230. loopx/capabilities/material_lifecycle/cli.py +161 -0
  231. loopx/capabilities/material_lifecycle/decision_planning.py +470 -0
  232. loopx/capabilities/material_lifecycle/explore_execution.py +306 -0
  233. loopx/capabilities/material_lifecycle/intake.py +869 -0
  234. loopx/capabilities/material_lifecycle/inventory.py +147 -0
  235. loopx/capabilities/material_lifecycle/lifecycle.py +98 -0
  236. loopx/capabilities/material_lifecycle/preparation.py +147 -0
  237. loopx/capabilities/material_lifecycle/project_skill.py +83 -0
  238. loopx/capabilities/material_lifecycle/ranking.py +267 -0
  239. loopx/capabilities/material_lifecycle/readable_projection.py +500 -0
  240. loopx/capabilities/material_lifecycle/rebuild.py +480 -0
  241. loopx/capabilities/material_lifecycle/settlement.py +238 -0
  242. loopx/capabilities/periodic_report/__init__.py +71 -0
  243. loopx/capabilities/periodic_report/adapters.py +939 -0
  244. loopx/capabilities/periodic_report/archive.py +422 -0
  245. loopx/capabilities/periodic_report/bindings.py +705 -0
  246. loopx/capabilities/periodic_report/cli.py +277 -0
  247. loopx/capabilities/periodic_report/core.py +691 -0
  248. loopx/capabilities/periodic_report/extension_envelope.py +66 -0
  249. loopx/capabilities/periodic_report/presets.py +103 -0
  250. loopx/capabilities/periodic_report/profile.py +235 -0
  251. loopx/capabilities/periodic_report/project_progress.py +179 -0
  252. loopx/capabilities/periodic_report/triggers.py +452 -0
  253. loopx/capabilities/pr_review_queue/__init__.py +17 -0
  254. loopx/capabilities/pr_review_queue/core.py +506 -0
  255. loopx/capabilities/pr_review_queue/review_contract.py +506 -0
  256. loopx/capabilities/registry.py +192 -0
  257. loopx/capabilities/reward_memory/__init__.py +75 -0
  258. loopx/capabilities/reward_memory/application.py +819 -0
  259. loopx/capabilities/reward_memory/architecture.py +572 -0
  260. loopx/capabilities/reward_memory/candidate_review.py +511 -0
  261. loopx/capabilities/reward_memory/cli.py +469 -0
  262. loopx/capabilities/reward_memory/dogfood.py +574 -0
  263. loopx/capabilities/reward_memory/evaluation.py +296 -0
  264. loopx/capabilities/reward_memory/evaluation_fixtures.py +362 -0
  265. loopx/capabilities/reward_memory/experiment.py +567 -0
  266. loopx/capabilities/reward_memory/health.py +222 -0
  267. loopx/capabilities/reward_memory/ingestion.py +519 -0
  268. loopx/capabilities/reward_memory/registry.py +600 -0
  269. loopx/capabilities/reward_memory/runtime_hooks.py +312 -0
  270. loopx/capabilities/reward_memory/scoped_feedback.py +173 -0
  271. loopx/capabilities/semantic_preference/__init__.py +12 -0
  272. loopx/capabilities/semantic_preference/cli.py +189 -0
  273. loopx/capabilities/semantic_preference/contract.py +592 -0
  274. loopx/capabilities/semantic_preference/reward_memory.py +62 -0
  275. loopx/capabilities/value_connectors/__init__.py +1 -0
  276. loopx/capabilities/value_connectors/cli.py +401 -0
  277. loopx/capabilities/value_connectors/finance_extension_migration.py +108 -0
  278. loopx/capabilities/value_connectors/install_check.py +147 -0
  279. loopx/capabilities/value_connectors/planner.py +733 -0
  280. loopx/capabilities/value_connectors/source_map.py +446 -0
  281. loopx/claude_goal_baseline.py +138 -0
  282. loopx/claude_goal_mode/__init__.py +23 -0
  283. loopx/claude_goal_mode/hooks/goal_policy.py +212 -0
  284. loopx/claude_goal_mode/hooks/goal_state.py +139 -0
  285. loopx/claude_goal_mode/mcp/loopx_mcp.py +167 -0
  286. loopx/claude_goal_mode/scripts/connect.py +103 -0
  287. loopx/claude_goal_mode/scripts/goalmode_cmd.py +241 -0
  288. loopx/claude_goal_mode/scripts/install.py +328 -0
  289. loopx/claude_goal_mode/statusline/goal_status.py +97 -0
  290. loopx/cli.py +836 -0
  291. loopx/cli_commands/__init__.py +334 -0
  292. loopx/cli_commands/_host_thread.py +13 -0
  293. loopx/cli_commands/agentissue_runner_flow.py +447 -0
  294. loopx/cli_commands/agents_last_exam.py +160 -0
  295. loopx/cli_commands/agents_last_exam_baked_input.py +302 -0
  296. loopx/cli_commands/agents_last_exam_host_codex.py +374 -0
  297. loopx/cli_commands/agents_last_exam_launch_dry_run.py +372 -0
  298. loopx/cli_commands/agents_last_exam_local_plan.py +322 -0
  299. loopx/cli_commands/agents_last_exam_runner_source.py +352 -0
  300. loopx/cli_commands/agents_last_exam_task_material.py +335 -0
  301. loopx/cli_commands/agents_last_exam_validation_gate.py +236 -0
  302. loopx/cli_commands/benchmark_boundary.py +499 -0
  303. loopx/cli_commands/benchmark_dispatch.py +161 -0
  304. loopx/cli_commands/benchmark_release_outcome.py +123 -0
  305. loopx/cli_commands/benchmark_review_lifecycle.py +1275 -0
  306. loopx/cli_commands/benchmark_run_ledger.py +763 -0
  307. loopx/cli_commands/benchmark_run_ledger_case_analysis.py +249 -0
  308. loopx/cli_commands/benchmark_run_ledger_classification.py +45 -0
  309. loopx/cli_commands/benchmark_run_ledger_maintenance.py +486 -0
  310. loopx/cli_commands/benchmark_run_ledger_maintenance_registration.py +342 -0
  311. loopx/cli_commands/benchmark_run_ledger_maintenance_rendering.py +233 -0
  312. loopx/cli_commands/benchmark_run_ledger_parity.py +92 -0
  313. loopx/cli_commands/bootstrap_connect.py +238 -0
  314. loopx/cli_commands/canary.py +707 -0
  315. loopx/cli_commands/canary_release_qualification.py +79 -0
  316. loopx/cli_commands/capability.py +96 -0
  317. loopx/cli_commands/doctor.py +43 -0
  318. loopx/cli_commands/dreaming.py +143 -0
  319. loopx/cli_commands/edgebench.py +205 -0
  320. loopx/cli_commands/evidence_log.py +275 -0
  321. loopx/cli_commands/explore.py +989 -0
  322. loopx/cli_commands/explore_planning_commands.py +157 -0
  323. loopx/cli_commands/extension.py +271 -0
  324. loopx/cli_commands/first_run_report.py +73 -0
  325. loopx/cli_commands/goal_channel.py +656 -0
  326. loopx/cli_commands/handoff_mode.py +158 -0
  327. loopx/cli_commands/history.py +622 -0
  328. loopx/cli_commands/host_mode_plan.py +113 -0
  329. loopx/cli_commands/lark_inbox.py +431 -0
  330. loopx/cli_commands/lark_kanban.py +629 -0
  331. loopx/cli_commands/ml_experiment.py +321 -0
  332. loopx/cli_commands/multi_agent.py +211 -0
  333. loopx/cli_commands/opencode2_goal_worker.py +217 -0
  334. loopx/cli_commands/pr_review.py +167 -0
  335. loopx/cli_commands/presentation.py +218 -0
  336. loopx/cli_commands/preset.py +96 -0
  337. loopx/cli_commands/project.py +150 -0
  338. loopx/cli_commands/project_lifecycle.py +915 -0
  339. loopx/cli_commands/quota.py +859 -0
  340. loopx/cli_commands/quota_registration.py +241 -0
  341. loopx/cli_commands/quota_request.py +113 -0
  342. loopx/cli_commands/ready_score.py +110 -0
  343. loopx/cli_commands/registry_admin.py +975 -0
  344. loopx/cli_commands/registry_admin_configure.py +344 -0
  345. loopx/cli_commands/registry_admin_peer.py +84 -0
  346. loopx/cli_commands/registry_authority.py +218 -0
  347. loopx/cli_commands/review_batch.py +146 -0
  348. loopx/cli_commands/slash_commands.py +145 -0
  349. loopx/cli_commands/start_goal.py +251 -0
  350. loopx/cli_commands/starter.py +175 -0
  351. loopx/cli_commands/starter_bootstrap.py +179 -0
  352. loopx/cli_commands/starter_bootstrap_registration.py +198 -0
  353. loopx/cli_commands/starter_runtime_idle.py +107 -0
  354. loopx/cli_commands/starter_scheduler.py +207 -0
  355. loopx/cli_commands/starter_session_runtime.py +152 -0
  356. loopx/cli_commands/starter_visible_common.py +54 -0
  357. loopx/cli_commands/starter_visible_driver.py +161 -0
  358. loopx/cli_commands/starter_visible_pilot.py +278 -0
  359. loopx/cli_commands/status.py +867 -0
  360. loopx/cli_commands/status_registration.py +239 -0
  361. loopx/cli_commands/summary_all.py +222 -0
  362. loopx/cli_commands/support_control.py +809 -0
  363. loopx/cli_commands/support_control_registry.py +68 -0
  364. loopx/cli_commands/support_control_supervisor.py +289 -0
  365. loopx/cli_commands/task_lease.py +306 -0
  366. loopx/cli_commands/terminal_bench_adapter.py +717 -0
  367. loopx/cli_commands/terminal_bench_environment_result.py +1246 -0
  368. loopx/cli_commands/todo.py +940 -0
  369. loopx/cli_commands/todo_argument_validation.py +572 -0
  370. loopx/cli_commands/todo_event.py +114 -0
  371. loopx/cli_commands/turn.py +804 -0
  372. loopx/cli_commands/version.py +46 -0
  373. loopx/cli_commands/worker_bridge.py +659 -0
  374. loopx/cli_rollout.py +314 -0
  375. loopx/codex_cli_goal_tui.py +672 -0
  376. loopx/codex_cli_probe.py +1530 -0
  377. loopx/codex_cli_probe_markdown.py +935 -0
  378. loopx/codex_cli_runtime_probe.py +733 -0
  379. loopx/codex_cli_scheduler.py +564 -0
  380. loopx/codex_goal_baseline.py +620 -0
  381. loopx/configuration_catalog.py +617 -0
  382. loopx/configure_goal.py +1375 -0
  383. loopx/contract.py +996 -0
  384. loopx/control_plane/__init__.py +71 -0
  385. loopx/control_plane/agents/__init__.py +1 -0
  386. loopx/control_plane/agents/agent_lane_recommendation.py +516 -0
  387. loopx/control_plane/agents/agent_scope.py +1578 -0
  388. loopx/control_plane/agents/agent_scope_frontier.py +60 -0
  389. loopx/control_plane/agents/capability_gate.py +531 -0
  390. loopx/control_plane/agents/identity.py +140 -0
  391. loopx/control_plane/agents/legacy_migration.py +169 -0
  392. loopx/control_plane/agents/management_projection.py +658 -0
  393. loopx/control_plane/agents/material_frontier.py +608 -0
  394. loopx/control_plane/agents/material_handoff.py +156 -0
  395. loopx/control_plane/agents/multi_agent/__init__.py +1 -0
  396. loopx/control_plane/agents/multi_agent/codex_executable.py +207 -0
  397. loopx/control_plane/agents/multi_agent/collective_round_ledger.py +387 -0
  398. loopx/control_plane/agents/multi_agent/contract.py +474 -0
  399. loopx/control_plane/agents/multi_agent/recipe.py +110 -0
  400. loopx/control_plane/agents/multi_agent/role_successor.py +297 -0
  401. loopx/control_plane/agents/multi_agent/runtime_scripts.py +426 -0
  402. loopx/control_plane/agents/multi_agent/visible_launch_policy.py +149 -0
  403. loopx/control_plane/agents/multi_agent/visible_wake_scheduler.py +392 -0
  404. loopx/control_plane/agents/profile.py +216 -0
  405. loopx/control_plane/agents/runtime_model.py +73 -0
  406. loopx/control_plane/agents/subagent_activity.py +164 -0
  407. loopx/control_plane/agents/supervisor.py +544 -0
  408. loopx/control_plane/agents/supervisor_events.py +462 -0
  409. loopx/control_plane/agents/supervisor_inject.py +204 -0
  410. loopx/control_plane/agents/work_mode.py +56 -0
  411. loopx/control_plane/agents/workspace_guard.py +364 -0
  412. loopx/control_plane/effect_program.py +644 -0
  413. loopx/control_plane/goals/__init__.py +1 -0
  414. loopx/control_plane/goals/active_state_event_projection.py +103 -0
  415. loopx/control_plane/goals/active_state_metadata.py +47 -0
  416. loopx/control_plane/goals/active_state_sections.py +58 -0
  417. loopx/control_plane/goals/configure_goal_service.py +354 -0
  418. loopx/control_plane/goals/contract_health.py +132 -0
  419. loopx/control_plane/goals/dreaming.py +152 -0
  420. loopx/control_plane/goals/global_registry_health.py +199 -0
  421. loopx/control_plane/goals/global_registry_shadow.py +33 -0
  422. loopx/control_plane/goals/goal_channel.py +34 -0
  423. loopx/control_plane/goals/goal_channel_projection.py +560 -0
  424. loopx/control_plane/goals/goal_frontier/__init__.py +1917 -0
  425. loopx/control_plane/goals/goal_frontier/ack_policy.py +149 -0
  426. loopx/control_plane/goals/goal_frontier/outcome_continuity.py +437 -0
  427. loopx/control_plane/goals/goal_frontier/replan_rules.py +210 -0
  428. loopx/control_plane/goals/goal_frontier/semantic_history.py +314 -0
  429. loopx/control_plane/goals/goal_frontier/terminal.py +180 -0
  430. loopx/control_plane/goals/goal_vision.py +443 -0
  431. loopx/control_plane/goals/goal_vision_policy.py +36 -0
  432. loopx/control_plane/goals/goal_vision_state.py +62 -0
  433. loopx/control_plane/goals/goal_vision_wait.py +290 -0
  434. loopx/control_plane/goals/path_resolution.py +20 -0
  435. loopx/control_plane/goals/start_contract.py +206 -0
  436. loopx/control_plane/goals/vision_checkpoint.py +92 -0
  437. loopx/control_plane/handoff/__init__.py +1 -0
  438. loopx/control_plane/handoff/cross_runtime_impl_review.py +311 -0
  439. loopx/control_plane/handoff/delivery_contract.py +161 -0
  440. loopx/control_plane/handoff/handoff_runs.py +71 -0
  441. loopx/control_plane/handoff/project_handoff.py +155 -0
  442. loopx/control_plane/handoff/review_batch.py +463 -0
  443. loopx/control_plane/handoff/review_packet_context.py +216 -0
  444. loopx/control_plane/heartbeat/agent.py +173 -0
  445. loopx/control_plane/heartbeat/budget.py +66 -0
  446. loopx/control_plane/heartbeat/builder.py +501 -0
  447. loopx/control_plane/heartbeat/host.py +64 -0
  448. loopx/control_plane/heartbeat/rules.py +68 -0
  449. loopx/control_plane/heartbeat/task_body.py +759 -0
  450. loopx/control_plane/heartbeat/visible_goal.py +86 -0
  451. loopx/control_plane/projects/__init__.py +1 -0
  452. loopx/control_plane/projects/contract.py +25 -0
  453. loopx/control_plane/projects/registry.py +663 -0
  454. loopx/control_plane/quota/__init__.py +1 -0
  455. loopx/control_plane/quota/cli_projection.py +704 -0
  456. loopx/control_plane/quota/decision_summary.py +431 -0
  457. loopx/control_plane/quota/effect_program.py +152 -0
  458. loopx/control_plane/quota/error_codes.py +19 -0
  459. loopx/control_plane/quota/goal_boundary.py +464 -0
  460. loopx/control_plane/quota/heartbeat_receipt.py +277 -0
  461. loopx/control_plane/quota/heartbeat_recommendation.py +718 -0
  462. loopx/control_plane/quota/host_poll_receipts.py +162 -0
  463. loopx/control_plane/quota/live_decision.py +142 -0
  464. loopx/control_plane/quota/monitor_poll.py +786 -0
  465. loopx/control_plane/quota/policy_constants.py +40 -0
  466. loopx/control_plane/quota/projection_repair.py +262 -0
  467. loopx/control_plane/quota/recent_runs.py +210 -0
  468. loopx/control_plane/quota/scheduler_ack.py +490 -0
  469. loopx/control_plane/quota/selected_todo_projection.py +139 -0
  470. loopx/control_plane/quota/settlement.py +437 -0
  471. loopx/control_plane/quota/settlement_cli.py +246 -0
  472. loopx/control_plane/quota/settlement_validation.py +64 -0
  473. loopx/control_plane/quota/settlement_workspace_causality.py +180 -0
  474. loopx/control_plane/quota/should_run.py +249 -0
  475. loopx/control_plane/quota/should_run_packet.py +1165 -0
  476. loopx/control_plane/quota/should_run_prepare.py +675 -0
  477. loopx/control_plane/quota/slot_accounting.py +1123 -0
  478. loopx/control_plane/quota/spend_sources.py +11 -0
  479. loopx/control_plane/quota/stall_repair.py +397 -0
  480. loopx/control_plane/quota/states.py +29 -0
  481. loopx/control_plane/quota/task_orchestration.py +448 -0
  482. loopx/control_plane/quota/task_orchestration_admission.py +497 -0
  483. loopx/control_plane/quota/turn_envelope.py +889 -0
  484. loopx/control_plane/quota/usage_summary.py +140 -0
  485. loopx/control_plane/reward_memory.py +43 -0
  486. loopx/control_plane/runtime/__init__.py +2 -0
  487. loopx/control_plane/runtime/active_user_assisted_pilot.py +275 -0
  488. loopx/control_plane/runtime/agent_scoped_evidence_log.py +435 -0
  489. loopx/control_plane/runtime/decision_freshness.py +203 -0
  490. loopx/control_plane/runtime/event_ledger.py +197 -0
  491. loopx/control_plane/runtime/event_store_migration_bridge.py +196 -0
  492. loopx/control_plane/runtime/goal_project_route.py +70 -0
  493. loopx/control_plane/runtime/local_state_write_correctness.py +242 -0
  494. loopx/control_plane/runtime/promotion_readiness.py +152 -0
  495. loopx/control_plane/runtime/public_safety.py +120 -0
  496. loopx/control_plane/runtime/run_artifacts.py +78 -0
  497. loopx/control_plane/runtime/run_compaction.py +397 -0
  498. loopx/control_plane/runtime/run_context_retention.py +241 -0
  499. loopx/control_plane/runtime/run_history.py +132 -0
  500. loopx/control_plane/runtime/run_index_duplicates.py +205 -0
  501. loopx/control_plane/runtime/run_index_rebuild.py +263 -0
  502. loopx/control_plane/runtime/run_ingest_health.py +336 -0
  503. loopx/control_plane/runtime/runtime_projection_route.py +624 -0
  504. loopx/control_plane/runtime/runtime_projection_writer.py +98 -0
  505. loopx/control_plane/runtime/session_runtime.py +339 -0
  506. loopx/control_plane/runtime/shared_runtime_material_projection.py +332 -0
  507. loopx/control_plane/runtime/shared_runtime_refresh_projection.py +183 -0
  508. loopx/control_plane/runtime/stale_latest_run.py +90 -0
  509. loopx/control_plane/runtime/status_classifications.py +49 -0
  510. loopx/control_plane/runtime/status_projection_cache.py +235 -0
  511. loopx/control_plane/runtime/stride_observation.py +144 -0
  512. loopx/control_plane/runtime/time.py +39 -0
  513. loopx/control_plane/runtime/trajectory_hygiene.py +149 -0
  514. loopx/control_plane/runtime/validation_command.py +69 -0
  515. loopx/control_plane/scheduler/__init__.py +1 -0
  516. loopx/control_plane/scheduler/ack.py +329 -0
  517. loopx/control_plane/scheduler/arbitration.py +188 -0
  518. loopx/control_plane/scheduler/automation_liveness.py +183 -0
  519. loopx/control_plane/scheduler/execution_context.py +555 -0
  520. loopx/control_plane/scheduler/external_evidence_observation.py +428 -0
  521. loopx/control_plane/scheduler/monitor_display.py +143 -0
  522. loopx/control_plane/scheduler/monitor_poll_policy.py +161 -0
  523. loopx/control_plane/scheduler/monitor_poll_writeback.py +351 -0
  524. loopx/control_plane/scheduler/monitor_target.py +64 -0
  525. loopx/control_plane/scheduler/monitor_todo.py +146 -0
  526. loopx/control_plane/scheduler/monitor_wait.py +237 -0
  527. loopx/control_plane/scheduler/scheduler_hint.py +1284 -0
  528. loopx/control_plane/scheduler/state.py +354 -0
  529. loopx/control_plane/scheduler/state_transition_rules.py +179 -0
  530. loopx/control_plane/scheduler/time.py +10 -0
  531. loopx/control_plane/settlement_driver.py +293 -0
  532. loopx/control_plane/status/__init__.py +6 -0
  533. loopx/control_plane/status/active_state_projection.py +105 -0
  534. loopx/control_plane/status/agent_lane_projection.py +375 -0
  535. loopx/control_plane/status/attention_projection.py +74 -0
  536. loopx/control_plane/status/autonomous_replan_projection.py +103 -0
  537. loopx/control_plane/status/collection.py +140 -0
  538. loopx/control_plane/status/contract_projection.py +31 -0
  539. loopx/control_plane/status/dreaming_projection.py +52 -0
  540. loopx/control_plane/status/goal_attention_projection.py +157 -0
  541. loopx/control_plane/status/lifecycle_projection.py +110 -0
  542. loopx/control_plane/status/monitor_display_projection.py +69 -0
  543. loopx/control_plane/status/registry_health_projection.py +75 -0
  544. loopx/control_plane/status/run_projection.py +70 -0
  545. loopx/control_plane/status/runtime_summaries.py +161 -0
  546. loopx/control_plane/testing/__init__.py +1 -0
  547. loopx/control_plane/testing/actual_default_model_behavior_portfolio.py +1371 -0
  548. loopx/control_plane/testing/canary_harness.py +182 -0
  549. loopx/control_plane/testing/capability_monitor_repair_tool_behavior.py +674 -0
  550. loopx/control_plane/testing/cli_output_budget.py +807 -0
  551. loopx/control_plane/testing/cli_output_differential.py +250 -0
  552. loopx/control_plane/testing/cli_output_semantics.py +87 -0
  553. loopx/control_plane/testing/control_plane_composition_scenarios.py +225 -0
  554. loopx/control_plane/testing/decision_replay.py +268 -0
  555. loopx/control_plane/testing/doubao_model_behavior_actor.py +559 -0
  556. loopx/control_plane/testing/model_behavior_corpus.py +344 -0
  557. loopx/control_plane/testing/model_behavior_qualification.py +769 -0
  558. loopx/control_plane/testing/model_behavior_retained_cases.py +235 -0
  559. loopx/control_plane/testing/model_tool_behavior.py +536 -0
  560. loopx/control_plane/testing/onboarding_model_behavior_qualification.py +642 -0
  561. loopx/control_plane/testing/quota_fixtures.py +208 -0
  562. loopx/control_plane/testing/quota_should_run_parity.py +57 -0
  563. loopx/control_plane/testing/release_commit_qualification.py +671 -0
  564. loopx/control_plane/testing/replan_semantic_action_behavior.py +1302 -0
  565. loopx/control_plane/testing/scoped_gate_successor_tool_behavior.py +527 -0
  566. loopx/control_plane/testing/selected_todo_tool_behavior.py +1002 -0
  567. loopx/control_plane/testing/terminal_settlement_tool_behavior.py +656 -0
  568. loopx/control_plane/todos/__init__.py +1 -0
  569. loopx/control_plane/todos/active_state_editing.py +296 -0
  570. loopx/control_plane/todos/active_state_todo_parser.py +138 -0
  571. loopx/control_plane/todos/active_state_todos.py +175 -0
  572. loopx/control_plane/todos/addition.py +103 -0
  573. loopx/control_plane/todos/claim_visibility.py +253 -0
  574. loopx/control_plane/todos/completed_archive.py +139 -0
  575. loopx/control_plane/todos/completion_fence.py +49 -0
  576. loopx/control_plane/todos/completion_policy.py +153 -0
  577. loopx/control_plane/todos/completion_validation.py +248 -0
  578. loopx/control_plane/todos/completion_validation_accountability.py +27 -0
  579. loopx/control_plane/todos/completion_validation_projection.py +57 -0
  580. loopx/control_plane/todos/contract.py +1476 -0
  581. loopx/control_plane/todos/decision_scope.py +554 -0
  582. loopx/control_plane/todos/deferred_resume.py +546 -0
  583. loopx/control_plane/todos/durable_completion.py +201 -0
  584. loopx/control_plane/todos/event_writeback.py +484 -0
  585. loopx/control_plane/todos/frontier_deadline.py +132 -0
  586. loopx/control_plane/todos/handoff_gate.py +283 -0
  587. loopx/control_plane/todos/handoff_mode.py +444 -0
  588. loopx/control_plane/todos/handoff_note.py +202 -0
  589. loopx/control_plane/todos/line_update.py +361 -0
  590. loopx/control_plane/todos/list_projection.py +205 -0
  591. loopx/control_plane/todos/markdown.py +199 -0
  592. loopx/control_plane/todos/monitor_metadata.py +88 -0
  593. loopx/control_plane/todos/mutation_authority.py +299 -0
  594. loopx/control_plane/todos/projection.py +655 -0
  595. loopx/control_plane/todos/quota_summary.py +1138 -0
  596. loopx/control_plane/todos/route_continuation.py +267 -0
  597. loopx/control_plane/todos/succession_warning.py +174 -0
  598. loopx/control_plane/todos/summary_item.py +223 -0
  599. loopx/control_plane/todos/text.py +30 -0
  600. loopx/control_plane/todos/todo_index.py +226 -0
  601. loopx/control_plane/todos/todo_summary.py +1458 -0
  602. loopx/control_plane/todos/unblock_resume.py +326 -0
  603. loopx/control_plane/todos/user_gate.py +263 -0
  604. loopx/control_plane/todos/write_hint.py +63 -0
  605. loopx/control_plane/todos/write_policy.py +135 -0
  606. loopx/control_plane/turn_driver/__init__.py +85 -0
  607. loopx/control_plane/turn_driver/codex_cli.py +502 -0
  608. loopx/control_plane/turn_driver/driver.py +355 -0
  609. loopx/control_plane/turn_driver/executor.py +1468 -0
  610. loopx/control_plane/turn_driver/loop_controller.py +669 -0
  611. loopx/control_plane/turn_driver/settlement.py +318 -0
  612. loopx/control_plane/turn_driver/transaction.py +375 -0
  613. loopx/control_plane/work_items/__init__.py +1 -0
  614. loopx/control_plane/work_items/attention_fields.py +56 -0
  615. loopx/control_plane/work_items/attention_item.py +77 -0
  616. loopx/control_plane/work_items/attention_queue.py +322 -0
  617. loopx/control_plane/work_items/attention_routing.py +213 -0
  618. loopx/control_plane/work_items/autonomous_candidates.py +135 -0
  619. loopx/control_plane/work_items/autonomous_replan_ack.py +276 -0
  620. loopx/control_plane/work_items/autonomous_replan_obligation.py +786 -0
  621. loopx/control_plane/work_items/backlog_hygiene.py +59 -0
  622. loopx/control_plane/work_items/capability_monitor_fallback.py +221 -0
  623. loopx/control_plane/work_items/delivery_batch_scale.py +66 -0
  624. loopx/control_plane/work_items/delivery_outcome.py +152 -0
  625. loopx/control_plane/work_items/delivery_signals.py +113 -0
  626. loopx/control_plane/work_items/execution_obligation.py +235 -0
  627. loopx/control_plane/work_items/goal_route_hint.py +320 -0
  628. loopx/control_plane/work_items/interaction_contract.py +1540 -0
  629. loopx/control_plane/work_items/issue_meta_surface.py +159 -0
  630. loopx/control_plane/work_items/lifecycle.py +139 -0
  631. loopx/control_plane/work_items/operator_inbox.py +266 -0
  632. loopx/control_plane/work_items/outcome_followthrough.py +69 -0
  633. loopx/control_plane/work_items/primary_action.py +326 -0
  634. loopx/control_plane/work_items/progress_observation.py +630 -0
  635. loopx/control_plane/work_items/project_asset.py +675 -0
  636. loopx/control_plane/work_items/repair_delta.py +693 -0
  637. loopx/control_plane/work_items/runtime_capability_reentry.py +168 -0
  638. loopx/control_plane/work_items/semantic_replan_writeback.py +177 -0
  639. loopx/control_plane/work_items/status_contract.py +49 -0
  640. loopx/control_plane/work_items/task_graph.py +1046 -0
  641. loopx/control_plane/work_items/task_lease.py +1254 -0
  642. loopx/control_plane/work_items/task_lease_settlement.py +422 -0
  643. loopx/control_plane/work_items/work_lane.py +510 -0
  644. loopx/control_plane/work_items/work_lane_context.py +161 -0
  645. loopx/demo.py +247 -0
  646. loopx/diagnose.py +633 -0
  647. loopx/doctor.py +1251 -0
  648. loopx/domain_packs/__init__.py +1 -0
  649. loopx/domain_packs/issue_fix.py +571 -0
  650. loopx/domain_packs/ml_experiment.py +854 -0
  651. loopx/domain_state.py +137 -0
  652. loopx/dreaming.py +706 -0
  653. loopx/entrypoint.py +16 -0
  654. loopx/event_sourced_state.py +981 -0
  655. loopx/execution_profile.py +286 -0
  656. loopx/experiments/__init__.py +1 -0
  657. loopx/experiments/planner_worker/__init__.py +1 -0
  658. loopx/experiments/planner_worker/contract.py +523 -0
  659. loopx/experiments/planner_worker/runtime.py +391 -0
  660. loopx/experiments/planner_worker/traex.py +461 -0
  661. loopx/explore_graph.py +11 -0
  662. loopx/extensions/__init__.py +1 -0
  663. loopx/extensions/bundled.py +28 -0
  664. loopx/extensions/execution_envelope.py +126 -0
  665. loopx/extensions/lark/__init__.py +11 -0
  666. loopx/extensions/lark/event_collector.py +478 -0
  667. loopx/extensions/lark/event_collector_runtime.py +506 -0
  668. loopx/extensions/lark/event_inbox.py +454 -0
  669. loopx/extensions/lark/extension.toml +88 -0
  670. loopx/extensions/lark/goal_channel.py +44 -0
  671. loopx/extensions/lark/goal_channel_contracts.py +388 -0
  672. loopx/extensions/lark/goal_channel_lifecycle.py +218 -0
  673. loopx/extensions/lark/goal_channel_runtime.py +792 -0
  674. loopx/extensions/lark/goal_channel_setup.py +805 -0
  675. loopx/extensions/lark/goal_channel_targets.py +215 -0
  676. loopx/extensions/lark/goal_channel_transport.py +281 -0
  677. loopx/extensions/lark/inbox_reactions.py +650 -0
  678. loopx/extensions/lark/inbox_reply.py +430 -0
  679. loopx/extensions/lark/presentation/__init__.py +11 -0
  680. loopx/extensions/lark/presentation/explore_results.py +2276 -0
  681. loopx/extensions/lark/presentation/explore_singleflight.py +127 -0
  682. loopx/extensions/lark/presentation/explore_source_guard.py +121 -0
  683. loopx/extensions/lark/presentation/explore_stage_document.py +703 -0
  684. loopx/extensions/lark/presentation/explore_visual_integrity.py +122 -0
  685. loopx/extensions/lark/presentation/explore_visual_readback.py +452 -0
  686. loopx/extensions/lark/presentation/explore_visual_styles.py +156 -0
  687. loopx/extensions/lark/presentation/issue_fix_surface.py +612 -0
  688. loopx/extensions/lark/presentation/kanban.py +2791 -0
  689. loopx/extensions/lark/presentation/message_card.py +112 -0
  690. loopx/extensions/lark/presentation/periodic_report.py +261 -0
  691. loopx/extensions/lark/presentation/projection_rows.py +600 -0
  692. loopx/extensions/lark/presentation/record_io.py +95 -0
  693. loopx/extensions/lark/presentation/sync_receipt.py +145 -0
  694. loopx/extensions/lark/private_json.py +40 -0
  695. loopx/extensions/lark/provider.py +86 -0
  696. loopx/extensions/lark/reviewer_notification.py +604 -0
  697. loopx/extensions/manifest.py +385 -0
  698. loopx/extensions/openviking_periodic_report/__init__.py +17 -0
  699. loopx/extensions/openviking_periodic_report/activation.py +173 -0
  700. loopx/extensions/openviking_periodic_report/extension.toml +17 -0
  701. loopx/extensions/openviking_periodic_report/provider.py +355 -0
  702. loopx/extensions/openviking_periodic_report/sink.py +117 -0
  703. loopx/extensions/openviking_semantic_preference/__init__.py +5 -0
  704. loopx/extensions/openviking_semantic_preference/extension.toml +16 -0
  705. loopx/extensions/openviking_semantic_preference/history_export.py +484 -0
  706. loopx/extensions/openviking_semantic_preference/project_peer.py +68 -0
  707. loopx/extensions/openviking_semantic_preference/provider.py +312 -0
  708. loopx/extensions/presentation.py +979 -0
  709. loopx/extensions/process_runtime.py +204 -0
  710. loopx/extensions/readiness.py +168 -0
  711. loopx/extensions/runtime.py +931 -0
  712. loopx/extensions/scaffold.py +335 -0
  713. loopx/feedback.py +581 -0
  714. loopx/file_lock.py +382 -0
  715. loopx/global_registry.py +842 -0
  716. loopx/global_risks.py +970 -0
  717. loopx/global_todos.py +568 -0
  718. loopx/handoff_budget.py +28 -0
  719. loopx/heartbeat_prequota.py +80 -0
  720. loopx/heartbeat_prompt.py +159 -0
  721. loopx/help_surface.py +516 -0
  722. loopx/history.py +1507 -0
  723. loopx/host_loop_activation.py +1311 -0
  724. loopx/host_mode_planner.py +991 -0
  725. loopx/install_contract.py +1 -0
  726. loopx/interface_budget.py +196 -0
  727. loopx/long_task_cadence.py +208 -0
  728. loopx/materials.py +185 -0
  729. loopx/ml_experiment.py +3 -0
  730. loopx/onboarding.py +214 -0
  731. loopx/opencode2_goal_mode/README.md +81 -0
  732. loopx/opencode2_goal_mode/__init__.py +9 -0
  733. loopx/opencode2_goal_mode/opencode2-goal-worker.mjs +1018 -0
  734. loopx/opencode_goal_mode/README.md +99 -0
  735. loopx/opencode_goal_mode/__init__.py +13 -0
  736. loopx/opencode_goal_mode/goal-bridge-runtime.mjs +858 -0
  737. loopx/opencode_goal_mode/loopx-goal.js +8 -0
  738. loopx/operator_gate.py +420 -0
  739. loopx/orchestration.py +127 -0
  740. loopx/paths.py +59 -0
  741. loopx/pi_goal_mode/README.md +67 -0
  742. loopx/pi_goal_mode/__init__.py +13 -0
  743. loopx/pi_goal_mode/loopx-goal.ts +254 -0
  744. loopx/pi_goal_mode/pi-goal-loop-runtime.mjs +574 -0
  745. loopx/pr_review.py +1206 -0
  746. loopx/presentation/__init__.py +1 -0
  747. loopx/presentation/explore_views.py +1334 -0
  748. loopx/presentation/markdown.py +61 -0
  749. loopx/presentation/projection_source_reconcile.py +140 -0
  750. loopx/presentation/public_safety.py +42 -0
  751. loopx/presentation/renderers/__init__.py +17 -0
  752. loopx/presentation/renderers/goal_channel_html.py +269 -0
  753. loopx/presentation/renderers/periodic_report_html.py +786 -0
  754. loopx/presentation/renderers/periodic_report_markdown.py +184 -0
  755. loopx/presentation/renderers/quota_event_markdown.py +116 -0
  756. loopx/presentation/renderers/quota_markdown.py +1112 -0
  757. loopx/presentation/renderers/status_markdown.py +1570 -0
  758. loopx/presentation/renderers/trajectory_hygiene_markdown.py +39 -0
  759. loopx/presentation/renderers/turn_envelope_markdown.py +33 -0
  760. loopx/presentation/sinks/__init__.py +5 -0
  761. loopx/presentation/sinks/openviking_periodic_report.py +7 -0
  762. loopx/presentation/static_site.py +691 -0
  763. loopx/presets.py +369 -0
  764. loopx/project_alias.py +217 -0
  765. loopx/project_map.py +589 -0
  766. loopx/project_prompt.py +1153 -0
  767. loopx/project_skill_cli.py +125 -0
  768. loopx/project_skill_delivery.py +470 -0
  769. loopx/project_uninstall.py +462 -0
  770. loopx/promotion_gate.py +197 -0
  771. loopx/quota.py +1197 -0
  772. loopx/ready_score.py +413 -0
  773. loopx/registry.py +621 -0
  774. loopx/registry_writability.py +64 -0
  775. loopx/release_candidate.py +148 -0
  776. loopx/release_manifest.py +316 -0
  777. loopx/repository_identity.py +100 -0
  778. loopx/review_packet.py +1024 -0
  779. loopx/rollout_event_log.py +505 -0
  780. loopx/runtime.py +112 -0
  781. loopx/self_update.py +750 -0
  782. loopx/session_runtime.py +418 -0
  783. loopx/skill_install_readback.py +500 -0
  784. loopx/slash_command_install.py +1393 -0
  785. loopx/slash_commands.py +264 -0
  786. loopx/state_backup.py +573 -0
  787. loopx/state_migration.py +350 -0
  788. loopx/state_projection.py +809 -0
  789. loopx/state_refresh.py +1416 -0
  790. loopx/status.py +1383 -0
  791. loopx/status_server.py +935 -0
  792. loopx/summary_all.py +725 -0
  793. loopx/terminal_bench_agent.py +2056 -0
  794. loopx/thread_agent_binding.py +408 -0
  795. loopx/todo_followups.py +168 -0
  796. loopx/todo_suggestion_prompt.py +204 -0
  797. loopx/todos.py +2229 -0
  798. loopx/turn_identity.py +17 -0
  799. loopx/upgrade.py +1083 -0
  800. loopx/visible_governance.py +667 -0
  801. loopx/visible_multi_agent_launcher.py +1253 -0
  802. loopx/visible_multi_agent_tmux.py +429 -0
  803. loopx/worker_bridge.py +1574 -0
  804. loopx-0.4.8.dist-info/METADATA +708 -0
  805. loopx-0.4.8.dist-info/RECORD +811 -0
  806. loopx-0.4.8.dist-info/WHEEL +5 -0
  807. loopx-0.4.8.dist-info/entry_points.txt +5 -0
  808. loopx-0.4.8.dist-info/licenses/LICENSE +202 -0
  809. loopx-0.4.8.dist-info/licenses/LICENSE-MIT +21 -0
  810. loopx-0.4.8.dist-info/licenses/NOTICE +6 -0
  811. loopx-0.4.8.dist-info/top_level.txt +1 -0
@@ -0,0 +1,3793 @@
1
+ from __future__ import annotations
2
+
3
+ import hashlib
4
+ import json
5
+ import os
6
+ import time
7
+ from datetime import datetime
8
+ from pathlib import Path
9
+ from typing import Any
10
+
11
+ from .benchmark_core import (
12
+ LEGACY_NONPRODUCT_PROMPT_POLLING_ROUTES,
13
+ classify_benchmark_artifact_path,
14
+ classify_product_mode_main_table_pair,
15
+ )
16
+ from .benchmark_core.lifecycle import compact_benchmark_live_worker_phase_from_run
17
+ from .benchmark_adapters.skillsbench_signals import (
18
+ build_skillsbench_solution_quality_signals,
19
+ )
20
+ from .benchmark_ledger_countability import (
21
+ benchmark_run_official_score_countability,
22
+ official_score_bool_fallback_allowed,
23
+ official_score_bool_fallback_used,
24
+ )
25
+
26
+
27
+ BENCHMARK_RUN_LEDGER_SCHEMA_VERSION = "benchmark_run_ledger_v0"
28
+ BENCHMARK_RUN_LEDGER_CURRENT_AGGREGATE_SCHEMA_VERSION = (
29
+ "benchmark_run_ledger_current_aggregate_v0"
30
+ )
31
+ OPERATOR_SIMULATOR_RUN_SCHEMA_VERSION = "operator_simulator_run_v0"
32
+ BENCHMARK_RUN_LEDGER_DEFAULT_PATH = Path(
33
+ "docs/research/long-horizon-agent-benchmarks/benchmark-run-ledger.json"
34
+ )
35
+ DEFAULT_CODEX_GOAL_MODE_REPAIR_MODEL_ROUTE = "gpt-5.5"
36
+ DEFAULT_AGENT_TIMEOUT_REPAIR_MULTIPLIER = 8
37
+ DEFAULT_AGENT_SETUP_TIMEOUT_REPAIR_MULTIPLIER = 8
38
+ DEFAULT_CODEX_SETUP_TIMEOUT_REPAIR_INSTALL_STRATEGY = "require_existing_codex"
39
+ RUNTIME_CODEX_INSTALL_STRATEGY = "runtime_install_if_missing"
40
+ LEDGER_LOGICAL_BACKFILL_FIELDS = (
41
+ "artifact_refs",
42
+ "solution_quality_signals",
43
+ "round_reward_count",
44
+ "round_success_observed",
45
+ "max_rounds_budget",
46
+ "app_server_goal_round_semantics",
47
+ "native_goal_session_policy",
48
+ "max_rounds_budget_applies_to",
49
+ "native_goal_initial_turn_budget",
50
+ "native_goal_same_thread_followup_budget",
51
+ "native_goal_independent_attempt_budget",
52
+ "native_goal_fresh_thread_per_independent_attempt",
53
+ "native_goal_official_reward_feedback_forwarded_to_worker",
54
+ "native_goal_verifier_output_forwarded_to_worker",
55
+ "official_feedback_blinded",
56
+ "reward_feedback_forwarded",
57
+ "benchmark_live_worker_phase",
58
+ "task_setup_preflight",
59
+ "task_staging",
60
+ )
61
+ TERMINAL_BENCH_JOB_CASE_ARM_MARKERS = (
62
+ "_codex_goal_mode_baseline",
63
+ "_codex_loopx_treatment",
64
+ "_codex_loopx",
65
+ "_hardened_codex_baseline",
66
+ "_baseline",
67
+ "_treatment",
68
+ )
69
+ PRIVATE_ARTIFACT_REF_PATH_MARKERS = (
70
+ ".local/",
71
+ "/private/",
72
+ "private-benchmark-jobs",
73
+ "/Users/",
74
+ "/Volumes/",
75
+ "/var/folders/",
76
+ "/tmp/",
77
+ )
78
+ PUBLIC_LEDGER_LINEAGE_RESULT_FILENAMES = {
79
+ "benchmark-run.json",
80
+ "benchmark_run.json",
81
+ "compact-run.json",
82
+ "loopx-worker-benchmark-run.json",
83
+ "result.json",
84
+ "skillsbench-compact-benchmark-run-v0.json",
85
+ }
86
+ PRIVATE_ARTIFACT_REF_PATH_PARTS = {
87
+ ".local",
88
+ "private",
89
+ "private-benchmark-jobs",
90
+ "users",
91
+ "volumes",
92
+ "var",
93
+ "tmp",
94
+ }
95
+
96
+
97
+ def _now_local_iso() -> str:
98
+ return datetime.now().astimezone().isoformat(timespec="seconds")
99
+
100
+
101
+ def _compact_text(value: Any, *, limit: int = 160) -> str:
102
+ if value is None:
103
+ return ""
104
+ text = " ".join(str(value).split())
105
+ if len(text) <= limit:
106
+ return text
107
+ return text[: max(0, limit - 3)] + "..."
108
+
109
+
110
+ def _compact_list(value: Any, *, limit: int = 8) -> list[str]:
111
+ if not isinstance(value, list):
112
+ return []
113
+ result: list[str] = []
114
+ for item in value:
115
+ text = _compact_text(item, limit=120)
116
+ if text and text not in result:
117
+ result.append(text)
118
+ if len(result) >= limit:
119
+ break
120
+ return result
121
+
122
+
123
+ def _compact_round_reward_records(value: Any, *, limit: int = 32) -> list[dict[str, Any]]:
124
+ if not isinstance(value, list):
125
+ return []
126
+ records: list[dict[str, Any]] = []
127
+ seen_rounds: set[int] = set()
128
+ for item in value:
129
+ if not isinstance(item, dict):
130
+ continue
131
+ agent_round = item.get("agent_round")
132
+ if (
133
+ not isinstance(agent_round, int)
134
+ or isinstance(agent_round, bool)
135
+ or agent_round <= 0
136
+ or agent_round in seen_rounds
137
+ ):
138
+ continue
139
+ seen_rounds.add(agent_round)
140
+ record: dict[str, Any] = {"agent_round": agent_round}
141
+ for field in ("reward_present", "passed"):
142
+ if isinstance(item.get(field), bool):
143
+ record[field] = item[field]
144
+ reward = item.get("reward")
145
+ if isinstance(reward, (int, float)) and not isinstance(reward, bool):
146
+ record["reward"] = float(reward)
147
+ tool_calls = item.get("tool_calls")
148
+ if (
149
+ isinstance(tool_calls, int)
150
+ and not isinstance(tool_calls, bool)
151
+ and tool_calls >= 0
152
+ ):
153
+ record["tool_calls"] = tool_calls
154
+ records.append(record)
155
+ if len(records) >= limit:
156
+ break
157
+ return sorted(records, key=lambda record: record["agent_round"])
158
+
159
+
160
+ def _codex_acp_runtime_preflight_passed(value: Any) -> bool:
161
+ if not isinstance(value, dict):
162
+ return False
163
+ if value.get("codex_acp_runtime_launch_preflight") is True:
164
+ return True
165
+ return (
166
+ _compact_text(value.get("codex_acp_runtime_launch_preflight_status"), limit=80)
167
+ == "passed"
168
+ )
169
+
170
+
171
+ def _compact_task_staging(value: Any) -> dict[str, Any]:
172
+ if not isinstance(value, dict):
173
+ return {}
174
+ compact: dict[str, Any] = {}
175
+ for field in (
176
+ "staged",
177
+ "include_task_skills",
178
+ "apt_setup_risk_detected",
179
+ "apt_retry_patch_required",
180
+ "dockerfile_pip_install_risk_detected",
181
+ "dockerfile_pip_bootstrap_patch_required",
182
+ "dockerfile_pip_bootstrap_patch_applied",
183
+ "dockerfile_uv_bootstrap_risk_detected",
184
+ "dockerfile_uv_bootstrap_mirror_patch_required",
185
+ "dockerfile_uv_bootstrap_mirror_patch_applied",
186
+ "dockerfile_uv_bootstrap_pip_fallback_patch_applied",
187
+ "dockerfile_package_bootstrap_risk_preflight_blocked",
188
+ "app_skills_mount_patch_applied",
189
+ "apt_retry_patch_applied",
190
+ "apt_risk_preflight_blocked",
191
+ "bootstrap_light_preflight_blocked",
192
+ "bootstrap_light_fail_fast_defaulted",
193
+ "verifier_bootstrap_risk_detected",
194
+ "verifier_uv_bootstrap_risk_detected",
195
+ "verifier_uv_bootstrap_mirror_patch_required",
196
+ "verifier_uv_bootstrap_mirror_patch_applied",
197
+ "verifier_bootstrap_risk_preflight_blocked",
198
+ "dockerfile_apache_archive_mirror_patch_required",
199
+ "dockerfile_apache_archive_mirror_patch_applied",
200
+ "dockerfile_apache_archive_raw_url_recorded",
201
+ "dockerfile_maven_mirror_patch_required",
202
+ "dockerfile_maven_mirror_patch_applied",
203
+ "dockerfile_maven_mirror_raw_url_recorded",
204
+ "benchmark_egress_proxy_dockerfile_env_patch_required",
205
+ "benchmark_egress_proxy_dockerfile_env_patch_applied",
206
+ "benchmark_egress_proxy_dockerfile_java_opts_patch_applied",
207
+ "benchmark_egress_proxy_dockerfile_env_raw_proxy_recorded",
208
+ "codex_acp_runtime_tools_patch_applied",
209
+ "task_skills_removed",
210
+ "original_task_mutated",
211
+ ):
212
+ if isinstance(value.get(field), bool):
213
+ compact[field] = value[field]
214
+ for field in (
215
+ "dockerfile_pip_index_host",
216
+ "bootstrap_light_blocker_kind",
217
+ "dockerfile_uv_bootstrap_version",
218
+ "dockerfile_uv_bootstrap_mirror_host",
219
+ "verifier_uv_bootstrap_version",
220
+ "verifier_uv_bootstrap_mirror_host",
221
+ "dockerfile_apache_archive_mirror_host",
222
+ "dockerfile_maven_mirror_host",
223
+ ):
224
+ text = _compact_text(value.get(field), limit=140)
225
+ if text:
226
+ compact[field] = text
227
+ count = value.get("bootstrap_light_blocking_field_count")
228
+ if isinstance(count, int) and not isinstance(count, bool) and count >= 0:
229
+ compact["bootstrap_light_blocking_field_count"] = count
230
+ count = value.get("benchmark_egress_proxy_dockerfile_env_key_count")
231
+ if isinstance(count, int) and not isinstance(count, bool) and count >= 0:
232
+ compact["benchmark_egress_proxy_dockerfile_env_key_count"] = count
233
+ cap = value.get("resource_cap_patch")
234
+ if isinstance(cap, dict):
235
+ safe_cap: dict[str, Any] = {}
236
+ for field in ("applied", "original_task_mutated"):
237
+ if isinstance(cap.get(field), bool):
238
+ safe_cap[field] = cap[field]
239
+ for field in ("host_cpus", "requested_cpus", "effective_cpus"):
240
+ raw = cap.get(field)
241
+ if isinstance(raw, (int, float)) and not isinstance(raw, bool):
242
+ safe_cap[field] = raw
243
+ reason = _compact_text(cap.get("reason"), limit=120)
244
+ if reason:
245
+ safe_cap["reason"] = reason
246
+ if safe_cap:
247
+ compact["resource_cap_patch"] = safe_cap
248
+ return compact
249
+
250
+
251
+ def _compact_task_setup_preflight(value: Any) -> dict[str, Any]:
252
+ if not isinstance(value, dict):
253
+ return {}
254
+ compact: dict[str, Any] = {}
255
+ for field in (
256
+ "schema_version",
257
+ "status",
258
+ "sandbox",
259
+ "task_id",
260
+ "first_blocker",
261
+ "alternate_source_kind",
262
+ "canonical_equivalent_status",
263
+ "registry_source_kind",
264
+ "registry_source_status",
265
+ "registry_task_path",
266
+ "selection_recommendation",
267
+ ):
268
+ text = _compact_text(value.get(field), limit=140)
269
+ if text:
270
+ compact[field] = text
271
+ for field in (
272
+ "raw_task_text_read",
273
+ "raw_logs_read",
274
+ "raw_trajectory_read",
275
+ "apt_setup_risk_detected",
276
+ "apt_retry_patch_required",
277
+ "dockerfile_pip_install_risk_detected",
278
+ "dockerfile_pip_bootstrap_patch_required",
279
+ "verifier_present",
280
+ "verifier_bootstrap_risk_detected",
281
+ "verifier_uv_bootstrap_risk_detected",
282
+ "verifier_external_download_risk_detected",
283
+ "verifier_package_install_risk_detected",
284
+ "dockerfile_present",
285
+ "canonical_task_present",
286
+ "alternate_source_supported_by_runner",
287
+ "registry_task_present",
288
+ "registry_task_path_recorded",
289
+ "registry_excluded",
290
+ "task_source_path_recorded",
291
+ "task_source_content_recorded",
292
+ "bootstrap_light_candidate_eligible",
293
+ ):
294
+ if isinstance(value.get(field), bool):
295
+ compact[field] = value[field]
296
+ text = _compact_text(value.get("verifier_uv_bootstrap_version"), limit=140)
297
+ if text:
298
+ compact["verifier_uv_bootstrap_version"] = text
299
+ nearest_ids = value.get("nearest_canonical_task_ids")
300
+ if isinstance(nearest_ids, list):
301
+ compact_nearest: list[str] = []
302
+ for item in nearest_ids[:5]:
303
+ text = _compact_text(item, limit=120)
304
+ if text:
305
+ compact_nearest.append(text)
306
+ if compact_nearest:
307
+ compact["nearest_canonical_task_ids"] = compact_nearest
308
+ verifier_categories = value.get("verifier_bootstrap_risk_categories")
309
+ if isinstance(verifier_categories, list):
310
+ compact_categories: list[str] = []
311
+ for item in verifier_categories[:5]:
312
+ text = _compact_text(item, limit=120)
313
+ if text:
314
+ compact_categories.append(text)
315
+ if compact_categories:
316
+ compact["verifier_bootstrap_risk_categories"] = compact_categories
317
+ return compact
318
+
319
+
320
+ def _compact_compose_setup_diagnostic(value: Any) -> dict[str, Any]:
321
+ if not isinstance(value, dict):
322
+ return {}
323
+ compact: dict[str, Any] = {}
324
+ for field in (
325
+ "schema_version",
326
+ "status",
327
+ "route",
328
+ "failure_class",
329
+ "runner_prerequisite_status",
330
+ "task_setup_preflight_status",
331
+ "runner_error_len_bucket",
332
+ "primary_setup_failure_category",
333
+ "retryability",
334
+ "next_diagnostic_action",
335
+ ):
336
+ text = _compact_text(value.get(field), limit=140)
337
+ if text:
338
+ compact[field] = text
339
+ for field in (
340
+ "compose_setup_failure",
341
+ "unclassified_compose_failure",
342
+ "docker_daemon_unavailable",
343
+ "apt_repository_failure",
344
+ "volume_mount_failure",
345
+ "environment_setup_failure",
346
+ "agent_rounds_started",
347
+ "official_score_missing",
348
+ "official_result_json_materialized",
349
+ "case_attempt_budget_should_count",
350
+ "runner_launch_preflight_passed",
351
+ "apt_setup_risk_detected",
352
+ "apt_retry_patch_required",
353
+ "verifier_uv_bootstrap_risk_detected",
354
+ "verifier_uv_bootstrap_mirror_patch_required",
355
+ "verifier_uv_bootstrap_mirror_patch_applied",
356
+ "staged_task_prepared",
357
+ "task_skills_removed",
358
+ "codex_acp_runtime_tools_patch_applied",
359
+ "resource_cap_applied",
360
+ "raw_error_recorded",
361
+ "raw_logs_read",
362
+ "raw_task_text_read",
363
+ "raw_trajectory_read",
364
+ ):
365
+ if isinstance(value.get(field), bool):
366
+ compact[field] = value[field]
367
+ for field in (
368
+ "heartbeat_count",
369
+ "controller_action_decision_count",
370
+ "trajectory_round_count",
371
+ "trajectory_tool_call_count",
372
+ "loopx_cli_call_count",
373
+ "round_reward_count",
374
+ "setup_stall_timeout_requested_sec",
375
+ "setup_stall_timeout_sec",
376
+ "progress_completed_trials",
377
+ "progress_errored_trials",
378
+ ):
379
+ if isinstance(value.get(field), int) and not isinstance(value.get(field), bool):
380
+ compact[field] = value[field]
381
+ patterns = _compact_list(value.get("fingerprint_matched_patterns"), limit=8)
382
+ if patterns:
383
+ compact["fingerprint_matched_patterns"] = patterns
384
+ for field in (
385
+ "terminal_failure_dependency_classes",
386
+ "terminal_failure_reason_codes",
387
+ "terminal_failure_dependency_endpoints",
388
+ ):
389
+ values = _compact_list(value.get(field), limit=8)
390
+ if values:
391
+ compact[field] = values
392
+ return compact
393
+
394
+
395
+ def _compact_positive_int(value: Any) -> int | None:
396
+ if isinstance(value, int) and not isinstance(value, bool) and value > 0:
397
+ return value
398
+ return None
399
+
400
+
401
+ def _compact_nonnegative_int(value: Any) -> int | None:
402
+ if isinstance(value, int) and not isinstance(value, bool) and value >= 0:
403
+ return value
404
+ return None
405
+
406
+
407
+ def _compact_number(value: Any) -> float | None:
408
+ if isinstance(value, (int, float)) and not isinstance(value, bool):
409
+ return float(value)
410
+ return None
411
+
412
+
413
+ def _numeric_score_value(value: Any) -> float | None:
414
+ if isinstance(value, bool) or value is None:
415
+ return None
416
+ if isinstance(value, (int, float)):
417
+ return float(value)
418
+ if isinstance(value, str):
419
+ try:
420
+ return float(value)
421
+ except ValueError:
422
+ return None
423
+ return None
424
+
425
+
426
+ def _round_reward_best_stats(records: list[dict[str, Any]]) -> dict[str, Any]:
427
+ numeric_records: list[dict[str, Any]] = []
428
+ for record in records:
429
+ if not isinstance(record, dict):
430
+ continue
431
+ agent_round = record.get("agent_round")
432
+ reward = record.get("reward")
433
+ if (
434
+ not isinstance(agent_round, int)
435
+ or isinstance(agent_round, bool)
436
+ or agent_round <= 0
437
+ or not isinstance(reward, (int, float))
438
+ or isinstance(reward, bool)
439
+ ):
440
+ continue
441
+ numeric_records.append(
442
+ {
443
+ "agent_round": agent_round,
444
+ "reward": float(reward),
445
+ "passed": (
446
+ record.get("passed")
447
+ if isinstance(record.get("passed"), bool)
448
+ else reward >= 1
449
+ ),
450
+ }
451
+ )
452
+ if not numeric_records:
453
+ return {}
454
+ by_round = sorted(numeric_records, key=lambda item: item["agent_round"])
455
+ best = max(by_round, key=lambda item: (item["reward"], -item["agent_round"]))
456
+ final = by_round[-1]
457
+ return {
458
+ "final_round": final["agent_round"],
459
+ "final_round_reward": final["reward"],
460
+ "final_round_passed": final["passed"],
461
+ "best_reward_round": best["agent_round"],
462
+ "best_round_reward": best["reward"],
463
+ "best_round_passed": best["passed"],
464
+ "best_round_is_final": final["reward"] == best["reward"],
465
+ }
466
+
467
+
468
+ def _round_reward_summary(run: dict[str, Any]) -> str:
469
+ records = run.get("round_rewards")
470
+ if not isinstance(records, list):
471
+ return ""
472
+ parts: list[str] = []
473
+ for record in records:
474
+ if not isinstance(record, dict):
475
+ continue
476
+ agent_round = record.get("agent_round")
477
+ if not isinstance(agent_round, int) or isinstance(agent_round, bool):
478
+ continue
479
+ reward = record.get("reward")
480
+ if isinstance(reward, (int, float)) and not isinstance(reward, bool):
481
+ reward_text = f"{float(reward):g}"
482
+ elif record.get("reward_present") is False:
483
+ reward_text = "missing"
484
+ else:
485
+ reward_text = "unknown"
486
+ if record.get("passed") is True:
487
+ reward_text += "*"
488
+ parts.append(f"{agent_round}:{reward_text}")
489
+ return ",".join(parts)
490
+
491
+
492
+ def _attempt_label_from_accounting(run: dict[str, Any]) -> str:
493
+ accounting = (
494
+ run.get("attempt_accounting")
495
+ if isinstance(run.get("attempt_accounting"), dict)
496
+ else {}
497
+ )
498
+ if not accounting:
499
+ return ""
500
+ for field, label in (
501
+ ("official_score_attempt_countable", "official_score_attempt"),
502
+ ("verifier_attempt_countable", "verifier_attempt"),
503
+ ("solver_attempt_countable", "solver_attempt"),
504
+ ("case_attempt_countable", "case_attempt"),
505
+ ("launcher_attempt_countable", "launcher_attempt"),
506
+ ):
507
+ if accounting.get(field) is True:
508
+ return label
509
+ return ""
510
+
511
+
512
+ def _compact_first_from_lists(
513
+ benchmark_run: dict[str, Any],
514
+ *field_names: str,
515
+ ) -> str:
516
+ for field_name in field_names:
517
+ for item in _compact_list(benchmark_run.get(field_name), limit=8):
518
+ if item and item not in {"none", "None"}:
519
+ return item
520
+ return ""
521
+
522
+
523
+ def _compact_counter(benchmark_run: dict[str, Any], field_name: str) -> int:
524
+ value = benchmark_run.get(field_name)
525
+ if isinstance(value, int) and not isinstance(value, bool):
526
+ return value
527
+ return 0
528
+
529
+
530
+ def _source_schema(payload: dict[str, Any]) -> str:
531
+ return _compact_text(payload.get("schema_version"), limit=120)
532
+
533
+
534
+ def _run_identity_tokens(run: dict[str, Any]) -> set[str]:
535
+ tokens: set[str] = set()
536
+ for field in ("arm_id", "mode", "route"):
537
+ text = _compact_text(run.get(field), limit=160)
538
+ if text:
539
+ tokens.add(text.lower())
540
+ contract = run.get("benchmark_loop_contract")
541
+ if isinstance(contract, dict):
542
+ for field in ("route", "protocol_id"):
543
+ text = _compact_text(contract.get(field), limit=160)
544
+ if text:
545
+ tokens.add(text.lower())
546
+ return tokens
547
+
548
+
549
+ def _run_matches_token(run: dict[str, Any], *needles: str) -> bool:
550
+ tokens = _run_identity_tokens(run)
551
+ for needle in needles:
552
+ lower_needle = needle.lower()
553
+ if lower_needle in tokens:
554
+ return True
555
+ if any(lower_needle in token for token in tokens):
556
+ return True
557
+ return False
558
+
559
+
560
+ def _ledger_missing_value(value: Any) -> bool:
561
+ return value is None or value in ("", [], {})
562
+
563
+
564
+ def _ledger_logical_backfill_key(run: dict[str, Any]) -> tuple[str, ...]:
565
+ values: list[str] = []
566
+ for field in ("run_group_id", "arm_id", "job_name", "mode"):
567
+ text = _compact_text(run.get(field), limit=220)
568
+ if not text:
569
+ return ()
570
+ values.append(text)
571
+ return tuple(values)
572
+
573
+
574
+ def _ledger_result_equivalent_for_backfill(
575
+ run: dict[str, Any],
576
+ entry: dict[str, Any],
577
+ ) -> bool:
578
+ for field in ("status", "score_status", "official_passed", "failure_class"):
579
+ if run.get(field) != entry.get(field):
580
+ return False
581
+ return run.get("official_score") == entry.get("official_score")
582
+
583
+
584
+ def _merge_ledger_logical_backfill_fields(
585
+ run: dict[str, Any],
586
+ entry: dict[str, Any],
587
+ ) -> tuple[dict[str, Any], bool]:
588
+ merged = dict(run)
589
+ changed = False
590
+ for field in LEDGER_LOGICAL_BACKFILL_FIELDS:
591
+ if _ledger_missing_value(merged.get(field)) and not _ledger_missing_value(
592
+ entry.get(field)
593
+ ):
594
+ merged[field] = entry[field]
595
+ changed = True
596
+ return merged, changed
597
+
598
+
599
+ def _product_mode_baseline_run(run: dict[str, Any]) -> bool:
600
+ return _run_matches_token(
601
+ run,
602
+ "raw-codex-autonomous-max5",
603
+ "raw_codex_autonomous_max5",
604
+ "skillsbench_raw_codex_autonomous_max5",
605
+ )
606
+
607
+
608
+ def _product_mode_treatment_run(run: dict[str, Any]) -> bool:
609
+ return _run_matches_token(
610
+ run,
611
+ "loopx-product-mode",
612
+ "loopx_product_mode",
613
+ "skillsbench_loopx_product_mode",
614
+ )
615
+
616
+
617
+ def _compact_product_mode_pair_review(review: dict[str, Any]) -> dict[str, Any]:
618
+ compact: dict[str, Any] = {}
619
+ for field in (
620
+ "schema_version",
621
+ "comparison_id",
622
+ "main_table_claim_allowed",
623
+ "product_mode_pair_complete",
624
+ "claim_blocker",
625
+ "benchmark_id",
626
+ "case_id",
627
+ "max_rounds_budget",
628
+ "baseline_route_valid",
629
+ "treatment_route_valid",
630
+ "treatment_loopx_lifecycle_observed",
631
+ "official_feedback_blinded",
632
+ ):
633
+ value = review.get(field)
634
+ if value not in (None, "", []):
635
+ compact[field] = value
636
+ headline_metrics = review.get("headline_metrics")
637
+ if isinstance(headline_metrics, list):
638
+ compact["headline_metrics"] = [
639
+ metric
640
+ for metric in (_compact_text(item, limit=80) for item in headline_metrics)
641
+ if metric
642
+ ][:8]
643
+ return compact
644
+
645
+
646
+ def _compact_product_mode_lifecycle_contract(value: Any) -> dict[str, Any]:
647
+ if not isinstance(value, dict):
648
+ return {}
649
+ compact: dict[str, Any] = {}
650
+ schema = _compact_text(value.get("schema_version"), limit=100)
651
+ if schema:
652
+ compact["schema_version"] = schema
653
+ for field in (
654
+ "required",
655
+ "satisfied",
656
+ "countable_treatment",
657
+ "checkpoint_required",
658
+ "orchestrated_driver_lifecycle_satisfied",
659
+ "orchestrated_driver_counts_as_product_mode",
660
+ ):
661
+ if isinstance(value.get(field), bool):
662
+ compact[field] = value[field]
663
+ for field in (
664
+ "state_read_count",
665
+ "state_write_count",
666
+ "checkpoint_count",
667
+ "checkpoint_round",
668
+ ):
669
+ raw = value.get(field)
670
+ if isinstance(raw, int) and not isinstance(raw, bool):
671
+ compact[field] = max(0, raw)
672
+ missing_reason = _compact_text(value.get("missing_reason"), limit=140)
673
+ if missing_reason:
674
+ compact["missing_reason"] = missing_reason
675
+ execution_style = _compact_text(value.get("execution_style"), limit=120)
676
+ if execution_style:
677
+ compact["execution_style"] = execution_style
678
+ return compact
679
+
680
+
681
+ def _compact_app_server_goal_round_semantics(value: Any) -> dict[str, Any]:
682
+ if not isinstance(value, dict):
683
+ return {}
684
+ compact: dict[str, Any] = {}
685
+ for field in (
686
+ "schema_version",
687
+ "route",
688
+ "session_policy",
689
+ "max_rounds_budget_applies_to",
690
+ ):
691
+ text = _compact_text(value.get(field), limit=140)
692
+ if text:
693
+ compact[field] = text
694
+ for field in (
695
+ "benchflow_max_rounds_budget",
696
+ "initial_goal_turn_budget",
697
+ "same_thread_followup_budget",
698
+ "independent_attempt_budget",
699
+ ):
700
+ number = _compact_nonnegative_int(value.get(field))
701
+ if number is not None:
702
+ compact[field] = number
703
+ for field in (
704
+ "fresh_goal_thread_per_independent_attempt",
705
+ "official_reward_feedback_forwarded_to_worker",
706
+ "verifier_output_forwarded_to_worker",
707
+ ):
708
+ if isinstance(value.get(field), bool):
709
+ compact[field] = value[field]
710
+ return compact
711
+
712
+
713
+ def _compact_skillsbench_solution_quality_signals(value: Any) -> dict[str, Any]:
714
+ if not isinstance(value, dict):
715
+ return {}
716
+ compact: dict[str, Any] = {}
717
+ for field in (
718
+ "schema_version",
719
+ "source",
720
+ "outcome_class",
721
+ "rubric_miss_label_status",
722
+ ):
723
+ text = _compact_text(value.get(field), limit=120)
724
+ if text:
725
+ compact[field] = text
726
+ for field in ("solution_action_labels", "rubric_miss_labels", "public_limits"):
727
+ labels = _compact_list(value.get(field), limit=12)
728
+ if labels:
729
+ compact[field] = labels
730
+ worker_activity = (
731
+ value.get("worker_activity")
732
+ if isinstance(value.get("worker_activity"), dict)
733
+ else {}
734
+ )
735
+ compact_worker_activity: dict[str, Any] = {}
736
+ for field in (
737
+ "task_facing_activity_observed",
738
+ "worker_turn_or_bridge_observed",
739
+ ):
740
+ if isinstance(worker_activity.get(field), bool):
741
+ compact_worker_activity[field] = worker_activity[field]
742
+ for field in (
743
+ "tool_call_count",
744
+ "bridge_task_facing_operation_count",
745
+ "bridge_task_facing_success_count",
746
+ ):
747
+ raw = worker_activity.get(field)
748
+ if isinstance(raw, int) and not isinstance(raw, bool):
749
+ compact_worker_activity[field] = max(0, raw)
750
+ if compact_worker_activity:
751
+ compact["worker_activity"] = compact_worker_activity
752
+ return compact
753
+
754
+
755
+ def _compact_operator_simulator_run(value: Any) -> dict[str, Any]:
756
+ if not isinstance(value, dict):
757
+ return {}
758
+ schema_version = _compact_text(value.get("schema_version"), limit=120)
759
+ if schema_version != OPERATOR_SIMULATOR_RUN_SCHEMA_VERSION:
760
+ return {}
761
+ compact: dict[str, Any] = {"schema_version": OPERATOR_SIMULATOR_RUN_SCHEMA_VERSION}
762
+ simulator_identity = (
763
+ value.get("simulator_identity")
764
+ if isinstance(value.get("simulator_identity"), dict)
765
+ else {}
766
+ )
767
+ for field in (
768
+ "arm_schema_version",
769
+ "benchmark_id",
770
+ "case_id",
771
+ "task_id",
772
+ "mode",
773
+ "simulator_setting",
774
+ ):
775
+ raw = (
776
+ simulator_identity.get("setting")
777
+ if field == "simulator_setting" and value.get(field) is None
778
+ else value.get(field)
779
+ )
780
+ text = _compact_text(raw, limit=140)
781
+ if text:
782
+ compact[field] = text
783
+ claim_boundary = (
784
+ value.get("claim_boundary")
785
+ if isinstance(value.get("claim_boundary"), dict)
786
+ else {}
787
+ )
788
+ for field in (
789
+ "rubric_generated_before_solver_start",
790
+ "official_score_claim_allowed",
791
+ "leaderboard_claim_allowed",
792
+ "assisted_collaboration_claim_allowed",
793
+ "assisted_score_kept_separate_from_official",
794
+ ):
795
+ if isinstance(value.get(field), bool):
796
+ compact[field] = value[field]
797
+ elif isinstance(claim_boundary.get(field), bool):
798
+ compact[field] = claim_boundary[field]
799
+ for field in ("intervention_count", "proactive_intervention_count"):
800
+ raw = value.get(field)
801
+ if isinstance(raw, int) and not isinstance(raw, bool) and raw >= 0:
802
+ compact[field] = raw
803
+ assisted_score = value.get("assisted_score")
804
+ if isinstance(assisted_score, (int, float)) and not isinstance(assisted_score, bool):
805
+ compact["assisted_score"] = float(assisted_score)
806
+ return compact
807
+
808
+
809
+ def _ledger_skillsbench_solution_quality_signals(
810
+ benchmark_run: dict[str, Any],
811
+ *,
812
+ benchmark_id: str,
813
+ ) -> dict[str, Any]:
814
+ signals = _compact_skillsbench_solution_quality_signals(
815
+ benchmark_run.get("solution_quality_signals")
816
+ )
817
+ if signals or not benchmark_id.lower().startswith("skillsbench"):
818
+ return signals
819
+ return _compact_skillsbench_solution_quality_signals(
820
+ build_skillsbench_solution_quality_signals(benchmark_run)
821
+ )
822
+
823
+
824
+ def _terminal_bench_case_id_from_job_name(
825
+ *,
826
+ benchmark_id: str,
827
+ job_name: str,
828
+ ) -> str:
829
+ if not job_name:
830
+ return ""
831
+ if benchmark_id != "terminal-bench@2.0" and not job_name.startswith("terminal_bench_"):
832
+ return ""
833
+ remainder = job_name
834
+ if remainder.startswith("terminal_bench_"):
835
+ remainder = remainder[len("terminal_bench_") :]
836
+ parts = remainder.split("_")
837
+ while parts and parts[0].isdigit():
838
+ parts.pop(0)
839
+ if not parts:
840
+ return ""
841
+ remainder = "_".join(parts)
842
+ for marker in TERMINAL_BENCH_JOB_CASE_ARM_MARKERS:
843
+ marker_index = remainder.find(marker)
844
+ if marker_index > 0:
845
+ candidate = remainder[:marker_index]
846
+ return candidate.replace("_", "-")
847
+ return ""
848
+
849
+
850
+ def _relative_ref(value: str | Path | None, *, cwd: Path | None = None) -> str | None:
851
+ if value is None:
852
+ return None
853
+ raw = str(value).strip()
854
+ if not raw:
855
+ return None
856
+ path = Path(raw).expanduser()
857
+ base = (cwd or Path.cwd()).resolve()
858
+ if path.is_absolute():
859
+ try:
860
+ return path.resolve().relative_to(base).as_posix()
861
+ except ValueError:
862
+ return None
863
+ return path.as_posix()
864
+
865
+
866
+ def _public_ledger_artifact_ref(
867
+ value: str | Path | None,
868
+ *,
869
+ cwd: Path | None = None,
870
+ ) -> str | None:
871
+ ref = _relative_ref(value, cwd=cwd)
872
+ if not ref:
873
+ return None
874
+ classification = classify_benchmark_artifact_path(ref)
875
+ if classification.get("allowed_to_read") is not True:
876
+ normalized = ref.replace("\\", "/").strip("/")
877
+ parts = [part for part in normalized.split("/") if part]
878
+ if (
879
+ not normalized
880
+ or ref.startswith("/")
881
+ or ref.startswith("~")
882
+ or ":" in normalized
883
+ or len(parts) != len(normalized.split("/"))
884
+ or any(part in {".", ".."} or part.startswith(".") for part in parts)
885
+ or any(part.lower() in PRIVATE_ARTIFACT_REF_PATH_PARTS for part in parts)
886
+ or classification.get("private_raw_surface") is True
887
+ ):
888
+ return None
889
+ basename = _compact_text(classification.get("basename"), limit=160)
890
+ if basename in PUBLIC_LEDGER_LINEAGE_RESULT_FILENAMES:
891
+ return normalized
892
+ if "." not in basename:
893
+ return normalized
894
+ return None
895
+ normalized = ref.replace("\\", "/")
896
+ if any(marker in normalized for marker in PRIVATE_ARTIFACT_REF_PATH_MARKERS):
897
+ basename = _compact_text(classification.get("basename"), limit=160)
898
+ return basename or None
899
+ return normalized
900
+
901
+
902
+ def _case_ids(benchmark_run: dict[str, Any]) -> list[str]:
903
+ ids: list[str] = []
904
+ case_id = _compact_text(benchmark_run.get("case_id"), limit=140)
905
+ if case_id:
906
+ ids.append(case_id)
907
+ case_ids = benchmark_run.get("case_ids")
908
+ if isinstance(case_ids, list):
909
+ for item in case_ids:
910
+ text = _compact_text(item, limit=140)
911
+ if text and text not in ids:
912
+ ids.append(text)
913
+ if ids:
914
+ return ids
915
+ trials = benchmark_run.get("trials")
916
+ if isinstance(trials, list):
917
+ for trial in trials:
918
+ if not isinstance(trial, dict):
919
+ continue
920
+ task_id = _compact_text(trial.get("task_id"), limit=140)
921
+ if task_id and task_id not in ids:
922
+ ids.append(task_id)
923
+ if ids:
924
+ return ids
925
+ job_name = _compact_text(benchmark_run.get("job_name"), limit=140)
926
+ fallback_case = _terminal_bench_case_id_from_job_name(
927
+ benchmark_id=_compact_text(benchmark_run.get("benchmark_id"), limit=120),
928
+ job_name=job_name,
929
+ )
930
+ if fallback_case:
931
+ return [fallback_case]
932
+ return [job_name] if job_name else ["unknown-case"]
933
+
934
+
935
+ def _official_score(benchmark_run: dict[str, Any]) -> tuple[float | int | None, bool | None]:
936
+ official = (
937
+ benchmark_run.get("official_task_score")
938
+ if isinstance(benchmark_run.get("official_task_score"), dict)
939
+ else {}
940
+ )
941
+ value = official.get("value")
942
+ if isinstance(value, (int, float)) and not isinstance(value, bool):
943
+ passed = official.get("passed")
944
+ return value, passed if isinstance(passed, bool) else value >= 1
945
+ value = benchmark_run.get("official_score")
946
+ if isinstance(value, (int, float)) and not isinstance(value, bool):
947
+ return value, value >= 1
948
+ if not official_score_bool_fallback_allowed(benchmark_run):
949
+ return None, None
950
+ return _official_score_passed_bool_fallback(benchmark_run)
951
+
952
+
953
+ def _official_task_score_bool_passed(benchmark_run: dict[str, Any]) -> bool | None:
954
+ official = (
955
+ benchmark_run.get("official_task_score")
956
+ if isinstance(benchmark_run.get("official_task_score"), dict)
957
+ else {}
958
+ )
959
+ passed = official.get("passed")
960
+ return passed if isinstance(passed, bool) else None
961
+
962
+
963
+ def _infer_arm_id_from_job_name(job_name: str) -> str:
964
+ if not job_name:
965
+ return ""
966
+ if "codex_app_server_goal_baseline" in job_name:
967
+ return "codex_app_server_goal_baseline"
968
+ if "codex_goal_mode_baseline" in job_name:
969
+ return "codex_goal_mode_baseline"
970
+ if "hardened_codex_baseline" in job_name:
971
+ return "hardened_codex_baseline"
972
+ if "codex_loopx_treatment" in job_name:
973
+ return "codex_loopx_treatment"
974
+ if "loopx_automation_loop_treatment" in job_name:
975
+ return "loopx_automation_loop_treatment"
976
+ if job_name.endswith("_baseline") or "_baseline_" in job_name:
977
+ return "baseline"
978
+ if job_name.endswith("_treatment") or "_treatment_" in job_name:
979
+ return "treatment"
980
+ return ""
981
+
982
+
983
+ def _infer_arm_id(benchmark_run: dict[str, Any]) -> str:
984
+ mode = _compact_text(benchmark_run.get("mode"), limit=120)
985
+ route = _compact_text(benchmark_run.get("route"), limit=120)
986
+ if (
987
+ route in LEGACY_NONPRODUCT_PROMPT_POLLING_ROUTES
988
+ or mode in {
989
+ "skillsbench_loopx_blind_loop_treatment",
990
+ "skillsbench_loopx_prompt_polling_test",
991
+ "loopx_prompt_polling_test",
992
+ }
993
+ or benchmark_run.get("historical_route_read_only") is True
994
+ or _compact_text(
995
+ benchmark_run.get("skillsbench_route_semantics"), limit=120
996
+ )
997
+ == "historical_nonproduct_invalid_for_comparison"
998
+ ):
999
+ return "historical_nonproduct_invalid_for_comparison"
1000
+ if mode == "skillsbench_codex_app_server_goal_baseline":
1001
+ return "codex_app_server_goal_baseline"
1002
+ if mode == "codex_goal_mode_baseline":
1003
+ return "codex_goal_mode_baseline"
1004
+ if mode in {"hardened_codex_baseline", "hardened-codex"}:
1005
+ return "hardened_codex_baseline"
1006
+ if "automation_loop" in mode and "loopx" in mode:
1007
+ return "loopx_automation_loop_treatment"
1008
+ if "curated_skills" in mode:
1009
+ return "curated_skills_baseline"
1010
+ if mode in {"no_skills_baseline", "skillsbench_no_skills_baseline"}:
1011
+ return "no_skills_baseline"
1012
+ if "loopx" in mode or "codex-loopx" in mode:
1013
+ return "codex_loopx_treatment"
1014
+ from_job_name = _infer_arm_id_from_job_name(
1015
+ _compact_text(benchmark_run.get("job_name"), limit=160)
1016
+ )
1017
+ if from_job_name:
1018
+ return from_job_name
1019
+ return mode or "unknown_arm"
1020
+
1021
+
1022
+ def _resolved_arm_id(benchmark_run: dict[str, Any], arm_id: str | None) -> str:
1023
+ inferred = _infer_arm_id(benchmark_run)
1024
+ if inferred == "historical_nonproduct_invalid_for_comparison":
1025
+ return inferred
1026
+ explicit = _compact_text(arm_id, limit=120)
1027
+ if explicit in {"baseline", "treatment"} and inferred not in {"", "unknown_arm"}:
1028
+ return inferred
1029
+ return explicit or inferred
1030
+
1031
+
1032
+ def _score_status(benchmark_run: dict[str, Any], score: float | int | None, passed: bool | None) -> str:
1033
+ if _source_schema(benchmark_run) == "terminal_bench_post_launch_materialization_v0":
1034
+ return "missing"
1035
+ explicit = _compact_text(benchmark_run.get("official_score_status"), limit=80)
1036
+ if explicit and explicit != "completed":
1037
+ if explicit == "missing" and score is not None and isinstance(passed, bool):
1038
+ return "passed" if passed else "failed"
1039
+ return explicit
1040
+ if score is None:
1041
+ return "missing"
1042
+ return "passed" if passed else "failed"
1043
+
1044
+
1045
+ def _official_score_passed_bool_fallback(
1046
+ benchmark_run: dict[str, Any],
1047
+ ) -> tuple[float | None, bool | None]:
1048
+ official_passed = _official_task_score_bool_passed(benchmark_run)
1049
+ if isinstance(official_passed, bool):
1050
+ return (1.0 if official_passed else 0.0), official_passed
1051
+
1052
+ score_status = _compact_text(
1053
+ benchmark_run.get("official_score_status") or benchmark_run.get("score_status"),
1054
+ limit=80,
1055
+ )
1056
+ if score_status not in {"completed", "passed", "failed"}:
1057
+ return None, None
1058
+ if _compact_text(benchmark_run.get("runner_return_status"), limit=120) == (
1059
+ "failed_before_official_result"
1060
+ ):
1061
+ return None, None
1062
+ for container, key in (
1063
+ (benchmark_run, "official_passed"),
1064
+ (benchmark_run, "passed"),
1065
+ ):
1066
+ value = container.get(key) if isinstance(container, dict) else None
1067
+ if isinstance(value, bool):
1068
+ return (1.0 if value else 0.0), value
1069
+ return None, None
1070
+
1071
+
1072
+ _SKILLSBENCH_PRE_AGENT_SETUP_STATUS_LABELS = {
1073
+ "compose_setup_blocked_before_agent_rounds": (
1074
+ "skillsbench_compose_setup_blocked_before_agent_rounds"
1075
+ ),
1076
+ "runner_setup_blocked_before_agent_rounds": (
1077
+ "skillsbench_runner_setup_blocked_before_agent_rounds"
1078
+ ),
1079
+ }
1080
+
1081
+ _SKILLSBENCH_PRE_AGENT_SETUP_FAILURE_CLASSES = frozenset(
1082
+ _SKILLSBENCH_PRE_AGENT_SETUP_STATUS_LABELS.values()
1083
+ )
1084
+
1085
+ _SKILLSBENCH_SETUP_PREFLIGHT_REPAIR_ATTRIBUTIONS = {
1086
+ "skillsbench_docker_apt_setup_risk_preflight_blocked",
1087
+ "skillsbench_dockerfile_package_bootstrap_risk_preflight_blocked",
1088
+ "skillsbench_verifier_bootstrap_risk_preflight_blocked",
1089
+ "skillsbench_task_source_preflight_blocked",
1090
+ }
1091
+
1092
+
1093
+ def _skillsbench_pre_agent_setup_failure_class(
1094
+ benchmark_run: dict[str, Any],
1095
+ ) -> str:
1096
+ diagnostic = (
1097
+ benchmark_run.get("compose_setup_diagnostic")
1098
+ if isinstance(benchmark_run.get("compose_setup_diagnostic"), dict)
1099
+ else {}
1100
+ )
1101
+ label = _SKILLSBENCH_PRE_AGENT_SETUP_STATUS_LABELS.get(
1102
+ _compact_text(diagnostic.get("status"), limit=120)
1103
+ )
1104
+ if not label:
1105
+ return ""
1106
+ mode = _compact_text(benchmark_run.get("mode"), limit=120)
1107
+ route = _compact_text(benchmark_run.get("route"), limit=120)
1108
+ if mode != "skillsbench_codex_app_server_goal_baseline" and route != (
1109
+ "codex-app-server-goal-baseline"
1110
+ ):
1111
+ return ""
1112
+ if diagnostic.get("agent_rounds_started") is True:
1113
+ return ""
1114
+ return label
1115
+
1116
+
1117
+ def _failure_class(benchmark_run: dict[str, Any], score: float | int | None) -> str:
1118
+ if _source_schema(benchmark_run) == "terminal_bench_post_launch_materialization_v0":
1119
+ compact_failure = _compact_text(
1120
+ benchmark_run.get("compact_failure_class"),
1121
+ limit=120,
1122
+ )
1123
+ if compact_failure:
1124
+ return compact_failure
1125
+ marker = (
1126
+ benchmark_run.get("compact_failure_marker")
1127
+ if isinstance(benchmark_run.get("compact_failure_marker"), dict)
1128
+ else {}
1129
+ )
1130
+ marker_failure = _compact_text(marker.get("failure_class"), limit=120)
1131
+ if marker_failure:
1132
+ return marker_failure
1133
+ first_blocker = _compact_text(benchmark_run.get("first_blocker"), limit=120)
1134
+ return first_blocker or "post_launch_compact_result_missing"
1135
+ if score is not None and score != 0:
1136
+ return "none"
1137
+ if score is None:
1138
+ pre_agent_setup = _skillsbench_pre_agent_setup_failure_class(benchmark_run)
1139
+ if pre_agent_setup:
1140
+ return pre_agent_setup
1141
+ setup_blocker = _compact_first_from_lists(
1142
+ benchmark_run,
1143
+ "worker_setup_diagnostic_blockers",
1144
+ "worker_startup_blockers",
1145
+ )
1146
+ if setup_blocker:
1147
+ return setup_blocker
1148
+ labels = _compact_list(benchmark_run.get("failure_attribution_labels"), limit=12)
1149
+ if (
1150
+ "official_verifier_solution_failure" in labels
1151
+ and "worker_bridge_connected_official_score_failure" in labels
1152
+ and _compact_text(benchmark_run.get("worker_bridge_materialization_status"), limit=80)
1153
+ == "verified"
1154
+ and _compact_counter(benchmark_run, "worker_self_validation_official_score_mismatch_count") == 0
1155
+ and _compact_counter(benchmark_run, "worker_validation_scope_ambiguous_official_score_failure_count") == 0
1156
+ and _compact_counter(benchmark_run, "worker_submit_eligible_mismatch_count") == 0
1157
+ and _compact_counter(benchmark_run, "worker_bridge_writeback_loss_count") == 0
1158
+ and _compact_counter(benchmark_run, "worker_startup_blocker_count") == 0
1159
+ and _compact_counter(benchmark_run, "environment_setup_failure_before_worker_count") == 0
1160
+ and _compact_counter(benchmark_run, "pre_worker_agent_setup_failure_count") == 0
1161
+ ):
1162
+ return "official_verifier_solution_failure"
1163
+ for label in (
1164
+ "codex_model_access_unsupported_for_account",
1165
+ "codex_model_access_failure_before_solution_attempt",
1166
+ "agent_setup_timeout_before_worker_start",
1167
+ "agent_setup_failed_before_worker_start",
1168
+ "environment_setup_failed_before_worker",
1169
+ "worker_self_validation_official_score_mismatch",
1170
+ "worker_validation_scope_ambiguous_official_score_failure",
1171
+ "worker_bridge_connected_official_score_failure",
1172
+ "verifier_dependency_install_failure",
1173
+ "verifier_platform_probe_failure",
1174
+ "agent_timeout_before_solution_completion",
1175
+ ):
1176
+ if label in labels:
1177
+ return label
1178
+ attribution = _compact_text(
1179
+ benchmark_run.get("score_failure_attribution"),
1180
+ limit=120,
1181
+ )
1182
+ if attribution and attribution != "none":
1183
+ return attribution
1184
+ trial_exception = _trial_exception_failure_class(benchmark_run)
1185
+ if trial_exception:
1186
+ return trial_exception
1187
+ blocker = _compact_text(benchmark_run.get("first_blocker"), limit=120)
1188
+ if blocker:
1189
+ return blocker
1190
+ return "score_failure_unattributed" if score is not None else "score_missing"
1191
+
1192
+
1193
+ def _trial_exception_failure_class(benchmark_run: dict[str, Any]) -> str:
1194
+ trials = benchmark_run.get("trials")
1195
+ if not isinstance(trials, list):
1196
+ return ""
1197
+ exceptions: list[str] = []
1198
+ for trial in trials:
1199
+ if not isinstance(trial, dict):
1200
+ continue
1201
+ exception_type = _compact_text(trial.get("exception_type"), limit=80)
1202
+ if exception_type and exception_type not in {"none", "None"}:
1203
+ exceptions.append(exception_type)
1204
+ if not exceptions:
1205
+ return ""
1206
+ lowered = {item.lower() for item in exceptions}
1207
+ if any("setup" in item and "timeout" in item for item in lowered):
1208
+ return "agent_setup_timeout_before_worker_start"
1209
+ if any("setup" in item for item in lowered):
1210
+ return "agent_setup_exception_before_solution_attempt"
1211
+ if any("timeout" in item for item in lowered):
1212
+ return "agent_timeout_before_solution_completion"
1213
+ return "agent_exception_before_solution_completion"
1214
+
1215
+
1216
+ def _failure_scope(failure_class: str, score: float | int | None, passed: bool | None) -> str:
1217
+ if passed is True:
1218
+ return "passed"
1219
+ if failure_class in {
1220
+ "not_applicable_worker_materialization_probe",
1221
+ "not_applicable_worker_materialization_probe_no_trial_result",
1222
+ }:
1223
+ return "startup_surface"
1224
+ if failure_class in {
1225
+ "stale_active_job_without_trial_result",
1226
+ "detached_worker_ended_active_without_trial_result",
1227
+ "detached_worker_ended_without_trial_result",
1228
+ "post_launch_compact_result_missing",
1229
+ }:
1230
+ return "runner_or_setup"
1231
+ if score is None:
1232
+ return "score_missing"
1233
+ if failure_class == "score_failure_unattributed":
1234
+ return "attribution_required"
1235
+ if failure_class in {
1236
+ "none",
1237
+ "official_verifier_solution_failure",
1238
+ "official_score_zero_case_failure",
1239
+ "model_solution_failure",
1240
+ "agent_solution_failure",
1241
+ "task_solution_failure",
1242
+ "solution_incorrect",
1243
+ "agent_timeout_before_solution_completion",
1244
+ "agent_exception_before_solution_completion",
1245
+ }:
1246
+ return "case_or_solution"
1247
+ if failure_class.startswith("verifier_"):
1248
+ return "verifier_or_infra"
1249
+ return "runner_or_setup"
1250
+
1251
+
1252
+ def _repair_route_failure_class(
1253
+ benchmark_run: dict[str, Any], failure_class: str
1254
+ ) -> str:
1255
+ if failure_class not in _SKILLSBENCH_PRE_AGENT_SETUP_FAILURE_CLASSES:
1256
+ return failure_class
1257
+ attribution = _compact_text(
1258
+ benchmark_run.get("score_failure_attribution"),
1259
+ limit=120,
1260
+ )
1261
+ if attribution in _SKILLSBENCH_SETUP_PREFLIGHT_REPAIR_ATTRIBUTIONS:
1262
+ return attribution
1263
+ return failure_class
1264
+
1265
+
1266
+ def _repair_route(
1267
+ failure_class: str,
1268
+ failure_scope: str,
1269
+ *,
1270
+ agent_model: str = "",
1271
+ round_success_observed: bool = False,
1272
+ runtime_preflight_passed: bool = False,
1273
+ ) -> dict[str, Any]:
1274
+ if (
1275
+ failure_class.startswith("codex_cli_")
1276
+ or failure_class.startswith("worker_install_failed")
1277
+ ):
1278
+ return {
1279
+ "repair_priority": "P0",
1280
+ "repair_class": "runner_codex_cli_materialization",
1281
+ "next_action": (
1282
+ "materialize an existing Codex CLI on the worker PATH or provide "
1283
+ "an equivalent launcher before rerunning; require a compact setup "
1284
+ "diagnostic that proves the Codex preflight reached ok instead of "
1285
+ "only a generic pre-worker agent failure"
1286
+ ),
1287
+ "repair_profile": {
1288
+ "schema_version": "benchmark_repair_profile_v0",
1289
+ "repair_class": "runner_codex_cli_materialization",
1290
+ "required_launch_overrides": {
1291
+ "codex_install_strategy": "require_existing_codex",
1292
+ },
1293
+ "disallowed_launch_overrides": {
1294
+ "codex_install_strategy": "runtime_install_if_missing",
1295
+ },
1296
+ "required_preflight": [
1297
+ "codex_cli_existing_in_worker_or_fail_fast_blocker",
1298
+ "worker_setup_diagnostic.schema_ok",
1299
+ "worker_setup_diagnostic.first_blocker_or_ok",
1300
+ ],
1301
+ "raw_logs_required": False,
1302
+ "raw_task_text_required": False,
1303
+ "rerun_allowed_after_profile_applied": True,
1304
+ },
1305
+ }
1306
+ if failure_class == "environment_setup_failed_before_worker":
1307
+ return {
1308
+ "repair_priority": "P0",
1309
+ "repair_class": "benchmark_environment_setup_contract",
1310
+ "next_action": (
1311
+ "repair or preflight the benchmark environment setup layer before "
1312
+ "rerunning this case; the failure occurred before Codex/worker "
1313
+ "startup, so require compact environment setup readiness evidence "
1314
+ "instead of treating it as an adapter startup issue"
1315
+ ),
1316
+ "repair_profile": {
1317
+ "schema_version": "benchmark_repair_profile_v0",
1318
+ "repair_class": "benchmark_environment_setup_contract",
1319
+ "rerun_allowed_after_profile_applied": True,
1320
+ "required_preflight": [
1321
+ "environment_setup_readiness_preflight_before_repeat",
1322
+ "compact_environment_setup_failure_context",
1323
+ "worker_not_started_before_environment_ready",
1324
+ ],
1325
+ "raw_logs_required": False,
1326
+ "raw_task_text_required": False,
1327
+ },
1328
+ }
1329
+ if failure_class in {
1330
+ "agent_setup_timeout_before_worker_start",
1331
+ "agent_setup_failed_before_worker_start",
1332
+ "agent_setup_exception_before_solution_attempt",
1333
+ }:
1334
+ return {
1335
+ "repair_priority": "P0",
1336
+ "repair_class": "runner_setup_timeout",
1337
+ "next_action": (
1338
+ "repair the Codex worker setup path before rerunning this case; "
1339
+ "do not rely on runtime Codex install inside Harbor setup; use "
1340
+ "a materialized launcher or require_existing_codex fail-fast probe "
1341
+ "with compact setup-readiness proof"
1342
+ ),
1343
+ "repair_profile": {
1344
+ "schema_version": "benchmark_repair_profile_v0",
1345
+ "repair_class": "runner_setup_timeout",
1346
+ "rerun_allowed_after_profile_applied": True,
1347
+ "disallowed_launch_overrides": {
1348
+ "codex_install_strategy": RUNTIME_CODEX_INSTALL_STRATEGY,
1349
+ },
1350
+ "required_launch_overrides": {
1351
+ "codex_install_strategy": (
1352
+ DEFAULT_CODEX_SETUP_TIMEOUT_REPAIR_INSTALL_STRATEGY
1353
+ ),
1354
+ "agent_setup_timeout_multiplier": DEFAULT_AGENT_SETUP_TIMEOUT_REPAIR_MULTIPLIER,
1355
+ "agent_timeout_multiplier": DEFAULT_AGENT_TIMEOUT_REPAIR_MULTIPLIER,
1356
+ },
1357
+ "required_preflight": [
1358
+ "private_runner_launch_summary.agent_setup_readiness",
1359
+ "private_runner_launch_summary.timeout_multiplier_policy",
1360
+ "codex_cli_existing_in_worker_or_fail_fast_blocker",
1361
+ ],
1362
+ "raw_logs_required": False,
1363
+ "raw_task_text_required": False,
1364
+ },
1365
+ }
1366
+ if failure_class in {
1367
+ "codex_model_access_unsupported_for_account",
1368
+ "codex_model_access_failure_before_solution_attempt",
1369
+ }:
1370
+ blocked_model = _compact_text(agent_model, limit=120)
1371
+ recommended_model = (
1372
+ DEFAULT_CODEX_GOAL_MODE_REPAIR_MODEL_ROUTE
1373
+ if blocked_model != DEFAULT_CODEX_GOAL_MODE_REPAIR_MODEL_ROUTE
1374
+ else "current_local_codex_config_model_after_probe"
1375
+ )
1376
+ return {
1377
+ "repair_priority": "P0",
1378
+ "repair_class": "runner_model_access",
1379
+ "next_action": (
1380
+ "rerun only after selecting a Codex model route proven usable for this account"
1381
+ ),
1382
+ "repair_profile": {
1383
+ "schema_version": "benchmark_repair_profile_v0",
1384
+ "repair_class": "runner_model_access",
1385
+ "blocked_model_route": blocked_model,
1386
+ "recommended_model_route": recommended_model,
1387
+ "required_preflight": ["codex_cli_minimal_model_probe"],
1388
+ "rerun_allowed_after_profile_applied": True,
1389
+ "raw_logs_required": False,
1390
+ "raw_task_text_required": False,
1391
+ },
1392
+ }
1393
+ if (
1394
+ failure_class.startswith("skillsbench_codex_acp_")
1395
+ and round_success_observed
1396
+ and runtime_preflight_passed
1397
+ ):
1398
+ return {
1399
+ "repair_priority": "P0",
1400
+ "repair_class": "skillsbench_codex_acp_post_success_finalization",
1401
+ "next_action": (
1402
+ "separate post-success Codex ACP transport/finalization closeout "
1403
+ "from runtime startup preflight: preserve the blinded round "
1404
+ "reward trace, keep official-score status separate, and rerun "
1405
+ "only after the compact finalization classifier is corrected"
1406
+ ),
1407
+ "repair_profile": {
1408
+ "schema_version": "benchmark_repair_profile_v0",
1409
+ "repair_class": "skillsbench_codex_acp_post_success_finalization",
1410
+ "required_preflight": [
1411
+ "round_reward_trace.success_observed",
1412
+ "codex_acp_runtime_launch_preflight",
1413
+ "skillsbench_compact_failure_class",
1414
+ ],
1415
+ "rerun_allowed_after_profile_applied": True,
1416
+ "raw_logs_required": False,
1417
+ "raw_task_text_required": False,
1418
+ },
1419
+ }
1420
+ if failure_class.startswith("skillsbench_codex_acp_"):
1421
+ return {
1422
+ "repair_priority": "P0",
1423
+ "repair_class": "skillsbench_codex_acp_runtime_preflight",
1424
+ "next_action": (
1425
+ "prove the Codex ACP runtime can start inside the SkillsBench "
1426
+ "sandbox before rerunning or launching treatment; require compact "
1427
+ "dependency and launch preflight evidence instead of a generic "
1428
+ "ACP launch failure"
1429
+ ),
1430
+ "repair_profile": {
1431
+ "schema_version": "benchmark_repair_profile_v0",
1432
+ "repair_class": "skillsbench_codex_acp_runtime_preflight",
1433
+ "required_preflight": [
1434
+ "codex_acp_runtime_dependency_preflight",
1435
+ "codex_acp_runtime_launch_preflight",
1436
+ "skillsbench_compact_failure_class",
1437
+ ],
1438
+ "rerun_allowed_after_profile_applied": True,
1439
+ "raw_logs_required": False,
1440
+ "raw_task_text_required": False,
1441
+ },
1442
+ }
1443
+ if failure_class in {
1444
+ "skillsbench_docker_apt_setup_risk_preflight_blocked",
1445
+ "skillsbench_dockerfile_package_bootstrap_risk_preflight_blocked",
1446
+ }:
1447
+ return {
1448
+ "repair_priority": "P1",
1449
+ "repair_class": "skillsbench_setup_preflight_selection",
1450
+ "next_action": (
1451
+ "select a SkillsBench task without Docker package-bootstrap "
1452
+ "setup risk for the next full baseline/treatment pair, or "
1453
+ "repair the Docker setup route before rerunning this task"
1454
+ ),
1455
+ "repair_profile": {
1456
+ "schema_version": "benchmark_repair_profile_v0",
1457
+ "repair_class": "skillsbench_setup_preflight_selection",
1458
+ "rerun_allowed_after_profile_applied": True,
1459
+ "required_preflight": [
1460
+ "skillsbench_task_setup_preflight",
1461
+ "task_staging.bootstrap_light_preflight_blocked",
1462
+ ],
1463
+ "raw_logs_required": False,
1464
+ "raw_task_text_required": False,
1465
+ },
1466
+ }
1467
+ if failure_class == "skillsbench_verifier_bootstrap_risk_preflight_blocked":
1468
+ return {
1469
+ "repair_priority": "P1",
1470
+ "repair_class": "skillsbench_verifier_bootstrap_preflight_selection",
1471
+ "next_action": (
1472
+ "select a SkillsBench task whose verifier does not require "
1473
+ "network/package bootstrap, or repair/cache the verifier "
1474
+ "bootstrap route before spending another full arm"
1475
+ ),
1476
+ "repair_profile": {
1477
+ "schema_version": "benchmark_repair_profile_v0",
1478
+ "repair_class": "skillsbench_verifier_bootstrap_preflight_selection",
1479
+ "rerun_allowed_after_profile_applied": True,
1480
+ "required_preflight": [
1481
+ "skillsbench_task_setup_preflight",
1482
+ "verifier_bootstrap_risk_detected",
1483
+ "task_staging.verifier_bootstrap_risk_preflight_blocked",
1484
+ ],
1485
+ "raw_logs_required": False,
1486
+ "raw_task_text_required": False,
1487
+ },
1488
+ }
1489
+ if failure_class == "skillsbench_task_source_preflight_blocked":
1490
+ return {
1491
+ "repair_priority": "P1",
1492
+ "repair_class": "skillsbench_task_source_preflight_selection",
1493
+ "next_action": (
1494
+ "select a SkillsBench task from the canonical tasks source, or "
1495
+ "use an explicit sanity-source runner before spending a full "
1496
+ "baseline/treatment arm"
1497
+ ),
1498
+ "repair_profile": {
1499
+ "schema_version": "benchmark_repair_profile_v0",
1500
+ "repair_class": "skillsbench_task_source_preflight_selection",
1501
+ "rerun_allowed_after_profile_applied": True,
1502
+ "required_preflight": [
1503
+ "skillsbench_task_setup_preflight",
1504
+ "canonical_task_present",
1505
+ "nearest_canonical_task_ids",
1506
+ ],
1507
+ "raw_logs_required": False,
1508
+ "raw_task_text_required": False,
1509
+ },
1510
+ }
1511
+ if failure_class == "skillsbench_task_source_excluded":
1512
+ return {
1513
+ "repair_priority": "P1",
1514
+ "repair_class": "skillsbench_task_source_excluded",
1515
+ "next_action": (
1516
+ "exclude this noncanonical SkillsBench source from formal "
1517
+ "87-case scoring, or rerun it only through an explicit "
1518
+ "sanity/source-extra runner"
1519
+ ),
1520
+ "repair_profile": {
1521
+ "schema_version": "benchmark_repair_profile_v0",
1522
+ "repair_class": "skillsbench_task_source_excluded",
1523
+ "rerun_allowed_after_profile_applied": True,
1524
+ "required_preflight": [
1525
+ "skillsbench_task_setup_preflight",
1526
+ "task_excluded_from_formal_tasks",
1527
+ "registry_source_kind=tasks_extra",
1528
+ "registry_excluded=true",
1529
+ ],
1530
+ "raw_logs_required": False,
1531
+ "raw_task_text_required": False,
1532
+ },
1533
+ }
1534
+ if failure_class == "score_missing":
1535
+ return {
1536
+ "repair_priority": "P0",
1537
+ "repair_class": "runner_result_materialization",
1538
+ "next_action": (
1539
+ "repair or ignore the incomplete runner materialization before treating this as case evidence"
1540
+ ),
1541
+ }
1542
+ if failure_class in {
1543
+ "stale_active_job_without_trial_result",
1544
+ "detached_worker_ended_active_without_trial_result",
1545
+ "detached_worker_ended_without_trial_result",
1546
+ "post_launch_compact_result_missing",
1547
+ }:
1548
+ return {
1549
+ "repair_priority": "P0",
1550
+ "repair_class": "runner_result_finalization",
1551
+ "next_action": (
1552
+ "repair Harbor/worker finalization or rerun after proving the "
1553
+ "worker can close with a compact trial result; do not treat "
1554
+ "connectivity or active job state as case success"
1555
+ ),
1556
+ "repair_profile": {
1557
+ "schema_version": "benchmark_repair_profile_v0",
1558
+ "repair_class": "runner_result_finalization",
1559
+ "rerun_allowed_after_profile_applied": True,
1560
+ "required_preflight": [
1561
+ "post_launch_compact_polling_contract",
1562
+ "stale_active_job_reconciliation_marker",
1563
+ "worker_closes_with_trial_result_or_terminal_marker",
1564
+ ],
1565
+ "raw_logs_required": False,
1566
+ "raw_task_text_required": False,
1567
+ "trajectory_required": False,
1568
+ },
1569
+ }
1570
+ if failure_class == "score_failure_unattributed" or failure_scope == "attribution_required":
1571
+ return {
1572
+ "repair_priority": "P0",
1573
+ "repair_class": "verifier_attribution_required",
1574
+ "next_action": (
1575
+ "collect finer compact failure attribution before launching treatment"
1576
+ ),
1577
+ }
1578
+ if failure_class in {
1579
+ "worker_self_validation_official_score_mismatch",
1580
+ "worker_validation_scope_ambiguous_official_score_failure",
1581
+ "worker_bridge_connected_official_score_failure",
1582
+ }:
1583
+ return {
1584
+ "repair_priority": "P0",
1585
+ "repair_class": "worker_verifier_alignment",
1586
+ "next_action": (
1587
+ "align worker self-validation with verifier-facing compact evidence before repeating"
1588
+ ),
1589
+ }
1590
+ if failure_class.startswith("verifier_") or failure_scope == "verifier_or_infra":
1591
+ return {
1592
+ "repair_priority": "P0",
1593
+ "repair_class": "verifier_or_infra_repair",
1594
+ "next_action": "repair verifier or infra attribution before comparing arms",
1595
+ }
1596
+ if failure_class == "agent_timeout_before_solution_completion":
1597
+ return {
1598
+ "repair_priority": "P1",
1599
+ "repair_class": "case_timeout_research",
1600
+ "next_action": (
1601
+ "inspect compact timeout context and decide whether the run needs a private long-horizon timeout tier"
1602
+ ),
1603
+ }
1604
+ if failure_class == "agent_exception_before_solution_completion":
1605
+ return {
1606
+ "repair_priority": "P1",
1607
+ "repair_class": "case_exception_research",
1608
+ "next_action": "inspect compact exception attribution and form a case-level intervention hypothesis",
1609
+ }
1610
+ return {}
1611
+
1612
+
1613
+ def _repair_profile_summary(value: Any) -> str:
1614
+ if not isinstance(value, dict):
1615
+ return ""
1616
+ repair_class = _compact_text(value.get("repair_class"), limit=80)
1617
+ if repair_class == "runner_model_access":
1618
+ blocked = _compact_text(value.get("blocked_model_route"), limit=80) or "unknown"
1619
+ recommended = _compact_text(value.get("recommended_model_route"), limit=80) or "probe_required"
1620
+ return f"blocked_model={blocked}; rerun_model={recommended}; preflight=codex_cli_minimal_model_probe"
1621
+ if repair_class == "runner_setup_timeout":
1622
+ overrides = value.get("required_launch_overrides")
1623
+ if not isinstance(overrides, dict):
1624
+ return "required_launch_overrides=missing"
1625
+ strategy = _compact_text(overrides.get("codex_install_strategy"), limit=80)
1626
+ setup = overrides.get("agent_setup_timeout_multiplier")
1627
+ agent = overrides.get("agent_timeout_multiplier")
1628
+ parts = []
1629
+ if strategy:
1630
+ parts.append(f"codex_install_strategy={strategy}")
1631
+ parts.extend(
1632
+ [
1633
+ f"agent_setup_timeout_multiplier={setup}",
1634
+ f"agent_timeout_multiplier={agent}",
1635
+ ]
1636
+ )
1637
+ disallowed = value.get("disallowed_launch_overrides")
1638
+ if isinstance(disallowed, dict):
1639
+ disallowed_strategy = _compact_text(
1640
+ disallowed.get("codex_install_strategy"),
1641
+ limit=80,
1642
+ )
1643
+ if disallowed_strategy:
1644
+ parts.append(f"disallow={disallowed_strategy}")
1645
+ return "; ".join(parts)
1646
+ if repair_class in {
1647
+ "skillsbench_codex_acp_runtime_preflight",
1648
+ "skillsbench_codex_acp_post_success_finalization",
1649
+ }:
1650
+ required = _compact_list(value.get("required_preflight"), limit=3)
1651
+ return "required_preflight=" + ",".join(required) if required else repair_class
1652
+ if repair_class == "runner_result_finalization":
1653
+ required = _compact_list(value.get("required_preflight"), limit=3)
1654
+ return "required_preflight=" + ",".join(required) if required else repair_class
1655
+ return repair_class
1656
+
1657
+
1658
+ def _case_routing_taxonomy(
1659
+ runs: list[dict[str, Any]],
1660
+ decision: dict[str, Any],
1661
+ ) -> dict[str, Any]:
1662
+ """Summarize case-history routing hints that should survive latest-pair churn."""
1663
+
1664
+ if not runs:
1665
+ return {}
1666
+ repair_counts: dict[str, int] = {}
1667
+ bridge_signal_run_count = 0
1668
+ for run in runs:
1669
+ repair_class = _compact_text(run.get("repair_class"), limit=120)
1670
+ if repair_class:
1671
+ repair_counts[repair_class] = repair_counts.get(repair_class, 0) + 1
1672
+ failure_class = _compact_text(run.get("failure_class"), limit=120)
1673
+ labels = _compact_list(run.get("failure_labels"), limit=20) or _compact_list(
1674
+ run.get("failure_attribution_labels"),
1675
+ limit=20,
1676
+ )
1677
+ if (
1678
+ failure_class == "worker_bridge_connected_official_score_failure"
1679
+ or "worker_bridge_connected_official_score_failure" in labels
1680
+ ):
1681
+ bridge_signal_run_count += 1
1682
+
1683
+ if repair_counts.get("case_exception_research", 0) > 0:
1684
+ return {
1685
+ "class": "case_exception_research",
1686
+ "priority": "P1",
1687
+ "evidence": (
1688
+ f"case_exception_research_count="
1689
+ f"{repair_counts['case_exception_research']}"
1690
+ ),
1691
+ "next_action": (
1692
+ "inspect compact exception attribution and form a case-level "
1693
+ "intervention hypothesis before rerunning this case"
1694
+ ),
1695
+ }
1696
+
1697
+ timeout_count = repair_counts.get("case_timeout_research", 0)
1698
+ if timeout_count >= 2:
1699
+ return {
1700
+ "class": "timeout_tier_policy_candidate",
1701
+ "priority": "P1",
1702
+ "evidence": f"case_timeout_research_count={timeout_count}",
1703
+ "next_action": (
1704
+ "decide the timeout tier and continuation cadence before "
1705
+ "rerunning; separate setup timeout from solver timeout evidence"
1706
+ ),
1707
+ }
1708
+ if timeout_count == 1:
1709
+ return {
1710
+ "class": "case_timeout_research",
1711
+ "priority": "P1",
1712
+ "evidence": "case_timeout_research_count=1",
1713
+ "next_action": (
1714
+ "inspect compact timeout context and decide whether this case "
1715
+ "needs a private long-horizon timeout tier"
1716
+ ),
1717
+ }
1718
+
1719
+ decision_name = _compact_text(decision.get("decision"), limit=120)
1720
+ no_score_uplift = (
1721
+ decision.get("official_score_delta") == 0
1722
+ or decision_name
1723
+ in {
1724
+ "paired_no_score_uplift",
1725
+ "paired_no_score_uplift_case_research_required",
1726
+ }
1727
+ )
1728
+ if bridge_signal_run_count and no_score_uplift:
1729
+ return {
1730
+ "class": "bridge_connected_no_uplift",
1731
+ "priority": "P1",
1732
+ "evidence": (
1733
+ f"worker_bridge_connected_official_score_failure_runs="
1734
+ f"{bridge_signal_run_count}; official_score_delta=0"
1735
+ ),
1736
+ "next_action": (
1737
+ "treat the bridge as connected and analyze case or solution "
1738
+ "quality; do not relaunch this case as a bridge repair"
1739
+ ),
1740
+ }
1741
+ return {}
1742
+
1743
+
1744
+ def _run_status(benchmark_run: dict[str, Any], score: float | int | None) -> str:
1745
+ if _source_schema(benchmark_run) == "terminal_bench_post_launch_materialization_v0":
1746
+ if benchmark_run.get("ready_for_compact_failure_marker") is True:
1747
+ return "blocked"
1748
+ if benchmark_run.get("job_active_without_trial_result") is True:
1749
+ return "running"
1750
+ return "recorded"
1751
+ progress = benchmark_run.get("progress") if isinstance(benchmark_run.get("progress"), dict) else {}
1752
+ running = progress.get("n_running_trials")
1753
+ if isinstance(running, int) and not isinstance(running, bool) and running > 0:
1754
+ return "running"
1755
+ if score is not None or _compact_text(benchmark_run.get("runner_return_status")):
1756
+ return "completed"
1757
+ return "recorded"
1758
+
1759
+
1760
+ def build_benchmark_run_ledger_entry(
1761
+ benchmark_run: dict[str, Any],
1762
+ *,
1763
+ artifact_ref: str | Path | None = None,
1764
+ result_ref: str | Path | None = None,
1765
+ compact_artifact_ref: str | Path | None = None,
1766
+ run_group_id: str | None = None,
1767
+ arm_id: str | None = None,
1768
+ notes: str | None = None,
1769
+ recorded_at: str | None = None,
1770
+ cwd: Path | None = None,
1771
+ ) -> dict[str, Any]:
1772
+ source_schema = _source_schema(benchmark_run)
1773
+ benchmark_id = _compact_text(benchmark_run.get("benchmark_id"), limit=120)
1774
+ if not benchmark_id and source_schema == "terminal_bench_post_launch_materialization_v0":
1775
+ benchmark_id = "terminal-bench@2.0"
1776
+ benchmark_id = benchmark_id or "unknown-benchmark"
1777
+ case_ids = _case_ids(benchmark_run)
1778
+ job_name = _compact_text(benchmark_run.get("job_name"), limit=160)
1779
+ mode = _compact_text(benchmark_run.get("mode"), limit=120)
1780
+ score, passed = _official_score(benchmark_run)
1781
+ bool_fallback_used = official_score_bool_fallback_used(
1782
+ benchmark_run
1783
+ ) and official_score_bool_fallback_allowed(benchmark_run)
1784
+ score_status = _score_status(benchmark_run, score, passed)
1785
+ failure_class = _failure_class(benchmark_run, score)
1786
+ failure_scope = _failure_scope(failure_class, score, passed)
1787
+ resolved_arm_id = _resolved_arm_id(benchmark_run, arm_id)
1788
+ identity_artifact = _relative_ref(artifact_ref, cwd=cwd)
1789
+ identity_result = _relative_ref(result_ref, cwd=cwd)
1790
+ identity_compact_artifact = _relative_ref(compact_artifact_ref, cwd=cwd)
1791
+ artifact = _public_ledger_artifact_ref(artifact_ref, cwd=cwd)
1792
+ result = _public_ledger_artifact_ref(result_ref, cwd=cwd)
1793
+ compact_artifact = _public_ledger_artifact_ref(compact_artifact_ref, cwd=cwd)
1794
+ resolved_run_group_id = _compact_text(run_group_id, limit=160) or job_name
1795
+ identity = "|".join(
1796
+ str(part)
1797
+ for part in (
1798
+ benchmark_id,
1799
+ case_ids[0],
1800
+ resolved_arm_id,
1801
+ resolved_run_group_id,
1802
+ job_name,
1803
+ identity_artifact
1804
+ or identity_result
1805
+ or identity_compact_artifact
1806
+ or artifact
1807
+ or result
1808
+ or compact_artifact
1809
+ or "",
1810
+ )
1811
+ )
1812
+ run_id = hashlib.sha1(identity.encode("utf-8")).hexdigest()[:12]
1813
+ agent = benchmark_run.get("agent") if isinstance(benchmark_run.get("agent"), dict) else {}
1814
+ agent_model = _compact_text(agent.get("model"), limit=120)
1815
+ model_control = (
1816
+ benchmark_run.get("model_control")
1817
+ if isinstance(benchmark_run.get("model_control"), dict)
1818
+ else {}
1819
+ )
1820
+ model_control_status = _compact_text(
1821
+ model_control.get("control_status"),
1822
+ limit=120,
1823
+ )
1824
+ model_actual_verified = model_control.get("actual_model_verified")
1825
+ model_warning_labels = _compact_list(
1826
+ model_control.get("warning_labels")
1827
+ or benchmark_run.get("runner_warning_labels"),
1828
+ limit=6,
1829
+ )
1830
+ round_reward_trace = (
1831
+ benchmark_run.get("round_reward_trace")
1832
+ if isinstance(benchmark_run.get("round_reward_trace"), dict)
1833
+ else {}
1834
+ )
1835
+ round_rewards = _compact_round_reward_records(
1836
+ round_reward_trace.get("records") if isinstance(round_reward_trace, dict) else []
1837
+ )
1838
+ first_success_round = (
1839
+ _compact_positive_int(round_reward_trace.get("first_success_round"))
1840
+ if isinstance(round_reward_trace, dict)
1841
+ else None
1842
+ )
1843
+ if first_success_round is None:
1844
+ for record in round_rewards:
1845
+ if record.get("passed") is True:
1846
+ first_success_round = int(record["agent_round"])
1847
+ break
1848
+ max_rounds_budget = (
1849
+ round_reward_trace.get("max_rounds_budget")
1850
+ if isinstance(round_reward_trace, dict)
1851
+ else None
1852
+ )
1853
+ round_reward_stats = _round_reward_best_stats(round_rewards)
1854
+ final_round = _compact_positive_int(round_reward_trace.get("final_round"))
1855
+ final_round_reward = _compact_number(round_reward_trace.get("final_round_reward"))
1856
+ final_round_passed = round_reward_trace.get("final_round_passed")
1857
+ best_reward_round = _compact_positive_int(
1858
+ round_reward_trace.get("best_reward_round")
1859
+ )
1860
+ best_round_reward = _compact_number(round_reward_trace.get("best_round_reward"))
1861
+ best_round_passed = round_reward_trace.get("best_round_passed")
1862
+ best_round_is_final = round_reward_trace.get("best_round_is_final")
1863
+ declared_done_round = _compact_positive_int(
1864
+ round_reward_trace.get("declared_done_round")
1865
+ )
1866
+ declared_done_score = _compact_number(round_reward_trace.get("declared_done_score"))
1867
+ agent_declared_done = round_reward_trace.get("agent_declared_done")
1868
+ if final_round is None:
1869
+ final_round = round_reward_stats.get("final_round")
1870
+ if final_round_reward is None:
1871
+ final_round_reward = round_reward_stats.get("final_round_reward")
1872
+ if not isinstance(final_round_passed, bool):
1873
+ final_round_passed = round_reward_stats.get("final_round_passed")
1874
+ if best_reward_round is None:
1875
+ best_reward_round = round_reward_stats.get("best_reward_round")
1876
+ if best_round_reward is None:
1877
+ best_round_reward = round_reward_stats.get("best_round_reward")
1878
+ if not isinstance(best_round_passed, bool):
1879
+ best_round_passed = round_reward_stats.get("best_round_passed")
1880
+ if not isinstance(best_round_is_final, bool):
1881
+ best_round_is_final = round_reward_stats.get("best_round_is_final")
1882
+ native_goal_worker_contract = (
1883
+ benchmark_run.get("native_goal_worker_contract")
1884
+ if isinstance(benchmark_run.get("native_goal_worker_contract"), dict)
1885
+ else {}
1886
+ )
1887
+ raw_app_server_goal_round_semantics = (
1888
+ benchmark_run.get("app_server_goal_round_semantics")
1889
+ if isinstance(benchmark_run.get("app_server_goal_round_semantics"), dict)
1890
+ else {}
1891
+ )
1892
+ if not raw_app_server_goal_round_semantics and (
1893
+ benchmark_run.get("route") == "codex-app-server-goal-baseline"
1894
+ or native_goal_worker_contract.get("required") is True
1895
+ ):
1896
+ raw_app_server_goal_round_semantics = native_goal_worker_contract
1897
+ app_server_goal_round_semantics = _compact_app_server_goal_round_semantics(
1898
+ raw_app_server_goal_round_semantics
1899
+ )
1900
+ if (
1901
+ not isinstance(max_rounds_budget, int)
1902
+ or isinstance(max_rounds_budget, bool)
1903
+ ) and (
1904
+ isinstance(app_server_goal_round_semantics.get("benchflow_max_rounds_budget"), int)
1905
+ and not isinstance(
1906
+ app_server_goal_round_semantics.get("benchflow_max_rounds_budget"), bool
1907
+ )
1908
+ ):
1909
+ max_rounds_budget = app_server_goal_round_semantics.get(
1910
+ "benchflow_max_rounds_budget"
1911
+ )
1912
+ runner_prerequisites = (
1913
+ benchmark_run.get("runner_prerequisites")
1914
+ if isinstance(benchmark_run.get("runner_prerequisites"), dict)
1915
+ else {}
1916
+ )
1917
+ runner_failure = (
1918
+ benchmark_run.get("runner_failure")
1919
+ if isinstance(benchmark_run.get("runner_failure"), dict)
1920
+ else {}
1921
+ )
1922
+ verifier_reward_artifact_recovery = (
1923
+ benchmark_run.get("verifier_reward_artifact_recovery")
1924
+ if isinstance(benchmark_run.get("verifier_reward_artifact_recovery"), dict)
1925
+ else {}
1926
+ )
1927
+ validation = (
1928
+ benchmark_run.get("validation")
1929
+ if isinstance(benchmark_run.get("validation"), dict)
1930
+ else {}
1931
+ )
1932
+ runtime_preflight_passed = _codex_acp_runtime_preflight_passed(
1933
+ runner_prerequisites
1934
+ )
1935
+ round_success_observed = (
1936
+ round_reward_trace.get("success_observed")
1937
+ if isinstance(round_reward_trace, dict)
1938
+ and isinstance(round_reward_trace.get("success_observed"), bool)
1939
+ else (first_success_round is not None)
1940
+ )
1941
+ repair_route = _repair_route(
1942
+ _repair_route_failure_class(benchmark_run, failure_class),
1943
+ failure_scope,
1944
+ agent_model=agent_model,
1945
+ round_success_observed=round_success_observed,
1946
+ runtime_preflight_passed=runtime_preflight_passed,
1947
+ )
1948
+
1949
+ entry: dict[str, Any] = {
1950
+ "run_id": run_id,
1951
+ "recorded_at": recorded_at or _now_local_iso(),
1952
+ "benchmark_id": benchmark_id,
1953
+ "case_id": case_ids[0],
1954
+ "case_ids": case_ids,
1955
+ "run_group_id": resolved_run_group_id or run_id,
1956
+ "arm_id": resolved_arm_id,
1957
+ "mode": mode,
1958
+ "route": _compact_text(benchmark_run.get("route"), limit=120),
1959
+ "job_name": job_name,
1960
+ "status": _run_status(benchmark_run, score),
1961
+ "score_status": score_status,
1962
+ "official_score": score,
1963
+ "official_passed": passed,
1964
+ "official_score_bool_fallback_used": bool_fallback_used,
1965
+ "first_success_round": first_success_round,
1966
+ "final_round": final_round,
1967
+ "final_round_reward": final_round_reward,
1968
+ "final_round_passed": final_round_passed,
1969
+ "best_reward_round": best_reward_round,
1970
+ "best_round_reward": best_round_reward,
1971
+ "best_round_passed": best_round_passed,
1972
+ "best_round_is_final": best_round_is_final,
1973
+ "agent_declared_done": agent_declared_done
1974
+ if isinstance(agent_declared_done, bool)
1975
+ else False,
1976
+ "declared_done_round": declared_done_round,
1977
+ "declared_done_score": declared_done_score,
1978
+ "loop_score_policy": _compact_text(
1979
+ round_reward_trace.get("loop_score_policy"),
1980
+ limit=120,
1981
+ )
1982
+ or ("best_round_for_offline_controller_analysis" if round_rewards else ""),
1983
+ "official_score_policy": _compact_text(
1984
+ round_reward_trace.get("official_score_policy"),
1985
+ limit=120,
1986
+ )
1987
+ or ("final_workspace_official_result" if round_rewards else ""),
1988
+ "round_rewards": round_rewards,
1989
+ "round_reward_count": len(round_rewards),
1990
+ "round_success_observed": round_success_observed,
1991
+ "codex_acp_runtime_preflight_passed": runtime_preflight_passed,
1992
+ "max_rounds_budget": max_rounds_budget
1993
+ if isinstance(max_rounds_budget, int) and not isinstance(max_rounds_budget, bool)
1994
+ else None,
1995
+ "app_server_goal_round_semantics": app_server_goal_round_semantics or None,
1996
+ "native_goal_session_policy": _compact_text(
1997
+ app_server_goal_round_semantics.get("session_policy"), limit=120
1998
+ ),
1999
+ "max_rounds_budget_applies_to": _compact_text(
2000
+ app_server_goal_round_semantics.get("max_rounds_budget_applies_to"),
2001
+ limit=140,
2002
+ ),
2003
+ "native_goal_initial_turn_budget": _compact_nonnegative_int(
2004
+ app_server_goal_round_semantics.get("initial_goal_turn_budget")
2005
+ ),
2006
+ "native_goal_same_thread_followup_budget": _compact_nonnegative_int(
2007
+ app_server_goal_round_semantics.get("same_thread_followup_budget")
2008
+ ),
2009
+ "native_goal_independent_attempt_budget": _compact_nonnegative_int(
2010
+ app_server_goal_round_semantics.get("independent_attempt_budget")
2011
+ ),
2012
+ "native_goal_fresh_thread_per_independent_attempt": (
2013
+ app_server_goal_round_semantics.get(
2014
+ "fresh_goal_thread_per_independent_attempt"
2015
+ )
2016
+ if isinstance(
2017
+ app_server_goal_round_semantics.get(
2018
+ "fresh_goal_thread_per_independent_attempt"
2019
+ ),
2020
+ bool,
2021
+ )
2022
+ else None
2023
+ ),
2024
+ "native_goal_official_reward_feedback_forwarded_to_worker": (
2025
+ app_server_goal_round_semantics.get(
2026
+ "official_reward_feedback_forwarded_to_worker"
2027
+ )
2028
+ if isinstance(
2029
+ app_server_goal_round_semantics.get(
2030
+ "official_reward_feedback_forwarded_to_worker"
2031
+ ),
2032
+ bool,
2033
+ )
2034
+ else None
2035
+ ),
2036
+ "native_goal_verifier_output_forwarded_to_worker": (
2037
+ app_server_goal_round_semantics.get(
2038
+ "verifier_output_forwarded_to_worker"
2039
+ )
2040
+ if isinstance(
2041
+ app_server_goal_round_semantics.get(
2042
+ "verifier_output_forwarded_to_worker"
2043
+ ),
2044
+ bool,
2045
+ )
2046
+ else None
2047
+ ),
2048
+ "official_feedback_blinded": round_reward_trace.get("official_feedback_blinded")
2049
+ if isinstance(round_reward_trace, dict)
2050
+ and isinstance(round_reward_trace.get("official_feedback_blinded"), bool)
2051
+ else None,
2052
+ "reward_feedback_forwarded": round_reward_trace.get("reward_feedback_forwarded")
2053
+ if isinstance(round_reward_trace, dict)
2054
+ and isinstance(round_reward_trace.get("reward_feedback_forwarded"), bool)
2055
+ else None,
2056
+ "failure_class": failure_class,
2057
+ "failure_scope": failure_scope,
2058
+ "score_failure_attribution": _compact_text(
2059
+ benchmark_run.get("score_failure_attribution"),
2060
+ limit=120,
2061
+ ),
2062
+ "failure_labels": _compact_list(
2063
+ benchmark_run.get("failure_attribution_labels"),
2064
+ limit=8,
2065
+ ),
2066
+ "runner_return_status": _compact_text(
2067
+ benchmark_run.get("runner_return_status"),
2068
+ limit=120,
2069
+ ),
2070
+ "runner_score_recovered_from_verifier_artifact": (
2071
+ runner_failure.get("score_recovered_from_verifier_artifact")
2072
+ if isinstance(
2073
+ runner_failure.get("score_recovered_from_verifier_artifact"),
2074
+ bool,
2075
+ )
2076
+ else None
2077
+ ),
2078
+ "verifier_reward_artifact_recovery_status": _compact_text(
2079
+ verifier_reward_artifact_recovery.get("status"),
2080
+ limit=120,
2081
+ ),
2082
+ "verifier_reward_artifact_recovered": (
2083
+ validation.get("verifier_reward_artifact_recovered")
2084
+ if isinstance(validation.get("verifier_reward_artifact_recovered"), bool)
2085
+ else (
2086
+ verifier_reward_artifact_recovery.get("reward_present")
2087
+ if isinstance(
2088
+ verifier_reward_artifact_recovery.get("reward_present"),
2089
+ bool,
2090
+ )
2091
+ else None
2092
+ )
2093
+ ),
2094
+ "official_result_json_materialized": (
2095
+ verifier_reward_artifact_recovery.get("official_result_json_materialized")
2096
+ if isinstance(
2097
+ verifier_reward_artifact_recovery.get("official_result_json_materialized"),
2098
+ bool,
2099
+ )
2100
+ else (
2101
+ validation.get("official_result_json_materialized")
2102
+ if isinstance(validation.get("official_result_json_materialized"), bool)
2103
+ else None
2104
+ )
2105
+ ),
2106
+ "setup_blockers": _compact_list(
2107
+ benchmark_run.get("worker_setup_diagnostic_blockers"),
2108
+ limit=4,
2109
+ )
2110
+ or _compact_list(benchmark_run.get("worker_startup_blockers"), limit=4),
2111
+ "loopx_inside_case": benchmark_run.get("loopx_inside_case")
2112
+ if isinstance(benchmark_run.get("loopx_inside_case"), bool)
2113
+ else None,
2114
+ "worker_bridge_status": _compact_text(
2115
+ benchmark_run.get("worker_bridge_materialization_status"),
2116
+ limit=120,
2117
+ ),
2118
+ "loopx_prompt_driven_lifecycle_observed": benchmark_run.get(
2119
+ "loopx_prompt_driven_lifecycle_observed"
2120
+ )
2121
+ if isinstance(benchmark_run.get("loopx_prompt_driven_lifecycle_observed"), bool)
2122
+ else None,
2123
+ "worker_loopx_cli_call_total": benchmark_run.get("worker_loopx_cli_call_total")
2124
+ if isinstance(benchmark_run.get("worker_loopx_cli_call_total"), int)
2125
+ and not isinstance(benchmark_run.get("worker_loopx_cli_call_total"), bool)
2126
+ else None,
2127
+ "loopx_prompt_driven_case_cli_call_count": benchmark_run.get(
2128
+ "loopx_prompt_driven_case_cli_call_count"
2129
+ )
2130
+ if isinstance(
2131
+ benchmark_run.get("loopx_prompt_driven_case_cli_call_count"),
2132
+ int,
2133
+ )
2134
+ and not isinstance(
2135
+ benchmark_run.get("loopx_prompt_driven_case_cli_call_count"),
2136
+ bool,
2137
+ )
2138
+ else None,
2139
+ "agent_model": agent_model,
2140
+ "model_control_status": model_control_status,
2141
+ "actual_model_verified": model_actual_verified
2142
+ if isinstance(model_actual_verified, bool)
2143
+ else None,
2144
+ "model_warning_labels": model_warning_labels,
2145
+ "submit_eligible": benchmark_run.get("submit_eligible")
2146
+ if isinstance(benchmark_run.get("submit_eligible"), bool)
2147
+ else None,
2148
+ "leaderboard_evidence": benchmark_run.get("leaderboard_evidence")
2149
+ if isinstance(benchmark_run.get("leaderboard_evidence"), bool)
2150
+ else None,
2151
+ "source_event_schema": source_schema,
2152
+ }
2153
+ product_mode_lifecycle_contract = _compact_product_mode_lifecycle_contract(
2154
+ benchmark_run.get("product_mode_lifecycle_contract")
2155
+ )
2156
+ if product_mode_lifecycle_contract:
2157
+ entry["product_mode_lifecycle_contract"] = product_mode_lifecycle_contract
2158
+ solution_quality_signals = _ledger_skillsbench_solution_quality_signals(
2159
+ benchmark_run,
2160
+ benchmark_id=benchmark_id,
2161
+ )
2162
+ if solution_quality_signals:
2163
+ entry["solution_quality_signals"] = solution_quality_signals
2164
+ operator_simulator_run = _compact_operator_simulator_run(
2165
+ benchmark_run.get("operator_simulator_run")
2166
+ )
2167
+ if operator_simulator_run:
2168
+ entry["operator_simulator_run"] = operator_simulator_run
2169
+ live_worker_phase = compact_benchmark_live_worker_phase_from_run(benchmark_run)
2170
+ if live_worker_phase:
2171
+ entry["benchmark_live_worker_phase"] = live_worker_phase
2172
+ attempt_accounting = (
2173
+ benchmark_run.get("attempt_accounting")
2174
+ if isinstance(benchmark_run.get("attempt_accounting"), dict)
2175
+ else {}
2176
+ )
2177
+ if (
2178
+ not attempt_accounting
2179
+ and source_schema == "terminal_bench_post_launch_materialization_v0"
2180
+ ):
2181
+ marker = (
2182
+ benchmark_run.get("compact_failure_marker")
2183
+ if isinstance(benchmark_run.get("compact_failure_marker"), dict)
2184
+ else {}
2185
+ )
2186
+ attempt_accounting = (
2187
+ marker.get("attempt_accounting")
2188
+ if isinstance(marker.get("attempt_accounting"), dict)
2189
+ else {}
2190
+ )
2191
+ has_official_bool_score = bool_fallback_used
2192
+ if attempt_accounting:
2193
+ for source_field, entry_field in (
2194
+ ("lifecycle_phase", "attempt_lifecycle_phase"),
2195
+ ("failure_label", "attempt_failure_label"),
2196
+ ("failure_class", "attempt_failure_class"),
2197
+ ):
2198
+ text = _compact_text(attempt_accounting.get(source_field), limit=120)
2199
+ if text:
2200
+ entry[entry_field] = text
2201
+ for field in (
2202
+ "launcher_attempt_countable",
2203
+ "case_attempt_countable",
2204
+ "solver_attempt_countable",
2205
+ "verifier_attempt_countable",
2206
+ "official_score_attempt_countable",
2207
+ ):
2208
+ if isinstance(attempt_accounting.get(field), bool):
2209
+ entry[field] = (
2210
+ True
2211
+ if field == "official_score_attempt_countable"
2212
+ and has_official_bool_score
2213
+ else attempt_accounting[field]
2214
+ )
2215
+ for field in (
2216
+ "launcher_attempt_countable",
2217
+ "case_attempt_countable",
2218
+ "solver_attempt_countable",
2219
+ "verifier_attempt_countable",
2220
+ "official_score_attempt_countable",
2221
+ ):
2222
+ if field not in entry and isinstance(benchmark_run.get(field), bool):
2223
+ entry[field] = (
2224
+ True
2225
+ if field == "official_score_attempt_countable"
2226
+ and has_official_bool_score
2227
+ else benchmark_run[field]
2228
+ )
2229
+ if source_schema == "terminal_bench_post_launch_materialization_v0":
2230
+ marker = (
2231
+ benchmark_run.get("compact_failure_marker")
2232
+ if isinstance(benchmark_run.get("compact_failure_marker"), dict)
2233
+ else {}
2234
+ )
2235
+ entry.update(
2236
+ {
2237
+ "post_launch_first_blocker": _compact_text(
2238
+ benchmark_run.get("first_blocker"),
2239
+ limit=120,
2240
+ ),
2241
+ "compact_monitor_class": _compact_text(
2242
+ benchmark_run.get("compact_monitor_class"),
2243
+ limit=120,
2244
+ ),
2245
+ "job_active_without_trial_result": benchmark_run.get(
2246
+ "job_active_without_trial_result"
2247
+ )
2248
+ if isinstance(
2249
+ benchmark_run.get("job_active_without_trial_result"),
2250
+ bool,
2251
+ )
2252
+ else None,
2253
+ "job_stale_active_without_trial_result": benchmark_run.get(
2254
+ "job_stale_active_without_trial_result"
2255
+ )
2256
+ if isinstance(
2257
+ benchmark_run.get("job_stale_active_without_trial_result"),
2258
+ bool,
2259
+ )
2260
+ else None,
2261
+ "stale_active_reconcile_requested": benchmark_run.get(
2262
+ "stale_active_reconcile_requested"
2263
+ )
2264
+ if isinstance(
2265
+ benchmark_run.get("stale_active_reconcile_requested"),
2266
+ bool,
2267
+ )
2268
+ else None,
2269
+ "compact_failure_evidence_kind": _compact_text(
2270
+ marker.get("evidence_kind"),
2271
+ limit=120,
2272
+ ),
2273
+ "ledger_attempt_kind": _compact_text(
2274
+ marker.get("ledger_attempt_kind"),
2275
+ limit=120,
2276
+ ),
2277
+ "terminal_closeout": marker.get("terminal_closeout")
2278
+ if isinstance(marker.get("terminal_closeout"), bool)
2279
+ else None,
2280
+ "case_attempt_countable": marker.get("case_attempt_countable")
2281
+ if isinstance(marker.get("case_attempt_countable"), bool)
2282
+ else None,
2283
+ "benchmark_budget_countable": marker.get(
2284
+ "benchmark_budget_countable"
2285
+ )
2286
+ if isinstance(marker.get("benchmark_budget_countable"), bool)
2287
+ else None,
2288
+ }
2289
+ )
2290
+ entry.update(repair_route)
2291
+ task_setup_preflight = _compact_task_setup_preflight(
2292
+ benchmark_run.get("task_setup_preflight")
2293
+ )
2294
+ if task_setup_preflight:
2295
+ entry["task_setup_preflight"] = task_setup_preflight
2296
+ task_staging = _compact_task_staging(benchmark_run.get("task_staging"))
2297
+ if task_staging:
2298
+ entry["task_staging"] = task_staging
2299
+ compose_setup_diagnostic = _compact_compose_setup_diagnostic(
2300
+ benchmark_run.get("compose_setup_diagnostic")
2301
+ )
2302
+ if compose_setup_diagnostic:
2303
+ entry["compose_setup_diagnostic"] = compose_setup_diagnostic
2304
+ refs: dict[str, str] = {}
2305
+ if artifact:
2306
+ refs["artifact_ref"] = artifact
2307
+ if result:
2308
+ refs["result_ref"] = result
2309
+ if compact_artifact:
2310
+ refs["compact_artifact_ref"] = compact_artifact
2311
+ if refs:
2312
+ entry["artifact_refs"] = refs
2313
+ note = _compact_text(notes, limit=220)
2314
+ if note:
2315
+ entry["notes"] = note
2316
+ countability = benchmark_run_official_score_countability(entry)
2317
+ entry["official_score_countable"] = countability["countable"]
2318
+ entry["official_score_countability_reason"] = countability["reason"]
2319
+ if countability["countable"] is True and countability.get("score") is not None:
2320
+ entry["countable_score"] = countability["score"]
2321
+ return {key: value for key, value in entry.items() if value not in (None, "", [])}
2322
+
2323
+
2324
+ def _entry_is_public_ledger_closeout(entry: dict[str, Any]) -> bool:
2325
+ """Return whether a compact run is terminal enough for the public run ledger."""
2326
+
2327
+ if entry.get("status") == "running":
2328
+ return False
2329
+ return True
2330
+
2331
+
2332
+ def _ledger_run_archived(run: dict[str, Any]) -> bool:
2333
+ return _compact_text(run.get("archive_state"), limit=40) == "archived"
2334
+
2335
+
2336
+ def _active_ledger_runs(runs: list[dict[str, Any]]) -> list[dict[str, Any]]:
2337
+ return [run for run in runs if not _ledger_run_archived(run)]
2338
+
2339
+
2340
+ def _archived_ledger_runs(runs: list[dict[str, Any]]) -> list[dict[str, Any]]:
2341
+ return [run for run in runs if _ledger_run_archived(run)]
2342
+
2343
+
2344
+ def _empty_ledger() -> dict[str, Any]:
2345
+ return {
2346
+ "schema_version": BENCHMARK_RUN_LEDGER_SCHEMA_VERSION,
2347
+ "updated_at": _now_local_iso(),
2348
+ "update_policy": {
2349
+ "source_of_truth": "benchmark_run_v0 compact run events",
2350
+ "raw_logs_recorded": False,
2351
+ "raw_task_text_recorded": False,
2352
+ "absolute_paths_recorded": False,
2353
+ "update_rule": "upsert one run entry when a benchmark case is ingested or closed",
2354
+ },
2355
+ "benchmarks": {},
2356
+ }
2357
+
2358
+
2359
+ def load_benchmark_run_ledger(path: str | Path) -> dict[str, Any]:
2360
+ ledger_path = Path(path)
2361
+ if not ledger_path.exists():
2362
+ return _empty_ledger()
2363
+ payload = json.loads(ledger_path.read_text(encoding="utf-8"))
2364
+ if not isinstance(payload, dict):
2365
+ raise ValueError("benchmark run ledger must be a JSON object")
2366
+ if payload.get("schema_version") != BENCHMARK_RUN_LEDGER_SCHEMA_VERSION:
2367
+ raise ValueError(
2368
+ f"benchmark run ledger must use schema_version={BENCHMARK_RUN_LEDGER_SCHEMA_VERSION}"
2369
+ )
2370
+ payload.setdefault("benchmarks", {})
2371
+ return _normalize_benchmark_run_ledger(payload)
2372
+
2373
+
2374
+ def _normalize_ledger_run(run: dict[str, Any], *, fallback_benchmark_id: str) -> dict[str, Any]:
2375
+ normalized = dict(run)
2376
+ benchmark_id = _compact_text(
2377
+ normalized.get("benchmark_id") or fallback_benchmark_id,
2378
+ limit=120,
2379
+ )
2380
+ job_name = _compact_text(normalized.get("job_name"), limit=160)
2381
+ case_id = _compact_text(normalized.get("case_id"), limit=160)
2382
+ if job_name and (
2383
+ not case_id
2384
+ or case_id.startswith("terminal_bench_")
2385
+ or case_id == "unknown-case"
2386
+ ):
2387
+ parsed_case = _terminal_bench_case_id_from_job_name(
2388
+ benchmark_id=benchmark_id,
2389
+ job_name=job_name,
2390
+ )
2391
+ if parsed_case:
2392
+ normalized["case_id"] = parsed_case
2393
+ normalized["case_ids"] = [parsed_case]
2394
+ resolved_arm = _resolved_arm_id(normalized, _compact_text(normalized.get("arm_id"), limit=120))
2395
+ if resolved_arm:
2396
+ normalized["arm_id"] = resolved_arm
2397
+ normalized["benchmark_id"] = benchmark_id
2398
+ repair_route = _repair_route(
2399
+ _repair_route_failure_class(
2400
+ normalized,
2401
+ _compact_text(normalized.get("failure_class"), limit=120),
2402
+ ),
2403
+ _compact_text(normalized.get("failure_scope"), limit=80),
2404
+ agent_model=_compact_text(normalized.get("agent_model"), limit=120),
2405
+ round_success_observed=normalized.get("round_success_observed") is True
2406
+ or normalized.get("first_success_round") is not None
2407
+ or normalized.get("best_round_passed") is True
2408
+ or normalized.get("final_round_passed") is True,
2409
+ runtime_preflight_passed=(
2410
+ normalized.get("codex_acp_runtime_preflight_passed") is True
2411
+ ),
2412
+ )
2413
+ for key in ("repair_priority", "repair_class", "next_action", "repair_profile"):
2414
+ normalized.pop(key, None)
2415
+ normalized.update(repair_route)
2416
+ countability = benchmark_run_official_score_countability(normalized)
2417
+ normalized["official_score_countable"] = countability["countable"]
2418
+ normalized["official_score_countability_reason"] = countability["reason"]
2419
+ if countability["countable"] is True and countability.get("score") is not None:
2420
+ normalized["countable_score"] = countability["score"]
2421
+ else:
2422
+ normalized.pop("countable_score", None)
2423
+ archive_state = _compact_text(normalized.get("archive_state"), limit=40)
2424
+ if archive_state == "archived":
2425
+ normalized["archive_state"] = "archived"
2426
+ for key, limit in (
2427
+ ("archive_reason", 220),
2428
+ ("archive_batch_id", 120),
2429
+ ("archived_at", 80),
2430
+ ):
2431
+ value = _compact_text(normalized.get(key), limit=limit)
2432
+ if value:
2433
+ normalized[key] = value
2434
+ else:
2435
+ normalized.pop(key, None)
2436
+ else:
2437
+ for key in ("archive_state", "archive_reason", "archive_batch_id", "archived_at"):
2438
+ normalized.pop(key, None)
2439
+ live_worker_phase = compact_benchmark_live_worker_phase_from_run(normalized)
2440
+ if live_worker_phase:
2441
+ normalized["benchmark_live_worker_phase"] = live_worker_phase
2442
+ else:
2443
+ normalized.pop("benchmark_live_worker_phase", None)
2444
+ refs = normalized.get("artifact_refs")
2445
+ if isinstance(refs, dict):
2446
+ safe_refs: dict[str, str] = {}
2447
+ for key in ("artifact_ref", "result_ref", "compact_artifact_ref"):
2448
+ value = refs.get(key)
2449
+ if not isinstance(value, str):
2450
+ continue
2451
+ safe_ref = _public_ledger_artifact_ref(value)
2452
+ if safe_ref:
2453
+ safe_refs[key] = safe_ref
2454
+ if safe_refs:
2455
+ normalized["artifact_refs"] = safe_refs
2456
+ else:
2457
+ normalized.pop("artifact_refs", None)
2458
+ return normalized
2459
+
2460
+
2461
+ def _normalize_benchmark_run_ledger(ledger: dict[str, Any]) -> dict[str, Any]:
2462
+ benchmarks = ledger.get("benchmarks")
2463
+ if not isinstance(benchmarks, dict):
2464
+ ledger["benchmarks"] = {}
2465
+ return ledger
2466
+ normalized_benchmarks: dict[str, Any] = {}
2467
+ for benchmark_id, benchmark in benchmarks.items():
2468
+ if not isinstance(benchmark, dict):
2469
+ continue
2470
+ normalized_benchmark = normalized_benchmarks.setdefault(
2471
+ benchmark_id,
2472
+ {"benchmark_id": benchmark_id, "cases": {}},
2473
+ )
2474
+ cases = benchmark.get("cases")
2475
+ if not isinstance(cases, dict):
2476
+ continue
2477
+ for fallback_case_id, case in cases.items():
2478
+ if not isinstance(case, dict):
2479
+ continue
2480
+ runs = case.get("runs")
2481
+ if not isinstance(runs, list):
2482
+ continue
2483
+ for run in runs:
2484
+ if not isinstance(run, dict):
2485
+ continue
2486
+ normalized_run = _normalize_ledger_run(
2487
+ run,
2488
+ fallback_benchmark_id=str(benchmark_id),
2489
+ )
2490
+ case_id = _compact_text(
2491
+ normalized_run.get("case_id") or fallback_case_id,
2492
+ limit=160,
2493
+ )
2494
+ normalized_case = normalized_benchmark["cases"].setdefault(
2495
+ case_id,
2496
+ {"case_id": case_id, "runs": []},
2497
+ )
2498
+ normalized_case["runs"].append(normalized_run)
2499
+ for benchmark in normalized_benchmarks.values():
2500
+ cases = benchmark.get("cases")
2501
+ if not isinstance(cases, dict):
2502
+ continue
2503
+ for case in cases.values():
2504
+ if not isinstance(case, dict):
2505
+ continue
2506
+ runs = [run for run in case.get("runs", []) if isinstance(run, dict)]
2507
+ deduped: dict[str, dict[str, Any]] = {}
2508
+ for run in runs:
2509
+ run_id = _compact_text(run.get("run_id"), limit=80)
2510
+ deduped[run_id or json.dumps(run, sort_keys=True)] = run
2511
+ ordered_runs = sorted(
2512
+ deduped.values(),
2513
+ key=lambda run: (str(run.get("recorded_at", "")), str(run.get("run_id", ""))),
2514
+ )
2515
+ case["runs"] = ordered_runs
2516
+ case["active_run_count"] = len(_active_ledger_runs(ordered_runs))
2517
+ case["archived_run_count"] = len(_archived_ledger_runs(ordered_runs))
2518
+ case["latest_decision"] = _case_decision(case)
2519
+ benchmark["case_count"] = len(cases)
2520
+ benchmark["run_count"] = sum(
2521
+ len(value.get("runs", []))
2522
+ for value in cases.values()
2523
+ if isinstance(value, dict)
2524
+ )
2525
+ benchmark["active_case_count"] = sum(
2526
+ 1
2527
+ for value in cases.values()
2528
+ if isinstance(value, dict)
2529
+ and value.get("active_run_count", 0)
2530
+ )
2531
+ benchmark["active_run_count"] = sum(
2532
+ int(value.get("active_run_count", 0))
2533
+ for value in cases.values()
2534
+ if isinstance(value, dict)
2535
+ )
2536
+ benchmark["archived_run_count"] = sum(
2537
+ int(value.get("archived_run_count", 0))
2538
+ for value in cases.values()
2539
+ if isinstance(value, dict)
2540
+ )
2541
+ ledger["benchmarks"] = normalized_benchmarks
2542
+ return ledger
2543
+
2544
+
2545
+ class _LedgerWriteLock:
2546
+ def __init__(self, path: Path, *, timeout_seconds: float = 10.0) -> None:
2547
+ self.path = path
2548
+ self.lock_path = path.with_suffix(path.suffix + ".lock")
2549
+ self.timeout_seconds = timeout_seconds
2550
+ self._fd: int | None = None
2551
+
2552
+ def __enter__(self) -> "_LedgerWriteLock":
2553
+ self.path.parent.mkdir(parents=True, exist_ok=True)
2554
+ deadline = time.monotonic() + self.timeout_seconds
2555
+ while True:
2556
+ try:
2557
+ self._fd = os.open(
2558
+ self.lock_path,
2559
+ os.O_CREAT | os.O_EXCL | os.O_WRONLY,
2560
+ )
2561
+ os.write(self._fd, str(os.getpid()).encode("utf-8"))
2562
+ return self
2563
+ except FileExistsError:
2564
+ if time.monotonic() >= deadline:
2565
+ raise TimeoutError(f"timed out waiting for ledger lock: {self.lock_path}")
2566
+ time.sleep(0.05)
2567
+
2568
+ def __exit__(self, exc_type: object, exc: object, tb: object) -> None:
2569
+ if self._fd is not None:
2570
+ os.close(self._fd)
2571
+ self._fd = None
2572
+ try:
2573
+ self.lock_path.unlink()
2574
+ except FileNotFoundError:
2575
+ pass
2576
+
2577
+
2578
+ def _case_decision(case: dict[str, Any]) -> dict[str, Any]:
2579
+ all_runs = [run for run in case.get("runs", []) if isinstance(run, dict)]
2580
+ runs = _active_ledger_runs(all_runs)
2581
+ if not runs and all_runs:
2582
+ return {
2583
+ "decision": "archived_only",
2584
+ "archived_run_count": len(all_runs),
2585
+ }
2586
+ baselines = [
2587
+ run
2588
+ for run in runs
2589
+ if "baseline" in str(run.get("arm_id", ""))
2590
+ or _product_mode_baseline_run(run)
2591
+ ]
2592
+ treatments = [
2593
+ run
2594
+ for run in runs
2595
+ if "treatment" in str(run.get("arm_id", ""))
2596
+ or _product_mode_treatment_run(run)
2597
+ ]
2598
+ latest_baseline = baselines[-1] if baselines else None
2599
+ latest_treatment = treatments[-1] if treatments else None
2600
+
2601
+ def with_case_routing(result: dict[str, Any]) -> dict[str, Any]:
2602
+ routing = _case_routing_taxonomy(runs, result)
2603
+ if routing:
2604
+ result["case_routing"] = routing
2605
+ return result
2606
+
2607
+ def repair_decision(prefix: str, run: dict[str, Any]) -> dict[str, Any]:
2608
+ failure_class = _compact_text(run.get("failure_class"), limit=120)
2609
+ repair_class = _compact_text(run.get("repair_class"), limit=120)
2610
+ if repair_class == "runner_setup_timeout":
2611
+ decision = f"{prefix}_setup_timeout_repair_required"
2612
+ elif repair_class == "benchmark_environment_setup_contract":
2613
+ decision = f"{prefix}_environment_setup_repair_required"
2614
+ elif repair_class == "runner_model_access":
2615
+ decision = f"{prefix}_model_access_repair_required"
2616
+ elif repair_class == "runner_result_materialization":
2617
+ decision = f"{prefix}_result_materialization_repair_required"
2618
+ elif repair_class == "runner_result_finalization":
2619
+ decision = f"{prefix}_result_finalization_repair_required"
2620
+ elif repair_class == "skillsbench_codex_acp_runtime_preflight":
2621
+ decision = f"{prefix}_codex_acp_runtime_preflight_required"
2622
+ elif repair_class == "skillsbench_codex_acp_post_success_finalization":
2623
+ decision = f"{prefix}_codex_acp_post_success_finalization_required"
2624
+ elif repair_class == "skillsbench_setup_preflight_selection":
2625
+ decision = f"{prefix}_setup_preflight_selection_required"
2626
+ elif repair_class == "skillsbench_verifier_bootstrap_preflight_selection":
2627
+ decision = f"{prefix}_verifier_bootstrap_preflight_selection_required"
2628
+ elif repair_class == "skillsbench_task_source_preflight_selection":
2629
+ decision = f"{prefix}_task_source_preflight_selection_required"
2630
+ elif repair_class == "skillsbench_task_source_excluded":
2631
+ decision = f"{prefix}_task_source_excluded_from_formal_scoring"
2632
+ elif repair_class == "worker_verifier_alignment":
2633
+ decision = f"{prefix}_worker_verifier_alignment_required"
2634
+ elif repair_class == "verifier_or_infra_repair":
2635
+ decision = f"{prefix}_verifier_or_infra_repair_required"
2636
+ else:
2637
+ decision = f"{prefix}_runner_or_setup_repair_required"
2638
+ return {
2639
+ "decision": decision,
2640
+ "repair_priority": _compact_text(run.get("repair_priority"), limit=20),
2641
+ "repair_class": repair_class,
2642
+ "failure_class": failure_class,
2643
+ "next_action": _compact_text(run.get("next_action"), limit=220),
2644
+ }
2645
+
2646
+ def case_research_decision(run: dict[str, Any]) -> dict[str, Any]:
2647
+ repair_class = _compact_text(run.get("repair_class"), limit=120)
2648
+ if repair_class == "case_timeout_research":
2649
+ decision = "paired_no_score_uplift_timeout_research_required"
2650
+ elif repair_class == "case_exception_research":
2651
+ decision = "paired_no_score_uplift_exception_research_required"
2652
+ else:
2653
+ decision = "paired_no_score_uplift_case_research_required"
2654
+ return {
2655
+ "decision": decision,
2656
+ "repair_priority": _compact_text(run.get("repair_priority"), limit=20),
2657
+ "repair_class": repair_class,
2658
+ "failure_class": _compact_text(run.get("failure_class"), limit=120),
2659
+ "next_action": _compact_text(run.get("next_action"), limit=220),
2660
+ }
2661
+
2662
+ if latest_baseline and latest_treatment:
2663
+ product_mode_pair_review: dict[str, Any] | None = None
2664
+ if _product_mode_treatment_run(latest_treatment):
2665
+ product_mode_pair_review = _compact_product_mode_pair_review(
2666
+ classify_product_mode_main_table_pair(
2667
+ baseline_run=latest_baseline,
2668
+ treatment_run=latest_treatment,
2669
+ benchmark_id=_compact_text(
2670
+ latest_baseline.get("benchmark_id")
2671
+ or latest_treatment.get("benchmark_id")
2672
+ or "skillsbench@1.1",
2673
+ limit=120,
2674
+ )
2675
+ or "skillsbench@1.1",
2676
+ )
2677
+ )
2678
+ if product_mode_pair_review.get("main_table_claim_allowed") is not True:
2679
+ return with_case_routing(
2680
+ {
2681
+ "decision": "product_mode_pair_incomplete",
2682
+ "baseline_run_id": latest_baseline.get("run_id"),
2683
+ "treatment_run_id": latest_treatment.get("run_id"),
2684
+ "product_mode_main_table_pair": product_mode_pair_review,
2685
+ }
2686
+ )
2687
+ b_score = latest_baseline.get("official_score")
2688
+ t_score = latest_treatment.get("official_score")
2689
+ b_scope = _compact_text(latest_baseline.get("failure_scope"), limit=80)
2690
+ t_scope = _compact_text(latest_treatment.get("failure_scope"), limit=80)
2691
+ delta = (
2692
+ t_score - b_score
2693
+ if isinstance(b_score, (int, float))
2694
+ and not isinstance(b_score, bool)
2695
+ and isinstance(t_score, (int, float))
2696
+ and not isinstance(t_score, bool)
2697
+ else None
2698
+ )
2699
+ if "attribution_required" in {b_scope, t_scope}:
2700
+ decision = "paired_result_requires_attribution"
2701
+ elif "verifier_or_infra" in {b_scope, t_scope}:
2702
+ decision = "paired_result_blocked_by_verifier_or_infra"
2703
+ elif b_scope in {"runner_or_setup", "score_missing"}:
2704
+ decision_info = repair_decision("paired_baseline", latest_baseline)
2705
+ decision = decision_info["decision"]
2706
+ elif t_scope in {"runner_or_setup", "score_missing"}:
2707
+ decision_info = repair_decision("paired_treatment", latest_treatment)
2708
+ decision = decision_info["decision"]
2709
+ elif delta is None:
2710
+ decision = "paired_result_needs_score_review"
2711
+ elif delta > 0:
2712
+ decision = "paired_treatment_improved"
2713
+ elif delta == 0:
2714
+ case_research_run = next(
2715
+ (
2716
+ run
2717
+ for run in (latest_treatment, latest_baseline)
2718
+ if _compact_text(run.get("repair_priority"), limit=20) == "P1"
2719
+ ),
2720
+ None,
2721
+ )
2722
+ if b_scope == "case_or_solution" and t_scope == "case_or_solution" and case_research_run:
2723
+ decision_info = case_research_decision(case_research_run)
2724
+ decision = decision_info["decision"]
2725
+ elif b_scope == "passed" and t_scope == "passed":
2726
+ decision = "paired_baseline_solved_treatment_preserved"
2727
+ else:
2728
+ decision = "paired_no_score_uplift"
2729
+ else:
2730
+ decision = "paired_treatment_regressed"
2731
+ result = {
2732
+ "decision": decision,
2733
+ "baseline_run_id": latest_baseline.get("run_id"),
2734
+ "treatment_run_id": latest_treatment.get("run_id"),
2735
+ "official_score_delta": delta,
2736
+ "baseline_failure_scope": b_scope,
2737
+ "treatment_failure_scope": t_scope,
2738
+ }
2739
+ if product_mode_pair_review:
2740
+ result["product_mode_main_table_pair"] = product_mode_pair_review
2741
+ if "decision_info" in locals():
2742
+ result.update(
2743
+ {
2744
+ key: value
2745
+ for key, value in decision_info.items()
2746
+ if key != "decision" and value
2747
+ }
2748
+ )
2749
+ del decision_info
2750
+ return with_case_routing(result)
2751
+ if latest_baseline:
2752
+ if latest_baseline.get("official_passed") is True:
2753
+ decision = "baseline_passed_not_current_treatment_priority"
2754
+ elif latest_baseline.get("failure_scope") == "attribution_required":
2755
+ decision = "baseline_failed_requires_attribution"
2756
+ decision_info = repair_decision("baseline", latest_baseline)
2757
+ elif latest_baseline.get("failure_scope") == "case_or_solution":
2758
+ decision = "baseline_failed_treatment_candidate"
2759
+ case_route = _repair_route(
2760
+ _compact_text(latest_baseline.get("failure_class"), limit=120),
2761
+ _compact_text(latest_baseline.get("failure_scope"), limit=120),
2762
+ agent_model=_compact_text(latest_baseline.get("agent_model"), limit=120),
2763
+ )
2764
+ if case_route:
2765
+ decision_info = {
2766
+ "decision": decision,
2767
+ **case_route,
2768
+ "failure_class": _compact_text(
2769
+ latest_baseline.get("failure_class"),
2770
+ limit=120,
2771
+ ),
2772
+ }
2773
+ else:
2774
+ decision_info = repair_decision("baseline", latest_baseline)
2775
+ decision = decision_info["decision"]
2776
+ result = {
2777
+ "decision": decision,
2778
+ "baseline_run_id": latest_baseline.get("run_id"),
2779
+ "failure_scope": latest_baseline.get("failure_scope"),
2780
+ }
2781
+ if "decision_info" in locals():
2782
+ result.update(
2783
+ {
2784
+ key: value
2785
+ for key, value in decision_info.items()
2786
+ if key != "decision" and value
2787
+ }
2788
+ )
2789
+ del decision_info
2790
+ return with_case_routing(result)
2791
+ if runs:
2792
+ return with_case_routing(
2793
+ {"decision": "single_arm_recorded", "latest_run_id": runs[-1].get("run_id")}
2794
+ )
2795
+ return {"decision": "no_runs_recorded"}
2796
+
2797
+
2798
+ def upsert_benchmark_run_ledger_entry(
2799
+ ledger: dict[str, Any],
2800
+ entry: dict[str, Any],
2801
+ ) -> dict[str, Any]:
2802
+ benchmark_id = entry["benchmark_id"]
2803
+ case_id = entry["case_id"]
2804
+ benchmarks = ledger.setdefault("benchmarks", {})
2805
+ benchmark = benchmarks.setdefault(
2806
+ benchmark_id,
2807
+ {"benchmark_id": benchmark_id, "cases": {}},
2808
+ )
2809
+ cases = benchmark.setdefault("cases", {})
2810
+ case = cases.setdefault(case_id, {"case_id": case_id, "runs": []})
2811
+ runs = [run for run in case.get("runs", []) if isinstance(run, dict)]
2812
+ replaced = False
2813
+ for index, run in enumerate(runs):
2814
+ if run.get("run_id") == entry.get("run_id"):
2815
+ if _ledger_run_archived(run) and "archive_state" not in entry:
2816
+ entry = {
2817
+ **entry,
2818
+ "archive_state": "archived",
2819
+ "archive_reason": run.get("archive_reason"),
2820
+ "archive_batch_id": run.get("archive_batch_id"),
2821
+ "archived_at": run.get("archived_at"),
2822
+ }
2823
+ entry = {
2824
+ key: value
2825
+ for key, value in entry.items()
2826
+ if value not in (None, "", [])
2827
+ }
2828
+ runs[index] = entry
2829
+ replaced = True
2830
+ break
2831
+ if not replaced:
2832
+ entry_backfill_key = _ledger_logical_backfill_key(entry)
2833
+ if entry_backfill_key:
2834
+ for index, run in enumerate(runs):
2835
+ if (
2836
+ run.get("run_id") == entry.get("run_id")
2837
+ or _ledger_logical_backfill_key(run) != entry_backfill_key
2838
+ or not _ledger_result_equivalent_for_backfill(run, entry)
2839
+ ):
2840
+ continue
2841
+ merged, changed = _merge_ledger_logical_backfill_fields(run, entry)
2842
+ if changed:
2843
+ runs[index] = merged
2844
+ replaced = True
2845
+ break
2846
+ if not replaced:
2847
+ runs.append(entry)
2848
+ runs.sort(key=lambda run: (str(run.get("recorded_at", "")), str(run.get("run_id", ""))))
2849
+ case["runs"] = runs
2850
+ case["active_run_count"] = len(_active_ledger_runs(runs))
2851
+ case["archived_run_count"] = len(_archived_ledger_runs(runs))
2852
+ case["latest_decision"] = _case_decision(case)
2853
+ benchmark["case_count"] = len(cases)
2854
+ benchmark["run_count"] = sum(
2855
+ len(value.get("runs", []))
2856
+ for value in cases.values()
2857
+ if isinstance(value, dict)
2858
+ )
2859
+ benchmark["active_case_count"] = sum(
2860
+ 1
2861
+ for value in cases.values()
2862
+ if isinstance(value, dict)
2863
+ and value.get("active_run_count", 0)
2864
+ )
2865
+ benchmark["active_run_count"] = sum(
2866
+ int(value.get("active_run_count", 0))
2867
+ for value in cases.values()
2868
+ if isinstance(value, dict)
2869
+ )
2870
+ benchmark["archived_run_count"] = sum(
2871
+ int(value.get("archived_run_count", 0))
2872
+ for value in cases.values()
2873
+ if isinstance(value, dict)
2874
+ )
2875
+ ledger["updated_at"] = _now_local_iso()
2876
+ return ledger
2877
+
2878
+
2879
+ def render_benchmark_run_ledger_markdown(ledger: dict[str, Any]) -> str:
2880
+ benchmarks = ledger.get("benchmarks") if isinstance(ledger.get("benchmarks"), dict) else {}
2881
+ total_active_runs = 0
2882
+ total_archived_runs = 0
2883
+ total_active_cases = 0
2884
+ for benchmark in benchmarks.values():
2885
+ if not isinstance(benchmark, dict):
2886
+ continue
2887
+ cases = benchmark.get("cases") if isinstance(benchmark.get("cases"), dict) else {}
2888
+ if not isinstance(cases, dict):
2889
+ continue
2890
+ for case in cases.values():
2891
+ if not isinstance(case, dict):
2892
+ continue
2893
+ runs = [run for run in case.get("runs", []) if isinstance(run, dict)]
2894
+ active_runs = _active_ledger_runs(runs)
2895
+ archived_runs = _archived_ledger_runs(runs)
2896
+ if active_runs:
2897
+ total_active_cases += 1
2898
+ total_active_runs += len(active_runs)
2899
+ total_archived_runs += len(archived_runs)
2900
+ lines = [
2901
+ "# Benchmark Run Ledger",
2902
+ "",
2903
+ "This file is generated from `benchmark_run_ledger_v0`. It records compact",
2904
+ "benchmark case outcomes and artifact references; it must not contain raw",
2905
+ "logs, task prompts, trajectories, credentials, uploads, or absolute paths.",
2906
+ "Archived runs remain in JSON for traceability but are excluded from the",
2907
+ "default case decisions, repair backlog, and active runs table.",
2908
+ "",
2909
+ f"- schema_version: `{ledger.get('schema_version')}`",
2910
+ f"- updated_at: `{ledger.get('updated_at')}`",
2911
+ f"- active_case_count: `{total_active_cases}`",
2912
+ f"- active_run_count: `{total_active_runs}`",
2913
+ f"- archived_run_count: `{total_archived_runs}`",
2914
+ "",
2915
+ "## Case Decisions",
2916
+ "",
2917
+ "| Benchmark | Case | Decision | Product Pair | Case Routing | Runs |",
2918
+ "| --- | --- | --- | --- | --- | --- |",
2919
+ ]
2920
+ for benchmark_id in sorted(benchmarks):
2921
+ benchmark = benchmarks[benchmark_id]
2922
+ cases = benchmark.get("cases") if isinstance(benchmark, dict) else {}
2923
+ if not isinstance(cases, dict):
2924
+ continue
2925
+ for case_id in sorted(cases):
2926
+ case = cases[case_id]
2927
+ runs = case.get("runs", []) if isinstance(case, dict) else []
2928
+ active_runs = _active_ledger_runs(
2929
+ [run for run in runs if isinstance(run, dict)]
2930
+ )
2931
+ archived_count = len(
2932
+ _archived_ledger_runs([run for run in runs if isinstance(run, dict)])
2933
+ )
2934
+ if not active_runs:
2935
+ continue
2936
+ decision = (
2937
+ case.get("latest_decision")
2938
+ if isinstance(case, dict) and isinstance(case.get("latest_decision"), dict)
2939
+ else {}
2940
+ )
2941
+ routing = (
2942
+ decision.get("case_routing")
2943
+ if isinstance(decision.get("case_routing"), dict)
2944
+ else {}
2945
+ )
2946
+ routing_class = _compact_text(routing.get("class"), limit=120)
2947
+ routing_cell = f"`{routing_class}`" if routing_class else "-"
2948
+ pair_review = (
2949
+ decision.get("product_mode_main_table_pair")
2950
+ if isinstance(decision.get("product_mode_main_table_pair"), dict)
2951
+ else {}
2952
+ )
2953
+ product_pair_cell = "-"
2954
+ if pair_review:
2955
+ if pair_review.get("main_table_claim_allowed") is True:
2956
+ product_pair_cell = "`main_table_ready`"
2957
+ else:
2958
+ blocker = _compact_text(
2959
+ pair_review.get("claim_blocker"),
2960
+ limit=120,
2961
+ )
2962
+ product_pair_cell = f"`{blocker or 'pair_incomplete'}`"
2963
+ run_cell = str(len(active_runs))
2964
+ if archived_count:
2965
+ run_cell = f"{run_cell} active / {archived_count} archived"
2966
+ lines.append(
2967
+ f"| `{benchmark_id}` | `{case_id}` | "
2968
+ f"`{decision.get('decision', 'unknown')}` | "
2969
+ f"{product_pair_cell} | "
2970
+ f"{routing_cell} | `{run_cell}` |"
2971
+ )
2972
+ repair_rows: list[tuple[str, str, str, str, str, str, str, str]] = []
2973
+ priority_order = {"P0": 0, "P1": 1, "P2": 2}
2974
+ for benchmark_id in sorted(benchmarks):
2975
+ benchmark = benchmarks[benchmark_id]
2976
+ cases = benchmark.get("cases") if isinstance(benchmark, dict) else {}
2977
+ if not isinstance(cases, dict):
2978
+ continue
2979
+ for case_id in sorted(cases):
2980
+ case = cases[case_id]
2981
+ runs = _active_ledger_runs(
2982
+ [
2983
+ run
2984
+ for run in (case.get("runs", []) if isinstance(case, dict) else [])
2985
+ if isinstance(run, dict)
2986
+ ]
2987
+ )
2988
+ latest_run_by_arm: dict[str, dict[str, Any]] = {}
2989
+ for run in runs:
2990
+ if not isinstance(run, dict):
2991
+ continue
2992
+ arm = _compact_text(run.get("arm_id"), limit=80)
2993
+ if arm:
2994
+ latest_run_by_arm[arm] = run
2995
+ for run in runs:
2996
+ if not isinstance(run, dict):
2997
+ continue
2998
+ priority = _compact_text(run.get("repair_priority"), limit=20)
2999
+ repair_class = _compact_text(run.get("repair_class"), limit=80)
3000
+ if not priority or not repair_class:
3001
+ continue
3002
+ arm = _compact_text(run.get("arm_id"), limit=80)
3003
+ if arm and latest_run_by_arm.get(arm) is not run:
3004
+ continue
3005
+ repair_rows.append(
3006
+ (
3007
+ priority,
3008
+ benchmark_id,
3009
+ case_id,
3010
+ arm,
3011
+ repair_class,
3012
+ _compact_text(run.get("failure_class"), limit=120),
3013
+ _repair_profile_summary(run.get("repair_profile")),
3014
+ _compact_text(run.get("next_action"), limit=180),
3015
+ )
3016
+ )
3017
+ repair_rows.sort(
3018
+ key=lambda row: (
3019
+ priority_order.get(row[0], 99),
3020
+ row[4],
3021
+ row[1],
3022
+ row[2],
3023
+ row[3],
3024
+ )
3025
+ )
3026
+ if repair_rows:
3027
+ lines.extend(
3028
+ [
3029
+ "",
3030
+ "## Repair Backlog",
3031
+ "",
3032
+ "| Priority | Benchmark | Case | Arm | Repair Class | Failure | Repair Profile | Next Action |",
3033
+ "| --- | --- | --- | --- | --- | --- | --- | --- |",
3034
+ ]
3035
+ )
3036
+ for priority, benchmark_id, case_id, arm, repair_class, failure, profile, next_action in repair_rows:
3037
+ lines.append(
3038
+ "| "
3039
+ f"`{priority}` | "
3040
+ f"`{benchmark_id}` | "
3041
+ f"`{case_id}` | "
3042
+ f"`{arm}` | "
3043
+ f"`{repair_class}` | "
3044
+ f"`{failure}` | "
3045
+ f"{profile} | "
3046
+ f"{next_action} |"
3047
+ )
3048
+ archived_summary: list[tuple[str, int, int]] = []
3049
+ for benchmark_id in sorted(benchmarks):
3050
+ benchmark = benchmarks[benchmark_id]
3051
+ cases = benchmark.get("cases") if isinstance(benchmark, dict) else {}
3052
+ if not isinstance(cases, dict):
3053
+ continue
3054
+ archived_case_count = 0
3055
+ archived_run_count = 0
3056
+ for case in cases.values():
3057
+ if not isinstance(case, dict):
3058
+ continue
3059
+ runs = [run for run in case.get("runs", []) if isinstance(run, dict)]
3060
+ archived_runs = _archived_ledger_runs(runs)
3061
+ if archived_runs:
3062
+ archived_case_count += 1
3063
+ archived_run_count += len(archived_runs)
3064
+ if archived_run_count:
3065
+ archived_summary.append(
3066
+ (benchmark_id, archived_case_count, archived_run_count)
3067
+ )
3068
+ if archived_summary:
3069
+ lines.extend(
3070
+ [
3071
+ "",
3072
+ "## Archived Run Summary",
3073
+ "",
3074
+ "| Benchmark | Archived Cases | Archived Runs |",
3075
+ "| --- | --- | --- |",
3076
+ ]
3077
+ )
3078
+ for benchmark_id, case_count, run_count in archived_summary:
3079
+ lines.append(
3080
+ f"| `{benchmark_id}` | `{case_count}` | `{run_count}` |"
3081
+ )
3082
+ lines.extend(
3083
+ [
3084
+ "",
3085
+ "## Runs",
3086
+ "",
3087
+ "| Benchmark | Case | Arm | Attempt | Score | First Success Round | Round Rewards | Failure | Artifact |",
3088
+ "| --- | --- | --- | --- | --- | --- | --- | --- | --- |",
3089
+ ]
3090
+ )
3091
+ for benchmark_id in sorted(benchmarks):
3092
+ benchmark = benchmarks[benchmark_id]
3093
+ cases = benchmark.get("cases") if isinstance(benchmark, dict) else {}
3094
+ if not isinstance(cases, dict):
3095
+ continue
3096
+ for case_id in sorted(cases):
3097
+ case = cases[case_id]
3098
+ runs = _active_ledger_runs(
3099
+ [
3100
+ run
3101
+ for run in (case.get("runs", []) if isinstance(case, dict) else [])
3102
+ if isinstance(run, dict)
3103
+ ]
3104
+ )
3105
+ for run in runs:
3106
+ refs = run.get("artifact_refs") if isinstance(run.get("artifact_refs"), dict) else {}
3107
+ artifact = refs.get("compact_artifact_ref") or refs.get("result_ref") or refs.get("artifact_ref") or ""
3108
+ score = run.get("official_score")
3109
+ score_text = "missing" if score is None else str(score)
3110
+ first_success_round = run.get("first_success_round")
3111
+ first_success_text = (
3112
+ str(first_success_round)
3113
+ if isinstance(first_success_round, int)
3114
+ and not isinstance(first_success_round, bool)
3115
+ else ""
3116
+ )
3117
+ round_rewards_text = _round_reward_summary(run)
3118
+ attempt = _compact_text(run.get("ledger_attempt_kind"), limit=80)
3119
+ if not attempt:
3120
+ attempt = _attempt_label_from_accounting(run)
3121
+ if not attempt:
3122
+ attempt = (
3123
+ "case_attempt"
3124
+ if run.get("case_attempt_countable") is True
3125
+ else ""
3126
+ )
3127
+ lines.append(
3128
+ "| "
3129
+ f"`{benchmark_id}` | "
3130
+ f"`{case_id}` | "
3131
+ f"`{run.get('arm_id', '')}` | "
3132
+ f"`{attempt}` | "
3133
+ f"`{score_text}` | "
3134
+ f"`{first_success_text}` | "
3135
+ f"`{round_rewards_text}` | "
3136
+ f"`{run.get('failure_class', 'none')}` | "
3137
+ f"`{artifact}` |"
3138
+ )
3139
+ return "\n".join(lines) + "\n"
3140
+
3141
+
3142
+ def update_benchmark_run_ledger_entries(
3143
+ *,
3144
+ ledger_path: str | Path,
3145
+ entries: list[dict[str, Any]],
3146
+ dry_run: bool = False,
3147
+ ) -> dict[str, Any]:
3148
+ """Atomically upsert a compact batch of terminal public ledger entries."""
3149
+
3150
+ path = Path(ledger_path)
3151
+ markdown_path = path.with_suffix(".md")
3152
+ accepted = [
3153
+ dict(entry)
3154
+ for entry in entries
3155
+ if isinstance(entry, dict) and _entry_is_public_ledger_closeout(entry)
3156
+ ]
3157
+
3158
+ def apply_entries(ledger: dict[str, Any]) -> dict[str, Any]:
3159
+ for entry in accepted:
3160
+ ledger = upsert_benchmark_run_ledger_entry(ledger, entry)
3161
+ return ledger
3162
+
3163
+ if dry_run:
3164
+ updated = apply_entries(load_benchmark_run_ledger(path))
3165
+ elif accepted:
3166
+ with _LedgerWriteLock(path):
3167
+ updated = apply_entries(load_benchmark_run_ledger(path))
3168
+ path.parent.mkdir(parents=True, exist_ok=True)
3169
+ tmp_path = path.with_suffix(path.suffix + ".tmp")
3170
+ tmp_markdown_path = markdown_path.with_suffix(markdown_path.suffix + ".tmp")
3171
+ tmp_path.write_text(
3172
+ json.dumps(updated, ensure_ascii=False, indent=2, sort_keys=True) + "\n",
3173
+ encoding="utf-8",
3174
+ )
3175
+ tmp_markdown_path.write_text(
3176
+ render_benchmark_run_ledger_markdown(updated),
3177
+ encoding="utf-8",
3178
+ )
3179
+ tmp_path.replace(path)
3180
+ tmp_markdown_path.replace(markdown_path)
3181
+ else:
3182
+ updated = load_benchmark_run_ledger(path)
3183
+
3184
+ case_decisions = []
3185
+ for entry in accepted:
3186
+ case = (
3187
+ updated.get("benchmarks", {})
3188
+ .get(entry["benchmark_id"], {})
3189
+ .get("cases", {})
3190
+ .get(entry["case_id"], {})
3191
+ )
3192
+ case_decisions.append(
3193
+ {
3194
+ "benchmark_id": entry["benchmark_id"],
3195
+ "case_id": entry["case_id"],
3196
+ "decision": case.get("latest_decision", {}),
3197
+ }
3198
+ )
3199
+ return {
3200
+ "ok": True,
3201
+ "dry_run": dry_run,
3202
+ "updated": bool(accepted) and not dry_run,
3203
+ "schema_version": BENCHMARK_RUN_LEDGER_SCHEMA_VERSION,
3204
+ "ledger_path": str(path),
3205
+ "markdown_path": str(markdown_path),
3206
+ "input_entry_count": len(entries),
3207
+ "accepted_entry_count": len(accepted),
3208
+ "skipped_entry_count": len(entries) - len(accepted),
3209
+ "upserted_count": len(accepted) if not dry_run else 0,
3210
+ "case_decisions": case_decisions,
3211
+ }
3212
+
3213
+
3214
+ def update_benchmark_run_ledger(
3215
+ *,
3216
+ ledger_path: str | Path,
3217
+ benchmark_run: dict[str, Any],
3218
+ artifact_ref: str | Path | None = None,
3219
+ result_ref: str | Path | None = None,
3220
+ compact_artifact_ref: str | Path | None = None,
3221
+ run_group_id: str | None = None,
3222
+ arm_id: str | None = None,
3223
+ notes: str | None = None,
3224
+ recorded_at: str | None = None,
3225
+ dry_run: bool = False,
3226
+ cwd: Path | None = None,
3227
+ ) -> dict[str, Any]:
3228
+ path = Path(ledger_path)
3229
+ entry = build_benchmark_run_ledger_entry(
3230
+ benchmark_run,
3231
+ artifact_ref=artifact_ref,
3232
+ result_ref=result_ref,
3233
+ compact_artifact_ref=compact_artifact_ref,
3234
+ run_group_id=run_group_id,
3235
+ arm_id=arm_id,
3236
+ notes=notes,
3237
+ recorded_at=recorded_at,
3238
+ cwd=cwd,
3239
+ )
3240
+ markdown_path = path.with_suffix(".md")
3241
+ if not _entry_is_public_ledger_closeout(entry):
3242
+ return {
3243
+ "ok": True,
3244
+ "dry_run": dry_run,
3245
+ "updated": False,
3246
+ "skipped": True,
3247
+ "skip_reason": "benchmark_run_not_terminal_for_public_ledger",
3248
+ "schema_version": BENCHMARK_RUN_LEDGER_SCHEMA_VERSION,
3249
+ "ledger_path": str(path),
3250
+ "markdown_path": str(markdown_path),
3251
+ "entry": entry,
3252
+ "case_decision": {},
3253
+ }
3254
+ batch = update_benchmark_run_ledger_entries(
3255
+ ledger_path=path,
3256
+ entries=[entry],
3257
+ dry_run=dry_run,
3258
+ )
3259
+ case_decision = (
3260
+ batch["case_decisions"][0]["decision"]
3261
+ if batch.get("case_decisions")
3262
+ else {}
3263
+ )
3264
+ return {
3265
+ "ok": True,
3266
+ "dry_run": dry_run,
3267
+ "updated": batch["updated"],
3268
+ "schema_version": BENCHMARK_RUN_LEDGER_SCHEMA_VERSION,
3269
+ "ledger_path": str(path),
3270
+ "markdown_path": str(markdown_path),
3271
+ "entry": entry,
3272
+ "case_decision": case_decision,
3273
+ }
3274
+
3275
+
3276
+ def _matches_any_pattern(value: str, patterns: list[str]) -> bool:
3277
+ return any(pattern and pattern in value for pattern in patterns)
3278
+
3279
+
3280
+ def archive_benchmark_run_ledger_runs(
3281
+ *,
3282
+ ledger_path: str | Path,
3283
+ benchmark_id: str,
3284
+ reason: str,
3285
+ run_group_contains: list[str] | None = None,
3286
+ keep_run_group_contains: list[str] | None = None,
3287
+ case_ids: list[str] | None = None,
3288
+ arm_ids: list[str] | None = None,
3289
+ archive_all_matching_benchmark: bool = False,
3290
+ archive_batch_id: str | None = None,
3291
+ dry_run: bool = False,
3292
+ ) -> dict[str, Any]:
3293
+ """Mark matching run-ledger rows archived without deleting traceable evidence."""
3294
+
3295
+ benchmark_filter = _compact_text(benchmark_id, limit=120)
3296
+ archive_reason = _compact_text(reason, limit=220)
3297
+ if not benchmark_filter:
3298
+ raise ValueError("benchmark_id is required")
3299
+ if not archive_reason:
3300
+ raise ValueError("archive reason is required")
3301
+ run_group_patterns = [
3302
+ _compact_text(item, limit=160)
3303
+ for item in (run_group_contains or [])
3304
+ if _compact_text(item, limit=160)
3305
+ ]
3306
+ keep_patterns = [
3307
+ _compact_text(item, limit=160)
3308
+ for item in (keep_run_group_contains or [])
3309
+ if _compact_text(item, limit=160)
3310
+ ]
3311
+ case_filters = {
3312
+ _compact_text(item, limit=160)
3313
+ for item in (case_ids or [])
3314
+ if _compact_text(item, limit=160)
3315
+ }
3316
+ arm_filters = {
3317
+ _compact_text(item, limit=120)
3318
+ for item in (arm_ids or [])
3319
+ if _compact_text(item, limit=120)
3320
+ }
3321
+ if not (
3322
+ archive_all_matching_benchmark
3323
+ or run_group_patterns
3324
+ or case_filters
3325
+ or arm_filters
3326
+ ):
3327
+ raise ValueError(
3328
+ "provide a run/case/arm filter, or pass --archive-all-matching-benchmark"
3329
+ )
3330
+
3331
+ path = Path(ledger_path)
3332
+ markdown_path = path.with_suffix(".md")
3333
+ archived_at = _now_local_iso()
3334
+ batch_id = _compact_text(archive_batch_id, limit=120) or hashlib.sha1(
3335
+ f"{benchmark_filter}|{archive_reason}|{archived_at}".encode("utf-8")
3336
+ ).hexdigest()[:12]
3337
+
3338
+ def apply_archive(ledger: dict[str, Any]) -> tuple[dict[str, Any], dict[str, Any]]:
3339
+ normalized = _normalize_benchmark_run_ledger(dict(ledger))
3340
+ benchmarks = (
3341
+ normalized.get("benchmarks")
3342
+ if isinstance(normalized.get("benchmarks"), dict)
3343
+ else {}
3344
+ )
3345
+ matched_run_count = 0
3346
+ newly_archived_run_count = 0
3347
+ already_archived_run_count = 0
3348
+ kept_run_count = 0
3349
+ archived_samples: list[dict[str, Any]] = []
3350
+ benchmark = benchmarks.get(benchmark_filter)
3351
+ if isinstance(benchmark, dict):
3352
+ cases = benchmark.get("cases") if isinstance(benchmark.get("cases"), dict) else {}
3353
+ for case in cases.values():
3354
+ if not isinstance(case, dict):
3355
+ continue
3356
+ case_id = _compact_text(case.get("case_id"), limit=160)
3357
+ runs = [run for run in case.get("runs", []) if isinstance(run, dict)]
3358
+ for run in runs:
3359
+ run_group_id = _compact_text(run.get("run_group_id"), limit=160)
3360
+ arm = _compact_text(run.get("arm_id"), limit=120)
3361
+ if keep_patterns and _matches_any_pattern(run_group_id, keep_patterns):
3362
+ kept_run_count += 1
3363
+ continue
3364
+ if case_filters and case_id not in case_filters:
3365
+ continue
3366
+ if arm_filters and arm not in arm_filters:
3367
+ continue
3368
+ if run_group_patterns and not _matches_any_pattern(
3369
+ run_group_id,
3370
+ run_group_patterns,
3371
+ ):
3372
+ continue
3373
+ matched_run_count += 1
3374
+ if _ledger_run_archived(run):
3375
+ already_archived_run_count += 1
3376
+ continue
3377
+ run["archive_state"] = "archived"
3378
+ run["archive_reason"] = archive_reason
3379
+ run["archive_batch_id"] = batch_id
3380
+ run["archived_at"] = archived_at
3381
+ newly_archived_run_count += 1
3382
+ if len(archived_samples) < 20:
3383
+ archived_samples.append(
3384
+ {
3385
+ "run_id": run.get("run_id"),
3386
+ "case_id": case_id,
3387
+ "arm_id": arm,
3388
+ "run_group_id": run_group_id,
3389
+ "official_score": run.get("official_score"),
3390
+ "failure_class": run.get("failure_class"),
3391
+ }
3392
+ )
3393
+ case["runs"] = sorted(
3394
+ runs,
3395
+ key=lambda run: (
3396
+ str(run.get("recorded_at", "")),
3397
+ str(run.get("run_id", "")),
3398
+ ),
3399
+ )
3400
+ case["active_run_count"] = len(_active_ledger_runs(case["runs"]))
3401
+ case["archived_run_count"] = len(_archived_ledger_runs(case["runs"]))
3402
+ case["latest_decision"] = _case_decision(case)
3403
+ benchmark["case_count"] = len(cases)
3404
+ benchmark["run_count"] = sum(
3405
+ len(value.get("runs", []))
3406
+ for value in cases.values()
3407
+ if isinstance(value, dict)
3408
+ )
3409
+ benchmark["active_case_count"] = sum(
3410
+ 1
3411
+ for value in cases.values()
3412
+ if isinstance(value, dict)
3413
+ and value.get("active_run_count", 0)
3414
+ )
3415
+ benchmark["active_run_count"] = sum(
3416
+ int(value.get("active_run_count", 0))
3417
+ for value in cases.values()
3418
+ if isinstance(value, dict)
3419
+ )
3420
+ benchmark["archived_run_count"] = sum(
3421
+ int(value.get("archived_run_count", 0))
3422
+ for value in cases.values()
3423
+ if isinstance(value, dict)
3424
+ )
3425
+ normalized["updated_at"] = archived_at
3426
+ summary = {
3427
+ "schema_version": "benchmark_run_ledger_archive_v0",
3428
+ "archive_batch_id": batch_id,
3429
+ "benchmark_id": benchmark_filter,
3430
+ "reason": archive_reason,
3431
+ "matched_run_count": matched_run_count,
3432
+ "newly_archived_run_count": newly_archived_run_count,
3433
+ "already_archived_run_count": already_archived_run_count,
3434
+ "kept_run_count": kept_run_count,
3435
+ "archived_samples": archived_samples,
3436
+ "truncated": newly_archived_run_count > len(archived_samples),
3437
+ }
3438
+ return normalized, summary
3439
+
3440
+ if dry_run:
3441
+ updated, summary = apply_archive(load_benchmark_run_ledger(path))
3442
+ else:
3443
+ with _LedgerWriteLock(path):
3444
+ updated, summary = apply_archive(load_benchmark_run_ledger(path))
3445
+ path.parent.mkdir(parents=True, exist_ok=True)
3446
+ tmp_path = path.with_suffix(path.suffix + ".tmp")
3447
+ tmp_markdown_path = markdown_path.with_suffix(markdown_path.suffix + ".tmp")
3448
+ tmp_path.write_text(
3449
+ json.dumps(updated, ensure_ascii=False, indent=2, sort_keys=True) + "\n",
3450
+ encoding="utf-8",
3451
+ )
3452
+ tmp_markdown_path.write_text(
3453
+ render_benchmark_run_ledger_markdown(updated),
3454
+ encoding="utf-8",
3455
+ )
3456
+ tmp_path.replace(path)
3457
+ tmp_markdown_path.replace(markdown_path)
3458
+ return {
3459
+ "ok": True,
3460
+ "dry_run": dry_run,
3461
+ "updated": not dry_run,
3462
+ "ledger_path": str(path),
3463
+ "markdown_path": str(markdown_path),
3464
+ "archive": summary,
3465
+ }
3466
+
3467
+
3468
+ def benchmark_run_ledger_entry_signature(
3469
+ entry: dict[str, Any],
3470
+ ) -> tuple[str, ...]:
3471
+ """Return the stable logical identity used for ledger drift detection."""
3472
+
3473
+ return (
3474
+ _compact_text(entry.get("benchmark_id"), limit=120),
3475
+ _compact_text(entry.get("case_id"), limit=160),
3476
+ _compact_text(entry.get("arm_id"), limit=120),
3477
+ _compact_text(entry.get("mode"), limit=120),
3478
+ _compact_text(entry.get("job_name"), limit=160),
3479
+ _compact_text(entry.get("score_status"), limit=80),
3480
+ _compact_text(entry.get("official_score"), limit=80),
3481
+ _compact_text(entry.get("failure_class"), limit=120),
3482
+ )
3483
+
3484
+
3485
+ def _iter_ledger_runs(ledger: dict[str, Any]) -> list[dict[str, Any]]:
3486
+ runs: list[dict[str, Any]] = []
3487
+ benchmarks = ledger.get("benchmarks") if isinstance(ledger.get("benchmarks"), dict) else {}
3488
+ for benchmark in benchmarks.values():
3489
+ if not isinstance(benchmark, dict):
3490
+ continue
3491
+ cases = benchmark.get("cases") if isinstance(benchmark.get("cases"), dict) else {}
3492
+ for case in cases.values():
3493
+ if not isinstance(case, dict):
3494
+ continue
3495
+ for run in case.get("runs") or []:
3496
+ if isinstance(run, dict):
3497
+ runs.append(run)
3498
+ return runs
3499
+
3500
+
3501
+ def build_benchmark_run_ledger_current_aggregate(
3502
+ ledger: dict[str, Any],
3503
+ *,
3504
+ benchmark_id: str = "skillsbench@1.1",
3505
+ canonical_case_ids: list[str] | None = None,
3506
+ active_case_ids: list[str] | None = None,
3507
+ source_ledger_count: int = 1,
3508
+ exclude_noncanonical_sanity_sources: bool = True,
3509
+ target_lane_id: str | None = None,
3510
+ target_run_group_contains: list[str] | None = None,
3511
+ target_current_run_group_contains: list[str] | None = None,
3512
+ target_backfill_run_group_contains: list[str] | None = None,
3513
+ ) -> dict[str, Any]:
3514
+ from .benchmark_ledger_current import (
3515
+ build_benchmark_run_ledger_current_aggregate as _build_current_aggregate,
3516
+ )
3517
+
3518
+ return _build_current_aggregate(
3519
+ ledger,
3520
+ benchmark_id=benchmark_id,
3521
+ canonical_case_ids=canonical_case_ids,
3522
+ active_case_ids=active_case_ids,
3523
+ source_ledger_count=source_ledger_count,
3524
+ exclude_noncanonical_sanity_sources=exclude_noncanonical_sanity_sources,
3525
+ target_lane_id=target_lane_id,
3526
+ target_run_group_contains=target_run_group_contains,
3527
+ target_current_run_group_contains=target_current_run_group_contains,
3528
+ target_backfill_run_group_contains=target_backfill_run_group_contains,
3529
+ )
3530
+
3531
+
3532
+ def merge_benchmark_run_ledgers(
3533
+ *,
3534
+ target_ledger_path: str | Path,
3535
+ source_ledger_paths: list[str | Path],
3536
+ benchmark_ids: list[str] | None = None,
3537
+ run_group_contains: list[str] | None = None,
3538
+ dry_run: bool = False,
3539
+ ) -> dict[str, Any]:
3540
+ """Merge public benchmark_run_ledger_v0 files into one canonical ledger."""
3541
+
3542
+ if not source_ledger_paths:
3543
+ raise ValueError("at least one source benchmark run ledger is required")
3544
+ target_path = Path(target_ledger_path).expanduser()
3545
+ markdown_path = target_path.with_suffix(".md")
3546
+ benchmark_filter = {
3547
+ _compact_text(value, limit=120)
3548
+ for value in (benchmark_ids or [])
3549
+ if _compact_text(value, limit=120)
3550
+ }
3551
+ run_group_filters = [
3552
+ _compact_text(value, limit=160)
3553
+ for value in (run_group_contains or [])
3554
+ if _compact_text(value, limit=160)
3555
+ ]
3556
+ unique_sources: list[Path] = []
3557
+ seen_sources: set[str] = set()
3558
+ for source in source_ledger_paths:
3559
+ source_path = Path(source).expanduser()
3560
+ source_key = str(source_path.resolve(strict=False))
3561
+ if source_key in seen_sources:
3562
+ continue
3563
+ seen_sources.add(source_key)
3564
+ unique_sources.append(source_path)
3565
+
3566
+ def apply_merge(start_ledger: dict[str, Any]) -> tuple[dict[str, Any], dict[str, Any]]:
3567
+ updated = _normalize_benchmark_run_ledger(dict(start_ledger))
3568
+ before_run_ids = {
3569
+ _compact_text(run.get("run_id"), limit=80)
3570
+ for run in _iter_ledger_runs(updated)
3571
+ if _compact_text(run.get("run_id"), limit=80)
3572
+ }
3573
+ before_signatures = {
3574
+ benchmark_run_ledger_entry_signature(run)
3575
+ for run in _iter_ledger_runs(updated)
3576
+ }
3577
+ source_ledger_count = 0
3578
+ missing_source_count = 0
3579
+ source_run_count = 0
3580
+ considered_run_count = 0
3581
+ merged_run_count = 0
3582
+ skipped_run_count = 0
3583
+ skipped_by_reason: dict[str, int] = {}
3584
+
3585
+ def skip(reason: str) -> None:
3586
+ nonlocal skipped_run_count
3587
+ skipped_run_count += 1
3588
+ skipped_by_reason[reason] = skipped_by_reason.get(reason, 0) + 1
3589
+
3590
+ for source_path in unique_sources:
3591
+ if not source_path.exists():
3592
+ missing_source_count += 1
3593
+ continue
3594
+ source_ledger_count += 1
3595
+ source_ledger = load_benchmark_run_ledger(source_path)
3596
+ for run in _iter_ledger_runs(source_ledger):
3597
+ if not isinstance(run, dict):
3598
+ continue
3599
+ source_run_count += 1
3600
+ benchmark_id = _compact_text(run.get("benchmark_id"), limit=120)
3601
+ if benchmark_filter and benchmark_id not in benchmark_filter:
3602
+ skip("benchmark_filter")
3603
+ continue
3604
+ run_group_id = _compact_text(run.get("run_group_id"), limit=160)
3605
+ if run_group_filters and not any(
3606
+ token in run_group_id for token in run_group_filters
3607
+ ):
3608
+ skip("run_group_filter")
3609
+ continue
3610
+ if not _entry_is_public_ledger_closeout(run):
3611
+ skip("non_terminal")
3612
+ continue
3613
+ considered_run_count += 1
3614
+ updated = upsert_benchmark_run_ledger_entry(updated, dict(run))
3615
+ merged_run_count += 1
3616
+
3617
+ updated = _normalize_benchmark_run_ledger(updated)
3618
+ after_runs = _iter_ledger_runs(updated)
3619
+ after_run_ids = {
3620
+ _compact_text(run.get("run_id"), limit=80)
3621
+ for run in after_runs
3622
+ if _compact_text(run.get("run_id"), limit=80)
3623
+ }
3624
+ after_signatures = {
3625
+ benchmark_run_ledger_entry_signature(run) for run in after_runs
3626
+ }
3627
+ summary = {
3628
+ "schema_version": "benchmark_run_ledger_merge_v0",
3629
+ "ok": True,
3630
+ "dry_run": dry_run,
3631
+ "updated": not dry_run,
3632
+ "ledger_path": str(target_path),
3633
+ "markdown_path": str(markdown_path),
3634
+ "source_ledger_count": source_ledger_count,
3635
+ "missing_source_count": missing_source_count,
3636
+ "source_run_count": source_run_count,
3637
+ "considered_run_count": considered_run_count,
3638
+ "merged_run_count": merged_run_count,
3639
+ "new_run_id_count": len(after_run_ids - before_run_ids),
3640
+ "new_signature_count": len(after_signatures - before_signatures),
3641
+ "target_run_count": len(after_runs),
3642
+ "skipped_run_count": skipped_run_count,
3643
+ "skipped_by_reason": skipped_by_reason,
3644
+ "benchmark_ids": sorted(benchmark_filter),
3645
+ "run_group_contains": run_group_filters,
3646
+ "source_paths_recorded": False,
3647
+ }
3648
+ return updated, summary
3649
+
3650
+ if dry_run:
3651
+ _, summary = apply_merge(load_benchmark_run_ledger(target_path))
3652
+ else:
3653
+ with _LedgerWriteLock(target_path):
3654
+ updated, summary = apply_merge(load_benchmark_run_ledger(target_path))
3655
+ target_path.parent.mkdir(parents=True, exist_ok=True)
3656
+ tmp_path = target_path.with_suffix(target_path.suffix + ".tmp")
3657
+ tmp_markdown_path = markdown_path.with_suffix(markdown_path.suffix + ".tmp")
3658
+ tmp_path.write_text(
3659
+ json.dumps(updated, ensure_ascii=False, indent=2, sort_keys=True) + "\n",
3660
+ encoding="utf-8",
3661
+ )
3662
+ tmp_markdown_path.write_text(
3663
+ render_benchmark_run_ledger_markdown(updated),
3664
+ encoding="utf-8",
3665
+ )
3666
+ tmp_path.replace(target_path)
3667
+ tmp_markdown_path.replace(markdown_path)
3668
+ return {
3669
+ "ok": True,
3670
+ "dry_run": dry_run,
3671
+ "updated": not dry_run,
3672
+ "ledger_path": str(target_path),
3673
+ "markdown_path": str(markdown_path),
3674
+ "merge": summary,
3675
+ }
3676
+
3677
+
3678
+ def _history_benchmark_run(record: dict[str, Any]) -> dict[str, Any] | None:
3679
+ if record.get("schema_version") == "benchmark_run_v0":
3680
+ return record
3681
+ nested = record.get("benchmark_run")
3682
+ if isinstance(nested, dict) and nested.get("schema_version") == "benchmark_run_v0":
3683
+ return nested
3684
+ return None
3685
+
3686
+
3687
+ def check_benchmark_run_ledger_drift(
3688
+ *,
3689
+ history_records: list[dict[str, Any]],
3690
+ ledger: dict[str, Any],
3691
+ ledger_path: str | Path | None = None,
3692
+ limit: int = 20,
3693
+ cwd: Path | None = None,
3694
+ ) -> dict[str, Any]:
3695
+ """Compare compact benchmark_run_v0 history events with the public run ledger."""
3696
+
3697
+ normalized_ledger = _normalize_benchmark_run_ledger(dict(ledger))
3698
+ ledger_runs = _iter_ledger_runs(normalized_ledger)
3699
+ ledger_run_ids = {
3700
+ _compact_text(run.get("run_id"), limit=80)
3701
+ for run in ledger_runs
3702
+ if _compact_text(run.get("run_id"), limit=80)
3703
+ }
3704
+ ledger_signatures = {
3705
+ benchmark_run_ledger_entry_signature(run) for run in ledger_runs
3706
+ }
3707
+
3708
+ checked_history_run_count = 0
3709
+ terminal_history_run_count = 0
3710
+ matched_count = 0
3711
+ non_terminal_skipped_count = 0
3712
+ missing: list[dict[str, Any]] = []
3713
+ for record in history_records:
3714
+ if not isinstance(record, dict):
3715
+ continue
3716
+ benchmark_run = _history_benchmark_run(record)
3717
+ if not benchmark_run:
3718
+ continue
3719
+ checked_history_run_count += 1
3720
+ entry = build_benchmark_run_ledger_entry(
3721
+ benchmark_run,
3722
+ compact_artifact_ref=record.get("json_path")
3723
+ if isinstance(record.get("json_path"), str)
3724
+ else None,
3725
+ recorded_at=record.get("generated_at")
3726
+ if isinstance(record.get("generated_at"), str)
3727
+ else None,
3728
+ cwd=cwd,
3729
+ )
3730
+ if not _entry_is_public_ledger_closeout(entry):
3731
+ non_terminal_skipped_count += 1
3732
+ continue
3733
+ terminal_history_run_count += 1
3734
+ run_id = _compact_text(entry.get("run_id"), limit=80)
3735
+ signature = benchmark_run_ledger_entry_signature(entry)
3736
+ if run_id in ledger_run_ids or signature in ledger_signatures:
3737
+ matched_count += 1
3738
+ continue
3739
+ catch_up = "loopx benchmark run-ledger-upsert --benchmark-run-json <compact-benchmark-run-v0.json>"
3740
+ if ledger_path:
3741
+ ledger_ref_path = Path(ledger_path)
3742
+ ledger_ref = (
3743
+ "<benchmark-run-ledger.json>"
3744
+ if ledger_ref_path.is_absolute()
3745
+ else ledger_ref_path.as_posix()
3746
+ )
3747
+ catch_up += f" --run-ledger-path {ledger_ref}"
3748
+ if entry.get("run_group_id"):
3749
+ catch_up += f" --run-group-id {entry['run_group_id']}"
3750
+ if entry.get("arm_id"):
3751
+ catch_up += f" --arm-id {entry['arm_id']}"
3752
+ catch_up += " --execute"
3753
+ missing.append(
3754
+ {
3755
+ "run_id": run_id,
3756
+ "generated_at": _compact_text(record.get("generated_at"), limit=80),
3757
+ "benchmark_id": entry.get("benchmark_id"),
3758
+ "case_id": entry.get("case_id"),
3759
+ "arm_id": entry.get("arm_id"),
3760
+ "mode": entry.get("mode"),
3761
+ "job_name": entry.get("job_name"),
3762
+ "score_status": entry.get("score_status"),
3763
+ "official_score": entry.get("official_score"),
3764
+ "failure_class": entry.get("failure_class"),
3765
+ "catch_up_command_template": catch_up,
3766
+ }
3767
+ )
3768
+
3769
+ limited_missing = missing[: max(0, limit)]
3770
+ return {
3771
+ "schema_version": "benchmark_run_ledger_drift_v0",
3772
+ "ok": True,
3773
+ "drift_detected": bool(missing),
3774
+ "ledger_schema_version": normalized_ledger.get("schema_version"),
3775
+ "ledger_run_count": len(ledger_runs),
3776
+ "checked_history_run_count": checked_history_run_count,
3777
+ "terminal_history_run_count": terminal_history_run_count,
3778
+ "matched_history_run_count": matched_count,
3779
+ "non_terminal_skipped_count": non_terminal_skipped_count,
3780
+ "missing_ledger_run_count": len(missing),
3781
+ "missing_runs": limited_missing,
3782
+ "truncated": len(missing) > len(limited_missing),
3783
+ "limit": limit,
3784
+ "read_boundary": {
3785
+ "compact_only": True,
3786
+ "raw_logs_read": False,
3787
+ "task_text_read": False,
3788
+ "trajectory_read": False,
3789
+ "docker_invoked": False,
3790
+ "model_api_invoked": False,
3791
+ "upload_invoked": False,
3792
+ },
3793
+ }