verifiers 0.3.2.dev45__tar.gz → 0.3.2.dev47__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (215) hide show
  1. {verifiers-0.3.2.dev45 → verifiers-0.3.2.dev47}/PKG-INFO +1 -1
  2. {verifiers-0.3.2.dev45 → verifiers-0.3.2.dev47}/tests/v1/test_scoring.py +13 -0
  3. {verifiers-0.3.2.dev45 → verifiers-0.3.2.dev47}/verifiers/v1/agent.py +26 -24
  4. {verifiers-0.3.2.dev45 → verifiers-0.3.2.dev47}/verifiers/v1/configs/harness.py +4 -0
  5. verifiers-0.3.2.dev47/verifiers/v1/envs/isolated_verifier/__init__.py +7 -0
  6. verifiers-0.3.2.dev47/verifiers/v1/envs/isolated_verifier/env.py +182 -0
  7. {verifiers-0.3.2.dev45 → verifiers-0.3.2.dev47}/verifiers/v1/harnesses/claude_code/harness.py +11 -25
  8. {verifiers-0.3.2.dev45 → verifiers-0.3.2.dev47}/verifiers/v1/harnesses/codex/harness.py +11 -23
  9. {verifiers-0.3.2.dev45 → verifiers-0.3.2.dev47}/verifiers/v1/harnesses/hermes_agent/harness.py +4 -9
  10. {verifiers-0.3.2.dev45 → verifiers-0.3.2.dev47}/verifiers/v1/harnesses/kimi_code/harness.py +9 -13
  11. {verifiers-0.3.2.dev45 → verifiers-0.3.2.dev47}/verifiers/v1/harnesses/mini_swe_agent/harness.py +2 -4
  12. {verifiers-0.3.2.dev45 → verifiers-0.3.2.dev47}/verifiers/v1/harnesses/node.py +9 -14
  13. {verifiers-0.3.2.dev45 → verifiers-0.3.2.dev47}/verifiers/v1/harnesses/openclaw/harness.py +14 -30
  14. {verifiers-0.3.2.dev45 → verifiers-0.3.2.dev47}/verifiers/v1/harnesses/pi/harness.py +9 -20
  15. {verifiers-0.3.2.dev45 → verifiers-0.3.2.dev47}/verifiers/v1/harnesses/pool/harness.py +11 -14
  16. {verifiers-0.3.2.dev45 → verifiers-0.3.2.dev47}/verifiers/v1/harnesses/prime_agent/harness.py +8 -18
  17. {verifiers-0.3.2.dev45 → verifiers-0.3.2.dev47}/verifiers/v1/harnesses/rlm/harness.py +9 -7
  18. {verifiers-0.3.2.dev45 → verifiers-0.3.2.dev47}/verifiers/v1/harnesses/terminus_2/harness.py +2 -4
  19. verifiers-0.3.2.dev47/verifiers/v1/harnesses/utils/install.py +65 -0
  20. {verifiers-0.3.2.dev45 → verifiers-0.3.2.dev47}/verifiers/v1/rollout.py +1 -0
  21. {verifiers-0.3.2.dev45 → verifiers-0.3.2.dev47}/verifiers/v1/task.py +21 -0
  22. verifiers-0.3.2.dev47/verifiers/v1/tasksets/harbor/env.py +82 -0
  23. {verifiers-0.3.2.dev45 → verifiers-0.3.2.dev47}/verifiers/v1/tasksets/harbor/taskset.py +22 -20
  24. verifiers-0.3.2.dev45/verifiers/v1/tasksets/harbor/env.py +0 -127
  25. {verifiers-0.3.2.dev45 → verifiers-0.3.2.dev47}/.gitignore +0 -0
  26. {verifiers-0.3.2.dev45 → verifiers-0.3.2.dev47}/LICENSE +0 -0
  27. {verifiers-0.3.2.dev45 → verifiers-0.3.2.dev47}/README.md +0 -0
  28. {verifiers-0.3.2.dev45 → verifiers-0.3.2.dev47}/pyproject.toml +0 -0
  29. {verifiers-0.3.2.dev45 → verifiers-0.3.2.dev47}/tests/v1/conftest.py +0 -0
  30. {verifiers-0.3.2.dev45 → verifiers-0.3.2.dev47}/tests/v1/fixtures/counter_tool_v1.py +0 -0
  31. {verifiers-0.3.2.dev45 → verifiers-0.3.2.dev47}/tests/v1/fixtures/duet_v1.py +0 -0
  32. {verifiers-0.3.2.dev45 → verifiers-0.3.2.dev47}/tests/v1/fixtures/echo_acp_resume_v1.py +0 -0
  33. {verifiers-0.3.2.dev45 → verifiers-0.3.2.dev47}/tests/v1/fixtures/echo_agentic_v1.py +0 -0
  34. {verifiers-0.3.2.dev45 → verifiers-0.3.2.dev47}/tests/v1/fixtures/echo_tool_v1.py +0 -0
  35. {verifiers-0.3.2.dev45 → verifiers-0.3.2.dev47}/tests/v1/fixtures/echo_user_sim_v1.py +0 -0
  36. {verifiers-0.3.2.dev45 → verifiers-0.3.2.dev47}/tests/v1/fixtures/echo_v1.py +0 -0
  37. {verifiers-0.3.2.dev45 → verifiers-0.3.2.dev47}/tests/v1/fixtures/pyproject.toml +0 -0
  38. {verifiers-0.3.2.dev45 → verifiers-0.3.2.dev47}/tests/v1/fixtures/tool_response_image_v1.py +0 -0
  39. {verifiers-0.3.2.dev45 → verifiers-0.3.2.dev47}/tests/v1/test_color_codeword_tasks.py +0 -0
  40. {verifiers-0.3.2.dev45 → verifiers-0.3.2.dev47}/tests/v1/test_configs.py +0 -0
  41. {verifiers-0.3.2.dev45 → verifiers-0.3.2.dev47}/tests/v1/test_e2e.py +0 -0
  42. {verifiers-0.3.2.dev45 → verifiers-0.3.2.dev47}/tests/v1/test_envs.py +0 -0
  43. {verifiers-0.3.2.dev45 → verifiers-0.3.2.dev47}/tests/v1/test_graph.py +0 -0
  44. {verifiers-0.3.2.dev45 → verifiers-0.3.2.dev47}/tests/v1/test_judges.py +0 -0
  45. {verifiers-0.3.2.dev45 → verifiers-0.3.2.dev47}/tests/v1/test_taskset.py +0 -0
  46. {verifiers-0.3.2.dev45 → verifiers-0.3.2.dev47}/tests/v1/test_trace.py +0 -0
  47. {verifiers-0.3.2.dev45 → verifiers-0.3.2.dev47}/verifiers/__init__.py +0 -0
  48. {verifiers-0.3.2.dev45 → verifiers-0.3.2.dev47}/verifiers/v1/__init__.py +0 -0
  49. {verifiers-0.3.2.dev45 → verifiers-0.3.2.dev47}/verifiers/v1/acp/__init__.py +0 -0
  50. {verifiers-0.3.2.dev45 → verifiers-0.3.2.dev47}/verifiers/v1/acp/runner.py +0 -0
  51. {verifiers-0.3.2.dev45 → verifiers-0.3.2.dev47}/verifiers/v1/cli/__init__.py +0 -0
  52. {verifiers-0.3.2.dev45 → verifiers-0.3.2.dev47}/verifiers/v1/cli/dashboard/__init__.py +0 -0
  53. {verifiers-0.3.2.dev45 → verifiers-0.3.2.dev47}/verifiers/v1/cli/dashboard/base.py +0 -0
  54. {verifiers-0.3.2.dev45 → verifiers-0.3.2.dev47}/verifiers/v1/cli/dashboard/eval.py +0 -0
  55. {verifiers-0.3.2.dev45 → verifiers-0.3.2.dev47}/verifiers/v1/cli/dashboard/replay.py +0 -0
  56. {verifiers-0.3.2.dev45 → verifiers-0.3.2.dev47}/verifiers/v1/cli/dashboard/validate.py +0 -0
  57. {verifiers-0.3.2.dev45 → verifiers-0.3.2.dev47}/verifiers/v1/cli/debug.py +0 -0
  58. {verifiers-0.3.2.dev45 → verifiers-0.3.2.dev47}/verifiers/v1/cli/eval/__init__.py +0 -0
  59. {verifiers-0.3.2.dev45 → verifiers-0.3.2.dev47}/verifiers/v1/cli/eval/main.py +0 -0
  60. {verifiers-0.3.2.dev45 → verifiers-0.3.2.dev47}/verifiers/v1/cli/eval/resume.py +0 -0
  61. {verifiers-0.3.2.dev45 → verifiers-0.3.2.dev47}/verifiers/v1/cli/eval/runner.py +0 -0
  62. {verifiers-0.3.2.dev45 → verifiers-0.3.2.dev47}/verifiers/v1/cli/gepa.py +0 -0
  63. {verifiers-0.3.2.dev45 → verifiers-0.3.2.dev47}/verifiers/v1/cli/init.py +0 -0
  64. {verifiers-0.3.2.dev45 → verifiers-0.3.2.dev47}/verifiers/v1/cli/output.py +0 -0
  65. {verifiers-0.3.2.dev45 → verifiers-0.3.2.dev47}/verifiers/v1/cli/replay.py +0 -0
  66. {verifiers-0.3.2.dev45 → verifiers-0.3.2.dev47}/verifiers/v1/cli/resolve.py +0 -0
  67. {verifiers-0.3.2.dev45 → verifiers-0.3.2.dev47}/verifiers/v1/cli/resume.py +0 -0
  68. {verifiers-0.3.2.dev45 → verifiers-0.3.2.dev47}/verifiers/v1/cli/validate.py +0 -0
  69. {verifiers-0.3.2.dev45 → verifiers-0.3.2.dev47}/verifiers/v1/clients/__init__.py +0 -0
  70. {verifiers-0.3.2.dev45 → verifiers-0.3.2.dev47}/verifiers/v1/clients/base.py +0 -0
  71. {verifiers-0.3.2.dev45 → verifiers-0.3.2.dev47}/verifiers/v1/clients/client.py +0 -0
  72. {verifiers-0.3.2.dev45 → verifiers-0.3.2.dev47}/verifiers/v1/clients/eval.py +0 -0
  73. {verifiers-0.3.2.dev45 → verifiers-0.3.2.dev47}/verifiers/v1/clients/train.py +0 -0
  74. {verifiers-0.3.2.dev45 → verifiers-0.3.2.dev47}/verifiers/v1/configs/__init__.py +0 -0
  75. {verifiers-0.3.2.dev45 → verifiers-0.3.2.dev47}/verifiers/v1/configs/agent.py +0 -0
  76. {verifiers-0.3.2.dev45 → verifiers-0.3.2.dev47}/verifiers/v1/configs/cli/__init__.py +0 -0
  77. {verifiers-0.3.2.dev45 → verifiers-0.3.2.dev47}/verifiers/v1/configs/cli/debug.py +0 -0
  78. {verifiers-0.3.2.dev45 → verifiers-0.3.2.dev47}/verifiers/v1/configs/cli/env.py +0 -0
  79. {verifiers-0.3.2.dev45 → verifiers-0.3.2.dev47}/verifiers/v1/configs/cli/eval.py +0 -0
  80. {verifiers-0.3.2.dev45 → verifiers-0.3.2.dev47}/verifiers/v1/configs/cli/init.py +0 -0
  81. {verifiers-0.3.2.dev45 → verifiers-0.3.2.dev47}/verifiers/v1/configs/cli/replay.py +0 -0
  82. {verifiers-0.3.2.dev45 → verifiers-0.3.2.dev47}/verifiers/v1/configs/cli/validate.py +0 -0
  83. {verifiers-0.3.2.dev45 → verifiers-0.3.2.dev47}/verifiers/v1/configs/client.py +0 -0
  84. {verifiers-0.3.2.dev45 → verifiers-0.3.2.dev47}/verifiers/v1/configs/env.py +0 -0
  85. {verifiers-0.3.2.dev45 → verifiers-0.3.2.dev47}/verifiers/v1/configs/judge.py +0 -0
  86. {verifiers-0.3.2.dev45 → verifiers-0.3.2.dev47}/verifiers/v1/configs/retries.py +0 -0
  87. {verifiers-0.3.2.dev45 → verifiers-0.3.2.dev47}/verifiers/v1/configs/runtime.py +0 -0
  88. {verifiers-0.3.2.dev45 → verifiers-0.3.2.dev47}/verifiers/v1/configs/serve.py +0 -0
  89. {verifiers-0.3.2.dev45 → verifiers-0.3.2.dev47}/verifiers/v1/configs/task.py +0 -0
  90. {verifiers-0.3.2.dev45 → verifiers-0.3.2.dev47}/verifiers/v1/configs/taskset.py +0 -0
  91. {verifiers-0.3.2.dev45 → verifiers-0.3.2.dev47}/verifiers/v1/dialects/__init__.py +0 -0
  92. {verifiers-0.3.2.dev45 → verifiers-0.3.2.dev47}/verifiers/v1/dialects/anthropic.py +0 -0
  93. {verifiers-0.3.2.dev45 → verifiers-0.3.2.dev47}/verifiers/v1/dialects/base.py +0 -0
  94. {verifiers-0.3.2.dev45 → verifiers-0.3.2.dev47}/verifiers/v1/dialects/chat.py +0 -0
  95. {verifiers-0.3.2.dev45 → verifiers-0.3.2.dev47}/verifiers/v1/dialects/responses.py +0 -0
  96. {verifiers-0.3.2.dev45 → verifiers-0.3.2.dev47}/verifiers/v1/env.py +0 -0
  97. {verifiers-0.3.2.dev45 → verifiers-0.3.2.dev47}/verifiers/v1/envs/__init__.py +0 -0
  98. {verifiers-0.3.2.dev45 → verifiers-0.3.2.dev47}/verifiers/v1/envs/agentic_judge/__init__.py +0 -0
  99. {verifiers-0.3.2.dev45 → verifiers-0.3.2.dev47}/verifiers/v1/envs/agentic_judge/env.py +0 -0
  100. {verifiers-0.3.2.dev45 → verifiers-0.3.2.dev47}/verifiers/v1/envs/best_of_n/__init__.py +0 -0
  101. {verifiers-0.3.2.dev45 → verifiers-0.3.2.dev47}/verifiers/v1/envs/best_of_n/env.py +0 -0
  102. {verifiers-0.3.2.dev45 → verifiers-0.3.2.dev47}/verifiers/v1/envs/shared_agentic_judge/__init__.py +0 -0
  103. {verifiers-0.3.2.dev45 → verifiers-0.3.2.dev47}/verifiers/v1/envs/single_agent/__init__.py +0 -0
  104. {verifiers-0.3.2.dev45 → verifiers-0.3.2.dev47}/verifiers/v1/envs/single_agent/env.py +0 -0
  105. {verifiers-0.3.2.dev45 → verifiers-0.3.2.dev47}/verifiers/v1/envs/user_sim/__init__.py +0 -0
  106. {verifiers-0.3.2.dev45 → verifiers-0.3.2.dev47}/verifiers/v1/envs/user_sim/env.py +0 -0
  107. {verifiers-0.3.2.dev45 → verifiers-0.3.2.dev47}/verifiers/v1/episode.py +0 -0
  108. {verifiers-0.3.2.dev45 → verifiers-0.3.2.dev47}/verifiers/v1/errors.py +0 -0
  109. {verifiers-0.3.2.dev45 → verifiers-0.3.2.dev47}/verifiers/v1/gepa/__init__.py +0 -0
  110. {verifiers-0.3.2.dev45 → verifiers-0.3.2.dev47}/verifiers/v1/gepa/adapter.py +0 -0
  111. {verifiers-0.3.2.dev45 → verifiers-0.3.2.dev47}/verifiers/v1/gepa/config.py +0 -0
  112. {verifiers-0.3.2.dev45 → verifiers-0.3.2.dev47}/verifiers/v1/gepa/dataset.py +0 -0
  113. {verifiers-0.3.2.dev45 → verifiers-0.3.2.dev47}/verifiers/v1/gepa/reflection.py +0 -0
  114. {verifiers-0.3.2.dev45 → verifiers-0.3.2.dev47}/verifiers/v1/gepa/runner.py +0 -0
  115. {verifiers-0.3.2.dev45 → verifiers-0.3.2.dev47}/verifiers/v1/graph.py +0 -0
  116. {verifiers-0.3.2.dev45 → verifiers-0.3.2.dev47}/verifiers/v1/harness.py +0 -0
  117. {verifiers-0.3.2.dev45 → verifiers-0.3.2.dev47}/verifiers/v1/harnesses/__init__.py +0 -0
  118. {verifiers-0.3.2.dev45 → verifiers-0.3.2.dev47}/verifiers/v1/harnesses/bash/__init__.py +0 -0
  119. {verifiers-0.3.2.dev45 → verifiers-0.3.2.dev47}/verifiers/v1/harnesses/bash/harness.py +0 -0
  120. {verifiers-0.3.2.dev45 → verifiers-0.3.2.dev47}/verifiers/v1/harnesses/browser_use/__init__.py +0 -0
  121. {verifiers-0.3.2.dev45 → verifiers-0.3.2.dev47}/verifiers/v1/harnesses/browser_use/harness.py +0 -0
  122. {verifiers-0.3.2.dev45 → verifiers-0.3.2.dev47}/verifiers/v1/harnesses/browser_use/program.py +0 -0
  123. {verifiers-0.3.2.dev45 → verifiers-0.3.2.dev47}/verifiers/v1/harnesses/claude_code/__init__.py +0 -0
  124. {verifiers-0.3.2.dev45 → verifiers-0.3.2.dev47}/verifiers/v1/harnesses/codex/__init__.py +0 -0
  125. {verifiers-0.3.2.dev45 → verifiers-0.3.2.dev47}/verifiers/v1/harnesses/hermes_agent/__init__.py +0 -0
  126. {verifiers-0.3.2.dev45 → verifiers-0.3.2.dev47}/verifiers/v1/harnesses/hermes_agent/program.py +0 -0
  127. {verifiers-0.3.2.dev45 → verifiers-0.3.2.dev47}/verifiers/v1/harnesses/kimi_code/__init__.py +0 -0
  128. {verifiers-0.3.2.dev45 → verifiers-0.3.2.dev47}/verifiers/v1/harnesses/mini_swe_agent/__init__.py +0 -0
  129. {verifiers-0.3.2.dev45 → verifiers-0.3.2.dev47}/verifiers/v1/harnesses/mini_swe_agent/program.py +0 -0
  130. {verifiers-0.3.2.dev45 → verifiers-0.3.2.dev47}/verifiers/v1/harnesses/null/__init__.py +0 -0
  131. {verifiers-0.3.2.dev45 → verifiers-0.3.2.dev47}/verifiers/v1/harnesses/null/harness.py +0 -0
  132. {verifiers-0.3.2.dev45 → verifiers-0.3.2.dev47}/verifiers/v1/harnesses/openclaw/__init__.py +0 -0
  133. {verifiers-0.3.2.dev45 → verifiers-0.3.2.dev47}/verifiers/v1/harnesses/pi/__init__.py +0 -0
  134. {verifiers-0.3.2.dev45 → verifiers-0.3.2.dev47}/verifiers/v1/harnesses/pool/__init__.py +0 -0
  135. {verifiers-0.3.2.dev45 → verifiers-0.3.2.dev47}/verifiers/v1/harnesses/prime_agent/__init__.py +0 -0
  136. {verifiers-0.3.2.dev45 → verifiers-0.3.2.dev47}/verifiers/v1/harnesses/rlm/__init__.py +0 -0
  137. {verifiers-0.3.2.dev45 → verifiers-0.3.2.dev47}/verifiers/v1/harnesses/terminus_2/__init__.py +0 -0
  138. {verifiers-0.3.2.dev45 → verifiers-0.3.2.dev47}/verifiers/v1/harnesses/terminus_2/program.py +0 -0
  139. {verifiers-0.3.2.dev45 → verifiers-0.3.2.dev47}/verifiers/v1/harnesses/utils/__init__.py +0 -0
  140. {verifiers-0.3.2.dev45 → verifiers-0.3.2.dev47}/verifiers/v1/harnesses/utils/compaction.py +0 -0
  141. {verifiers-0.3.2.dev45 → verifiers-0.3.2.dev47}/verifiers/v1/harnesses/utils/core.py +0 -0
  142. {verifiers-0.3.2.dev45 → verifiers-0.3.2.dev47}/verifiers/v1/harnesses/utils/launch.py +0 -0
  143. {verifiers-0.3.2.dev45 → verifiers-0.3.2.dev47}/verifiers/v1/harnesses/utils/mcp.py +0 -0
  144. {verifiers-0.3.2.dev45 → verifiers-0.3.2.dev47}/verifiers/v1/interception/__init__.py +0 -0
  145. {verifiers-0.3.2.dev45 → verifiers-0.3.2.dev47}/verifiers/v1/interception/base.py +0 -0
  146. {verifiers-0.3.2.dev45 → verifiers-0.3.2.dev47}/verifiers/v1/interception/pool.py +0 -0
  147. {verifiers-0.3.2.dev45 → verifiers-0.3.2.dev47}/verifiers/v1/interception/server.py +0 -0
  148. {verifiers-0.3.2.dev45 → verifiers-0.3.2.dev47}/verifiers/v1/interception/tool.py +0 -0
  149. {verifiers-0.3.2.dev45 → verifiers-0.3.2.dev47}/verifiers/v1/interception/tunnel/__init__.py +0 -0
  150. {verifiers-0.3.2.dev45 → verifiers-0.3.2.dev47}/verifiers/v1/interception/tunnel/base.py +0 -0
  151. {verifiers-0.3.2.dev45 → verifiers-0.3.2.dev47}/verifiers/v1/interception/tunnel/custom.py +0 -0
  152. {verifiers-0.3.2.dev45 → verifiers-0.3.2.dev47}/verifiers/v1/interception/tunnel/prime.py +0 -0
  153. {verifiers-0.3.2.dev45 → verifiers-0.3.2.dev47}/verifiers/v1/judge.py +0 -0
  154. {verifiers-0.3.2.dev45 → verifiers-0.3.2.dev47}/verifiers/v1/judges/__init__.py +0 -0
  155. {verifiers-0.3.2.dev45 → verifiers-0.3.2.dev47}/verifiers/v1/judges/reference.py +0 -0
  156. {verifiers-0.3.2.dev45 → verifiers-0.3.2.dev47}/verifiers/v1/judges/reference.txt +0 -0
  157. {verifiers-0.3.2.dev45 → verifiers-0.3.2.dev47}/verifiers/v1/judges/rubric.py +0 -0
  158. {verifiers-0.3.2.dev45 → verifiers-0.3.2.dev47}/verifiers/v1/judges/rubric.txt +0 -0
  159. {verifiers-0.3.2.dev45 → verifiers-0.3.2.dev47}/verifiers/v1/mcp/__init__.py +0 -0
  160. {verifiers-0.3.2.dev45 → verifiers-0.3.2.dev47}/verifiers/v1/mcp/launch.py +0 -0
  161. {verifiers-0.3.2.dev45 → verifiers-0.3.2.dev47}/verifiers/v1/mcp/server.py +0 -0
  162. {verifiers-0.3.2.dev45 → verifiers-0.3.2.dev47}/verifiers/v1/mcp/toolset.py +0 -0
  163. {verifiers-0.3.2.dev45 → verifiers-0.3.2.dev47}/verifiers/v1/runtimes/__init__.py +0 -0
  164. {verifiers-0.3.2.dev45 → verifiers-0.3.2.dev47}/verifiers/v1/runtimes/base.py +0 -0
  165. {verifiers-0.3.2.dev45 → verifiers-0.3.2.dev47}/verifiers/v1/runtimes/docker/__init__.py +0 -0
  166. {verifiers-0.3.2.dev45 → verifiers-0.3.2.dev47}/verifiers/v1/runtimes/docker/egress.py +0 -0
  167. {verifiers-0.3.2.dev45 → verifiers-0.3.2.dev47}/verifiers/v1/runtimes/limiters.py +0 -0
  168. {verifiers-0.3.2.dev45 → verifiers-0.3.2.dev47}/verifiers/v1/runtimes/modal.py +0 -0
  169. {verifiers-0.3.2.dev45 → verifiers-0.3.2.dev47}/verifiers/v1/runtimes/prime.py +0 -0
  170. {verifiers-0.3.2.dev45 → verifiers-0.3.2.dev47}/verifiers/v1/runtimes/subprocess.py +0 -0
  171. {verifiers-0.3.2.dev45 → verifiers-0.3.2.dev47}/verifiers/v1/semantic.py +0 -0
  172. {verifiers-0.3.2.dev45 → verifiers-0.3.2.dev47}/verifiers/v1/serve/__init__.py +0 -0
  173. {verifiers-0.3.2.dev45 → verifiers-0.3.2.dev47}/verifiers/v1/serve/client.py +0 -0
  174. {verifiers-0.3.2.dev45 → verifiers-0.3.2.dev47}/verifiers/v1/serve/encoding.py +0 -0
  175. {verifiers-0.3.2.dev45 → verifiers-0.3.2.dev47}/verifiers/v1/serve/pool.py +0 -0
  176. {verifiers-0.3.2.dev45 → verifiers-0.3.2.dev47}/verifiers/v1/serve/server.py +0 -0
  177. {verifiers-0.3.2.dev45 → verifiers-0.3.2.dev47}/verifiers/v1/serve/types.py +0 -0
  178. {verifiers-0.3.2.dev45 → verifiers-0.3.2.dev47}/verifiers/v1/session.py +0 -0
  179. {verifiers-0.3.2.dev45 → verifiers-0.3.2.dev47}/verifiers/v1/state.py +0 -0
  180. {verifiers-0.3.2.dev45 → verifiers-0.3.2.dev47}/verifiers/v1/taskset.py +0 -0
  181. {verifiers-0.3.2.dev45 → verifiers-0.3.2.dev47}/verifiers/v1/tasksets/__init__.py +0 -0
  182. {verifiers-0.3.2.dev45 → verifiers-0.3.2.dev47}/verifiers/v1/tasksets/harbor/__init__.py +0 -0
  183. {verifiers-0.3.2.dev45 → verifiers-0.3.2.dev47}/verifiers/v1/tasksets/lean/__init__.py +0 -0
  184. {verifiers-0.3.2.dev45 → verifiers-0.3.2.dev47}/verifiers/v1/tasksets/lean/scoring.py +0 -0
  185. {verifiers-0.3.2.dev45 → verifiers-0.3.2.dev47}/verifiers/v1/tasksets/lean/taskset.py +0 -0
  186. {verifiers-0.3.2.dev45 → verifiers-0.3.2.dev47}/verifiers/v1/tasksets/nemo_gym/__init__.py +0 -0
  187. {verifiers-0.3.2.dev45 → verifiers-0.3.2.dev47}/verifiers/v1/tasksets/nemo_gym/response.py +0 -0
  188. {verifiers-0.3.2.dev45 → verifiers-0.3.2.dev47}/verifiers/v1/tasksets/nemo_gym/server.py +0 -0
  189. {verifiers-0.3.2.dev45 → verifiers-0.3.2.dev47}/verifiers/v1/tasksets/nemo_gym/taskset.py +0 -0
  190. {verifiers-0.3.2.dev45 → verifiers-0.3.2.dev47}/verifiers/v1/tasksets/nemo_gym/toolset.py +0 -0
  191. {verifiers-0.3.2.dev45 → verifiers-0.3.2.dev47}/verifiers/v1/tasksets/openenv/__init__.py +0 -0
  192. {verifiers-0.3.2.dev45 → verifiers-0.3.2.dev47}/verifiers/v1/tasksets/openenv/taskset.py +0 -0
  193. {verifiers-0.3.2.dev45 → verifiers-0.3.2.dev47}/verifiers/v1/tasksets/textarena/__init__.py +0 -0
  194. {verifiers-0.3.2.dev45 → verifiers-0.3.2.dev47}/verifiers/v1/tasksets/textarena/taskset.py +0 -0
  195. {verifiers-0.3.2.dev45 → verifiers-0.3.2.dev47}/verifiers/v1/trace.py +0 -0
  196. {verifiers-0.3.2.dev45 → verifiers-0.3.2.dev47}/verifiers/v1/types.py +0 -0
  197. {verifiers-0.3.2.dev45 → verifiers-0.3.2.dev47}/verifiers/v1/utils/__init__.py +0 -0
  198. {verifiers-0.3.2.dev45 → verifiers-0.3.2.dev47}/verifiers/v1/utils/aio.py +0 -0
  199. {verifiers-0.3.2.dev45 → verifiers-0.3.2.dev47}/verifiers/v1/utils/artifacts.py +0 -0
  200. {verifiers-0.3.2.dev45 → verifiers-0.3.2.dev47}/verifiers/v1/utils/compile.py +0 -0
  201. {verifiers-0.3.2.dev45 → verifiers-0.3.2.dev47}/verifiers/v1/utils/decorators.py +0 -0
  202. {verifiers-0.3.2.dev45 → verifiers-0.3.2.dev47}/verifiers/v1/utils/format.py +0 -0
  203. {verifiers-0.3.2.dev45 → verifiers-0.3.2.dev47}/verifiers/v1/utils/generic.py +0 -0
  204. {verifiers-0.3.2.dev45 → verifiers-0.3.2.dev47}/verifiers/v1/utils/git.py +0 -0
  205. {verifiers-0.3.2.dev45 → verifiers-0.3.2.dev47}/verifiers/v1/utils/image.py +0 -0
  206. {verifiers-0.3.2.dev45 → verifiers-0.3.2.dev47}/verifiers/v1/utils/interrupt.py +0 -0
  207. {verifiers-0.3.2.dev45 → verifiers-0.3.2.dev47}/verifiers/v1/utils/loaders.py +0 -0
  208. {verifiers-0.3.2.dev45 → verifiers-0.3.2.dev47}/verifiers/v1/utils/logging.py +0 -0
  209. {verifiers-0.3.2.dev45 → verifiers-0.3.2.dev47}/verifiers/v1/utils/memory.py +0 -0
  210. {verifiers-0.3.2.dev45 → verifiers-0.3.2.dev47}/verifiers/v1/utils/paths.py +0 -0
  211. {verifiers-0.3.2.dev45 → verifiers-0.3.2.dev47}/verifiers/v1/utils/platform.py +0 -0
  212. {verifiers-0.3.2.dev45 → verifiers-0.3.2.dev47}/verifiers/v1/utils/prime.py +0 -0
  213. {verifiers-0.3.2.dev45 → verifiers-0.3.2.dev47}/verifiers/v1/utils/retries.py +0 -0
  214. {verifiers-0.3.2.dev45 → verifiers-0.3.2.dev47}/verifiers/v1/utils/score.py +0 -0
  215. {verifiers-0.3.2.dev45 → verifiers-0.3.2.dev47}/verifiers/v1/utils/version.py +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.5
2
2
  Name: verifiers
3
- Version: 0.3.2.dev45
3
+ Version: 0.3.2.dev47
4
4
  Summary: Verifiers: Environments for LLM Reinforcement Learning
5
5
  Project-URL: Homepage, https://github.com/primeintellect-ai/verifiers
6
6
  Project-URL: Documentation, https://github.com/primeintellect-ai/verifiers
@@ -89,6 +89,19 @@ async def test_config_plugged_fns_merge_and_override(tmp_path) -> None:
89
89
  HookTask(HookData(idx=0, prompt="abc"), config).hooks("reward")
90
90
 
91
91
 
92
+ async def test_defer_scoring_defers_only_task_scoring() -> None:
93
+ task = HookTask(HookData(idx=0, prompt="abc"))
94
+ trace = vf.Trace(
95
+ agent=vf.AgentInfo(config=vf.AgentConfig()),
96
+ task=vf.TraceTask(type="HookTask", data=task.data),
97
+ )
98
+
99
+ await task.defer_scoring().score(trace)
100
+ assert trace.rewards == {}
101
+ await task.score(trace)
102
+ assert set(trace.rewards) == {"fmt", "lcs"}
103
+
104
+
92
105
  def test_compare_stdout_results_accepts_token_equal_text() -> None:
93
106
  assert vf.compare_stdout_results("hello world\n", "hello world\n")
94
107
 
@@ -11,7 +11,7 @@ import asyncio
11
11
  import logging
12
12
  from collections.abc import AsyncIterator, Callable, Mapping
13
13
  from contextlib import asynccontextmanager, nullcontext
14
- from dataclasses import dataclass
14
+ from dataclasses import dataclass, replace
15
15
  from typing import Self
16
16
 
17
17
  from verifiers.v1.clients import (
@@ -54,6 +54,27 @@ __all__ = ["Agent", "AgentConfig", "Agents", "TimeoutConfig", "make_agent"]
54
54
  logger = logging.getLogger(__name__)
55
55
 
56
56
 
57
+ def resolve_rollout_timeouts(timeout: TimeoutConfig, task: Task) -> RolloutTimeouts:
58
+ """Apply an agent's stage-timeout precedence to one task."""
59
+ agent_timeout = (
60
+ timeout.rollout if timeout.rollout is not None else task.data.timeout.agent
61
+ )
62
+ return RolloutTimeouts(
63
+ setup=timeout.setup if timeout.setup is not None else task.data.timeout.setup,
64
+ agent=agent_timeout,
65
+ finalize=(
66
+ timeout.finalize
67
+ if timeout.finalize is not None
68
+ else task.data.timeout.finalize
69
+ ),
70
+ scoring=(
71
+ timeout.scoring
72
+ if timeout.scoring is not None
73
+ else task.data.timeout.scoring
74
+ ),
75
+ )
76
+
77
+
57
78
  def _check_borrowed_placement(
58
79
  task: Task, runtime: Runtime, base_config: RuntimeConfig
59
80
  ) -> None:
@@ -526,34 +547,15 @@ class Agent:
526
547
  runtime_config,
527
548
  tools=[*task.toolsets(task.config), *shared_tools.values()],
528
549
  )
529
- # Timeout precedence: agent-level wins, else the task's, else no limit.
530
- agent_timeout = (
531
- self.timeout.rollout
532
- if self.timeout.rollout is not None
533
- else task.data.timeout.agent
534
- )
550
+ timeouts = resolve_rollout_timeouts(self.timeout, task)
535
551
  return {
536
552
  "agent_config": self.config,
537
553
  "harness": self.harness,
538
554
  "ctx": self.ctx,
539
555
  "runtime_config": runtime_config,
540
- "timeouts": RolloutTimeouts(
541
- setup=(
542
- self.timeout.setup
543
- if self.timeout.setup is not None
544
- else task.data.timeout.setup
545
- ),
546
- agent=cap_remote_agent_timeout(agent_timeout, runtime_config, task),
547
- finalize=(
548
- self.timeout.finalize
549
- if self.timeout.finalize is not None
550
- else task.data.timeout.finalize
551
- ),
552
- scoring=(
553
- self.timeout.scoring
554
- if self.timeout.scoring is not None
555
- else task.data.timeout.scoring
556
- ),
556
+ "timeouts": replace(
557
+ timeouts,
558
+ agent=cap_remote_agent_timeout(timeouts.agent, runtime_config, task),
557
559
  ),
558
560
  "limits": self.limits,
559
561
  "shared_tools": shared_tools,
@@ -4,12 +4,16 @@ from __future__ import annotations
4
4
 
5
5
  import os
6
6
  from pathlib import Path
7
+ from typing import Annotated
7
8
 
8
9
  from pydantic import ConfigDict, Field, FiniteFloat
9
10
  from pydantic_config import BaseConfig
10
11
 
11
12
  from verifiers.v1.types import ID
12
13
 
14
+ PinnedVersion = Annotated[str, Field(pattern=r"^[A-Za-z0-9._+-]+$")]
15
+ """A release/tag a harness pins its program install to."""
16
+
13
17
 
14
18
  class HarnessConfig(BaseConfig):
15
19
  id: ID = "bash"
@@ -0,0 +1,7 @@
1
+ from verifiers.v1.envs.isolated_verifier.env import (
2
+ IsolatedVerifierEnv,
3
+ IsolatedVerifierEnvConfig,
4
+ VerifierConfig,
5
+ )
6
+
7
+ __all__ = ["IsolatedVerifierEnv", "IsolatedVerifierEnvConfig", "VerifierConfig"]
@@ -0,0 +1,182 @@
1
+ """Deterministic task verification in a fresh runtime.
2
+
3
+ One solver agent runs the task. Its task scoring is deferred, declared artifacts
4
+ are collected after normal task finalization, and the solver runtime is destroyed.
5
+ The task is then set up with a fresh controller in a fresh runtime, its artifacts
6
+ are restored, and its ordinary metrics and rewards run there onto the solver's trace.
7
+ """
8
+
9
+ import asyncio
10
+ import copy
11
+ import logging
12
+ from contextlib import AsyncExitStack
13
+ from pathlib import PurePosixPath
14
+ from typing import Any
15
+
16
+ from pydantic import Field, SerializationInfo, field_serializer
17
+
18
+ import verifiers.v1 as vf
19
+ from verifiers.v1.agent import resolve_rollout_timeouts
20
+ from verifiers.v1.errors import TaskError, boundary
21
+ from verifiers.v1.runtimes import Runtime, RuntimeConfig, provision_runtime
22
+ from verifiers.v1.utils.compile import resolve_runtime_config
23
+ from verifiers.v1.utils.decorators import invoke
24
+ from verifiers.v1.utils.retries import backoff
25
+
26
+ logger = logging.getLogger(__name__)
27
+
28
+
29
+ class VerifierConfig(vf.BaseConfig):
30
+ runtime: RuntimeConfig | None = None
31
+ """Independent verifier placement and policy. None provisions a fresh runtime
32
+ equivalent to the solver's resolved task runtime."""
33
+ env: dict[str, str] | None = None
34
+ """Process environment for verifier setup and scoring. None uses the task's
35
+ normal runtime environment."""
36
+ retries: int = Field(2, ge=0)
37
+ """Extra fresh-runtime attempts after setup, restoration, staging, or scoring
38
+ failures."""
39
+
40
+ @field_serializer("runtime")
41
+ def serialize_runtime(
42
+ self, runtime: RuntimeConfig | None, info: SerializationInfo
43
+ ) -> dict | None:
44
+ """Keep an omitted image omitted across a resolved-config round trip."""
45
+ if runtime is None:
46
+ return None
47
+ values = runtime.model_dump(mode=info.mode)
48
+ if "image" not in runtime.model_fields_set:
49
+ values.pop("image", None)
50
+ return values
51
+
52
+
53
+ class IsolatedVerifierEnvConfig(vf.EnvConfig):
54
+ agent: vf.AgentConfig = vf.AgentConfig()
55
+ """The one solver seat."""
56
+ verifier: VerifierConfig = VerifierConfig()
57
+ """The fresh verifier's runtime, process environment, and retry policy."""
58
+
59
+
60
+ class IsolatedVerifierEnv(vf.Env[IsolatedVerifierEnvConfig]):
61
+ """Run one solver, then its deterministic task scoring in a fresh runtime."""
62
+
63
+ async def run(self, task: vf.Task, agents: vf.Agents) -> None:
64
+ if task.config.judges:
65
+ raise ValueError(
66
+ "isolated-verifier runs deterministic task metrics and rewards; "
67
+ "model-backed task judges are not supported"
68
+ )
69
+ self.verifier_config(task) # Refuse an impossible verifier before solving.
70
+ await agents.agent.run(task.defer_scoring(), collect_artifacts=True)
71
+
72
+ def verifier_config(self, task: vf.Task) -> RuntimeConfig:
73
+ base = self.config.verifier.runtime or self.config.agent.runtime
74
+ config = resolve_runtime_config(base, task)
75
+ image_spec = type(base).model_fields.get("image")
76
+ if (
77
+ self.config.verifier.runtime is not None
78
+ and image_spec is not None
79
+ and "image" in base.model_fields_set
80
+ ):
81
+ config = config.model_copy(update={"image": base.image})
82
+ if isinstance(config, vf.SubprocessConfig):
83
+ raise TypeError(
84
+ "isolated-verifier requires a container runtime so artifacts can be "
85
+ "restored safely; configure the agent or verifier runtime as docker, "
86
+ "prime, or modal"
87
+ )
88
+ relative = [
89
+ artifact.source
90
+ for artifact in task.data.artifacts
91
+ if not PurePosixPath(artifact.source).is_absolute()
92
+ ]
93
+ solver = resolve_runtime_config(self.config.agent.runtime, task)
94
+ solver_workdir = PurePosixPath(getattr(solver, "workdir", "") or "/")
95
+ verifier_workdir = PurePosixPath(config.workdir)
96
+ if relative and solver_workdir != verifier_workdir:
97
+ raise ValueError(
98
+ "isolated-verifier cannot transfer relative artifacts "
99
+ f"{relative!r} between solver workdir {str(solver_workdir)!r} and "
100
+ f"verifier workdir {str(verifier_workdir)!r}; use matching workdirs "
101
+ "or absolute artifact paths"
102
+ )
103
+ return config
104
+
105
+ async def finalize(self, task: vf.Task, episode: vf.Episode) -> None:
106
+ solution = episode.traces[0]
107
+ if solution.ok:
108
+ graded = await self.grade(self.verifier_config(task), task, solution)
109
+ episode.traces[0] = graded[1]
110
+
111
+ async def stage_verifier(
112
+ self, task: vf.Task, solution: vf.Trace, runtime: Runtime
113
+ ) -> None:
114
+ artifacts = dict(solution.state.artifacts)
115
+ async with boundary(TaskError, "verifier task setup"):
116
+ await invoke(task.setup, {"trace": solution, "runtime": runtime})
117
+ await vf.restore(runtime, artifacts)
118
+ async with boundary(TaskError, "verifier staging"):
119
+ await invoke(task.stage_verifier, {"trace": solution, "runtime": runtime})
120
+
121
+ async def verify(self, task: vf.Task, solution: vf.Trace, runtime: Runtime) -> Any:
122
+ await task.score(solution, runtime)
123
+
124
+ async def grade(
125
+ self,
126
+ config: RuntimeConfig,
127
+ task: vf.Task,
128
+ solution: vf.Trace,
129
+ *,
130
+ scoring_timeout_covers_attempt: bool = False,
131
+ ) -> tuple[Any, vf.Trace]:
132
+ timeouts = resolve_rollout_timeouts(self.config.agent.timeout, task)
133
+ last: Exception | None = None
134
+ for attempt in range(self.config.verifier.retries + 1):
135
+ if attempt:
136
+ delay = backoff(attempt - 1)
137
+ logger.warning(
138
+ "isolated verifier attempt %d/%d failed (%s); retrying in %.1fs",
139
+ attempt,
140
+ self.config.verifier.retries + 1,
141
+ last,
142
+ delay,
143
+ )
144
+ await asyncio.sleep(delay)
145
+ try:
146
+ # Teardown is outside the stage deadlines: a completed score must
147
+ # survive a slow cleanup of the verifier runtime.
148
+ async with (
149
+ AsyncExitStack() as boxes,
150
+ asyncio.timeout(
151
+ timeouts.scoring if scoring_timeout_covers_attempt else None
152
+ ),
153
+ ):
154
+ async with asyncio.timeout(timeouts.setup):
155
+ # Failed setup or scoring must not alter the next attempt.
156
+ # Only the successful controller and trace leave this scope.
157
+ verifier_task = copy.deepcopy(task)
158
+ verifier_solution = copy.deepcopy(solution)
159
+ runtime = await boxes.enter_async_context(
160
+ provision_runtime(
161
+ config,
162
+ env=(
163
+ verifier_task.runtime_env()
164
+ if self.config.verifier.env is None
165
+ else self.config.verifier.env
166
+ ),
167
+ )
168
+ )
169
+ await runtime.prepare_setup()
170
+ await self.stage_verifier(
171
+ verifier_task, verifier_solution, runtime
172
+ )
173
+ await runtime.prepare_execution([])
174
+ async with asyncio.timeout(timeouts.scoring):
175
+ result = await self.verify(
176
+ verifier_task, verifier_solution, runtime
177
+ )
178
+ return result, verifier_solution
179
+ except Exception as error: # noqa: BLE001 - retry the whole fresh box
180
+ last = error
181
+ assert last is not None
182
+ raise last
@@ -1,13 +1,10 @@
1
1
  """Run Claude Code through the Claude Agent SDK ACP adapter."""
2
2
 
3
- import shlex
4
-
5
- from pydantic import Field
6
-
7
3
  from verifiers.v1.acp import ACPConfig, ACPHarness
8
4
  from verifiers.v1.clients import ModelContext
9
- from verifiers.v1.configs.harness import HarnessConfig
5
+ from verifiers.v1.configs.harness import HarnessConfig, PinnedVersion
10
6
  from verifiers.v1.harnesses.node import NODE_BIN_DIR, ensure_node
7
+ from verifiers.v1.harnesses.utils.install import ensure_installed, remove_dir
11
8
  from verifiers.v1.runtimes import Runtime
12
9
  from verifiers.v1.task import TaskData
13
10
  from verifiers.v1.trace import Trace
@@ -32,7 +29,7 @@ touch {ready}
32
29
 
33
30
 
34
31
  class ClaudeCodeHarnessConfig(HarnessConfig):
35
- version: str = Field(default="2.1.232", pattern=r"^[A-Za-z0-9._+-]+$")
32
+ version: PinnedVersion = "2.1.232"
36
33
  """Claude Code release to install, pinned for reproducibility."""
37
34
 
38
35
 
@@ -51,25 +48,18 @@ class ClaudeCodeHarness(ACPHarness[ClaudeCodeHarnessConfig]):
51
48
  acp_bin = ACP_BIN.format(**versions)
52
49
  ready = f"{directory}/.ready"
53
50
  script = ACP_INSTALL.replace("{packages}", packages).replace("{ready}", ready)
54
- ensure = shlex.quote(
55
- f"[ -f {ready} ] && [ -x {claude_bin} ] && [ -x {acp_bin} ] || ({script})"
56
- )
57
- acp_guarded = (
58
- f"mkdir -p {directory} && "
59
- f'"$(command -v flock || command -v lockf)" {directory}/install.lock '
60
- f"sh -c {ensure}"
61
- )
62
- acp_result = await runtime.run(
63
- ["sh", "-c", acp_guarded],
64
- {
51
+ await ensure_installed(
52
+ runtime,
53
+ directory=directory,
54
+ ready=f"[ -f {ready} ] && [ -x {claude_bin} ] && [ -x {acp_bin} ]",
55
+ install=script,
56
+ env={
65
57
  **self.config.resolved_env,
66
58
  "VF_CLAUDE_CODE_VERSION": self.config.version,
67
59
  "VF_CLAUDE_ACP_VERSION": ACP_VERSION,
68
60
  },
61
+ label="Claude Agent ACP",
69
62
  )
70
- if acp_result.exit_code != 0:
71
- detail = (acp_result.stderr or acp_result.stdout).strip()[-500:]
72
- raise RuntimeError(f"Claude Agent ACP install failed: {detail}")
73
63
  await super().setup(runtime)
74
64
 
75
65
  async def prepare_acp(
@@ -113,11 +103,7 @@ class ClaudeCodeHarness(ACPHarness[ClaudeCodeHarnessConfig]):
113
103
  )
114
104
 
115
105
  async def cleanup(self, trace: Trace, runtime: Runtime) -> None:
116
- result = await runtime.run(["rm", "-rf", self.config_dir(trace)], {})
117
- if result.exit_code != 0:
118
- raise RuntimeError(
119
- f"failed to clean up Claude config: {result.stderr.strip()[-500:]}"
120
- )
106
+ await remove_dir(runtime, self.config_dir(trace), "Claude config")
121
107
 
122
108
  @staticmethod
123
109
  def config_dir(trace: Trace) -> str:
@@ -4,15 +4,13 @@ import hashlib
4
4
  import json
5
5
  import logging
6
6
  import re
7
- import shlex
8
7
  from collections import Counter
9
8
 
10
- from pydantic import Field
11
-
12
9
  from verifiers.v1.acp import ACPConfig, ACPHarness
13
10
  from verifiers.v1.clients import ModelContext
14
- from verifiers.v1.configs.harness import HarnessConfig
11
+ from verifiers.v1.configs.harness import HarnessConfig, PinnedVersion
15
12
  from verifiers.v1.harnesses.node import NODE_BIN_DIR, ensure_node
13
+ from verifiers.v1.harnesses.utils.install import ensure_installed, remove_dir
16
14
  from verifiers.v1.runtimes import Runtime
17
15
  from verifiers.v1.task import TaskData
18
16
  from verifiers.v1.trace import Trace
@@ -38,7 +36,7 @@ touch {ready}
38
36
 
39
37
 
40
38
  class CodexHarnessConfig(HarnessConfig):
41
- version: str = Field(default="0.147.0", pattern=r"^[A-Za-z0-9._+-]+$")
39
+ version: PinnedVersion = "0.147.0"
42
40
  """Codex release to install, pinned for reproducibility."""
43
41
  multi_agent: bool = False
44
42
  """Enable Codex's native multi-agent v2 tools."""
@@ -64,24 +62,18 @@ class CodexHarness(ACPHarness[CodexHarnessConfig]):
64
62
  acp_bin = ACP_BIN.format(**versions)
65
63
  ready = f"{directory}/.ready"
66
64
  script = INSTALL.replace("{packages}", packages).replace("{ready}", ready)
67
- ensure = shlex.quote(
68
- f"[ -f {ready} ] && [ -x {codex_bin} ] && [ -x {acp_bin} ] || ({script})"
69
- )
70
- guarded = (
71
- f"mkdir -p {directory} && "
72
- f'"$(command -v flock || command -v lockf)" {directory}/install.lock '
73
- f"sh -c {ensure}"
74
- )
75
- install = await runtime.run(
76
- ["sh", "-c", guarded],
77
- {
65
+ await ensure_installed(
66
+ runtime,
67
+ directory=directory,
68
+ ready=f"[ -f {ready} ] && [ -x {codex_bin} ] && [ -x {acp_bin} ]",
69
+ install=script,
70
+ env={
78
71
  **self.config.resolved_env,
79
72
  "VF_CODEX_VERSION": self.config.version,
80
73
  "VF_CODEX_ACP_VERSION": ACP_VERSION,
81
74
  },
75
+ label="codex",
82
76
  )
83
- if install.exit_code != 0:
84
- raise RuntimeError(f"codex install failed: {install.stderr.strip()[-500:]}")
85
77
  await super().setup(runtime)
86
78
 
87
79
  async def prepare_acp(
@@ -112,11 +104,7 @@ class CodexHarness(ACPHarness[CodexHarnessConfig]):
112
104
  )
113
105
 
114
106
  async def cleanup(self, trace: Trace, runtime: Runtime) -> None:
115
- result = await runtime.run(["rm", "-rf", self.trace_home(trace)], {})
116
- if result.exit_code != 0:
117
- raise RuntimeError(
118
- f"failed to clean up Codex home: {result.stderr.strip()[-500:]}"
119
- )
107
+ await remove_dir(runtime, self.trace_home(trace), "Codex home")
120
108
 
121
109
  @staticmethod
122
110
  def trace_home(trace: Trace) -> str:
@@ -3,11 +3,10 @@
3
3
  import json
4
4
  from pathlib import Path
5
5
 
6
- from pydantic import Field
7
-
8
6
  from verifiers.v1.acp import ACPConfig, ACPHarness
9
7
  from verifiers.v1.clients import ModelContext
10
- from verifiers.v1.configs.harness import HarnessConfig
8
+ from verifiers.v1.configs.harness import HarnessConfig, PinnedVersion
9
+ from verifiers.v1.harnesses.utils.install import remove_dir
11
10
  from verifiers.v1.runtimes import Runtime
12
11
  from verifiers.v1.task import TaskData
13
12
  from verifiers.v1.trace import Trace
@@ -16,7 +15,7 @@ PROGRAM_SOURCE = (Path(__file__).resolve().parent / "program.py").read_text()
16
15
 
17
16
 
18
17
  class HermesAgentHarnessConfig(HarnessConfig):
19
- version: str = Field(default="0.19.0", pattern=r"^[A-Za-z0-9._+-]+$")
18
+ version: PinnedVersion = "0.19.0"
20
19
  """Hermes Agent release to install, pinned for reproducibility."""
21
20
  use_bundled_skill: bool = False
22
21
  """Enable Hermes Agent's bundled skill catalog in addition to uploaded skills."""
@@ -96,8 +95,4 @@ class HermesAgentHarness(ACPHarness[HermesAgentHarnessConfig]):
96
95
  )
97
96
 
98
97
  async def cleanup(self, trace: Trace, runtime: Runtime) -> None:
99
- result = await runtime.run(["rm", "-rf", f"/tmp/vf-hermes/{trace.id}"], {})
100
- if result.exit_code:
101
- raise RuntimeError(
102
- f"failed to clean up Hermes home: {result.stderr.strip()[-500:]}"
103
- )
98
+ await remove_dir(runtime, f"/tmp/vf-hermes/{trace.id}", "Hermes home")
@@ -1,15 +1,14 @@
1
1
  """Run Kimi Code's native ACP server against interception."""
2
2
 
3
3
  import logging
4
- import shlex
5
4
  from typing import Literal
6
5
 
7
6
  import tomli_w
8
- from pydantic import Field
9
7
 
10
8
  from verifiers.v1.acp import ACPConfig, ACPHarness
11
9
  from verifiers.v1.clients import ModelContext
12
- from verifiers.v1.configs.harness import HarnessConfig
10
+ from verifiers.v1.configs.harness import HarnessConfig, PinnedVersion
11
+ from verifiers.v1.harnesses.utils.install import ensure_installed
13
12
  from verifiers.v1.runtimes import Runtime
14
13
  from verifiers.v1.task import TaskData
15
14
  from verifiers.v1.trace import Trace
@@ -39,7 +38,7 @@ env \
39
38
 
40
39
 
41
40
  class KimiCodeHarnessConfig(HarnessConfig):
42
- version: str = Field(default="0.36.0", pattern=r"^[A-Za-z0-9._+-]+$")
41
+ version: PinnedVersion = "0.36.0"
43
42
  """Kimi Code release to install, pinned for reproducibility."""
44
43
  transport: Literal["chat_completions", "responses", "anthropic_messages"] = (
45
44
  "chat_completions"
@@ -58,16 +57,13 @@ class KimiCodeHarness(ACPHarness[KimiCodeHarnessConfig]):
58
57
  "kimi-code: ensuring Kimi Code %s is installed", self.config.version
59
58
  )
60
59
  script = INSTALL.replace("{version}", self.config.version)
61
- guarded = (
62
- "mkdir -p /tmp/vf-kimi-code && "
63
- '"$(command -v flock || command -v lockf)" '
64
- f"/tmp/vf-kimi-code/install.lock sh -c {shlex.quote(script)}"
60
+ await ensure_installed(
61
+ runtime,
62
+ directory="/tmp/vf-kimi-code",
63
+ install=script,
64
+ env={},
65
+ label="Kimi Code",
65
66
  )
66
- install = await runtime.run(["sh", "-c", guarded], {})
67
- if install.exit_code != 0:
68
- raise RuntimeError(
69
- f"Kimi Code install failed: {install.stderr.strip()[-500:]}"
70
- )
71
67
  await super().setup(runtime)
72
68
 
73
69
  async def prepare_acp(
@@ -1,9 +1,7 @@
1
1
  from pathlib import Path
2
2
 
3
- from pydantic import Field
4
-
5
3
  from verifiers.v1.clients import ModelContext
6
- from verifiers.v1.configs.harness import HarnessConfig
4
+ from verifiers.v1.configs.harness import HarnessConfig, PinnedVersion
7
5
  from verifiers.v1.harness import Harness
8
6
  from verifiers.v1.runtimes import ProgramResult, Runtime
9
7
  from verifiers.v1.task import TaskData
@@ -13,7 +11,7 @@ PROGRAM_SOURCE = (Path(__file__).resolve().parent / "program.py").read_text()
13
11
 
14
12
 
15
13
  class MiniSWEAgentHarnessConfig(HarnessConfig):
16
- version: str = Field(default="2.4.6", pattern=r"^[A-Za-z0-9._+-]+$")
14
+ version: PinnedVersion = "2.4.6"
17
15
  """mini-swe-agent release to install, pinned for reproducibility."""
18
16
 
19
17
 
@@ -1,5 +1,4 @@
1
- import shlex
2
-
1
+ from verifiers.v1.harnesses.utils.install import ensure_installed
3
2
  from verifiers.v1.runtimes import Runtime
4
3
 
5
4
  NODE_DIR = "/var/tmp/vf-node"
@@ -39,16 +38,12 @@ node_ok || { echo "ACP adapters require Node.js 22.19 or newer" >&2; exit 1; }
39
38
 
40
39
  async def ensure_node(runtime: Runtime) -> None:
41
40
  """Install the shared Node runtime used by ACP adapter harnesses."""
42
- lock = f"{NODE_DIR}.install.lock"
43
- guarded = (
44
- f'until ln -s "$$" {lock} 2>/dev/null; do '
45
- f"owner=$(readlink {lock}); "
46
- f'if ! kill -0 "$owner" 2>/dev/null; then '
47
- f'[ "$(readlink {lock})" != "$owner" ] || rm -f {lock}; fi; '
48
- f"sleep 0.1; done; "
49
- f'trap \'[ "$(readlink {lock})" != "$$" ] || rm -f {lock}\' EXIT; '
50
- f"sh -c {shlex.quote(INSTALL)}"
41
+ # The install replaces NODE_DIR wholesale, so the lock lives beside it.
42
+ await ensure_installed(
43
+ runtime,
44
+ directory=NODE_DIR,
45
+ lock=f"{NODE_DIR}.install.lock",
46
+ install=INSTALL,
47
+ env={"VF_NODE_VERSION": NODE_VERSION},
48
+ label="Node.js",
51
49
  )
52
- result = await runtime.run(["sh", "-c", guarded], {"VF_NODE_VERSION": NODE_VERSION})
53
- if result.exit_code != 0:
54
- raise RuntimeError(f"Node.js install failed: {result.stderr.strip()[-500:]}")