verifiers 0.3.2.dev167__tar.gz → 0.3.2.dev169__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (224) hide show
  1. {verifiers-0.3.2.dev167 → verifiers-0.3.2.dev169}/PKG-INFO +4 -2
  2. {verifiers-0.3.2.dev167 → verifiers-0.3.2.dev169}/pyproject.toml +7 -1
  3. {verifiers-0.3.2.dev167 → verifiers-0.3.2.dev169}/tests/v1/conftest.py +5 -4
  4. verifiers-0.3.2.dev169/tests/v1/fixtures/agent_user_v1.py +82 -0
  5. verifiers-0.3.2.dev169/tests/v1/test_configs.py +95 -0
  6. {verifiers-0.3.2.dev167 → verifiers-0.3.2.dev169}/tests/v1/test_e2e.py +48 -3
  7. {verifiers-0.3.2.dev167 → verifiers-0.3.2.dev169}/verifiers/v1/rollout.py +35 -1
  8. {verifiers-0.3.2.dev167 → verifiers-0.3.2.dev169}/verifiers/v1/runtimes/__init__.py +9 -2
  9. {verifiers-0.3.2.dev167 → verifiers-0.3.2.dev169}/verifiers/v1/runtimes/base.py +19 -1
  10. {verifiers-0.3.2.dev167 → verifiers-0.3.2.dev169}/verifiers/v1/runtimes/docker/__init__.py +2 -0
  11. verifiers-0.3.2.dev169/verifiers/v1/runtimes/e2b.py +711 -0
  12. {verifiers-0.3.2.dev167 → verifiers-0.3.2.dev169}/verifiers/v1/runtimes/prime.py +5 -0
  13. {verifiers-0.3.2.dev167 → verifiers-0.3.2.dev169}/verifiers/v1/task.py +4 -0
  14. {verifiers-0.3.2.dev167 → verifiers-0.3.2.dev169}/verifiers/v1/tasksets/harbor/taskset.py +1 -0
  15. verifiers-0.3.2.dev167/tests/v1/test_configs.py +0 -20
  16. {verifiers-0.3.2.dev167 → verifiers-0.3.2.dev169}/.gitignore +0 -0
  17. {verifiers-0.3.2.dev167 → verifiers-0.3.2.dev169}/LICENSE +0 -0
  18. {verifiers-0.3.2.dev167 → verifiers-0.3.2.dev169}/README.md +0 -0
  19. {verifiers-0.3.2.dev167 → verifiers-0.3.2.dev169}/tests/v1/fixtures/counter_tool_v1.py +0 -0
  20. {verifiers-0.3.2.dev167 → verifiers-0.3.2.dev169}/tests/v1/fixtures/duet_v1.py +0 -0
  21. {verifiers-0.3.2.dev167 → verifiers-0.3.2.dev169}/tests/v1/fixtures/echo_acp_resume_v1.py +0 -0
  22. {verifiers-0.3.2.dev167 → verifiers-0.3.2.dev169}/tests/v1/fixtures/echo_agentic_v1.py +0 -0
  23. {verifiers-0.3.2.dev167 → verifiers-0.3.2.dev169}/tests/v1/fixtures/echo_tool_v1.py +0 -0
  24. {verifiers-0.3.2.dev167 → verifiers-0.3.2.dev169}/tests/v1/fixtures/echo_user_sim_v1.py +0 -0
  25. {verifiers-0.3.2.dev167 → verifiers-0.3.2.dev169}/tests/v1/fixtures/echo_v1.py +0 -0
  26. {verifiers-0.3.2.dev167 → verifiers-0.3.2.dev169}/tests/v1/fixtures/pyproject.toml +0 -0
  27. {verifiers-0.3.2.dev167 → verifiers-0.3.2.dev169}/tests/v1/fixtures/tool_response_image_v1.py +0 -0
  28. {verifiers-0.3.2.dev167 → verifiers-0.3.2.dev169}/tests/v1/test_color_codeword_tasks.py +0 -0
  29. {verifiers-0.3.2.dev167 → verifiers-0.3.2.dev169}/tests/v1/test_envs.py +0 -0
  30. {verifiers-0.3.2.dev167 → verifiers-0.3.2.dev169}/tests/v1/test_graph.py +0 -0
  31. {verifiers-0.3.2.dev167 → verifiers-0.3.2.dev169}/tests/v1/test_judges.py +0 -0
  32. {verifiers-0.3.2.dev167 → verifiers-0.3.2.dev169}/tests/v1/test_platform_run.py +0 -0
  33. {verifiers-0.3.2.dev167 → verifiers-0.3.2.dev169}/tests/v1/test_scoring.py +0 -0
  34. {verifiers-0.3.2.dev167 → verifiers-0.3.2.dev169}/tests/v1/test_serve_delta.py +0 -0
  35. {verifiers-0.3.2.dev167 → verifiers-0.3.2.dev169}/tests/v1/test_taskset.py +0 -0
  36. {verifiers-0.3.2.dev167 → verifiers-0.3.2.dev169}/tests/v1/test_trace.py +0 -0
  37. {verifiers-0.3.2.dev167 → verifiers-0.3.2.dev169}/verifiers/__init__.py +0 -0
  38. {verifiers-0.3.2.dev167 → verifiers-0.3.2.dev169}/verifiers/v1/__init__.py +0 -0
  39. {verifiers-0.3.2.dev167 → verifiers-0.3.2.dev169}/verifiers/v1/acp/__init__.py +0 -0
  40. {verifiers-0.3.2.dev167 → verifiers-0.3.2.dev169}/verifiers/v1/acp/runner.py +0 -0
  41. {verifiers-0.3.2.dev167 → verifiers-0.3.2.dev169}/verifiers/v1/agent.py +0 -0
  42. {verifiers-0.3.2.dev167 → verifiers-0.3.2.dev169}/verifiers/v1/cli/__init__.py +0 -0
  43. {verifiers-0.3.2.dev167 → verifiers-0.3.2.dev169}/verifiers/v1/cli/dashboard/__init__.py +0 -0
  44. {verifiers-0.3.2.dev167 → verifiers-0.3.2.dev169}/verifiers/v1/cli/dashboard/base.py +0 -0
  45. {verifiers-0.3.2.dev167 → verifiers-0.3.2.dev169}/verifiers/v1/cli/dashboard/eval.py +0 -0
  46. {verifiers-0.3.2.dev167 → verifiers-0.3.2.dev169}/verifiers/v1/cli/dashboard/replay.py +0 -0
  47. {verifiers-0.3.2.dev167 → verifiers-0.3.2.dev169}/verifiers/v1/cli/dashboard/validate.py +0 -0
  48. {verifiers-0.3.2.dev167 → verifiers-0.3.2.dev169}/verifiers/v1/cli/debug.py +0 -0
  49. {verifiers-0.3.2.dev167 → verifiers-0.3.2.dev169}/verifiers/v1/cli/eval/__init__.py +0 -0
  50. {verifiers-0.3.2.dev167 → verifiers-0.3.2.dev169}/verifiers/v1/cli/eval/hint.py +0 -0
  51. {verifiers-0.3.2.dev167 → verifiers-0.3.2.dev169}/verifiers/v1/cli/eval/main.py +0 -0
  52. {verifiers-0.3.2.dev167 → verifiers-0.3.2.dev169}/verifiers/v1/cli/eval/resume.py +0 -0
  53. {verifiers-0.3.2.dev167 → verifiers-0.3.2.dev169}/verifiers/v1/cli/eval/runner.py +0 -0
  54. {verifiers-0.3.2.dev167 → verifiers-0.3.2.dev169}/verifiers/v1/cli/gepa.py +0 -0
  55. {verifiers-0.3.2.dev167 → verifiers-0.3.2.dev169}/verifiers/v1/cli/init.py +0 -0
  56. {verifiers-0.3.2.dev167 → verifiers-0.3.2.dev169}/verifiers/v1/cli/output.py +0 -0
  57. {verifiers-0.3.2.dev167 → verifiers-0.3.2.dev169}/verifiers/v1/cli/replay.py +0 -0
  58. {verifiers-0.3.2.dev167 → verifiers-0.3.2.dev169}/verifiers/v1/cli/resolve.py +0 -0
  59. {verifiers-0.3.2.dev167 → verifiers-0.3.2.dev169}/verifiers/v1/cli/resume.py +0 -0
  60. {verifiers-0.3.2.dev167 → verifiers-0.3.2.dev169}/verifiers/v1/cli/validate.py +0 -0
  61. {verifiers-0.3.2.dev167 → verifiers-0.3.2.dev169}/verifiers/v1/clients/__init__.py +0 -0
  62. {verifiers-0.3.2.dev167 → verifiers-0.3.2.dev169}/verifiers/v1/clients/base.py +0 -0
  63. {verifiers-0.3.2.dev167 → verifiers-0.3.2.dev169}/verifiers/v1/clients/client.py +0 -0
  64. {verifiers-0.3.2.dev167 → verifiers-0.3.2.dev169}/verifiers/v1/clients/eval.py +0 -0
  65. {verifiers-0.3.2.dev167 → verifiers-0.3.2.dev169}/verifiers/v1/clients/train.py +0 -0
  66. {verifiers-0.3.2.dev167 → verifiers-0.3.2.dev169}/verifiers/v1/configs/__init__.py +0 -0
  67. {verifiers-0.3.2.dev167 → verifiers-0.3.2.dev169}/verifiers/v1/configs/agent.py +0 -0
  68. {verifiers-0.3.2.dev167 → verifiers-0.3.2.dev169}/verifiers/v1/configs/cli/__init__.py +0 -0
  69. {verifiers-0.3.2.dev167 → verifiers-0.3.2.dev169}/verifiers/v1/configs/cli/debug.py +0 -0
  70. {verifiers-0.3.2.dev167 → verifiers-0.3.2.dev169}/verifiers/v1/configs/cli/env.py +0 -0
  71. {verifiers-0.3.2.dev167 → verifiers-0.3.2.dev169}/verifiers/v1/configs/cli/eval.py +0 -0
  72. {verifiers-0.3.2.dev167 → verifiers-0.3.2.dev169}/verifiers/v1/configs/cli/init.py +0 -0
  73. {verifiers-0.3.2.dev167 → verifiers-0.3.2.dev169}/verifiers/v1/configs/cli/replay.py +0 -0
  74. {verifiers-0.3.2.dev167 → verifiers-0.3.2.dev169}/verifiers/v1/configs/cli/validate.py +0 -0
  75. {verifiers-0.3.2.dev167 → verifiers-0.3.2.dev169}/verifiers/v1/configs/client.py +0 -0
  76. {verifiers-0.3.2.dev167 → verifiers-0.3.2.dev169}/verifiers/v1/configs/env.py +0 -0
  77. {verifiers-0.3.2.dev167 → verifiers-0.3.2.dev169}/verifiers/v1/configs/harness.py +0 -0
  78. {verifiers-0.3.2.dev167 → verifiers-0.3.2.dev169}/verifiers/v1/configs/judge.py +0 -0
  79. {verifiers-0.3.2.dev167 → verifiers-0.3.2.dev169}/verifiers/v1/configs/retries.py +0 -0
  80. {verifiers-0.3.2.dev167 → verifiers-0.3.2.dev169}/verifiers/v1/configs/runtime.py +0 -0
  81. {verifiers-0.3.2.dev167 → verifiers-0.3.2.dev169}/verifiers/v1/configs/select.py +0 -0
  82. {verifiers-0.3.2.dev167 → verifiers-0.3.2.dev169}/verifiers/v1/configs/serve.py +0 -0
  83. {verifiers-0.3.2.dev167 → verifiers-0.3.2.dev169}/verifiers/v1/configs/task.py +0 -0
  84. {verifiers-0.3.2.dev167 → verifiers-0.3.2.dev169}/verifiers/v1/configs/taskset.py +0 -0
  85. {verifiers-0.3.2.dev167 → verifiers-0.3.2.dev169}/verifiers/v1/dialects/__init__.py +0 -0
  86. {verifiers-0.3.2.dev167 → verifiers-0.3.2.dev169}/verifiers/v1/dialects/anthropic.py +0 -0
  87. {verifiers-0.3.2.dev167 → verifiers-0.3.2.dev169}/verifiers/v1/dialects/base.py +0 -0
  88. {verifiers-0.3.2.dev167 → verifiers-0.3.2.dev169}/verifiers/v1/dialects/chat.py +0 -0
  89. {verifiers-0.3.2.dev167 → verifiers-0.3.2.dev169}/verifiers/v1/dialects/responses.py +0 -0
  90. {verifiers-0.3.2.dev167 → verifiers-0.3.2.dev169}/verifiers/v1/env.py +0 -0
  91. {verifiers-0.3.2.dev167 → verifiers-0.3.2.dev169}/verifiers/v1/envs/__init__.py +0 -0
  92. {verifiers-0.3.2.dev167 → verifiers-0.3.2.dev169}/verifiers/v1/envs/agentic_judge/__init__.py +0 -0
  93. {verifiers-0.3.2.dev167 → verifiers-0.3.2.dev169}/verifiers/v1/envs/agentic_judge/env.py +0 -0
  94. {verifiers-0.3.2.dev167 → verifiers-0.3.2.dev169}/verifiers/v1/envs/best_of_n/__init__.py +0 -0
  95. {verifiers-0.3.2.dev167 → verifiers-0.3.2.dev169}/verifiers/v1/envs/best_of_n/env.py +0 -0
  96. {verifiers-0.3.2.dev167 → verifiers-0.3.2.dev169}/verifiers/v1/envs/isolated_verifier/__init__.py +0 -0
  97. {verifiers-0.3.2.dev167 → verifiers-0.3.2.dev169}/verifiers/v1/envs/isolated_verifier/env.py +0 -0
  98. {verifiers-0.3.2.dev167 → verifiers-0.3.2.dev169}/verifiers/v1/envs/shared_agentic_judge/__init__.py +0 -0
  99. {verifiers-0.3.2.dev167 → verifiers-0.3.2.dev169}/verifiers/v1/envs/single_agent/__init__.py +0 -0
  100. {verifiers-0.3.2.dev167 → verifiers-0.3.2.dev169}/verifiers/v1/envs/single_agent/env.py +0 -0
  101. {verifiers-0.3.2.dev167 → verifiers-0.3.2.dev169}/verifiers/v1/envs/user_sim/__init__.py +0 -0
  102. {verifiers-0.3.2.dev167 → verifiers-0.3.2.dev169}/verifiers/v1/envs/user_sim/env.py +0 -0
  103. {verifiers-0.3.2.dev167 → verifiers-0.3.2.dev169}/verifiers/v1/episode.py +0 -0
  104. {verifiers-0.3.2.dev167 → verifiers-0.3.2.dev169}/verifiers/v1/errors.py +0 -0
  105. {verifiers-0.3.2.dev167 → verifiers-0.3.2.dev169}/verifiers/v1/gepa/__init__.py +0 -0
  106. {verifiers-0.3.2.dev167 → verifiers-0.3.2.dev169}/verifiers/v1/gepa/adapter.py +0 -0
  107. {verifiers-0.3.2.dev167 → verifiers-0.3.2.dev169}/verifiers/v1/gepa/config.py +0 -0
  108. {verifiers-0.3.2.dev167 → verifiers-0.3.2.dev169}/verifiers/v1/gepa/dataset.py +0 -0
  109. {verifiers-0.3.2.dev167 → verifiers-0.3.2.dev169}/verifiers/v1/gepa/reflection.py +0 -0
  110. {verifiers-0.3.2.dev167 → verifiers-0.3.2.dev169}/verifiers/v1/gepa/runner.py +0 -0
  111. {verifiers-0.3.2.dev167 → verifiers-0.3.2.dev169}/verifiers/v1/graph.py +0 -0
  112. {verifiers-0.3.2.dev167 → verifiers-0.3.2.dev169}/verifiers/v1/harness.py +0 -0
  113. {verifiers-0.3.2.dev167 → verifiers-0.3.2.dev169}/verifiers/v1/harnesses/__init__.py +0 -0
  114. {verifiers-0.3.2.dev167 → verifiers-0.3.2.dev169}/verifiers/v1/harnesses/bash/__init__.py +0 -0
  115. {verifiers-0.3.2.dev167 → verifiers-0.3.2.dev169}/verifiers/v1/harnesses/bash/harness.py +0 -0
  116. {verifiers-0.3.2.dev167 → verifiers-0.3.2.dev169}/verifiers/v1/harnesses/browser_use/__init__.py +0 -0
  117. {verifiers-0.3.2.dev167 → verifiers-0.3.2.dev169}/verifiers/v1/harnesses/browser_use/harness.py +0 -0
  118. {verifiers-0.3.2.dev167 → verifiers-0.3.2.dev169}/verifiers/v1/harnesses/browser_use/program.py +0 -0
  119. {verifiers-0.3.2.dev167 → verifiers-0.3.2.dev169}/verifiers/v1/harnesses/claude_code/__init__.py +0 -0
  120. {verifiers-0.3.2.dev167 → verifiers-0.3.2.dev169}/verifiers/v1/harnesses/claude_code/harness.py +0 -0
  121. {verifiers-0.3.2.dev167 → verifiers-0.3.2.dev169}/verifiers/v1/harnesses/codex/__init__.py +0 -0
  122. {verifiers-0.3.2.dev167 → verifiers-0.3.2.dev169}/verifiers/v1/harnesses/codex/gate.mjs +0 -0
  123. {verifiers-0.3.2.dev167 → verifiers-0.3.2.dev169}/verifiers/v1/harnesses/codex/harness.py +0 -0
  124. {verifiers-0.3.2.dev167 → verifiers-0.3.2.dev169}/verifiers/v1/harnesses/hermes_agent/__init__.py +0 -0
  125. {verifiers-0.3.2.dev167 → verifiers-0.3.2.dev169}/verifiers/v1/harnesses/hermes_agent/harness.py +0 -0
  126. {verifiers-0.3.2.dev167 → verifiers-0.3.2.dev169}/verifiers/v1/harnesses/hermes_agent/program.py +0 -0
  127. {verifiers-0.3.2.dev167 → verifiers-0.3.2.dev169}/verifiers/v1/harnesses/kimi_code/__init__.py +0 -0
  128. {verifiers-0.3.2.dev167 → verifiers-0.3.2.dev169}/verifiers/v1/harnesses/kimi_code/harness.py +0 -0
  129. {verifiers-0.3.2.dev167 → verifiers-0.3.2.dev169}/verifiers/v1/harnesses/mini_swe_agent/__init__.py +0 -0
  130. {verifiers-0.3.2.dev167 → verifiers-0.3.2.dev169}/verifiers/v1/harnesses/mini_swe_agent/harness.py +0 -0
  131. {verifiers-0.3.2.dev167 → verifiers-0.3.2.dev169}/verifiers/v1/harnesses/mini_swe_agent/program.py +0 -0
  132. {verifiers-0.3.2.dev167 → verifiers-0.3.2.dev169}/verifiers/v1/harnesses/node.py +0 -0
  133. {verifiers-0.3.2.dev167 → verifiers-0.3.2.dev169}/verifiers/v1/harnesses/null/__init__.py +0 -0
  134. {verifiers-0.3.2.dev167 → verifiers-0.3.2.dev169}/verifiers/v1/harnesses/null/harness.py +0 -0
  135. {verifiers-0.3.2.dev167 → verifiers-0.3.2.dev169}/verifiers/v1/harnesses/openclaw/__init__.py +0 -0
  136. {verifiers-0.3.2.dev167 → verifiers-0.3.2.dev169}/verifiers/v1/harnesses/openclaw/harness.py +0 -0
  137. {verifiers-0.3.2.dev167 → verifiers-0.3.2.dev169}/verifiers/v1/harnesses/pi/__init__.py +0 -0
  138. {verifiers-0.3.2.dev167 → verifiers-0.3.2.dev169}/verifiers/v1/harnesses/pi/gate.mjs +0 -0
  139. {verifiers-0.3.2.dev167 → verifiers-0.3.2.dev169}/verifiers/v1/harnesses/pi/harness.py +0 -0
  140. {verifiers-0.3.2.dev167 → verifiers-0.3.2.dev169}/verifiers/v1/harnesses/prime_agent/__init__.py +0 -0
  141. {verifiers-0.3.2.dev167 → verifiers-0.3.2.dev169}/verifiers/v1/harnesses/prime_agent/harness.py +0 -0
  142. {verifiers-0.3.2.dev167 → verifiers-0.3.2.dev169}/verifiers/v1/harnesses/rlm/__init__.py +0 -0
  143. {verifiers-0.3.2.dev167 → verifiers-0.3.2.dev169}/verifiers/v1/harnesses/rlm/harness.py +0 -0
  144. {verifiers-0.3.2.dev167 → verifiers-0.3.2.dev169}/verifiers/v1/harnesses/terminus_2/__init__.py +0 -0
  145. {verifiers-0.3.2.dev167 → verifiers-0.3.2.dev169}/verifiers/v1/harnesses/terminus_2/harness.py +0 -0
  146. {verifiers-0.3.2.dev167 → verifiers-0.3.2.dev169}/verifiers/v1/harnesses/terminus_2/program.py +0 -0
  147. {verifiers-0.3.2.dev167 → verifiers-0.3.2.dev169}/verifiers/v1/harnesses/utils/__init__.py +0 -0
  148. {verifiers-0.3.2.dev167 → verifiers-0.3.2.dev169}/verifiers/v1/harnesses/utils/compaction.py +0 -0
  149. {verifiers-0.3.2.dev167 → verifiers-0.3.2.dev169}/verifiers/v1/harnesses/utils/core.py +0 -0
  150. {verifiers-0.3.2.dev167 → verifiers-0.3.2.dev169}/verifiers/v1/harnesses/utils/install.py +0 -0
  151. {verifiers-0.3.2.dev167 → verifiers-0.3.2.dev169}/verifiers/v1/harnesses/utils/launch.py +0 -0
  152. {verifiers-0.3.2.dev167 → verifiers-0.3.2.dev169}/verifiers/v1/harnesses/utils/mcp.py +0 -0
  153. {verifiers-0.3.2.dev167 → verifiers-0.3.2.dev169}/verifiers/v1/interception/__init__.py +0 -0
  154. {verifiers-0.3.2.dev167 → verifiers-0.3.2.dev169}/verifiers/v1/interception/base.py +0 -0
  155. {verifiers-0.3.2.dev167 → verifiers-0.3.2.dev169}/verifiers/v1/interception/pool.py +0 -0
  156. {verifiers-0.3.2.dev167 → verifiers-0.3.2.dev169}/verifiers/v1/interception/server.py +0 -0
  157. {verifiers-0.3.2.dev167 → verifiers-0.3.2.dev169}/verifiers/v1/interception/tunnel/__init__.py +0 -0
  158. {verifiers-0.3.2.dev167 → verifiers-0.3.2.dev169}/verifiers/v1/interception/tunnel/base.py +0 -0
  159. {verifiers-0.3.2.dev167 → verifiers-0.3.2.dev169}/verifiers/v1/interception/tunnel/custom.py +0 -0
  160. {verifiers-0.3.2.dev167 → verifiers-0.3.2.dev169}/verifiers/v1/interception/tunnel/prime.py +0 -0
  161. {verifiers-0.3.2.dev167 → verifiers-0.3.2.dev169}/verifiers/v1/judge.py +0 -0
  162. {verifiers-0.3.2.dev167 → verifiers-0.3.2.dev169}/verifiers/v1/judges/__init__.py +0 -0
  163. {verifiers-0.3.2.dev167 → verifiers-0.3.2.dev169}/verifiers/v1/judges/reference.py +0 -0
  164. {verifiers-0.3.2.dev167 → verifiers-0.3.2.dev169}/verifiers/v1/judges/reference.txt +0 -0
  165. {verifiers-0.3.2.dev167 → verifiers-0.3.2.dev169}/verifiers/v1/judges/rubric.py +0 -0
  166. {verifiers-0.3.2.dev167 → verifiers-0.3.2.dev169}/verifiers/v1/judges/rubric.txt +0 -0
  167. {verifiers-0.3.2.dev167 → verifiers-0.3.2.dev169}/verifiers/v1/mcp/__init__.py +0 -0
  168. {verifiers-0.3.2.dev167 → verifiers-0.3.2.dev169}/verifiers/v1/mcp/launch.py +0 -0
  169. {verifiers-0.3.2.dev167 → verifiers-0.3.2.dev169}/verifiers/v1/mcp/server.py +0 -0
  170. {verifiers-0.3.2.dev167 → verifiers-0.3.2.dev169}/verifiers/v1/mcp/toolset.py +0 -0
  171. {verifiers-0.3.2.dev167 → verifiers-0.3.2.dev169}/verifiers/v1/runtimes/apptainer.py +0 -0
  172. {verifiers-0.3.2.dev167 → verifiers-0.3.2.dev169}/verifiers/v1/runtimes/container.py +0 -0
  173. {verifiers-0.3.2.dev167 → verifiers-0.3.2.dev169}/verifiers/v1/runtimes/docker/egress.py +0 -0
  174. {verifiers-0.3.2.dev167 → verifiers-0.3.2.dev169}/verifiers/v1/runtimes/limiters.py +0 -0
  175. {verifiers-0.3.2.dev167 → verifiers-0.3.2.dev169}/verifiers/v1/runtimes/modal.py +0 -0
  176. {verifiers-0.3.2.dev167 → verifiers-0.3.2.dev169}/verifiers/v1/runtimes/subprocess.py +0 -0
  177. {verifiers-0.3.2.dev167 → verifiers-0.3.2.dev169}/verifiers/v1/semantic.py +0 -0
  178. {verifiers-0.3.2.dev167 → verifiers-0.3.2.dev169}/verifiers/v1/serve/__init__.py +0 -0
  179. {verifiers-0.3.2.dev167 → verifiers-0.3.2.dev169}/verifiers/v1/serve/client.py +0 -0
  180. {verifiers-0.3.2.dev167 → verifiers-0.3.2.dev169}/verifiers/v1/serve/delta.py +0 -0
  181. {verifiers-0.3.2.dev167 → verifiers-0.3.2.dev169}/verifiers/v1/serve/encoding.py +0 -0
  182. {verifiers-0.3.2.dev167 → verifiers-0.3.2.dev169}/verifiers/v1/serve/pool.py +0 -0
  183. {verifiers-0.3.2.dev167 → verifiers-0.3.2.dev169}/verifiers/v1/serve/server.py +0 -0
  184. {verifiers-0.3.2.dev167 → verifiers-0.3.2.dev169}/verifiers/v1/serve/types.py +0 -0
  185. {verifiers-0.3.2.dev167 → verifiers-0.3.2.dev169}/verifiers/v1/session.py +0 -0
  186. {verifiers-0.3.2.dev167 → verifiers-0.3.2.dev169}/verifiers/v1/state.py +0 -0
  187. {verifiers-0.3.2.dev167 → verifiers-0.3.2.dev169}/verifiers/v1/taskset.py +0 -0
  188. {verifiers-0.3.2.dev167 → verifiers-0.3.2.dev169}/verifiers/v1/tasksets/__init__.py +0 -0
  189. {verifiers-0.3.2.dev167 → verifiers-0.3.2.dev169}/verifiers/v1/tasksets/harbor/__init__.py +0 -0
  190. {verifiers-0.3.2.dev167 → verifiers-0.3.2.dev169}/verifiers/v1/tasksets/harbor/compose.py +0 -0
  191. {verifiers-0.3.2.dev167 → verifiers-0.3.2.dev169}/verifiers/v1/tasksets/harbor/env.py +0 -0
  192. {verifiers-0.3.2.dev167 → verifiers-0.3.2.dev169}/verifiers/v1/tasksets/harbor/toolset.py +0 -0
  193. {verifiers-0.3.2.dev167 → verifiers-0.3.2.dev169}/verifiers/v1/tasksets/nemo_gym/__init__.py +0 -0
  194. {verifiers-0.3.2.dev167 → verifiers-0.3.2.dev169}/verifiers/v1/tasksets/nemo_gym/response.py +0 -0
  195. {verifiers-0.3.2.dev167 → verifiers-0.3.2.dev169}/verifiers/v1/tasksets/nemo_gym/server.py +0 -0
  196. {verifiers-0.3.2.dev167 → verifiers-0.3.2.dev169}/verifiers/v1/tasksets/nemo_gym/taskset.py +0 -0
  197. {verifiers-0.3.2.dev167 → verifiers-0.3.2.dev169}/verifiers/v1/tasksets/nemo_gym/toolset.py +0 -0
  198. {verifiers-0.3.2.dev167 → verifiers-0.3.2.dev169}/verifiers/v1/tasksets/openenv/__init__.py +0 -0
  199. {verifiers-0.3.2.dev167 → verifiers-0.3.2.dev169}/verifiers/v1/tasksets/openenv/taskset.py +0 -0
  200. {verifiers-0.3.2.dev167 → verifiers-0.3.2.dev169}/verifiers/v1/tasksets/textarena/__init__.py +0 -0
  201. {verifiers-0.3.2.dev167 → verifiers-0.3.2.dev169}/verifiers/v1/tasksets/textarena/taskset.py +0 -0
  202. {verifiers-0.3.2.dev167 → verifiers-0.3.2.dev169}/verifiers/v1/trace.py +0 -0
  203. {verifiers-0.3.2.dev167 → verifiers-0.3.2.dev169}/verifiers/v1/types.py +0 -0
  204. {verifiers-0.3.2.dev167 → verifiers-0.3.2.dev169}/verifiers/v1/utils/__init__.py +0 -0
  205. {verifiers-0.3.2.dev167 → verifiers-0.3.2.dev169}/verifiers/v1/utils/aio.py +0 -0
  206. {verifiers-0.3.2.dev167 → verifiers-0.3.2.dev169}/verifiers/v1/utils/artifacts.py +0 -0
  207. {verifiers-0.3.2.dev167 → verifiers-0.3.2.dev169}/verifiers/v1/utils/compile.py +0 -0
  208. {verifiers-0.3.2.dev167 → verifiers-0.3.2.dev169}/verifiers/v1/utils/decorators.py +0 -0
  209. {verifiers-0.3.2.dev167 → verifiers-0.3.2.dev169}/verifiers/v1/utils/format.py +0 -0
  210. {verifiers-0.3.2.dev167 → verifiers-0.3.2.dev169}/verifiers/v1/utils/generic.py +0 -0
  211. {verifiers-0.3.2.dev167 → verifiers-0.3.2.dev169}/verifiers/v1/utils/git.py +0 -0
  212. {verifiers-0.3.2.dev167 → verifiers-0.3.2.dev169}/verifiers/v1/utils/image.py +0 -0
  213. {verifiers-0.3.2.dev167 → verifiers-0.3.2.dev169}/verifiers/v1/utils/interrupt.py +0 -0
  214. {verifiers-0.3.2.dev167 → verifiers-0.3.2.dev169}/verifiers/v1/utils/loaders.py +0 -0
  215. {verifiers-0.3.2.dev167 → verifiers-0.3.2.dev169}/verifiers/v1/utils/logging.py +0 -0
  216. {verifiers-0.3.2.dev167 → verifiers-0.3.2.dev169}/verifiers/v1/utils/memory.py +0 -0
  217. {verifiers-0.3.2.dev167 → verifiers-0.3.2.dev169}/verifiers/v1/utils/paths.py +0 -0
  218. {verifiers-0.3.2.dev167 → verifiers-0.3.2.dev169}/verifiers/v1/utils/platform.py +0 -0
  219. {verifiers-0.3.2.dev167 → verifiers-0.3.2.dev169}/verifiers/v1/utils/prime.py +0 -0
  220. {verifiers-0.3.2.dev167 → verifiers-0.3.2.dev169}/verifiers/v1/utils/retries.py +0 -0
  221. {verifiers-0.3.2.dev167 → verifiers-0.3.2.dev169}/verifiers/v1/utils/scope.py +0 -0
  222. {verifiers-0.3.2.dev167 → verifiers-0.3.2.dev169}/verifiers/v1/utils/score.py +0 -0
  223. {verifiers-0.3.2.dev167 → verifiers-0.3.2.dev169}/verifiers/v1/utils/trace_store.py +0 -0
  224. {verifiers-0.3.2.dev167 → verifiers-0.3.2.dev169}/verifiers/v1/utils/version.py +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.5
2
2
  Name: verifiers
3
- Version: 0.3.2.dev167
3
+ Version: 0.3.2.dev169
4
4
  Summary: Verifiers: Environments for LLM Reinforcement Learning
5
5
  Project-URL: Homepage, https://github.com/primeintellect-ai/verifiers
6
6
  Project-URL: Documentation, https://github.com/primeintellect-ai/verifiers
@@ -36,7 +36,7 @@ Requires-Dist: numpy>=2.1.0
36
36
  Requires-Dist: openai<3.0.0,>=2.54.0
37
37
  Requires-Dist: prime-pydantic-config[toml]>=0.4.3
38
38
  Requires-Dist: prime-runs>=0.1.3
39
- Requires-Dist: prime-sandboxes>=0.3.0
39
+ Requires-Dist: prime-sandboxes>=0.4.1
40
40
  Requires-Dist: prime-tunnel>=0.1.8
41
41
  Requires-Dist: pydantic>=2.12.3
42
42
  Requires-Dist: pyzmq>=27.1.0
@@ -46,6 +46,8 @@ Requires-Dist: tenacity>=8.5.0
46
46
  Requires-Dist: tomli-w>=1.0.0
47
47
  Requires-Dist: typing-extensions>=4.12.2
48
48
  Requires-Dist: uvicorn>=0.52.0
49
+ Provides-Extra: e2b
50
+ Requires-Dist: e2b<3,>=2.35.0; extra == 'e2b'
49
51
  Provides-Extra: harbor
50
52
  Requires-Dist: harbor==0.21.0; (python_full_version >= '3.12') and extra == 'harbor'
51
53
  Provides-Extra: modal
@@ -33,7 +33,7 @@ dependencies = [
33
33
  "math-verify>=0.8.0",
34
34
  "openai>=2.54.0,<3.0.0",
35
35
  "prime-tunnel>=0.1.8",
36
- "prime-sandboxes>=0.3.0",
36
+ "prime-sandboxes>=0.4.1",
37
37
  "prime-runs>=0.1.3",
38
38
  "pydantic>=2.12.3",
39
39
  "rich>=11.0.0",
@@ -100,6 +100,11 @@ harbor = [
100
100
  modal = [
101
101
  "modal>=1.5.4",
102
102
  ]
103
+ e2b = [
104
+ # <3: the runtime swaps the SDK command handle's private stream decoders for
105
+ # byte-accurate live output (see runtimes/e2b.py _byte_handle).
106
+ "e2b>=2.35.0,<3",
107
+ ]
103
108
  openenv = [
104
109
  "openenv>=0.4.1",
105
110
  ]
@@ -228,6 +233,7 @@ markers = [
228
233
  "apptainer: v1 e2e cases touching the apptainer runtime",
229
234
  "prime: v1 e2e cases touching the prime runtime (real sandbox + tunnel; local-only, excluded in CI)",
230
235
  "modal: v1 e2e cases touching the modal runtime (local-only setup; excluded in CI)",
236
+ "e2b: v1 e2e cases touching the e2b runtime (real sandbox; local-only, excluded in CI)",
231
237
  "colocated: v1 e2e cases with a user/tool server colocated in the harness runtime",
232
238
  "null: v1 e2e cases on the null harness",
233
239
  "bash: v1 e2e cases on the bash harness",
@@ -16,20 +16,21 @@ one without `indirect=True` fails loudly.
16
16
 
17
17
  Every combination carries its axes' pytest marks, so subsets select with `-m`:
18
18
 
19
- uv run pytest tests/v1 -n auto # everything (needs modal setup)
19
+ uv run pytest tests/v1 -n auto # everything (needs remote setup)
20
20
  uv run pytest tests/v1 -n auto -m "not e2e" # deterministic CI matrix
21
- uv run pytest tests/v1 -n auto -m "e2e and not prime and not modal" # live CI job
21
+ uv run pytest tests/v1 -n auto -m "e2e and not prime and not modal and not e2b" # live CI job
22
22
  uv run pytest tests/v1 -n auto -m docker # any case touching the docker runtime
23
23
  uv run pytest tests/v1 -n auto -m bash # only the bash harness
24
24
  uv run pytest tests/v1 -n auto -m prime # only prime (real sandboxes; local)
25
25
  uv run pytest tests/v1 -n auto -m modal # only modal (needs local setup)
26
+ uv run pytest tests/v1 -n auto -m e2b # only e2b (real sandboxes; local)
26
27
 
27
- Marks: runtimes `subprocess` / `docker` / `podman` / `apptainer` / `prime` / `modal`, placement `colocated`,
28
+ Marks: runtimes `subprocess` / `docker` / `podman` / `apptainer` / `prime` / `modal` / `e2b`, placement `colocated`,
28
29
  harnesses `null` / `bash` / `rlm` / `kimi_code` / `pi` / `openclaw` / `codex` /
29
30
  `claude_code` / `hermes_agent`.
30
31
  A mark is applied per axis, so it selects every case touching that value on ANY axis; for one exact
31
32
  combination use `-k` on the test id (e.g. `-k "harness-in-docker-with-tool-in-subprocess"`).
32
- prime/modal provision real remote sandboxes (slow, infra-flaky, need setup), so they're local-only.
33
+ prime/modal/e2b provision real remote sandboxes (slow, infra-flaky, need setup), so they're local-only.
33
34
  CI runs deterministic tests across the Python matrix and the remaining live E2Es once.
34
35
  """
35
36
 
@@ -0,0 +1,82 @@
1
+ """agent-user-v1: a scripted, model-free agent that runs as a non-root user and tries root actions.
2
+
3
+ Task setup runs as root: it creates the task's `agent_user` (images normally ship one),
4
+ plants a root-only secret, and hands the workdir to that user. The scripted harness then runs
5
+ as that user, attempts four actions only root may take, and records each outcome in the
6
+ workdir. The reward is 1 only if every root action was denied while the agent still saw the
7
+ secret and could write its workspace.
8
+ Needs a runtime that can switch users (docker or prime).
9
+ """
10
+
11
+ import json
12
+
13
+ import verifiers.v1 as vf
14
+ from verifiers.v1.clients import ModelContext
15
+ from verifiers.v1.configs.harness import HarnessConfig
16
+ from verifiers.v1.harness import Harness
17
+ from verifiers.v1.runtimes import ProgramResult, Runtime
18
+
19
+ __all__ = ["AgentUserTaskset", "ScriptedAgentUserHarness"]
20
+
21
+ AGENT_USER = "vf-agent"
22
+ SECRET = "/opt/vf-agent-user-secret"
23
+ REPORT = "agent-user-report.json"
24
+ ROOT_ACTIONS = ("read_secret", "write_etc", "chmod_secret", "signal_init")
25
+
26
+ SETUP = (
27
+ f"useradd -m {AGENT_USER} && echo secret > {SECRET} && chmod 600 {SECRET} "
28
+ f"&& chown -R {AGENT_USER} ."
29
+ )
30
+
31
+ PROBE = f"""
32
+ r() {{ "$@" >/dev/null 2>&1 && echo ok || echo denied; }}
33
+ printf '{{"user":"%s","secret_exists":"%s","read_secret":"%s","write_etc":"%s","chmod_secret":"%s","signal_init":"%s","write_workdir":"%s"}}' \\
34
+ "$(id -un)" "$(r test -e {SECRET})" "$(r cat {SECRET})" "$(r touch /etc/vf-agent-user-probe)" \\
35
+ "$(r chmod 644 {SECRET})" "$(r kill -0 1)" "$(r touch probe)" > {REPORT}
36
+ """
37
+
38
+
39
+ class ScriptedAgentUserHarness(Harness[HarnessConfig]):
40
+ async def launch(
41
+ self,
42
+ ctx: ModelContext,
43
+ trace: vf.Trace,
44
+ runtime: Runtime,
45
+ endpoint: str,
46
+ secret: str,
47
+ mcp_urls: dict[str, str],
48
+ data: vf.TaskData,
49
+ tool_interception_url: str | None = None,
50
+ ) -> ProgramResult:
51
+ return await runtime.run_program(["sh", "-c", PROBE], {})
52
+
53
+
54
+ class AgentUserTask(vf.Task):
55
+ async def setup(self, trace: vf.Trace, runtime: Runtime) -> None:
56
+ result = await runtime.run(["sh", "-c", SETUP], {})
57
+ if result.exit_code != 0:
58
+ raise RuntimeError(f"root setup failed: {result.stderr}")
59
+
60
+ @vf.reward(weight=1.0)
61
+ async def root_actions_denied(self, trace: vf.Trace, runtime: Runtime) -> float:
62
+ report = json.loads(await runtime.read(REPORT))
63
+ trace.info["agent_user_report"] = report
64
+ return float(
65
+ report["user"] == AGENT_USER
66
+ and report["secret_exists"] == "ok"
67
+ and report["write_workdir"] == "ok"
68
+ and all(report[action] == "denied" for action in ROOT_ACTIONS)
69
+ )
70
+
71
+
72
+ class AgentUserTaskset(vf.Taskset[AgentUserTask, vf.TasksetConfig]):
73
+ def load(self) -> list[AgentUserTask]:
74
+ return [
75
+ AgentUserTask(
76
+ vf.TaskData(
77
+ idx=0,
78
+ prompt="Scripted: try root-only actions as a non-root user.",
79
+ agent_user=AGENT_USER,
80
+ )
81
+ )
82
+ ]
@@ -0,0 +1,95 @@
1
+ """Every checked-in v1 eval config parses.
2
+
3
+ Mirrors prime-rl's config test: glob the configs and assert each validates into its config
4
+ type. The root `configs/*.toml` are the `uv run vf-eval @ <file>` v1 configs (EvalConfig).
5
+ """
6
+
7
+ import tomllib
8
+ from pathlib import Path
9
+
10
+ import pytest
11
+
12
+ from verifiers.v1.configs.cli.eval import EvalConfig
13
+ from verifiers.v1.runtimes import E2BConfig
14
+ from verifiers.v1.runtimes.e2b import _egress_update
15
+
16
+ CONFIGS = sorted((Path(__file__).resolve().parents[2] / "configs").glob("*.toml"))
17
+
18
+
19
+ @pytest.mark.parametrize("path", CONFIGS, ids=lambda p: p.name)
20
+ def test_eval_config_parses(path: Path) -> None:
21
+ config = EvalConfig.model_validate(tomllib.load(path.open("rb")))
22
+ assert config.env.taskset.id
23
+
24
+
25
+ @pytest.mark.parametrize(
26
+ "rule",
27
+ [
28
+ "",
29
+ "https://api.example.com",
30
+ "example.com:8443",
31
+ "example.com/path",
32
+ "user@example.com",
33
+ "10.0.0.0/99",
34
+ "example..com",
35
+ ],
36
+ )
37
+ def test_e2b_config_rejects_unenforceable_egress_rules(rule: str) -> None:
38
+ with pytest.raises(ValueError, match="allow rules"):
39
+ E2BConfig(allow=[rule])
40
+
41
+
42
+ @pytest.mark.parametrize("rule", ["api.example.com", "*.example.com"])
43
+ def test_e2b_config_rejects_hostname_block_rules(rule: str) -> None:
44
+ with pytest.raises(ValueError, match="block rules must be IP addresses or CIDR"):
45
+ E2BConfig(block=[rule])
46
+
47
+
48
+ @pytest.mark.parametrize(
49
+ "rule",
50
+ [
51
+ "api.example.com",
52
+ "*.example.com",
53
+ "10.0.0.1",
54
+ "10.0.0.0/24",
55
+ "2001:db8::1",
56
+ "2001:db8::/32",
57
+ ],
58
+ )
59
+ def test_e2b_config_accepts_supported_egress_rules(rule: str) -> None:
60
+ assert E2BConfig(allow=[rule]).allow == [rule]
61
+
62
+
63
+ @pytest.mark.parametrize(
64
+ "rule", ["10.0.0.1", "10.0.0.0/24", "2001:db8::1", "2001:db8::/32"]
65
+ )
66
+ def test_e2b_config_accepts_supported_block_rules(rule: str) -> None:
67
+ assert E2BConfig(block=[rule]).block == [rule]
68
+
69
+
70
+ def test_e2b_egress_update_states_the_complete_policy() -> None:
71
+ # The security-critical policy table: whether a "restricted" sandbox actually
72
+ # gets a deny-everything floor, with framework routes kept reachable. No e2e
73
+ # placement runs network-restricted, so this table is pinned here.
74
+ routes = ["https://tunnel.example.com/intercept"]
75
+
76
+ unrestricted = _egress_update(E2BConfig(), None)
77
+ assert unrestricted == {"allow_internet_access": True}
78
+
79
+ blocklist = _egress_update(E2BConfig(block=["203.0.113.0/24"]), routes)
80
+ assert blocklist == {"deny_out": ["203.0.113.0/24"]}
81
+
82
+ allowlist = _egress_update(E2BConfig(allow=["api.example.com"]), routes)
83
+ assert allowlist == {
84
+ "allow_out": ["tunnel.example.com", "api.example.com"],
85
+ "deny_out": ["0.0.0.0/0"],
86
+ }
87
+
88
+ framework_only = _egress_update(E2BConfig(allow=[]), routes)
89
+ assert framework_only == {
90
+ "allow_out": ["tunnel.example.com"],
91
+ "deny_out": ["0.0.0.0/0"],
92
+ }
93
+
94
+ no_routes = _egress_update(E2BConfig(allow=[]), [])
95
+ assert no_routes == {"allow_internet_access": False}
@@ -2,8 +2,8 @@
2
2
 
3
3
  Placement coverage is pairwise (see tests/v1/conftest.py): each list below names the
4
4
  combinations a test runs — every axis value at least once plus the cross-boundary pairs
5
- with distinct networking — instead of fanning the full cross product. prime/modal rows
6
- are local-only (their marks are excluded in CI)."""
5
+ with distinct networking — instead of fanning the full cross product. prime/modal/e2b
6
+ rows are local-only (their marks are excluded in CI)."""
7
7
 
8
8
  import shutil
9
9
  import subprocess
@@ -142,6 +142,7 @@ CHAT_PLACEMENTS = [
142
142
  ),
143
143
  pair("bash", "prime", "bash-harness-in-prime"),
144
144
  pair("bash", "modal", "bash-harness-in-modal"),
145
+ pair("bash", "e2b", "bash-harness-in-e2b"),
145
146
  ]
146
147
 
147
148
  # harness x harness runtime for the shell task: every coding agent once (null is a chat
@@ -176,6 +177,7 @@ AGENTIC_PLACEMENTS = [
176
177
  marks=[mark.bash, mark.modal],
177
178
  id="bash-harness-in-modal-framework-only",
178
179
  ),
180
+ pair("bash", "e2b", "bash-harness-in-e2b"),
179
181
  ]
180
182
 
181
183
  # The scripted user runs in the eval process itself (no placement axis); the harness
@@ -185,6 +187,13 @@ USER_RUNTIMES = [
185
187
  pytest.param("docker", marks=[mark.docker], id="harness-in-docker"),
186
188
  pytest.param("prime", marks=[mark.prime], id="harness-in-prime"),
187
189
  pytest.param("modal", marks=[mark.modal], id="harness-in-modal"),
190
+ pytest.param("e2b", marks=[mark.e2b], id="harness-in-e2b"),
191
+ ]
192
+
193
+ # Runtimes that can switch users (`Runtime.with_user`).
194
+ AGENT_USER_RUNTIMES = [
195
+ pytest.param("docker", marks=[mark.docker], id="harness-in-docker"),
196
+ pytest.param("prime", marks=[mark.prime], id="harness-in-prime"),
188
197
  ]
189
198
 
190
199
  # ACP-backed harnesses: each must preserve an exchange across interaction segments and
@@ -218,6 +227,13 @@ ACP_RESUME_PLACEMENTS = [
218
227
  ),
219
228
  pair("openclaw", "docker", "openclaw-acp-in-docker"),
220
229
  pair("rlm", "prime", "rlm-acp-in-prime-vm"),
230
+ # Give RLM 2 GB for ACP session startup.
231
+ pytest.param(
232
+ "rlm",
233
+ {"type": "e2b", "memory": 2.0},
234
+ marks=[mark.rlm, mark.e2b],
235
+ id="rlm-acp-in-e2b",
236
+ ),
221
237
  pytest.param(
222
238
  "prime-agent",
223
239
  "prime",
@@ -247,6 +263,8 @@ TOOL_PLACEMENTS = [
247
263
  pair("prime", "colocated", "harness-in-prime-with-tool-colocated"),
248
264
  pair("modal", "colocated", "harness-in-modal-with-tool-colocated"),
249
265
  pair("subprocess", "modal", "harness-in-subprocess-with-tool-in-modal"),
266
+ pair("e2b", "colocated", "harness-in-e2b-with-tool-colocated"),
267
+ pair("subprocess", "e2b", "harness-in-subprocess-with-tool-in-e2b"),
250
268
  ]
251
269
 
252
270
  # The state channel rides the same reachability as TOOL_PLACEMENTS; cover each axis
@@ -256,6 +274,7 @@ TOOL_STATE_PLACEMENTS = [
256
274
  pair("docker", "subprocess", "harness-in-docker-with-tool-in-subprocess"),
257
275
  pair("subprocess", "docker", "harness-in-subprocess-with-tool-in-docker"),
258
276
  pair("modal", "colocated", "harness-in-modal-with-tool-colocated"),
277
+ pair("e2b", "colocated", "harness-in-e2b-with-tool-colocated"),
259
278
  ]
260
279
 
261
280
  # Shared servers always run in their own runtime (colocation is per-rollout, shared is
@@ -265,6 +284,7 @@ SHARED_TOOL_PLACEMENTS = [
265
284
  pair("docker", "docker", "harness-in-docker-with-tool-in-docker"),
266
285
  pair("subprocess", "docker", "harness-in-subprocess-with-tool-in-docker"),
267
286
  pair("modal", "modal", "harness-in-modal-with-tool-in-modal"),
287
+ pair("e2b", "e2b", "harness-in-e2b-with-tool-in-e2b"),
268
288
  ]
269
289
 
270
290
 
@@ -318,6 +338,29 @@ async def test_browser_use(run_v1, tmp_path):
318
338
  assert trace.reward == 1.0
319
339
 
320
340
 
341
+ @pytest.mark.e2e
342
+ @pytest.mark.parametrize("harness_runtime", AGENT_USER_RUNTIMES, indirect=True)
343
+ async def test_agent_user_cannot_act_as_root(run_v1, harness_runtime, tmp_path):
344
+ """A scripted, model-free agent runs as the task's non-root `agent_user`: it can see
345
+ the secret the root-run setup planted and write its own workspace, but every
346
+ root-only action (read the secret, write /etc, chmod the secret, signal PID 1) is
347
+ denied."""
348
+ (trace,) = await run_v1(
349
+ "agent-user-v1",
350
+ harness="agent-user-v1",
351
+ runtime={"type": harness_runtime},
352
+ output_dir=tmp_path,
353
+ )
354
+ report = trace.info["agent_user_report"]
355
+ assert report["user"] == "vf-agent"
356
+ assert report["secret_exists"] == "ok" and report["write_workdir"] == "ok"
357
+ root_actions = ("read_secret", "write_etc", "chmod_secret", "signal_init")
358
+ assert {action: report[action] for action in root_actions} == dict.fromkeys(
359
+ root_actions, "denied"
360
+ )
361
+ assert trace.reward == 1.0
362
+
363
+
321
364
  @pytest.mark.e2e
322
365
  @pytest.mark.parametrize("harness_runtime", USER_RUNTIMES, indirect=True)
323
366
  async def test_user(run_v1, harness_runtime, tmp_path):
@@ -389,7 +432,9 @@ async def test_acp_resume_with_tool(run_v1, harness, harness_runtime, tmp_path):
389
432
  (trace,) = await run_v1(
390
433
  "echo-acp-resume-v1",
391
434
  harness=harness,
392
- runtime={"type": harness_runtime},
435
+ runtime=harness_runtime
436
+ if isinstance(harness_runtime, dict)
437
+ else {"type": harness_runtime},
393
438
  output_dir=tmp_path,
394
439
  max_turns=8,
395
440
  max_tokens=8192,
@@ -14,6 +14,7 @@ from verifiers.v1.configs.runtime import NetworkPolicyConfig
14
14
  from verifiers.v1.errors import (
15
15
  HarnessError,
16
16
  RolloutError,
17
+ SandboxError,
17
18
  TaskError,
18
19
  ToolsetError,
19
20
  boundary,
@@ -52,6 +53,39 @@ class RolloutTimeouts:
52
53
  """Timeout (in seconds) for the task + harness metrics + scoring hooks."""
53
54
 
54
55
 
56
+ _HOME_PROBE = (
57
+ "home=$(awk -F: -v u=\"$1\" '$1 == u { print $6 }' /etc/passwd); "
58
+ 'echo "$home"; test -n "$home" && test -d "$home"'
59
+ )
60
+
61
+
62
+ async def agent_runtime(runtime: Runtime, user: str | None) -> Runtime:
63
+ """The runtime the harness runs on: `runtime` itself, or a view running as `user`
64
+ with `HOME` set to its home. `user` must exist with a home directory it can write
65
+ (task authors ship it in the image), and the view must really run as it, so a
66
+ runtime ignoring the user never runs the agent as root."""
67
+ if user is None:
68
+ return runtime
69
+ lookup = await runtime.run(["sh", "-c", _HOME_PROBE, "sh", user], {})
70
+ home = lookup.stdout.strip()
71
+ if not home:
72
+ raise TaskError(f"agent_user {user!r} does not exist in the image")
73
+ if lookup.exit_code != 0:
74
+ raise TaskError(f"agent_user {user!r} has no home directory ({home})")
75
+ view = runtime.with_user(user)
76
+ view = view.with_env({**view.env, "HOME": home})
77
+ probe = await view.run(["sh", "-c", 'id -un; test -w "$HOME" && echo writable'], {})
78
+ effective, *rest = probe.stdout.split() or [""]
79
+ if effective != user:
80
+ raise SandboxError(
81
+ f"{type(runtime).__name__} ran as {effective!r} instead of agent_user "
82
+ f"{user!r}; refusing to run the agent"
83
+ )
84
+ if rest != ["writable"]:
85
+ raise TaskError(f"agent_user {user!r} cannot write its home directory ({home})")
86
+ return view
87
+
88
+
55
89
  class Rollout:
56
90
  """Manages one rollout's lifecycle (open, step, close)."""
57
91
 
@@ -341,7 +375,7 @@ class Rollout:
341
375
  self._harness_session = await self.harness.session(
342
376
  self.ctx,
343
377
  self.trace,
344
- runtime,
378
+ await agent_runtime(runtime, harness_data.agent_user),
345
379
  self._endpoint,
346
380
  self._secret,
347
381
  self._urls,
@@ -25,6 +25,7 @@ from verifiers.v1.runtimes.docker import (
25
25
  PodmanRuntime,
26
26
  PodmanRuntimeInfo,
27
27
  )
28
+ from verifiers.v1.runtimes.e2b import E2BConfig, E2BRuntime, E2BRuntimeInfo
28
29
  from verifiers.v1.runtimes.modal import ModalConfig, ModalRuntime, ModalRuntimeInfo
29
30
  from verifiers.v1.runtimes.prime import (
30
31
  PrimeConfig,
@@ -44,7 +45,8 @@ RuntimeConfig = Annotated[
44
45
  | PodmanConfig
45
46
  | ApptainerConfig
46
47
  | PrimeConfig
47
- | ModalConfig,
48
+ | ModalConfig
49
+ | E2BConfig,
48
50
  Field(discriminator="type"),
49
51
  ]
50
52
 
@@ -54,7 +56,8 @@ RuntimeInfo = Annotated[
54
56
  | PodmanRuntimeInfo
55
57
  | ApptainerRuntimeInfo
56
58
  | PrimeRuntimeInfo
57
- | ModalRuntimeInfo,
59
+ | ModalRuntimeInfo
60
+ | E2BRuntimeInfo,
58
61
  Field(discriminator="type"),
59
62
  ]
60
63
 
@@ -67,6 +70,7 @@ def _runtime_cls(config: RuntimeConfig) -> type[Runtime]:
67
70
  "apptainer": ApptainerRuntime,
68
71
  "prime": PrimeRuntime,
69
72
  "modal": ModalRuntime,
73
+ "e2b": E2BRuntime,
70
74
  }[config.type]
71
75
 
72
76
 
@@ -109,6 +113,9 @@ __all__ = [
109
113
  "DockerConfig",
110
114
  "DockerRuntime",
111
115
  "DockerRuntimeInfo",
116
+ "E2BConfig",
117
+ "E2BRuntime",
118
+ "E2BRuntimeInfo",
112
119
  "ModalConfig",
113
120
  "ModalRuntime",
114
121
  "ModalRuntimeInfo",
@@ -136,13 +136,16 @@ class BaseRuntimeInfo(BaseConfig):
136
136
 
137
137
 
138
138
  class Runtime(ABC):
139
- __slots__ = ("env",)
139
+ __slots__ = ("env", "user")
140
140
 
141
141
  is_local: bool = True
142
142
  """Whether this runtime exchanges host-local URLs without a public tunnel. True for
143
143
  subprocess and the local container runtimes; remote runtimes override to False and
144
144
  use a host `Tunnel` inward plus `expose` outward."""
145
145
 
146
+ supports_user: ClassVar[bool] = False
147
+ """Whether `with_user` can run this runtime's commands as another existing user."""
148
+
146
149
  scripts_dir: ClassVar[str] = "/tmp/vf-scripts"
147
150
  """Digest-keyed PEP 723 scripts inside the runtime. Sandboxes own their `/tmp`;
148
151
  the host subprocess runtime overrides this with its per-user cache."""
@@ -159,6 +162,8 @@ class Runtime(ABC):
159
162
  # Per-run task values live on the runtime rather than its serializable config/info.
160
163
  # Explicit process values (model credentials, proxy settings, etc.) override these.
161
164
  self.env: dict[str, str] = {}
165
+ self.user: str | None = None
166
+ """Existing user this view runs commands as; None uses the runtime's default."""
162
167
  self._uv_interpreters: dict[str, str] = {}
163
168
  self._uv_script_locks: dict[str, asyncio.Lock] = {}
164
169
  self._mcp_sources: set[str] = set()
@@ -215,6 +220,19 @@ class Runtime(ABC):
215
220
  runtime.env = dict(env)
216
221
  return runtime
217
222
 
223
+ def with_user(self, user: str | None) -> "Runtime":
224
+ """Share this physical runtime through a view that runs commands as `user`, an
225
+ existing user in the box (None keeps the default). Only the view runs as `user`:
226
+ setup, collection, and grading keep using the original runtime."""
227
+ if user is not None and not self.supports_user:
228
+ raise ValueError(
229
+ f"{type(self).__name__} cannot run commands as another user"
230
+ )
231
+ runtime = copy.copy(self)
232
+ runtime.__dict__ = self.__dict__
233
+ runtime.user = user
234
+ return runtime
235
+
218
236
  async def alive(self) -> bool:
219
237
  """Whether the box still executes anything. Not every runtime raises when
220
238
  the box is gone — some surface it as `exec`'s own non-zero result,
@@ -70,6 +70,7 @@ control.sendmsg([b"listener"], [(socket.SOL_SOCKET, socket.SCM_RIGHTS, array.arr
70
70
 
71
71
 
72
72
  class DockerRuntime(ContainerRuntime):
73
+ supports_user: ClassVar[bool] = True
73
74
  engine: ClassVar[str] = "docker"
74
75
  """The CLI binary for the shared OCI container operations."""
75
76
  info_cls: ClassVar[type[BaseRuntimeInfo]] = DockerRuntimeInfo
@@ -529,6 +530,7 @@ class DockerRuntime(ContainerRuntime):
529
530
  "exec",
530
531
  *(("-i",) if stdin else ()),
531
532
  *(arg for key, value in env.items() for arg in ("--env", f"{key}={value}")),
533
+ *(("--user", self.user) if self.user is not None else ()),
532
534
  "--workdir",
533
535
  self.config.workdir,
534
536
  self._container,