verifiers 0.3.2.dev167__tar.gz → 0.3.2.dev169__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {verifiers-0.3.2.dev167 → verifiers-0.3.2.dev169}/PKG-INFO +4 -2
- {verifiers-0.3.2.dev167 → verifiers-0.3.2.dev169}/pyproject.toml +7 -1
- {verifiers-0.3.2.dev167 → verifiers-0.3.2.dev169}/tests/v1/conftest.py +5 -4
- verifiers-0.3.2.dev169/tests/v1/fixtures/agent_user_v1.py +82 -0
- verifiers-0.3.2.dev169/tests/v1/test_configs.py +95 -0
- {verifiers-0.3.2.dev167 → verifiers-0.3.2.dev169}/tests/v1/test_e2e.py +48 -3
- {verifiers-0.3.2.dev167 → verifiers-0.3.2.dev169}/verifiers/v1/rollout.py +35 -1
- {verifiers-0.3.2.dev167 → verifiers-0.3.2.dev169}/verifiers/v1/runtimes/__init__.py +9 -2
- {verifiers-0.3.2.dev167 → verifiers-0.3.2.dev169}/verifiers/v1/runtimes/base.py +19 -1
- {verifiers-0.3.2.dev167 → verifiers-0.3.2.dev169}/verifiers/v1/runtimes/docker/__init__.py +2 -0
- verifiers-0.3.2.dev169/verifiers/v1/runtimes/e2b.py +711 -0
- {verifiers-0.3.2.dev167 → verifiers-0.3.2.dev169}/verifiers/v1/runtimes/prime.py +5 -0
- {verifiers-0.3.2.dev167 → verifiers-0.3.2.dev169}/verifiers/v1/task.py +4 -0
- {verifiers-0.3.2.dev167 → verifiers-0.3.2.dev169}/verifiers/v1/tasksets/harbor/taskset.py +1 -0
- verifiers-0.3.2.dev167/tests/v1/test_configs.py +0 -20
- {verifiers-0.3.2.dev167 → verifiers-0.3.2.dev169}/.gitignore +0 -0
- {verifiers-0.3.2.dev167 → verifiers-0.3.2.dev169}/LICENSE +0 -0
- {verifiers-0.3.2.dev167 → verifiers-0.3.2.dev169}/README.md +0 -0
- {verifiers-0.3.2.dev167 → verifiers-0.3.2.dev169}/tests/v1/fixtures/counter_tool_v1.py +0 -0
- {verifiers-0.3.2.dev167 → verifiers-0.3.2.dev169}/tests/v1/fixtures/duet_v1.py +0 -0
- {verifiers-0.3.2.dev167 → verifiers-0.3.2.dev169}/tests/v1/fixtures/echo_acp_resume_v1.py +0 -0
- {verifiers-0.3.2.dev167 → verifiers-0.3.2.dev169}/tests/v1/fixtures/echo_agentic_v1.py +0 -0
- {verifiers-0.3.2.dev167 → verifiers-0.3.2.dev169}/tests/v1/fixtures/echo_tool_v1.py +0 -0
- {verifiers-0.3.2.dev167 → verifiers-0.3.2.dev169}/tests/v1/fixtures/echo_user_sim_v1.py +0 -0
- {verifiers-0.3.2.dev167 → verifiers-0.3.2.dev169}/tests/v1/fixtures/echo_v1.py +0 -0
- {verifiers-0.3.2.dev167 → verifiers-0.3.2.dev169}/tests/v1/fixtures/pyproject.toml +0 -0
- {verifiers-0.3.2.dev167 → verifiers-0.3.2.dev169}/tests/v1/fixtures/tool_response_image_v1.py +0 -0
- {verifiers-0.3.2.dev167 → verifiers-0.3.2.dev169}/tests/v1/test_color_codeword_tasks.py +0 -0
- {verifiers-0.3.2.dev167 → verifiers-0.3.2.dev169}/tests/v1/test_envs.py +0 -0
- {verifiers-0.3.2.dev167 → verifiers-0.3.2.dev169}/tests/v1/test_graph.py +0 -0
- {verifiers-0.3.2.dev167 → verifiers-0.3.2.dev169}/tests/v1/test_judges.py +0 -0
- {verifiers-0.3.2.dev167 → verifiers-0.3.2.dev169}/tests/v1/test_platform_run.py +0 -0
- {verifiers-0.3.2.dev167 → verifiers-0.3.2.dev169}/tests/v1/test_scoring.py +0 -0
- {verifiers-0.3.2.dev167 → verifiers-0.3.2.dev169}/tests/v1/test_serve_delta.py +0 -0
- {verifiers-0.3.2.dev167 → verifiers-0.3.2.dev169}/tests/v1/test_taskset.py +0 -0
- {verifiers-0.3.2.dev167 → verifiers-0.3.2.dev169}/tests/v1/test_trace.py +0 -0
- {verifiers-0.3.2.dev167 → verifiers-0.3.2.dev169}/verifiers/__init__.py +0 -0
- {verifiers-0.3.2.dev167 → verifiers-0.3.2.dev169}/verifiers/v1/__init__.py +0 -0
- {verifiers-0.3.2.dev167 → verifiers-0.3.2.dev169}/verifiers/v1/acp/__init__.py +0 -0
- {verifiers-0.3.2.dev167 → verifiers-0.3.2.dev169}/verifiers/v1/acp/runner.py +0 -0
- {verifiers-0.3.2.dev167 → verifiers-0.3.2.dev169}/verifiers/v1/agent.py +0 -0
- {verifiers-0.3.2.dev167 → verifiers-0.3.2.dev169}/verifiers/v1/cli/__init__.py +0 -0
- {verifiers-0.3.2.dev167 → verifiers-0.3.2.dev169}/verifiers/v1/cli/dashboard/__init__.py +0 -0
- {verifiers-0.3.2.dev167 → verifiers-0.3.2.dev169}/verifiers/v1/cli/dashboard/base.py +0 -0
- {verifiers-0.3.2.dev167 → verifiers-0.3.2.dev169}/verifiers/v1/cli/dashboard/eval.py +0 -0
- {verifiers-0.3.2.dev167 → verifiers-0.3.2.dev169}/verifiers/v1/cli/dashboard/replay.py +0 -0
- {verifiers-0.3.2.dev167 → verifiers-0.3.2.dev169}/verifiers/v1/cli/dashboard/validate.py +0 -0
- {verifiers-0.3.2.dev167 → verifiers-0.3.2.dev169}/verifiers/v1/cli/debug.py +0 -0
- {verifiers-0.3.2.dev167 → verifiers-0.3.2.dev169}/verifiers/v1/cli/eval/__init__.py +0 -0
- {verifiers-0.3.2.dev167 → verifiers-0.3.2.dev169}/verifiers/v1/cli/eval/hint.py +0 -0
- {verifiers-0.3.2.dev167 → verifiers-0.3.2.dev169}/verifiers/v1/cli/eval/main.py +0 -0
- {verifiers-0.3.2.dev167 → verifiers-0.3.2.dev169}/verifiers/v1/cli/eval/resume.py +0 -0
- {verifiers-0.3.2.dev167 → verifiers-0.3.2.dev169}/verifiers/v1/cli/eval/runner.py +0 -0
- {verifiers-0.3.2.dev167 → verifiers-0.3.2.dev169}/verifiers/v1/cli/gepa.py +0 -0
- {verifiers-0.3.2.dev167 → verifiers-0.3.2.dev169}/verifiers/v1/cli/init.py +0 -0
- {verifiers-0.3.2.dev167 → verifiers-0.3.2.dev169}/verifiers/v1/cli/output.py +0 -0
- {verifiers-0.3.2.dev167 → verifiers-0.3.2.dev169}/verifiers/v1/cli/replay.py +0 -0
- {verifiers-0.3.2.dev167 → verifiers-0.3.2.dev169}/verifiers/v1/cli/resolve.py +0 -0
- {verifiers-0.3.2.dev167 → verifiers-0.3.2.dev169}/verifiers/v1/cli/resume.py +0 -0
- {verifiers-0.3.2.dev167 → verifiers-0.3.2.dev169}/verifiers/v1/cli/validate.py +0 -0
- {verifiers-0.3.2.dev167 → verifiers-0.3.2.dev169}/verifiers/v1/clients/__init__.py +0 -0
- {verifiers-0.3.2.dev167 → verifiers-0.3.2.dev169}/verifiers/v1/clients/base.py +0 -0
- {verifiers-0.3.2.dev167 → verifiers-0.3.2.dev169}/verifiers/v1/clients/client.py +0 -0
- {verifiers-0.3.2.dev167 → verifiers-0.3.2.dev169}/verifiers/v1/clients/eval.py +0 -0
- {verifiers-0.3.2.dev167 → verifiers-0.3.2.dev169}/verifiers/v1/clients/train.py +0 -0
- {verifiers-0.3.2.dev167 → verifiers-0.3.2.dev169}/verifiers/v1/configs/__init__.py +0 -0
- {verifiers-0.3.2.dev167 → verifiers-0.3.2.dev169}/verifiers/v1/configs/agent.py +0 -0
- {verifiers-0.3.2.dev167 → verifiers-0.3.2.dev169}/verifiers/v1/configs/cli/__init__.py +0 -0
- {verifiers-0.3.2.dev167 → verifiers-0.3.2.dev169}/verifiers/v1/configs/cli/debug.py +0 -0
- {verifiers-0.3.2.dev167 → verifiers-0.3.2.dev169}/verifiers/v1/configs/cli/env.py +0 -0
- {verifiers-0.3.2.dev167 → verifiers-0.3.2.dev169}/verifiers/v1/configs/cli/eval.py +0 -0
- {verifiers-0.3.2.dev167 → verifiers-0.3.2.dev169}/verifiers/v1/configs/cli/init.py +0 -0
- {verifiers-0.3.2.dev167 → verifiers-0.3.2.dev169}/verifiers/v1/configs/cli/replay.py +0 -0
- {verifiers-0.3.2.dev167 → verifiers-0.3.2.dev169}/verifiers/v1/configs/cli/validate.py +0 -0
- {verifiers-0.3.2.dev167 → verifiers-0.3.2.dev169}/verifiers/v1/configs/client.py +0 -0
- {verifiers-0.3.2.dev167 → verifiers-0.3.2.dev169}/verifiers/v1/configs/env.py +0 -0
- {verifiers-0.3.2.dev167 → verifiers-0.3.2.dev169}/verifiers/v1/configs/harness.py +0 -0
- {verifiers-0.3.2.dev167 → verifiers-0.3.2.dev169}/verifiers/v1/configs/judge.py +0 -0
- {verifiers-0.3.2.dev167 → verifiers-0.3.2.dev169}/verifiers/v1/configs/retries.py +0 -0
- {verifiers-0.3.2.dev167 → verifiers-0.3.2.dev169}/verifiers/v1/configs/runtime.py +0 -0
- {verifiers-0.3.2.dev167 → verifiers-0.3.2.dev169}/verifiers/v1/configs/select.py +0 -0
- {verifiers-0.3.2.dev167 → verifiers-0.3.2.dev169}/verifiers/v1/configs/serve.py +0 -0
- {verifiers-0.3.2.dev167 → verifiers-0.3.2.dev169}/verifiers/v1/configs/task.py +0 -0
- {verifiers-0.3.2.dev167 → verifiers-0.3.2.dev169}/verifiers/v1/configs/taskset.py +0 -0
- {verifiers-0.3.2.dev167 → verifiers-0.3.2.dev169}/verifiers/v1/dialects/__init__.py +0 -0
- {verifiers-0.3.2.dev167 → verifiers-0.3.2.dev169}/verifiers/v1/dialects/anthropic.py +0 -0
- {verifiers-0.3.2.dev167 → verifiers-0.3.2.dev169}/verifiers/v1/dialects/base.py +0 -0
- {verifiers-0.3.2.dev167 → verifiers-0.3.2.dev169}/verifiers/v1/dialects/chat.py +0 -0
- {verifiers-0.3.2.dev167 → verifiers-0.3.2.dev169}/verifiers/v1/dialects/responses.py +0 -0
- {verifiers-0.3.2.dev167 → verifiers-0.3.2.dev169}/verifiers/v1/env.py +0 -0
- {verifiers-0.3.2.dev167 → verifiers-0.3.2.dev169}/verifiers/v1/envs/__init__.py +0 -0
- {verifiers-0.3.2.dev167 → verifiers-0.3.2.dev169}/verifiers/v1/envs/agentic_judge/__init__.py +0 -0
- {verifiers-0.3.2.dev167 → verifiers-0.3.2.dev169}/verifiers/v1/envs/agentic_judge/env.py +0 -0
- {verifiers-0.3.2.dev167 → verifiers-0.3.2.dev169}/verifiers/v1/envs/best_of_n/__init__.py +0 -0
- {verifiers-0.3.2.dev167 → verifiers-0.3.2.dev169}/verifiers/v1/envs/best_of_n/env.py +0 -0
- {verifiers-0.3.2.dev167 → verifiers-0.3.2.dev169}/verifiers/v1/envs/isolated_verifier/__init__.py +0 -0
- {verifiers-0.3.2.dev167 → verifiers-0.3.2.dev169}/verifiers/v1/envs/isolated_verifier/env.py +0 -0
- {verifiers-0.3.2.dev167 → verifiers-0.3.2.dev169}/verifiers/v1/envs/shared_agentic_judge/__init__.py +0 -0
- {verifiers-0.3.2.dev167 → verifiers-0.3.2.dev169}/verifiers/v1/envs/single_agent/__init__.py +0 -0
- {verifiers-0.3.2.dev167 → verifiers-0.3.2.dev169}/verifiers/v1/envs/single_agent/env.py +0 -0
- {verifiers-0.3.2.dev167 → verifiers-0.3.2.dev169}/verifiers/v1/envs/user_sim/__init__.py +0 -0
- {verifiers-0.3.2.dev167 → verifiers-0.3.2.dev169}/verifiers/v1/envs/user_sim/env.py +0 -0
- {verifiers-0.3.2.dev167 → verifiers-0.3.2.dev169}/verifiers/v1/episode.py +0 -0
- {verifiers-0.3.2.dev167 → verifiers-0.3.2.dev169}/verifiers/v1/errors.py +0 -0
- {verifiers-0.3.2.dev167 → verifiers-0.3.2.dev169}/verifiers/v1/gepa/__init__.py +0 -0
- {verifiers-0.3.2.dev167 → verifiers-0.3.2.dev169}/verifiers/v1/gepa/adapter.py +0 -0
- {verifiers-0.3.2.dev167 → verifiers-0.3.2.dev169}/verifiers/v1/gepa/config.py +0 -0
- {verifiers-0.3.2.dev167 → verifiers-0.3.2.dev169}/verifiers/v1/gepa/dataset.py +0 -0
- {verifiers-0.3.2.dev167 → verifiers-0.3.2.dev169}/verifiers/v1/gepa/reflection.py +0 -0
- {verifiers-0.3.2.dev167 → verifiers-0.3.2.dev169}/verifiers/v1/gepa/runner.py +0 -0
- {verifiers-0.3.2.dev167 → verifiers-0.3.2.dev169}/verifiers/v1/graph.py +0 -0
- {verifiers-0.3.2.dev167 → verifiers-0.3.2.dev169}/verifiers/v1/harness.py +0 -0
- {verifiers-0.3.2.dev167 → verifiers-0.3.2.dev169}/verifiers/v1/harnesses/__init__.py +0 -0
- {verifiers-0.3.2.dev167 → verifiers-0.3.2.dev169}/verifiers/v1/harnesses/bash/__init__.py +0 -0
- {verifiers-0.3.2.dev167 → verifiers-0.3.2.dev169}/verifiers/v1/harnesses/bash/harness.py +0 -0
- {verifiers-0.3.2.dev167 → verifiers-0.3.2.dev169}/verifiers/v1/harnesses/browser_use/__init__.py +0 -0
- {verifiers-0.3.2.dev167 → verifiers-0.3.2.dev169}/verifiers/v1/harnesses/browser_use/harness.py +0 -0
- {verifiers-0.3.2.dev167 → verifiers-0.3.2.dev169}/verifiers/v1/harnesses/browser_use/program.py +0 -0
- {verifiers-0.3.2.dev167 → verifiers-0.3.2.dev169}/verifiers/v1/harnesses/claude_code/__init__.py +0 -0
- {verifiers-0.3.2.dev167 → verifiers-0.3.2.dev169}/verifiers/v1/harnesses/claude_code/harness.py +0 -0
- {verifiers-0.3.2.dev167 → verifiers-0.3.2.dev169}/verifiers/v1/harnesses/codex/__init__.py +0 -0
- {verifiers-0.3.2.dev167 → verifiers-0.3.2.dev169}/verifiers/v1/harnesses/codex/gate.mjs +0 -0
- {verifiers-0.3.2.dev167 → verifiers-0.3.2.dev169}/verifiers/v1/harnesses/codex/harness.py +0 -0
- {verifiers-0.3.2.dev167 → verifiers-0.3.2.dev169}/verifiers/v1/harnesses/hermes_agent/__init__.py +0 -0
- {verifiers-0.3.2.dev167 → verifiers-0.3.2.dev169}/verifiers/v1/harnesses/hermes_agent/harness.py +0 -0
- {verifiers-0.3.2.dev167 → verifiers-0.3.2.dev169}/verifiers/v1/harnesses/hermes_agent/program.py +0 -0
- {verifiers-0.3.2.dev167 → verifiers-0.3.2.dev169}/verifiers/v1/harnesses/kimi_code/__init__.py +0 -0
- {verifiers-0.3.2.dev167 → verifiers-0.3.2.dev169}/verifiers/v1/harnesses/kimi_code/harness.py +0 -0
- {verifiers-0.3.2.dev167 → verifiers-0.3.2.dev169}/verifiers/v1/harnesses/mini_swe_agent/__init__.py +0 -0
- {verifiers-0.3.2.dev167 → verifiers-0.3.2.dev169}/verifiers/v1/harnesses/mini_swe_agent/harness.py +0 -0
- {verifiers-0.3.2.dev167 → verifiers-0.3.2.dev169}/verifiers/v1/harnesses/mini_swe_agent/program.py +0 -0
- {verifiers-0.3.2.dev167 → verifiers-0.3.2.dev169}/verifiers/v1/harnesses/node.py +0 -0
- {verifiers-0.3.2.dev167 → verifiers-0.3.2.dev169}/verifiers/v1/harnesses/null/__init__.py +0 -0
- {verifiers-0.3.2.dev167 → verifiers-0.3.2.dev169}/verifiers/v1/harnesses/null/harness.py +0 -0
- {verifiers-0.3.2.dev167 → verifiers-0.3.2.dev169}/verifiers/v1/harnesses/openclaw/__init__.py +0 -0
- {verifiers-0.3.2.dev167 → verifiers-0.3.2.dev169}/verifiers/v1/harnesses/openclaw/harness.py +0 -0
- {verifiers-0.3.2.dev167 → verifiers-0.3.2.dev169}/verifiers/v1/harnesses/pi/__init__.py +0 -0
- {verifiers-0.3.2.dev167 → verifiers-0.3.2.dev169}/verifiers/v1/harnesses/pi/gate.mjs +0 -0
- {verifiers-0.3.2.dev167 → verifiers-0.3.2.dev169}/verifiers/v1/harnesses/pi/harness.py +0 -0
- {verifiers-0.3.2.dev167 → verifiers-0.3.2.dev169}/verifiers/v1/harnesses/prime_agent/__init__.py +0 -0
- {verifiers-0.3.2.dev167 → verifiers-0.3.2.dev169}/verifiers/v1/harnesses/prime_agent/harness.py +0 -0
- {verifiers-0.3.2.dev167 → verifiers-0.3.2.dev169}/verifiers/v1/harnesses/rlm/__init__.py +0 -0
- {verifiers-0.3.2.dev167 → verifiers-0.3.2.dev169}/verifiers/v1/harnesses/rlm/harness.py +0 -0
- {verifiers-0.3.2.dev167 → verifiers-0.3.2.dev169}/verifiers/v1/harnesses/terminus_2/__init__.py +0 -0
- {verifiers-0.3.2.dev167 → verifiers-0.3.2.dev169}/verifiers/v1/harnesses/terminus_2/harness.py +0 -0
- {verifiers-0.3.2.dev167 → verifiers-0.3.2.dev169}/verifiers/v1/harnesses/terminus_2/program.py +0 -0
- {verifiers-0.3.2.dev167 → verifiers-0.3.2.dev169}/verifiers/v1/harnesses/utils/__init__.py +0 -0
- {verifiers-0.3.2.dev167 → verifiers-0.3.2.dev169}/verifiers/v1/harnesses/utils/compaction.py +0 -0
- {verifiers-0.3.2.dev167 → verifiers-0.3.2.dev169}/verifiers/v1/harnesses/utils/core.py +0 -0
- {verifiers-0.3.2.dev167 → verifiers-0.3.2.dev169}/verifiers/v1/harnesses/utils/install.py +0 -0
- {verifiers-0.3.2.dev167 → verifiers-0.3.2.dev169}/verifiers/v1/harnesses/utils/launch.py +0 -0
- {verifiers-0.3.2.dev167 → verifiers-0.3.2.dev169}/verifiers/v1/harnesses/utils/mcp.py +0 -0
- {verifiers-0.3.2.dev167 → verifiers-0.3.2.dev169}/verifiers/v1/interception/__init__.py +0 -0
- {verifiers-0.3.2.dev167 → verifiers-0.3.2.dev169}/verifiers/v1/interception/base.py +0 -0
- {verifiers-0.3.2.dev167 → verifiers-0.3.2.dev169}/verifiers/v1/interception/pool.py +0 -0
- {verifiers-0.3.2.dev167 → verifiers-0.3.2.dev169}/verifiers/v1/interception/server.py +0 -0
- {verifiers-0.3.2.dev167 → verifiers-0.3.2.dev169}/verifiers/v1/interception/tunnel/__init__.py +0 -0
- {verifiers-0.3.2.dev167 → verifiers-0.3.2.dev169}/verifiers/v1/interception/tunnel/base.py +0 -0
- {verifiers-0.3.2.dev167 → verifiers-0.3.2.dev169}/verifiers/v1/interception/tunnel/custom.py +0 -0
- {verifiers-0.3.2.dev167 → verifiers-0.3.2.dev169}/verifiers/v1/interception/tunnel/prime.py +0 -0
- {verifiers-0.3.2.dev167 → verifiers-0.3.2.dev169}/verifiers/v1/judge.py +0 -0
- {verifiers-0.3.2.dev167 → verifiers-0.3.2.dev169}/verifiers/v1/judges/__init__.py +0 -0
- {verifiers-0.3.2.dev167 → verifiers-0.3.2.dev169}/verifiers/v1/judges/reference.py +0 -0
- {verifiers-0.3.2.dev167 → verifiers-0.3.2.dev169}/verifiers/v1/judges/reference.txt +0 -0
- {verifiers-0.3.2.dev167 → verifiers-0.3.2.dev169}/verifiers/v1/judges/rubric.py +0 -0
- {verifiers-0.3.2.dev167 → verifiers-0.3.2.dev169}/verifiers/v1/judges/rubric.txt +0 -0
- {verifiers-0.3.2.dev167 → verifiers-0.3.2.dev169}/verifiers/v1/mcp/__init__.py +0 -0
- {verifiers-0.3.2.dev167 → verifiers-0.3.2.dev169}/verifiers/v1/mcp/launch.py +0 -0
- {verifiers-0.3.2.dev167 → verifiers-0.3.2.dev169}/verifiers/v1/mcp/server.py +0 -0
- {verifiers-0.3.2.dev167 → verifiers-0.3.2.dev169}/verifiers/v1/mcp/toolset.py +0 -0
- {verifiers-0.3.2.dev167 → verifiers-0.3.2.dev169}/verifiers/v1/runtimes/apptainer.py +0 -0
- {verifiers-0.3.2.dev167 → verifiers-0.3.2.dev169}/verifiers/v1/runtimes/container.py +0 -0
- {verifiers-0.3.2.dev167 → verifiers-0.3.2.dev169}/verifiers/v1/runtimes/docker/egress.py +0 -0
- {verifiers-0.3.2.dev167 → verifiers-0.3.2.dev169}/verifiers/v1/runtimes/limiters.py +0 -0
- {verifiers-0.3.2.dev167 → verifiers-0.3.2.dev169}/verifiers/v1/runtimes/modal.py +0 -0
- {verifiers-0.3.2.dev167 → verifiers-0.3.2.dev169}/verifiers/v1/runtimes/subprocess.py +0 -0
- {verifiers-0.3.2.dev167 → verifiers-0.3.2.dev169}/verifiers/v1/semantic.py +0 -0
- {verifiers-0.3.2.dev167 → verifiers-0.3.2.dev169}/verifiers/v1/serve/__init__.py +0 -0
- {verifiers-0.3.2.dev167 → verifiers-0.3.2.dev169}/verifiers/v1/serve/client.py +0 -0
- {verifiers-0.3.2.dev167 → verifiers-0.3.2.dev169}/verifiers/v1/serve/delta.py +0 -0
- {verifiers-0.3.2.dev167 → verifiers-0.3.2.dev169}/verifiers/v1/serve/encoding.py +0 -0
- {verifiers-0.3.2.dev167 → verifiers-0.3.2.dev169}/verifiers/v1/serve/pool.py +0 -0
- {verifiers-0.3.2.dev167 → verifiers-0.3.2.dev169}/verifiers/v1/serve/server.py +0 -0
- {verifiers-0.3.2.dev167 → verifiers-0.3.2.dev169}/verifiers/v1/serve/types.py +0 -0
- {verifiers-0.3.2.dev167 → verifiers-0.3.2.dev169}/verifiers/v1/session.py +0 -0
- {verifiers-0.3.2.dev167 → verifiers-0.3.2.dev169}/verifiers/v1/state.py +0 -0
- {verifiers-0.3.2.dev167 → verifiers-0.3.2.dev169}/verifiers/v1/taskset.py +0 -0
- {verifiers-0.3.2.dev167 → verifiers-0.3.2.dev169}/verifiers/v1/tasksets/__init__.py +0 -0
- {verifiers-0.3.2.dev167 → verifiers-0.3.2.dev169}/verifiers/v1/tasksets/harbor/__init__.py +0 -0
- {verifiers-0.3.2.dev167 → verifiers-0.3.2.dev169}/verifiers/v1/tasksets/harbor/compose.py +0 -0
- {verifiers-0.3.2.dev167 → verifiers-0.3.2.dev169}/verifiers/v1/tasksets/harbor/env.py +0 -0
- {verifiers-0.3.2.dev167 → verifiers-0.3.2.dev169}/verifiers/v1/tasksets/harbor/toolset.py +0 -0
- {verifiers-0.3.2.dev167 → verifiers-0.3.2.dev169}/verifiers/v1/tasksets/nemo_gym/__init__.py +0 -0
- {verifiers-0.3.2.dev167 → verifiers-0.3.2.dev169}/verifiers/v1/tasksets/nemo_gym/response.py +0 -0
- {verifiers-0.3.2.dev167 → verifiers-0.3.2.dev169}/verifiers/v1/tasksets/nemo_gym/server.py +0 -0
- {verifiers-0.3.2.dev167 → verifiers-0.3.2.dev169}/verifiers/v1/tasksets/nemo_gym/taskset.py +0 -0
- {verifiers-0.3.2.dev167 → verifiers-0.3.2.dev169}/verifiers/v1/tasksets/nemo_gym/toolset.py +0 -0
- {verifiers-0.3.2.dev167 → verifiers-0.3.2.dev169}/verifiers/v1/tasksets/openenv/__init__.py +0 -0
- {verifiers-0.3.2.dev167 → verifiers-0.3.2.dev169}/verifiers/v1/tasksets/openenv/taskset.py +0 -0
- {verifiers-0.3.2.dev167 → verifiers-0.3.2.dev169}/verifiers/v1/tasksets/textarena/__init__.py +0 -0
- {verifiers-0.3.2.dev167 → verifiers-0.3.2.dev169}/verifiers/v1/tasksets/textarena/taskset.py +0 -0
- {verifiers-0.3.2.dev167 → verifiers-0.3.2.dev169}/verifiers/v1/trace.py +0 -0
- {verifiers-0.3.2.dev167 → verifiers-0.3.2.dev169}/verifiers/v1/types.py +0 -0
- {verifiers-0.3.2.dev167 → verifiers-0.3.2.dev169}/verifiers/v1/utils/__init__.py +0 -0
- {verifiers-0.3.2.dev167 → verifiers-0.3.2.dev169}/verifiers/v1/utils/aio.py +0 -0
- {verifiers-0.3.2.dev167 → verifiers-0.3.2.dev169}/verifiers/v1/utils/artifacts.py +0 -0
- {verifiers-0.3.2.dev167 → verifiers-0.3.2.dev169}/verifiers/v1/utils/compile.py +0 -0
- {verifiers-0.3.2.dev167 → verifiers-0.3.2.dev169}/verifiers/v1/utils/decorators.py +0 -0
- {verifiers-0.3.2.dev167 → verifiers-0.3.2.dev169}/verifiers/v1/utils/format.py +0 -0
- {verifiers-0.3.2.dev167 → verifiers-0.3.2.dev169}/verifiers/v1/utils/generic.py +0 -0
- {verifiers-0.3.2.dev167 → verifiers-0.3.2.dev169}/verifiers/v1/utils/git.py +0 -0
- {verifiers-0.3.2.dev167 → verifiers-0.3.2.dev169}/verifiers/v1/utils/image.py +0 -0
- {verifiers-0.3.2.dev167 → verifiers-0.3.2.dev169}/verifiers/v1/utils/interrupt.py +0 -0
- {verifiers-0.3.2.dev167 → verifiers-0.3.2.dev169}/verifiers/v1/utils/loaders.py +0 -0
- {verifiers-0.3.2.dev167 → verifiers-0.3.2.dev169}/verifiers/v1/utils/logging.py +0 -0
- {verifiers-0.3.2.dev167 → verifiers-0.3.2.dev169}/verifiers/v1/utils/memory.py +0 -0
- {verifiers-0.3.2.dev167 → verifiers-0.3.2.dev169}/verifiers/v1/utils/paths.py +0 -0
- {verifiers-0.3.2.dev167 → verifiers-0.3.2.dev169}/verifiers/v1/utils/platform.py +0 -0
- {verifiers-0.3.2.dev167 → verifiers-0.3.2.dev169}/verifiers/v1/utils/prime.py +0 -0
- {verifiers-0.3.2.dev167 → verifiers-0.3.2.dev169}/verifiers/v1/utils/retries.py +0 -0
- {verifiers-0.3.2.dev167 → verifiers-0.3.2.dev169}/verifiers/v1/utils/scope.py +0 -0
- {verifiers-0.3.2.dev167 → verifiers-0.3.2.dev169}/verifiers/v1/utils/score.py +0 -0
- {verifiers-0.3.2.dev167 → verifiers-0.3.2.dev169}/verifiers/v1/utils/trace_store.py +0 -0
- {verifiers-0.3.2.dev167 → verifiers-0.3.2.dev169}/verifiers/v1/utils/version.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.5
|
|
2
2
|
Name: verifiers
|
|
3
|
-
Version: 0.3.2.
|
|
3
|
+
Version: 0.3.2.dev169
|
|
4
4
|
Summary: Verifiers: Environments for LLM Reinforcement Learning
|
|
5
5
|
Project-URL: Homepage, https://github.com/primeintellect-ai/verifiers
|
|
6
6
|
Project-URL: Documentation, https://github.com/primeintellect-ai/verifiers
|
|
@@ -36,7 +36,7 @@ Requires-Dist: numpy>=2.1.0
|
|
|
36
36
|
Requires-Dist: openai<3.0.0,>=2.54.0
|
|
37
37
|
Requires-Dist: prime-pydantic-config[toml]>=0.4.3
|
|
38
38
|
Requires-Dist: prime-runs>=0.1.3
|
|
39
|
-
Requires-Dist: prime-sandboxes>=0.
|
|
39
|
+
Requires-Dist: prime-sandboxes>=0.4.1
|
|
40
40
|
Requires-Dist: prime-tunnel>=0.1.8
|
|
41
41
|
Requires-Dist: pydantic>=2.12.3
|
|
42
42
|
Requires-Dist: pyzmq>=27.1.0
|
|
@@ -46,6 +46,8 @@ Requires-Dist: tenacity>=8.5.0
|
|
|
46
46
|
Requires-Dist: tomli-w>=1.0.0
|
|
47
47
|
Requires-Dist: typing-extensions>=4.12.2
|
|
48
48
|
Requires-Dist: uvicorn>=0.52.0
|
|
49
|
+
Provides-Extra: e2b
|
|
50
|
+
Requires-Dist: e2b<3,>=2.35.0; extra == 'e2b'
|
|
49
51
|
Provides-Extra: harbor
|
|
50
52
|
Requires-Dist: harbor==0.21.0; (python_full_version >= '3.12') and extra == 'harbor'
|
|
51
53
|
Provides-Extra: modal
|
|
@@ -33,7 +33,7 @@ dependencies = [
|
|
|
33
33
|
"math-verify>=0.8.0",
|
|
34
34
|
"openai>=2.54.0,<3.0.0",
|
|
35
35
|
"prime-tunnel>=0.1.8",
|
|
36
|
-
"prime-sandboxes>=0.
|
|
36
|
+
"prime-sandboxes>=0.4.1",
|
|
37
37
|
"prime-runs>=0.1.3",
|
|
38
38
|
"pydantic>=2.12.3",
|
|
39
39
|
"rich>=11.0.0",
|
|
@@ -100,6 +100,11 @@ harbor = [
|
|
|
100
100
|
modal = [
|
|
101
101
|
"modal>=1.5.4",
|
|
102
102
|
]
|
|
103
|
+
e2b = [
|
|
104
|
+
# <3: the runtime swaps the SDK command handle's private stream decoders for
|
|
105
|
+
# byte-accurate live output (see runtimes/e2b.py _byte_handle).
|
|
106
|
+
"e2b>=2.35.0,<3",
|
|
107
|
+
]
|
|
103
108
|
openenv = [
|
|
104
109
|
"openenv>=0.4.1",
|
|
105
110
|
]
|
|
@@ -228,6 +233,7 @@ markers = [
|
|
|
228
233
|
"apptainer: v1 e2e cases touching the apptainer runtime",
|
|
229
234
|
"prime: v1 e2e cases touching the prime runtime (real sandbox + tunnel; local-only, excluded in CI)",
|
|
230
235
|
"modal: v1 e2e cases touching the modal runtime (local-only setup; excluded in CI)",
|
|
236
|
+
"e2b: v1 e2e cases touching the e2b runtime (real sandbox; local-only, excluded in CI)",
|
|
231
237
|
"colocated: v1 e2e cases with a user/tool server colocated in the harness runtime",
|
|
232
238
|
"null: v1 e2e cases on the null harness",
|
|
233
239
|
"bash: v1 e2e cases on the bash harness",
|
|
@@ -16,20 +16,21 @@ one without `indirect=True` fails loudly.
|
|
|
16
16
|
|
|
17
17
|
Every combination carries its axes' pytest marks, so subsets select with `-m`:
|
|
18
18
|
|
|
19
|
-
uv run pytest tests/v1 -n auto # everything (needs
|
|
19
|
+
uv run pytest tests/v1 -n auto # everything (needs remote setup)
|
|
20
20
|
uv run pytest tests/v1 -n auto -m "not e2e" # deterministic CI matrix
|
|
21
|
-
uv run pytest tests/v1 -n auto -m "e2e and not prime and not modal" # live CI job
|
|
21
|
+
uv run pytest tests/v1 -n auto -m "e2e and not prime and not modal and not e2b" # live CI job
|
|
22
22
|
uv run pytest tests/v1 -n auto -m docker # any case touching the docker runtime
|
|
23
23
|
uv run pytest tests/v1 -n auto -m bash # only the bash harness
|
|
24
24
|
uv run pytest tests/v1 -n auto -m prime # only prime (real sandboxes; local)
|
|
25
25
|
uv run pytest tests/v1 -n auto -m modal # only modal (needs local setup)
|
|
26
|
+
uv run pytest tests/v1 -n auto -m e2b # only e2b (real sandboxes; local)
|
|
26
27
|
|
|
27
|
-
Marks: runtimes `subprocess` / `docker` / `podman` / `apptainer` / `prime` / `modal`, placement `colocated`,
|
|
28
|
+
Marks: runtimes `subprocess` / `docker` / `podman` / `apptainer` / `prime` / `modal` / `e2b`, placement `colocated`,
|
|
28
29
|
harnesses `null` / `bash` / `rlm` / `kimi_code` / `pi` / `openclaw` / `codex` /
|
|
29
30
|
`claude_code` / `hermes_agent`.
|
|
30
31
|
A mark is applied per axis, so it selects every case touching that value on ANY axis; for one exact
|
|
31
32
|
combination use `-k` on the test id (e.g. `-k "harness-in-docker-with-tool-in-subprocess"`).
|
|
32
|
-
prime/modal provision real remote sandboxes (slow, infra-flaky, need setup), so they're local-only.
|
|
33
|
+
prime/modal/e2b provision real remote sandboxes (slow, infra-flaky, need setup), so they're local-only.
|
|
33
34
|
CI runs deterministic tests across the Python matrix and the remaining live E2Es once.
|
|
34
35
|
"""
|
|
35
36
|
|
|
@@ -0,0 +1,82 @@
|
|
|
1
|
+
"""agent-user-v1: a scripted, model-free agent that runs as a non-root user and tries root actions.
|
|
2
|
+
|
|
3
|
+
Task setup runs as root: it creates the task's `agent_user` (images normally ship one),
|
|
4
|
+
plants a root-only secret, and hands the workdir to that user. The scripted harness then runs
|
|
5
|
+
as that user, attempts four actions only root may take, and records each outcome in the
|
|
6
|
+
workdir. The reward is 1 only if every root action was denied while the agent still saw the
|
|
7
|
+
secret and could write its workspace.
|
|
8
|
+
Needs a runtime that can switch users (docker or prime).
|
|
9
|
+
"""
|
|
10
|
+
|
|
11
|
+
import json
|
|
12
|
+
|
|
13
|
+
import verifiers.v1 as vf
|
|
14
|
+
from verifiers.v1.clients import ModelContext
|
|
15
|
+
from verifiers.v1.configs.harness import HarnessConfig
|
|
16
|
+
from verifiers.v1.harness import Harness
|
|
17
|
+
from verifiers.v1.runtimes import ProgramResult, Runtime
|
|
18
|
+
|
|
19
|
+
__all__ = ["AgentUserTaskset", "ScriptedAgentUserHarness"]
|
|
20
|
+
|
|
21
|
+
AGENT_USER = "vf-agent"
|
|
22
|
+
SECRET = "/opt/vf-agent-user-secret"
|
|
23
|
+
REPORT = "agent-user-report.json"
|
|
24
|
+
ROOT_ACTIONS = ("read_secret", "write_etc", "chmod_secret", "signal_init")
|
|
25
|
+
|
|
26
|
+
SETUP = (
|
|
27
|
+
f"useradd -m {AGENT_USER} && echo secret > {SECRET} && chmod 600 {SECRET} "
|
|
28
|
+
f"&& chown -R {AGENT_USER} ."
|
|
29
|
+
)
|
|
30
|
+
|
|
31
|
+
PROBE = f"""
|
|
32
|
+
r() {{ "$@" >/dev/null 2>&1 && echo ok || echo denied; }}
|
|
33
|
+
printf '{{"user":"%s","secret_exists":"%s","read_secret":"%s","write_etc":"%s","chmod_secret":"%s","signal_init":"%s","write_workdir":"%s"}}' \\
|
|
34
|
+
"$(id -un)" "$(r test -e {SECRET})" "$(r cat {SECRET})" "$(r touch /etc/vf-agent-user-probe)" \\
|
|
35
|
+
"$(r chmod 644 {SECRET})" "$(r kill -0 1)" "$(r touch probe)" > {REPORT}
|
|
36
|
+
"""
|
|
37
|
+
|
|
38
|
+
|
|
39
|
+
class ScriptedAgentUserHarness(Harness[HarnessConfig]):
|
|
40
|
+
async def launch(
|
|
41
|
+
self,
|
|
42
|
+
ctx: ModelContext,
|
|
43
|
+
trace: vf.Trace,
|
|
44
|
+
runtime: Runtime,
|
|
45
|
+
endpoint: str,
|
|
46
|
+
secret: str,
|
|
47
|
+
mcp_urls: dict[str, str],
|
|
48
|
+
data: vf.TaskData,
|
|
49
|
+
tool_interception_url: str | None = None,
|
|
50
|
+
) -> ProgramResult:
|
|
51
|
+
return await runtime.run_program(["sh", "-c", PROBE], {})
|
|
52
|
+
|
|
53
|
+
|
|
54
|
+
class AgentUserTask(vf.Task):
|
|
55
|
+
async def setup(self, trace: vf.Trace, runtime: Runtime) -> None:
|
|
56
|
+
result = await runtime.run(["sh", "-c", SETUP], {})
|
|
57
|
+
if result.exit_code != 0:
|
|
58
|
+
raise RuntimeError(f"root setup failed: {result.stderr}")
|
|
59
|
+
|
|
60
|
+
@vf.reward(weight=1.0)
|
|
61
|
+
async def root_actions_denied(self, trace: vf.Trace, runtime: Runtime) -> float:
|
|
62
|
+
report = json.loads(await runtime.read(REPORT))
|
|
63
|
+
trace.info["agent_user_report"] = report
|
|
64
|
+
return float(
|
|
65
|
+
report["user"] == AGENT_USER
|
|
66
|
+
and report["secret_exists"] == "ok"
|
|
67
|
+
and report["write_workdir"] == "ok"
|
|
68
|
+
and all(report[action] == "denied" for action in ROOT_ACTIONS)
|
|
69
|
+
)
|
|
70
|
+
|
|
71
|
+
|
|
72
|
+
class AgentUserTaskset(vf.Taskset[AgentUserTask, vf.TasksetConfig]):
|
|
73
|
+
def load(self) -> list[AgentUserTask]:
|
|
74
|
+
return [
|
|
75
|
+
AgentUserTask(
|
|
76
|
+
vf.TaskData(
|
|
77
|
+
idx=0,
|
|
78
|
+
prompt="Scripted: try root-only actions as a non-root user.",
|
|
79
|
+
agent_user=AGENT_USER,
|
|
80
|
+
)
|
|
81
|
+
)
|
|
82
|
+
]
|
|
@@ -0,0 +1,95 @@
|
|
|
1
|
+
"""Every checked-in v1 eval config parses.
|
|
2
|
+
|
|
3
|
+
Mirrors prime-rl's config test: glob the configs and assert each validates into its config
|
|
4
|
+
type. The root `configs/*.toml` are the `uv run vf-eval @ <file>` v1 configs (EvalConfig).
|
|
5
|
+
"""
|
|
6
|
+
|
|
7
|
+
import tomllib
|
|
8
|
+
from pathlib import Path
|
|
9
|
+
|
|
10
|
+
import pytest
|
|
11
|
+
|
|
12
|
+
from verifiers.v1.configs.cli.eval import EvalConfig
|
|
13
|
+
from verifiers.v1.runtimes import E2BConfig
|
|
14
|
+
from verifiers.v1.runtimes.e2b import _egress_update
|
|
15
|
+
|
|
16
|
+
CONFIGS = sorted((Path(__file__).resolve().parents[2] / "configs").glob("*.toml"))
|
|
17
|
+
|
|
18
|
+
|
|
19
|
+
@pytest.mark.parametrize("path", CONFIGS, ids=lambda p: p.name)
|
|
20
|
+
def test_eval_config_parses(path: Path) -> None:
|
|
21
|
+
config = EvalConfig.model_validate(tomllib.load(path.open("rb")))
|
|
22
|
+
assert config.env.taskset.id
|
|
23
|
+
|
|
24
|
+
|
|
25
|
+
@pytest.mark.parametrize(
|
|
26
|
+
"rule",
|
|
27
|
+
[
|
|
28
|
+
"",
|
|
29
|
+
"https://api.example.com",
|
|
30
|
+
"example.com:8443",
|
|
31
|
+
"example.com/path",
|
|
32
|
+
"user@example.com",
|
|
33
|
+
"10.0.0.0/99",
|
|
34
|
+
"example..com",
|
|
35
|
+
],
|
|
36
|
+
)
|
|
37
|
+
def test_e2b_config_rejects_unenforceable_egress_rules(rule: str) -> None:
|
|
38
|
+
with pytest.raises(ValueError, match="allow rules"):
|
|
39
|
+
E2BConfig(allow=[rule])
|
|
40
|
+
|
|
41
|
+
|
|
42
|
+
@pytest.mark.parametrize("rule", ["api.example.com", "*.example.com"])
|
|
43
|
+
def test_e2b_config_rejects_hostname_block_rules(rule: str) -> None:
|
|
44
|
+
with pytest.raises(ValueError, match="block rules must be IP addresses or CIDR"):
|
|
45
|
+
E2BConfig(block=[rule])
|
|
46
|
+
|
|
47
|
+
|
|
48
|
+
@pytest.mark.parametrize(
|
|
49
|
+
"rule",
|
|
50
|
+
[
|
|
51
|
+
"api.example.com",
|
|
52
|
+
"*.example.com",
|
|
53
|
+
"10.0.0.1",
|
|
54
|
+
"10.0.0.0/24",
|
|
55
|
+
"2001:db8::1",
|
|
56
|
+
"2001:db8::/32",
|
|
57
|
+
],
|
|
58
|
+
)
|
|
59
|
+
def test_e2b_config_accepts_supported_egress_rules(rule: str) -> None:
|
|
60
|
+
assert E2BConfig(allow=[rule]).allow == [rule]
|
|
61
|
+
|
|
62
|
+
|
|
63
|
+
@pytest.mark.parametrize(
|
|
64
|
+
"rule", ["10.0.0.1", "10.0.0.0/24", "2001:db8::1", "2001:db8::/32"]
|
|
65
|
+
)
|
|
66
|
+
def test_e2b_config_accepts_supported_block_rules(rule: str) -> None:
|
|
67
|
+
assert E2BConfig(block=[rule]).block == [rule]
|
|
68
|
+
|
|
69
|
+
|
|
70
|
+
def test_e2b_egress_update_states_the_complete_policy() -> None:
|
|
71
|
+
# The security-critical policy table: whether a "restricted" sandbox actually
|
|
72
|
+
# gets a deny-everything floor, with framework routes kept reachable. No e2e
|
|
73
|
+
# placement runs network-restricted, so this table is pinned here.
|
|
74
|
+
routes = ["https://tunnel.example.com/intercept"]
|
|
75
|
+
|
|
76
|
+
unrestricted = _egress_update(E2BConfig(), None)
|
|
77
|
+
assert unrestricted == {"allow_internet_access": True}
|
|
78
|
+
|
|
79
|
+
blocklist = _egress_update(E2BConfig(block=["203.0.113.0/24"]), routes)
|
|
80
|
+
assert blocklist == {"deny_out": ["203.0.113.0/24"]}
|
|
81
|
+
|
|
82
|
+
allowlist = _egress_update(E2BConfig(allow=["api.example.com"]), routes)
|
|
83
|
+
assert allowlist == {
|
|
84
|
+
"allow_out": ["tunnel.example.com", "api.example.com"],
|
|
85
|
+
"deny_out": ["0.0.0.0/0"],
|
|
86
|
+
}
|
|
87
|
+
|
|
88
|
+
framework_only = _egress_update(E2BConfig(allow=[]), routes)
|
|
89
|
+
assert framework_only == {
|
|
90
|
+
"allow_out": ["tunnel.example.com"],
|
|
91
|
+
"deny_out": ["0.0.0.0/0"],
|
|
92
|
+
}
|
|
93
|
+
|
|
94
|
+
no_routes = _egress_update(E2BConfig(allow=[]), [])
|
|
95
|
+
assert no_routes == {"allow_internet_access": False}
|
|
@@ -2,8 +2,8 @@
|
|
|
2
2
|
|
|
3
3
|
Placement coverage is pairwise (see tests/v1/conftest.py): each list below names the
|
|
4
4
|
combinations a test runs — every axis value at least once plus the cross-boundary pairs
|
|
5
|
-
with distinct networking — instead of fanning the full cross product. prime/modal
|
|
6
|
-
are local-only (their marks are excluded in CI)."""
|
|
5
|
+
with distinct networking — instead of fanning the full cross product. prime/modal/e2b
|
|
6
|
+
rows are local-only (their marks are excluded in CI)."""
|
|
7
7
|
|
|
8
8
|
import shutil
|
|
9
9
|
import subprocess
|
|
@@ -142,6 +142,7 @@ CHAT_PLACEMENTS = [
|
|
|
142
142
|
),
|
|
143
143
|
pair("bash", "prime", "bash-harness-in-prime"),
|
|
144
144
|
pair("bash", "modal", "bash-harness-in-modal"),
|
|
145
|
+
pair("bash", "e2b", "bash-harness-in-e2b"),
|
|
145
146
|
]
|
|
146
147
|
|
|
147
148
|
# harness x harness runtime for the shell task: every coding agent once (null is a chat
|
|
@@ -176,6 +177,7 @@ AGENTIC_PLACEMENTS = [
|
|
|
176
177
|
marks=[mark.bash, mark.modal],
|
|
177
178
|
id="bash-harness-in-modal-framework-only",
|
|
178
179
|
),
|
|
180
|
+
pair("bash", "e2b", "bash-harness-in-e2b"),
|
|
179
181
|
]
|
|
180
182
|
|
|
181
183
|
# The scripted user runs in the eval process itself (no placement axis); the harness
|
|
@@ -185,6 +187,13 @@ USER_RUNTIMES = [
|
|
|
185
187
|
pytest.param("docker", marks=[mark.docker], id="harness-in-docker"),
|
|
186
188
|
pytest.param("prime", marks=[mark.prime], id="harness-in-prime"),
|
|
187
189
|
pytest.param("modal", marks=[mark.modal], id="harness-in-modal"),
|
|
190
|
+
pytest.param("e2b", marks=[mark.e2b], id="harness-in-e2b"),
|
|
191
|
+
]
|
|
192
|
+
|
|
193
|
+
# Runtimes that can switch users (`Runtime.with_user`).
|
|
194
|
+
AGENT_USER_RUNTIMES = [
|
|
195
|
+
pytest.param("docker", marks=[mark.docker], id="harness-in-docker"),
|
|
196
|
+
pytest.param("prime", marks=[mark.prime], id="harness-in-prime"),
|
|
188
197
|
]
|
|
189
198
|
|
|
190
199
|
# ACP-backed harnesses: each must preserve an exchange across interaction segments and
|
|
@@ -218,6 +227,13 @@ ACP_RESUME_PLACEMENTS = [
|
|
|
218
227
|
),
|
|
219
228
|
pair("openclaw", "docker", "openclaw-acp-in-docker"),
|
|
220
229
|
pair("rlm", "prime", "rlm-acp-in-prime-vm"),
|
|
230
|
+
# Give RLM 2 GB for ACP session startup.
|
|
231
|
+
pytest.param(
|
|
232
|
+
"rlm",
|
|
233
|
+
{"type": "e2b", "memory": 2.0},
|
|
234
|
+
marks=[mark.rlm, mark.e2b],
|
|
235
|
+
id="rlm-acp-in-e2b",
|
|
236
|
+
),
|
|
221
237
|
pytest.param(
|
|
222
238
|
"prime-agent",
|
|
223
239
|
"prime",
|
|
@@ -247,6 +263,8 @@ TOOL_PLACEMENTS = [
|
|
|
247
263
|
pair("prime", "colocated", "harness-in-prime-with-tool-colocated"),
|
|
248
264
|
pair("modal", "colocated", "harness-in-modal-with-tool-colocated"),
|
|
249
265
|
pair("subprocess", "modal", "harness-in-subprocess-with-tool-in-modal"),
|
|
266
|
+
pair("e2b", "colocated", "harness-in-e2b-with-tool-colocated"),
|
|
267
|
+
pair("subprocess", "e2b", "harness-in-subprocess-with-tool-in-e2b"),
|
|
250
268
|
]
|
|
251
269
|
|
|
252
270
|
# The state channel rides the same reachability as TOOL_PLACEMENTS; cover each axis
|
|
@@ -256,6 +274,7 @@ TOOL_STATE_PLACEMENTS = [
|
|
|
256
274
|
pair("docker", "subprocess", "harness-in-docker-with-tool-in-subprocess"),
|
|
257
275
|
pair("subprocess", "docker", "harness-in-subprocess-with-tool-in-docker"),
|
|
258
276
|
pair("modal", "colocated", "harness-in-modal-with-tool-colocated"),
|
|
277
|
+
pair("e2b", "colocated", "harness-in-e2b-with-tool-colocated"),
|
|
259
278
|
]
|
|
260
279
|
|
|
261
280
|
# Shared servers always run in their own runtime (colocation is per-rollout, shared is
|
|
@@ -265,6 +284,7 @@ SHARED_TOOL_PLACEMENTS = [
|
|
|
265
284
|
pair("docker", "docker", "harness-in-docker-with-tool-in-docker"),
|
|
266
285
|
pair("subprocess", "docker", "harness-in-subprocess-with-tool-in-docker"),
|
|
267
286
|
pair("modal", "modal", "harness-in-modal-with-tool-in-modal"),
|
|
287
|
+
pair("e2b", "e2b", "harness-in-e2b-with-tool-in-e2b"),
|
|
268
288
|
]
|
|
269
289
|
|
|
270
290
|
|
|
@@ -318,6 +338,29 @@ async def test_browser_use(run_v1, tmp_path):
|
|
|
318
338
|
assert trace.reward == 1.0
|
|
319
339
|
|
|
320
340
|
|
|
341
|
+
@pytest.mark.e2e
|
|
342
|
+
@pytest.mark.parametrize("harness_runtime", AGENT_USER_RUNTIMES, indirect=True)
|
|
343
|
+
async def test_agent_user_cannot_act_as_root(run_v1, harness_runtime, tmp_path):
|
|
344
|
+
"""A scripted, model-free agent runs as the task's non-root `agent_user`: it can see
|
|
345
|
+
the secret the root-run setup planted and write its own workspace, but every
|
|
346
|
+
root-only action (read the secret, write /etc, chmod the secret, signal PID 1) is
|
|
347
|
+
denied."""
|
|
348
|
+
(trace,) = await run_v1(
|
|
349
|
+
"agent-user-v1",
|
|
350
|
+
harness="agent-user-v1",
|
|
351
|
+
runtime={"type": harness_runtime},
|
|
352
|
+
output_dir=tmp_path,
|
|
353
|
+
)
|
|
354
|
+
report = trace.info["agent_user_report"]
|
|
355
|
+
assert report["user"] == "vf-agent"
|
|
356
|
+
assert report["secret_exists"] == "ok" and report["write_workdir"] == "ok"
|
|
357
|
+
root_actions = ("read_secret", "write_etc", "chmod_secret", "signal_init")
|
|
358
|
+
assert {action: report[action] for action in root_actions} == dict.fromkeys(
|
|
359
|
+
root_actions, "denied"
|
|
360
|
+
)
|
|
361
|
+
assert trace.reward == 1.0
|
|
362
|
+
|
|
363
|
+
|
|
321
364
|
@pytest.mark.e2e
|
|
322
365
|
@pytest.mark.parametrize("harness_runtime", USER_RUNTIMES, indirect=True)
|
|
323
366
|
async def test_user(run_v1, harness_runtime, tmp_path):
|
|
@@ -389,7 +432,9 @@ async def test_acp_resume_with_tool(run_v1, harness, harness_runtime, tmp_path):
|
|
|
389
432
|
(trace,) = await run_v1(
|
|
390
433
|
"echo-acp-resume-v1",
|
|
391
434
|
harness=harness,
|
|
392
|
-
runtime=
|
|
435
|
+
runtime=harness_runtime
|
|
436
|
+
if isinstance(harness_runtime, dict)
|
|
437
|
+
else {"type": harness_runtime},
|
|
393
438
|
output_dir=tmp_path,
|
|
394
439
|
max_turns=8,
|
|
395
440
|
max_tokens=8192,
|
|
@@ -14,6 +14,7 @@ from verifiers.v1.configs.runtime import NetworkPolicyConfig
|
|
|
14
14
|
from verifiers.v1.errors import (
|
|
15
15
|
HarnessError,
|
|
16
16
|
RolloutError,
|
|
17
|
+
SandboxError,
|
|
17
18
|
TaskError,
|
|
18
19
|
ToolsetError,
|
|
19
20
|
boundary,
|
|
@@ -52,6 +53,39 @@ class RolloutTimeouts:
|
|
|
52
53
|
"""Timeout (in seconds) for the task + harness metrics + scoring hooks."""
|
|
53
54
|
|
|
54
55
|
|
|
56
|
+
_HOME_PROBE = (
|
|
57
|
+
"home=$(awk -F: -v u=\"$1\" '$1 == u { print $6 }' /etc/passwd); "
|
|
58
|
+
'echo "$home"; test -n "$home" && test -d "$home"'
|
|
59
|
+
)
|
|
60
|
+
|
|
61
|
+
|
|
62
|
+
async def agent_runtime(runtime: Runtime, user: str | None) -> Runtime:
|
|
63
|
+
"""The runtime the harness runs on: `runtime` itself, or a view running as `user`
|
|
64
|
+
with `HOME` set to its home. `user` must exist with a home directory it can write
|
|
65
|
+
(task authors ship it in the image), and the view must really run as it, so a
|
|
66
|
+
runtime ignoring the user never runs the agent as root."""
|
|
67
|
+
if user is None:
|
|
68
|
+
return runtime
|
|
69
|
+
lookup = await runtime.run(["sh", "-c", _HOME_PROBE, "sh", user], {})
|
|
70
|
+
home = lookup.stdout.strip()
|
|
71
|
+
if not home:
|
|
72
|
+
raise TaskError(f"agent_user {user!r} does not exist in the image")
|
|
73
|
+
if lookup.exit_code != 0:
|
|
74
|
+
raise TaskError(f"agent_user {user!r} has no home directory ({home})")
|
|
75
|
+
view = runtime.with_user(user)
|
|
76
|
+
view = view.with_env({**view.env, "HOME": home})
|
|
77
|
+
probe = await view.run(["sh", "-c", 'id -un; test -w "$HOME" && echo writable'], {})
|
|
78
|
+
effective, *rest = probe.stdout.split() or [""]
|
|
79
|
+
if effective != user:
|
|
80
|
+
raise SandboxError(
|
|
81
|
+
f"{type(runtime).__name__} ran as {effective!r} instead of agent_user "
|
|
82
|
+
f"{user!r}; refusing to run the agent"
|
|
83
|
+
)
|
|
84
|
+
if rest != ["writable"]:
|
|
85
|
+
raise TaskError(f"agent_user {user!r} cannot write its home directory ({home})")
|
|
86
|
+
return view
|
|
87
|
+
|
|
88
|
+
|
|
55
89
|
class Rollout:
|
|
56
90
|
"""Manages one rollout's lifecycle (open, step, close)."""
|
|
57
91
|
|
|
@@ -341,7 +375,7 @@ class Rollout:
|
|
|
341
375
|
self._harness_session = await self.harness.session(
|
|
342
376
|
self.ctx,
|
|
343
377
|
self.trace,
|
|
344
|
-
runtime,
|
|
378
|
+
await agent_runtime(runtime, harness_data.agent_user),
|
|
345
379
|
self._endpoint,
|
|
346
380
|
self._secret,
|
|
347
381
|
self._urls,
|
|
@@ -25,6 +25,7 @@ from verifiers.v1.runtimes.docker import (
|
|
|
25
25
|
PodmanRuntime,
|
|
26
26
|
PodmanRuntimeInfo,
|
|
27
27
|
)
|
|
28
|
+
from verifiers.v1.runtimes.e2b import E2BConfig, E2BRuntime, E2BRuntimeInfo
|
|
28
29
|
from verifiers.v1.runtimes.modal import ModalConfig, ModalRuntime, ModalRuntimeInfo
|
|
29
30
|
from verifiers.v1.runtimes.prime import (
|
|
30
31
|
PrimeConfig,
|
|
@@ -44,7 +45,8 @@ RuntimeConfig = Annotated[
|
|
|
44
45
|
| PodmanConfig
|
|
45
46
|
| ApptainerConfig
|
|
46
47
|
| PrimeConfig
|
|
47
|
-
| ModalConfig
|
|
48
|
+
| ModalConfig
|
|
49
|
+
| E2BConfig,
|
|
48
50
|
Field(discriminator="type"),
|
|
49
51
|
]
|
|
50
52
|
|
|
@@ -54,7 +56,8 @@ RuntimeInfo = Annotated[
|
|
|
54
56
|
| PodmanRuntimeInfo
|
|
55
57
|
| ApptainerRuntimeInfo
|
|
56
58
|
| PrimeRuntimeInfo
|
|
57
|
-
| ModalRuntimeInfo
|
|
59
|
+
| ModalRuntimeInfo
|
|
60
|
+
| E2BRuntimeInfo,
|
|
58
61
|
Field(discriminator="type"),
|
|
59
62
|
]
|
|
60
63
|
|
|
@@ -67,6 +70,7 @@ def _runtime_cls(config: RuntimeConfig) -> type[Runtime]:
|
|
|
67
70
|
"apptainer": ApptainerRuntime,
|
|
68
71
|
"prime": PrimeRuntime,
|
|
69
72
|
"modal": ModalRuntime,
|
|
73
|
+
"e2b": E2BRuntime,
|
|
70
74
|
}[config.type]
|
|
71
75
|
|
|
72
76
|
|
|
@@ -109,6 +113,9 @@ __all__ = [
|
|
|
109
113
|
"DockerConfig",
|
|
110
114
|
"DockerRuntime",
|
|
111
115
|
"DockerRuntimeInfo",
|
|
116
|
+
"E2BConfig",
|
|
117
|
+
"E2BRuntime",
|
|
118
|
+
"E2BRuntimeInfo",
|
|
112
119
|
"ModalConfig",
|
|
113
120
|
"ModalRuntime",
|
|
114
121
|
"ModalRuntimeInfo",
|
|
@@ -136,13 +136,16 @@ class BaseRuntimeInfo(BaseConfig):
|
|
|
136
136
|
|
|
137
137
|
|
|
138
138
|
class Runtime(ABC):
|
|
139
|
-
__slots__ = ("env",)
|
|
139
|
+
__slots__ = ("env", "user")
|
|
140
140
|
|
|
141
141
|
is_local: bool = True
|
|
142
142
|
"""Whether this runtime exchanges host-local URLs without a public tunnel. True for
|
|
143
143
|
subprocess and the local container runtimes; remote runtimes override to False and
|
|
144
144
|
use a host `Tunnel` inward plus `expose` outward."""
|
|
145
145
|
|
|
146
|
+
supports_user: ClassVar[bool] = False
|
|
147
|
+
"""Whether `with_user` can run this runtime's commands as another existing user."""
|
|
148
|
+
|
|
146
149
|
scripts_dir: ClassVar[str] = "/tmp/vf-scripts"
|
|
147
150
|
"""Digest-keyed PEP 723 scripts inside the runtime. Sandboxes own their `/tmp`;
|
|
148
151
|
the host subprocess runtime overrides this with its per-user cache."""
|
|
@@ -159,6 +162,8 @@ class Runtime(ABC):
|
|
|
159
162
|
# Per-run task values live on the runtime rather than its serializable config/info.
|
|
160
163
|
# Explicit process values (model credentials, proxy settings, etc.) override these.
|
|
161
164
|
self.env: dict[str, str] = {}
|
|
165
|
+
self.user: str | None = None
|
|
166
|
+
"""Existing user this view runs commands as; None uses the runtime's default."""
|
|
162
167
|
self._uv_interpreters: dict[str, str] = {}
|
|
163
168
|
self._uv_script_locks: dict[str, asyncio.Lock] = {}
|
|
164
169
|
self._mcp_sources: set[str] = set()
|
|
@@ -215,6 +220,19 @@ class Runtime(ABC):
|
|
|
215
220
|
runtime.env = dict(env)
|
|
216
221
|
return runtime
|
|
217
222
|
|
|
223
|
+
def with_user(self, user: str | None) -> "Runtime":
|
|
224
|
+
"""Share this physical runtime through a view that runs commands as `user`, an
|
|
225
|
+
existing user in the box (None keeps the default). Only the view runs as `user`:
|
|
226
|
+
setup, collection, and grading keep using the original runtime."""
|
|
227
|
+
if user is not None and not self.supports_user:
|
|
228
|
+
raise ValueError(
|
|
229
|
+
f"{type(self).__name__} cannot run commands as another user"
|
|
230
|
+
)
|
|
231
|
+
runtime = copy.copy(self)
|
|
232
|
+
runtime.__dict__ = self.__dict__
|
|
233
|
+
runtime.user = user
|
|
234
|
+
return runtime
|
|
235
|
+
|
|
218
236
|
async def alive(self) -> bool:
|
|
219
237
|
"""Whether the box still executes anything. Not every runtime raises when
|
|
220
238
|
the box is gone — some surface it as `exec`'s own non-zero result,
|
|
@@ -70,6 +70,7 @@ control.sendmsg([b"listener"], [(socket.SOL_SOCKET, socket.SCM_RIGHTS, array.arr
|
|
|
70
70
|
|
|
71
71
|
|
|
72
72
|
class DockerRuntime(ContainerRuntime):
|
|
73
|
+
supports_user: ClassVar[bool] = True
|
|
73
74
|
engine: ClassVar[str] = "docker"
|
|
74
75
|
"""The CLI binary for the shared OCI container operations."""
|
|
75
76
|
info_cls: ClassVar[type[BaseRuntimeInfo]] = DockerRuntimeInfo
|
|
@@ -529,6 +530,7 @@ class DockerRuntime(ContainerRuntime):
|
|
|
529
530
|
"exec",
|
|
530
531
|
*(("-i",) if stdin else ()),
|
|
531
532
|
*(arg for key, value in env.items() for arg in ("--env", f"{key}={value}")),
|
|
533
|
+
*(("--user", self.user) if self.user is not None else ()),
|
|
532
534
|
"--workdir",
|
|
533
535
|
self.config.workdir,
|
|
534
536
|
self._container,
|