verifiers 0.3.2.dev105__tar.gz → 0.3.2.dev107__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {verifiers-0.3.2.dev105 → verifiers-0.3.2.dev107}/PKG-INFO +1 -1
- {verifiers-0.3.2.dev105 → verifiers-0.3.2.dev107}/pyproject.toml +2 -0
- {verifiers-0.3.2.dev105 → verifiers-0.3.2.dev107}/tests/v1/conftest.py +1 -1
- {verifiers-0.3.2.dev105 → verifiers-0.3.2.dev107}/tests/v1/test_e2e.py +28 -0
- {verifiers-0.3.2.dev105 → verifiers-0.3.2.dev107}/verifiers/v1/__init__.py +4 -0
- {verifiers-0.3.2.dev105 → verifiers-0.3.2.dev107}/verifiers/v1/harnesses/browser_use/harness.py +14 -1
- {verifiers-0.3.2.dev105 → verifiers-0.3.2.dev107}/verifiers/v1/mcp/launch.py +14 -15
- {verifiers-0.3.2.dev105 → verifiers-0.3.2.dev107}/verifiers/v1/rollout.py +6 -2
- {verifiers-0.3.2.dev105 → verifiers-0.3.2.dev107}/verifiers/v1/runtimes/__init__.py +39 -10
- verifiers-0.3.2.dev107/verifiers/v1/runtimes/apptainer.py +214 -0
- {verifiers-0.3.2.dev105 → verifiers-0.3.2.dev107}/verifiers/v1/runtimes/base.py +14 -13
- verifiers-0.3.2.dev107/verifiers/v1/runtimes/container.py +249 -0
- verifiers-0.3.2.dev107/verifiers/v1/runtimes/docker/__init__.py +490 -0
- {verifiers-0.3.2.dev105 → verifiers-0.3.2.dev107}/verifiers/v1/runtimes/docker/egress.py +233 -39
- {verifiers-0.3.2.dev105 → verifiers-0.3.2.dev107}/verifiers/v1/task.py +1 -1
- verifiers-0.3.2.dev105/verifiers/v1/runtimes/docker/__init__.py +0 -574
- {verifiers-0.3.2.dev105 → verifiers-0.3.2.dev107}/.gitignore +0 -0
- {verifiers-0.3.2.dev105 → verifiers-0.3.2.dev107}/LICENSE +0 -0
- {verifiers-0.3.2.dev105 → verifiers-0.3.2.dev107}/README.md +0 -0
- {verifiers-0.3.2.dev105 → verifiers-0.3.2.dev107}/tests/v1/fixtures/counter_tool_v1.py +0 -0
- {verifiers-0.3.2.dev105 → verifiers-0.3.2.dev107}/tests/v1/fixtures/duet_v1.py +0 -0
- {verifiers-0.3.2.dev105 → verifiers-0.3.2.dev107}/tests/v1/fixtures/echo_acp_resume_v1.py +0 -0
- {verifiers-0.3.2.dev105 → verifiers-0.3.2.dev107}/tests/v1/fixtures/echo_agentic_v1.py +0 -0
- {verifiers-0.3.2.dev105 → verifiers-0.3.2.dev107}/tests/v1/fixtures/echo_tool_v1.py +0 -0
- {verifiers-0.3.2.dev105 → verifiers-0.3.2.dev107}/tests/v1/fixtures/echo_user_sim_v1.py +0 -0
- {verifiers-0.3.2.dev105 → verifiers-0.3.2.dev107}/tests/v1/fixtures/echo_v1.py +0 -0
- {verifiers-0.3.2.dev105 → verifiers-0.3.2.dev107}/tests/v1/fixtures/pyproject.toml +0 -0
- {verifiers-0.3.2.dev105 → verifiers-0.3.2.dev107}/tests/v1/fixtures/tool_response_image_v1.py +0 -0
- {verifiers-0.3.2.dev105 → verifiers-0.3.2.dev107}/tests/v1/test_color_codeword_tasks.py +0 -0
- {verifiers-0.3.2.dev105 → verifiers-0.3.2.dev107}/tests/v1/test_configs.py +0 -0
- {verifiers-0.3.2.dev105 → verifiers-0.3.2.dev107}/tests/v1/test_envs.py +0 -0
- {verifiers-0.3.2.dev105 → verifiers-0.3.2.dev107}/tests/v1/test_graph.py +0 -0
- {verifiers-0.3.2.dev105 → verifiers-0.3.2.dev107}/tests/v1/test_judges.py +0 -0
- {verifiers-0.3.2.dev105 → verifiers-0.3.2.dev107}/tests/v1/test_platform_run.py +0 -0
- {verifiers-0.3.2.dev105 → verifiers-0.3.2.dev107}/tests/v1/test_scoring.py +0 -0
- {verifiers-0.3.2.dev105 → verifiers-0.3.2.dev107}/tests/v1/test_serve_delta.py +0 -0
- {verifiers-0.3.2.dev105 → verifiers-0.3.2.dev107}/tests/v1/test_taskset.py +0 -0
- {verifiers-0.3.2.dev105 → verifiers-0.3.2.dev107}/tests/v1/test_trace.py +0 -0
- {verifiers-0.3.2.dev105 → verifiers-0.3.2.dev107}/verifiers/__init__.py +0 -0
- {verifiers-0.3.2.dev105 → verifiers-0.3.2.dev107}/verifiers/cli/commands/init.py +0 -0
- {verifiers-0.3.2.dev105 → verifiers-0.3.2.dev107}/verifiers/v1/acp/__init__.py +0 -0
- {verifiers-0.3.2.dev105 → verifiers-0.3.2.dev107}/verifiers/v1/acp/runner.py +0 -0
- {verifiers-0.3.2.dev105 → verifiers-0.3.2.dev107}/verifiers/v1/agent.py +0 -0
- {verifiers-0.3.2.dev105 → verifiers-0.3.2.dev107}/verifiers/v1/cli/__init__.py +0 -0
- {verifiers-0.3.2.dev105 → verifiers-0.3.2.dev107}/verifiers/v1/cli/dashboard/__init__.py +0 -0
- {verifiers-0.3.2.dev105 → verifiers-0.3.2.dev107}/verifiers/v1/cli/dashboard/base.py +0 -0
- {verifiers-0.3.2.dev105 → verifiers-0.3.2.dev107}/verifiers/v1/cli/dashboard/eval.py +0 -0
- {verifiers-0.3.2.dev105 → verifiers-0.3.2.dev107}/verifiers/v1/cli/dashboard/replay.py +0 -0
- {verifiers-0.3.2.dev105 → verifiers-0.3.2.dev107}/verifiers/v1/cli/dashboard/validate.py +0 -0
- {verifiers-0.3.2.dev105 → verifiers-0.3.2.dev107}/verifiers/v1/cli/debug.py +0 -0
- {verifiers-0.3.2.dev105 → verifiers-0.3.2.dev107}/verifiers/v1/cli/eval/__init__.py +0 -0
- {verifiers-0.3.2.dev105 → verifiers-0.3.2.dev107}/verifiers/v1/cli/eval/hint.py +0 -0
- {verifiers-0.3.2.dev105 → verifiers-0.3.2.dev107}/verifiers/v1/cli/eval/main.py +0 -0
- {verifiers-0.3.2.dev105 → verifiers-0.3.2.dev107}/verifiers/v1/cli/eval/resume.py +0 -0
- {verifiers-0.3.2.dev105 → verifiers-0.3.2.dev107}/verifiers/v1/cli/eval/runner.py +0 -0
- {verifiers-0.3.2.dev105 → verifiers-0.3.2.dev107}/verifiers/v1/cli/gepa.py +0 -0
- {verifiers-0.3.2.dev105 → verifiers-0.3.2.dev107}/verifiers/v1/cli/init.py +0 -0
- {verifiers-0.3.2.dev105 → verifiers-0.3.2.dev107}/verifiers/v1/cli/output.py +0 -0
- {verifiers-0.3.2.dev105 → verifiers-0.3.2.dev107}/verifiers/v1/cli/replay.py +0 -0
- {verifiers-0.3.2.dev105 → verifiers-0.3.2.dev107}/verifiers/v1/cli/resolve.py +0 -0
- {verifiers-0.3.2.dev105 → verifiers-0.3.2.dev107}/verifiers/v1/cli/resume.py +0 -0
- {verifiers-0.3.2.dev105 → verifiers-0.3.2.dev107}/verifiers/v1/cli/validate.py +0 -0
- {verifiers-0.3.2.dev105 → verifiers-0.3.2.dev107}/verifiers/v1/clients/__init__.py +0 -0
- {verifiers-0.3.2.dev105 → verifiers-0.3.2.dev107}/verifiers/v1/clients/base.py +0 -0
- {verifiers-0.3.2.dev105 → verifiers-0.3.2.dev107}/verifiers/v1/clients/client.py +0 -0
- {verifiers-0.3.2.dev105 → verifiers-0.3.2.dev107}/verifiers/v1/clients/eval.py +0 -0
- {verifiers-0.3.2.dev105 → verifiers-0.3.2.dev107}/verifiers/v1/clients/train.py +0 -0
- {verifiers-0.3.2.dev105 → verifiers-0.3.2.dev107}/verifiers/v1/configs/__init__.py +0 -0
- {verifiers-0.3.2.dev105 → verifiers-0.3.2.dev107}/verifiers/v1/configs/agent.py +0 -0
- {verifiers-0.3.2.dev105 → verifiers-0.3.2.dev107}/verifiers/v1/configs/cli/__init__.py +0 -0
- {verifiers-0.3.2.dev105 → verifiers-0.3.2.dev107}/verifiers/v1/configs/cli/debug.py +0 -0
- {verifiers-0.3.2.dev105 → verifiers-0.3.2.dev107}/verifiers/v1/configs/cli/env.py +0 -0
- {verifiers-0.3.2.dev105 → verifiers-0.3.2.dev107}/verifiers/v1/configs/cli/eval.py +0 -0
- {verifiers-0.3.2.dev105 → verifiers-0.3.2.dev107}/verifiers/v1/configs/cli/init.py +0 -0
- {verifiers-0.3.2.dev105 → verifiers-0.3.2.dev107}/verifiers/v1/configs/cli/replay.py +0 -0
- {verifiers-0.3.2.dev105 → verifiers-0.3.2.dev107}/verifiers/v1/configs/cli/validate.py +0 -0
- {verifiers-0.3.2.dev105 → verifiers-0.3.2.dev107}/verifiers/v1/configs/client.py +0 -0
- {verifiers-0.3.2.dev105 → verifiers-0.3.2.dev107}/verifiers/v1/configs/env.py +0 -0
- {verifiers-0.3.2.dev105 → verifiers-0.3.2.dev107}/verifiers/v1/configs/harness.py +0 -0
- {verifiers-0.3.2.dev105 → verifiers-0.3.2.dev107}/verifiers/v1/configs/judge.py +0 -0
- {verifiers-0.3.2.dev105 → verifiers-0.3.2.dev107}/verifiers/v1/configs/retries.py +0 -0
- {verifiers-0.3.2.dev105 → verifiers-0.3.2.dev107}/verifiers/v1/configs/runtime.py +0 -0
- {verifiers-0.3.2.dev105 → verifiers-0.3.2.dev107}/verifiers/v1/configs/serve.py +0 -0
- {verifiers-0.3.2.dev105 → verifiers-0.3.2.dev107}/verifiers/v1/configs/task.py +0 -0
- {verifiers-0.3.2.dev105 → verifiers-0.3.2.dev107}/verifiers/v1/configs/taskset.py +0 -0
- {verifiers-0.3.2.dev105 → verifiers-0.3.2.dev107}/verifiers/v1/dialects/__init__.py +0 -0
- {verifiers-0.3.2.dev105 → verifiers-0.3.2.dev107}/verifiers/v1/dialects/anthropic.py +0 -0
- {verifiers-0.3.2.dev105 → verifiers-0.3.2.dev107}/verifiers/v1/dialects/base.py +0 -0
- {verifiers-0.3.2.dev105 → verifiers-0.3.2.dev107}/verifiers/v1/dialects/chat.py +0 -0
- {verifiers-0.3.2.dev105 → verifiers-0.3.2.dev107}/verifiers/v1/dialects/responses.py +0 -0
- {verifiers-0.3.2.dev105 → verifiers-0.3.2.dev107}/verifiers/v1/env.py +0 -0
- {verifiers-0.3.2.dev105 → verifiers-0.3.2.dev107}/verifiers/v1/envs/__init__.py +0 -0
- {verifiers-0.3.2.dev105 → verifiers-0.3.2.dev107}/verifiers/v1/envs/agentic_judge/__init__.py +0 -0
- {verifiers-0.3.2.dev105 → verifiers-0.3.2.dev107}/verifiers/v1/envs/agentic_judge/env.py +0 -0
- {verifiers-0.3.2.dev105 → verifiers-0.3.2.dev107}/verifiers/v1/envs/best_of_n/__init__.py +0 -0
- {verifiers-0.3.2.dev105 → verifiers-0.3.2.dev107}/verifiers/v1/envs/best_of_n/env.py +0 -0
- {verifiers-0.3.2.dev105 → verifiers-0.3.2.dev107}/verifiers/v1/envs/isolated_verifier/__init__.py +0 -0
- {verifiers-0.3.2.dev105 → verifiers-0.3.2.dev107}/verifiers/v1/envs/isolated_verifier/env.py +0 -0
- {verifiers-0.3.2.dev105 → verifiers-0.3.2.dev107}/verifiers/v1/envs/shared_agentic_judge/__init__.py +0 -0
- {verifiers-0.3.2.dev105 → verifiers-0.3.2.dev107}/verifiers/v1/envs/single_agent/__init__.py +0 -0
- {verifiers-0.3.2.dev105 → verifiers-0.3.2.dev107}/verifiers/v1/envs/single_agent/env.py +0 -0
- {verifiers-0.3.2.dev105 → verifiers-0.3.2.dev107}/verifiers/v1/envs/user_sim/__init__.py +0 -0
- {verifiers-0.3.2.dev105 → verifiers-0.3.2.dev107}/verifiers/v1/envs/user_sim/env.py +0 -0
- {verifiers-0.3.2.dev105 → verifiers-0.3.2.dev107}/verifiers/v1/episode.py +0 -0
- {verifiers-0.3.2.dev105 → verifiers-0.3.2.dev107}/verifiers/v1/errors.py +0 -0
- {verifiers-0.3.2.dev105 → verifiers-0.3.2.dev107}/verifiers/v1/gepa/__init__.py +0 -0
- {verifiers-0.3.2.dev105 → verifiers-0.3.2.dev107}/verifiers/v1/gepa/adapter.py +0 -0
- {verifiers-0.3.2.dev105 → verifiers-0.3.2.dev107}/verifiers/v1/gepa/config.py +0 -0
- {verifiers-0.3.2.dev105 → verifiers-0.3.2.dev107}/verifiers/v1/gepa/dataset.py +0 -0
- {verifiers-0.3.2.dev105 → verifiers-0.3.2.dev107}/verifiers/v1/gepa/reflection.py +0 -0
- {verifiers-0.3.2.dev105 → verifiers-0.3.2.dev107}/verifiers/v1/gepa/runner.py +0 -0
- {verifiers-0.3.2.dev105 → verifiers-0.3.2.dev107}/verifiers/v1/graph.py +0 -0
- {verifiers-0.3.2.dev105 → verifiers-0.3.2.dev107}/verifiers/v1/harness.py +0 -0
- {verifiers-0.3.2.dev105 → verifiers-0.3.2.dev107}/verifiers/v1/harnesses/__init__.py +0 -0
- {verifiers-0.3.2.dev105 → verifiers-0.3.2.dev107}/verifiers/v1/harnesses/bash/__init__.py +0 -0
- {verifiers-0.3.2.dev105 → verifiers-0.3.2.dev107}/verifiers/v1/harnesses/bash/harness.py +0 -0
- {verifiers-0.3.2.dev105 → verifiers-0.3.2.dev107}/verifiers/v1/harnesses/browser_use/__init__.py +0 -0
- {verifiers-0.3.2.dev105 → verifiers-0.3.2.dev107}/verifiers/v1/harnesses/browser_use/program.py +0 -0
- {verifiers-0.3.2.dev105 → verifiers-0.3.2.dev107}/verifiers/v1/harnesses/claude_code/__init__.py +0 -0
- {verifiers-0.3.2.dev105 → verifiers-0.3.2.dev107}/verifiers/v1/harnesses/claude_code/harness.py +0 -0
- {verifiers-0.3.2.dev105 → verifiers-0.3.2.dev107}/verifiers/v1/harnesses/codex/__init__.py +0 -0
- {verifiers-0.3.2.dev105 → verifiers-0.3.2.dev107}/verifiers/v1/harnesses/codex/harness.py +0 -0
- {verifiers-0.3.2.dev105 → verifiers-0.3.2.dev107}/verifiers/v1/harnesses/hermes_agent/__init__.py +0 -0
- {verifiers-0.3.2.dev105 → verifiers-0.3.2.dev107}/verifiers/v1/harnesses/hermes_agent/harness.py +0 -0
- {verifiers-0.3.2.dev105 → verifiers-0.3.2.dev107}/verifiers/v1/harnesses/hermes_agent/program.py +0 -0
- {verifiers-0.3.2.dev105 → verifiers-0.3.2.dev107}/verifiers/v1/harnesses/kimi_code/__init__.py +0 -0
- {verifiers-0.3.2.dev105 → verifiers-0.3.2.dev107}/verifiers/v1/harnesses/kimi_code/harness.py +0 -0
- {verifiers-0.3.2.dev105 → verifiers-0.3.2.dev107}/verifiers/v1/harnesses/mini_swe_agent/__init__.py +0 -0
- {verifiers-0.3.2.dev105 → verifiers-0.3.2.dev107}/verifiers/v1/harnesses/mini_swe_agent/harness.py +0 -0
- {verifiers-0.3.2.dev105 → verifiers-0.3.2.dev107}/verifiers/v1/harnesses/mini_swe_agent/program.py +0 -0
- {verifiers-0.3.2.dev105 → verifiers-0.3.2.dev107}/verifiers/v1/harnesses/node.py +0 -0
- {verifiers-0.3.2.dev105 → verifiers-0.3.2.dev107}/verifiers/v1/harnesses/null/__init__.py +0 -0
- {verifiers-0.3.2.dev105 → verifiers-0.3.2.dev107}/verifiers/v1/harnesses/null/harness.py +0 -0
- {verifiers-0.3.2.dev105 → verifiers-0.3.2.dev107}/verifiers/v1/harnesses/openclaw/__init__.py +0 -0
- {verifiers-0.3.2.dev105 → verifiers-0.3.2.dev107}/verifiers/v1/harnesses/openclaw/harness.py +0 -0
- {verifiers-0.3.2.dev105 → verifiers-0.3.2.dev107}/verifiers/v1/harnesses/pi/__init__.py +0 -0
- {verifiers-0.3.2.dev105 → verifiers-0.3.2.dev107}/verifiers/v1/harnesses/pi/harness.py +0 -0
- {verifiers-0.3.2.dev105 → verifiers-0.3.2.dev107}/verifiers/v1/harnesses/prime_agent/__init__.py +0 -0
- {verifiers-0.3.2.dev105 → verifiers-0.3.2.dev107}/verifiers/v1/harnesses/prime_agent/harness.py +0 -0
- {verifiers-0.3.2.dev105 → verifiers-0.3.2.dev107}/verifiers/v1/harnesses/rlm/__init__.py +0 -0
- {verifiers-0.3.2.dev105 → verifiers-0.3.2.dev107}/verifiers/v1/harnesses/rlm/harness.py +0 -0
- {verifiers-0.3.2.dev105 → verifiers-0.3.2.dev107}/verifiers/v1/harnesses/terminus_2/__init__.py +0 -0
- {verifiers-0.3.2.dev105 → verifiers-0.3.2.dev107}/verifiers/v1/harnesses/terminus_2/harness.py +0 -0
- {verifiers-0.3.2.dev105 → verifiers-0.3.2.dev107}/verifiers/v1/harnesses/terminus_2/program.py +0 -0
- {verifiers-0.3.2.dev105 → verifiers-0.3.2.dev107}/verifiers/v1/harnesses/utils/__init__.py +0 -0
- {verifiers-0.3.2.dev105 → verifiers-0.3.2.dev107}/verifiers/v1/harnesses/utils/compaction.py +0 -0
- {verifiers-0.3.2.dev105 → verifiers-0.3.2.dev107}/verifiers/v1/harnesses/utils/core.py +0 -0
- {verifiers-0.3.2.dev105 → verifiers-0.3.2.dev107}/verifiers/v1/harnesses/utils/install.py +0 -0
- {verifiers-0.3.2.dev105 → verifiers-0.3.2.dev107}/verifiers/v1/harnesses/utils/launch.py +0 -0
- {verifiers-0.3.2.dev105 → verifiers-0.3.2.dev107}/verifiers/v1/harnesses/utils/mcp.py +0 -0
- {verifiers-0.3.2.dev105 → verifiers-0.3.2.dev107}/verifiers/v1/interception/__init__.py +0 -0
- {verifiers-0.3.2.dev105 → verifiers-0.3.2.dev107}/verifiers/v1/interception/base.py +0 -0
- {verifiers-0.3.2.dev105 → verifiers-0.3.2.dev107}/verifiers/v1/interception/pool.py +0 -0
- {verifiers-0.3.2.dev105 → verifiers-0.3.2.dev107}/verifiers/v1/interception/server.py +0 -0
- {verifiers-0.3.2.dev105 → verifiers-0.3.2.dev107}/verifiers/v1/interception/tool.py +0 -0
- {verifiers-0.3.2.dev105 → verifiers-0.3.2.dev107}/verifiers/v1/interception/tunnel/__init__.py +0 -0
- {verifiers-0.3.2.dev105 → verifiers-0.3.2.dev107}/verifiers/v1/interception/tunnel/base.py +0 -0
- {verifiers-0.3.2.dev105 → verifiers-0.3.2.dev107}/verifiers/v1/interception/tunnel/custom.py +0 -0
- {verifiers-0.3.2.dev105 → verifiers-0.3.2.dev107}/verifiers/v1/interception/tunnel/prime.py +0 -0
- {verifiers-0.3.2.dev105 → verifiers-0.3.2.dev107}/verifiers/v1/judge.py +0 -0
- {verifiers-0.3.2.dev105 → verifiers-0.3.2.dev107}/verifiers/v1/judges/__init__.py +0 -0
- {verifiers-0.3.2.dev105 → verifiers-0.3.2.dev107}/verifiers/v1/judges/reference.py +0 -0
- {verifiers-0.3.2.dev105 → verifiers-0.3.2.dev107}/verifiers/v1/judges/reference.txt +0 -0
- {verifiers-0.3.2.dev105 → verifiers-0.3.2.dev107}/verifiers/v1/judges/rubric.py +0 -0
- {verifiers-0.3.2.dev105 → verifiers-0.3.2.dev107}/verifiers/v1/judges/rubric.txt +0 -0
- {verifiers-0.3.2.dev105 → verifiers-0.3.2.dev107}/verifiers/v1/mcp/__init__.py +0 -0
- {verifiers-0.3.2.dev105 → verifiers-0.3.2.dev107}/verifiers/v1/mcp/server.py +0 -0
- {verifiers-0.3.2.dev105 → verifiers-0.3.2.dev107}/verifiers/v1/mcp/toolset.py +0 -0
- {verifiers-0.3.2.dev105 → verifiers-0.3.2.dev107}/verifiers/v1/runtimes/limiters.py +0 -0
- {verifiers-0.3.2.dev105 → verifiers-0.3.2.dev107}/verifiers/v1/runtimes/modal.py +0 -0
- {verifiers-0.3.2.dev105 → verifiers-0.3.2.dev107}/verifiers/v1/runtimes/prime.py +0 -0
- {verifiers-0.3.2.dev105 → verifiers-0.3.2.dev107}/verifiers/v1/runtimes/subprocess.py +0 -0
- {verifiers-0.3.2.dev105 → verifiers-0.3.2.dev107}/verifiers/v1/semantic.py +0 -0
- {verifiers-0.3.2.dev105 → verifiers-0.3.2.dev107}/verifiers/v1/serve/__init__.py +0 -0
- {verifiers-0.3.2.dev105 → verifiers-0.3.2.dev107}/verifiers/v1/serve/client.py +0 -0
- {verifiers-0.3.2.dev105 → verifiers-0.3.2.dev107}/verifiers/v1/serve/delta.py +0 -0
- {verifiers-0.3.2.dev105 → verifiers-0.3.2.dev107}/verifiers/v1/serve/encoding.py +0 -0
- {verifiers-0.3.2.dev105 → verifiers-0.3.2.dev107}/verifiers/v1/serve/pool.py +0 -0
- {verifiers-0.3.2.dev105 → verifiers-0.3.2.dev107}/verifiers/v1/serve/server.py +0 -0
- {verifiers-0.3.2.dev105 → verifiers-0.3.2.dev107}/verifiers/v1/serve/types.py +0 -0
- {verifiers-0.3.2.dev105 → verifiers-0.3.2.dev107}/verifiers/v1/session.py +0 -0
- {verifiers-0.3.2.dev105 → verifiers-0.3.2.dev107}/verifiers/v1/state.py +0 -0
- {verifiers-0.3.2.dev105 → verifiers-0.3.2.dev107}/verifiers/v1/taskset.py +0 -0
- {verifiers-0.3.2.dev105 → verifiers-0.3.2.dev107}/verifiers/v1/tasksets/__init__.py +0 -0
- {verifiers-0.3.2.dev105 → verifiers-0.3.2.dev107}/verifiers/v1/tasksets/harbor/__init__.py +0 -0
- {verifiers-0.3.2.dev105 → verifiers-0.3.2.dev107}/verifiers/v1/tasksets/harbor/env.py +0 -0
- {verifiers-0.3.2.dev105 → verifiers-0.3.2.dev107}/verifiers/v1/tasksets/harbor/taskset.py +0 -0
- {verifiers-0.3.2.dev105 → verifiers-0.3.2.dev107}/verifiers/v1/tasksets/harbor/toolset.py +0 -0
- {verifiers-0.3.2.dev105 → verifiers-0.3.2.dev107}/verifiers/v1/tasksets/lean/__init__.py +0 -0
- {verifiers-0.3.2.dev105 → verifiers-0.3.2.dev107}/verifiers/v1/tasksets/lean/scoring.py +0 -0
- {verifiers-0.3.2.dev105 → verifiers-0.3.2.dev107}/verifiers/v1/tasksets/lean/taskset.py +0 -0
- {verifiers-0.3.2.dev105 → verifiers-0.3.2.dev107}/verifiers/v1/tasksets/nemo_gym/__init__.py +0 -0
- {verifiers-0.3.2.dev105 → verifiers-0.3.2.dev107}/verifiers/v1/tasksets/nemo_gym/response.py +0 -0
- {verifiers-0.3.2.dev105 → verifiers-0.3.2.dev107}/verifiers/v1/tasksets/nemo_gym/server.py +0 -0
- {verifiers-0.3.2.dev105 → verifiers-0.3.2.dev107}/verifiers/v1/tasksets/nemo_gym/taskset.py +0 -0
- {verifiers-0.3.2.dev105 → verifiers-0.3.2.dev107}/verifiers/v1/tasksets/nemo_gym/toolset.py +0 -0
- {verifiers-0.3.2.dev105 → verifiers-0.3.2.dev107}/verifiers/v1/tasksets/openenv/__init__.py +0 -0
- {verifiers-0.3.2.dev105 → verifiers-0.3.2.dev107}/verifiers/v1/tasksets/openenv/taskset.py +0 -0
- {verifiers-0.3.2.dev105 → verifiers-0.3.2.dev107}/verifiers/v1/tasksets/textarena/__init__.py +0 -0
- {verifiers-0.3.2.dev105 → verifiers-0.3.2.dev107}/verifiers/v1/tasksets/textarena/taskset.py +0 -0
- {verifiers-0.3.2.dev105 → verifiers-0.3.2.dev107}/verifiers/v1/trace.py +0 -0
- {verifiers-0.3.2.dev105 → verifiers-0.3.2.dev107}/verifiers/v1/types.py +0 -0
- {verifiers-0.3.2.dev105 → verifiers-0.3.2.dev107}/verifiers/v1/utils/__init__.py +0 -0
- {verifiers-0.3.2.dev105 → verifiers-0.3.2.dev107}/verifiers/v1/utils/aio.py +0 -0
- {verifiers-0.3.2.dev105 → verifiers-0.3.2.dev107}/verifiers/v1/utils/artifacts.py +0 -0
- {verifiers-0.3.2.dev105 → verifiers-0.3.2.dev107}/verifiers/v1/utils/compile.py +0 -0
- {verifiers-0.3.2.dev105 → verifiers-0.3.2.dev107}/verifiers/v1/utils/decorators.py +0 -0
- {verifiers-0.3.2.dev105 → verifiers-0.3.2.dev107}/verifiers/v1/utils/format.py +0 -0
- {verifiers-0.3.2.dev105 → verifiers-0.3.2.dev107}/verifiers/v1/utils/generic.py +0 -0
- {verifiers-0.3.2.dev105 → verifiers-0.3.2.dev107}/verifiers/v1/utils/git.py +0 -0
- {verifiers-0.3.2.dev105 → verifiers-0.3.2.dev107}/verifiers/v1/utils/image.py +0 -0
- {verifiers-0.3.2.dev105 → verifiers-0.3.2.dev107}/verifiers/v1/utils/interrupt.py +0 -0
- {verifiers-0.3.2.dev105 → verifiers-0.3.2.dev107}/verifiers/v1/utils/loaders.py +0 -0
- {verifiers-0.3.2.dev105 → verifiers-0.3.2.dev107}/verifiers/v1/utils/logging.py +0 -0
- {verifiers-0.3.2.dev105 → verifiers-0.3.2.dev107}/verifiers/v1/utils/memory.py +0 -0
- {verifiers-0.3.2.dev105 → verifiers-0.3.2.dev107}/verifiers/v1/utils/paths.py +0 -0
- {verifiers-0.3.2.dev105 → verifiers-0.3.2.dev107}/verifiers/v1/utils/platform.py +0 -0
- {verifiers-0.3.2.dev105 → verifiers-0.3.2.dev107}/verifiers/v1/utils/prime.py +0 -0
- {verifiers-0.3.2.dev105 → verifiers-0.3.2.dev107}/verifiers/v1/utils/retries.py +0 -0
- {verifiers-0.3.2.dev105 → verifiers-0.3.2.dev107}/verifiers/v1/utils/score.py +0 -0
- {verifiers-0.3.2.dev105 → verifiers-0.3.2.dev107}/verifiers/v1/utils/version.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.5
|
|
2
2
|
Name: verifiers
|
|
3
|
-
Version: 0.3.2.
|
|
3
|
+
Version: 0.3.2.dev107
|
|
4
4
|
Summary: Verifiers: Environments for LLM Reinforcement Learning
|
|
5
5
|
Project-URL: Homepage, https://github.com/primeintellect-ai/verifiers
|
|
6
6
|
Project-URL: Documentation, https://github.com/primeintellect-ai/verifiers
|
|
@@ -227,6 +227,8 @@ markers = [
|
|
|
227
227
|
# per axis, so it matches every case touching that value on any axis.
|
|
228
228
|
"subprocess: v1 e2e cases touching the subprocess (host) runtime",
|
|
229
229
|
"docker: v1 e2e cases touching the docker runtime",
|
|
230
|
+
"podman: v1 e2e cases touching the podman runtime",
|
|
231
|
+
"apptainer: v1 e2e cases touching the apptainer runtime",
|
|
230
232
|
"prime: v1 e2e cases touching the prime runtime (real sandbox + tunnel; local-only, excluded in CI)",
|
|
231
233
|
"modal: v1 e2e cases touching the modal runtime (local-only setup; excluded in CI)",
|
|
232
234
|
"colocated: v1 e2e cases with a user/tool server colocated in the harness runtime",
|
|
@@ -24,7 +24,7 @@ Every combination carries its axes' pytest marks, so subsets select with `-m`:
|
|
|
24
24
|
uv run pytest tests/v1 -n auto -m prime # only prime (real sandboxes; local)
|
|
25
25
|
uv run pytest tests/v1 -n auto -m modal # only modal (needs local setup)
|
|
26
26
|
|
|
27
|
-
Marks: runtimes `subprocess` / `docker` / `prime` / `modal`, placement `colocated`,
|
|
27
|
+
Marks: runtimes `subprocess` / `docker` / `podman` / `apptainer` / `prime` / `modal`, placement `colocated`,
|
|
28
28
|
harnesses `null` / `bash` / `rlm` / `kimi_code` / `pi` / `openclaw` / `codex` /
|
|
29
29
|
`claude_code` / `hermes_agent`.
|
|
30
30
|
A mark is applied per axis, so it selects every case touching that value on ANY axis; for one exact
|
|
@@ -5,6 +5,7 @@ combinations a test runs — every axis value at least once plus the cross-bound
|
|
|
5
5
|
with distinct networking — instead of fanning the full cross product. prime/modal rows
|
|
6
6
|
are local-only (their marks are excluded in CI)."""
|
|
7
7
|
|
|
8
|
+
import shutil
|
|
8
9
|
import subprocess
|
|
9
10
|
import sys
|
|
10
11
|
|
|
@@ -115,6 +116,15 @@ CHAT_PLACEMENTS = [
|
|
|
115
116
|
# remote row per provider.
|
|
116
117
|
AGENTIC_PLACEMENTS = [
|
|
117
118
|
pair("bash", "subprocess", "bash-harness-in-subprocess"),
|
|
119
|
+
*[
|
|
120
|
+
pair(
|
|
121
|
+
"bash",
|
|
122
|
+
engine,
|
|
123
|
+
f"bash-harness-in-{engine}",
|
|
124
|
+
mark.skipif(shutil.which(engine) is None, reason=f"needs {engine}"),
|
|
125
|
+
)
|
|
126
|
+
for engine in ("podman", "apptainer")
|
|
127
|
+
],
|
|
118
128
|
pair("rlm", "docker", "rlm-harness-in-docker"),
|
|
119
129
|
pytest.param(
|
|
120
130
|
{"id": "kimi-code", "transport": "responses"},
|
|
@@ -148,6 +158,15 @@ USER_RUNTIMES = [
|
|
|
148
158
|
# retain MCP access after resuming. Cover every harness in the local container runtime,
|
|
149
159
|
# plus remote placements for the sandbox/tunnel and native-process boundaries.
|
|
150
160
|
ACP_RESUME_PLACEMENTS = [
|
|
161
|
+
*[
|
|
162
|
+
pair(
|
|
163
|
+
"rlm",
|
|
164
|
+
engine,
|
|
165
|
+
f"rlm-acp-in-{engine}",
|
|
166
|
+
mark.skipif(shutil.which(engine) is None, reason=f"needs {engine}"),
|
|
167
|
+
)
|
|
168
|
+
for engine in ("podman", "apptainer")
|
|
169
|
+
],
|
|
151
170
|
pair("codex", "docker", "codex-acp-in-docker"),
|
|
152
171
|
pair("claude-code", "docker", "claude-code-acp-in-docker"),
|
|
153
172
|
pair("hermes-agent", "docker", "hermes-agent-acp-in-docker"),
|
|
@@ -178,6 +197,15 @@ ACP_RESUME_PLACEMENTS = [
|
|
|
178
197
|
# (harness and tool in separate docker boxes) and a prime-colocated row (a tool in its
|
|
179
198
|
# OWN prime sandbox needs port exposure; colocated rides the harness's box).
|
|
180
199
|
TOOL_PLACEMENTS = [
|
|
200
|
+
*[
|
|
201
|
+
pair(
|
|
202
|
+
"subprocess",
|
|
203
|
+
engine,
|
|
204
|
+
f"harness-in-subprocess-with-tool-in-{engine}",
|
|
205
|
+
mark.skipif(shutil.which(engine) is None, reason=f"needs {engine}"),
|
|
206
|
+
)
|
|
207
|
+
for engine in ("podman", "apptainer")
|
|
208
|
+
],
|
|
181
209
|
pair("subprocess", "colocated", "harness-in-subprocess-with-tool-colocated"),
|
|
182
210
|
pair("docker", "colocated", "harness-in-docker-with-tool-colocated"),
|
|
183
211
|
pair("subprocess", "docker", "harness-in-subprocess-with-tool-in-docker"),
|
|
@@ -73,7 +73,9 @@ from verifiers.v1.mcp import (
|
|
|
73
73
|
ToolsetConfig,
|
|
74
74
|
)
|
|
75
75
|
from verifiers.v1.runtimes import (
|
|
76
|
+
ApptainerConfig,
|
|
76
77
|
DockerConfig,
|
|
78
|
+
PodmanConfig,
|
|
77
79
|
PrimeConfig,
|
|
78
80
|
ProgramResult,
|
|
79
81
|
Runtime,
|
|
@@ -301,6 +303,8 @@ __all__ = [ # noqa: RUF022 - grouped by public API area
|
|
|
301
303
|
"ProgramResult",
|
|
302
304
|
"SubprocessConfig",
|
|
303
305
|
"DockerConfig",
|
|
306
|
+
"PodmanConfig",
|
|
307
|
+
"ApptainerConfig",
|
|
304
308
|
"PrimeConfig",
|
|
305
309
|
"Env",
|
|
306
310
|
"SingleAgentEnv",
|
{verifiers-0.3.2.dev105 → verifiers-0.3.2.dev107}/verifiers/v1/harnesses/browser_use/harness.py
RENAMED
|
@@ -1,6 +1,8 @@
|
|
|
1
1
|
from pathlib import Path
|
|
2
2
|
from typing import Literal
|
|
3
|
+
from urllib.parse import urlsplit
|
|
3
4
|
|
|
5
|
+
import httpx
|
|
4
6
|
from pydantic import model_validator
|
|
5
7
|
|
|
6
8
|
from verifiers.v1.clients import ModelContext
|
|
@@ -92,7 +94,18 @@ class BrowserUseHarness(Harness[BrowserUseHarnessConfig]):
|
|
|
92
94
|
f"--state-dir={state}",
|
|
93
95
|
]
|
|
94
96
|
if self.config.cdp_url:
|
|
95
|
-
|
|
97
|
+
cdp_url = runtime.host_url(self.config.cdp_url)
|
|
98
|
+
if cdp_url != self.config.cdp_url and urlsplit(
|
|
99
|
+
self.config.cdp_url
|
|
100
|
+
).scheme in ("http", "https"):
|
|
101
|
+
# CDP discovery returns a host-side WebSocket URL; translate that too.
|
|
102
|
+
async with httpx.AsyncClient() as client:
|
|
103
|
+
response = await client.get(
|
|
104
|
+
f"{self.config.cdp_url.rstrip('/')}/json/version"
|
|
105
|
+
)
|
|
106
|
+
response.raise_for_status()
|
|
107
|
+
cdp_url = runtime.host_url(response.json()["webSocketDebuggerUrl"])
|
|
108
|
+
args.append(f"--cdp-url={cdp_url}")
|
|
96
109
|
return await launch_chat_program(
|
|
97
110
|
PROGRAM_SOURCE,
|
|
98
111
|
self.config,
|
|
@@ -28,7 +28,7 @@ from verifiers.v1.mcp.server import (
|
|
|
28
28
|
)
|
|
29
29
|
from verifiers.v1.runtimes import (
|
|
30
30
|
Runtime,
|
|
31
|
-
|
|
31
|
+
provision_runtime,
|
|
32
32
|
)
|
|
33
33
|
from verifiers.v1.runtimes.base import _ENSURE_UV
|
|
34
34
|
from verifiers.v1.state import State
|
|
@@ -269,7 +269,7 @@ async def serve_in_runtime(
|
|
|
269
269
|
# Keep provider temp files in the runtime workdir so cleanup removes them.
|
|
270
270
|
assert runtime.info.id is not None
|
|
271
271
|
env["TMPDIR"] = runtime.info.id
|
|
272
|
-
if runtime.published_port is not None:
|
|
272
|
+
if exposed and runtime.published_port is not None:
|
|
273
273
|
env["MCP_HOST"] = "0.0.0.0"
|
|
274
274
|
fixed = runtime.published_port if exposed else None
|
|
275
275
|
port_file = None
|
|
@@ -318,17 +318,18 @@ async def reachable_url(
|
|
|
318
318
|
runtime; `consumer_is_local` = the consumer can use a host-local URL without a tunnel.
|
|
319
319
|
|
|
320
320
|
- `colocated` -> localhost (same runtime, in-sandbox or host loopback);
|
|
321
|
-
- the
|
|
322
|
-
|
|
321
|
+
- else the runtime publishes the port (`expose`): a remote sandbox's URL is reachable
|
|
322
|
+
anywhere, a host-local URL directly by a local consumer and through a host tunnel
|
|
323
|
+
by a remote one."""
|
|
323
324
|
if colocated:
|
|
324
325
|
yield f"http://127.0.0.1:{port}"
|
|
325
|
-
|
|
326
|
-
|
|
327
|
-
|
|
328
|
-
|
|
329
|
-
|
|
330
|
-
|
|
331
|
-
|
|
326
|
+
return
|
|
327
|
+
url = await service.expose(port)
|
|
328
|
+
if service.is_local and not consumer_is_local:
|
|
329
|
+
async with PrimeTunnel().expose(urlsplit(url).port or 80) as public:
|
|
330
|
+
yield public
|
|
331
|
+
else:
|
|
332
|
+
yield url
|
|
332
333
|
|
|
333
334
|
|
|
334
335
|
@dataclass(frozen=True)
|
|
@@ -365,9 +366,7 @@ async def _serve(
|
|
|
365
366
|
if colocated and harness_runtime is not None:
|
|
366
367
|
runtime = harness_runtime
|
|
367
368
|
else:
|
|
368
|
-
runtime =
|
|
369
|
-
await runtime.start()
|
|
370
|
-
stack.push_async_callback(runtime.stop)
|
|
369
|
+
runtime = await stack.enter_async_context(provision_runtime(cfg.runtime))
|
|
371
370
|
# Only consumers outside the server runtime need its fixed published port. Colocated tools
|
|
372
371
|
# use independent OS-assigned ports, avoiding clashes on the runtime's service port.
|
|
373
372
|
exposed = runtime is not harness_runtime
|
|
@@ -376,7 +375,7 @@ async def _serve(
|
|
|
376
375
|
# whenever any consumer is remote). Eval-level shared servers get no per-rollout channel
|
|
377
376
|
# (`state_base` is None for them).
|
|
378
377
|
state_url = (
|
|
379
|
-
|
|
378
|
+
runtime.host_url(f"{state_base.rstrip('/')}/state") if state_base else None
|
|
380
379
|
)
|
|
381
380
|
port = await serve_in_runtime(
|
|
382
381
|
server,
|
|
@@ -256,7 +256,7 @@ class Rollout:
|
|
|
256
256
|
self._shared_tools,
|
|
257
257
|
)
|
|
258
258
|
)
|
|
259
|
-
self._endpoint = f"{
|
|
259
|
+
self._endpoint = runtime.host_url(f"{base_url.rstrip('/')}/v1")
|
|
260
260
|
self._secret = model_secret
|
|
261
261
|
self._urls = await self._stack.enter_async_context(
|
|
262
262
|
serve_tools(
|
|
@@ -314,7 +314,11 @@ class Rollout:
|
|
|
314
314
|
)
|
|
315
315
|
if not self._session.stopped:
|
|
316
316
|
session_kwargs = (
|
|
317
|
-
{
|
|
317
|
+
{
|
|
318
|
+
"tool_interception_url": runtime.host_url(
|
|
319
|
+
f"{base_url.rstrip('/')}/tool"
|
|
320
|
+
)
|
|
321
|
+
}
|
|
318
322
|
if self.harness.SUPPORTS_TOOL_INTERCEPTION
|
|
319
323
|
and (
|
|
320
324
|
self._session.request_interceptors
|
|
@@ -5,6 +5,11 @@ from typing import Annotated
|
|
|
5
5
|
from pydantic import Field
|
|
6
6
|
|
|
7
7
|
from verifiers.v1.configs.runtime import NetworkPolicyConfig
|
|
8
|
+
from verifiers.v1.runtimes.apptainer import (
|
|
9
|
+
ApptainerConfig,
|
|
10
|
+
ApptainerRuntime,
|
|
11
|
+
ApptainerRuntimeInfo,
|
|
12
|
+
)
|
|
8
13
|
from verifiers.v1.runtimes.base import (
|
|
9
14
|
BaseRuntimeInfo,
|
|
10
15
|
ProgramResult,
|
|
@@ -12,7 +17,14 @@ from verifiers.v1.runtimes.base import (
|
|
|
12
17
|
RuntimeProcess,
|
|
13
18
|
register,
|
|
14
19
|
)
|
|
15
|
-
from verifiers.v1.runtimes.docker import
|
|
20
|
+
from verifiers.v1.runtimes.docker import (
|
|
21
|
+
DockerConfig,
|
|
22
|
+
DockerRuntime,
|
|
23
|
+
DockerRuntimeInfo,
|
|
24
|
+
PodmanConfig,
|
|
25
|
+
PodmanRuntime,
|
|
26
|
+
PodmanRuntimeInfo,
|
|
27
|
+
)
|
|
16
28
|
from verifiers.v1.runtimes.modal import ModalConfig, ModalRuntime, ModalRuntimeInfo
|
|
17
29
|
from verifiers.v1.runtimes.prime import (
|
|
18
30
|
PrimeConfig,
|
|
@@ -27,24 +39,35 @@ from verifiers.v1.runtimes.subprocess import (
|
|
|
27
39
|
)
|
|
28
40
|
|
|
29
41
|
RuntimeConfig = Annotated[
|
|
30
|
-
SubprocessConfig
|
|
42
|
+
SubprocessConfig
|
|
43
|
+
| DockerConfig
|
|
44
|
+
| PodmanConfig
|
|
45
|
+
| ApptainerConfig
|
|
46
|
+
| PrimeConfig
|
|
47
|
+
| ModalConfig,
|
|
31
48
|
Field(discriminator="type"),
|
|
32
49
|
]
|
|
33
50
|
|
|
34
51
|
RuntimeInfo = Annotated[
|
|
35
|
-
SubprocessRuntimeInfo
|
|
52
|
+
SubprocessRuntimeInfo
|
|
53
|
+
| DockerRuntimeInfo
|
|
54
|
+
| PodmanRuntimeInfo
|
|
55
|
+
| ApptainerRuntimeInfo
|
|
56
|
+
| PrimeRuntimeInfo
|
|
57
|
+
| ModalRuntimeInfo,
|
|
36
58
|
Field(discriminator="type"),
|
|
37
59
|
]
|
|
38
60
|
|
|
39
61
|
|
|
40
62
|
def _runtime_cls(config: RuntimeConfig) -> type[Runtime]:
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
|
|
63
|
+
return {
|
|
64
|
+
"subprocess": SubprocessRuntime,
|
|
65
|
+
"docker": DockerRuntime,
|
|
66
|
+
"podman": PodmanRuntime,
|
|
67
|
+
"apptainer": ApptainerRuntime,
|
|
68
|
+
"prime": PrimeRuntime,
|
|
69
|
+
"modal": ModalRuntime,
|
|
70
|
+
}[config.type]
|
|
48
71
|
|
|
49
72
|
|
|
50
73
|
def make_runtime(config: RuntimeConfig, name: str | None = None) -> Runtime:
|
|
@@ -79,6 +102,9 @@ def runtime_is_local(config: RuntimeConfig) -> bool:
|
|
|
79
102
|
|
|
80
103
|
|
|
81
104
|
__all__ = [
|
|
105
|
+
"ApptainerConfig",
|
|
106
|
+
"ApptainerRuntime",
|
|
107
|
+
"ApptainerRuntimeInfo",
|
|
82
108
|
"BaseRuntimeInfo",
|
|
83
109
|
"DockerConfig",
|
|
84
110
|
"DockerRuntime",
|
|
@@ -87,6 +113,9 @@ __all__ = [
|
|
|
87
113
|
"ModalRuntime",
|
|
88
114
|
"ModalRuntimeInfo",
|
|
89
115
|
"NetworkPolicyConfig",
|
|
116
|
+
"PodmanConfig",
|
|
117
|
+
"PodmanRuntime",
|
|
118
|
+
"PodmanRuntimeInfo",
|
|
90
119
|
"PrimeConfig",
|
|
91
120
|
"PrimeRuntime",
|
|
92
121
|
"PrimeRuntimeInfo",
|
|
@@ -0,0 +1,214 @@
|
|
|
1
|
+
"""Local Apptainer runtime: an unprivileged instance sharing the host network."""
|
|
2
|
+
|
|
3
|
+
import asyncio
|
|
4
|
+
import contextlib
|
|
5
|
+
import hashlib
|
|
6
|
+
import logging
|
|
7
|
+
import shutil
|
|
8
|
+
import subprocess
|
|
9
|
+
import tempfile
|
|
10
|
+
import uuid
|
|
11
|
+
from pathlib import Path
|
|
12
|
+
from typing import ClassVar, Literal, Self
|
|
13
|
+
from weakref import WeakValueDictionary
|
|
14
|
+
|
|
15
|
+
from pydantic import model_validator
|
|
16
|
+
|
|
17
|
+
from verifiers.v1.errors import SandboxError
|
|
18
|
+
from verifiers.v1.runtimes.base import BaseRuntimeInfo, parse_gpu
|
|
19
|
+
from verifiers.v1.runtimes.container import ContainerConfig, ContainerRuntime, cli
|
|
20
|
+
from verifiers.v1.utils.paths import CACHE_DIR
|
|
21
|
+
|
|
22
|
+
logger = logging.getLogger(__name__)
|
|
23
|
+
|
|
24
|
+
_ROOT = CACHE_DIR / "runtimes" / "apptainer"
|
|
25
|
+
|
|
26
|
+
|
|
27
|
+
class ApptainerConfig(ContainerConfig):
|
|
28
|
+
type: Literal["apptainer"] = "apptainer"
|
|
29
|
+
"""Apptainer has no egress policy: instances run unprivileged on the host network,
|
|
30
|
+
as on HPC clusters without Docker. `image` is a Docker reference (pulled to a SIF
|
|
31
|
+
once per reference), any `scheme://` URI Apptainer can pull, or a local SIF path."""
|
|
32
|
+
|
|
33
|
+
@model_validator(mode="after")
|
|
34
|
+
def validate_workdir(self) -> Self:
|
|
35
|
+
if self.workdir is not None and (
|
|
36
|
+
not self.workdir.startswith("/")
|
|
37
|
+
or Path(self.workdir) == Path("/")
|
|
38
|
+
or ".." in Path(self.workdir).parts
|
|
39
|
+
or any(char in self.workdir for char in ":,")
|
|
40
|
+
):
|
|
41
|
+
raise ValueError(
|
|
42
|
+
"Apptainer workdir must be an absolute non-root path without '..', ':' or ','"
|
|
43
|
+
)
|
|
44
|
+
return self
|
|
45
|
+
|
|
46
|
+
|
|
47
|
+
class ApptainerRuntimeInfo(ApptainerConfig, BaseRuntimeInfo):
|
|
48
|
+
pass
|
|
49
|
+
|
|
50
|
+
|
|
51
|
+
class ApptainerRuntime(ContainerRuntime):
|
|
52
|
+
_pulls: ClassVar[
|
|
53
|
+
WeakValueDictionary[tuple[asyncio.AbstractEventLoop, str], asyncio.Lock]
|
|
54
|
+
] = WeakValueDictionary()
|
|
55
|
+
|
|
56
|
+
def __init__(self, config: ApptainerConfig, name: str | None = None) -> None:
|
|
57
|
+
super().__init__(name)
|
|
58
|
+
self.config = config.model_copy(update={"workdir": config.workdir or "/app"})
|
|
59
|
+
self.info = ApptainerRuntimeInfo(**self.config.model_dump())
|
|
60
|
+
# Apptainer names are user-global; never stop an instance named by a caller.
|
|
61
|
+
self._instance = f"vf-{uuid.uuid4().hex}"
|
|
62
|
+
self._dir: Path | None = None # host backing for the workspace, /tmp and $HOME
|
|
63
|
+
self._stopped = False
|
|
64
|
+
|
|
65
|
+
def _exec(self, env: dict[str, str], *, stdin: bool = False) -> list[str]:
|
|
66
|
+
# `--env` is a comma-separated map flag, so values pass through `env` inside.
|
|
67
|
+
return [
|
|
68
|
+
"apptainer",
|
|
69
|
+
"exec",
|
|
70
|
+
"--cleanenv",
|
|
71
|
+
"--no-eval",
|
|
72
|
+
"--cwd",
|
|
73
|
+
self.config.workdir,
|
|
74
|
+
f"instance://{self._instance}",
|
|
75
|
+
"env",
|
|
76
|
+
*(f"{key}={value}" for key, value in env.items()),
|
|
77
|
+
]
|
|
78
|
+
|
|
79
|
+
async def start(self) -> None:
|
|
80
|
+
try:
|
|
81
|
+
version = await cli("apptainer", "version")
|
|
82
|
+
except FileNotFoundError as e:
|
|
83
|
+
raise RuntimeError(
|
|
84
|
+
"apptainer runtime selected but the `apptainer` CLI is not installed"
|
|
85
|
+
) from e
|
|
86
|
+
if version.exit_code != 0:
|
|
87
|
+
detail = (version.stderr or version.stdout).strip()
|
|
88
|
+
raise RuntimeError(
|
|
89
|
+
f"apptainer runtime selected but Apptainer is not usable: {detail}"
|
|
90
|
+
)
|
|
91
|
+
self._dir = Path(tempfile.mkdtemp(prefix="vf-apptainer-"))
|
|
92
|
+
(self._dir / "workspace").mkdir()
|
|
93
|
+
(self._dir / "session").mkdir()
|
|
94
|
+
# Bind targets below a contained directory must also exist in its backing tree.
|
|
95
|
+
workdir = Path(self.config.workdir)
|
|
96
|
+
for parent, backing in (
|
|
97
|
+
(Path("/tmp"), "tmp"),
|
|
98
|
+
(Path("/var/tmp"), "var_tmp"),
|
|
99
|
+
(Path.home(), "home"),
|
|
100
|
+
):
|
|
101
|
+
if workdir.is_relative_to(parent):
|
|
102
|
+
(self._dir / "session" / backing / workdir.relative_to(parent)).mkdir(
|
|
103
|
+
parents=True, exist_ok=True
|
|
104
|
+
)
|
|
105
|
+
image = await self._image()
|
|
106
|
+
inspected = await cli("apptainer", "inspect", "--startscript", image)
|
|
107
|
+
# Apptainer's default startscript contains only a shebang and comments.
|
|
108
|
+
if inspected.exit_code or any(
|
|
109
|
+
line.strip() and not line.lstrip().startswith("#")
|
|
110
|
+
for line in inspected.stdout.splitlines()
|
|
111
|
+
):
|
|
112
|
+
raise SandboxError("Apptainer requires an image without a startscript")
|
|
113
|
+
containment = [
|
|
114
|
+
"--cleanenv",
|
|
115
|
+
"--containall",
|
|
116
|
+
"--no-mount",
|
|
117
|
+
"cwd,hostfs,bind-paths",
|
|
118
|
+
"--no-eval",
|
|
119
|
+
]
|
|
120
|
+
# Seed the writable bind from the image before it hides the original workdir.
|
|
121
|
+
destination = f"/{self._instance}"
|
|
122
|
+
copied = await cli(
|
|
123
|
+
"apptainer",
|
|
124
|
+
"exec",
|
|
125
|
+
*containment,
|
|
126
|
+
"--no-mount",
|
|
127
|
+
"home,tmp",
|
|
128
|
+
"--bind",
|
|
129
|
+
f"{self._dir / 'workspace'}:{destination}",
|
|
130
|
+
image,
|
|
131
|
+
"sh",
|
|
132
|
+
"-c",
|
|
133
|
+
'if [ -d "$1" ]; then cp -R -P "$1"/. "$2"/; fi',
|
|
134
|
+
"vf-workspace",
|
|
135
|
+
self.config.workdir,
|
|
136
|
+
destination,
|
|
137
|
+
)
|
|
138
|
+
if copied.exit_code != 0:
|
|
139
|
+
raise SandboxError(
|
|
140
|
+
f"apptainer workdir copy failed: {copied.stderr.strip()}"
|
|
141
|
+
)
|
|
142
|
+
limits: list[str] = []
|
|
143
|
+
if self.config.cpu is not None:
|
|
144
|
+
limits += ["--cpus", str(self.config.cpu)]
|
|
145
|
+
if self.config.memory is not None:
|
|
146
|
+
limits += ["--memory", f"{self.config.memory}g"]
|
|
147
|
+
if parse_gpu(self.config.gpu)[1]:
|
|
148
|
+
limits += ["--nv"]
|
|
149
|
+
started = await cli(
|
|
150
|
+
"apptainer",
|
|
151
|
+
"instance",
|
|
152
|
+
"start",
|
|
153
|
+
*containment,
|
|
154
|
+
# A contained /tmp and $HOME live in tmpfs unless a workdir backs them.
|
|
155
|
+
"--workdir",
|
|
156
|
+
str(self._dir / "session"),
|
|
157
|
+
"--writable-tmpfs",
|
|
158
|
+
"--bind",
|
|
159
|
+
f"{self._dir / 'workspace'}:{self.config.workdir}",
|
|
160
|
+
*limits,
|
|
161
|
+
image,
|
|
162
|
+
self._instance,
|
|
163
|
+
)
|
|
164
|
+
if started.exit_code != 0:
|
|
165
|
+
raise SandboxError(
|
|
166
|
+
f"apptainer instance start failed: {started.stderr.strip()}"
|
|
167
|
+
)
|
|
168
|
+
self.info.id = self._instance
|
|
169
|
+
logger.info(
|
|
170
|
+
"apptainer: started instance %s (image=%s)", self.name, self.config.image
|
|
171
|
+
)
|
|
172
|
+
|
|
173
|
+
async def _image(self) -> str:
|
|
174
|
+
"""The SIF to run: a local file as is, else the reference pulled once into the
|
|
175
|
+
cache, like a container engine keeping a pulled image."""
|
|
176
|
+
local = Path(self.config.image).expanduser()
|
|
177
|
+
if local.is_file():
|
|
178
|
+
return str(local)
|
|
179
|
+
image = self.config.image
|
|
180
|
+
ref = image if "://" in image else f"docker://{image}"
|
|
181
|
+
sif = _ROOT / "images" / f"{hashlib.sha256(ref.encode()).hexdigest()}.sif"
|
|
182
|
+
async with self._pulls.setdefault(
|
|
183
|
+
(asyncio.get_running_loop(), ref), asyncio.Lock()
|
|
184
|
+
):
|
|
185
|
+
if not sif.exists():
|
|
186
|
+
sif.parent.mkdir(parents=True, exist_ok=True)
|
|
187
|
+
# Pull to a unique path and publish it atomically, so concurrent
|
|
188
|
+
# workers never read a half-written SIF.
|
|
189
|
+
tmp = sif.with_name(f"{uuid.uuid4().hex}.sif")
|
|
190
|
+
try:
|
|
191
|
+
pulled = await cli("apptainer", "pull", str(tmp), ref)
|
|
192
|
+
if pulled.exit_code != 0:
|
|
193
|
+
raise SandboxError(
|
|
194
|
+
f"apptainer pull {ref} failed: {pulled.stderr.strip()}"
|
|
195
|
+
)
|
|
196
|
+
tmp.replace(sif)
|
|
197
|
+
finally:
|
|
198
|
+
tmp.unlink(missing_ok=True)
|
|
199
|
+
return str(sif)
|
|
200
|
+
|
|
201
|
+
def cleanup(self) -> None:
|
|
202
|
+
if self._dir is None or self._stopped:
|
|
203
|
+
return
|
|
204
|
+
self._stopped = True
|
|
205
|
+
logger.debug("apptainer: stopping instance %s", self.name)
|
|
206
|
+
with contextlib.suppress(Exception):
|
|
207
|
+
subprocess.run(
|
|
208
|
+
["apptainer", "instance", "stop", "--force", self._instance],
|
|
209
|
+
stdout=subprocess.DEVNULL,
|
|
210
|
+
stderr=subprocess.DEVNULL,
|
|
211
|
+
timeout=30,
|
|
212
|
+
check=False,
|
|
213
|
+
)
|
|
214
|
+
shutil.rmtree(self._dir, ignore_errors=True)
|
|
@@ -45,9 +45,9 @@ _ENSURE_UV = (
|
|
|
45
45
|
f"|| {{ {_INSTALL_CURL}; {_DOWNLOAD_UV}; }}"
|
|
46
46
|
)
|
|
47
47
|
|
|
48
|
-
# The single port a
|
|
49
|
-
#
|
|
50
|
-
# at the
|
|
48
|
+
# The single port a sandbox runtime forwards out for a server hosted in it: a public URL on
|
|
49
|
+
# modal/prime, a host loopback port on the local container engines. A server placed in such
|
|
50
|
+
# a runtime binds this (on 0.0.0.0) and is reached at the URL `expose` returns.
|
|
51
51
|
SERVICE_PORT = 8000
|
|
52
52
|
|
|
53
53
|
|
|
@@ -136,8 +136,8 @@ class Runtime(ABC):
|
|
|
136
136
|
|
|
137
137
|
is_local: ClassVar[bool] = True
|
|
138
138
|
"""Whether this runtime exchanges host-local URLs without a public tunnel. True for
|
|
139
|
-
subprocess and
|
|
140
|
-
|
|
139
|
+
subprocess and the local container runtimes; remote runtimes override to False and
|
|
140
|
+
use a host `Tunnel` inward plus `expose` outward."""
|
|
141
141
|
|
|
142
142
|
scripts_dir: ClassVar[str] = "/tmp/vf-scripts"
|
|
143
143
|
"""Digest-keyed PEP 723 scripts inside the runtime. Sandboxes own their `/tmp`;
|
|
@@ -397,13 +397,14 @@ class Runtime(ABC):
|
|
|
397
397
|
"""A fixed port this runtime exposes to the outside at startup, declared up front to the
|
|
398
398
|
provider (Modal forwards only ports named at `Sandbox.create`). When set, a server placed
|
|
399
399
|
here binds it instead of a host-chosen free port, and `expose` returns its public URL.
|
|
400
|
-
`None` for
|
|
400
|
+
`None` for runtimes whose services already sit on host loopback (subprocess,
|
|
401
|
+
apptainer), which pick a free port."""
|
|
401
402
|
return None
|
|
402
403
|
|
|
403
|
-
async def expose(self, port: int) -> str
|
|
404
|
-
"""Publish a port running *inside this runtime* to a URL reachable from the host/outside
|
|
405
|
-
|
|
406
|
-
exposure (modal `tunnels()`, prime `client.expose
|
|
407
|
-
`stop()`. The reverse of a
|
|
408
|
-
port from inside a runtime)."""
|
|
409
|
-
return
|
|
404
|
+
async def expose(self, port: int) -> str:
|
|
405
|
+
"""Publish a port running *inside this runtime* to a URL reachable from the host/outside.
|
|
406
|
+
A runtime on the host network is reached at host loopback as is; a sandbox overrides
|
|
407
|
+
this with its port exposure (modal `tunnels()`, prime `client.expose`, a container
|
|
408
|
+
engine's published port), torn down with the sandbox in `stop()`. The reverse of a
|
|
409
|
+
host `Tunnel` (interception.tunnel, which reaches a host port from inside a runtime)."""
|
|
410
|
+
return f"http://127.0.0.1:{port}"
|