verifiers 0.3.2.dev136__tar.gz → 0.3.2.dev138__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {verifiers-0.3.2.dev136 → verifiers-0.3.2.dev138}/PKG-INFO +1 -1
- {verifiers-0.3.2.dev136 → verifiers-0.3.2.dev138}/tests/v1/test_e2e.py +1 -1
- {verifiers-0.3.2.dev136 → verifiers-0.3.2.dev138}/tests/v1/test_serve_delta.py +25 -30
- {verifiers-0.3.2.dev136 → verifiers-0.3.2.dev138}/tests/v1/test_trace.py +1 -1
- {verifiers-0.3.2.dev136 → verifiers-0.3.2.dev138}/verifiers/v1/agent.py +14 -17
- {verifiers-0.3.2.dev136 → verifiers-0.3.2.dev138}/verifiers/v1/cli/dashboard/eval.py +2 -2
- {verifiers-0.3.2.dev136 → verifiers-0.3.2.dev138}/verifiers/v1/cli/debug.py +2 -1
- {verifiers-0.3.2.dev136 → verifiers-0.3.2.dev138}/verifiers/v1/cli/eval/main.py +1 -1
- {verifiers-0.3.2.dev136 → verifiers-0.3.2.dev138}/verifiers/v1/cli/eval/resume.py +1 -1
- {verifiers-0.3.2.dev136 → verifiers-0.3.2.dev138}/verifiers/v1/cli/eval/runner.py +2 -5
- {verifiers-0.3.2.dev136 → verifiers-0.3.2.dev138}/verifiers/v1/cli/gepa.py +2 -1
- {verifiers-0.3.2.dev136 → verifiers-0.3.2.dev138}/verifiers/v1/cli/output.py +4 -87
- {verifiers-0.3.2.dev136 → verifiers-0.3.2.dev138}/verifiers/v1/cli/replay.py +2 -7
- {verifiers-0.3.2.dev136 → verifiers-0.3.2.dev138}/verifiers/v1/configs/agent.py +51 -1
- {verifiers-0.3.2.dev136 → verifiers-0.3.2.dev138}/verifiers/v1/configs/env.py +7 -24
- {verifiers-0.3.2.dev136 → verifiers-0.3.2.dev138}/verifiers/v1/env.py +13 -24
- {verifiers-0.3.2.dev136 → verifiers-0.3.2.dev138}/verifiers/v1/gepa/runner.py +2 -1
- {verifiers-0.3.2.dev136 → verifiers-0.3.2.dev138}/verifiers/v1/harnesses/utils/core.py +17 -15
- {verifiers-0.3.2.dev136 → verifiers-0.3.2.dev138}/verifiers/v1/runtimes/docker/__init__.py +10 -1
- {verifiers-0.3.2.dev136 → verifiers-0.3.2.dev138}/verifiers/v1/runtimes/prime.py +4 -4
- {verifiers-0.3.2.dev136 → verifiers-0.3.2.dev138}/verifiers/v1/serve/delta.py +14 -11
- {verifiers-0.3.2.dev136 → verifiers-0.3.2.dev138}/verifiers/v1/serve/server.py +4 -4
- verifiers-0.3.2.dev138/verifiers/v1/utils/trace_store.py +80 -0
- {verifiers-0.3.2.dev136 → verifiers-0.3.2.dev138}/.gitignore +0 -0
- {verifiers-0.3.2.dev136 → verifiers-0.3.2.dev138}/LICENSE +0 -0
- {verifiers-0.3.2.dev136 → verifiers-0.3.2.dev138}/README.md +0 -0
- {verifiers-0.3.2.dev136 → verifiers-0.3.2.dev138}/pyproject.toml +0 -0
- {verifiers-0.3.2.dev136 → verifiers-0.3.2.dev138}/tests/v1/conftest.py +0 -0
- {verifiers-0.3.2.dev136 → verifiers-0.3.2.dev138}/tests/v1/fixtures/counter_tool_v1.py +0 -0
- {verifiers-0.3.2.dev136 → verifiers-0.3.2.dev138}/tests/v1/fixtures/duet_v1.py +0 -0
- {verifiers-0.3.2.dev136 → verifiers-0.3.2.dev138}/tests/v1/fixtures/echo_acp_resume_v1.py +0 -0
- {verifiers-0.3.2.dev136 → verifiers-0.3.2.dev138}/tests/v1/fixtures/echo_agentic_v1.py +0 -0
- {verifiers-0.3.2.dev136 → verifiers-0.3.2.dev138}/tests/v1/fixtures/echo_tool_v1.py +0 -0
- {verifiers-0.3.2.dev136 → verifiers-0.3.2.dev138}/tests/v1/fixtures/echo_user_sim_v1.py +0 -0
- {verifiers-0.3.2.dev136 → verifiers-0.3.2.dev138}/tests/v1/fixtures/echo_v1.py +0 -0
- {verifiers-0.3.2.dev136 → verifiers-0.3.2.dev138}/tests/v1/fixtures/pyproject.toml +0 -0
- {verifiers-0.3.2.dev136 → verifiers-0.3.2.dev138}/tests/v1/fixtures/tool_response_image_v1.py +0 -0
- {verifiers-0.3.2.dev136 → verifiers-0.3.2.dev138}/tests/v1/test_color_codeword_tasks.py +0 -0
- {verifiers-0.3.2.dev136 → verifiers-0.3.2.dev138}/tests/v1/test_configs.py +0 -0
- {verifiers-0.3.2.dev136 → verifiers-0.3.2.dev138}/tests/v1/test_envs.py +0 -0
- {verifiers-0.3.2.dev136 → verifiers-0.3.2.dev138}/tests/v1/test_graph.py +0 -0
- {verifiers-0.3.2.dev136 → verifiers-0.3.2.dev138}/tests/v1/test_judges.py +0 -0
- {verifiers-0.3.2.dev136 → verifiers-0.3.2.dev138}/tests/v1/test_platform_run.py +0 -0
- {verifiers-0.3.2.dev136 → verifiers-0.3.2.dev138}/tests/v1/test_scoring.py +0 -0
- {verifiers-0.3.2.dev136 → verifiers-0.3.2.dev138}/tests/v1/test_taskset.py +0 -0
- {verifiers-0.3.2.dev136 → verifiers-0.3.2.dev138}/verifiers/__init__.py +0 -0
- {verifiers-0.3.2.dev136 → verifiers-0.3.2.dev138}/verifiers/v1/__init__.py +0 -0
- {verifiers-0.3.2.dev136 → verifiers-0.3.2.dev138}/verifiers/v1/acp/__init__.py +0 -0
- {verifiers-0.3.2.dev136 → verifiers-0.3.2.dev138}/verifiers/v1/acp/runner.py +0 -0
- {verifiers-0.3.2.dev136 → verifiers-0.3.2.dev138}/verifiers/v1/cli/__init__.py +0 -0
- {verifiers-0.3.2.dev136 → verifiers-0.3.2.dev138}/verifiers/v1/cli/dashboard/__init__.py +0 -0
- {verifiers-0.3.2.dev136 → verifiers-0.3.2.dev138}/verifiers/v1/cli/dashboard/base.py +0 -0
- {verifiers-0.3.2.dev136 → verifiers-0.3.2.dev138}/verifiers/v1/cli/dashboard/replay.py +0 -0
- {verifiers-0.3.2.dev136 → verifiers-0.3.2.dev138}/verifiers/v1/cli/dashboard/validate.py +0 -0
- {verifiers-0.3.2.dev136 → verifiers-0.3.2.dev138}/verifiers/v1/cli/eval/__init__.py +0 -0
- {verifiers-0.3.2.dev136 → verifiers-0.3.2.dev138}/verifiers/v1/cli/eval/hint.py +0 -0
- {verifiers-0.3.2.dev136 → verifiers-0.3.2.dev138}/verifiers/v1/cli/init.py +0 -0
- {verifiers-0.3.2.dev136 → verifiers-0.3.2.dev138}/verifiers/v1/cli/resolve.py +0 -0
- {verifiers-0.3.2.dev136 → verifiers-0.3.2.dev138}/verifiers/v1/cli/resume.py +0 -0
- {verifiers-0.3.2.dev136 → verifiers-0.3.2.dev138}/verifiers/v1/cli/validate.py +0 -0
- {verifiers-0.3.2.dev136 → verifiers-0.3.2.dev138}/verifiers/v1/clients/__init__.py +0 -0
- {verifiers-0.3.2.dev136 → verifiers-0.3.2.dev138}/verifiers/v1/clients/base.py +0 -0
- {verifiers-0.3.2.dev136 → verifiers-0.3.2.dev138}/verifiers/v1/clients/client.py +0 -0
- {verifiers-0.3.2.dev136 → verifiers-0.3.2.dev138}/verifiers/v1/clients/eval.py +0 -0
- {verifiers-0.3.2.dev136 → verifiers-0.3.2.dev138}/verifiers/v1/clients/train.py +0 -0
- {verifiers-0.3.2.dev136 → verifiers-0.3.2.dev138}/verifiers/v1/configs/__init__.py +0 -0
- {verifiers-0.3.2.dev136 → verifiers-0.3.2.dev138}/verifiers/v1/configs/cli/__init__.py +0 -0
- {verifiers-0.3.2.dev136 → verifiers-0.3.2.dev138}/verifiers/v1/configs/cli/debug.py +0 -0
- {verifiers-0.3.2.dev136 → verifiers-0.3.2.dev138}/verifiers/v1/configs/cli/env.py +0 -0
- {verifiers-0.3.2.dev136 → verifiers-0.3.2.dev138}/verifiers/v1/configs/cli/eval.py +0 -0
- {verifiers-0.3.2.dev136 → verifiers-0.3.2.dev138}/verifiers/v1/configs/cli/init.py +0 -0
- {verifiers-0.3.2.dev136 → verifiers-0.3.2.dev138}/verifiers/v1/configs/cli/replay.py +0 -0
- {verifiers-0.3.2.dev136 → verifiers-0.3.2.dev138}/verifiers/v1/configs/cli/validate.py +0 -0
- {verifiers-0.3.2.dev136 → verifiers-0.3.2.dev138}/verifiers/v1/configs/client.py +0 -0
- {verifiers-0.3.2.dev136 → verifiers-0.3.2.dev138}/verifiers/v1/configs/harness.py +0 -0
- {verifiers-0.3.2.dev136 → verifiers-0.3.2.dev138}/verifiers/v1/configs/judge.py +0 -0
- {verifiers-0.3.2.dev136 → verifiers-0.3.2.dev138}/verifiers/v1/configs/retries.py +0 -0
- {verifiers-0.3.2.dev136 → verifiers-0.3.2.dev138}/verifiers/v1/configs/runtime.py +0 -0
- {verifiers-0.3.2.dev136 → verifiers-0.3.2.dev138}/verifiers/v1/configs/serve.py +0 -0
- {verifiers-0.3.2.dev136 → verifiers-0.3.2.dev138}/verifiers/v1/configs/task.py +0 -0
- {verifiers-0.3.2.dev136 → verifiers-0.3.2.dev138}/verifiers/v1/configs/taskset.py +0 -0
- {verifiers-0.3.2.dev136 → verifiers-0.3.2.dev138}/verifiers/v1/dialects/__init__.py +0 -0
- {verifiers-0.3.2.dev136 → verifiers-0.3.2.dev138}/verifiers/v1/dialects/anthropic.py +0 -0
- {verifiers-0.3.2.dev136 → verifiers-0.3.2.dev138}/verifiers/v1/dialects/base.py +0 -0
- {verifiers-0.3.2.dev136 → verifiers-0.3.2.dev138}/verifiers/v1/dialects/chat.py +0 -0
- {verifiers-0.3.2.dev136 → verifiers-0.3.2.dev138}/verifiers/v1/dialects/responses.py +0 -0
- {verifiers-0.3.2.dev136 → verifiers-0.3.2.dev138}/verifiers/v1/envs/__init__.py +0 -0
- {verifiers-0.3.2.dev136 → verifiers-0.3.2.dev138}/verifiers/v1/envs/agentic_judge/__init__.py +0 -0
- {verifiers-0.3.2.dev136 → verifiers-0.3.2.dev138}/verifiers/v1/envs/agentic_judge/env.py +0 -0
- {verifiers-0.3.2.dev136 → verifiers-0.3.2.dev138}/verifiers/v1/envs/best_of_n/__init__.py +0 -0
- {verifiers-0.3.2.dev136 → verifiers-0.3.2.dev138}/verifiers/v1/envs/best_of_n/env.py +0 -0
- {verifiers-0.3.2.dev136 → verifiers-0.3.2.dev138}/verifiers/v1/envs/isolated_verifier/__init__.py +0 -0
- {verifiers-0.3.2.dev136 → verifiers-0.3.2.dev138}/verifiers/v1/envs/isolated_verifier/env.py +0 -0
- {verifiers-0.3.2.dev136 → verifiers-0.3.2.dev138}/verifiers/v1/envs/shared_agentic_judge/__init__.py +0 -0
- {verifiers-0.3.2.dev136 → verifiers-0.3.2.dev138}/verifiers/v1/envs/single_agent/__init__.py +0 -0
- {verifiers-0.3.2.dev136 → verifiers-0.3.2.dev138}/verifiers/v1/envs/single_agent/env.py +0 -0
- {verifiers-0.3.2.dev136 → verifiers-0.3.2.dev138}/verifiers/v1/envs/user_sim/__init__.py +0 -0
- {verifiers-0.3.2.dev136 → verifiers-0.3.2.dev138}/verifiers/v1/envs/user_sim/env.py +0 -0
- {verifiers-0.3.2.dev136 → verifiers-0.3.2.dev138}/verifiers/v1/episode.py +0 -0
- {verifiers-0.3.2.dev136 → verifiers-0.3.2.dev138}/verifiers/v1/errors.py +0 -0
- {verifiers-0.3.2.dev136 → verifiers-0.3.2.dev138}/verifiers/v1/gepa/__init__.py +0 -0
- {verifiers-0.3.2.dev136 → verifiers-0.3.2.dev138}/verifiers/v1/gepa/adapter.py +0 -0
- {verifiers-0.3.2.dev136 → verifiers-0.3.2.dev138}/verifiers/v1/gepa/config.py +0 -0
- {verifiers-0.3.2.dev136 → verifiers-0.3.2.dev138}/verifiers/v1/gepa/dataset.py +0 -0
- {verifiers-0.3.2.dev136 → verifiers-0.3.2.dev138}/verifiers/v1/gepa/reflection.py +0 -0
- {verifiers-0.3.2.dev136 → verifiers-0.3.2.dev138}/verifiers/v1/graph.py +0 -0
- {verifiers-0.3.2.dev136 → verifiers-0.3.2.dev138}/verifiers/v1/harness.py +0 -0
- {verifiers-0.3.2.dev136 → verifiers-0.3.2.dev138}/verifiers/v1/harnesses/__init__.py +0 -0
- {verifiers-0.3.2.dev136 → verifiers-0.3.2.dev138}/verifiers/v1/harnesses/bash/__init__.py +0 -0
- {verifiers-0.3.2.dev136 → verifiers-0.3.2.dev138}/verifiers/v1/harnesses/bash/harness.py +0 -0
- {verifiers-0.3.2.dev136 → verifiers-0.3.2.dev138}/verifiers/v1/harnesses/browser_use/__init__.py +0 -0
- {verifiers-0.3.2.dev136 → verifiers-0.3.2.dev138}/verifiers/v1/harnesses/browser_use/harness.py +0 -0
- {verifiers-0.3.2.dev136 → verifiers-0.3.2.dev138}/verifiers/v1/harnesses/browser_use/program.py +0 -0
- {verifiers-0.3.2.dev136 → verifiers-0.3.2.dev138}/verifiers/v1/harnesses/claude_code/__init__.py +0 -0
- {verifiers-0.3.2.dev136 → verifiers-0.3.2.dev138}/verifiers/v1/harnesses/claude_code/harness.py +0 -0
- {verifiers-0.3.2.dev136 → verifiers-0.3.2.dev138}/verifiers/v1/harnesses/codex/__init__.py +0 -0
- {verifiers-0.3.2.dev136 → verifiers-0.3.2.dev138}/verifiers/v1/harnesses/codex/gate.mjs +0 -0
- {verifiers-0.3.2.dev136 → verifiers-0.3.2.dev138}/verifiers/v1/harnesses/codex/harness.py +0 -0
- {verifiers-0.3.2.dev136 → verifiers-0.3.2.dev138}/verifiers/v1/harnesses/hermes_agent/__init__.py +0 -0
- {verifiers-0.3.2.dev136 → verifiers-0.3.2.dev138}/verifiers/v1/harnesses/hermes_agent/harness.py +0 -0
- {verifiers-0.3.2.dev136 → verifiers-0.3.2.dev138}/verifiers/v1/harnesses/hermes_agent/program.py +0 -0
- {verifiers-0.3.2.dev136 → verifiers-0.3.2.dev138}/verifiers/v1/harnesses/kimi_code/__init__.py +0 -0
- {verifiers-0.3.2.dev136 → verifiers-0.3.2.dev138}/verifiers/v1/harnesses/kimi_code/harness.py +0 -0
- {verifiers-0.3.2.dev136 → verifiers-0.3.2.dev138}/verifiers/v1/harnesses/mini_swe_agent/__init__.py +0 -0
- {verifiers-0.3.2.dev136 → verifiers-0.3.2.dev138}/verifiers/v1/harnesses/mini_swe_agent/harness.py +0 -0
- {verifiers-0.3.2.dev136 → verifiers-0.3.2.dev138}/verifiers/v1/harnesses/mini_swe_agent/program.py +0 -0
- {verifiers-0.3.2.dev136 → verifiers-0.3.2.dev138}/verifiers/v1/harnesses/node.py +0 -0
- {verifiers-0.3.2.dev136 → verifiers-0.3.2.dev138}/verifiers/v1/harnesses/null/__init__.py +0 -0
- {verifiers-0.3.2.dev136 → verifiers-0.3.2.dev138}/verifiers/v1/harnesses/null/harness.py +0 -0
- {verifiers-0.3.2.dev136 → verifiers-0.3.2.dev138}/verifiers/v1/harnesses/openclaw/__init__.py +0 -0
- {verifiers-0.3.2.dev136 → verifiers-0.3.2.dev138}/verifiers/v1/harnesses/openclaw/harness.py +0 -0
- {verifiers-0.3.2.dev136 → verifiers-0.3.2.dev138}/verifiers/v1/harnesses/pi/__init__.py +0 -0
- {verifiers-0.3.2.dev136 → verifiers-0.3.2.dev138}/verifiers/v1/harnesses/pi/gate.mjs +0 -0
- {verifiers-0.3.2.dev136 → verifiers-0.3.2.dev138}/verifiers/v1/harnesses/pi/harness.py +0 -0
- {verifiers-0.3.2.dev136 → verifiers-0.3.2.dev138}/verifiers/v1/harnesses/prime_agent/__init__.py +0 -0
- {verifiers-0.3.2.dev136 → verifiers-0.3.2.dev138}/verifiers/v1/harnesses/prime_agent/harness.py +0 -0
- {verifiers-0.3.2.dev136 → verifiers-0.3.2.dev138}/verifiers/v1/harnesses/rlm/__init__.py +0 -0
- {verifiers-0.3.2.dev136 → verifiers-0.3.2.dev138}/verifiers/v1/harnesses/rlm/harness.py +0 -0
- {verifiers-0.3.2.dev136 → verifiers-0.3.2.dev138}/verifiers/v1/harnesses/terminus_2/__init__.py +0 -0
- {verifiers-0.3.2.dev136 → verifiers-0.3.2.dev138}/verifiers/v1/harnesses/terminus_2/harness.py +0 -0
- {verifiers-0.3.2.dev136 → verifiers-0.3.2.dev138}/verifiers/v1/harnesses/terminus_2/program.py +0 -0
- {verifiers-0.3.2.dev136 → verifiers-0.3.2.dev138}/verifiers/v1/harnesses/utils/__init__.py +0 -0
- {verifiers-0.3.2.dev136 → verifiers-0.3.2.dev138}/verifiers/v1/harnesses/utils/compaction.py +0 -0
- {verifiers-0.3.2.dev136 → verifiers-0.3.2.dev138}/verifiers/v1/harnesses/utils/install.py +0 -0
- {verifiers-0.3.2.dev136 → verifiers-0.3.2.dev138}/verifiers/v1/harnesses/utils/launch.py +0 -0
- {verifiers-0.3.2.dev136 → verifiers-0.3.2.dev138}/verifiers/v1/harnesses/utils/mcp.py +0 -0
- {verifiers-0.3.2.dev136 → verifiers-0.3.2.dev138}/verifiers/v1/interception/__init__.py +0 -0
- {verifiers-0.3.2.dev136 → verifiers-0.3.2.dev138}/verifiers/v1/interception/base.py +0 -0
- {verifiers-0.3.2.dev136 → verifiers-0.3.2.dev138}/verifiers/v1/interception/pool.py +0 -0
- {verifiers-0.3.2.dev136 → verifiers-0.3.2.dev138}/verifiers/v1/interception/server.py +0 -0
- {verifiers-0.3.2.dev136 → verifiers-0.3.2.dev138}/verifiers/v1/interception/tunnel/__init__.py +0 -0
- {verifiers-0.3.2.dev136 → verifiers-0.3.2.dev138}/verifiers/v1/interception/tunnel/base.py +0 -0
- {verifiers-0.3.2.dev136 → verifiers-0.3.2.dev138}/verifiers/v1/interception/tunnel/custom.py +0 -0
- {verifiers-0.3.2.dev136 → verifiers-0.3.2.dev138}/verifiers/v1/interception/tunnel/prime.py +0 -0
- {verifiers-0.3.2.dev136 → verifiers-0.3.2.dev138}/verifiers/v1/judge.py +0 -0
- {verifiers-0.3.2.dev136 → verifiers-0.3.2.dev138}/verifiers/v1/judges/__init__.py +0 -0
- {verifiers-0.3.2.dev136 → verifiers-0.3.2.dev138}/verifiers/v1/judges/reference.py +0 -0
- {verifiers-0.3.2.dev136 → verifiers-0.3.2.dev138}/verifiers/v1/judges/reference.txt +0 -0
- {verifiers-0.3.2.dev136 → verifiers-0.3.2.dev138}/verifiers/v1/judges/rubric.py +0 -0
- {verifiers-0.3.2.dev136 → verifiers-0.3.2.dev138}/verifiers/v1/judges/rubric.txt +0 -0
- {verifiers-0.3.2.dev136 → verifiers-0.3.2.dev138}/verifiers/v1/mcp/__init__.py +0 -0
- {verifiers-0.3.2.dev136 → verifiers-0.3.2.dev138}/verifiers/v1/mcp/launch.py +0 -0
- {verifiers-0.3.2.dev136 → verifiers-0.3.2.dev138}/verifiers/v1/mcp/server.py +0 -0
- {verifiers-0.3.2.dev136 → verifiers-0.3.2.dev138}/verifiers/v1/mcp/toolset.py +0 -0
- {verifiers-0.3.2.dev136 → verifiers-0.3.2.dev138}/verifiers/v1/rollout.py +0 -0
- {verifiers-0.3.2.dev136 → verifiers-0.3.2.dev138}/verifiers/v1/runtimes/__init__.py +0 -0
- {verifiers-0.3.2.dev136 → verifiers-0.3.2.dev138}/verifiers/v1/runtimes/apptainer.py +0 -0
- {verifiers-0.3.2.dev136 → verifiers-0.3.2.dev138}/verifiers/v1/runtimes/base.py +0 -0
- {verifiers-0.3.2.dev136 → verifiers-0.3.2.dev138}/verifiers/v1/runtimes/container.py +0 -0
- {verifiers-0.3.2.dev136 → verifiers-0.3.2.dev138}/verifiers/v1/runtimes/docker/egress.py +0 -0
- {verifiers-0.3.2.dev136 → verifiers-0.3.2.dev138}/verifiers/v1/runtimes/limiters.py +0 -0
- {verifiers-0.3.2.dev136 → verifiers-0.3.2.dev138}/verifiers/v1/runtimes/modal.py +0 -0
- {verifiers-0.3.2.dev136 → verifiers-0.3.2.dev138}/verifiers/v1/runtimes/subprocess.py +0 -0
- {verifiers-0.3.2.dev136 → verifiers-0.3.2.dev138}/verifiers/v1/semantic.py +0 -0
- {verifiers-0.3.2.dev136 → verifiers-0.3.2.dev138}/verifiers/v1/serve/__init__.py +0 -0
- {verifiers-0.3.2.dev136 → verifiers-0.3.2.dev138}/verifiers/v1/serve/client.py +0 -0
- {verifiers-0.3.2.dev136 → verifiers-0.3.2.dev138}/verifiers/v1/serve/encoding.py +0 -0
- {verifiers-0.3.2.dev136 → verifiers-0.3.2.dev138}/verifiers/v1/serve/pool.py +0 -0
- {verifiers-0.3.2.dev136 → verifiers-0.3.2.dev138}/verifiers/v1/serve/types.py +0 -0
- {verifiers-0.3.2.dev136 → verifiers-0.3.2.dev138}/verifiers/v1/session.py +0 -0
- {verifiers-0.3.2.dev136 → verifiers-0.3.2.dev138}/verifiers/v1/state.py +0 -0
- {verifiers-0.3.2.dev136 → verifiers-0.3.2.dev138}/verifiers/v1/task.py +0 -0
- {verifiers-0.3.2.dev136 → verifiers-0.3.2.dev138}/verifiers/v1/taskset.py +0 -0
- {verifiers-0.3.2.dev136 → verifiers-0.3.2.dev138}/verifiers/v1/tasksets/__init__.py +0 -0
- {verifiers-0.3.2.dev136 → verifiers-0.3.2.dev138}/verifiers/v1/tasksets/harbor/__init__.py +0 -0
- {verifiers-0.3.2.dev136 → verifiers-0.3.2.dev138}/verifiers/v1/tasksets/harbor/env.py +0 -0
- {verifiers-0.3.2.dev136 → verifiers-0.3.2.dev138}/verifiers/v1/tasksets/harbor/taskset.py +0 -0
- {verifiers-0.3.2.dev136 → verifiers-0.3.2.dev138}/verifiers/v1/tasksets/harbor/toolset.py +0 -0
- {verifiers-0.3.2.dev136 → verifiers-0.3.2.dev138}/verifiers/v1/tasksets/nemo_gym/__init__.py +0 -0
- {verifiers-0.3.2.dev136 → verifiers-0.3.2.dev138}/verifiers/v1/tasksets/nemo_gym/response.py +0 -0
- {verifiers-0.3.2.dev136 → verifiers-0.3.2.dev138}/verifiers/v1/tasksets/nemo_gym/server.py +0 -0
- {verifiers-0.3.2.dev136 → verifiers-0.3.2.dev138}/verifiers/v1/tasksets/nemo_gym/taskset.py +0 -0
- {verifiers-0.3.2.dev136 → verifiers-0.3.2.dev138}/verifiers/v1/tasksets/nemo_gym/toolset.py +0 -0
- {verifiers-0.3.2.dev136 → verifiers-0.3.2.dev138}/verifiers/v1/tasksets/openenv/__init__.py +0 -0
- {verifiers-0.3.2.dev136 → verifiers-0.3.2.dev138}/verifiers/v1/tasksets/openenv/taskset.py +0 -0
- {verifiers-0.3.2.dev136 → verifiers-0.3.2.dev138}/verifiers/v1/tasksets/textarena/__init__.py +0 -0
- {verifiers-0.3.2.dev136 → verifiers-0.3.2.dev138}/verifiers/v1/tasksets/textarena/taskset.py +0 -0
- {verifiers-0.3.2.dev136 → verifiers-0.3.2.dev138}/verifiers/v1/trace.py +0 -0
- {verifiers-0.3.2.dev136 → verifiers-0.3.2.dev138}/verifiers/v1/types.py +0 -0
- {verifiers-0.3.2.dev136 → verifiers-0.3.2.dev138}/verifiers/v1/utils/__init__.py +0 -0
- {verifiers-0.3.2.dev136 → verifiers-0.3.2.dev138}/verifiers/v1/utils/aio.py +0 -0
- {verifiers-0.3.2.dev136 → verifiers-0.3.2.dev138}/verifiers/v1/utils/artifacts.py +0 -0
- {verifiers-0.3.2.dev136 → verifiers-0.3.2.dev138}/verifiers/v1/utils/compile.py +0 -0
- {verifiers-0.3.2.dev136 → verifiers-0.3.2.dev138}/verifiers/v1/utils/decorators.py +0 -0
- {verifiers-0.3.2.dev136 → verifiers-0.3.2.dev138}/verifiers/v1/utils/format.py +0 -0
- {verifiers-0.3.2.dev136 → verifiers-0.3.2.dev138}/verifiers/v1/utils/generic.py +0 -0
- {verifiers-0.3.2.dev136 → verifiers-0.3.2.dev138}/verifiers/v1/utils/git.py +0 -0
- {verifiers-0.3.2.dev136 → verifiers-0.3.2.dev138}/verifiers/v1/utils/image.py +0 -0
- {verifiers-0.3.2.dev136 → verifiers-0.3.2.dev138}/verifiers/v1/utils/interrupt.py +0 -0
- {verifiers-0.3.2.dev136 → verifiers-0.3.2.dev138}/verifiers/v1/utils/loaders.py +0 -0
- {verifiers-0.3.2.dev136 → verifiers-0.3.2.dev138}/verifiers/v1/utils/logging.py +0 -0
- {verifiers-0.3.2.dev136 → verifiers-0.3.2.dev138}/verifiers/v1/utils/memory.py +0 -0
- {verifiers-0.3.2.dev136 → verifiers-0.3.2.dev138}/verifiers/v1/utils/paths.py +0 -0
- {verifiers-0.3.2.dev136 → verifiers-0.3.2.dev138}/verifiers/v1/utils/platform.py +0 -0
- {verifiers-0.3.2.dev136 → verifiers-0.3.2.dev138}/verifiers/v1/utils/prime.py +0 -0
- {verifiers-0.3.2.dev136 → verifiers-0.3.2.dev138}/verifiers/v1/utils/retries.py +0 -0
- {verifiers-0.3.2.dev136 → verifiers-0.3.2.dev138}/verifiers/v1/utils/scope.py +0 -0
- {verifiers-0.3.2.dev136 → verifiers-0.3.2.dev138}/verifiers/v1/utils/score.py +0 -0
- {verifiers-0.3.2.dev136 → verifiers-0.3.2.dev138}/verifiers/v1/utils/version.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.5
|
|
2
2
|
Name: verifiers
|
|
3
|
-
Version: 0.3.2.
|
|
3
|
+
Version: 0.3.2.dev138
|
|
4
4
|
Summary: Verifiers: Environments for LLM Reinforcement Learning
|
|
5
5
|
Project-URL: Homepage, https://github.com/primeintellect-ai/verifiers
|
|
6
6
|
Project-URL: Documentation, https://github.com/primeintellect-ai/verifiers
|
|
@@ -740,8 +740,8 @@ async def test_env_id_user_sim(run_v1, tmp_path):
|
|
|
740
740
|
# both sides land as ONE durable episode.
|
|
741
741
|
assert assistant.task.data.prompt is None
|
|
742
742
|
assert "echoed" in assistant.rewards
|
|
743
|
-
from verifiers.v1.cli.output import read_episodes
|
|
744
743
|
from verifiers.v1.trace import WireTrace
|
|
744
|
+
from verifiers.v1.utils.trace_store import read_episodes
|
|
745
745
|
|
|
746
746
|
(record,) = read_episodes(tmp_path, WireTrace)
|
|
747
747
|
assert {t.agent.name for t in record.traces} == {"assistant", "user"}
|
|
@@ -28,11 +28,6 @@ class MyTask(vf.TaskData):
|
|
|
28
28
|
answer: str
|
|
29
29
|
|
|
30
30
|
|
|
31
|
-
class Slot:
|
|
32
|
-
def __init__(self) -> None:
|
|
33
|
-
self.traces: list[vf.Trace] = []
|
|
34
|
-
|
|
35
|
-
|
|
36
31
|
async def settle() -> None:
|
|
37
32
|
"""Let a notified change reach the wire: the flush is a callback that starts a task."""
|
|
38
33
|
for _ in range(3):
|
|
@@ -86,18 +81,18 @@ def test_delta_fields_cover_every_serialized_trace_field():
|
|
|
86
81
|
async def test_failed_send_is_diffed_again():
|
|
87
82
|
"""A delta the wire refused is not lost: its cursor stays and the next flush
|
|
88
83
|
carries the same content, so the client still assembles the whole trace."""
|
|
89
|
-
|
|
84
|
+
traces: list[vf.Trace] = []
|
|
90
85
|
frames: list[bytes] = []
|
|
91
86
|
fail = {"on": False}
|
|
92
87
|
|
|
93
|
-
async def send(data:
|
|
88
|
+
async def send(data: dict) -> None:
|
|
94
89
|
if fail["on"]:
|
|
95
90
|
raise OSError("host unreachable")
|
|
96
|
-
frames.append(data)
|
|
91
|
+
frames.append(pack(data))
|
|
97
92
|
|
|
98
|
-
async with DeltaStreamer(
|
|
93
|
+
async with DeltaStreamer(lambda: traces, send) as streamer:
|
|
99
94
|
trace = make_trace()
|
|
100
|
-
|
|
95
|
+
traces.append(trace)
|
|
101
96
|
streamer.watch(trace)
|
|
102
97
|
await settle()
|
|
103
98
|
fail["on"] = True
|
|
@@ -106,7 +101,7 @@ async def test_failed_send_is_diffed_again():
|
|
|
106
101
|
fail["on"] = False
|
|
107
102
|
add_turn(trace, "a2")
|
|
108
103
|
await settle()
|
|
109
|
-
|
|
104
|
+
traces = [trace]
|
|
110
105
|
assembly = EpisodeAssembly()
|
|
111
106
|
for frame in frames:
|
|
112
107
|
assembly.apply(unpack(frame))
|
|
@@ -116,15 +111,15 @@ async def test_failed_send_is_diffed_again():
|
|
|
116
111
|
|
|
117
112
|
@pytest.mark.asyncio
|
|
118
113
|
async def test_pending_preview_streams_and_clears_on_commit():
|
|
119
|
-
|
|
114
|
+
traces: list[vf.Trace] = []
|
|
120
115
|
frames: list[bytes] = []
|
|
121
116
|
|
|
122
|
-
async def send(data:
|
|
123
|
-
frames.append(data)
|
|
117
|
+
async def send(data: dict) -> None:
|
|
118
|
+
frames.append(pack(data))
|
|
124
119
|
|
|
125
|
-
async with DeltaStreamer(
|
|
120
|
+
async with DeltaStreamer(lambda: traces, send) as streamer:
|
|
126
121
|
trace = make_trace()
|
|
127
|
-
|
|
122
|
+
traces.append(trace)
|
|
128
123
|
streamer.watch(trace)
|
|
129
124
|
add_turn(trace, "a1")
|
|
130
125
|
await settle()
|
|
@@ -161,7 +156,7 @@ async def test_pending_preview_streams_and_clears_on_commit():
|
|
|
161
156
|
episode = Episode(
|
|
162
157
|
env=EnvInfo(id="my-env"), task=trace.task, ok=True, traces=[trace]
|
|
163
158
|
)
|
|
164
|
-
|
|
159
|
+
traces = list(episode.traces)
|
|
165
160
|
assembly = EpisodeAssembly()
|
|
166
161
|
for frame in frames:
|
|
167
162
|
assembly.apply(unpack(frame))
|
|
@@ -179,15 +174,15 @@ async def test_pending_preview_streams_and_clears_on_commit():
|
|
|
179
174
|
@pytest.mark.asyncio
|
|
180
175
|
@pytest.mark.parametrize("routing_dtype", [np.uint8, np.uint16])
|
|
181
176
|
async def test_deltas_stream_once_and_reassemble_the_episode(routing_dtype):
|
|
182
|
-
|
|
177
|
+
traces: list[vf.Trace] = []
|
|
183
178
|
frames: list[bytes] = []
|
|
184
179
|
|
|
185
|
-
async def send(data:
|
|
186
|
-
frames.append(data)
|
|
180
|
+
async def send(data: dict) -> None:
|
|
181
|
+
frames.append(pack(data))
|
|
187
182
|
|
|
188
|
-
async with DeltaStreamer(
|
|
183
|
+
async with DeltaStreamer(lambda: traces, send) as streamer:
|
|
189
184
|
trace = make_trace()
|
|
190
|
-
|
|
185
|
+
traces.append(trace)
|
|
191
186
|
streamer.watch(trace)
|
|
192
187
|
trace.timing.boot.start = 1.0
|
|
193
188
|
trace.notify()
|
|
@@ -211,7 +206,7 @@ async def test_deltas_stream_once_and_reassemble_the_episode(routing_dtype):
|
|
|
211
206
|
episode = Episode(
|
|
212
207
|
env=EnvInfo(id="my-env"), task=trace.task, ok=True, traces=[trace]
|
|
213
208
|
)
|
|
214
|
-
|
|
209
|
+
traces = list(episode.traces)
|
|
215
210
|
# The exit flushed the tail (reward, stop, ok) without re-sending any node.
|
|
216
211
|
deltas = [unpack(frame) for frame in frames]
|
|
217
212
|
sent_nodes = sum(len(delta.get("nodes", [])) for delta in deltas)
|
|
@@ -247,21 +242,21 @@ async def test_deltas_stream_once_and_reassemble_the_episode(routing_dtype):
|
|
|
247
242
|
|
|
248
243
|
@pytest.mark.asyncio
|
|
249
244
|
async def test_discarded_attempt_drops_its_trace():
|
|
250
|
-
|
|
245
|
+
traces: list[vf.Trace] = []
|
|
251
246
|
frames: list[bytes] = []
|
|
252
247
|
|
|
253
|
-
async def send(data:
|
|
254
|
-
frames.append(data)
|
|
248
|
+
async def send(data: dict) -> None:
|
|
249
|
+
frames.append(pack(data))
|
|
255
250
|
|
|
256
|
-
async with DeltaStreamer(
|
|
251
|
+
async with DeltaStreamer(lambda: traces, send) as streamer:
|
|
257
252
|
first = make_trace()
|
|
258
|
-
|
|
253
|
+
traces.append(first)
|
|
259
254
|
streamer.watch(first)
|
|
260
255
|
add_turn(first, "a1")
|
|
261
256
|
await settle()
|
|
262
|
-
|
|
257
|
+
traces = [] # the attempt is retried
|
|
263
258
|
second = make_trace()
|
|
264
|
-
|
|
259
|
+
traces.append(second)
|
|
265
260
|
streamer.watch(second)
|
|
266
261
|
await settle()
|
|
267
262
|
deltas = [unpack(frame) for frame in frames]
|
|
@@ -10,7 +10,6 @@ import pytest
|
|
|
10
10
|
|
|
11
11
|
import verifiers.v1 as vf
|
|
12
12
|
from verifiers.v1.agent import Interaction
|
|
13
|
-
from verifiers.v1.cli.output import write_episode
|
|
14
13
|
from verifiers.v1.dialects.chat import ChatDialect
|
|
15
14
|
from verifiers.v1.dialects.responses import ResponsesDialect, fold_assistant
|
|
16
15
|
from verifiers.v1.graph import MessageNode, prepare_turn
|
|
@@ -26,6 +25,7 @@ from verifiers.v1.semantic import (
|
|
|
26
25
|
extract_acp_info,
|
|
27
26
|
)
|
|
28
27
|
from verifiers.v1.types import AssistantMessage, UserMessage
|
|
28
|
+
from verifiers.v1.utils.trace_store import write_episode
|
|
29
29
|
|
|
30
30
|
|
|
31
31
|
class MyTask(vf.TaskData):
|
|
@@ -12,13 +12,15 @@ import logging
|
|
|
12
12
|
from collections.abc import AsyncIterator, Callable, Mapping
|
|
13
13
|
from contextlib import asynccontextmanager, nullcontext
|
|
14
14
|
from dataclasses import dataclass, replace
|
|
15
|
-
from typing import Self
|
|
15
|
+
from typing import Generic, Self, cast
|
|
16
|
+
|
|
17
|
+
from typing_extensions import TypeVar
|
|
16
18
|
|
|
17
19
|
from verifiers.v1.clients import (
|
|
18
20
|
EvalClientConfig,
|
|
19
21
|
ModelContext,
|
|
20
22
|
)
|
|
21
|
-
from verifiers.v1.configs.agent import AgentConfig, TimeoutConfig
|
|
23
|
+
from verifiers.v1.configs.agent import AgentConfig, TimeoutConfig, agent_config_fields
|
|
22
24
|
from verifiers.v1.configs.runtime import NetworkPolicyConfig
|
|
23
25
|
from verifiers.v1.dialects import parse_message
|
|
24
26
|
from verifiers.v1.harness import Harness
|
|
@@ -42,6 +44,7 @@ from verifiers.v1.types import (
|
|
|
42
44
|
ToolMessage,
|
|
43
45
|
UserMessage,
|
|
44
46
|
)
|
|
47
|
+
from verifiers.v1.utils.aio import run_shielded
|
|
45
48
|
from verifiers.v1.utils.compile import (
|
|
46
49
|
cap_remote_agent_timeout,
|
|
47
50
|
resolve_runtime_config,
|
|
@@ -462,9 +465,8 @@ class Agent:
|
|
|
462
465
|
run.trace.stop("agent_completed")
|
|
463
466
|
trace = await run.close()
|
|
464
467
|
except BaseException:
|
|
465
|
-
#
|
|
466
|
-
|
|
467
|
-
await run.abort()
|
|
468
|
+
# Finish cleanup even if another cancellation arrives during abort().
|
|
469
|
+
await run_shielded(run.abort())
|
|
468
470
|
raise
|
|
469
471
|
if trace.agent.runtime is not None:
|
|
470
472
|
trace.agent.runtime.borrowed = runtime is not None
|
|
@@ -720,27 +722,22 @@ def make_agent(
|
|
|
720
722
|
return Agent(config, interception=interception)
|
|
721
723
|
|
|
722
724
|
|
|
723
|
-
|
|
725
|
+
AgentT = TypeVar("AgentT", bound=Agent, default=Agent)
|
|
726
|
+
MakeAgent = Callable[[str, AgentConfig], AgentT]
|
|
724
727
|
"""An agent factory keyed by name — what `Agents` calls per scraped config field."""
|
|
725
728
|
|
|
726
729
|
|
|
727
|
-
|
|
728
|
-
"""The top-level `AgentConfig` fields declared on a config, in declaration
|
|
729
|
-
order — the env's agents, keyed by field name (the only naming site)."""
|
|
730
|
-
return {name: value for name, value in config if isinstance(value, AgentConfig)}
|
|
731
|
-
|
|
732
|
-
|
|
733
|
-
class Agents:
|
|
730
|
+
class Agents(Generic[AgentT]):
|
|
734
731
|
"""A config's agents, addressed by attribute: every top-level `AgentConfig`
|
|
735
732
|
field becomes an `Agent` under the field's name (`agents.solver`)."""
|
|
736
733
|
|
|
737
|
-
def __init__(self, config, make: MakeAgent | None = None) -> None:
|
|
738
|
-
self._agents: dict[str,
|
|
739
|
-
name: make_agent(value) if make is None else make(name, value)
|
|
734
|
+
def __init__(self, config, make: MakeAgent[AgentT] | None = None) -> None:
|
|
735
|
+
self._agents: dict[str, AgentT] = {
|
|
736
|
+
name: cast(AgentT, make_agent(value)) if make is None else make(name, value)
|
|
740
737
|
for name, value in agent_config_fields(config).items()
|
|
741
738
|
}
|
|
742
739
|
|
|
743
|
-
def __getattr__(self, name: str) ->
|
|
740
|
+
def __getattr__(self, name: str) -> AgentT:
|
|
744
741
|
# self.__dict__ directly: attribute lookup re-entering __getattr__ before
|
|
745
742
|
# __init__ ran (copy/unpickle) must raise, not recurse.
|
|
746
743
|
agents = self.__dict__.get("_agents")
|
|
@@ -81,9 +81,9 @@ def format_cost_usd(cost_usd: float) -> str:
|
|
|
81
81
|
def _seat_value(config: EvalConfig, read):
|
|
82
82
|
"""A cap as the overview shows it: the declared seats' shared value, the
|
|
83
83
|
string 'per-seat' when they disagree (caps live on the seats)."""
|
|
84
|
-
from verifiers.v1.configs.
|
|
84
|
+
from verifiers.v1.configs.agent import agent_config_fields
|
|
85
85
|
|
|
86
|
-
values = {read(spec) for spec in
|
|
86
|
+
values = {read(spec) for spec in agent_config_fields(config.env).values()}
|
|
87
87
|
return values.pop() if len(values) == 1 else "per-seat"
|
|
88
88
|
|
|
89
89
|
|
|
@@ -17,7 +17,7 @@ from uuid import uuid4
|
|
|
17
17
|
from pydantic_config import cli
|
|
18
18
|
|
|
19
19
|
import verifiers.v1 as vf
|
|
20
|
-
from verifiers.v1.cli.output import
|
|
20
|
+
from verifiers.v1.cli.output import save_config
|
|
21
21
|
from verifiers.v1.cli.resolve import (
|
|
22
22
|
extract_id,
|
|
23
23
|
narrow_taskset_config,
|
|
@@ -34,6 +34,7 @@ from verifiers.v1.utils.compile import resolve_runtime_config
|
|
|
34
34
|
from verifiers.v1.utils.decorators import invoke
|
|
35
35
|
from verifiers.v1.utils.interrupt import install_interrupt
|
|
36
36
|
from verifiers.v1.utils.logging import setup_logging
|
|
37
|
+
from verifiers.v1.utils.trace_store import append_trace
|
|
37
38
|
|
|
38
39
|
logger = logging.getLogger(__name__)
|
|
39
40
|
|
|
@@ -10,7 +10,6 @@ from pydantic_config import cli
|
|
|
10
10
|
|
|
11
11
|
from verifiers.v1.cli.eval.runner import run_eval
|
|
12
12
|
from verifiers.v1.cli.output import (
|
|
13
|
-
TRACES_FILE,
|
|
14
13
|
create_attempt_log_dir,
|
|
15
14
|
output_path,
|
|
16
15
|
saved_config_path,
|
|
@@ -26,6 +25,7 @@ from verifiers.v1.cli.resolve import (
|
|
|
26
25
|
from verifiers.v1.configs.cli.eval import EvalConfig
|
|
27
26
|
from verifiers.v1.utils.interrupt import install_interrupt
|
|
28
27
|
from verifiers.v1.utils.logging import setup_logging
|
|
28
|
+
from verifiers.v1.utils.trace_store import TRACES_FILE
|
|
29
29
|
|
|
30
30
|
logger = logging.getLogger(__name__)
|
|
31
31
|
|
|
@@ -16,9 +16,9 @@ from pathlib import Path
|
|
|
16
16
|
|
|
17
17
|
from pydantic_core import from_json
|
|
18
18
|
|
|
19
|
-
from verifiers.v1.cli.output import TRACES_FILE
|
|
20
19
|
from verifiers.v1.episode import WireEpisode
|
|
21
20
|
from verifiers.v1.task import task_key
|
|
21
|
+
from verifiers.v1.utils.trace_store import TRACES_FILE
|
|
22
22
|
|
|
23
23
|
|
|
24
24
|
def load(
|
|
@@ -17,11 +17,7 @@ from typing import TypeVar, cast
|
|
|
17
17
|
from verifiers.v1.cli.dashboard import dashboard
|
|
18
18
|
from verifiers.v1.cli.eval import resume
|
|
19
19
|
from verifiers.v1.cli.eval.hint import PRIME_RL_HINT
|
|
20
|
-
from verifiers.v1.cli.output import
|
|
21
|
-
append_episode,
|
|
22
|
-
output_path,
|
|
23
|
-
save_config,
|
|
24
|
-
)
|
|
20
|
+
from verifiers.v1.cli.output import output_path, save_config
|
|
25
21
|
from verifiers.v1.cli.resume import distribute
|
|
26
22
|
from verifiers.v1.clients import ModelContext
|
|
27
23
|
from verifiers.v1.configs.cli.eval import EvalConfig
|
|
@@ -35,6 +31,7 @@ from verifiers.v1.utils.platform import (
|
|
|
35
31
|
log_episodes,
|
|
36
32
|
open_run,
|
|
37
33
|
)
|
|
34
|
+
from verifiers.v1.utils.trace_store import append_episode
|
|
38
35
|
|
|
39
36
|
logger = logging.getLogger(__name__)
|
|
40
37
|
|
|
@@ -16,7 +16,7 @@ import uuid
|
|
|
16
16
|
from pydantic_config import cli
|
|
17
17
|
|
|
18
18
|
import verifiers.v1 as vf
|
|
19
|
-
from verifiers.v1.cli.output import
|
|
19
|
+
from verifiers.v1.cli.output import output_path, write_config
|
|
20
20
|
from verifiers.v1.cli.resolve import (
|
|
21
21
|
extract_id,
|
|
22
22
|
narrow_config,
|
|
@@ -27,6 +27,7 @@ from verifiers.v1.cli.resolve import (
|
|
|
27
27
|
from verifiers.v1.gepa import GEPAConfig, run_gepa
|
|
28
28
|
from verifiers.v1.utils.interrupt import install_interrupt
|
|
29
29
|
from verifiers.v1.utils.logging import setup_logging
|
|
30
|
+
from verifiers.v1.utils.trace_store import TRACES_FILE
|
|
30
31
|
|
|
31
32
|
logger = logging.getLogger(__name__)
|
|
32
33
|
|
|
@@ -1,31 +1,13 @@
|
|
|
1
|
-
"""
|
|
2
|
-
|
|
3
|
-
Each line is an `Episode` — the episode's standing (`id`/`env`/`errors`) inlined
|
|
4
|
-
next to its flat, self-contained traces — so an episode persists whole or not at all: a torn line is the
|
|
5
|
-
whole episode owed on resume, and a failure before any trace minted still leaves
|
|
6
|
-
its errors on disk. The JSON file is the run's resolved config in the format the
|
|
7
|
-
CLI reads (`@ configs/<cli>.json`), so a run is re-runnable from its own output. Lines
|
|
8
|
-
append as episodes complete, so results are durable mid-run. Files written
|
|
9
|
-
by this surface contain episodes only.
|
|
10
|
-
"""
|
|
11
|
-
|
|
12
|
-
import asyncio
|
|
1
|
+
"""CLI output directories, saved launch/config files, and per-attempt log paths."""
|
|
2
|
+
|
|
13
3
|
import json
|
|
14
4
|
import os
|
|
15
|
-
from functools import cache
|
|
16
5
|
from pathlib import Path
|
|
17
6
|
|
|
18
|
-
from pydantic import BaseModel
|
|
7
|
+
from pydantic import BaseModel
|
|
19
8
|
|
|
20
9
|
from verifiers.v1.configs.cli.eval import EvalConfig
|
|
21
|
-
from verifiers.v1.
|
|
22
|
-
from verifiers.v1.state import StateT
|
|
23
|
-
from verifiers.v1.task import DataT
|
|
24
|
-
from verifiers.v1.trace import AgentConfigT, Trace
|
|
25
|
-
from verifiers.v1.utils.aio import run_shielded
|
|
26
|
-
|
|
27
|
-
TRACES_FILE = "traces.jsonl"
|
|
28
|
-
"""Filename a run's rollout episodes are written to (one JSON episode per line)."""
|
|
10
|
+
from verifiers.v1.utils.trace_store import TRACES_FILE
|
|
29
11
|
|
|
30
12
|
CONFIG_DIR = "configs"
|
|
31
13
|
"""Directory inside a run dir holding its configs: the launch TOML copied verbatim to
|
|
@@ -78,10 +60,6 @@ def saved_config_path(run_dir: Path) -> Path | None:
|
|
|
78
60
|
return candidates[0] if candidates else None
|
|
79
61
|
|
|
80
62
|
|
|
81
|
-
# Compiling an adapter is the expensive part; run output reuses only a few model classes.
|
|
82
|
-
type_adapter = cache(TypeAdapter)
|
|
83
|
-
|
|
84
|
-
|
|
85
63
|
def output_path(config: EvalConfig) -> Path:
|
|
86
64
|
"""Where this run writes: `output_dir / run.dir` — the same grouping convention as
|
|
87
65
|
training. The run directory defaults to the auto-generated run name
|
|
@@ -142,64 +120,3 @@ def save_config(
|
|
|
142
120
|
(results_dir / TRACES_FILE).write_text(
|
|
143
121
|
""
|
|
144
122
|
) # fresh; appended to as rollouts complete
|
|
145
|
-
|
|
146
|
-
|
|
147
|
-
def write_episode(
|
|
148
|
-
results_dir: Path, episode: Episode[DataT, StateT, AgentConfigT]
|
|
149
|
-
) -> None:
|
|
150
|
-
"""Serialize and append one rollout episode in the worker thread."""
|
|
151
|
-
# Preserve fields declared by typed Trace subclasses nested in the episode.
|
|
152
|
-
data = type_adapter(type(episode)).dump_json(episode, exclude_none=True)
|
|
153
|
-
with (results_dir / TRACES_FILE).open("ab") as f:
|
|
154
|
-
f.write(data + b"\n")
|
|
155
|
-
|
|
156
|
-
|
|
157
|
-
def read_episodes(results_dir: Path, trace_type: type) -> list[WireEpisode]:
|
|
158
|
-
"""Load a run's saved rollouts from `traces.jsonl` with traces typed as
|
|
159
|
-
`trace_type` (`Trace[WireTaskData, ...]` reads any taskset's file without
|
|
160
|
-
importing it)."""
|
|
161
|
-
trace_adapter = type_adapter(trace_type)
|
|
162
|
-
episodes: list[WireEpisode] = []
|
|
163
|
-
with (results_dir / TRACES_FILE).open(encoding="utf-8") as f:
|
|
164
|
-
for line in f:
|
|
165
|
-
if not line.strip():
|
|
166
|
-
continue
|
|
167
|
-
row = json.loads(line)
|
|
168
|
-
record = WireEpisode.model_validate({**row, "traces": []})
|
|
169
|
-
record.traces = [
|
|
170
|
-
trace_adapter.validate_python(trace) for trace in row["traces"]
|
|
171
|
-
]
|
|
172
|
-
episodes.append(record)
|
|
173
|
-
return episodes
|
|
174
|
-
|
|
175
|
-
|
|
176
|
-
async def append_episode(
|
|
177
|
-
results_dir: Path,
|
|
178
|
-
episode: Episode[DataT, StateT, AgentConfigT],
|
|
179
|
-
lock: asyncio.Lock,
|
|
180
|
-
) -> None:
|
|
181
|
-
"""Append one finished rollout episode without blocking the event loop. The run's
|
|
182
|
-
shared lock preserves whole-line ordering, and awaiting the worker preserves
|
|
183
|
-
per-episode durability."""
|
|
184
|
-
|
|
185
|
-
async def persist() -> None:
|
|
186
|
-
async with lock:
|
|
187
|
-
await asyncio.to_thread(write_episode, results_dir, episode)
|
|
188
|
-
|
|
189
|
-
# Run lock acquisition and the worker to completion even under cancellation, so
|
|
190
|
-
# finalized episodes are never lost mid-write (`run_shielded` re-raises the cancellation).
|
|
191
|
-
await run_shielded(persist())
|
|
192
|
-
|
|
193
|
-
|
|
194
|
-
async def append_trace(
|
|
195
|
-
results_dir: Path, trace: Trace, lock: asyncio.Lock, env: str = ""
|
|
196
|
-
) -> None:
|
|
197
|
-
"""Append one finished trace as a single-agent rollout episode — debug and replay,
|
|
198
|
-
which complete trace-at-a-time, both go through here."""
|
|
199
|
-
episode = Episode(
|
|
200
|
-
env=EnvInfo(id=env),
|
|
201
|
-
task=trace.task,
|
|
202
|
-
traces=[trace],
|
|
203
|
-
ok=trace.ok,
|
|
204
|
-
)
|
|
205
|
-
await append_episode(results_dir, episode, lock)
|
|
@@ -22,13 +22,7 @@ from pydantic_config import cli
|
|
|
22
22
|
|
|
23
23
|
import verifiers.v1 as vf
|
|
24
24
|
from verifiers.v1.cli.dashboard.replay import ReplayProgress, replay_dashboard
|
|
25
|
-
from verifiers.v1.cli.output import
|
|
26
|
-
append_trace,
|
|
27
|
-
read_episodes,
|
|
28
|
-
save_config,
|
|
29
|
-
saved_config_path,
|
|
30
|
-
write_config,
|
|
31
|
-
)
|
|
25
|
+
from verifiers.v1.cli.output import save_config, saved_config_path, write_config
|
|
32
26
|
from verifiers.v1.cli.resolve import narrow_taskset_config
|
|
33
27
|
from verifiers.v1.configs.agent import WireAgentConfig
|
|
34
28
|
from verifiers.v1.configs.cli.replay import ReplayConfig
|
|
@@ -37,6 +31,7 @@ from verifiers.v1.task import Task, WireTaskData
|
|
|
37
31
|
from verifiers.v1.trace import Trace
|
|
38
32
|
from verifiers.v1.utils.interrupt import install_interrupt
|
|
39
33
|
from verifiers.v1.utils.logging import setup_logging
|
|
34
|
+
from verifiers.v1.utils.trace_store import append_trace, read_episodes
|
|
40
35
|
|
|
41
36
|
logger = logging.getLogger(__name__)
|
|
42
37
|
|
|
@@ -1,6 +1,8 @@
|
|
|
1
1
|
"""One env agent's config: who plays the seat, and its per-run caps."""
|
|
2
2
|
|
|
3
|
-
from
|
|
3
|
+
from typing import Any
|
|
4
|
+
|
|
5
|
+
from pydantic import BaseModel, SerializeAsAny, model_validator
|
|
4
6
|
from pydantic_config import BaseConfig
|
|
5
7
|
|
|
6
8
|
from verifiers.v1.clients import ClientConfig
|
|
@@ -8,6 +10,7 @@ from verifiers.v1.configs.harness import HarnessConfig, WireHarnessConfig
|
|
|
8
10
|
from verifiers.v1.configs.retries import RetryConfig
|
|
9
11
|
from verifiers.v1.runtimes import PrimeConfig, RuntimeConfig
|
|
10
12
|
from verifiers.v1.types import SamplingConfig
|
|
13
|
+
from verifiers.v1.utils.generic import deep_merge
|
|
11
14
|
|
|
12
15
|
|
|
13
16
|
class TimeoutConfig(BaseConfig):
|
|
@@ -78,3 +81,50 @@ class WireAgentConfig(AgentConfig):
|
|
|
78
81
|
def _resolve_harness(cls, data):
|
|
79
82
|
"""Override: a record read resolves no plugins."""
|
|
80
83
|
return data
|
|
84
|
+
|
|
85
|
+
|
|
86
|
+
def agent_config_fields(config: BaseModel) -> dict[str, AgentConfig]:
|
|
87
|
+
"""Top-level agent configs, in declaration order, keyed by their field names."""
|
|
88
|
+
return {name: value for name, value in config if isinstance(value, AgentConfig)}
|
|
89
|
+
|
|
90
|
+
|
|
91
|
+
def merge_agent_defaults(config: type[BaseModel], data: Any) -> Any:
|
|
92
|
+
"""Merge partial agent overrides onto their declared defaults."""
|
|
93
|
+
if isinstance(data, dict):
|
|
94
|
+
for name, field in config.model_fields.items():
|
|
95
|
+
if isinstance(field.default, AgentConfig) and isinstance(
|
|
96
|
+
data.get(name), dict
|
|
97
|
+
):
|
|
98
|
+
data[name] = deep_merge(
|
|
99
|
+
field.default.model_dump(exclude_none=True), data[name]
|
|
100
|
+
)
|
|
101
|
+
return data
|
|
102
|
+
|
|
103
|
+
|
|
104
|
+
def resolve_agent(
|
|
105
|
+
spec: AgentConfig,
|
|
106
|
+
*,
|
|
107
|
+
model: str | None = None,
|
|
108
|
+
client: ClientConfig | None = None,
|
|
109
|
+
sampling: SamplingConfig | None = None,
|
|
110
|
+
harness: HarnessConfig | None = None,
|
|
111
|
+
) -> AgentConfig:
|
|
112
|
+
"""`spec` with what it leaves unset filled from the run's defaults; its own
|
|
113
|
+
sampling values merge over the run's. The one place a seat's identity resolves,
|
|
114
|
+
for configured agent roles."""
|
|
115
|
+
merged = spec.sampling if sampling is None else sampling
|
|
116
|
+
if sampling is not None and spec.sampling is not None:
|
|
117
|
+
merged = sampling.model_copy(
|
|
118
|
+
update=deep_merge(
|
|
119
|
+
sampling.model_dump(exclude_unset=True),
|
|
120
|
+
spec.sampling.model_dump(exclude_unset=True),
|
|
121
|
+
)
|
|
122
|
+
)
|
|
123
|
+
return spec.model_copy(
|
|
124
|
+
update={
|
|
125
|
+
"harness": spec.harness if spec.harness is not None else harness,
|
|
126
|
+
"model": spec.model if spec.model is not None else model,
|
|
127
|
+
"client": spec.client if spec.client is not None else client,
|
|
128
|
+
"sampling": merged,
|
|
129
|
+
}
|
|
130
|
+
)
|
|
@@ -6,13 +6,16 @@ from typing import get_args
|
|
|
6
6
|
from pydantic import Field, SerializeAsAny, model_validator
|
|
7
7
|
from pydantic_config import BaseConfig
|
|
8
8
|
|
|
9
|
-
from verifiers.v1.configs.agent import
|
|
9
|
+
from verifiers.v1.configs.agent import (
|
|
10
|
+
AgentConfig,
|
|
11
|
+
agent_config_fields,
|
|
12
|
+
merge_agent_defaults,
|
|
13
|
+
)
|
|
10
14
|
from verifiers.v1.configs.harness import HarnessConfig
|
|
11
15
|
from verifiers.v1.configs.retries import RetryConfig
|
|
12
16
|
from verifiers.v1.configs.taskset import TasksetConfig
|
|
13
17
|
from verifiers.v1.interception import ElasticInterceptionPoolConfig, InterceptionConfig
|
|
14
18
|
from verifiers.v1.types import ID
|
|
15
|
-
from verifiers.v1.utils.generic import deep_merge
|
|
16
19
|
|
|
17
20
|
|
|
18
21
|
class TimeoutConfig(BaseConfig):
|
|
@@ -73,7 +76,7 @@ class EnvConfig(BaseConfig):
|
|
|
73
76
|
default = default_agent_harness(self.taskset.id)
|
|
74
77
|
return {
|
|
75
78
|
name: cfg.harness if cfg.harness is not None else default
|
|
76
|
-
for name, cfg in
|
|
79
|
+
for name, cfg in agent_config_fields(self).items()
|
|
77
80
|
}
|
|
78
81
|
|
|
79
82
|
@model_validator(mode="before")
|
|
@@ -110,17 +113,7 @@ class EnvConfig(BaseConfig):
|
|
|
110
113
|
@model_validator(mode="before")
|
|
111
114
|
@classmethod
|
|
112
115
|
def _merge_role_defaults(cls, data):
|
|
113
|
-
|
|
114
|
-
validation would replace the instance wholesale, resetting its other pins."""
|
|
115
|
-
if isinstance(data, dict):
|
|
116
|
-
for name, field in cls.model_fields.items():
|
|
117
|
-
if isinstance(field.default, AgentConfig) and isinstance(
|
|
118
|
-
data.get(name), dict
|
|
119
|
-
):
|
|
120
|
-
data[name] = deep_merge(
|
|
121
|
-
field.default.model_dump(exclude_none=True), data[name]
|
|
122
|
-
)
|
|
123
|
-
return data
|
|
116
|
+
return merge_agent_defaults(cls, data)
|
|
124
117
|
|
|
125
118
|
@classmethod
|
|
126
119
|
def __pydantic_init_subclass__(cls, **kwargs):
|
|
@@ -146,16 +139,6 @@ class EnvConfig(BaseConfig):
|
|
|
146
139
|
)
|
|
147
140
|
|
|
148
141
|
|
|
149
|
-
def _declared_agent_configs(config: EnvConfig) -> dict[str, AgentConfig]:
|
|
150
|
-
"""The `AgentConfig` fields declared on an env's config, in declaration order —
|
|
151
|
-
the env's roles, each seat keyed by its field name (the only naming site)."""
|
|
152
|
-
return {
|
|
153
|
-
name: getattr(config, name)
|
|
154
|
-
for name, field in type(config).model_fields.items()
|
|
155
|
-
if isinstance(field.default, AgentConfig)
|
|
156
|
-
}
|
|
157
|
-
|
|
158
|
-
|
|
159
142
|
def default_agent_harness(taskset_id: str) -> HarnessConfig:
|
|
160
143
|
"""What an unpinned role's `harness=None` resolves to: the taskset's bundled
|
|
161
144
|
harness when it ships one, else the built-in `bash`."""
|