verifiers 0.1.15.dev189__tar.gz → 0.1.15.dev359__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {verifiers-0.1.15.dev189 → verifiers-0.1.15.dev359}/PKG-INFO +17 -91
- {verifiers-0.1.15.dev189 → verifiers-0.1.15.dev359}/README.md +0 -60
- {verifiers-0.1.15.dev189 → verifiers-0.1.15.dev359}/pyproject.toml +75 -95
- {verifiers-0.1.15.dev189 → verifiers-0.1.15.dev359}/tests/test_envs.py +8 -57
- {verifiers-0.1.15.dev189 → verifiers-0.1.15.dev359}/tests/test_imports.py +0 -80
- verifiers-0.1.15.dev359/tests/test_init_script.py +19 -0
- {verifiers-0.1.15.dev189 → verifiers-0.1.15.dev359}/tests/test_trajectory_processing.py +0 -42
- verifiers-0.1.15.dev359/tests/v1/conftest.py +262 -0
- verifiers-0.1.15.dev359/tests/v1/fixtures/counter_tool_v1.py +67 -0
- verifiers-0.1.15.dev359/tests/v1/fixtures/echo_agentic_v1.py +63 -0
- verifiers-0.1.15.dev359/tests/v1/fixtures/echo_multi_v0.py +54 -0
- verifiers-0.1.15.dev359/tests/v1/fixtures/echo_tool_v1.py +64 -0
- verifiers-0.1.15.dev359/tests/v1/fixtures/echo_user_sim_v1.py +82 -0
- verifiers-0.1.15.dev359/tests/v1/fixtures/echo_v0.py +33 -0
- verifiers-0.1.15.dev359/tests/v1/fixtures/echo_v1.py +51 -0
- verifiers-0.1.15.dev359/tests/v1/fixtures/pyproject.toml +21 -0
- verifiers-0.1.15.dev359/tests/v1/fixtures/tool_response_image_v1.py +63 -0
- verifiers-0.1.15.dev359/tests/v1/test_color_codeword_tasks.py +18 -0
- verifiers-0.1.15.dev359/tests/v1/test_configs.py +26 -0
- verifiers-0.1.15.dev359/tests/v1/test_e2e.py +225 -0
- verifiers-0.1.15.dev359/tests/v1/test_envs.py +85 -0
- verifiers-0.1.15.dev359/tests/v1/test_graph.py +240 -0
- verifiers-0.1.15.dev359/tests/v1/test_legacy.py +44 -0
- verifiers-0.1.15.dev359/tests/v1/test_trace.py +84 -0
- {verifiers-0.1.15.dev189 → verifiers-0.1.15.dev359}/verifiers/__init__.py +2 -158
- {verifiers-0.1.15.dev189 → verifiers-0.1.15.dev359}/verifiers/clients/__init__.py +5 -17
- {verifiers-0.1.15.dev189 → verifiers-0.1.15.dev359}/verifiers/envs/environment.py +15 -9
- {verifiers-0.1.15.dev189 → verifiers-0.1.15.dev359}/verifiers/envs/experimental/composable/tasksets/search/quest/README.md +2 -0
- {verifiers-0.1.15.dev189 → verifiers-0.1.15.dev359}/verifiers/envs/experimental/composable/tasksets/search/quest/obj_task_eval/api_tools/tool_pdf.py +13 -4
- {verifiers-0.1.15.dev189 → verifiers-0.1.15.dev359}/verifiers/envs/python_env.py +1 -7
- {verifiers-0.1.15.dev189 → verifiers-0.1.15.dev359}/verifiers/envs/sandbox_env.py +1 -8
- {verifiers-0.1.15.dev189 → verifiers-0.1.15.dev359}/verifiers/scripts/eval.py +2 -141
- verifiers-0.1.15.dev359/verifiers/scripts/init.py +249 -0
- {verifiers-0.1.15.dev189 → verifiers-0.1.15.dev359}/verifiers/serve/types.py +3 -7
- {verifiers-0.1.15.dev189 → verifiers-0.1.15.dev359}/verifiers/types.py +5 -350
- verifiers-0.1.15.dev359/verifiers/utils/env_utils.py +71 -0
- {verifiers-0.1.15.dev189 → verifiers-0.1.15.dev359}/verifiers/utils/interception_utils.py +1 -1
- {verifiers-0.1.15.dev189 → verifiers-0.1.15.dev359}/verifiers/utils/process_utils.py +15 -8
- verifiers-0.1.15.dev359/verifiers/v1/ARCHITECTURE.md +243 -0
- verifiers-0.1.15.dev359/verifiers/v1/GUIDE.md +762 -0
- verifiers-0.1.15.dev359/verifiers/v1/README.md +254 -0
- verifiers-0.1.15.dev359/verifiers/v1/__init__.py +203 -0
- verifiers-0.1.15.dev359/verifiers/v1/cli/__init__.py +3 -0
- verifiers-0.1.15.dev359/verifiers/v1/cli/dashboard/__init__.py +6 -0
- verifiers-0.1.15.dev359/verifiers/v1/cli/dashboard/base.py +32 -0
- verifiers-0.1.15.dev359/verifiers/v1/cli/dashboard/eval.py +364 -0
- verifiers-0.1.15.dev359/verifiers/v1/cli/dashboard/validate.py +114 -0
- verifiers-0.1.15.dev359/verifiers/v1/cli/eval/__init__.py +1 -0
- verifiers-0.1.15.dev359/verifiers/v1/cli/eval/main.py +110 -0
- verifiers-0.1.15.dev359/verifiers/v1/cli/eval/resume.py +105 -0
- verifiers-0.1.15.dev359/verifiers/v1/cli/eval/runner.py +230 -0
- verifiers-0.1.15.dev359/verifiers/v1/cli/init.py +324 -0
- verifiers-0.1.15.dev359/verifiers/v1/cli/output.py +74 -0
- verifiers-0.1.15.dev359/verifiers/v1/cli/resolve.py +67 -0
- verifiers-0.1.15.dev359/verifiers/v1/cli/serve.py +74 -0
- verifiers-0.1.15.dev359/verifiers/v1/cli/validate.py +189 -0
- verifiers-0.1.15.dev359/verifiers/v1/clients/__init__.py +24 -0
- verifiers-0.1.15.dev359/verifiers/v1/clients/client.py +86 -0
- verifiers-0.1.15.dev359/verifiers/v1/clients/config.py +115 -0
- verifiers-0.1.15.dev359/verifiers/v1/clients/eval.py +217 -0
- verifiers-0.1.15.dev359/verifiers/v1/clients/train.py +304 -0
- verifiers-0.1.15.dev359/verifiers/v1/configs/__init__.py +8 -0
- verifiers-0.1.15.dev359/verifiers/v1/configs/eval.py +74 -0
- verifiers-0.1.15.dev359/verifiers/v1/configs/init.py +30 -0
- verifiers-0.1.15.dev359/verifiers/v1/configs/serve.py +22 -0
- verifiers-0.1.15.dev359/verifiers/v1/configs/validate.py +62 -0
- verifiers-0.1.15.dev359/verifiers/v1/decorators.py +126 -0
- verifiers-0.1.15.dev359/verifiers/v1/dialects/__init__.py +34 -0
- verifiers-0.1.15.dev359/verifiers/v1/dialects/anthropic.py +286 -0
- verifiers-0.1.15.dev359/verifiers/v1/dialects/base.py +130 -0
- verifiers-0.1.15.dev359/verifiers/v1/dialects/chat.py +307 -0
- verifiers-0.1.15.dev359/verifiers/v1/dialects/responses.py +356 -0
- verifiers-0.1.15.dev359/verifiers/v1/env.py +405 -0
- verifiers-0.1.15.dev359/verifiers/v1/episode.py +79 -0
- verifiers-0.1.15.dev359/verifiers/v1/errors.py +138 -0
- verifiers-0.1.15.dev359/verifiers/v1/graph.py +510 -0
- verifiers-0.1.15.dev359/verifiers/v1/harness.py +178 -0
- verifiers-0.1.15.dev359/verifiers/v1/harnesses/__init__.py +31 -0
- verifiers-0.1.15.dev359/verifiers/v1/harnesses/bash/__init__.py +7 -0
- verifiers-0.1.15.dev359/verifiers/v1/harnesses/bash/harness.py +76 -0
- verifiers-0.1.15.dev359/verifiers/v1/harnesses/bash/program.py +187 -0
- verifiers-0.1.15.dev359/verifiers/v1/harnesses/codex/__init__.py +3 -0
- verifiers-0.1.15.dev359/verifiers/v1/harnesses/codex/harness.py +110 -0
- verifiers-0.1.15.dev359/verifiers/v1/harnesses/default/__init__.py +6 -0
- verifiers-0.1.15.dev359/verifiers/v1/harnesses/default/harness.py +73 -0
- verifiers-0.1.15.dev359/verifiers/v1/harnesses/default/program.py +155 -0
- verifiers-0.1.15.dev359/verifiers/v1/harnesses/kimi_code/__init__.py +6 -0
- verifiers-0.1.15.dev359/verifiers/v1/harnesses/kimi_code/harness.py +106 -0
- verifiers-0.1.15.dev359/verifiers/v1/harnesses/mini_swe_agent/__init__.py +6 -0
- verifiers-0.1.15.dev359/verifiers/v1/harnesses/mini_swe_agent/harness.py +74 -0
- verifiers-0.1.15.dev359/verifiers/v1/harnesses/mini_swe_agent/program.py +8 -0
- verifiers-0.1.15.dev359/verifiers/v1/harnesses/rlm/__init__.py +3 -0
- verifiers-0.1.15.dev359/verifiers/v1/harnesses/rlm/harness.py +108 -0
- verifiers-0.1.15.dev359/verifiers/v1/harnesses/terminus_2/__init__.py +6 -0
- verifiers-0.1.15.dev359/verifiers/v1/harnesses/terminus_2/harness.py +89 -0
- verifiers-0.1.15.dev359/verifiers/v1/harnesses/terminus_2/program.py +91 -0
- verifiers-0.1.15.dev359/verifiers/v1/interception/__init__.py +21 -0
- verifiers-0.1.15.dev359/verifiers/v1/interception/pool.py +94 -0
- verifiers-0.1.15.dev359/verifiers/v1/interception/server.py +566 -0
- verifiers-0.1.15.dev359/verifiers/v1/legacy.py +498 -0
- verifiers-0.1.15.dev359/verifiers/v1/loaders.py +179 -0
- verifiers-0.1.15.dev359/verifiers/v1/mcp/__init__.py +34 -0
- verifiers-0.1.15.dev359/verifiers/v1/mcp/launch.py +572 -0
- verifiers-0.1.15.dev359/verifiers/v1/mcp/multiplex.py +288 -0
- verifiers-0.1.15.dev359/verifiers/v1/mcp/server.py +394 -0
- verifiers-0.1.15.dev359/verifiers/v1/mcp/toolset.py +105 -0
- verifiers-0.1.15.dev359/verifiers/v1/mcp/user.py +89 -0
- verifiers-0.1.15.dev359/verifiers/v1/retries.py +141 -0
- verifiers-0.1.15.dev359/verifiers/v1/rollout.py +296 -0
- verifiers-0.1.15.dev359/verifiers/v1/runtimes/__init__.py +74 -0
- verifiers-0.1.15.dev359/verifiers/v1/runtimes/base.py +368 -0
- verifiers-0.1.15.dev359/verifiers/v1/runtimes/docker.py +202 -0
- verifiers-0.1.15.dev359/verifiers/v1/runtimes/limiters.py +76 -0
- verifiers-0.1.15.dev359/verifiers/v1/runtimes/modal.py +201 -0
- verifiers-0.1.15.dev359/verifiers/v1/runtimes/prime.py +233 -0
- verifiers-0.1.15.dev359/verifiers/v1/runtimes/subprocess.py +124 -0
- verifiers-0.1.15.dev359/verifiers/v1/serve/__init__.py +36 -0
- verifiers-0.1.15.dev359/verifiers/v1/serve/client.py +157 -0
- verifiers-0.1.15.dev359/verifiers/v1/serve/pool.py +344 -0
- verifiers-0.1.15.dev359/verifiers/v1/serve/server.py +218 -0
- verifiers-0.1.15.dev359/verifiers/v1/serve/types.py +79 -0
- verifiers-0.1.15.dev359/verifiers/v1/state.py +46 -0
- verifiers-0.1.15.dev359/verifiers/v1/task.py +97 -0
- verifiers-0.1.15.dev359/verifiers/v1/taskset.py +164 -0
- verifiers-0.1.15.dev359/verifiers/v1/tasksets/__init__.py +9 -0
- verifiers-0.1.15.dev359/verifiers/v1/tasksets/harbor_v1/__init__.py +7 -0
- verifiers-0.1.15.dev359/verifiers/v1/tasksets/harbor_v1/taskset.py +245 -0
- verifiers-0.1.15.dev359/verifiers/v1/tasksets/textarena_v1/__init__.py +15 -0
- verifiers-0.1.15.dev359/verifiers/v1/tasksets/textarena_v1/taskset.py +182 -0
- verifiers-0.1.15.dev359/verifiers/v1/trace.py +392 -0
- verifiers-0.1.15.dev359/verifiers/v1/types.py +283 -0
- verifiers-0.1.15.dev359/verifiers/v1/utils/__init__.py +3 -0
- verifiers-0.1.15.dev359/verifiers/v1/utils/format.py +52 -0
- verifiers-0.1.15.dev359/verifiers/v1/utils/install.py +43 -0
- verifiers-0.1.15.dev359/verifiers/v1/utils/logging.py +56 -0
- verifiers-0.1.15.dev359/verifiers/v1/utils/memory.py +43 -0
- verifiers-0.1.15.dev189/tests/test_eval_cli.py +0 -1746
- verifiers-0.1.15.dev189/tests/test_init_script.py +0 -124
- verifiers-0.1.15.dev189/tests/test_langchain_deep_agents_wikispeedia.py +0 -395
- verifiers-0.1.15.dev189/tests/test_mcp_search_env.py +0 -89
- verifiers-0.1.15.dev189/tests/test_opencode_harbor.py +0 -106
- verifiers-0.1.15.dev189/tests/test_v1_bfcl.py +0 -178
- verifiers-0.1.15.dev189/tests/test_v1_config_extension.py +0 -3707
- verifiers-0.1.15.dev189/tests/test_v1_empty_completions.py +0 -57
- verifiers-0.1.15.dev189/tests/test_v1_endpoint_protocols.py +0 -224
- verifiers-0.1.15.dev189/tests/test_v1_example_counts.py +0 -115
- verifiers-0.1.15.dev189/tests/test_v1_group_reward_env.py +0 -40
- verifiers-0.1.15.dev189/tests/test_v1_harbor_cli.py +0 -555
- verifiers-0.1.15.dev189/tests/test_v1_mini_swe_agent.py +0 -126
- verifiers-0.1.15.dev189/tests/test_v1_nemo_gym_harness.py +0 -427
- verifiers-0.1.15.dev189/tests/test_v1_openenv_taskset.py +0 -236
- verifiers-0.1.15.dev189/tests/test_v1_openreward_taskset.py +0 -243
- verifiers-0.1.15.dev189/tests/test_v1_replay_harness.py +0 -327
- verifiers-0.1.15.dev189/tests/test_v1_rlm_swe.py +0 -855
- verifiers-0.1.15.dev189/tests/test_v1_runtime_lifecycle.py +0 -3683
- verifiers-0.1.15.dev189/tests/test_v1_scoring_functions.py +0 -163
- verifiers-0.1.15.dev189/tests/test_v1_taskset_bindings.py +0 -374
- verifiers-0.1.15.dev189/tests/test_v1_taskset_utils.py +0 -142
- verifiers-0.1.15.dev189/tests/test_v1_textarena_taskset.py +0 -288
- verifiers-0.1.15.dev189/tests/test_wiki_search_v1.py +0 -136
- verifiers-0.1.15.dev189/tests/test_wordle_v1_env.py +0 -171
- verifiers-0.1.15.dev189/verifiers/scripts/init.py +0 -563
- verifiers-0.1.15.dev189/verifiers/utils/env_utils.py +0 -426
- verifiers-0.1.15.dev189/verifiers/v1/ENVIRONMENT_BEST_PRACTICES.md +0 -186
- verifiers-0.1.15.dev189/verifiers/v1/README.md +0 -339
- verifiers-0.1.15.dev189/verifiers/v1/RE_MIGRATION.md +0 -417
- verifiers-0.1.15.dev189/verifiers/v1/__init__.py +0 -150
- verifiers-0.1.15.dev189/verifiers/v1/artifact.py +0 -86
- verifiers-0.1.15.dev189/verifiers/v1/config.py +0 -127
- verifiers-0.1.15.dev189/verifiers/v1/env.py +0 -203
- verifiers-0.1.15.dev189/verifiers/v1/harness.py +0 -694
- verifiers-0.1.15.dev189/verifiers/v1/model.py +0 -54
- verifiers-0.1.15.dev189/verifiers/v1/program.py +0 -301
- verifiers-0.1.15.dev189/verifiers/v1/runtime.py +0 -2828
- verifiers-0.1.15.dev189/verifiers/v1/runtime_handles.py +0 -51
- verifiers-0.1.15.dev189/verifiers/v1/sandbox.py +0 -66
- verifiers-0.1.15.dev189/verifiers/v1/state.py +0 -10
- verifiers-0.1.15.dev189/verifiers/v1/task.py +0 -150
- verifiers-0.1.15.dev189/verifiers/v1/taskset.py +0 -157
- verifiers-0.1.15.dev189/verifiers/v1/toolset.py +0 -292
- verifiers-0.1.15.dev189/verifiers/v1/types.py +0 -53
- verifiers-0.1.15.dev189/verifiers/v1/user.py +0 -132
- verifiers-0.1.15.dev189/verifiers/v1/utils/__init__.py +0 -1
- verifiers-0.1.15.dev189/verifiers/v1/utils/binding_utils.py +0 -327
- verifiers-0.1.15.dev189/verifiers/v1/utils/config_callable_utils.py +0 -127
- verifiers-0.1.15.dev189/verifiers/v1/utils/config_utils.py +0 -298
- verifiers-0.1.15.dev189/verifiers/v1/utils/endpoint_utils.py +0 -833
- verifiers-0.1.15.dev189/verifiers/v1/utils/json_utils.py +0 -10
- verifiers-0.1.15.dev189/verifiers/v1/utils/judge_utils.py +0 -54
- verifiers-0.1.15.dev189/verifiers/v1/utils/lifecycle_utils.py +0 -105
- verifiers-0.1.15.dev189/verifiers/v1/utils/logging_utils.py +0 -61
- verifiers-0.1.15.dev189/verifiers/v1/utils/mcp_proxy_utils.py +0 -223
- verifiers-0.1.15.dev189/verifiers/v1/utils/mcp_utils.py +0 -150
- verifiers-0.1.15.dev189/verifiers/v1/utils/object_utils.py +0 -61
- verifiers-0.1.15.dev189/verifiers/v1/utils/program_utils.py +0 -555
- verifiers-0.1.15.dev189/verifiers/v1/utils/prompt_utils.py +0 -249
- verifiers-0.1.15.dev189/verifiers/v1/utils/runtime_owner_utils.py +0 -139
- verifiers-0.1.15.dev189/verifiers/v1/utils/runtime_registry.py +0 -33
- verifiers-0.1.15.dev189/verifiers/v1/utils/sandbox_program_utils.py +0 -588
- verifiers-0.1.15.dev189/verifiers/v1/utils/sandbox_python_utils.py +0 -99
- verifiers-0.1.15.dev189/verifiers/v1/utils/sandbox_utils.py +0 -1139
- verifiers-0.1.15.dev189/verifiers/v1/utils/scoring_utils.py +0 -438
- verifiers-0.1.15.dev189/verifiers/v1/utils/serialization_utils.py +0 -11
- verifiers-0.1.15.dev189/verifiers/v1/utils/task_freeze_utils.py +0 -89
- verifiers-0.1.15.dev189/verifiers/v1/utils/taskset_utils.py +0 -153
- verifiers-0.1.15.dev189/verifiers/v1/utils/tool_utils.py +0 -58
- verifiers-0.1.15.dev189/verifiers/v1/utils/toolset_utils.py +0 -217
- verifiers-0.1.15.dev189/verifiers/v1/utils/trajectory_utils.py +0 -82
- verifiers-0.1.15.dev189/verifiers/v1/utils/usage_utils.py +0 -34
- {verifiers-0.1.15.dev189 → verifiers-0.1.15.dev359}/.gitignore +0 -0
- {verifiers-0.1.15.dev189 → verifiers-0.1.15.dev359}/LICENSE +0 -0
- {verifiers-0.1.15.dev189 → verifiers-0.1.15.dev359}/tests/AGENTS.md +0 -0
- {verifiers-0.1.15.dev189 → verifiers-0.1.15.dev359}/tests/README.md +0 -0
- {verifiers-0.1.15.dev189 → verifiers-0.1.15.dev359}/tests/__init__.py +0 -0
- {verifiers-0.1.15.dev189 → verifiers-0.1.15.dev359}/tests/conftest.py +0 -0
- {verifiers-0.1.15.dev189 → verifiers-0.1.15.dev359}/tests/test_browser_env.py +0 -0
- {verifiers-0.1.15.dev189 → verifiers-0.1.15.dev359}/tests/test_build_script.py +0 -0
- {verifiers-0.1.15.dev189 → verifiers-0.1.15.dev359}/tests/test_cli_agent_env.py +0 -0
- {verifiers-0.1.15.dev189 → verifiers-0.1.15.dev359}/tests/test_client_auth_errors.py +0 -0
- {verifiers-0.1.15.dev189 → verifiers-0.1.15.dev359}/tests/test_client_config.py +0 -0
- {verifiers-0.1.15.dev189 → verifiers-0.1.15.dev359}/tests/test_client_multimodal_types.py +0 -0
- {verifiers-0.1.15.dev189 → verifiers-0.1.15.dev359}/tests/test_composable_env.py +0 -0
- {verifiers-0.1.15.dev189 → verifiers-0.1.15.dev359}/tests/test_context_token_metrics.py +0 -0
- {verifiers-0.1.15.dev189 → verifiers-0.1.15.dev359}/tests/test_decorator_ranks.py +0 -0
- {verifiers-0.1.15.dev189 → verifiers-0.1.15.dev359}/tests/test_endpoint_registry.py +0 -0
- {verifiers-0.1.15.dev189 → verifiers-0.1.15.dev359}/tests/test_env_group.py +0 -0
- {verifiers-0.1.15.dev189 → verifiers-0.1.15.dev359}/tests/test_env_server.py +0 -0
- {verifiers-0.1.15.dev189 → verifiers-0.1.15.dev359}/tests/test_environment.py +0 -0
- {verifiers-0.1.15.dev189 → verifiers-0.1.15.dev359}/tests/test_environment_extra.py +0 -0
- {verifiers-0.1.15.dev189 → verifiers-0.1.15.dev359}/tests/test_error_chain.py +0 -0
- {verifiers-0.1.15.dev189 → verifiers-0.1.15.dev359}/tests/test_eval_display.py +0 -0
- {verifiers-0.1.15.dev189 → verifiers-0.1.15.dev359}/tests/test_eval_utils.py +0 -0
- {verifiers-0.1.15.dev189 → verifiers-0.1.15.dev359}/tests/test_gepa_cli.py +0 -0
- {verifiers-0.1.15.dev189 → verifiers-0.1.15.dev359}/tests/test_gepa_utils.py +0 -0
- {verifiers-0.1.15.dev189 → verifiers-0.1.15.dev359}/tests/test_gym_env.py +0 -0
- {verifiers-0.1.15.dev189 → verifiers-0.1.15.dev359}/tests/test_harbor_env_mcp.py +0 -0
- {verifiers-0.1.15.dev189 → verifiers-0.1.15.dev359}/tests/test_install_utils.py +0 -0
- {verifiers-0.1.15.dev189 → verifiers-0.1.15.dev359}/tests/test_interception_utils.py +0 -0
- {verifiers-0.1.15.dev189 → verifiers-0.1.15.dev359}/tests/test_lean_task.py +0 -0
- {verifiers-0.1.15.dev189 → verifiers-0.1.15.dev359}/tests/test_logging.py +0 -0
- {verifiers-0.1.15.dev189 → verifiers-0.1.15.dev359}/tests/test_math_rubric.py +0 -0
- {verifiers-0.1.15.dev189 → verifiers-0.1.15.dev359}/tests/test_maybe_think_parser.py +0 -0
- {verifiers-0.1.15.dev189 → verifiers-0.1.15.dev359}/tests/test_message_utils.py +0 -0
- {verifiers-0.1.15.dev189 → verifiers-0.1.15.dev359}/tests/test_message_utils_multimodal.py +0 -0
- {verifiers-0.1.15.dev189 → verifiers-0.1.15.dev359}/tests/test_multiturn_env.py +0 -0
- {verifiers-0.1.15.dev189 → verifiers-0.1.15.dev359}/tests/test_nemorl_client.py +0 -0
- {verifiers-0.1.15.dev189 → verifiers-0.1.15.dev359}/tests/test_openai_chat_completions_token_client.py +0 -0
- {verifiers-0.1.15.dev189 → verifiers-0.1.15.dev359}/tests/test_openai_responses_client.py +0 -0
- {verifiers-0.1.15.dev189 → verifiers-0.1.15.dev359}/tests/test_opencode_rlm_env.py +0 -0
- {verifiers-0.1.15.dev189 → verifiers-0.1.15.dev359}/tests/test_openenv_client.py +0 -0
- {verifiers-0.1.15.dev189 → verifiers-0.1.15.dev359}/tests/test_parser.py +0 -0
- {verifiers-0.1.15.dev189 → verifiers-0.1.15.dev359}/tests/test_path_utils.py +0 -0
- {verifiers-0.1.15.dev189 → verifiers-0.1.15.dev359}/tests/test_per_turn_timing.py +0 -0
- {verifiers-0.1.15.dev189 → verifiers-0.1.15.dev359}/tests/test_pricing_utils.py +0 -0
- {verifiers-0.1.15.dev189 → verifiers-0.1.15.dev359}/tests/test_prime_plugin.py +0 -0
- {verifiers-0.1.15.dev189 → verifiers-0.1.15.dev359}/tests/test_renderer_client.py +0 -0
- {verifiers-0.1.15.dev189 → verifiers-0.1.15.dev359}/tests/test_renderer_e2e.py +0 -0
- {verifiers-0.1.15.dev189 → verifiers-0.1.15.dev359}/tests/test_rlm_composable_env.py +0 -0
- {verifiers-0.1.15.dev189 → verifiers-0.1.15.dev359}/tests/test_rubric.py +0 -0
- {verifiers-0.1.15.dev189 → verifiers-0.1.15.dev359}/tests/test_rubric_group.py +0 -0
- {verifiers-0.1.15.dev189 → verifiers-0.1.15.dev359}/tests/test_sandbox_debug_env.py +0 -0
- {verifiers-0.1.15.dev189 → verifiers-0.1.15.dev359}/tests/test_sandbox_env.py +0 -0
- {verifiers-0.1.15.dev189 → verifiers-0.1.15.dev359}/tests/test_sandbox_mixin.py +0 -0
- {verifiers-0.1.15.dev189 → verifiers-0.1.15.dev359}/tests/test_save_utils.py +0 -0
- {verifiers-0.1.15.dev189 → verifiers-0.1.15.dev359}/tests/test_setup_script.py +0 -0
- {verifiers-0.1.15.dev189 → verifiers-0.1.15.dev359}/tests/test_singleturn_env.py +0 -0
- {verifiers-0.1.15.dev189 → verifiers-0.1.15.dev359}/tests/test_stateful_tool_env.py +0 -0
- {verifiers-0.1.15.dev189 → verifiers-0.1.15.dev359}/tests/test_think_parser.py +0 -0
- {verifiers-0.1.15.dev189 → verifiers-0.1.15.dev359}/tests/test_tool_env.py +0 -0
- {verifiers-0.1.15.dev189 → verifiers-0.1.15.dev359}/tests/test_tool_utils.py +0 -0
- {verifiers-0.1.15.dev189 → verifiers-0.1.15.dev359}/tests/test_tui_info_formatting.py +0 -0
- {verifiers-0.1.15.dev189 → verifiers-0.1.15.dev359}/tests/test_types.py +0 -0
- {verifiers-0.1.15.dev189 → verifiers-0.1.15.dev359}/tests/test_wordle_env.py +0 -0
- {verifiers-0.1.15.dev189 → verifiers-0.1.15.dev359}/tests/test_xml_parser.py +0 -0
- {verifiers-0.1.15.dev189 → verifiers-0.1.15.dev359}/verifiers/AGENTS.md +0 -0
- {verifiers-0.1.15.dev189 → verifiers-0.1.15.dev359}/verifiers/cli/__init__.py +0 -0
- {verifiers-0.1.15.dev189 → verifiers-0.1.15.dev359}/verifiers/cli/commands/__init__.py +0 -0
- {verifiers-0.1.15.dev189 → verifiers-0.1.15.dev359}/verifiers/cli/commands/build.py +0 -0
- {verifiers-0.1.15.dev189 → verifiers-0.1.15.dev359}/verifiers/cli/commands/eval.py +0 -0
- {verifiers-0.1.15.dev189 → verifiers-0.1.15.dev359}/verifiers/cli/commands/gepa.py +0 -0
- {verifiers-0.1.15.dev189 → verifiers-0.1.15.dev359}/verifiers/cli/commands/init.py +0 -0
- {verifiers-0.1.15.dev189 → verifiers-0.1.15.dev359}/verifiers/cli/commands/install.py +0 -0
- {verifiers-0.1.15.dev189 → verifiers-0.1.15.dev359}/verifiers/cli/commands/setup.py +0 -0
- {verifiers-0.1.15.dev189 → verifiers-0.1.15.dev359}/verifiers/cli/plugins/__init__.py +0 -0
- {verifiers-0.1.15.dev189 → verifiers-0.1.15.dev359}/verifiers/cli/plugins/prime.py +0 -0
- {verifiers-0.1.15.dev189 → verifiers-0.1.15.dev359}/verifiers/cli/tui.py +0 -0
- {verifiers-0.1.15.dev189 → verifiers-0.1.15.dev359}/verifiers/clients/anthropic_messages_client.py +0 -0
- {verifiers-0.1.15.dev189 → verifiers-0.1.15.dev359}/verifiers/clients/client.py +0 -0
- {verifiers-0.1.15.dev189 → verifiers-0.1.15.dev359}/verifiers/clients/nemorl_chat_completions_client.py +0 -0
- {verifiers-0.1.15.dev189 → verifiers-0.1.15.dev359}/verifiers/clients/openai_chat_completions_client.py +0 -0
- {verifiers-0.1.15.dev189 → verifiers-0.1.15.dev359}/verifiers/clients/openai_chat_completions_token_client.py +0 -0
- {verifiers-0.1.15.dev189 → verifiers-0.1.15.dev359}/verifiers/clients/openai_completions_client.py +0 -0
- {verifiers-0.1.15.dev189 → verifiers-0.1.15.dev359}/verifiers/clients/openai_responses_client.py +0 -0
- {verifiers-0.1.15.dev189 → verifiers-0.1.15.dev359}/verifiers/clients/renderer_client.py +0 -0
- {verifiers-0.1.15.dev189 → verifiers-0.1.15.dev359}/verifiers/decorators.py +0 -0
- {verifiers-0.1.15.dev189 → verifiers-0.1.15.dev359}/verifiers/envs/AGENTS.md +0 -0
- {verifiers-0.1.15.dev189 → verifiers-0.1.15.dev359}/verifiers/envs/__init__.py +0 -0
- {verifiers-0.1.15.dev189 → verifiers-0.1.15.dev359}/verifiers/envs/env_group.py +0 -0
- {verifiers-0.1.15.dev189 → verifiers-0.1.15.dev359}/verifiers/envs/experimental/README.md +0 -0
- {verifiers-0.1.15.dev189 → verifiers-0.1.15.dev359}/verifiers/envs/experimental/__init__.py +0 -0
- {verifiers-0.1.15.dev189 → verifiers-0.1.15.dev359}/verifiers/envs/experimental/cli_agent_env.py +0 -0
- {verifiers-0.1.15.dev189 → verifiers-0.1.15.dev359}/verifiers/envs/experimental/composable/README.md +0 -0
- {verifiers-0.1.15.dev189 → verifiers-0.1.15.dev359}/verifiers/envs/experimental/composable/__init__.py +0 -0
- {verifiers-0.1.15.dev189 → verifiers-0.1.15.dev359}/verifiers/envs/experimental/composable/_filter.py +0 -0
- {verifiers-0.1.15.dev189 → verifiers-0.1.15.dev359}/verifiers/envs/experimental/composable/composable_env.py +0 -0
- {verifiers-0.1.15.dev189 → verifiers-0.1.15.dev359}/verifiers/envs/experimental/composable/harness.py +0 -0
- {verifiers-0.1.15.dev189 → verifiers-0.1.15.dev359}/verifiers/envs/experimental/composable/harnesses/__init__.py +0 -0
- {verifiers-0.1.15.dev189 → verifiers-0.1.15.dev359}/verifiers/envs/experimental/composable/harnesses/mini_swe_agent.py +0 -0
- {verifiers-0.1.15.dev189 → verifiers-0.1.15.dev359}/verifiers/envs/experimental/composable/harnesses/opencode.py +0 -0
- {verifiers-0.1.15.dev189 → verifiers-0.1.15.dev359}/verifiers/envs/experimental/composable/harnesses/prompt.txt +0 -0
- {verifiers-0.1.15.dev189 → verifiers-0.1.15.dev359}/verifiers/envs/experimental/composable/harnesses/rlm.py +0 -0
- {verifiers-0.1.15.dev189 → verifiers-0.1.15.dev359}/verifiers/envs/experimental/composable/sandbox_debug_env.py +0 -0
- {verifiers-0.1.15.dev189 → verifiers-0.1.15.dev359}/verifiers/envs/experimental/composable/swe_debug_env.py +0 -0
- {verifiers-0.1.15.dev189 → verifiers-0.1.15.dev359}/verifiers/envs/experimental/composable/task.py +0 -0
- {verifiers-0.1.15.dev189 → verifiers-0.1.15.dev359}/verifiers/envs/experimental/composable/tasksets/__init__.py +0 -0
- {verifiers-0.1.15.dev189 → verifiers-0.1.15.dev359}/verifiers/envs/experimental/composable/tasksets/cp/__init__.py +0 -0
- {verifiers-0.1.15.dev189 → verifiers-0.1.15.dev359}/verifiers/envs/experimental/composable/tasksets/cp/cp_task.py +0 -0
- {verifiers-0.1.15.dev189 → verifiers-0.1.15.dev359}/verifiers/envs/experimental/composable/tasksets/cp/test_utils.py +0 -0
- {verifiers-0.1.15.dev189 → verifiers-0.1.15.dev359}/verifiers/envs/experimental/composable/tasksets/harbor/__init__.py +0 -0
- {verifiers-0.1.15.dev189 → verifiers-0.1.15.dev359}/verifiers/envs/experimental/composable/tasksets/harbor/cli_gym.py +0 -0
- {verifiers-0.1.15.dev189 → verifiers-0.1.15.dev359}/verifiers/envs/experimental/composable/tasksets/harbor/harbor.py +0 -0
- {verifiers-0.1.15.dev189 → verifiers-0.1.15.dev359}/verifiers/envs/experimental/composable/tasksets/harbor/terminal_lego.py +0 -0
- {verifiers-0.1.15.dev189 → verifiers-0.1.15.dev359}/verifiers/envs/experimental/composable/tasksets/lean/__init__.py +0 -0
- {verifiers-0.1.15.dev189 → verifiers-0.1.15.dev359}/verifiers/envs/experimental/composable/tasksets/lean/lean_task.py +0 -0
- {verifiers-0.1.15.dev189 → verifiers-0.1.15.dev359}/verifiers/envs/experimental/composable/tasksets/math/__init__.py +0 -0
- {verifiers-0.1.15.dev189 → verifiers-0.1.15.dev359}/verifiers/envs/experimental/composable/tasksets/math/math_task.py +0 -0
- {verifiers-0.1.15.dev189 → verifiers-0.1.15.dev359}/verifiers/envs/experimental/composable/tasksets/search/README.md +0 -0
- {verifiers-0.1.15.dev189 → verifiers-0.1.15.dev359}/verifiers/envs/experimental/composable/tasksets/search/__init__.py +0 -0
- {verifiers-0.1.15.dev189 → verifiers-0.1.15.dev359}/verifiers/envs/experimental/composable/tasksets/search/openseeker/README.md +0 -0
- {verifiers-0.1.15.dev189 → verifiers-0.1.15.dev359}/verifiers/envs/experimental/composable/tasksets/search/openseeker/__init__.py +0 -0
- {verifiers-0.1.15.dev189 → verifiers-0.1.15.dev359}/verifiers/envs/experimental/composable/tasksets/search/openseeker/taskset.py +0 -0
- {verifiers-0.1.15.dev189 → verifiers-0.1.15.dev359}/verifiers/envs/experimental/composable/tasksets/search/quest/__init__.py +0 -0
- {verifiers-0.1.15.dev189 → verifiers-0.1.15.dev359}/verifiers/envs/experimental/composable/tasksets/search/quest/obj_task_eval/__init__.py +0 -0
- {verifiers-0.1.15.dev189 → verifiers-0.1.15.dev359}/verifiers/envs/experimental/composable/tasksets/search/quest/obj_task_eval/api_tools/__init__.py +0 -0
- {verifiers-0.1.15.dev189 → verifiers-0.1.15.dev359}/verifiers/envs/experimental/composable/tasksets/search/quest/obj_task_eval/eval_toolkit.py +0 -0
- {verifiers-0.1.15.dev189 → verifiers-0.1.15.dev359}/verifiers/envs/experimental/composable/tasksets/search/quest/obj_task_eval/evaluator.py +0 -0
- {verifiers-0.1.15.dev189 → verifiers-0.1.15.dev359}/verifiers/envs/experimental/composable/tasksets/search/quest/obj_task_eval/llm_client/__init__.py +0 -0
- {verifiers-0.1.15.dev189 → verifiers-0.1.15.dev359}/verifiers/envs/experimental/composable/tasksets/search/quest/obj_task_eval/llm_client/base_client.py +0 -0
- {verifiers-0.1.15.dev189 → verifiers-0.1.15.dev359}/verifiers/envs/experimental/composable/tasksets/search/quest/obj_task_eval/prompts/__init__.py +0 -0
- {verifiers-0.1.15.dev189 → verifiers-0.1.15.dev359}/verifiers/envs/experimental/composable/tasksets/search/quest/obj_task_eval/prompts/cache_prompts.py +0 -0
- {verifiers-0.1.15.dev189 → verifiers-0.1.15.dev359}/verifiers/envs/experimental/composable/tasksets/search/quest/obj_task_eval/utils/__init__.py +0 -0
- {verifiers-0.1.15.dev189 → verifiers-0.1.15.dev359}/verifiers/envs/experimental/composable/tasksets/search/quest/obj_task_eval/utils/cache_filesys.py +0 -0
- {verifiers-0.1.15.dev189 → verifiers-0.1.15.dev359}/verifiers/envs/experimental/composable/tasksets/search/quest/obj_task_eval/utils/load_eval_script.py +0 -0
- {verifiers-0.1.15.dev189 → verifiers-0.1.15.dev359}/verifiers/envs/experimental/composable/tasksets/search/quest/obj_task_eval/utils/misc.py +0 -0
- {verifiers-0.1.15.dev189 → verifiers-0.1.15.dev359}/verifiers/envs/experimental/composable/tasksets/search/quest/obj_task_eval/utils/tool_visit.py +0 -0
- {verifiers-0.1.15.dev189 → verifiers-0.1.15.dev359}/verifiers/envs/experimental/composable/tasksets/search/quest/obj_task_eval/utils/url_tools.py +0 -0
- {verifiers-0.1.15.dev189 → verifiers-0.1.15.dev359}/verifiers/envs/experimental/composable/tasksets/search/quest/obj_task_eval/verification_tree.py +0 -0
- {verifiers-0.1.15.dev189 → verifiers-0.1.15.dev359}/verifiers/envs/experimental/composable/tasksets/search/quest/open_ended.py +0 -0
- {verifiers-0.1.15.dev189 → verifiers-0.1.15.dev359}/verifiers/envs/experimental/composable/tasksets/search/quest/taskset.py +0 -0
- {verifiers-0.1.15.dev189 → verifiers-0.1.15.dev359}/verifiers/envs/experimental/composable/tasksets/search/redsearcher/README.md +0 -0
- {verifiers-0.1.15.dev189 → verifiers-0.1.15.dev359}/verifiers/envs/experimental/composable/tasksets/search/redsearcher/__init__.py +0 -0
- {verifiers-0.1.15.dev189 → verifiers-0.1.15.dev359}/verifiers/envs/experimental/composable/tasksets/search/redsearcher/taskset.py +0 -0
- {verifiers-0.1.15.dev189 → verifiers-0.1.15.dev359}/verifiers/envs/experimental/composable/tasksets/search/search_tasksets.py +0 -0
- {verifiers-0.1.15.dev189 → verifiers-0.1.15.dev359}/verifiers/envs/experimental/composable/tasksets/swe/README.md +0 -0
- {verifiers-0.1.15.dev189 → verifiers-0.1.15.dev359}/verifiers/envs/experimental/composable/tasksets/swe/__init__.py +0 -0
- {verifiers-0.1.15.dev189 → verifiers-0.1.15.dev359}/verifiers/envs/experimental/composable/tasksets/swe/multi_swe/__init__.py +0 -0
- {verifiers-0.1.15.dev189 → verifiers-0.1.15.dev359}/verifiers/envs/experimental/composable/tasksets/swe/multi_swe/extract_fix_patch.sh +0 -0
- {verifiers-0.1.15.dev189 → verifiers-0.1.15.dev359}/verifiers/envs/experimental/composable/tasksets/swe/multi_swe/taskset.py +0 -0
- {verifiers-0.1.15.dev189 → verifiers-0.1.15.dev359}/verifiers/envs/experimental/composable/tasksets/swe/openswe/__init__.py +0 -0
- {verifiers-0.1.15.dev189 → verifiers-0.1.15.dev359}/verifiers/envs/experimental/composable/tasksets/swe/openswe/taskset.py +0 -0
- {verifiers-0.1.15.dev189 → verifiers-0.1.15.dev359}/verifiers/envs/experimental/composable/tasksets/swe/r2e_gym/__init__.py +0 -0
- {verifiers-0.1.15.dev189 → verifiers-0.1.15.dev359}/verifiers/envs/experimental/composable/tasksets/swe/r2e_gym/log_parser.py +0 -0
- {verifiers-0.1.15.dev189 → verifiers-0.1.15.dev359}/verifiers/envs/experimental/composable/tasksets/swe/r2e_gym/taskset.py +0 -0
- {verifiers-0.1.15.dev189 → verifiers-0.1.15.dev359}/verifiers/envs/experimental/composable/tasksets/swe/scale_swe/__init__.py +0 -0
- {verifiers-0.1.15.dev189 → verifiers-0.1.15.dev359}/verifiers/envs/experimental/composable/tasksets/swe/scale_swe/taskset.py +0 -0
- {verifiers-0.1.15.dev189 → verifiers-0.1.15.dev359}/verifiers/envs/experimental/composable/tasksets/swe/shared/__init__.py +0 -0
- {verifiers-0.1.15.dev189 → verifiers-0.1.15.dev359}/verifiers/envs/experimental/composable/tasksets/swe/shared/test_patch.py +0 -0
- {verifiers-0.1.15.dev189 → verifiers-0.1.15.dev359}/verifiers/envs/experimental/composable/tasksets/swe/swe_bench/__init__.py +0 -0
- {verifiers-0.1.15.dev189 → verifiers-0.1.15.dev359}/verifiers/envs/experimental/composable/tasksets/swe/swe_bench/taskset.py +0 -0
- {verifiers-0.1.15.dev189 → verifiers-0.1.15.dev359}/verifiers/envs/experimental/composable/tasksets/swe/swe_lego/__init__.py +0 -0
- {verifiers-0.1.15.dev189 → verifiers-0.1.15.dev359}/verifiers/envs/experimental/composable/tasksets/swe/swe_lego/taskset.py +0 -0
- {verifiers-0.1.15.dev189 → verifiers-0.1.15.dev359}/verifiers/envs/experimental/composable/tasksets/swe/swe_rebench_v2/__init__.py +0 -0
- {verifiers-0.1.15.dev189 → verifiers-0.1.15.dev359}/verifiers/envs/experimental/composable/tasksets/swe/swe_rebench_v2/log_parsers.py +0 -0
- {verifiers-0.1.15.dev189 → verifiers-0.1.15.dev359}/verifiers/envs/experimental/composable/tasksets/swe/swe_rebench_v2/taskset.py +0 -0
- {verifiers-0.1.15.dev189 → verifiers-0.1.15.dev359}/verifiers/envs/experimental/composable/tasksets/swe/swe_smith/__init__.py +0 -0
- {verifiers-0.1.15.dev189 → verifiers-0.1.15.dev359}/verifiers/envs/experimental/composable/tasksets/swe/swe_smith/taskset.py +0 -0
- {verifiers-0.1.15.dev189 → verifiers-0.1.15.dev359}/verifiers/envs/experimental/composable/tasksets/swe/swe_tasksets.py +0 -0
- {verifiers-0.1.15.dev189 → verifiers-0.1.15.dev359}/verifiers/envs/experimental/gym_env.py +0 -0
- {verifiers-0.1.15.dev189 → verifiers-0.1.15.dev359}/verifiers/envs/experimental/harbor_env/__init__.py +0 -0
- {verifiers-0.1.15.dev189 → verifiers-0.1.15.dev359}/verifiers/envs/experimental/harbor_env/env.py +0 -0
- {verifiers-0.1.15.dev189 → verifiers-0.1.15.dev359}/verifiers/envs/experimental/harbor_env/mcp.py +0 -0
- {verifiers-0.1.15.dev189 → verifiers-0.1.15.dev359}/verifiers/envs/experimental/mcp_env.py +0 -0
- {verifiers-0.1.15.dev189 → verifiers-0.1.15.dev359}/verifiers/envs/experimental/opencode_env.py +0 -0
- {verifiers-0.1.15.dev189 → verifiers-0.1.15.dev359}/verifiers/envs/experimental/opencode_qa_env.py +0 -0
- {verifiers-0.1.15.dev189 → verifiers-0.1.15.dev359}/verifiers/envs/experimental/opencode_rlm_env.py +0 -0
- {verifiers-0.1.15.dev189 → verifiers-0.1.15.dev359}/verifiers/envs/experimental/sandbox_mixin.py +0 -0
- {verifiers-0.1.15.dev189 → verifiers-0.1.15.dev359}/verifiers/envs/experimental/utils/__init__.py +0 -0
- {verifiers-0.1.15.dev189 → verifiers-0.1.15.dev359}/verifiers/envs/experimental/utils/file_locks.py +0 -0
- {verifiers-0.1.15.dev189 → verifiers-0.1.15.dev359}/verifiers/envs/experimental/utils/git_checkout_cache.py +0 -0
- {verifiers-0.1.15.dev189 → verifiers-0.1.15.dev359}/verifiers/envs/integrations/README.md +0 -0
- {verifiers-0.1.15.dev189 → verifiers-0.1.15.dev359}/verifiers/envs/integrations/__init__.py +0 -0
- {verifiers-0.1.15.dev189 → verifiers-0.1.15.dev359}/verifiers/envs/integrations/browser_env/README.md +0 -0
- {verifiers-0.1.15.dev189 → verifiers-0.1.15.dev359}/verifiers/envs/integrations/browser_env/__init__.py +0 -0
- {verifiers-0.1.15.dev189 → verifiers-0.1.15.dev359}/verifiers/envs/integrations/browser_env/browser_env.py +0 -0
- {verifiers-0.1.15.dev189 → verifiers-0.1.15.dev359}/verifiers/envs/integrations/browser_env/modes/__init__.py +0 -0
- {verifiers-0.1.15.dev189 → verifiers-0.1.15.dev359}/verifiers/envs/integrations/browser_env/modes/base.py +0 -0
- {verifiers-0.1.15.dev189 → verifiers-0.1.15.dev359}/verifiers/envs/integrations/browser_env/modes/cua_mode.py +0 -0
- {verifiers-0.1.15.dev189 → verifiers-0.1.15.dev359}/verifiers/envs/integrations/browser_env/modes/dom_mode.py +0 -0
- {verifiers-0.1.15.dev189 → verifiers-0.1.15.dev359}/verifiers/envs/integrations/openenv_env.py +0 -0
- {verifiers-0.1.15.dev189 → verifiers-0.1.15.dev359}/verifiers/envs/integrations/reasoninggym_env.py +0 -0
- {verifiers-0.1.15.dev189 → verifiers-0.1.15.dev359}/verifiers/envs/integrations/textarena_env.py +0 -0
- {verifiers-0.1.15.dev189 → verifiers-0.1.15.dev359}/verifiers/envs/multiturn_env.py +0 -0
- {verifiers-0.1.15.dev189 → verifiers-0.1.15.dev359}/verifiers/envs/singleturn_env.py +0 -0
- {verifiers-0.1.15.dev189 → verifiers-0.1.15.dev359}/verifiers/envs/stateful_tool_env.py +0 -0
- {verifiers-0.1.15.dev189 → verifiers-0.1.15.dev359}/verifiers/envs/tool_env.py +0 -0
- {verifiers-0.1.15.dev189 → verifiers-0.1.15.dev359}/verifiers/errors.py +0 -0
- {verifiers-0.1.15.dev189 → verifiers-0.1.15.dev359}/verifiers/gepa/__init__.py +0 -0
- {verifiers-0.1.15.dev189 → verifiers-0.1.15.dev359}/verifiers/gepa/adapter.py +0 -0
- {verifiers-0.1.15.dev189 → verifiers-0.1.15.dev359}/verifiers/gepa/config.py +0 -0
- {verifiers-0.1.15.dev189 → verifiers-0.1.15.dev359}/verifiers/gepa/display.py +0 -0
- {verifiers-0.1.15.dev189 → verifiers-0.1.15.dev359}/verifiers/gepa/gepa_utils.py +0 -0
- {verifiers-0.1.15.dev189 → verifiers-0.1.15.dev359}/verifiers/parsers/__init__.py +0 -0
- {verifiers-0.1.15.dev189 → verifiers-0.1.15.dev359}/verifiers/parsers/maybe_think_parser.py +0 -0
- {verifiers-0.1.15.dev189 → verifiers-0.1.15.dev359}/verifiers/parsers/parser.py +0 -0
- {verifiers-0.1.15.dev189 → verifiers-0.1.15.dev359}/verifiers/parsers/think_parser.py +0 -0
- {verifiers-0.1.15.dev189 → verifiers-0.1.15.dev359}/verifiers/parsers/xml_parser.py +0 -0
- {verifiers-0.1.15.dev189 → verifiers-0.1.15.dev359}/verifiers/rl/README.md +0 -0
- {verifiers-0.1.15.dev189 → verifiers-0.1.15.dev359}/verifiers/rl/__init__.py +0 -0
- {verifiers-0.1.15.dev189 → verifiers-0.1.15.dev359}/verifiers/rl/inference/__init__.py +0 -0
- {verifiers-0.1.15.dev189 → verifiers-0.1.15.dev359}/verifiers/rl/inference/client.py +0 -0
- {verifiers-0.1.15.dev189 → verifiers-0.1.15.dev359}/verifiers/rl/inference/server.py +0 -0
- {verifiers-0.1.15.dev189 → verifiers-0.1.15.dev359}/verifiers/rl/trainer/__init__.py +0 -0
- {verifiers-0.1.15.dev189 → verifiers-0.1.15.dev359}/verifiers/rl/trainer/config.py +0 -0
- {verifiers-0.1.15.dev189 → verifiers-0.1.15.dev359}/verifiers/rl/trainer/orchestrator.py +0 -0
- {verifiers-0.1.15.dev189 → verifiers-0.1.15.dev359}/verifiers/rl/trainer/trainer.py +0 -0
- {verifiers-0.1.15.dev189 → verifiers-0.1.15.dev359}/verifiers/rl/trainer/utils.py +0 -0
- {verifiers-0.1.15.dev189 → verifiers-0.1.15.dev359}/verifiers/rubrics/__init__.py +0 -0
- {verifiers-0.1.15.dev189 → verifiers-0.1.15.dev359}/verifiers/rubrics/experimental/hybrid_math_rubric.py +0 -0
- {verifiers-0.1.15.dev189 → verifiers-0.1.15.dev359}/verifiers/rubrics/judge_rubric.py +0 -0
- {verifiers-0.1.15.dev189 → verifiers-0.1.15.dev359}/verifiers/rubrics/math_rubric.py +0 -0
- {verifiers-0.1.15.dev189 → verifiers-0.1.15.dev359}/verifiers/rubrics/rubric.py +0 -0
- {verifiers-0.1.15.dev189 → verifiers-0.1.15.dev359}/verifiers/rubrics/rubric_group.py +0 -0
- {verifiers-0.1.15.dev189 → verifiers-0.1.15.dev359}/verifiers/scripts/__init__.py +0 -0
- {verifiers-0.1.15.dev189 → verifiers-0.1.15.dev359}/verifiers/scripts/build.py +0 -0
- {verifiers-0.1.15.dev189 → verifiers-0.1.15.dev359}/verifiers/scripts/gepa.py +0 -0
- {verifiers-0.1.15.dev189 → verifiers-0.1.15.dev359}/verifiers/scripts/install.py +0 -0
- {verifiers-0.1.15.dev189 → verifiers-0.1.15.dev359}/verifiers/scripts/rl.py +0 -0
- {verifiers-0.1.15.dev189 → verifiers-0.1.15.dev359}/verifiers/scripts/setup.py +0 -0
- {verifiers-0.1.15.dev189 → verifiers-0.1.15.dev359}/verifiers/scripts/train.py +0 -0
- {verifiers-0.1.15.dev189 → verifiers-0.1.15.dev359}/verifiers/scripts/tui.py +0 -0
- {verifiers-0.1.15.dev189 → verifiers-0.1.15.dev359}/verifiers/scripts/vllm.py +0 -0
- {verifiers-0.1.15.dev189 → verifiers-0.1.15.dev359}/verifiers/serve/__init__.py +0 -0
- {verifiers-0.1.15.dev189 → verifiers-0.1.15.dev359}/verifiers/serve/client/env_client.py +0 -0
- {verifiers-0.1.15.dev189 → verifiers-0.1.15.dev359}/verifiers/serve/client/zmq_env_client.py +0 -0
- {verifiers-0.1.15.dev189 → verifiers-0.1.15.dev359}/verifiers/serve/server/__init__.py +0 -0
- {verifiers-0.1.15.dev189 → verifiers-0.1.15.dev359}/verifiers/serve/server/env_router.py +0 -0
- {verifiers-0.1.15.dev189 → verifiers-0.1.15.dev359}/verifiers/serve/server/env_server.py +0 -0
- {verifiers-0.1.15.dev189 → verifiers-0.1.15.dev359}/verifiers/serve/server/env_worker.py +0 -0
- {verifiers-0.1.15.dev189 → verifiers-0.1.15.dev359}/verifiers/serve/server/zmq_env_server.py +0 -0
- {verifiers-0.1.15.dev189 → verifiers-0.1.15.dev359}/verifiers/utils/__init__.py +0 -0
- {verifiers-0.1.15.dev189 → verifiers-0.1.15.dev359}/verifiers/utils/async_utils.py +0 -0
- {verifiers-0.1.15.dev189 → verifiers-0.1.15.dev359}/verifiers/utils/client_utils.py +0 -0
- {verifiers-0.1.15.dev189 → verifiers-0.1.15.dev359}/verifiers/utils/config_utils.py +0 -0
- {verifiers-0.1.15.dev189 → verifiers-0.1.15.dev359}/verifiers/utils/data_utils.py +0 -0
- {verifiers-0.1.15.dev189 → verifiers-0.1.15.dev359}/verifiers/utils/display_utils.py +0 -0
- {verifiers-0.1.15.dev189 → verifiers-0.1.15.dev359}/verifiers/utils/env_config_utils.py +0 -0
- {verifiers-0.1.15.dev189 → verifiers-0.1.15.dev359}/verifiers/utils/error_utils.py +0 -0
- {verifiers-0.1.15.dev189 → verifiers-0.1.15.dev359}/verifiers/utils/eval_display.py +0 -0
- {verifiers-0.1.15.dev189 → verifiers-0.1.15.dev359}/verifiers/utils/eval_utils.py +0 -0
- {verifiers-0.1.15.dev189 → verifiers-0.1.15.dev359}/verifiers/utils/heartbeat.py +0 -0
- {verifiers-0.1.15.dev189 → verifiers-0.1.15.dev359}/verifiers/utils/import_utils.py +0 -0
- {verifiers-0.1.15.dev189 → verifiers-0.1.15.dev359}/verifiers/utils/install_utils.py +0 -0
- {verifiers-0.1.15.dev189 → verifiers-0.1.15.dev359}/verifiers/utils/logging_utils.py +0 -0
- {verifiers-0.1.15.dev189 → verifiers-0.1.15.dev359}/verifiers/utils/message_utils.py +0 -0
- {verifiers-0.1.15.dev189 → verifiers-0.1.15.dev359}/verifiers/utils/metric_utils.py +0 -0
- {verifiers-0.1.15.dev189 → verifiers-0.1.15.dev359}/verifiers/utils/path_utils.py +0 -0
- {verifiers-0.1.15.dev189 → verifiers-0.1.15.dev359}/verifiers/utils/pricing_utils.py +0 -0
- {verifiers-0.1.15.dev189 → verifiers-0.1.15.dev359}/verifiers/utils/response_utils.py +0 -0
- {verifiers-0.1.15.dev189 → verifiers-0.1.15.dev359}/verifiers/utils/save_utils.py +0 -0
- {verifiers-0.1.15.dev189 → verifiers-0.1.15.dev359}/verifiers/utils/serve_utils.py +0 -0
- {verifiers-0.1.15.dev189 → verifiers-0.1.15.dev359}/verifiers/utils/thread_utils.py +0 -0
- {verifiers-0.1.15.dev189 → verifiers-0.1.15.dev359}/verifiers/utils/threaded_sandbox_client.py +0 -0
- {verifiers-0.1.15.dev189 → verifiers-0.1.15.dev359}/verifiers/utils/tool_utils.py +0 -0
- {verifiers-0.1.15.dev189 → verifiers-0.1.15.dev359}/verifiers/utils/usage_utils.py +0 -0
- {verifiers-0.1.15.dev189 → verifiers-0.1.15.dev359}/verifiers/utils/version_utils.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: verifiers
|
|
3
|
-
Version: 0.1.15.
|
|
3
|
+
Version: 0.1.15.dev359
|
|
4
4
|
Summary: Verifiers: Environments for LLM Reinforcement Learning
|
|
5
5
|
Project-URL: Homepage, https://github.com/primeintellect-ai/verifiers
|
|
6
6
|
Project-URL: Documentation, https://github.com/primeintellect-ai/verifiers
|
|
@@ -15,62 +15,50 @@ Classifier: Intended Audience :: Developers
|
|
|
15
15
|
Classifier: Intended Audience :: Science/Research
|
|
16
16
|
Classifier: License :: OSI Approved :: MIT License
|
|
17
17
|
Classifier: Programming Language :: Python :: 3
|
|
18
|
-
Classifier: Programming Language :: Python :: 3.10
|
|
19
18
|
Classifier: Programming Language :: Python :: 3.11
|
|
20
19
|
Classifier: Programming Language :: Python :: 3.12
|
|
21
20
|
Classifier: Programming Language :: Python :: 3.13
|
|
22
21
|
Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
|
|
23
22
|
Classifier: Topic :: Software Development :: Libraries :: Python Modules
|
|
24
|
-
Requires-Python: <3.14,>=3.
|
|
23
|
+
Requires-Python: <3.14,>=3.11
|
|
25
24
|
Requires-Dist: aiohttp>=3.9.0
|
|
26
25
|
Requires-Dist: aiolimiter>=1.2.1
|
|
27
26
|
Requires-Dist: anthropic>=0.78.0
|
|
28
|
-
Requires-Dist: certifi
|
|
29
27
|
Requires-Dist: datasets<4.7.0,>=3.0.0
|
|
30
28
|
Requires-Dist: gepa
|
|
31
29
|
Requires-Dist: httpx>=0.27.0
|
|
32
|
-
Requires-Dist:
|
|
30
|
+
Requires-Dist: loguru>=0.7.0
|
|
33
31
|
Requires-Dist: math-verify>=0.8.0
|
|
34
32
|
Requires-Dist: mcp>=1.14.1
|
|
35
33
|
Requires-Dist: msgpack>=1.1.2
|
|
36
|
-
Requires-Dist: nest-asyncio>=1.6.0
|
|
37
34
|
Requires-Dist: numpy
|
|
38
35
|
Requires-Dist: openai-agents>=0.0.7
|
|
39
36
|
Requires-Dist: openai>=1.108.1
|
|
40
|
-
Requires-Dist:
|
|
41
|
-
Requires-Dist: prime-
|
|
42
|
-
Requires-Dist: prime-sandboxes>=0.2.25
|
|
37
|
+
Requires-Dist: prime-pydantic-config[toml]>=0.3.0.dev86
|
|
38
|
+
Requires-Dist: prime-sandboxes>=0.2.27
|
|
43
39
|
Requires-Dist: prime-tunnel>=0.1.8
|
|
44
40
|
Requires-Dist: pydantic>=2.11.9
|
|
45
|
-
Requires-Dist: pymupdf
|
|
46
41
|
Requires-Dist: pyzmq>=27.1.0
|
|
47
|
-
Requires-Dist:
|
|
42
|
+
Requires-Dist: renderers>=0.1.8.dev40
|
|
48
43
|
Requires-Dist: requests
|
|
49
44
|
Requires-Dist: rich
|
|
50
45
|
Requires-Dist: setproctitle>=1.3.0
|
|
51
46
|
Requires-Dist: tenacity>=8.5.0
|
|
52
|
-
Requires-Dist:
|
|
53
|
-
Requires-Dist: tomli; python_version < '3.11'
|
|
47
|
+
Requires-Dist: tomli-w>=1.0.0
|
|
54
48
|
Requires-Dist: typing-extensions; python_version < '3.12'
|
|
55
49
|
Requires-Dist: uvloop>=0.21.0; sys_platform != 'win32' and sys_platform != 'cygwin' and platform_python_implementation != 'PyPy'
|
|
56
50
|
Provides-Extra: browser
|
|
57
51
|
Requires-Dist: aiohttp>=3.9.0; extra == 'browser'
|
|
58
52
|
Requires-Dist: python-dotenv>=1.0.0; extra == 'browser'
|
|
59
53
|
Requires-Dist: stagehand>=3.0.0; extra == 'browser'
|
|
60
|
-
Provides-Extra:
|
|
61
|
-
Requires-Dist:
|
|
62
|
-
Provides-Extra:
|
|
63
|
-
Requires-Dist:
|
|
64
|
-
|
|
65
|
-
|
|
66
|
-
Requires-Dist:
|
|
67
|
-
|
|
68
|
-
Requires-Dist: tasksets[openreward]>=0.1.5; extra == 'openreward'
|
|
69
|
-
Provides-Extra: packages
|
|
70
|
-
Requires-Dist: harnesses>=0.1.2; extra == 'packages'
|
|
71
|
-
Requires-Dist: tasksets[openenv,openreward,ta]>=0.1.5; extra == 'packages'
|
|
72
|
-
Provides-Extra: renderers
|
|
73
|
-
Requires-Dist: renderers>=0.1.8.dev28; extra == 'renderers'
|
|
54
|
+
Provides-Extra: modal
|
|
55
|
+
Requires-Dist: modal>=1.4.0; extra == 'modal'
|
|
56
|
+
Provides-Extra: notebook
|
|
57
|
+
Requires-Dist: nest-asyncio>=1.6.0; extra == 'notebook'
|
|
58
|
+
Provides-Extra: quest
|
|
59
|
+
Requires-Dist: certifi; extra == 'quest'
|
|
60
|
+
Requires-Dist: pillow; extra == 'quest'
|
|
61
|
+
Requires-Dist: pymupdf; extra == 'quest'
|
|
74
62
|
Provides-Extra: rg
|
|
75
63
|
Requires-Dist: reasoning-gym; extra == 'rg'
|
|
76
64
|
Provides-Extra: rl
|
|
@@ -85,10 +73,8 @@ Requires-Dist: transformers>=4.56.2; extra == 'rl'
|
|
|
85
73
|
Requires-Dist: vllm<0.11.0,>=0.10.0; extra == 'rl'
|
|
86
74
|
Requires-Dist: wandb; extra == 'rl'
|
|
87
75
|
Provides-Extra: ta
|
|
88
|
-
Requires-Dist: nltk; extra == 'ta'
|
|
89
|
-
Requires-Dist: textarena; extra == 'ta'
|
|
90
|
-
Provides-Extra: tasksets
|
|
91
|
-
Requires-Dist: tasksets[openenv,openreward,ta]>=0.1.5; extra == 'tasksets'
|
|
76
|
+
Requires-Dist: nltk>=3.9.2; extra == 'ta'
|
|
77
|
+
Requires-Dist: textarena==0.7.4; extra == 'ta'
|
|
92
78
|
Description-Content-Type: text/markdown
|
|
93
79
|
|
|
94
80
|
<p align="center">
|
|
@@ -230,64 +216,6 @@ def load_environment(dataset_name: str = 'gsm8k') -> vf.Environment:
|
|
|
230
216
|
return env
|
|
231
217
|
```
|
|
232
218
|
|
|
233
|
-
For new environments with reusable tasksets, toolsets, custom programs, or
|
|
234
|
-
custom harnesses, use the v1 Taskset/Harness path:
|
|
235
|
-
```python
|
|
236
|
-
# my_env.py
|
|
237
|
-
import verifiers as vf
|
|
238
|
-
|
|
239
|
-
|
|
240
|
-
class MyTasksetConfig(vf.TasksetConfig):
|
|
241
|
-
system_prompt: vf.SystemPrompt = "Reverse text exactly."
|
|
242
|
-
|
|
243
|
-
|
|
244
|
-
class MyTaskset(vf.Taskset[MyTasksetConfig]):
|
|
245
|
-
def load_tasks(self, split: vf.TaskSplit = "train") -> vf.Tasks:
|
|
246
|
-
rows = [
|
|
247
|
-
{
|
|
248
|
-
"prompt": [{"role": "user", "content": "Reverse abc."}],
|
|
249
|
-
"answer": "cba",
|
|
250
|
-
"split": "train",
|
|
251
|
-
"max_turns": 1,
|
|
252
|
-
}
|
|
253
|
-
]
|
|
254
|
-
return [row for row in rows if row["split"] == split]
|
|
255
|
-
|
|
256
|
-
@vf.reward(weight=1.0)
|
|
257
|
-
async def contains_answer(self, task, state) -> float:
|
|
258
|
-
return float(task["answer"] in str(state.get("completion") or ""))
|
|
259
|
-
|
|
260
|
-
|
|
261
|
-
def load_taskset(config: MyTasksetConfig) -> MyTaskset:
|
|
262
|
-
return MyTaskset(config=config)
|
|
263
|
-
|
|
264
|
-
|
|
265
|
-
def load_environment(config: vf.EnvConfig) -> vf.Env:
|
|
266
|
-
"""Loader pattern for all Taskset/Harness environments."""
|
|
267
|
-
return vf.Env(
|
|
268
|
-
taskset=vf.load_taskset(config=config.taskset),
|
|
269
|
-
harness=vf.load_harness(config=config.harness),
|
|
270
|
-
)
|
|
271
|
-
```
|
|
272
|
-
The child loader annotation defines the taskset config shape; root
|
|
273
|
-
`load_environment` stays typed as `vf.EnvConfig`. See
|
|
274
|
-
**[BYO Harness](docs/byo-harness.md)** for the advanced v1 taskset/harness API.
|
|
275
|
-
Reusable taskset and harness packages live in `tasksets` and `harnesses`.
|
|
276
|
-
Install them with `uv add "verifiers[packages]"`, or with the narrower
|
|
277
|
-
`verifiers[tasksets]`, `verifiers[harnesses]`, and backend-specific extras. For
|
|
278
|
-
example, Harbor task directories can run through the bundled OpenCode CLI
|
|
279
|
-
harness with:
|
|
280
|
-
|
|
281
|
-
```python
|
|
282
|
-
from harnesses import OpenCode, OpenCodeConfig
|
|
283
|
-
from tasksets import HarborTaskset, HarborTasksetConfig
|
|
284
|
-
|
|
285
|
-
env = vf.Env(
|
|
286
|
-
taskset=HarborTaskset(config=HarborTasksetConfig(bundle_package=__name__)),
|
|
287
|
-
harness=OpenCode(config=OpenCodeConfig()),
|
|
288
|
-
)
|
|
289
|
-
```
|
|
290
|
-
|
|
291
219
|
The same environment package is the unit used by evals and `prime-rl`. The
|
|
292
220
|
trainer owns model, endpoint, sampling, and rollout count; v1-specific options
|
|
293
221
|
stay on the taskset or harness config that owns them:
|
|
@@ -346,8 +274,6 @@ prime eval run primeintellect/math-python
|
|
|
346
274
|
|
|
347
275
|
**[Environments](docs/environments.md)** — Create datasets, rubrics, and custom multi-turn interaction protocols.
|
|
348
276
|
|
|
349
|
-
**[BYO Harness](docs/byo-harness.md)** — Build v1 Taskset/Harness environments with custom tools, sandboxes, users, and custom programs.
|
|
350
|
-
|
|
351
277
|
**[Evaluation](docs/evaluation.md)** - Evaluate models using your environments.
|
|
352
278
|
|
|
353
279
|
**[Training](docs/training.md)** — Train models in your environments with reinforcement learning.
|
|
@@ -137,64 +137,6 @@ def load_environment(dataset_name: str = 'gsm8k') -> vf.Environment:
|
|
|
137
137
|
return env
|
|
138
138
|
```
|
|
139
139
|
|
|
140
|
-
For new environments with reusable tasksets, toolsets, custom programs, or
|
|
141
|
-
custom harnesses, use the v1 Taskset/Harness path:
|
|
142
|
-
```python
|
|
143
|
-
# my_env.py
|
|
144
|
-
import verifiers as vf
|
|
145
|
-
|
|
146
|
-
|
|
147
|
-
class MyTasksetConfig(vf.TasksetConfig):
|
|
148
|
-
system_prompt: vf.SystemPrompt = "Reverse text exactly."
|
|
149
|
-
|
|
150
|
-
|
|
151
|
-
class MyTaskset(vf.Taskset[MyTasksetConfig]):
|
|
152
|
-
def load_tasks(self, split: vf.TaskSplit = "train") -> vf.Tasks:
|
|
153
|
-
rows = [
|
|
154
|
-
{
|
|
155
|
-
"prompt": [{"role": "user", "content": "Reverse abc."}],
|
|
156
|
-
"answer": "cba",
|
|
157
|
-
"split": "train",
|
|
158
|
-
"max_turns": 1,
|
|
159
|
-
}
|
|
160
|
-
]
|
|
161
|
-
return [row for row in rows if row["split"] == split]
|
|
162
|
-
|
|
163
|
-
@vf.reward(weight=1.0)
|
|
164
|
-
async def contains_answer(self, task, state) -> float:
|
|
165
|
-
return float(task["answer"] in str(state.get("completion") or ""))
|
|
166
|
-
|
|
167
|
-
|
|
168
|
-
def load_taskset(config: MyTasksetConfig) -> MyTaskset:
|
|
169
|
-
return MyTaskset(config=config)
|
|
170
|
-
|
|
171
|
-
|
|
172
|
-
def load_environment(config: vf.EnvConfig) -> vf.Env:
|
|
173
|
-
"""Loader pattern for all Taskset/Harness environments."""
|
|
174
|
-
return vf.Env(
|
|
175
|
-
taskset=vf.load_taskset(config=config.taskset),
|
|
176
|
-
harness=vf.load_harness(config=config.harness),
|
|
177
|
-
)
|
|
178
|
-
```
|
|
179
|
-
The child loader annotation defines the taskset config shape; root
|
|
180
|
-
`load_environment` stays typed as `vf.EnvConfig`. See
|
|
181
|
-
**[BYO Harness](docs/byo-harness.md)** for the advanced v1 taskset/harness API.
|
|
182
|
-
Reusable taskset and harness packages live in `tasksets` and `harnesses`.
|
|
183
|
-
Install them with `uv add "verifiers[packages]"`, or with the narrower
|
|
184
|
-
`verifiers[tasksets]`, `verifiers[harnesses]`, and backend-specific extras. For
|
|
185
|
-
example, Harbor task directories can run through the bundled OpenCode CLI
|
|
186
|
-
harness with:
|
|
187
|
-
|
|
188
|
-
```python
|
|
189
|
-
from harnesses import OpenCode, OpenCodeConfig
|
|
190
|
-
from tasksets import HarborTaskset, HarborTasksetConfig
|
|
191
|
-
|
|
192
|
-
env = vf.Env(
|
|
193
|
-
taskset=HarborTaskset(config=HarborTasksetConfig(bundle_package=__name__)),
|
|
194
|
-
harness=OpenCode(config=OpenCodeConfig()),
|
|
195
|
-
)
|
|
196
|
-
```
|
|
197
|
-
|
|
198
140
|
The same environment package is the unit used by evals and `prime-rl`. The
|
|
199
141
|
trainer owns model, endpoint, sampling, and rollout count; v1-specific options
|
|
200
142
|
stay on the taskset or harness config that owns them:
|
|
@@ -253,8 +195,6 @@ prime eval run primeintellect/math-python
|
|
|
253
195
|
|
|
254
196
|
**[Environments](docs/environments.md)** — Create datasets, rubrics, and custom multi-turn interaction protocols.
|
|
255
197
|
|
|
256
|
-
**[BYO Harness](docs/byo-harness.md)** — Build v1 Taskset/Harness environments with custom tools, sandboxes, users, and custom programs.
|
|
257
|
-
|
|
258
198
|
**[Evaluation](docs/evaluation.md)** - Evaluate models using your environments.
|
|
259
199
|
|
|
260
200
|
**[Training](docs/training.md)** — Train models in your environments with reinforcement learning.
|
|
@@ -11,7 +11,7 @@ dynamic = ["version"] # derived from git tags via hatch-vcs
|
|
|
11
11
|
description = "Verifiers: Environments for LLM Reinforcement Learning"
|
|
12
12
|
readme = "README.md"
|
|
13
13
|
license = "MIT"
|
|
14
|
-
requires-python = ">=3.
|
|
14
|
+
requires-python = ">=3.11,<3.14"
|
|
15
15
|
keywords = ["reinforcement-learning", "llm", "rl", "grpo", "environments", "multi-turn", "agents", "agentic-rl", "tool-use", "train", "eval", "harness", "verifiers", "rlvr"]
|
|
16
16
|
classifiers = [
|
|
17
17
|
"Development Status :: 4 - Beta",
|
|
@@ -19,7 +19,6 @@ classifiers = [
|
|
|
19
19
|
"Intended Audience :: Science/Research",
|
|
20
20
|
"License :: OSI Approved :: MIT License",
|
|
21
21
|
"Programming Language :: Python :: 3",
|
|
22
|
-
"Programming Language :: Python :: 3.10",
|
|
23
22
|
"Programming Language :: Python :: 3.11",
|
|
24
23
|
"Programming Language :: Python :: 3.12",
|
|
25
24
|
"Programming Language :: Python :: 3.13",
|
|
@@ -30,35 +29,30 @@ classifiers = [
|
|
|
30
29
|
dependencies = [
|
|
31
30
|
"anthropic>=0.78.0",
|
|
32
31
|
"datasets>=3.0.0, <4.7.0",
|
|
33
|
-
"jinja2>=3.1.6",
|
|
34
32
|
"numpy",
|
|
35
33
|
"math-verify>=0.8.0",
|
|
36
34
|
"mcp>=1.14.1",
|
|
37
|
-
"nest-asyncio>=1.6.0", # for jupyter notebooks
|
|
38
35
|
"openai>=1.108.1",
|
|
39
36
|
"openai-agents>=0.0.7",
|
|
40
37
|
"prime-tunnel>=0.1.8",
|
|
41
|
-
"prime-sandboxes>=0.2.
|
|
38
|
+
"prime-sandboxes>=0.2.27",
|
|
42
39
|
"pydantic>=2.11.9",
|
|
43
40
|
"requests",
|
|
44
41
|
"rich",
|
|
45
42
|
"tenacity>=8.5.0",
|
|
46
|
-
"textual",
|
|
47
|
-
"tomli; python_version < '3.11'",
|
|
48
|
-
"typing_extensions; python_version < '3.12'",
|
|
49
43
|
"gepa",
|
|
50
44
|
"pyzmq>=27.1.0",
|
|
51
45
|
"msgpack>=1.1.2",
|
|
52
46
|
"aiolimiter>=1.2.1",
|
|
53
47
|
"setproctitle>=1.3.0",
|
|
54
|
-
"regex<2026.4.4",
|
|
55
48
|
"httpx>=0.27.0",
|
|
56
49
|
"aiohttp>=3.9.0",
|
|
57
|
-
"
|
|
58
|
-
"pillow",
|
|
59
|
-
"certifi",
|
|
60
|
-
"prime-pydantic-config[toml]",
|
|
50
|
+
"prime-pydantic-config[toml]>=0.3.0.dev86",
|
|
61
51
|
"uvloop>=0.21.0; sys_platform != 'win32' and sys_platform != 'cygwin' and platform_python_implementation != 'PyPy'",
|
|
52
|
+
"loguru>=0.7.0",
|
|
53
|
+
"tomli-w>=1.0.0",
|
|
54
|
+
"renderers>=0.1.8.dev40",
|
|
55
|
+
"typing_extensions; python_version < '3.12'",
|
|
62
56
|
]
|
|
63
57
|
|
|
64
58
|
[dependency-groups]
|
|
@@ -76,12 +70,19 @@ dev = [
|
|
|
76
70
|
"stagehand>=3.0.0",
|
|
77
71
|
"aiohttp>=3.9.0",
|
|
78
72
|
"python-dotenv>=1.0.0",
|
|
79
|
-
"
|
|
80
|
-
"
|
|
81
|
-
"tasksets[openenv,openreward,ta]",
|
|
73
|
+
"nltk>=3.9.2",
|
|
74
|
+
"textarena==0.7.4",
|
|
82
75
|
]
|
|
83
|
-
|
|
84
|
-
|
|
76
|
+
# v1 example environments vendored under environments/, resolved by id via [tool.uv.sources].
|
|
77
|
+
# Installed by default (tool.uv.default-groups) so the v1 CLIs (eval <id>, serve) run straight
|
|
78
|
+
# from this repo. The built-in tasksets/harnesses ship inside verifiers (verifiers/v1/...).
|
|
79
|
+
examples = [
|
|
80
|
+
"compact",
|
|
81
|
+
"gsm8k-v1", "wikispeedia-v1", "glossary-v1", "deepwiki-v1", "wiki-search-v1", "code-golf-v1",
|
|
82
|
+
"reverse-text-v1", "math-env-v1", "aime24-v1", "color-codeword-v1",
|
|
83
|
+
"wordle-v1", "terminal-bench-2-v1", "alphabet-sort-v1",
|
|
84
|
+
"r2e-gym-v1", "scaleswe-v1", "swelego-v1", "scratchpad-v1",
|
|
85
|
+
"general-agent-v1", "swebench-verified-v1", "tau2-bench-v1",
|
|
85
86
|
]
|
|
86
87
|
|
|
87
88
|
[project.optional-dependencies]
|
|
@@ -89,36 +90,24 @@ rg = [
|
|
|
89
90
|
"reasoning-gym",
|
|
90
91
|
]
|
|
91
92
|
ta = [
|
|
92
|
-
"
|
|
93
|
-
"
|
|
94
|
-
]
|
|
95
|
-
tasksets = [
|
|
96
|
-
"tasksets[openenv,openreward,ta]>=0.1.5",
|
|
97
|
-
]
|
|
98
|
-
harnesses = [
|
|
99
|
-
"harnesses>=0.1.2",
|
|
100
|
-
]
|
|
101
|
-
packages = [
|
|
102
|
-
"harnesses>=0.1.2",
|
|
103
|
-
"tasksets[openenv,openreward,ta]>=0.1.5",
|
|
93
|
+
"textarena==0.7.4",
|
|
94
|
+
"nltk>=3.9.2",
|
|
104
95
|
]
|
|
105
96
|
browser = [
|
|
106
97
|
"stagehand>=3.0.0",
|
|
107
98
|
"aiohttp>=3.9.0",
|
|
108
99
|
"python-dotenv>=1.0.0",
|
|
109
100
|
]
|
|
110
|
-
|
|
111
|
-
"
|
|
101
|
+
modal = [
|
|
102
|
+
"modal>=1.4.0",
|
|
112
103
|
]
|
|
113
|
-
|
|
114
|
-
"
|
|
104
|
+
notebook = [
|
|
105
|
+
"nest-asyncio>=1.6.0",
|
|
115
106
|
]
|
|
116
|
-
|
|
117
|
-
"
|
|
118
|
-
"
|
|
119
|
-
|
|
120
|
-
renderers = [
|
|
121
|
-
"renderers>=0.1.8.dev28",
|
|
107
|
+
quest = [
|
|
108
|
+
"certifi",
|
|
109
|
+
"pillow",
|
|
110
|
+
"pymupdf",
|
|
122
111
|
]
|
|
123
112
|
rl = [
|
|
124
113
|
"torch>=2.8.0,<2.9.0",
|
|
@@ -137,48 +126,7 @@ rl = [
|
|
|
137
126
|
preview = true
|
|
138
127
|
required-version = ">=0.11.1"
|
|
139
128
|
exclude-newer = "7 days"
|
|
140
|
-
|
|
141
|
-
[
|
|
142
|
-
{ extra = "openenv" },
|
|
143
|
-
{ group = "policy" },
|
|
144
|
-
],
|
|
145
|
-
[
|
|
146
|
-
{ extra = "nemogym" },
|
|
147
|
-
{ extra = "openenv" },
|
|
148
|
-
],
|
|
149
|
-
[
|
|
150
|
-
{ extra = "nemogym" },
|
|
151
|
-
{ group = "dev" },
|
|
152
|
-
],
|
|
153
|
-
[
|
|
154
|
-
{ extra = "nemogym" },
|
|
155
|
-
{ extra = "tasksets" },
|
|
156
|
-
],
|
|
157
|
-
[
|
|
158
|
-
{ extra = "nemogym" },
|
|
159
|
-
{ extra = "packages" },
|
|
160
|
-
],
|
|
161
|
-
[
|
|
162
|
-
{ extra = "ta" },
|
|
163
|
-
{ group = "policy" },
|
|
164
|
-
],
|
|
165
|
-
[
|
|
166
|
-
{ extra = "tasksets" },
|
|
167
|
-
{ group = "policy" },
|
|
168
|
-
],
|
|
169
|
-
[
|
|
170
|
-
{ extra = "packages" },
|
|
171
|
-
{ group = "policy" },
|
|
172
|
-
],
|
|
173
|
-
[
|
|
174
|
-
{ extra = "openreward" },
|
|
175
|
-
{ group = "policy" },
|
|
176
|
-
],
|
|
177
|
-
[
|
|
178
|
-
{ group = "dev" },
|
|
179
|
-
{ group = "policy" },
|
|
180
|
-
],
|
|
181
|
-
]
|
|
129
|
+
default-groups = ["dev", "examples"]
|
|
182
130
|
|
|
183
131
|
[[tool.uv.index]]
|
|
184
132
|
name = "pypi"
|
|
@@ -186,8 +134,28 @@ url = "https://pypi.org/simple"
|
|
|
186
134
|
default = true
|
|
187
135
|
|
|
188
136
|
[tool.uv.sources]
|
|
189
|
-
|
|
190
|
-
|
|
137
|
+
compact = { path = "environments/compact", editable = true }
|
|
138
|
+
glossary-v1 = { path = "environments/glossary_v1", editable = true }
|
|
139
|
+
deepwiki-v1 = { path = "environments/deepwiki_v1", editable = true }
|
|
140
|
+
wikispeedia-v1 = { path = "environments/wikispeedia_v1", editable = true }
|
|
141
|
+
wiki-search-v1 = { path = "environments/wiki_search_v1", editable = true }
|
|
142
|
+
gsm8k-v1 = { path = "environments/gsm8k_v1", editable = true }
|
|
143
|
+
code-golf-v1 = { path = "environments/code_golf_v1", editable = true }
|
|
144
|
+
reverse-text-v1 = { path = "environments/reverse_text_v1", editable = true }
|
|
145
|
+
reverse-text = { path = "environments/reverse_text", editable = true }
|
|
146
|
+
wordle-v1 = { path = "environments/wordle_v1", editable = true }
|
|
147
|
+
terminal-bench-2-v1 = { path = "environments/terminal_bench_2_v1", editable = true }
|
|
148
|
+
math-env-v1 = { path = "environments/math_env_v1", editable = true }
|
|
149
|
+
aime24-v1 = { path = "environments/aime24_v1", editable = true }
|
|
150
|
+
color-codeword-v1 = { path = "environments/color_codeword_v1", editable = true }
|
|
151
|
+
r2e-gym-v1 = { path = "environments/r2e_gym_v1", editable = true }
|
|
152
|
+
scaleswe-v1 = { path = "environments/scaleswe_v1", editable = true }
|
|
153
|
+
swelego-v1 = { path = "environments/swelego_v1", editable = true }
|
|
154
|
+
alphabet-sort-v1 = { path = "environments/alphabet_sort_v1", editable = true }
|
|
155
|
+
scratchpad-v1 = { path = "environments/scratchpad_v1", editable = true }
|
|
156
|
+
general-agent-v1 = { path = "environments/general_agent_v1", editable = true }
|
|
157
|
+
swebench-verified-v1 = { path = "environments/swebench_verified_v1", editable = true }
|
|
158
|
+
tau2-bench-v1 = { path = "environments/tau2_bench_v1", editable = true }
|
|
191
159
|
|
|
192
160
|
[tool.uv.exclude-newer-package]
|
|
193
161
|
# PrimeIntellect-published on PyPI (trusted publisher)
|
|
@@ -195,9 +163,6 @@ prime-tunnel = false
|
|
|
195
163
|
prime-sandboxes = false
|
|
196
164
|
prime-pydantic-config = false
|
|
197
165
|
renderers = false
|
|
198
|
-
openenv-core = false
|
|
199
|
-
harnesses = false
|
|
200
|
-
tasksets = false
|
|
201
166
|
|
|
202
167
|
[tool.uv.extra-build-dependencies]
|
|
203
168
|
flash-attn = [{ requirement = "torch", match-runtime = true }]
|
|
@@ -205,13 +170,12 @@ flash-attn = [{ requirement = "torch", match-runtime = true }]
|
|
|
205
170
|
[tool.uv.extra-build-variables]
|
|
206
171
|
flash-attn = { FLASH_ATTENTION_SKIP_CUDA_BUILD = "TRUE" }
|
|
207
172
|
|
|
208
|
-
[tool.ruff]
|
|
209
|
-
exclude = [
|
|
210
|
-
".semgrep",
|
|
211
|
-
]
|
|
212
|
-
|
|
213
173
|
[project.scripts]
|
|
214
174
|
vf-eval = "verifiers.scripts.eval:main"
|
|
175
|
+
eval = "verifiers.v1.cli.eval.main:main"
|
|
176
|
+
validate = "verifiers.v1.cli.validate:main"
|
|
177
|
+
serve = "verifiers.v1.cli.serve:main"
|
|
178
|
+
init = "verifiers.v1.cli.init:main"
|
|
215
179
|
vf-gepa = "verifiers.scripts.gepa:main"
|
|
216
180
|
vf-init = "verifiers.scripts.init:main"
|
|
217
181
|
vf-install = "verifiers.scripts.install:main"
|
|
@@ -271,6 +235,7 @@ Issues = "https://github.com/primeintellect-ai/verifiers/issues"
|
|
|
271
235
|
[tool.pytest.ini_options]
|
|
272
236
|
minversion = "7.0"
|
|
273
237
|
testpaths = ["tests"]
|
|
238
|
+
pythonpath = ["tests/v1/fixtures"]
|
|
274
239
|
python_files = ["test_*.py"]
|
|
275
240
|
python_classes = ["Test*"]
|
|
276
241
|
python_functions = ["test_*"]
|
|
@@ -281,9 +246,22 @@ addopts = [
|
|
|
281
246
|
"--quiet",
|
|
282
247
|
]
|
|
283
248
|
markers = [
|
|
284
|
-
"
|
|
249
|
+
"e2e: marks end-to-end eval-run tests (need a model API key; skipped without one)",
|
|
285
250
|
"integration: marks tests as integration tests",
|
|
286
|
-
"
|
|
251
|
+
"slow: marks slow-running tests (deselect with -m 'not slow')",
|
|
252
|
+
# v1 e2e matrix axes — select subsets with `-m` (see tests/v1/conftest.py). A mark is applied
|
|
253
|
+
# per axis, so it matches every case touching that value on any axis.
|
|
254
|
+
"subprocess: v1 e2e cases touching the subprocess (host) runtime",
|
|
255
|
+
"docker: v1 e2e cases touching the docker runtime",
|
|
256
|
+
"prime: v1 e2e cases touching the prime runtime (real sandbox + tunnel; local-only, excluded in CI)",
|
|
257
|
+
"modal: v1 e2e cases touching the modal runtime (local-only setup; excluded in CI)",
|
|
258
|
+
"colocated: v1 e2e cases with a user/tool server colocated in the harness runtime",
|
|
259
|
+
"shared: v1 e2e cases with a shared (one-per-eval) tool server",
|
|
260
|
+
"default: v1 e2e cases on the default harness",
|
|
261
|
+
"bash: v1 e2e cases on the bash harness",
|
|
262
|
+
"rlm: v1 e2e cases on the rlm harness",
|
|
263
|
+
"kimi_code: v1 e2e cases on the kimi-code harness",
|
|
264
|
+
"codex: v1 e2e cases on the codex harness",
|
|
287
265
|
"unit: marks tests as unit tests",
|
|
288
266
|
"asyncio: marks tests as async tests",
|
|
289
267
|
"parsers: marks tests for parser components",
|
|
@@ -309,7 +287,9 @@ unknown-argument = "warn"
|
|
|
309
287
|
redundant-cast = "ignore"
|
|
310
288
|
|
|
311
289
|
[tool.ty.src]
|
|
312
|
-
|
|
290
|
+
# verifiers/v1 is vendored (heavily typed upstream for a different checker); it is not part
|
|
291
|
+
# of verifiers' ty gate. environments/ are user-authored envs, also out of scope.
|
|
292
|
+
exclude = ["environments", "verifiers/v1"]
|
|
313
293
|
|
|
314
294
|
[[tool.ty.overrides]]
|
|
315
295
|
include = ["verifiers/envs/experimental/composable/tasksets/**"]
|
|
@@ -1,7 +1,5 @@
|
|
|
1
1
|
import os
|
|
2
|
-
import importlib.util
|
|
3
2
|
import subprocess
|
|
4
|
-
import sys
|
|
5
3
|
from pathlib import Path
|
|
6
4
|
|
|
7
5
|
import pytest
|
|
@@ -49,6 +47,14 @@ def get_environments() -> list[Path]:
|
|
|
49
47
|
# Filter out skipped environments
|
|
50
48
|
all_envs = [env for env in all_envs if env.name not in SKIPPED_ENVS]
|
|
51
49
|
|
|
50
|
+
# These are v0 smoke tests (vf.load_environment + vf-eval). The v1 plugins (the `_v1`
|
|
51
|
+
# tasksets + the `compact` harness) are id-referenced and covered by tests/v1/test_envs.py.
|
|
52
|
+
all_envs = [
|
|
53
|
+
env
|
|
54
|
+
for env in all_envs
|
|
55
|
+
if not env.name.endswith("_v1") and env.name != "compact"
|
|
56
|
+
]
|
|
57
|
+
|
|
52
58
|
# Filter environments if CHANGED_ENVS is set (for PRs)
|
|
53
59
|
changed_envs = os.getenv("CHANGED_ENVS")
|
|
54
60
|
if changed_envs == "none":
|
|
@@ -92,55 +98,6 @@ def test_readme_exists(env_dir: Path):
|
|
|
92
98
|
assert (env_dir / "README.md").exists(), "README.md does not exist"
|
|
93
99
|
|
|
94
100
|
|
|
95
|
-
def test_alphabet_sort_v1_validates_parameters():
|
|
96
|
-
module_path = Path("environments/alphabet_sort/alphabet_sort_v1.py").resolve()
|
|
97
|
-
spec = importlib.util.spec_from_file_location("alphabet_sort_v1_test", module_path)
|
|
98
|
-
assert spec is not None and spec.loader is not None
|
|
99
|
-
module = importlib.util.module_from_spec(spec)
|
|
100
|
-
sys.modules[spec.name] = module
|
|
101
|
-
spec.loader.exec_module(module)
|
|
102
|
-
|
|
103
|
-
with pytest.raises(ValueError, match="min_turns must be at least 1"):
|
|
104
|
-
module.AlphabetSortTaskset(config=module.AlphabetSortTasksetConfig(min_turns=0))
|
|
105
|
-
with pytest.raises(
|
|
106
|
-
ValueError, match="min_turns must be less than or equal to max_turns"
|
|
107
|
-
):
|
|
108
|
-
module.AlphabetSortTaskset(
|
|
109
|
-
config=module.AlphabetSortTasksetConfig(min_turns=3, max_turns=2)
|
|
110
|
-
)
|
|
111
|
-
with pytest.raises(ValueError, match="min_names_per_turn must be at least 1"):
|
|
112
|
-
module.AlphabetSortTaskset(
|
|
113
|
-
config=module.AlphabetSortTasksetConfig(min_names_per_turn=0)
|
|
114
|
-
)
|
|
115
|
-
with pytest.raises(
|
|
116
|
-
ValueError,
|
|
117
|
-
match="min_names_per_turn must be less than or equal to max_names_per_turn",
|
|
118
|
-
):
|
|
119
|
-
module.AlphabetSortTaskset(
|
|
120
|
-
config=module.AlphabetSortTasksetConfig(
|
|
121
|
-
min_names_per_turn=3,
|
|
122
|
-
max_names_per_turn=2,
|
|
123
|
-
)
|
|
124
|
-
)
|
|
125
|
-
|
|
126
|
-
|
|
127
|
-
@pytest.mark.parametrize("env_name", ["alphabet_sort", "math_python"])
|
|
128
|
-
def test_v1_wrapper_rejects_unknown_kwargs(env_name: str):
|
|
129
|
-
module_path = Path("environments") / env_name / f"{env_name}.py"
|
|
130
|
-
spec = importlib.util.spec_from_file_location(
|
|
131
|
-
f"{env_name}_wrapper_test", module_path
|
|
132
|
-
)
|
|
133
|
-
assert spec is not None and spec.loader is not None
|
|
134
|
-
module = importlib.util.module_from_spec(spec)
|
|
135
|
-
sys.modules[spec.name] = module
|
|
136
|
-
spec.loader.exec_module(module)
|
|
137
|
-
|
|
138
|
-
with pytest.raises(
|
|
139
|
-
TypeError, match="Unsupported v1 load_environment kwargs: extra"
|
|
140
|
-
):
|
|
141
|
-
module.load_environment(v1=True, extra=True)
|
|
142
|
-
|
|
143
|
-
|
|
144
101
|
@pytest.mark.slow
|
|
145
102
|
@pytest.mark.parametrize("env_dir", get_environments(), ids=lambda x: x.name)
|
|
146
103
|
def test_env(env_dir: Path, tmp_path_factory: pytest.TempPathFactory):
|
|
@@ -154,14 +111,8 @@ def test_env(env_dir: Path, tmp_path_factory: pytest.TempPathFactory):
|
|
|
154
111
|
cmd = (
|
|
155
112
|
f"cd {tmp_venv_dir} && uv venv --clear && source .venv/bin/activate && "
|
|
156
113
|
"uv pip install "
|
|
157
|
-
"--exclude-newer-package prime-pydantic-config=2026-05-20T00:00:00Z "
|
|
158
114
|
f"{repo_root.as_posix()} && "
|
|
159
115
|
"uv pip install "
|
|
160
|
-
"--exclude-newer-package prime-pydantic-config=2026-05-20T00:00:00Z "
|
|
161
|
-
f"{(repo_root / 'packages' / 'tasksets').as_posix()} "
|
|
162
|
-
f"{(repo_root / 'packages' / 'harnesses').as_posix()} && "
|
|
163
|
-
"uv pip install "
|
|
164
|
-
"--exclude-newer-package prime-pydantic-config=2026-05-20T00:00:00Z "
|
|
165
116
|
f"{env_dir.absolute().as_posix()}"
|
|
166
117
|
)
|
|
167
118
|
try:
|