verifiers 0.2.2.dev62__tar.gz → 0.2.2.dev64__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {verifiers-0.2.2.dev62 → verifiers-0.2.2.dev64}/PKG-INFO +1 -1
- {verifiers-0.2.2.dev62 → verifiers-0.2.2.dev64}/tests/v1/conftest.py +1 -1
- {verifiers-0.2.2.dev62 → verifiers-0.2.2.dev64}/tests/v1/test_judges.py +1 -1
- {verifiers-0.2.2.dev62 → verifiers-0.2.2.dev64}/verifiers/v1/__init__.py +43 -43
- {verifiers-0.2.2.dev62 → verifiers-0.2.2.dev64}/verifiers/v1/agent.py +6 -2
- {verifiers-0.2.2.dev62 → verifiers-0.2.2.dev64}/verifiers/v1/cli/dashboard/eval.py +2 -2
- {verifiers-0.2.2.dev62 → verifiers-0.2.2.dev64}/verifiers/v1/cli/debug.py +1 -1
- {verifiers-0.2.2.dev62 → verifiers-0.2.2.dev64}/verifiers/v1/cli/eval/main.py +1 -1
- {verifiers-0.2.2.dev62 → verifiers-0.2.2.dev64}/verifiers/v1/cli/eval/runner.py +9 -5
- {verifiers-0.2.2.dev62 → verifiers-0.2.2.dev64}/verifiers/v1/cli/validate.py +1 -1
- {verifiers-0.2.2.dev62 → verifiers-0.2.2.dev64}/verifiers/v1/configs/agent.py +4 -1
- {verifiers-0.2.2.dev62 → verifiers-0.2.2.dev64}/verifiers/v1/configs/cli/debug.py +1 -1
- {verifiers-0.2.2.dev62 → verifiers-0.2.2.dev64}/verifiers/v1/configs/cli/env.py +1 -1
- {verifiers-0.2.2.dev62 → verifiers-0.2.2.dev64}/verifiers/v1/configs/cli/replay.py +1 -1
- {verifiers-0.2.2.dev62 → verifiers-0.2.2.dev64}/verifiers/v1/configs/cli/validate.py +1 -1
- {verifiers-0.2.2.dev62 → verifiers-0.2.2.dev64}/verifiers/v1/configs/env.py +5 -2
- {verifiers-0.2.2.dev62 → verifiers-0.2.2.dev64}/verifiers/v1/configs/judge.py +1 -1
- {verifiers-0.2.2.dev62 → verifiers-0.2.2.dev64}/verifiers/v1/env.py +9 -3
- {verifiers-0.2.2.dev62 → verifiers-0.2.2.dev64}/verifiers/v1/harness.py +1 -1
- {verifiers-0.2.2.dev62 → verifiers-0.2.2.dev64}/verifiers/v1/harnesses/rlm/harness.py +1 -1
- {verifiers-0.2.2.dev62 → verifiers-0.2.2.dev64}/verifiers/v1/interception/__init__.py +7 -4
- {verifiers-0.2.2.dev62 → verifiers-0.2.2.dev64}/verifiers/v1/interception/base.py +3 -5
- {verifiers-0.2.2.dev62 → verifiers-0.2.2.dev64}/verifiers/v1/interception/pool.py +14 -6
- {verifiers-0.2.2.dev62 → verifiers-0.2.2.dev64}/verifiers/v1/interception/server.py +39 -27
- {verifiers-0.2.2.dev62 → verifiers-0.2.2.dev64}/verifiers/v1/interception/tunnel/prime.py +1 -1
- {verifiers-0.2.2.dev62 → verifiers-0.2.2.dev64}/verifiers/v1/judge.py +2 -2
- {verifiers-0.2.2.dev62 → verifiers-0.2.2.dev64}/verifiers/v1/legacy.py +6 -2
- {verifiers-0.2.2.dev62 → verifiers-0.2.2.dev64}/verifiers/v1/mcp/launch.py +81 -35
- {verifiers-0.2.2.dev62 → verifiers-0.2.2.dev64}/verifiers/v1/mcp/server.py +51 -14
- {verifiers-0.2.2.dev62 → verifiers-0.2.2.dev64}/verifiers/v1/mcp/toolset.py +1 -1
- {verifiers-0.2.2.dev62 → verifiers-0.2.2.dev64}/verifiers/v1/rollout.py +15 -5
- {verifiers-0.2.2.dev62 → verifiers-0.2.2.dev64}/verifiers/v1/serve/pool.py +1 -1
- {verifiers-0.2.2.dev62 → verifiers-0.2.2.dev64}/verifiers/v1/serve/server.py +1 -1
- {verifiers-0.2.2.dev62 → verifiers-0.2.2.dev64}/verifiers/v1/state.py +2 -5
- {verifiers-0.2.2.dev62 → verifiers-0.2.2.dev64}/verifiers/v1/task.py +3 -3
- {verifiers-0.2.2.dev62 → verifiers-0.2.2.dev64}/verifiers/v1/taskset.py +2 -1
- {verifiers-0.2.2.dev62 → verifiers-0.2.2.dev64}/verifiers/v1/tasksets/harbor/taskset.py +2 -2
- {verifiers-0.2.2.dev62 → verifiers-0.2.2.dev64}/verifiers/v1/tasksets/lean/taskset.py +1 -1
- verifiers-0.2.2.dev62/verifiers/v1/utils/sampling.py +0 -24
- {verifiers-0.2.2.dev62 → verifiers-0.2.2.dev64}/.gitignore +0 -0
- {verifiers-0.2.2.dev62 → verifiers-0.2.2.dev64}/LICENSE +0 -0
- {verifiers-0.2.2.dev62 → verifiers-0.2.2.dev64}/README.md +0 -0
- {verifiers-0.2.2.dev62 → verifiers-0.2.2.dev64}/pyproject.toml +0 -0
- {verifiers-0.2.2.dev62 → verifiers-0.2.2.dev64}/tests/__init__.py +0 -0
- {verifiers-0.2.2.dev62 → verifiers-0.2.2.dev64}/tests/conftest.py +0 -0
- {verifiers-0.2.2.dev62 → verifiers-0.2.2.dev64}/tests/test_browser_env.py +0 -0
- {verifiers-0.2.2.dev62 → verifiers-0.2.2.dev64}/tests/test_build_script.py +0 -0
- {verifiers-0.2.2.dev62 → verifiers-0.2.2.dev64}/tests/test_cli_agent_env.py +0 -0
- {verifiers-0.2.2.dev62 → verifiers-0.2.2.dev64}/tests/test_client_auth_errors.py +0 -0
- {verifiers-0.2.2.dev62 → verifiers-0.2.2.dev64}/tests/test_client_config.py +0 -0
- {verifiers-0.2.2.dev62 → verifiers-0.2.2.dev64}/tests/test_client_multimodal_types.py +0 -0
- {verifiers-0.2.2.dev62 → verifiers-0.2.2.dev64}/tests/test_composable_env.py +0 -0
- {verifiers-0.2.2.dev62 → verifiers-0.2.2.dev64}/tests/test_context_token_metrics.py +0 -0
- {verifiers-0.2.2.dev62 → verifiers-0.2.2.dev64}/tests/test_decorator_ranks.py +0 -0
- {verifiers-0.2.2.dev62 → verifiers-0.2.2.dev64}/tests/test_endpoint_registry.py +0 -0
- {verifiers-0.2.2.dev62 → verifiers-0.2.2.dev64}/tests/test_env_group.py +0 -0
- {verifiers-0.2.2.dev62 → verifiers-0.2.2.dev64}/tests/test_env_server.py +0 -0
- {verifiers-0.2.2.dev62 → verifiers-0.2.2.dev64}/tests/test_environment.py +0 -0
- {verifiers-0.2.2.dev62 → verifiers-0.2.2.dev64}/tests/test_environment_extra.py +0 -0
- {verifiers-0.2.2.dev62 → verifiers-0.2.2.dev64}/tests/test_error_chain.py +0 -0
- {verifiers-0.2.2.dev62 → verifiers-0.2.2.dev64}/tests/test_eval_display.py +0 -0
- {verifiers-0.2.2.dev62 → verifiers-0.2.2.dev64}/tests/test_eval_utils.py +0 -0
- {verifiers-0.2.2.dev62 → verifiers-0.2.2.dev64}/tests/test_gepa_cli.py +0 -0
- {verifiers-0.2.2.dev62 → verifiers-0.2.2.dev64}/tests/test_gepa_utils.py +0 -0
- {verifiers-0.2.2.dev62 → verifiers-0.2.2.dev64}/tests/test_gym_env.py +0 -0
- {verifiers-0.2.2.dev62 → verifiers-0.2.2.dev64}/tests/test_harbor_env_mcp.py +0 -0
- {verifiers-0.2.2.dev62 → verifiers-0.2.2.dev64}/tests/test_imports.py +0 -0
- {verifiers-0.2.2.dev62 → verifiers-0.2.2.dev64}/tests/test_init_script.py +0 -0
- {verifiers-0.2.2.dev62 → verifiers-0.2.2.dev64}/tests/test_install_utils.py +0 -0
- {verifiers-0.2.2.dev62 → verifiers-0.2.2.dev64}/tests/test_interception_utils.py +0 -0
- {verifiers-0.2.2.dev62 → verifiers-0.2.2.dev64}/tests/test_logging.py +0 -0
- {verifiers-0.2.2.dev62 → verifiers-0.2.2.dev64}/tests/test_math_rubric.py +0 -0
- {verifiers-0.2.2.dev62 → verifiers-0.2.2.dev64}/tests/test_maybe_think_parser.py +0 -0
- {verifiers-0.2.2.dev62 → verifiers-0.2.2.dev64}/tests/test_message_utils.py +0 -0
- {verifiers-0.2.2.dev62 → verifiers-0.2.2.dev64}/tests/test_message_utils_multimodal.py +0 -0
- {verifiers-0.2.2.dev62 → verifiers-0.2.2.dev64}/tests/test_multiturn_env.py +0 -0
- {verifiers-0.2.2.dev62 → verifiers-0.2.2.dev64}/tests/test_nemorl_client.py +0 -0
- {verifiers-0.2.2.dev62 → verifiers-0.2.2.dev64}/tests/test_openai_chat_completions_token_client.py +0 -0
- {verifiers-0.2.2.dev62 → verifiers-0.2.2.dev64}/tests/test_openai_responses_client.py +0 -0
- {verifiers-0.2.2.dev62 → verifiers-0.2.2.dev64}/tests/test_opencode_rlm_env.py +0 -0
- {verifiers-0.2.2.dev62 → verifiers-0.2.2.dev64}/tests/test_openenv_client.py +0 -0
- {verifiers-0.2.2.dev62 → verifiers-0.2.2.dev64}/tests/test_parser.py +0 -0
- {verifiers-0.2.2.dev62 → verifiers-0.2.2.dev64}/tests/test_path_utils.py +0 -0
- {verifiers-0.2.2.dev62 → verifiers-0.2.2.dev64}/tests/test_per_turn_timing.py +0 -0
- {verifiers-0.2.2.dev62 → verifiers-0.2.2.dev64}/tests/test_pricing_utils.py +0 -0
- {verifiers-0.2.2.dev62 → verifiers-0.2.2.dev64}/tests/test_prime_plugin.py +0 -0
- {verifiers-0.2.2.dev62 → verifiers-0.2.2.dev64}/tests/test_renderer_client.py +0 -0
- {verifiers-0.2.2.dev62 → verifiers-0.2.2.dev64}/tests/test_renderer_e2e.py +0 -0
- {verifiers-0.2.2.dev62 → verifiers-0.2.2.dev64}/tests/test_rlm_composable_env.py +0 -0
- {verifiers-0.2.2.dev62 → verifiers-0.2.2.dev64}/tests/test_rubric.py +0 -0
- {verifiers-0.2.2.dev62 → verifiers-0.2.2.dev64}/tests/test_rubric_group.py +0 -0
- {verifiers-0.2.2.dev62 → verifiers-0.2.2.dev64}/tests/test_sandbox_debug_env.py +0 -0
- {verifiers-0.2.2.dev62 → verifiers-0.2.2.dev64}/tests/test_sandbox_env.py +0 -0
- {verifiers-0.2.2.dev62 → verifiers-0.2.2.dev64}/tests/test_sandbox_mixin.py +0 -0
- {verifiers-0.2.2.dev62 → verifiers-0.2.2.dev64}/tests/test_save_utils.py +0 -0
- {verifiers-0.2.2.dev62 → verifiers-0.2.2.dev64}/tests/test_setup_script.py +0 -0
- {verifiers-0.2.2.dev62 → verifiers-0.2.2.dev64}/tests/test_singleturn_env.py +0 -0
- {verifiers-0.2.2.dev62 → verifiers-0.2.2.dev64}/tests/test_stateful_tool_env.py +0 -0
- {verifiers-0.2.2.dev62 → verifiers-0.2.2.dev64}/tests/test_think_parser.py +0 -0
- {verifiers-0.2.2.dev62 → verifiers-0.2.2.dev64}/tests/test_tool_env.py +0 -0
- {verifiers-0.2.2.dev62 → verifiers-0.2.2.dev64}/tests/test_tool_utils.py +0 -0
- {verifiers-0.2.2.dev62 → verifiers-0.2.2.dev64}/tests/test_trajectory_processing.py +0 -0
- {verifiers-0.2.2.dev62 → verifiers-0.2.2.dev64}/tests/test_tui_info_formatting.py +0 -0
- {verifiers-0.2.2.dev62 → verifiers-0.2.2.dev64}/tests/test_types.py +0 -0
- {verifiers-0.2.2.dev62 → verifiers-0.2.2.dev64}/tests/test_xml_parser.py +0 -0
- {verifiers-0.2.2.dev62 → verifiers-0.2.2.dev64}/tests/v1/fixtures/counter_tool_v1.py +0 -0
- {verifiers-0.2.2.dev62 → verifiers-0.2.2.dev64}/tests/v1/fixtures/duet_v1.py +0 -0
- {verifiers-0.2.2.dev62 → verifiers-0.2.2.dev64}/tests/v1/fixtures/echo_acp_resume_v1.py +0 -0
- {verifiers-0.2.2.dev62 → verifiers-0.2.2.dev64}/tests/v1/fixtures/echo_agentic_v1.py +0 -0
- {verifiers-0.2.2.dev62 → verifiers-0.2.2.dev64}/tests/v1/fixtures/echo_multi_v0.py +0 -0
- {verifiers-0.2.2.dev62 → verifiers-0.2.2.dev64}/tests/v1/fixtures/echo_tool_v1.py +0 -0
- {verifiers-0.2.2.dev62 → verifiers-0.2.2.dev64}/tests/v1/fixtures/echo_user_sim_v1.py +0 -0
- {verifiers-0.2.2.dev62 → verifiers-0.2.2.dev64}/tests/v1/fixtures/echo_v0.py +0 -0
- {verifiers-0.2.2.dev62 → verifiers-0.2.2.dev64}/tests/v1/fixtures/echo_v1.py +0 -0
- {verifiers-0.2.2.dev62 → verifiers-0.2.2.dev64}/tests/v1/fixtures/pyproject.toml +0 -0
- {verifiers-0.2.2.dev62 → verifiers-0.2.2.dev64}/tests/v1/fixtures/tool_response_image_v1.py +0 -0
- {verifiers-0.2.2.dev62 → verifiers-0.2.2.dev64}/tests/v1/test_color_codeword_tasks.py +0 -0
- {verifiers-0.2.2.dev62 → verifiers-0.2.2.dev64}/tests/v1/test_configs.py +0 -0
- {verifiers-0.2.2.dev62 → verifiers-0.2.2.dev64}/tests/v1/test_e2e.py +0 -0
- {verifiers-0.2.2.dev62 → verifiers-0.2.2.dev64}/tests/v1/test_envs.py +0 -0
- {verifiers-0.2.2.dev62 → verifiers-0.2.2.dev64}/tests/v1/test_graph.py +0 -0
- {verifiers-0.2.2.dev62 → verifiers-0.2.2.dev64}/tests/v1/test_legacy.py +0 -0
- {verifiers-0.2.2.dev62 → verifiers-0.2.2.dev64}/tests/v1/test_scoring.py +0 -0
- {verifiers-0.2.2.dev62 → verifiers-0.2.2.dev64}/tests/v1/test_taskset.py +0 -0
- {verifiers-0.2.2.dev62 → verifiers-0.2.2.dev64}/tests/v1/test_trace.py +0 -0
- {verifiers-0.2.2.dev62 → verifiers-0.2.2.dev64}/verifiers/__init__.py +0 -0
- {verifiers-0.2.2.dev62 → verifiers-0.2.2.dev64}/verifiers/cli/__init__.py +0 -0
- {verifiers-0.2.2.dev62 → verifiers-0.2.2.dev64}/verifiers/cli/commands/__init__.py +0 -0
- {verifiers-0.2.2.dev62 → verifiers-0.2.2.dev64}/verifiers/cli/commands/build.py +0 -0
- {verifiers-0.2.2.dev62 → verifiers-0.2.2.dev64}/verifiers/cli/commands/eval.py +0 -0
- {verifiers-0.2.2.dev62 → verifiers-0.2.2.dev64}/verifiers/cli/commands/gepa.py +0 -0
- {verifiers-0.2.2.dev62 → verifiers-0.2.2.dev64}/verifiers/cli/commands/init.py +0 -0
- {verifiers-0.2.2.dev62 → verifiers-0.2.2.dev64}/verifiers/cli/commands/install.py +0 -0
- {verifiers-0.2.2.dev62 → verifiers-0.2.2.dev64}/verifiers/cli/commands/setup.py +0 -0
- {verifiers-0.2.2.dev62 → verifiers-0.2.2.dev64}/verifiers/cli/plugins/__init__.py +0 -0
- {verifiers-0.2.2.dev62 → verifiers-0.2.2.dev64}/verifiers/cli/plugins/prime.py +0 -0
- {verifiers-0.2.2.dev62 → verifiers-0.2.2.dev64}/verifiers/cli/tui.py +0 -0
- {verifiers-0.2.2.dev62 → verifiers-0.2.2.dev64}/verifiers/clients/__init__.py +0 -0
- {verifiers-0.2.2.dev62 → verifiers-0.2.2.dev64}/verifiers/clients/anthropic_messages_client.py +0 -0
- {verifiers-0.2.2.dev62 → verifiers-0.2.2.dev64}/verifiers/clients/client.py +0 -0
- {verifiers-0.2.2.dev62 → verifiers-0.2.2.dev64}/verifiers/clients/nemorl_chat_completions_client.py +0 -0
- {verifiers-0.2.2.dev62 → verifiers-0.2.2.dev64}/verifiers/clients/openai_chat_completions_client.py +0 -0
- {verifiers-0.2.2.dev62 → verifiers-0.2.2.dev64}/verifiers/clients/openai_chat_completions_token_client.py +0 -0
- {verifiers-0.2.2.dev62 → verifiers-0.2.2.dev64}/verifiers/clients/openai_completions_client.py +0 -0
- {verifiers-0.2.2.dev62 → verifiers-0.2.2.dev64}/verifiers/clients/openai_responses_client.py +0 -0
- {verifiers-0.2.2.dev62 → verifiers-0.2.2.dev64}/verifiers/clients/renderer_client.py +0 -0
- {verifiers-0.2.2.dev62 → verifiers-0.2.2.dev64}/verifiers/decorators.py +0 -0
- {verifiers-0.2.2.dev62 → verifiers-0.2.2.dev64}/verifiers/envs/__init__.py +0 -0
- {verifiers-0.2.2.dev62 → verifiers-0.2.2.dev64}/verifiers/envs/env_group.py +0 -0
- {verifiers-0.2.2.dev62 → verifiers-0.2.2.dev64}/verifiers/envs/environment.py +0 -0
- {verifiers-0.2.2.dev62 → verifiers-0.2.2.dev64}/verifiers/envs/experimental/README.md +0 -0
- {verifiers-0.2.2.dev62 → verifiers-0.2.2.dev64}/verifiers/envs/experimental/__init__.py +0 -0
- {verifiers-0.2.2.dev62 → verifiers-0.2.2.dev64}/verifiers/envs/experimental/cli_agent_env.py +0 -0
- {verifiers-0.2.2.dev62 → verifiers-0.2.2.dev64}/verifiers/envs/experimental/composable/README.md +0 -0
- {verifiers-0.2.2.dev62 → verifiers-0.2.2.dev64}/verifiers/envs/experimental/composable/__init__.py +0 -0
- {verifiers-0.2.2.dev62 → verifiers-0.2.2.dev64}/verifiers/envs/experimental/composable/_filter.py +0 -0
- {verifiers-0.2.2.dev62 → verifiers-0.2.2.dev64}/verifiers/envs/experimental/composable/composable_env.py +0 -0
- {verifiers-0.2.2.dev62 → verifiers-0.2.2.dev64}/verifiers/envs/experimental/composable/harness.py +0 -0
- {verifiers-0.2.2.dev62 → verifiers-0.2.2.dev64}/verifiers/envs/experimental/composable/harnesses/__init__.py +0 -0
- {verifiers-0.2.2.dev62 → verifiers-0.2.2.dev64}/verifiers/envs/experimental/composable/harnesses/mini_swe_agent.py +0 -0
- {verifiers-0.2.2.dev62 → verifiers-0.2.2.dev64}/verifiers/envs/experimental/composable/harnesses/opencode.py +0 -0
- {verifiers-0.2.2.dev62 → verifiers-0.2.2.dev64}/verifiers/envs/experimental/composable/harnesses/prompt.txt +0 -0
- {verifiers-0.2.2.dev62 → verifiers-0.2.2.dev64}/verifiers/envs/experimental/composable/harnesses/rlm.py +0 -0
- {verifiers-0.2.2.dev62 → verifiers-0.2.2.dev64}/verifiers/envs/experimental/composable/sandbox_debug_env.py +0 -0
- {verifiers-0.2.2.dev62 → verifiers-0.2.2.dev64}/verifiers/envs/experimental/composable/swe_debug_env.py +0 -0
- {verifiers-0.2.2.dev62 → verifiers-0.2.2.dev64}/verifiers/envs/experimental/composable/task.py +0 -0
- {verifiers-0.2.2.dev62 → verifiers-0.2.2.dev64}/verifiers/envs/experimental/composable/tasksets/__init__.py +0 -0
- {verifiers-0.2.2.dev62 → verifiers-0.2.2.dev64}/verifiers/envs/experimental/composable/tasksets/cp/__init__.py +0 -0
- {verifiers-0.2.2.dev62 → verifiers-0.2.2.dev64}/verifiers/envs/experimental/composable/tasksets/cp/cp_task.py +0 -0
- {verifiers-0.2.2.dev62 → verifiers-0.2.2.dev64}/verifiers/envs/experimental/composable/tasksets/cp/test_utils.py +0 -0
- {verifiers-0.2.2.dev62 → verifiers-0.2.2.dev64}/verifiers/envs/experimental/composable/tasksets/harbor/__init__.py +0 -0
- {verifiers-0.2.2.dev62 → verifiers-0.2.2.dev64}/verifiers/envs/experimental/composable/tasksets/harbor/cli_gym.py +0 -0
- {verifiers-0.2.2.dev62 → verifiers-0.2.2.dev64}/verifiers/envs/experimental/composable/tasksets/harbor/harbor.py +0 -0
- {verifiers-0.2.2.dev62 → verifiers-0.2.2.dev64}/verifiers/envs/experimental/composable/tasksets/harbor/terminal_lego.py +0 -0
- {verifiers-0.2.2.dev62 → verifiers-0.2.2.dev64}/verifiers/envs/experimental/composable/tasksets/math/__init__.py +0 -0
- {verifiers-0.2.2.dev62 → verifiers-0.2.2.dev64}/verifiers/envs/experimental/composable/tasksets/math/math_task.py +0 -0
- {verifiers-0.2.2.dev62 → verifiers-0.2.2.dev64}/verifiers/envs/experimental/composable/tasksets/swe/README.md +0 -0
- {verifiers-0.2.2.dev62 → verifiers-0.2.2.dev64}/verifiers/envs/experimental/composable/tasksets/swe/__init__.py +0 -0
- {verifiers-0.2.2.dev62 → verifiers-0.2.2.dev64}/verifiers/envs/experimental/composable/tasksets/swe/multi_swe/__init__.py +0 -0
- {verifiers-0.2.2.dev62 → verifiers-0.2.2.dev64}/verifiers/envs/experimental/composable/tasksets/swe/multi_swe/extract_fix_patch.sh +0 -0
- {verifiers-0.2.2.dev62 → verifiers-0.2.2.dev64}/verifiers/envs/experimental/composable/tasksets/swe/multi_swe/taskset.py +0 -0
- {verifiers-0.2.2.dev62 → verifiers-0.2.2.dev64}/verifiers/envs/experimental/composable/tasksets/swe/openswe/__init__.py +0 -0
- {verifiers-0.2.2.dev62 → verifiers-0.2.2.dev64}/verifiers/envs/experimental/composable/tasksets/swe/openswe/taskset.py +0 -0
- {verifiers-0.2.2.dev62 → verifiers-0.2.2.dev64}/verifiers/envs/experimental/composable/tasksets/swe/r2e_gym/__init__.py +0 -0
- {verifiers-0.2.2.dev62 → verifiers-0.2.2.dev64}/verifiers/envs/experimental/composable/tasksets/swe/r2e_gym/log_parser.py +0 -0
- {verifiers-0.2.2.dev62 → verifiers-0.2.2.dev64}/verifiers/envs/experimental/composable/tasksets/swe/r2e_gym/taskset.py +0 -0
- {verifiers-0.2.2.dev62 → verifiers-0.2.2.dev64}/verifiers/envs/experimental/composable/tasksets/swe/scale_swe/__init__.py +0 -0
- {verifiers-0.2.2.dev62 → verifiers-0.2.2.dev64}/verifiers/envs/experimental/composable/tasksets/swe/scale_swe/taskset.py +0 -0
- {verifiers-0.2.2.dev62 → verifiers-0.2.2.dev64}/verifiers/envs/experimental/composable/tasksets/swe/shared/__init__.py +0 -0
- {verifiers-0.2.2.dev62 → verifiers-0.2.2.dev64}/verifiers/envs/experimental/composable/tasksets/swe/shared/test_patch.py +0 -0
- {verifiers-0.2.2.dev62 → verifiers-0.2.2.dev64}/verifiers/envs/experimental/composable/tasksets/swe/swe_bench/__init__.py +0 -0
- {verifiers-0.2.2.dev62 → verifiers-0.2.2.dev64}/verifiers/envs/experimental/composable/tasksets/swe/swe_bench/taskset.py +0 -0
- {verifiers-0.2.2.dev62 → verifiers-0.2.2.dev64}/verifiers/envs/experimental/composable/tasksets/swe/swe_lego/__init__.py +0 -0
- {verifiers-0.2.2.dev62 → verifiers-0.2.2.dev64}/verifiers/envs/experimental/composable/tasksets/swe/swe_lego/taskset.py +0 -0
- {verifiers-0.2.2.dev62 → verifiers-0.2.2.dev64}/verifiers/envs/experimental/composable/tasksets/swe/swe_rebench_v2/__init__.py +0 -0
- {verifiers-0.2.2.dev62 → verifiers-0.2.2.dev64}/verifiers/envs/experimental/composable/tasksets/swe/swe_rebench_v2/log_parsers.py +0 -0
- {verifiers-0.2.2.dev62 → verifiers-0.2.2.dev64}/verifiers/envs/experimental/composable/tasksets/swe/swe_rebench_v2/taskset.py +0 -0
- {verifiers-0.2.2.dev62 → verifiers-0.2.2.dev64}/verifiers/envs/experimental/composable/tasksets/swe/swe_smith/__init__.py +0 -0
- {verifiers-0.2.2.dev62 → verifiers-0.2.2.dev64}/verifiers/envs/experimental/composable/tasksets/swe/swe_smith/taskset.py +0 -0
- {verifiers-0.2.2.dev62 → verifiers-0.2.2.dev64}/verifiers/envs/experimental/composable/tasksets/swe/swe_tasksets.py +0 -0
- {verifiers-0.2.2.dev62 → verifiers-0.2.2.dev64}/verifiers/envs/experimental/gym_env.py +0 -0
- {verifiers-0.2.2.dev62 → verifiers-0.2.2.dev64}/verifiers/envs/experimental/harbor_env/__init__.py +0 -0
- {verifiers-0.2.2.dev62 → verifiers-0.2.2.dev64}/verifiers/envs/experimental/harbor_env/env.py +0 -0
- {verifiers-0.2.2.dev62 → verifiers-0.2.2.dev64}/verifiers/envs/experimental/harbor_env/mcp.py +0 -0
- {verifiers-0.2.2.dev62 → verifiers-0.2.2.dev64}/verifiers/envs/experimental/mcp_env.py +0 -0
- {verifiers-0.2.2.dev62 → verifiers-0.2.2.dev64}/verifiers/envs/experimental/opencode_env.py +0 -0
- {verifiers-0.2.2.dev62 → verifiers-0.2.2.dev64}/verifiers/envs/experimental/opencode_qa_env.py +0 -0
- {verifiers-0.2.2.dev62 → verifiers-0.2.2.dev64}/verifiers/envs/experimental/opencode_rlm_env.py +0 -0
- {verifiers-0.2.2.dev62 → verifiers-0.2.2.dev64}/verifiers/envs/experimental/sandbox_mixin.py +0 -0
- {verifiers-0.2.2.dev62 → verifiers-0.2.2.dev64}/verifiers/envs/experimental/utils/__init__.py +0 -0
- {verifiers-0.2.2.dev62 → verifiers-0.2.2.dev64}/verifiers/envs/experimental/utils/file_locks.py +0 -0
- {verifiers-0.2.2.dev62 → verifiers-0.2.2.dev64}/verifiers/envs/experimental/utils/git_checkout_cache.py +0 -0
- {verifiers-0.2.2.dev62 → verifiers-0.2.2.dev64}/verifiers/envs/integrations/README.md +0 -0
- {verifiers-0.2.2.dev62 → verifiers-0.2.2.dev64}/verifiers/envs/integrations/__init__.py +0 -0
- {verifiers-0.2.2.dev62 → verifiers-0.2.2.dev64}/verifiers/envs/integrations/browser_env/README.md +0 -0
- {verifiers-0.2.2.dev62 → verifiers-0.2.2.dev64}/verifiers/envs/integrations/browser_env/__init__.py +0 -0
- {verifiers-0.2.2.dev62 → verifiers-0.2.2.dev64}/verifiers/envs/integrations/browser_env/browser_env.py +0 -0
- {verifiers-0.2.2.dev62 → verifiers-0.2.2.dev64}/verifiers/envs/integrations/browser_env/modes/__init__.py +0 -0
- {verifiers-0.2.2.dev62 → verifiers-0.2.2.dev64}/verifiers/envs/integrations/browser_env/modes/base.py +0 -0
- {verifiers-0.2.2.dev62 → verifiers-0.2.2.dev64}/verifiers/envs/integrations/browser_env/modes/cua_mode.py +0 -0
- {verifiers-0.2.2.dev62 → verifiers-0.2.2.dev64}/verifiers/envs/integrations/browser_env/modes/dom_mode.py +0 -0
- {verifiers-0.2.2.dev62 → verifiers-0.2.2.dev64}/verifiers/envs/integrations/openenv_env.py +0 -0
- {verifiers-0.2.2.dev62 → verifiers-0.2.2.dev64}/verifiers/envs/integrations/reasoninggym_env.py +0 -0
- {verifiers-0.2.2.dev62 → verifiers-0.2.2.dev64}/verifiers/envs/integrations/textarena_env.py +0 -0
- {verifiers-0.2.2.dev62 → verifiers-0.2.2.dev64}/verifiers/envs/multiturn_env.py +0 -0
- {verifiers-0.2.2.dev62 → verifiers-0.2.2.dev64}/verifiers/envs/python_env.py +0 -0
- {verifiers-0.2.2.dev62 → verifiers-0.2.2.dev64}/verifiers/envs/sandbox_env.py +0 -0
- {verifiers-0.2.2.dev62 → verifiers-0.2.2.dev64}/verifiers/envs/singleturn_env.py +0 -0
- {verifiers-0.2.2.dev62 → verifiers-0.2.2.dev64}/verifiers/envs/stateful_tool_env.py +0 -0
- {verifiers-0.2.2.dev62 → verifiers-0.2.2.dev64}/verifiers/envs/tool_env.py +0 -0
- {verifiers-0.2.2.dev62 → verifiers-0.2.2.dev64}/verifiers/errors.py +0 -0
- {verifiers-0.2.2.dev62 → verifiers-0.2.2.dev64}/verifiers/gepa/__init__.py +0 -0
- {verifiers-0.2.2.dev62 → verifiers-0.2.2.dev64}/verifiers/gepa/adapter.py +0 -0
- {verifiers-0.2.2.dev62 → verifiers-0.2.2.dev64}/verifiers/gepa/config.py +0 -0
- {verifiers-0.2.2.dev62 → verifiers-0.2.2.dev64}/verifiers/gepa/display.py +0 -0
- {verifiers-0.2.2.dev62 → verifiers-0.2.2.dev64}/verifiers/gepa/gepa_utils.py +0 -0
- {verifiers-0.2.2.dev62 → verifiers-0.2.2.dev64}/verifiers/parsers/__init__.py +0 -0
- {verifiers-0.2.2.dev62 → verifiers-0.2.2.dev64}/verifiers/parsers/maybe_think_parser.py +0 -0
- {verifiers-0.2.2.dev62 → verifiers-0.2.2.dev64}/verifiers/parsers/parser.py +0 -0
- {verifiers-0.2.2.dev62 → verifiers-0.2.2.dev64}/verifiers/parsers/think_parser.py +0 -0
- {verifiers-0.2.2.dev62 → verifiers-0.2.2.dev64}/verifiers/parsers/xml_parser.py +0 -0
- {verifiers-0.2.2.dev62 → verifiers-0.2.2.dev64}/verifiers/rubrics/__init__.py +0 -0
- {verifiers-0.2.2.dev62 → verifiers-0.2.2.dev64}/verifiers/rubrics/experimental/hybrid_math_rubric.py +0 -0
- {verifiers-0.2.2.dev62 → verifiers-0.2.2.dev64}/verifiers/rubrics/judge_rubric.py +0 -0
- {verifiers-0.2.2.dev62 → verifiers-0.2.2.dev64}/verifiers/rubrics/math_rubric.py +0 -0
- {verifiers-0.2.2.dev62 → verifiers-0.2.2.dev64}/verifiers/rubrics/rubric.py +0 -0
- {verifiers-0.2.2.dev62 → verifiers-0.2.2.dev64}/verifiers/rubrics/rubric_group.py +0 -0
- {verifiers-0.2.2.dev62 → verifiers-0.2.2.dev64}/verifiers/scripts/__init__.py +0 -0
- {verifiers-0.2.2.dev62 → verifiers-0.2.2.dev64}/verifiers/scripts/build.py +0 -0
- {verifiers-0.2.2.dev62 → verifiers-0.2.2.dev64}/verifiers/scripts/eval.py +0 -0
- {verifiers-0.2.2.dev62 → verifiers-0.2.2.dev64}/verifiers/scripts/gepa.py +0 -0
- {verifiers-0.2.2.dev62 → verifiers-0.2.2.dev64}/verifiers/scripts/init.py +0 -0
- {verifiers-0.2.2.dev62 → verifiers-0.2.2.dev64}/verifiers/scripts/install.py +0 -0
- {verifiers-0.2.2.dev62 → verifiers-0.2.2.dev64}/verifiers/scripts/setup.py +0 -0
- {verifiers-0.2.2.dev62 → verifiers-0.2.2.dev64}/verifiers/scripts/tui.py +0 -0
- {verifiers-0.2.2.dev62 → verifiers-0.2.2.dev64}/verifiers/serve/__init__.py +0 -0
- {verifiers-0.2.2.dev62 → verifiers-0.2.2.dev64}/verifiers/serve/client/env_client.py +0 -0
- {verifiers-0.2.2.dev62 → verifiers-0.2.2.dev64}/verifiers/serve/client/zmq_env_client.py +0 -0
- {verifiers-0.2.2.dev62 → verifiers-0.2.2.dev64}/verifiers/serve/server/__init__.py +0 -0
- {verifiers-0.2.2.dev62 → verifiers-0.2.2.dev64}/verifiers/serve/server/env_router.py +0 -0
- {verifiers-0.2.2.dev62 → verifiers-0.2.2.dev64}/verifiers/serve/server/env_server.py +0 -0
- {verifiers-0.2.2.dev62 → verifiers-0.2.2.dev64}/verifiers/serve/server/env_worker.py +0 -0
- {verifiers-0.2.2.dev62 → verifiers-0.2.2.dev64}/verifiers/serve/server/zmq_env_server.py +0 -0
- {verifiers-0.2.2.dev62 → verifiers-0.2.2.dev64}/verifiers/serve/types.py +0 -0
- {verifiers-0.2.2.dev62 → verifiers-0.2.2.dev64}/verifiers/types.py +0 -0
- {verifiers-0.2.2.dev62 → verifiers-0.2.2.dev64}/verifiers/utils/__init__.py +0 -0
- {verifiers-0.2.2.dev62 → verifiers-0.2.2.dev64}/verifiers/utils/async_utils.py +0 -0
- {verifiers-0.2.2.dev62 → verifiers-0.2.2.dev64}/verifiers/utils/client_utils.py +0 -0
- {verifiers-0.2.2.dev62 → verifiers-0.2.2.dev64}/verifiers/utils/config_utils.py +0 -0
- {verifiers-0.2.2.dev62 → verifiers-0.2.2.dev64}/verifiers/utils/data_utils.py +0 -0
- {verifiers-0.2.2.dev62 → verifiers-0.2.2.dev64}/verifiers/utils/display_utils.py +0 -0
- {verifiers-0.2.2.dev62 → verifiers-0.2.2.dev64}/verifiers/utils/env_config_utils.py +0 -0
- {verifiers-0.2.2.dev62 → verifiers-0.2.2.dev64}/verifiers/utils/env_utils.py +0 -0
- {verifiers-0.2.2.dev62 → verifiers-0.2.2.dev64}/verifiers/utils/error_utils.py +0 -0
- {verifiers-0.2.2.dev62 → verifiers-0.2.2.dev64}/verifiers/utils/eval_display.py +0 -0
- {verifiers-0.2.2.dev62 → verifiers-0.2.2.dev64}/verifiers/utils/eval_utils.py +0 -0
- {verifiers-0.2.2.dev62 → verifiers-0.2.2.dev64}/verifiers/utils/heartbeat.py +0 -0
- {verifiers-0.2.2.dev62 → verifiers-0.2.2.dev64}/verifiers/utils/import_utils.py +0 -0
- {verifiers-0.2.2.dev62 → verifiers-0.2.2.dev64}/verifiers/utils/install_utils.py +0 -0
- {verifiers-0.2.2.dev62 → verifiers-0.2.2.dev64}/verifiers/utils/interception_utils.py +0 -0
- {verifiers-0.2.2.dev62 → verifiers-0.2.2.dev64}/verifiers/utils/logging_utils.py +0 -0
- {verifiers-0.2.2.dev62 → verifiers-0.2.2.dev64}/verifiers/utils/message_utils.py +0 -0
- {verifiers-0.2.2.dev62 → verifiers-0.2.2.dev64}/verifiers/utils/metric_utils.py +0 -0
- {verifiers-0.2.2.dev62 → verifiers-0.2.2.dev64}/verifiers/utils/path_utils.py +0 -0
- {verifiers-0.2.2.dev62 → verifiers-0.2.2.dev64}/verifiers/utils/pricing_utils.py +0 -0
- {verifiers-0.2.2.dev62 → verifiers-0.2.2.dev64}/verifiers/utils/process_utils.py +0 -0
- {verifiers-0.2.2.dev62 → verifiers-0.2.2.dev64}/verifiers/utils/response_utils.py +0 -0
- {verifiers-0.2.2.dev62 → verifiers-0.2.2.dev64}/verifiers/utils/save_utils.py +0 -0
- {verifiers-0.2.2.dev62 → verifiers-0.2.2.dev64}/verifiers/utils/serve_utils.py +0 -0
- {verifiers-0.2.2.dev62 → verifiers-0.2.2.dev64}/verifiers/utils/thread_utils.py +0 -0
- {verifiers-0.2.2.dev62 → verifiers-0.2.2.dev64}/verifiers/utils/threaded_sandbox_client.py +0 -0
- {verifiers-0.2.2.dev62 → verifiers-0.2.2.dev64}/verifiers/utils/tool_utils.py +0 -0
- {verifiers-0.2.2.dev62 → verifiers-0.2.2.dev64}/verifiers/utils/usage_utils.py +0 -0
- {verifiers-0.2.2.dev62 → verifiers-0.2.2.dev64}/verifiers/utils/version_utils.py +0 -0
- {verifiers-0.2.2.dev62 → verifiers-0.2.2.dev64}/verifiers/v1/acp/__init__.py +0 -0
- {verifiers-0.2.2.dev62 → verifiers-0.2.2.dev64}/verifiers/v1/acp/_runner.py +0 -0
- {verifiers-0.2.2.dev62 → verifiers-0.2.2.dev64}/verifiers/v1/cli/__init__.py +0 -0
- {verifiers-0.2.2.dev62 → verifiers-0.2.2.dev64}/verifiers/v1/cli/dashboard/__init__.py +0 -0
- {verifiers-0.2.2.dev62 → verifiers-0.2.2.dev64}/verifiers/v1/cli/dashboard/base.py +0 -0
- {verifiers-0.2.2.dev62 → verifiers-0.2.2.dev64}/verifiers/v1/cli/dashboard/replay.py +0 -0
- {verifiers-0.2.2.dev62 → verifiers-0.2.2.dev64}/verifiers/v1/cli/dashboard/validate.py +0 -0
- {verifiers-0.2.2.dev62 → verifiers-0.2.2.dev64}/verifiers/v1/cli/eval/__init__.py +0 -0
- {verifiers-0.2.2.dev62 → verifiers-0.2.2.dev64}/verifiers/v1/cli/eval/resume.py +0 -0
- {verifiers-0.2.2.dev62 → verifiers-0.2.2.dev64}/verifiers/v1/cli/gepa.py +0 -0
- {verifiers-0.2.2.dev62 → verifiers-0.2.2.dev64}/verifiers/v1/cli/init.py +0 -0
- {verifiers-0.2.2.dev62 → verifiers-0.2.2.dev64}/verifiers/v1/cli/output.py +0 -0
- {verifiers-0.2.2.dev62 → verifiers-0.2.2.dev64}/verifiers/v1/cli/replay.py +0 -0
- {verifiers-0.2.2.dev62 → verifiers-0.2.2.dev64}/verifiers/v1/cli/resolve.py +0 -0
- {verifiers-0.2.2.dev62 → verifiers-0.2.2.dev64}/verifiers/v1/cli/serve.py +0 -0
- {verifiers-0.2.2.dev62 → verifiers-0.2.2.dev64}/verifiers/v1/clients/__init__.py +0 -0
- {verifiers-0.2.2.dev62 → verifiers-0.2.2.dev64}/verifiers/v1/clients/client.py +0 -0
- {verifiers-0.2.2.dev62 → verifiers-0.2.2.dev64}/verifiers/v1/clients/config.py +0 -0
- {verifiers-0.2.2.dev62 → verifiers-0.2.2.dev64}/verifiers/v1/clients/eval.py +0 -0
- {verifiers-0.2.2.dev62 → verifiers-0.2.2.dev64}/verifiers/v1/clients/train.py +0 -0
- {verifiers-0.2.2.dev62 → verifiers-0.2.2.dev64}/verifiers/v1/configs/__init__.py +0 -0
- {verifiers-0.2.2.dev62 → verifiers-0.2.2.dev64}/verifiers/v1/configs/cli/__init__.py +0 -0
- {verifiers-0.2.2.dev62 → verifiers-0.2.2.dev64}/verifiers/v1/configs/cli/eval.py +0 -0
- {verifiers-0.2.2.dev62 → verifiers-0.2.2.dev64}/verifiers/v1/configs/cli/init.py +0 -0
- {verifiers-0.2.2.dev62 → verifiers-0.2.2.dev64}/verifiers/v1/configs/cli/serve.py +0 -0
- {verifiers-0.2.2.dev62 → verifiers-0.2.2.dev64}/verifiers/v1/configs/harness.py +0 -0
- {verifiers-0.2.2.dev62 → verifiers-0.2.2.dev64}/verifiers/v1/configs/legacy.py +0 -0
- {verifiers-0.2.2.dev62 → verifiers-0.2.2.dev64}/verifiers/v1/configs/retries.py +0 -0
- {verifiers-0.2.2.dev62 → verifiers-0.2.2.dev64}/verifiers/v1/configs/serve.py +0 -0
- {verifiers-0.2.2.dev62 → verifiers-0.2.2.dev64}/verifiers/v1/configs/task.py +0 -0
- {verifiers-0.2.2.dev62 → verifiers-0.2.2.dev64}/verifiers/v1/configs/taskset.py +0 -0
- {verifiers-0.2.2.dev62 → verifiers-0.2.2.dev64}/verifiers/v1/dialects/__init__.py +0 -0
- {verifiers-0.2.2.dev62 → verifiers-0.2.2.dev64}/verifiers/v1/dialects/anthropic.py +0 -0
- {verifiers-0.2.2.dev62 → verifiers-0.2.2.dev64}/verifiers/v1/dialects/base.py +0 -0
- {verifiers-0.2.2.dev62 → verifiers-0.2.2.dev64}/verifiers/v1/dialects/chat.py +0 -0
- {verifiers-0.2.2.dev62 → verifiers-0.2.2.dev64}/verifiers/v1/dialects/responses.py +0 -0
- {verifiers-0.2.2.dev62 → verifiers-0.2.2.dev64}/verifiers/v1/envs/__init__.py +0 -0
- {verifiers-0.2.2.dev62 → verifiers-0.2.2.dev64}/verifiers/v1/envs/agentic_judge/__init__.py +0 -0
- {verifiers-0.2.2.dev62 → verifiers-0.2.2.dev64}/verifiers/v1/envs/agentic_judge/env.py +0 -0
- {verifiers-0.2.2.dev62 → verifiers-0.2.2.dev64}/verifiers/v1/envs/best_of_n/__init__.py +0 -0
- {verifiers-0.2.2.dev62 → verifiers-0.2.2.dev64}/verifiers/v1/envs/best_of_n/env.py +0 -0
- {verifiers-0.2.2.dev62 → verifiers-0.2.2.dev64}/verifiers/v1/envs/single_agent/__init__.py +0 -0
- {verifiers-0.2.2.dev62 → verifiers-0.2.2.dev64}/verifiers/v1/envs/single_agent/env.py +0 -0
- {verifiers-0.2.2.dev62 → verifiers-0.2.2.dev64}/verifiers/v1/envs/user_sim/__init__.py +0 -0
- {verifiers-0.2.2.dev62 → verifiers-0.2.2.dev64}/verifiers/v1/envs/user_sim/env.py +0 -0
- {verifiers-0.2.2.dev62 → verifiers-0.2.2.dev64}/verifiers/v1/episode.py +0 -0
- {verifiers-0.2.2.dev62 → verifiers-0.2.2.dev64}/verifiers/v1/errors.py +0 -0
- {verifiers-0.2.2.dev62 → verifiers-0.2.2.dev64}/verifiers/v1/gepa/__init__.py +0 -0
- {verifiers-0.2.2.dev62 → verifiers-0.2.2.dev64}/verifiers/v1/gepa/adapter.py +0 -0
- {verifiers-0.2.2.dev62 → verifiers-0.2.2.dev64}/verifiers/v1/gepa/config.py +0 -0
- {verifiers-0.2.2.dev62 → verifiers-0.2.2.dev64}/verifiers/v1/gepa/dataset.py +0 -0
- {verifiers-0.2.2.dev62 → verifiers-0.2.2.dev64}/verifiers/v1/gepa/reflection.py +0 -0
- {verifiers-0.2.2.dev62 → verifiers-0.2.2.dev64}/verifiers/v1/gepa/runner.py +0 -0
- {verifiers-0.2.2.dev62 → verifiers-0.2.2.dev64}/verifiers/v1/graph.py +0 -0
- {verifiers-0.2.2.dev62 → verifiers-0.2.2.dev64}/verifiers/v1/harnesses/__init__.py +0 -0
- {verifiers-0.2.2.dev62 → verifiers-0.2.2.dev64}/verifiers/v1/harnesses/bash/__init__.py +0 -0
- {verifiers-0.2.2.dev62 → verifiers-0.2.2.dev64}/verifiers/v1/harnesses/bash/harness.py +0 -0
- {verifiers-0.2.2.dev62 → verifiers-0.2.2.dev64}/verifiers/v1/harnesses/bash/program.py +0 -0
- {verifiers-0.2.2.dev62 → verifiers-0.2.2.dev64}/verifiers/v1/harnesses/browser_use/__init__.py +0 -0
- {verifiers-0.2.2.dev62 → verifiers-0.2.2.dev64}/verifiers/v1/harnesses/browser_use/harness.py +0 -0
- {verifiers-0.2.2.dev62 → verifiers-0.2.2.dev64}/verifiers/v1/harnesses/browser_use/program.py +0 -0
- {verifiers-0.2.2.dev62 → verifiers-0.2.2.dev64}/verifiers/v1/harnesses/claude_code/__init__.py +0 -0
- {verifiers-0.2.2.dev62 → verifiers-0.2.2.dev64}/verifiers/v1/harnesses/claude_code/harness.py +0 -0
- {verifiers-0.2.2.dev62 → verifiers-0.2.2.dev64}/verifiers/v1/harnesses/codex/__init__.py +0 -0
- {verifiers-0.2.2.dev62 → verifiers-0.2.2.dev64}/verifiers/v1/harnesses/codex/harness.py +0 -0
- {verifiers-0.2.2.dev62 → verifiers-0.2.2.dev64}/verifiers/v1/harnesses/hermes_agent/__init__.py +0 -0
- {verifiers-0.2.2.dev62 → verifiers-0.2.2.dev64}/verifiers/v1/harnesses/hermes_agent/harness.py +0 -0
- {verifiers-0.2.2.dev62 → verifiers-0.2.2.dev64}/verifiers/v1/harnesses/hermes_agent/program.py +0 -0
- {verifiers-0.2.2.dev62 → verifiers-0.2.2.dev64}/verifiers/v1/harnesses/kimi_code/__init__.py +0 -0
- {verifiers-0.2.2.dev62 → verifiers-0.2.2.dev64}/verifiers/v1/harnesses/kimi_code/harness.py +0 -0
- {verifiers-0.2.2.dev62 → verifiers-0.2.2.dev64}/verifiers/v1/harnesses/mini_swe_agent/__init__.py +0 -0
- {verifiers-0.2.2.dev62 → verifiers-0.2.2.dev64}/verifiers/v1/harnesses/mini_swe_agent/harness.py +0 -0
- {verifiers-0.2.2.dev62 → verifiers-0.2.2.dev64}/verifiers/v1/harnesses/mini_swe_agent/program.py +0 -0
- {verifiers-0.2.2.dev62 → verifiers-0.2.2.dev64}/verifiers/v1/harnesses/null/__init__.py +0 -0
- {verifiers-0.2.2.dev62 → verifiers-0.2.2.dev64}/verifiers/v1/harnesses/null/harness.py +0 -0
- {verifiers-0.2.2.dev62 → verifiers-0.2.2.dev64}/verifiers/v1/harnesses/null/program.py +0 -0
- {verifiers-0.2.2.dev62 → verifiers-0.2.2.dev64}/verifiers/v1/harnesses/openclaw/__init__.py +0 -0
- {verifiers-0.2.2.dev62 → verifiers-0.2.2.dev64}/verifiers/v1/harnesses/openclaw/harness.py +0 -0
- {verifiers-0.2.2.dev62 → verifiers-0.2.2.dev64}/verifiers/v1/harnesses/pi/__init__.py +0 -0
- {verifiers-0.2.2.dev62 → verifiers-0.2.2.dev64}/verifiers/v1/harnesses/pi/harness.py +0 -0
- {verifiers-0.2.2.dev62 → verifiers-0.2.2.dev64}/verifiers/v1/harnesses/pool/__init__.py +0 -0
- {verifiers-0.2.2.dev62 → verifiers-0.2.2.dev64}/verifiers/v1/harnesses/pool/harness.py +0 -0
- {verifiers-0.2.2.dev62 → verifiers-0.2.2.dev64}/verifiers/v1/harnesses/rlm/__init__.py +0 -0
- {verifiers-0.2.2.dev62 → verifiers-0.2.2.dev64}/verifiers/v1/harnesses/terminus_2/__init__.py +0 -0
- {verifiers-0.2.2.dev62 → verifiers-0.2.2.dev64}/verifiers/v1/harnesses/terminus_2/harness.py +0 -0
- {verifiers-0.2.2.dev62 → verifiers-0.2.2.dev64}/verifiers/v1/harnesses/terminus_2/program.py +0 -0
- {verifiers-0.2.2.dev62 → verifiers-0.2.2.dev64}/verifiers/v1/interception/tunnel/__init__.py +0 -0
- {verifiers-0.2.2.dev62 → verifiers-0.2.2.dev64}/verifiers/v1/interception/tunnel/base.py +0 -0
- {verifiers-0.2.2.dev62 → verifiers-0.2.2.dev64}/verifiers/v1/interception/tunnel/custom.py +0 -0
- {verifiers-0.2.2.dev62 → verifiers-0.2.2.dev64}/verifiers/v1/judges/__init__.py +0 -0
- {verifiers-0.2.2.dev62 → verifiers-0.2.2.dev64}/verifiers/v1/judges/reference.py +0 -0
- {verifiers-0.2.2.dev62 → verifiers-0.2.2.dev64}/verifiers/v1/judges/reference.txt +0 -0
- {verifiers-0.2.2.dev62 → verifiers-0.2.2.dev64}/verifiers/v1/judges/rubric.py +0 -0
- {verifiers-0.2.2.dev62 → verifiers-0.2.2.dev64}/verifiers/v1/judges/rubric.txt +0 -0
- {verifiers-0.2.2.dev62 → verifiers-0.2.2.dev64}/verifiers/v1/mcp/__init__.py +0 -0
- {verifiers-0.2.2.dev62 → verifiers-0.2.2.dev64}/verifiers/v1/runtimes/__init__.py +0 -0
- {verifiers-0.2.2.dev62 → verifiers-0.2.2.dev64}/verifiers/v1/runtimes/base.py +0 -0
- {verifiers-0.2.2.dev62 → verifiers-0.2.2.dev64}/verifiers/v1/runtimes/docker/__init__.py +0 -0
- {verifiers-0.2.2.dev62 → verifiers-0.2.2.dev64}/verifiers/v1/runtimes/docker/egress.py +0 -0
- {verifiers-0.2.2.dev62 → verifiers-0.2.2.dev64}/verifiers/v1/runtimes/limiters.py +0 -0
- {verifiers-0.2.2.dev62 → verifiers-0.2.2.dev64}/verifiers/v1/runtimes/modal.py +0 -0
- {verifiers-0.2.2.dev62 → verifiers-0.2.2.dev64}/verifiers/v1/runtimes/prime.py +0 -0
- {verifiers-0.2.2.dev62 → verifiers-0.2.2.dev64}/verifiers/v1/runtimes/subprocess.py +0 -0
- {verifiers-0.2.2.dev62 → verifiers-0.2.2.dev64}/verifiers/v1/serve/__init__.py +0 -0
- {verifiers-0.2.2.dev62 → verifiers-0.2.2.dev64}/verifiers/v1/serve/client.py +0 -0
- {verifiers-0.2.2.dev62 → verifiers-0.2.2.dev64}/verifiers/v1/serve/types.py +0 -0
- {verifiers-0.2.2.dev62 → verifiers-0.2.2.dev64}/verifiers/v1/session.py +0 -0
- {verifiers-0.2.2.dev62 → verifiers-0.2.2.dev64}/verifiers/v1/tasksets/__init__.py +0 -0
- {verifiers-0.2.2.dev62 → verifiers-0.2.2.dev64}/verifiers/v1/tasksets/harbor/__init__.py +0 -0
- {verifiers-0.2.2.dev62 → verifiers-0.2.2.dev64}/verifiers/v1/tasksets/lean/__init__.py +0 -0
- {verifiers-0.2.2.dev62 → verifiers-0.2.2.dev64}/verifiers/v1/tasksets/lean/scoring.py +0 -0
- {verifiers-0.2.2.dev62 → verifiers-0.2.2.dev64}/verifiers/v1/tasksets/openenv/__init__.py +0 -0
- {verifiers-0.2.2.dev62 → verifiers-0.2.2.dev64}/verifiers/v1/tasksets/openenv/taskset.py +0 -0
- {verifiers-0.2.2.dev62 → verifiers-0.2.2.dev64}/verifiers/v1/tasksets/textarena/__init__.py +0 -0
- {verifiers-0.2.2.dev62 → verifiers-0.2.2.dev64}/verifiers/v1/tasksets/textarena/taskset.py +0 -0
- {verifiers-0.2.2.dev62 → verifiers-0.2.2.dev64}/verifiers/v1/trace.py +0 -0
- {verifiers-0.2.2.dev62 → verifiers-0.2.2.dev64}/verifiers/v1/types.py +0 -0
- {verifiers-0.2.2.dev62 → verifiers-0.2.2.dev64}/verifiers/v1/utils/__init__.py +0 -0
- {verifiers-0.2.2.dev62 → verifiers-0.2.2.dev64}/verifiers/v1/utils/aio.py +0 -0
- {verifiers-0.2.2.dev62/verifiers/v1 → verifiers-0.2.2.dev64/verifiers/v1/utils}/artifacts.py +0 -0
- {verifiers-0.2.2.dev62 → verifiers-0.2.2.dev64}/verifiers/v1/utils/compile.py +0 -0
- {verifiers-0.2.2.dev62/verifiers/v1 → verifiers-0.2.2.dev64/verifiers/v1/utils}/decorators.py +0 -0
- {verifiers-0.2.2.dev62 → verifiers-0.2.2.dev64}/verifiers/v1/utils/format.py +0 -0
- {verifiers-0.2.2.dev62 → verifiers-0.2.2.dev64}/verifiers/v1/utils/generic.py +0 -0
- {verifiers-0.2.2.dev62 → verifiers-0.2.2.dev64}/verifiers/v1/utils/git.py +0 -0
- {verifiers-0.2.2.dev62 → verifiers-0.2.2.dev64}/verifiers/v1/utils/image.py +0 -0
- {verifiers-0.2.2.dev62 → verifiers-0.2.2.dev64}/verifiers/v1/utils/install.py +0 -0
- {verifiers-0.2.2.dev62 → verifiers-0.2.2.dev64}/verifiers/v1/utils/interrupt.py +0 -0
- {verifiers-0.2.2.dev62/verifiers/v1 → verifiers-0.2.2.dev64/verifiers/v1/utils}/loaders.py +0 -0
- {verifiers-0.2.2.dev62 → verifiers-0.2.2.dev64}/verifiers/v1/utils/logging.py +0 -0
- {verifiers-0.2.2.dev62 → verifiers-0.2.2.dev64}/verifiers/v1/utils/memory.py +0 -0
- /verifiers-0.2.2.dev62/verifiers/v1/push.py → /verifiers-0.2.2.dev64/verifiers/v1/utils/platform.py +0 -0
- {verifiers-0.2.2.dev62/verifiers/v1 → verifiers-0.2.2.dev64/verifiers/v1/utils}/retries.py +0 -0
- /verifiers-0.2.2.dev62/verifiers/v1/scoring.py → /verifiers-0.2.2.dev64/verifiers/v1/utils/score.py +0 -0
- {verifiers-0.2.2.dev62 → verifiers-0.2.2.dev64}/verifiers/v1/utils/version.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: verifiers
|
|
3
|
-
Version: 0.2.2.
|
|
3
|
+
Version: 0.2.2.dev64
|
|
4
4
|
Summary: Verifiers: Environments for LLM Reinforcement Learning
|
|
5
5
|
Project-URL: Homepage, https://github.com/primeintellect-ai/verifiers
|
|
6
6
|
Project-URL: Documentation, https://github.com/primeintellect-ai/verifiers
|
|
@@ -41,8 +41,8 @@ import pytest
|
|
|
41
41
|
import verifiers.v1 as vf
|
|
42
42
|
from verifiers.v1.cli.eval.runner import run_eval
|
|
43
43
|
from verifiers.v1.configs.cli.eval import EvalConfig
|
|
44
|
-
from verifiers.v1.loaders import load_environment
|
|
45
44
|
from verifiers.v1.trace import Trace
|
|
45
|
+
from verifiers.v1.utils.loaders import load_environment
|
|
46
46
|
|
|
47
47
|
# Fixture tasksets/envs (echo-v1, echo-agentic-v1, echo-v0, echo-multi-v0) live in
|
|
48
48
|
# tests/v1/fixtures, added to the path via `pythonpath` in pyproject so the v1 loader and the
|
|
@@ -11,8 +11,8 @@ from pydantic import Field
|
|
|
11
11
|
import verifiers.v1 as vf
|
|
12
12
|
from verifiers.v1.graph import MessageNode
|
|
13
13
|
from verifiers.v1.judge import Judge, JudgeResponse
|
|
14
|
-
from verifiers.v1.loaders import judge_class, judge_config_type, load_judge
|
|
15
14
|
from verifiers.v1.types import AssistantMessage, UserMessage
|
|
15
|
+
from verifiers.v1.utils.loaders import judge_class, judge_config_type, load_judge
|
|
16
16
|
|
|
17
17
|
RUBRIC_TOML = """
|
|
18
18
|
[[criteria]]
|
|
@@ -4,12 +4,6 @@ from pydantic_config import BaseConfig
|
|
|
4
4
|
|
|
5
5
|
from verifiers.v1.acp import ACP
|
|
6
6
|
from verifiers.v1.agent import Agent, Agents, Interaction, Segment, make_agent
|
|
7
|
-
from verifiers.v1.artifacts import (
|
|
8
|
-
ARTIFACTS_DIR,
|
|
9
|
-
Artifact,
|
|
10
|
-
collect,
|
|
11
|
-
restore,
|
|
12
|
-
)
|
|
13
7
|
from verifiers.v1.clients import (
|
|
14
8
|
BaseClientConfig,
|
|
15
9
|
Client,
|
|
@@ -34,7 +28,6 @@ from verifiers.v1.configs.serve import (
|
|
|
34
28
|
)
|
|
35
29
|
from verifiers.v1.configs.task import TaskConfig
|
|
36
30
|
from verifiers.v1.configs.taskset import TasksetConfig
|
|
37
|
-
from verifiers.v1.decorators import metric, reward, stop, tool
|
|
38
31
|
from verifiers.v1.env import Env
|
|
39
32
|
from verifiers.v1.envs.single_agent import SingleAgentEnv, SingleAgentEnvConfig
|
|
40
33
|
from verifiers.v1.episode import Episode, WireEpisode
|
|
@@ -59,24 +52,6 @@ from verifiers.v1.judges import (
|
|
|
59
52
|
RubricJudge,
|
|
60
53
|
RubricJudgeConfig,
|
|
61
54
|
)
|
|
62
|
-
from verifiers.v1.loaders import (
|
|
63
|
-
default_harness_id,
|
|
64
|
-
env_config_type,
|
|
65
|
-
environment_class,
|
|
66
|
-
harness_config_type,
|
|
67
|
-
import_environment,
|
|
68
|
-
import_harness,
|
|
69
|
-
import_judge,
|
|
70
|
-
import_taskset,
|
|
71
|
-
judge_config_type,
|
|
72
|
-
load_environment,
|
|
73
|
-
load_harness,
|
|
74
|
-
load_judge,
|
|
75
|
-
load_taskset,
|
|
76
|
-
resolve_env_config,
|
|
77
|
-
task_type,
|
|
78
|
-
taskset_config_type,
|
|
79
|
-
)
|
|
80
55
|
from verifiers.v1.mcp import (
|
|
81
56
|
SharedToolsetConfig,
|
|
82
57
|
Toolset,
|
|
@@ -91,24 +66,6 @@ from verifiers.v1.runtimes import (
|
|
|
91
66
|
RuntimeInfo,
|
|
92
67
|
SubprocessConfig,
|
|
93
68
|
)
|
|
94
|
-
from verifiers.v1.scoring import (
|
|
95
|
-
compare_stdout_results as compare_stdout_results,
|
|
96
|
-
)
|
|
97
|
-
from verifiers.v1.scoring import (
|
|
98
|
-
extract_boxed_answer as extract_boxed_answer,
|
|
99
|
-
)
|
|
100
|
-
from verifiers.v1.scoring import (
|
|
101
|
-
parse_judge_choice as parse_judge_choice,
|
|
102
|
-
)
|
|
103
|
-
from verifiers.v1.scoring import (
|
|
104
|
-
parse_pytest_outcomes as parse_pytest_outcomes,
|
|
105
|
-
)
|
|
106
|
-
from verifiers.v1.scoring import (
|
|
107
|
-
read_answer_file_or_last_reply as read_answer_file_or_last_reply,
|
|
108
|
-
)
|
|
109
|
-
from verifiers.v1.scoring import (
|
|
110
|
-
verify_boxed_math_answer as verify_boxed_math_answer,
|
|
111
|
-
)
|
|
112
69
|
from verifiers.v1.state import State, StateT
|
|
113
70
|
from verifiers.v1.task import Task, TaskData, TaskResources, TaskTimeout, WireTaskData
|
|
114
71
|
from verifiers.v1.taskset import Taskset
|
|
@@ -153,6 +110,13 @@ from verifiers.v1.types import (
|
|
|
153
110
|
Usage,
|
|
154
111
|
UserMessage,
|
|
155
112
|
)
|
|
113
|
+
from verifiers.v1.utils.artifacts import (
|
|
114
|
+
ARTIFACTS_DIR,
|
|
115
|
+
Artifact,
|
|
116
|
+
collect,
|
|
117
|
+
restore,
|
|
118
|
+
)
|
|
119
|
+
from verifiers.v1.utils.decorators import metric, reward, stop, tool
|
|
156
120
|
from verifiers.v1.utils.git import (
|
|
157
121
|
PATCH_CAP_BYTES as PATCH_CAP_BYTES,
|
|
158
122
|
)
|
|
@@ -162,6 +126,42 @@ from verifiers.v1.utils.git import (
|
|
|
162
126
|
from verifiers.v1.utils.git import (
|
|
163
127
|
resolve_head as resolve_head,
|
|
164
128
|
)
|
|
129
|
+
from verifiers.v1.utils.loaders import (
|
|
130
|
+
default_harness_id,
|
|
131
|
+
env_config_type,
|
|
132
|
+
environment_class,
|
|
133
|
+
harness_config_type,
|
|
134
|
+
import_environment,
|
|
135
|
+
import_harness,
|
|
136
|
+
import_judge,
|
|
137
|
+
import_taskset,
|
|
138
|
+
judge_config_type,
|
|
139
|
+
load_environment,
|
|
140
|
+
load_harness,
|
|
141
|
+
load_judge,
|
|
142
|
+
load_taskset,
|
|
143
|
+
resolve_env_config,
|
|
144
|
+
task_type,
|
|
145
|
+
taskset_config_type,
|
|
146
|
+
)
|
|
147
|
+
from verifiers.v1.utils.score import (
|
|
148
|
+
compare_stdout_results as compare_stdout_results,
|
|
149
|
+
)
|
|
150
|
+
from verifiers.v1.utils.score import (
|
|
151
|
+
extract_boxed_answer as extract_boxed_answer,
|
|
152
|
+
)
|
|
153
|
+
from verifiers.v1.utils.score import (
|
|
154
|
+
parse_judge_choice as parse_judge_choice,
|
|
155
|
+
)
|
|
156
|
+
from verifiers.v1.utils.score import (
|
|
157
|
+
parse_pytest_outcomes as parse_pytest_outcomes,
|
|
158
|
+
)
|
|
159
|
+
from verifiers.v1.utils.score import (
|
|
160
|
+
read_answer_file_or_last_reply as read_answer_file_or_last_reply,
|
|
161
|
+
)
|
|
162
|
+
from verifiers.v1.utils.score import (
|
|
163
|
+
verify_boxed_math_answer as verify_boxed_math_answer,
|
|
164
|
+
)
|
|
165
165
|
|
|
166
166
|
__all__ = [ # noqa: RUF022 - grouped by public API area
|
|
167
167
|
# types
|
|
@@ -24,7 +24,6 @@ from verifiers.v1.configs.agent import AgentConfig, TimeoutConfig
|
|
|
24
24
|
from verifiers.v1.harness import Harness
|
|
25
25
|
from verifiers.v1.interception import Interception, InterceptionServer
|
|
26
26
|
from verifiers.v1.mcp import SharedToolServer
|
|
27
|
-
from verifiers.v1.retries import backoff, trace_should_retry
|
|
28
27
|
from verifiers.v1.rollout import RolloutRun, RolloutTimeouts, _as_messages
|
|
29
28
|
from verifiers.v1.runtimes import (
|
|
30
29
|
NetworkPolicyConfig,
|
|
@@ -49,6 +48,7 @@ from verifiers.v1.utils.compile import (
|
|
|
49
48
|
resolve_runtime_config,
|
|
50
49
|
validate_pairing,
|
|
51
50
|
)
|
|
51
|
+
from verifiers.v1.utils.retries import backoff, trace_should_retry
|
|
52
52
|
|
|
53
53
|
__all__ = ["Agent", "AgentConfig", "Agents", "TimeoutConfig", "make_agent"]
|
|
54
54
|
|
|
@@ -241,7 +241,7 @@ class Agent:
|
|
|
241
241
|
client: Client | None = None,
|
|
242
242
|
interception: Interception | None = None,
|
|
243
243
|
) -> None:
|
|
244
|
-
from verifiers.v1.loaders import harness_config_type, load_harness
|
|
244
|
+
from verifiers.v1.utils.loaders import harness_config_type, load_harness
|
|
245
245
|
|
|
246
246
|
if config.model is None:
|
|
247
247
|
raise ValueError(
|
|
@@ -334,6 +334,10 @@ class Agent:
|
|
|
334
334
|
return self.interception
|
|
335
335
|
if self._server is None:
|
|
336
336
|
return None
|
|
337
|
+
if any(tool.state_secret for tool in shared_tools.values()):
|
|
338
|
+
# Shared state credentials are attached per run, after this owned
|
|
339
|
+
# server was created; let the rollout size a scoped server instead.
|
|
340
|
+
return None
|
|
337
341
|
if self._server.tunnel is not None or (
|
|
338
342
|
run_is_local and not shared_tools and not task.toolsets(task.config)
|
|
339
343
|
):
|
|
@@ -29,7 +29,7 @@ from verifiers.v1.utils.install import env_name
|
|
|
29
29
|
from verifiers.v1.utils.interrupt import cleaning_up
|
|
30
30
|
|
|
31
31
|
if TYPE_CHECKING:
|
|
32
|
-
from verifiers.v1.
|
|
32
|
+
from verifiers.v1.utils.platform import PushState
|
|
33
33
|
|
|
34
34
|
# For sizing pages to the terminal: detects the real terminal height/width each access (the live
|
|
35
35
|
# view writes to the same terminal). Reused so we don't rebuild it every refresh tick.
|
|
@@ -160,7 +160,7 @@ def _warning(config: EvalConfig) -> Text | None:
|
|
|
160
160
|
"""A local-runtime caution when any code-running seat resolves to the subprocess
|
|
161
161
|
runtime (the tool-less chat loops are exempt), shown above the overview rather
|
|
162
162
|
than as a row in it."""
|
|
163
|
-
from verifiers.v1.loaders import harness_class
|
|
163
|
+
from verifiers.v1.utils.loaders import harness_class
|
|
164
164
|
|
|
165
165
|
if any(
|
|
166
166
|
getattr(config.env, role).runtime.type == "subprocess"
|
|
@@ -24,12 +24,12 @@ from verifiers.v1.cli.resolve import (
|
|
|
24
24
|
with_positional_taskset,
|
|
25
25
|
)
|
|
26
26
|
from verifiers.v1.configs.cli.debug import DebugConfig
|
|
27
|
-
from verifiers.v1.decorators import invoke
|
|
28
27
|
from verifiers.v1.runtimes import ProgramResult, Runtime, make_runtime
|
|
29
28
|
from verifiers.v1.state import state_cls
|
|
30
29
|
from verifiers.v1.task import Task
|
|
31
30
|
from verifiers.v1.trace import AgentInfo, Error, Trace, TraceTask
|
|
32
31
|
from verifiers.v1.utils.compile import resolve_runtime_config
|
|
32
|
+
from verifiers.v1.utils.decorators import invoke
|
|
33
33
|
from verifiers.v1.utils.interrupt import install_interrupt
|
|
34
34
|
from verifiers.v1.utils.logging import setup_logging
|
|
35
35
|
|
|
@@ -120,7 +120,7 @@ def main(argv: list[str] | None = None) -> None:
|
|
|
120
120
|
# disk. Exit on the conventional Ctrl-C code without a traceback.
|
|
121
121
|
raise SystemExit(130)
|
|
122
122
|
if config.push and not rich:
|
|
123
|
-
from verifiers.v1.
|
|
123
|
+
from verifiers.v1.utils.platform import push_traces
|
|
124
124
|
|
|
125
125
|
push_traces(episodes, config)
|
|
126
126
|
if not rich: # --rich is the whole output; otherwise dump each trace as JSON
|
|
@@ -3,6 +3,7 @@
|
|
|
3
3
|
import asyncio
|
|
4
4
|
import contextlib
|
|
5
5
|
import logging
|
|
6
|
+
import random
|
|
6
7
|
import time
|
|
7
8
|
|
|
8
9
|
from verifiers.v1.cli.dashboard import dashboard
|
|
@@ -17,8 +18,8 @@ from verifiers.v1.clients import ModelContext, resolve_client
|
|
|
17
18
|
from verifiers.v1.configs.cli.eval import EvalConfig
|
|
18
19
|
from verifiers.v1.env import Env, RunSlot
|
|
19
20
|
from verifiers.v1.episode import Episode
|
|
21
|
+
from verifiers.v1.taskset import SEED
|
|
20
22
|
from verifiers.v1.trace import EvalRunInfo
|
|
21
|
-
from verifiers.v1.utils.sampling import sample
|
|
22
23
|
|
|
23
24
|
logger = logging.getLogger(__name__)
|
|
24
25
|
|
|
@@ -86,7 +87,7 @@ async def run_eval(env: Env, config: EvalConfig) -> list[Episode]:
|
|
|
86
87
|
slots = [RunSlot.finished(episode) for episode in finished] + planned
|
|
87
88
|
push_state = None
|
|
88
89
|
if config.push and config.rich:
|
|
89
|
-
from verifiers.v1.
|
|
90
|
+
from verifiers.v1.utils.platform import PushState
|
|
90
91
|
|
|
91
92
|
push_state = PushState()
|
|
92
93
|
display = (
|
|
@@ -102,7 +103,7 @@ async def run_eval(env: Env, config: EvalConfig) -> list[Episode]:
|
|
|
102
103
|
if (
|
|
103
104
|
push_state is not None
|
|
104
105
|
): # upload off the event loop so the view keeps refreshing
|
|
105
|
-
from verifiers.v1.
|
|
106
|
+
from verifiers.v1.utils.platform import push_traces
|
|
106
107
|
|
|
107
108
|
push_state.started = True
|
|
108
109
|
await asyncio.to_thread(push_traces, episodes, config, push_state)
|
|
@@ -136,7 +137,7 @@ async def run_eval_server(config: EvalConfig) -> list[Episode]:
|
|
|
136
137
|
)
|
|
137
138
|
tasks = []
|
|
138
139
|
if not legacy:
|
|
139
|
-
from verifiers.v1.loaders import load_taskset
|
|
140
|
+
from verifiers.v1.utils.loaders import load_taskset
|
|
140
141
|
|
|
141
142
|
# The client owns the taskset: load it here, once — the server (and its pool
|
|
142
143
|
# workers) never load data, they rebuild each dispatched task from its request.
|
|
@@ -184,7 +185,10 @@ async def run_eval_server(config: EvalConfig) -> list[Episode]:
|
|
|
184
185
|
if legacy:
|
|
185
186
|
info = await client.info()
|
|
186
187
|
group_scored = info.requires_group_scoring
|
|
187
|
-
idxs =
|
|
188
|
+
idxs = list(range(info.num_tasks))
|
|
189
|
+
if config.shuffle:
|
|
190
|
+
random.Random(SEED).shuffle(idxs)
|
|
191
|
+
idxs = idxs[: config.num_tasks]
|
|
188
192
|
plan = [({"task_idx": idx}, config.num_rollouts) for idx in idxs]
|
|
189
193
|
else:
|
|
190
194
|
group_scored = False
|
|
@@ -20,12 +20,12 @@ from verifiers.v1.cli.resolve import (
|
|
|
20
20
|
with_positional_taskset,
|
|
21
21
|
)
|
|
22
22
|
from verifiers.v1.configs.cli.validate import ValidateConfig
|
|
23
|
-
from verifiers.v1.decorators import invoke
|
|
24
23
|
from verifiers.v1.runtimes import make_runtime
|
|
25
24
|
from verifiers.v1.state import state_cls
|
|
26
25
|
from verifiers.v1.task import Task
|
|
27
26
|
from verifiers.v1.trace import Trace, TraceTask
|
|
28
27
|
from verifiers.v1.utils.compile import resolve_runtime_config
|
|
28
|
+
from verifiers.v1.utils.decorators import invoke
|
|
29
29
|
from verifiers.v1.utils.interrupt import install_interrupt
|
|
30
30
|
from verifiers.v1.utils.logging import setup_logging
|
|
31
31
|
|
|
@@ -57,7 +57,10 @@ class AgentConfig(BaseConfig):
|
|
|
57
57
|
stays None = the taskset's default). The lazy import keeps class-body
|
|
58
58
|
`AgentConfig()` defaults constructible while this module initializes."""
|
|
59
59
|
if isinstance(data, dict) and data.get("harness") is not None:
|
|
60
|
-
from verifiers.v1.loaders import
|
|
60
|
+
from verifiers.v1.utils.loaders import (
|
|
61
|
+
harness_config_type,
|
|
62
|
+
narrow_plugin_field,
|
|
63
|
+
)
|
|
61
64
|
|
|
62
65
|
narrow_plugin_field(data, "harness", harness_config_type, "bash")
|
|
63
66
|
return data
|
|
@@ -52,7 +52,7 @@ class DebugConfig(BaseConfig):
|
|
|
52
52
|
@model_validator(mode="before")
|
|
53
53
|
@classmethod
|
|
54
54
|
def resolve_taskset(cls, data):
|
|
55
|
-
from verifiers.v1.loaders import narrow_plugin_field, taskset_config_type
|
|
55
|
+
from verifiers.v1.utils.loaders import narrow_plugin_field, taskset_config_type
|
|
56
56
|
|
|
57
57
|
narrow_plugin_field(data, "taskset", taskset_config_type)
|
|
58
58
|
return data
|
|
@@ -35,7 +35,7 @@ def resolve_env_field(data: dict, narrowed: "type[EnvConfig] | None" = None) ->
|
|
|
35
35
|
raw.model_dump() if isinstance(raw, BaseConfig) else raw
|
|
36
36
|
)
|
|
37
37
|
return data
|
|
38
|
-
from verifiers.v1.loaders import resolve_env_config
|
|
38
|
+
from verifiers.v1.utils.loaders import resolve_env_config
|
|
39
39
|
|
|
40
40
|
data["env"] = resolve_env_config(raw)
|
|
41
41
|
except ValidationError as e:
|
|
@@ -50,7 +50,7 @@ class ReplayConfig(BaseConfig):
|
|
|
50
50
|
@model_validator(mode="before")
|
|
51
51
|
@classmethod
|
|
52
52
|
def _resolve_taskset(cls, data):
|
|
53
|
-
from verifiers.v1.loaders import narrow_plugin_field, taskset_config_type
|
|
53
|
+
from verifiers.v1.utils.loaders import narrow_plugin_field, taskset_config_type
|
|
54
54
|
|
|
55
55
|
if isinstance(data, dict) and not data.get("taskset"):
|
|
56
56
|
# The base layer is usually a saved eval config, which keeps its
|
|
@@ -54,7 +54,7 @@ class ValidateConfig(BaseConfig):
|
|
|
54
54
|
@model_validator(mode="before")
|
|
55
55
|
@classmethod
|
|
56
56
|
def _resolve_taskset(cls, data):
|
|
57
|
-
from verifiers.v1.loaders import narrow_plugin_field, taskset_config_type
|
|
57
|
+
from verifiers.v1.utils.loaders import narrow_plugin_field, taskset_config_type
|
|
58
58
|
|
|
59
59
|
narrow_plugin_field(data, "taskset", taskset_config_type)
|
|
60
60
|
return data
|
|
@@ -100,7 +100,10 @@ class EnvConfig(BaseConfig):
|
|
|
100
100
|
"""Narrow `taskset` to its concrete config type by `id`; lazy import for
|
|
101
101
|
the same reason as `AgentConfig._resolve_harness`."""
|
|
102
102
|
if isinstance(data, dict) and data.get("taskset") is not None:
|
|
103
|
-
from verifiers.v1.loaders import
|
|
103
|
+
from verifiers.v1.utils.loaders import (
|
|
104
|
+
narrow_plugin_field,
|
|
105
|
+
taskset_config_type,
|
|
106
|
+
)
|
|
104
107
|
|
|
105
108
|
narrow_plugin_field(data, "taskset", taskset_config_type)
|
|
106
109
|
return data
|
|
@@ -157,7 +160,7 @@ def _declared_agent_configs(config: EnvConfig) -> dict[str, AgentConfig]:
|
|
|
157
160
|
def default_agent_harness(taskset_id: str) -> HarnessConfig:
|
|
158
161
|
"""What an unpinned role's `harness=None` resolves to: the taskset's bundled
|
|
159
162
|
harness when it ships one, else the built-in `bash`."""
|
|
160
|
-
from verifiers.v1.loaders import default_harness_id, harness_config_type
|
|
163
|
+
from verifiers.v1.utils.loaders import default_harness_id, harness_config_type
|
|
161
164
|
|
|
162
165
|
ident = default_harness_id(taskset_id)
|
|
163
166
|
return harness_config_type(ident).model_validate({"id": ident})
|
|
@@ -33,7 +33,7 @@ def judge_key(config: JudgeConfig) -> str:
|
|
|
33
33
|
|
|
34
34
|
|
|
35
35
|
def resolve_judges(entries: Sequence[Any]) -> list[JudgeConfig]:
|
|
36
|
-
from verifiers.v1.loaders import judge_config_type
|
|
36
|
+
from verifiers.v1.utils.loaders import judge_config_type
|
|
37
37
|
|
|
38
38
|
resolved = []
|
|
39
39
|
for entry in entries:
|
|
@@ -29,12 +29,12 @@ from verifiers.v1.interception import (
|
|
|
29
29
|
requires_tunnel,
|
|
30
30
|
)
|
|
31
31
|
from verifiers.v1.mcp import SharedToolServer, serve_shared
|
|
32
|
-
from verifiers.v1.retries import run_episode_with_retry
|
|
33
32
|
from verifiers.v1.runtimes import SubprocessConfig, runtime_is_local
|
|
34
33
|
from verifiers.v1.task import Task
|
|
35
34
|
from verifiers.v1.trace import Error, Trace
|
|
36
35
|
from verifiers.v1.utils.generic import concrete_type
|
|
37
36
|
from verifiers.v1.utils.memory import trim_memory_periodically
|
|
37
|
+
from verifiers.v1.utils.retries import run_episode_with_retry
|
|
38
38
|
|
|
39
39
|
logger = logging.getLogger(__name__)
|
|
40
40
|
|
|
@@ -81,7 +81,7 @@ class Env(ABC, Generic[ConfigT]):
|
|
|
81
81
|
task the agent actually receives — an env-minted task carries its own needs."""
|
|
82
82
|
|
|
83
83
|
def __init__(self, config: ConfigT) -> None:
|
|
84
|
-
from verifiers.v1.loaders import load_harness, load_taskset
|
|
84
|
+
from verifiers.v1.utils.loaders import load_harness, load_taskset
|
|
85
85
|
|
|
86
86
|
config_cls = concrete_type(type(self), EnvConfig, origin=Env) or EnvConfig
|
|
87
87
|
if not isinstance(config, config_cls):
|
|
@@ -350,7 +350,13 @@ class Env(ABC, Generic[ConfigT]):
|
|
|
350
350
|
run slots inside. Torn down on exit (`teardown()`, then the framework's)."""
|
|
351
351
|
async with self.shared_tools() as shared:
|
|
352
352
|
interception = make_interception(
|
|
353
|
-
self.config.interception,
|
|
353
|
+
self.config.interception,
|
|
354
|
+
requires_tunnel=self._requires_tunnel(shared),
|
|
355
|
+
state_service_secrets=tuple(
|
|
356
|
+
server.state_secret
|
|
357
|
+
for server in shared.values()
|
|
358
|
+
if server.state_secret
|
|
359
|
+
),
|
|
354
360
|
)
|
|
355
361
|
async with interception:
|
|
356
362
|
self._shared_tools = shared
|
|
@@ -8,11 +8,11 @@ from typing import TYPE_CHECKING, ClassVar, Generic, TypeVar
|
|
|
8
8
|
|
|
9
9
|
from verifiers.v1.clients import ModelContext
|
|
10
10
|
from verifiers.v1.configs.harness import HarnessConfig
|
|
11
|
-
from verifiers.v1.decorators import discover_decorated, invoke_all
|
|
12
11
|
from verifiers.v1.errors import HarnessError, SandboxError, boundary
|
|
13
12
|
from verifiers.v1.runtimes import ProgramResult, Runtime
|
|
14
13
|
from verifiers.v1.task import TaskData
|
|
15
14
|
from verifiers.v1.types import Messages
|
|
15
|
+
from verifiers.v1.utils.decorators import discover_decorated, invoke_all
|
|
16
16
|
|
|
17
17
|
if TYPE_CHECKING:
|
|
18
18
|
# Annotation-only: `Trace` appears in signatures only, so this module stays
|
|
@@ -10,12 +10,12 @@ from pydantic import Field, model_validator
|
|
|
10
10
|
|
|
11
11
|
from verifiers.v1.clients import ModelContext
|
|
12
12
|
from verifiers.v1.configs.harness import HarnessConfig
|
|
13
|
-
from verifiers.v1.decorators import metric
|
|
14
13
|
from verifiers.v1.dialects.chat import message_to_wire
|
|
15
14
|
from verifiers.v1.harness import Harness
|
|
16
15
|
from verifiers.v1.runtimes import ProgramResult, Runtime
|
|
17
16
|
from verifiers.v1.task import TaskData
|
|
18
17
|
from verifiers.v1.trace import Trace
|
|
18
|
+
from verifiers.v1.utils.decorators import metric
|
|
19
19
|
|
|
20
20
|
logger = logging.getLogger(__name__)
|
|
21
21
|
|
|
@@ -54,16 +54,19 @@ def requires_tunnel(
|
|
|
54
54
|
|
|
55
55
|
|
|
56
56
|
def make_interception(
|
|
57
|
-
config: InterceptionConfig,
|
|
57
|
+
config: InterceptionConfig,
|
|
58
|
+
*,
|
|
59
|
+
requires_tunnel: bool,
|
|
60
|
+
state_service_secrets: tuple[str, ...] = (),
|
|
58
61
|
) -> Interception:
|
|
59
62
|
"""The interception for a config, picked by type (the host-side counterpart to
|
|
60
63
|
`make_runtime`). With `requires_tunnel`, each server is exposed through its configured
|
|
61
64
|
tunnel; otherwise it remains on host loopback. The caller computes this requirement."""
|
|
62
65
|
if isinstance(config, InterceptionServerConfig):
|
|
63
|
-
return InterceptionServer(config, requires_tunnel)
|
|
66
|
+
return InterceptionServer(config, requires_tunnel, state_service_secrets)
|
|
64
67
|
if isinstance(config, StaticInterceptionPoolConfig):
|
|
65
|
-
return StaticInterceptionPool(config, requires_tunnel)
|
|
66
|
-
return ElasticInterceptionPool(config, requires_tunnel)
|
|
68
|
+
return StaticInterceptionPool(config, requires_tunnel, state_service_secrets)
|
|
69
|
+
return ElasticInterceptionPool(config, requires_tunnel, state_service_secrets)
|
|
67
70
|
|
|
68
71
|
|
|
69
72
|
__all__ = [
|
|
@@ -26,12 +26,10 @@ class BaseInterceptionConfig(BaseConfig):
|
|
|
26
26
|
`multiplex`)."""
|
|
27
27
|
|
|
28
28
|
|
|
29
|
-
# (base_url,
|
|
30
|
-
#
|
|
31
|
-
# `{base_url}/v1`; tool servers reach this rollout's shared state at `{base_url}/state`
|
|
32
|
-
# + `/task`. `base_url` is universally reachable — the interception is exposed (tunnel)
|
|
29
|
+
# (base_url, model_secret, state_secret): model inference and task state deliberately use
|
|
30
|
+
# separate capabilities. `base_url` is universally reachable — the interception is exposed
|
|
33
31
|
# whenever any consumer is remote.
|
|
34
|
-
Slot = tuple[str, str]
|
|
32
|
+
Slot = tuple[str, str, str]
|
|
35
33
|
|
|
36
34
|
|
|
37
35
|
class Interception(ABC):
|
|
@@ -48,12 +48,16 @@ class StaticInterceptionPool(Interception):
|
|
|
48
48
|
operator's call (it's the shape for pre-provisioned/bring-your-own endpoints)."""
|
|
49
49
|
|
|
50
50
|
def __init__(
|
|
51
|
-
self,
|
|
51
|
+
self,
|
|
52
|
+
config: StaticInterceptionPoolConfig,
|
|
53
|
+
requires_tunnel: bool = False,
|
|
54
|
+
state_service_secrets: tuple[str, ...] = (),
|
|
52
55
|
) -> None:
|
|
53
56
|
super().__init__()
|
|
54
57
|
self.config = config
|
|
55
58
|
self.servers = [
|
|
56
|
-
InterceptionServer(server, requires_tunnel)
|
|
59
|
+
InterceptionServer(server, requires_tunnel, state_service_secrets)
|
|
60
|
+
for server in config.servers
|
|
57
61
|
]
|
|
58
62
|
|
|
59
63
|
async def start(self) -> None:
|
|
@@ -89,10 +93,12 @@ class ElasticInterceptionPool(Interception):
|
|
|
89
93
|
self,
|
|
90
94
|
config: ElasticInterceptionPoolConfig | None = None,
|
|
91
95
|
requires_tunnel: bool = False,
|
|
96
|
+
state_service_secrets: tuple[str, ...] = (),
|
|
92
97
|
) -> None:
|
|
93
98
|
super().__init__()
|
|
94
99
|
self.config = config or ElasticInterceptionPoolConfig()
|
|
95
100
|
self.requires_tunnel = requires_tunnel
|
|
101
|
+
self.state_service_secrets = state_service_secrets
|
|
96
102
|
self.servers: list[InterceptionServer] = []
|
|
97
103
|
self._lock = asyncio.Lock()
|
|
98
104
|
self._warm_task: asyncio.Task[InterceptionServer] | None = None
|
|
@@ -116,7 +122,9 @@ class ElasticInterceptionPool(Interception):
|
|
|
116
122
|
return server
|
|
117
123
|
# Pin prime explicitly — the only tunnel kind that can be minted on demand.
|
|
118
124
|
server = InterceptionServer(
|
|
119
|
-
InterceptionServerConfig(tunnel=PrimeTunnelConfig()),
|
|
125
|
+
InterceptionServerConfig(tunnel=PrimeTunnelConfig()),
|
|
126
|
+
self.requires_tunnel,
|
|
127
|
+
self.state_service_secrets,
|
|
120
128
|
)
|
|
121
129
|
await self.stack.enter_async_context(server)
|
|
122
130
|
self.servers.append(server)
|
|
@@ -136,8 +144,8 @@ class ElasticInterceptionPool(Interception):
|
|
|
136
144
|
# Register under the lock so concurrent acquires see each other's load.
|
|
137
145
|
async with self._lock:
|
|
138
146
|
server = await self._server()
|
|
139
|
-
|
|
147
|
+
model_secret, state_secret = server.register(session)
|
|
140
148
|
try:
|
|
141
|
-
yield server.base_url,
|
|
149
|
+
yield server.base_url, model_secret, state_secret
|
|
142
150
|
finally:
|
|
143
|
-
server.unregister(
|
|
151
|
+
server.unregister(model_secret, state_secret)
|