verifiers 0.2.2.dev86__tar.gz → 0.2.2.dev91__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {verifiers-0.2.2.dev86 → verifiers-0.2.2.dev91}/PKG-INFO +2 -2
- {verifiers-0.2.2.dev86 → verifiers-0.2.2.dev91}/pyproject.toml +1 -1
- {verifiers-0.2.2.dev86 → verifiers-0.2.2.dev91}/tests/v1/conftest.py +6 -4
- {verifiers-0.2.2.dev86 → verifiers-0.2.2.dev91}/tests/v1/test_e2e.py +55 -47
- {verifiers-0.2.2.dev86 → verifiers-0.2.2.dev91}/tests/v1/test_envs.py +1 -1
- {verifiers-0.2.2.dev86 → verifiers-0.2.2.dev91}/verifiers/v1/__init__.py +4 -1
- verifiers-0.2.2.dev91/verifiers/v1/acp/__init__.py +312 -0
- verifiers-0.2.2.dev91/verifiers/v1/acp/runner.py +427 -0
- {verifiers-0.2.2.dev86 → verifiers-0.2.2.dev91}/verifiers/v1/configs/harness.py +2 -2
- {verifiers-0.2.2.dev86 → verifiers-0.2.2.dev91}/verifiers/v1/harness.py +106 -15
- verifiers-0.2.2.dev91/verifiers/v1/harnesses/claude_code/harness.py +165 -0
- verifiers-0.2.2.dev91/verifiers/v1/harnesses/codex/harness.py +246 -0
- verifiers-0.2.2.dev91/verifiers/v1/harnesses/node.py +54 -0
- {verifiers-0.2.2.dev86 → verifiers-0.2.2.dev91}/verifiers/v1/harnesses/pi/harness.py +6 -31
- {verifiers-0.2.2.dev86 → verifiers-0.2.2.dev91}/verifiers/v1/harnesses/rlm/harness.py +74 -29
- {verifiers-0.2.2.dev86 → verifiers-0.2.2.dev91}/verifiers/v1/interception/server.py +1 -1
- {verifiers-0.2.2.dev86 → verifiers-0.2.2.dev91}/verifiers/v1/mcp/launch.py +22 -9
- {verifiers-0.2.2.dev86 → verifiers-0.2.2.dev91}/verifiers/v1/rollout.py +33 -11
- {verifiers-0.2.2.dev86 → verifiers-0.2.2.dev91}/verifiers/v1/runtimes/__init__.py +2 -0
- {verifiers-0.2.2.dev86 → verifiers-0.2.2.dev91}/verifiers/v1/runtimes/base.py +37 -0
- {verifiers-0.2.2.dev86 → verifiers-0.2.2.dev91}/verifiers/v1/runtimes/docker/__init__.py +162 -1
- {verifiers-0.2.2.dev86 → verifiers-0.2.2.dev91}/verifiers/v1/runtimes/modal.py +99 -0
- {verifiers-0.2.2.dev86 → verifiers-0.2.2.dev91}/verifiers/v1/runtimes/prime.py +44 -0
- {verifiers-0.2.2.dev86 → verifiers-0.2.2.dev91}/verifiers/v1/runtimes/subprocess.py +67 -13
- {verifiers-0.2.2.dev86 → verifiers-0.2.2.dev91}/verifiers/v1/session.py +1 -1
- verifiers-0.2.2.dev86/verifiers/v1/acp/__init__.py +0 -66
- verifiers-0.2.2.dev86/verifiers/v1/acp/_runner.py +0 -213
- verifiers-0.2.2.dev86/verifiers/v1/harnesses/claude_code/harness.py +0 -105
- verifiers-0.2.2.dev86/verifiers/v1/harnesses/codex/harness.py +0 -332
- {verifiers-0.2.2.dev86 → verifiers-0.2.2.dev91}/.gitignore +0 -0
- {verifiers-0.2.2.dev86 → verifiers-0.2.2.dev91}/LICENSE +0 -0
- {verifiers-0.2.2.dev86 → verifiers-0.2.2.dev91}/README.md +0 -0
- {verifiers-0.2.2.dev86 → verifiers-0.2.2.dev91}/tests/__init__.py +0 -0
- {verifiers-0.2.2.dev86 → verifiers-0.2.2.dev91}/tests/conftest.py +0 -0
- {verifiers-0.2.2.dev86 → verifiers-0.2.2.dev91}/tests/test_browser_env.py +0 -0
- {verifiers-0.2.2.dev86 → verifiers-0.2.2.dev91}/tests/test_build_script.py +0 -0
- {verifiers-0.2.2.dev86 → verifiers-0.2.2.dev91}/tests/test_cli_agent_env.py +0 -0
- {verifiers-0.2.2.dev86 → verifiers-0.2.2.dev91}/tests/test_client_auth_errors.py +0 -0
- {verifiers-0.2.2.dev86 → verifiers-0.2.2.dev91}/tests/test_client_config.py +0 -0
- {verifiers-0.2.2.dev86 → verifiers-0.2.2.dev91}/tests/test_client_multimodal_types.py +0 -0
- {verifiers-0.2.2.dev86 → verifiers-0.2.2.dev91}/tests/test_composable_env.py +0 -0
- {verifiers-0.2.2.dev86 → verifiers-0.2.2.dev91}/tests/test_context_token_metrics.py +0 -0
- {verifiers-0.2.2.dev86 → verifiers-0.2.2.dev91}/tests/test_decorator_ranks.py +0 -0
- {verifiers-0.2.2.dev86 → verifiers-0.2.2.dev91}/tests/test_endpoint_registry.py +0 -0
- {verifiers-0.2.2.dev86 → verifiers-0.2.2.dev91}/tests/test_env_group.py +0 -0
- {verifiers-0.2.2.dev86 → verifiers-0.2.2.dev91}/tests/test_env_server.py +0 -0
- {verifiers-0.2.2.dev86 → verifiers-0.2.2.dev91}/tests/test_environment.py +0 -0
- {verifiers-0.2.2.dev86 → verifiers-0.2.2.dev91}/tests/test_environment_extra.py +0 -0
- {verifiers-0.2.2.dev86 → verifiers-0.2.2.dev91}/tests/test_error_chain.py +0 -0
- {verifiers-0.2.2.dev86 → verifiers-0.2.2.dev91}/tests/test_eval_display.py +0 -0
- {verifiers-0.2.2.dev86 → verifiers-0.2.2.dev91}/tests/test_eval_utils.py +0 -0
- {verifiers-0.2.2.dev86 → verifiers-0.2.2.dev91}/tests/test_gepa_cli.py +0 -0
- {verifiers-0.2.2.dev86 → verifiers-0.2.2.dev91}/tests/test_gepa_utils.py +0 -0
- {verifiers-0.2.2.dev86 → verifiers-0.2.2.dev91}/tests/test_gym_env.py +0 -0
- {verifiers-0.2.2.dev86 → verifiers-0.2.2.dev91}/tests/test_harbor_env_mcp.py +0 -0
- {verifiers-0.2.2.dev86 → verifiers-0.2.2.dev91}/tests/test_imports.py +0 -0
- {verifiers-0.2.2.dev86 → verifiers-0.2.2.dev91}/tests/test_init_script.py +0 -0
- {verifiers-0.2.2.dev86 → verifiers-0.2.2.dev91}/tests/test_install_utils.py +0 -0
- {verifiers-0.2.2.dev86 → verifiers-0.2.2.dev91}/tests/test_interception_utils.py +0 -0
- {verifiers-0.2.2.dev86 → verifiers-0.2.2.dev91}/tests/test_logging.py +0 -0
- {verifiers-0.2.2.dev86 → verifiers-0.2.2.dev91}/tests/test_math_rubric.py +0 -0
- {verifiers-0.2.2.dev86 → verifiers-0.2.2.dev91}/tests/test_maybe_think_parser.py +0 -0
- {verifiers-0.2.2.dev86 → verifiers-0.2.2.dev91}/tests/test_message_utils.py +0 -0
- {verifiers-0.2.2.dev86 → verifiers-0.2.2.dev91}/tests/test_message_utils_multimodal.py +0 -0
- {verifiers-0.2.2.dev86 → verifiers-0.2.2.dev91}/tests/test_multiturn_env.py +0 -0
- {verifiers-0.2.2.dev86 → verifiers-0.2.2.dev91}/tests/test_nemorl_client.py +0 -0
- {verifiers-0.2.2.dev86 → verifiers-0.2.2.dev91}/tests/test_openai_chat_completions_token_client.py +0 -0
- {verifiers-0.2.2.dev86 → verifiers-0.2.2.dev91}/tests/test_openai_responses_client.py +0 -0
- {verifiers-0.2.2.dev86 → verifiers-0.2.2.dev91}/tests/test_opencode_rlm_env.py +0 -0
- {verifiers-0.2.2.dev86 → verifiers-0.2.2.dev91}/tests/test_openenv_client.py +0 -0
- {verifiers-0.2.2.dev86 → verifiers-0.2.2.dev91}/tests/test_parser.py +0 -0
- {verifiers-0.2.2.dev86 → verifiers-0.2.2.dev91}/tests/test_path_utils.py +0 -0
- {verifiers-0.2.2.dev86 → verifiers-0.2.2.dev91}/tests/test_per_turn_timing.py +0 -0
- {verifiers-0.2.2.dev86 → verifiers-0.2.2.dev91}/tests/test_pricing_utils.py +0 -0
- {verifiers-0.2.2.dev86 → verifiers-0.2.2.dev91}/tests/test_prime_plugin.py +0 -0
- {verifiers-0.2.2.dev86 → verifiers-0.2.2.dev91}/tests/test_renderer_client.py +0 -0
- {verifiers-0.2.2.dev86 → verifiers-0.2.2.dev91}/tests/test_renderer_e2e.py +0 -0
- {verifiers-0.2.2.dev86 → verifiers-0.2.2.dev91}/tests/test_rlm_composable_env.py +0 -0
- {verifiers-0.2.2.dev86 → verifiers-0.2.2.dev91}/tests/test_rubric.py +0 -0
- {verifiers-0.2.2.dev86 → verifiers-0.2.2.dev91}/tests/test_rubric_group.py +0 -0
- {verifiers-0.2.2.dev86 → verifiers-0.2.2.dev91}/tests/test_sandbox_debug_env.py +0 -0
- {verifiers-0.2.2.dev86 → verifiers-0.2.2.dev91}/tests/test_sandbox_env.py +0 -0
- {verifiers-0.2.2.dev86 → verifiers-0.2.2.dev91}/tests/test_sandbox_mixin.py +0 -0
- {verifiers-0.2.2.dev86 → verifiers-0.2.2.dev91}/tests/test_save_utils.py +0 -0
- {verifiers-0.2.2.dev86 → verifiers-0.2.2.dev91}/tests/test_setup_script.py +0 -0
- {verifiers-0.2.2.dev86 → verifiers-0.2.2.dev91}/tests/test_singleturn_env.py +0 -0
- {verifiers-0.2.2.dev86 → verifiers-0.2.2.dev91}/tests/test_stateful_tool_env.py +0 -0
- {verifiers-0.2.2.dev86 → verifiers-0.2.2.dev91}/tests/test_think_parser.py +0 -0
- {verifiers-0.2.2.dev86 → verifiers-0.2.2.dev91}/tests/test_tool_env.py +0 -0
- {verifiers-0.2.2.dev86 → verifiers-0.2.2.dev91}/tests/test_tool_utils.py +0 -0
- {verifiers-0.2.2.dev86 → verifiers-0.2.2.dev91}/tests/test_trajectory_processing.py +0 -0
- {verifiers-0.2.2.dev86 → verifiers-0.2.2.dev91}/tests/test_tui_info_formatting.py +0 -0
- {verifiers-0.2.2.dev86 → verifiers-0.2.2.dev91}/tests/test_types.py +0 -0
- {verifiers-0.2.2.dev86 → verifiers-0.2.2.dev91}/tests/test_xml_parser.py +0 -0
- {verifiers-0.2.2.dev86 → verifiers-0.2.2.dev91}/tests/v1/fixtures/counter_tool_v1.py +0 -0
- {verifiers-0.2.2.dev86 → verifiers-0.2.2.dev91}/tests/v1/fixtures/duet_v1.py +0 -0
- {verifiers-0.2.2.dev86 → verifiers-0.2.2.dev91}/tests/v1/fixtures/echo_acp_resume_v1.py +0 -0
- {verifiers-0.2.2.dev86 → verifiers-0.2.2.dev91}/tests/v1/fixtures/echo_agentic_v1.py +0 -0
- {verifiers-0.2.2.dev86 → verifiers-0.2.2.dev91}/tests/v1/fixtures/echo_multi_v0.py +0 -0
- {verifiers-0.2.2.dev86 → verifiers-0.2.2.dev91}/tests/v1/fixtures/echo_tool_v1.py +0 -0
- {verifiers-0.2.2.dev86 → verifiers-0.2.2.dev91}/tests/v1/fixtures/echo_user_sim_v1.py +0 -0
- {verifiers-0.2.2.dev86 → verifiers-0.2.2.dev91}/tests/v1/fixtures/echo_v0.py +0 -0
- {verifiers-0.2.2.dev86 → verifiers-0.2.2.dev91}/tests/v1/fixtures/echo_v1.py +0 -0
- {verifiers-0.2.2.dev86 → verifiers-0.2.2.dev91}/tests/v1/fixtures/pyproject.toml +0 -0
- {verifiers-0.2.2.dev86 → verifiers-0.2.2.dev91}/tests/v1/fixtures/tool_response_image_v1.py +0 -0
- {verifiers-0.2.2.dev86 → verifiers-0.2.2.dev91}/tests/v1/test_color_codeword_tasks.py +0 -0
- {verifiers-0.2.2.dev86 → verifiers-0.2.2.dev91}/tests/v1/test_configs.py +0 -0
- {verifiers-0.2.2.dev86 → verifiers-0.2.2.dev91}/tests/v1/test_graph.py +0 -0
- {verifiers-0.2.2.dev86 → verifiers-0.2.2.dev91}/tests/v1/test_judges.py +0 -0
- {verifiers-0.2.2.dev86 → verifiers-0.2.2.dev91}/tests/v1/test_legacy.py +0 -0
- {verifiers-0.2.2.dev86 → verifiers-0.2.2.dev91}/tests/v1/test_scoring.py +0 -0
- {verifiers-0.2.2.dev86 → verifiers-0.2.2.dev91}/tests/v1/test_taskset.py +0 -0
- {verifiers-0.2.2.dev86 → verifiers-0.2.2.dev91}/tests/v1/test_trace.py +0 -0
- {verifiers-0.2.2.dev86 → verifiers-0.2.2.dev91}/verifiers/__init__.py +0 -0
- {verifiers-0.2.2.dev86 → verifiers-0.2.2.dev91}/verifiers/cli/__init__.py +0 -0
- {verifiers-0.2.2.dev86 → verifiers-0.2.2.dev91}/verifiers/cli/commands/__init__.py +0 -0
- {verifiers-0.2.2.dev86 → verifiers-0.2.2.dev91}/verifiers/cli/commands/build.py +0 -0
- {verifiers-0.2.2.dev86 → verifiers-0.2.2.dev91}/verifiers/cli/commands/eval.py +0 -0
- {verifiers-0.2.2.dev86 → verifiers-0.2.2.dev91}/verifiers/cli/commands/gepa.py +0 -0
- {verifiers-0.2.2.dev86 → verifiers-0.2.2.dev91}/verifiers/cli/commands/init.py +0 -0
- {verifiers-0.2.2.dev86 → verifiers-0.2.2.dev91}/verifiers/cli/commands/install.py +0 -0
- {verifiers-0.2.2.dev86 → verifiers-0.2.2.dev91}/verifiers/cli/commands/setup.py +0 -0
- {verifiers-0.2.2.dev86 → verifiers-0.2.2.dev91}/verifiers/cli/plugins/__init__.py +0 -0
- {verifiers-0.2.2.dev86 → verifiers-0.2.2.dev91}/verifiers/cli/plugins/prime.py +0 -0
- {verifiers-0.2.2.dev86 → verifiers-0.2.2.dev91}/verifiers/cli/tui.py +0 -0
- {verifiers-0.2.2.dev86 → verifiers-0.2.2.dev91}/verifiers/clients/__init__.py +0 -0
- {verifiers-0.2.2.dev86 → verifiers-0.2.2.dev91}/verifiers/clients/anthropic_messages_client.py +0 -0
- {verifiers-0.2.2.dev86 → verifiers-0.2.2.dev91}/verifiers/clients/client.py +0 -0
- {verifiers-0.2.2.dev86 → verifiers-0.2.2.dev91}/verifiers/clients/nemorl_chat_completions_client.py +0 -0
- {verifiers-0.2.2.dev86 → verifiers-0.2.2.dev91}/verifiers/clients/openai_chat_completions_client.py +0 -0
- {verifiers-0.2.2.dev86 → verifiers-0.2.2.dev91}/verifiers/clients/openai_chat_completions_token_client.py +0 -0
- {verifiers-0.2.2.dev86 → verifiers-0.2.2.dev91}/verifiers/clients/openai_completions_client.py +0 -0
- {verifiers-0.2.2.dev86 → verifiers-0.2.2.dev91}/verifiers/clients/openai_responses_client.py +0 -0
- {verifiers-0.2.2.dev86 → verifiers-0.2.2.dev91}/verifiers/clients/renderer_client.py +0 -0
- {verifiers-0.2.2.dev86 → verifiers-0.2.2.dev91}/verifiers/decorators.py +0 -0
- {verifiers-0.2.2.dev86 → verifiers-0.2.2.dev91}/verifiers/envs/__init__.py +0 -0
- {verifiers-0.2.2.dev86 → verifiers-0.2.2.dev91}/verifiers/envs/env_group.py +0 -0
- {verifiers-0.2.2.dev86 → verifiers-0.2.2.dev91}/verifiers/envs/environment.py +0 -0
- {verifiers-0.2.2.dev86 → verifiers-0.2.2.dev91}/verifiers/envs/experimental/README.md +0 -0
- {verifiers-0.2.2.dev86 → verifiers-0.2.2.dev91}/verifiers/envs/experimental/__init__.py +0 -0
- {verifiers-0.2.2.dev86 → verifiers-0.2.2.dev91}/verifiers/envs/experimental/cli_agent_env.py +0 -0
- {verifiers-0.2.2.dev86 → verifiers-0.2.2.dev91}/verifiers/envs/experimental/composable/README.md +0 -0
- {verifiers-0.2.2.dev86 → verifiers-0.2.2.dev91}/verifiers/envs/experimental/composable/__init__.py +0 -0
- {verifiers-0.2.2.dev86 → verifiers-0.2.2.dev91}/verifiers/envs/experimental/composable/_filter.py +0 -0
- {verifiers-0.2.2.dev86 → verifiers-0.2.2.dev91}/verifiers/envs/experimental/composable/composable_env.py +0 -0
- {verifiers-0.2.2.dev86 → verifiers-0.2.2.dev91}/verifiers/envs/experimental/composable/harness.py +0 -0
- {verifiers-0.2.2.dev86 → verifiers-0.2.2.dev91}/verifiers/envs/experimental/composable/harnesses/__init__.py +0 -0
- {verifiers-0.2.2.dev86 → verifiers-0.2.2.dev91}/verifiers/envs/experimental/composable/harnesses/mini_swe_agent.py +0 -0
- {verifiers-0.2.2.dev86 → verifiers-0.2.2.dev91}/verifiers/envs/experimental/composable/harnesses/opencode.py +0 -0
- {verifiers-0.2.2.dev86 → verifiers-0.2.2.dev91}/verifiers/envs/experimental/composable/harnesses/prompt.txt +0 -0
- {verifiers-0.2.2.dev86 → verifiers-0.2.2.dev91}/verifiers/envs/experimental/composable/harnesses/rlm.py +0 -0
- {verifiers-0.2.2.dev86 → verifiers-0.2.2.dev91}/verifiers/envs/experimental/composable/sandbox_debug_env.py +0 -0
- {verifiers-0.2.2.dev86 → verifiers-0.2.2.dev91}/verifiers/envs/experimental/composable/swe_debug_env.py +0 -0
- {verifiers-0.2.2.dev86 → verifiers-0.2.2.dev91}/verifiers/envs/experimental/composable/task.py +0 -0
- {verifiers-0.2.2.dev86 → verifiers-0.2.2.dev91}/verifiers/envs/experimental/composable/tasksets/__init__.py +0 -0
- {verifiers-0.2.2.dev86 → verifiers-0.2.2.dev91}/verifiers/envs/experimental/composable/tasksets/cp/__init__.py +0 -0
- {verifiers-0.2.2.dev86 → verifiers-0.2.2.dev91}/verifiers/envs/experimental/composable/tasksets/cp/cp_task.py +0 -0
- {verifiers-0.2.2.dev86 → verifiers-0.2.2.dev91}/verifiers/envs/experimental/composable/tasksets/cp/test_utils.py +0 -0
- {verifiers-0.2.2.dev86 → verifiers-0.2.2.dev91}/verifiers/envs/experimental/composable/tasksets/harbor/__init__.py +0 -0
- {verifiers-0.2.2.dev86 → verifiers-0.2.2.dev91}/verifiers/envs/experimental/composable/tasksets/harbor/cli_gym.py +0 -0
- {verifiers-0.2.2.dev86 → verifiers-0.2.2.dev91}/verifiers/envs/experimental/composable/tasksets/harbor/harbor.py +0 -0
- {verifiers-0.2.2.dev86 → verifiers-0.2.2.dev91}/verifiers/envs/experimental/composable/tasksets/harbor/terminal_lego.py +0 -0
- {verifiers-0.2.2.dev86 → verifiers-0.2.2.dev91}/verifiers/envs/experimental/composable/tasksets/math/__init__.py +0 -0
- {verifiers-0.2.2.dev86 → verifiers-0.2.2.dev91}/verifiers/envs/experimental/composable/tasksets/math/math_task.py +0 -0
- {verifiers-0.2.2.dev86 → verifiers-0.2.2.dev91}/verifiers/envs/experimental/composable/tasksets/swe/README.md +0 -0
- {verifiers-0.2.2.dev86 → verifiers-0.2.2.dev91}/verifiers/envs/experimental/composable/tasksets/swe/__init__.py +0 -0
- {verifiers-0.2.2.dev86 → verifiers-0.2.2.dev91}/verifiers/envs/experimental/composable/tasksets/swe/multi_swe/__init__.py +0 -0
- {verifiers-0.2.2.dev86 → verifiers-0.2.2.dev91}/verifiers/envs/experimental/composable/tasksets/swe/multi_swe/extract_fix_patch.sh +0 -0
- {verifiers-0.2.2.dev86 → verifiers-0.2.2.dev91}/verifiers/envs/experimental/composable/tasksets/swe/multi_swe/taskset.py +0 -0
- {verifiers-0.2.2.dev86 → verifiers-0.2.2.dev91}/verifiers/envs/experimental/composable/tasksets/swe/openswe/__init__.py +0 -0
- {verifiers-0.2.2.dev86 → verifiers-0.2.2.dev91}/verifiers/envs/experimental/composable/tasksets/swe/openswe/taskset.py +0 -0
- {verifiers-0.2.2.dev86 → verifiers-0.2.2.dev91}/verifiers/envs/experimental/composable/tasksets/swe/r2e_gym/__init__.py +0 -0
- {verifiers-0.2.2.dev86 → verifiers-0.2.2.dev91}/verifiers/envs/experimental/composable/tasksets/swe/r2e_gym/log_parser.py +0 -0
- {verifiers-0.2.2.dev86 → verifiers-0.2.2.dev91}/verifiers/envs/experimental/composable/tasksets/swe/r2e_gym/taskset.py +0 -0
- {verifiers-0.2.2.dev86 → verifiers-0.2.2.dev91}/verifiers/envs/experimental/composable/tasksets/swe/scale_swe/__init__.py +0 -0
- {verifiers-0.2.2.dev86 → verifiers-0.2.2.dev91}/verifiers/envs/experimental/composable/tasksets/swe/scale_swe/taskset.py +0 -0
- {verifiers-0.2.2.dev86 → verifiers-0.2.2.dev91}/verifiers/envs/experimental/composable/tasksets/swe/shared/__init__.py +0 -0
- {verifiers-0.2.2.dev86 → verifiers-0.2.2.dev91}/verifiers/envs/experimental/composable/tasksets/swe/shared/test_patch.py +0 -0
- {verifiers-0.2.2.dev86 → verifiers-0.2.2.dev91}/verifiers/envs/experimental/composable/tasksets/swe/swe_bench/__init__.py +0 -0
- {verifiers-0.2.2.dev86 → verifiers-0.2.2.dev91}/verifiers/envs/experimental/composable/tasksets/swe/swe_bench/taskset.py +0 -0
- {verifiers-0.2.2.dev86 → verifiers-0.2.2.dev91}/verifiers/envs/experimental/composable/tasksets/swe/swe_lego/__init__.py +0 -0
- {verifiers-0.2.2.dev86 → verifiers-0.2.2.dev91}/verifiers/envs/experimental/composable/tasksets/swe/swe_lego/taskset.py +0 -0
- {verifiers-0.2.2.dev86 → verifiers-0.2.2.dev91}/verifiers/envs/experimental/composable/tasksets/swe/swe_rebench_v2/__init__.py +0 -0
- {verifiers-0.2.2.dev86 → verifiers-0.2.2.dev91}/verifiers/envs/experimental/composable/tasksets/swe/swe_rebench_v2/log_parsers.py +0 -0
- {verifiers-0.2.2.dev86 → verifiers-0.2.2.dev91}/verifiers/envs/experimental/composable/tasksets/swe/swe_rebench_v2/taskset.py +0 -0
- {verifiers-0.2.2.dev86 → verifiers-0.2.2.dev91}/verifiers/envs/experimental/composable/tasksets/swe/swe_smith/__init__.py +0 -0
- {verifiers-0.2.2.dev86 → verifiers-0.2.2.dev91}/verifiers/envs/experimental/composable/tasksets/swe/swe_smith/taskset.py +0 -0
- {verifiers-0.2.2.dev86 → verifiers-0.2.2.dev91}/verifiers/envs/experimental/composable/tasksets/swe/swe_tasksets.py +0 -0
- {verifiers-0.2.2.dev86 → verifiers-0.2.2.dev91}/verifiers/envs/experimental/gym_env.py +0 -0
- {verifiers-0.2.2.dev86 → verifiers-0.2.2.dev91}/verifiers/envs/experimental/harbor_env/__init__.py +0 -0
- {verifiers-0.2.2.dev86 → verifiers-0.2.2.dev91}/verifiers/envs/experimental/harbor_env/env.py +0 -0
- {verifiers-0.2.2.dev86 → verifiers-0.2.2.dev91}/verifiers/envs/experimental/harbor_env/mcp.py +0 -0
- {verifiers-0.2.2.dev86 → verifiers-0.2.2.dev91}/verifiers/envs/experimental/mcp_env.py +0 -0
- {verifiers-0.2.2.dev86 → verifiers-0.2.2.dev91}/verifiers/envs/experimental/opencode_env.py +0 -0
- {verifiers-0.2.2.dev86 → verifiers-0.2.2.dev91}/verifiers/envs/experimental/opencode_qa_env.py +0 -0
- {verifiers-0.2.2.dev86 → verifiers-0.2.2.dev91}/verifiers/envs/experimental/opencode_rlm_env.py +0 -0
- {verifiers-0.2.2.dev86 → verifiers-0.2.2.dev91}/verifiers/envs/experimental/sandbox_mixin.py +0 -0
- {verifiers-0.2.2.dev86 → verifiers-0.2.2.dev91}/verifiers/envs/experimental/utils/__init__.py +0 -0
- {verifiers-0.2.2.dev86 → verifiers-0.2.2.dev91}/verifiers/envs/experimental/utils/file_locks.py +0 -0
- {verifiers-0.2.2.dev86 → verifiers-0.2.2.dev91}/verifiers/envs/experimental/utils/git_checkout_cache.py +0 -0
- {verifiers-0.2.2.dev86 → verifiers-0.2.2.dev91}/verifiers/envs/integrations/README.md +0 -0
- {verifiers-0.2.2.dev86 → verifiers-0.2.2.dev91}/verifiers/envs/integrations/__init__.py +0 -0
- {verifiers-0.2.2.dev86 → verifiers-0.2.2.dev91}/verifiers/envs/integrations/browser_env/README.md +0 -0
- {verifiers-0.2.2.dev86 → verifiers-0.2.2.dev91}/verifiers/envs/integrations/browser_env/__init__.py +0 -0
- {verifiers-0.2.2.dev86 → verifiers-0.2.2.dev91}/verifiers/envs/integrations/browser_env/browser_env.py +0 -0
- {verifiers-0.2.2.dev86 → verifiers-0.2.2.dev91}/verifiers/envs/integrations/browser_env/modes/__init__.py +0 -0
- {verifiers-0.2.2.dev86 → verifiers-0.2.2.dev91}/verifiers/envs/integrations/browser_env/modes/base.py +0 -0
- {verifiers-0.2.2.dev86 → verifiers-0.2.2.dev91}/verifiers/envs/integrations/browser_env/modes/cua_mode.py +0 -0
- {verifiers-0.2.2.dev86 → verifiers-0.2.2.dev91}/verifiers/envs/integrations/browser_env/modes/dom_mode.py +0 -0
- {verifiers-0.2.2.dev86 → verifiers-0.2.2.dev91}/verifiers/envs/integrations/openenv_env.py +0 -0
- {verifiers-0.2.2.dev86 → verifiers-0.2.2.dev91}/verifiers/envs/integrations/reasoninggym_env.py +0 -0
- {verifiers-0.2.2.dev86 → verifiers-0.2.2.dev91}/verifiers/envs/integrations/textarena_env.py +0 -0
- {verifiers-0.2.2.dev86 → verifiers-0.2.2.dev91}/verifiers/envs/multiturn_env.py +0 -0
- {verifiers-0.2.2.dev86 → verifiers-0.2.2.dev91}/verifiers/envs/python_env.py +0 -0
- {verifiers-0.2.2.dev86 → verifiers-0.2.2.dev91}/verifiers/envs/sandbox_env.py +0 -0
- {verifiers-0.2.2.dev86 → verifiers-0.2.2.dev91}/verifiers/envs/singleturn_env.py +0 -0
- {verifiers-0.2.2.dev86 → verifiers-0.2.2.dev91}/verifiers/envs/stateful_tool_env.py +0 -0
- {verifiers-0.2.2.dev86 → verifiers-0.2.2.dev91}/verifiers/envs/tool_env.py +0 -0
- {verifiers-0.2.2.dev86 → verifiers-0.2.2.dev91}/verifiers/errors.py +0 -0
- {verifiers-0.2.2.dev86 → verifiers-0.2.2.dev91}/verifiers/gepa/__init__.py +0 -0
- {verifiers-0.2.2.dev86 → verifiers-0.2.2.dev91}/verifiers/gepa/adapter.py +0 -0
- {verifiers-0.2.2.dev86 → verifiers-0.2.2.dev91}/verifiers/gepa/config.py +0 -0
- {verifiers-0.2.2.dev86 → verifiers-0.2.2.dev91}/verifiers/gepa/display.py +0 -0
- {verifiers-0.2.2.dev86 → verifiers-0.2.2.dev91}/verifiers/gepa/gepa_utils.py +0 -0
- {verifiers-0.2.2.dev86 → verifiers-0.2.2.dev91}/verifiers/parsers/__init__.py +0 -0
- {verifiers-0.2.2.dev86 → verifiers-0.2.2.dev91}/verifiers/parsers/maybe_think_parser.py +0 -0
- {verifiers-0.2.2.dev86 → verifiers-0.2.2.dev91}/verifiers/parsers/parser.py +0 -0
- {verifiers-0.2.2.dev86 → verifiers-0.2.2.dev91}/verifiers/parsers/think_parser.py +0 -0
- {verifiers-0.2.2.dev86 → verifiers-0.2.2.dev91}/verifiers/parsers/xml_parser.py +0 -0
- {verifiers-0.2.2.dev86 → verifiers-0.2.2.dev91}/verifiers/rubrics/__init__.py +0 -0
- {verifiers-0.2.2.dev86 → verifiers-0.2.2.dev91}/verifiers/rubrics/experimental/hybrid_math_rubric.py +0 -0
- {verifiers-0.2.2.dev86 → verifiers-0.2.2.dev91}/verifiers/rubrics/judge_rubric.py +0 -0
- {verifiers-0.2.2.dev86 → verifiers-0.2.2.dev91}/verifiers/rubrics/math_rubric.py +0 -0
- {verifiers-0.2.2.dev86 → verifiers-0.2.2.dev91}/verifiers/rubrics/rubric.py +0 -0
- {verifiers-0.2.2.dev86 → verifiers-0.2.2.dev91}/verifiers/rubrics/rubric_group.py +0 -0
- {verifiers-0.2.2.dev86 → verifiers-0.2.2.dev91}/verifiers/scripts/__init__.py +0 -0
- {verifiers-0.2.2.dev86 → verifiers-0.2.2.dev91}/verifiers/scripts/build.py +0 -0
- {verifiers-0.2.2.dev86 → verifiers-0.2.2.dev91}/verifiers/scripts/eval.py +0 -0
- {verifiers-0.2.2.dev86 → verifiers-0.2.2.dev91}/verifiers/scripts/gepa.py +0 -0
- {verifiers-0.2.2.dev86 → verifiers-0.2.2.dev91}/verifiers/scripts/init.py +0 -0
- {verifiers-0.2.2.dev86 → verifiers-0.2.2.dev91}/verifiers/scripts/install.py +0 -0
- {verifiers-0.2.2.dev86 → verifiers-0.2.2.dev91}/verifiers/scripts/setup.py +0 -0
- {verifiers-0.2.2.dev86 → verifiers-0.2.2.dev91}/verifiers/scripts/tui.py +0 -0
- {verifiers-0.2.2.dev86 → verifiers-0.2.2.dev91}/verifiers/serve/__init__.py +0 -0
- {verifiers-0.2.2.dev86 → verifiers-0.2.2.dev91}/verifiers/serve/client/env_client.py +0 -0
- {verifiers-0.2.2.dev86 → verifiers-0.2.2.dev91}/verifiers/serve/client/zmq_env_client.py +0 -0
- {verifiers-0.2.2.dev86 → verifiers-0.2.2.dev91}/verifiers/serve/server/__init__.py +0 -0
- {verifiers-0.2.2.dev86 → verifiers-0.2.2.dev91}/verifiers/serve/server/env_router.py +0 -0
- {verifiers-0.2.2.dev86 → verifiers-0.2.2.dev91}/verifiers/serve/server/env_server.py +0 -0
- {verifiers-0.2.2.dev86 → verifiers-0.2.2.dev91}/verifiers/serve/server/env_worker.py +0 -0
- {verifiers-0.2.2.dev86 → verifiers-0.2.2.dev91}/verifiers/serve/server/zmq_env_server.py +0 -0
- {verifiers-0.2.2.dev86 → verifiers-0.2.2.dev91}/verifiers/serve/types.py +0 -0
- {verifiers-0.2.2.dev86 → verifiers-0.2.2.dev91}/verifiers/types.py +0 -0
- {verifiers-0.2.2.dev86 → verifiers-0.2.2.dev91}/verifiers/utils/__init__.py +0 -0
- {verifiers-0.2.2.dev86 → verifiers-0.2.2.dev91}/verifiers/utils/async_utils.py +0 -0
- {verifiers-0.2.2.dev86 → verifiers-0.2.2.dev91}/verifiers/utils/client_utils.py +0 -0
- {verifiers-0.2.2.dev86 → verifiers-0.2.2.dev91}/verifiers/utils/config_utils.py +0 -0
- {verifiers-0.2.2.dev86 → verifiers-0.2.2.dev91}/verifiers/utils/data_utils.py +0 -0
- {verifiers-0.2.2.dev86 → verifiers-0.2.2.dev91}/verifiers/utils/display_utils.py +0 -0
- {verifiers-0.2.2.dev86 → verifiers-0.2.2.dev91}/verifiers/utils/env_config_utils.py +0 -0
- {verifiers-0.2.2.dev86 → verifiers-0.2.2.dev91}/verifiers/utils/env_utils.py +0 -0
- {verifiers-0.2.2.dev86 → verifiers-0.2.2.dev91}/verifiers/utils/error_utils.py +0 -0
- {verifiers-0.2.2.dev86 → verifiers-0.2.2.dev91}/verifiers/utils/eval_display.py +0 -0
- {verifiers-0.2.2.dev86 → verifiers-0.2.2.dev91}/verifiers/utils/eval_utils.py +0 -0
- {verifiers-0.2.2.dev86 → verifiers-0.2.2.dev91}/verifiers/utils/heartbeat.py +0 -0
- {verifiers-0.2.2.dev86 → verifiers-0.2.2.dev91}/verifiers/utils/import_utils.py +0 -0
- {verifiers-0.2.2.dev86 → verifiers-0.2.2.dev91}/verifiers/utils/install_utils.py +0 -0
- {verifiers-0.2.2.dev86 → verifiers-0.2.2.dev91}/verifiers/utils/interception_utils.py +0 -0
- {verifiers-0.2.2.dev86 → verifiers-0.2.2.dev91}/verifiers/utils/logging_utils.py +0 -0
- {verifiers-0.2.2.dev86 → verifiers-0.2.2.dev91}/verifiers/utils/message_utils.py +0 -0
- {verifiers-0.2.2.dev86 → verifiers-0.2.2.dev91}/verifiers/utils/metric_utils.py +0 -0
- {verifiers-0.2.2.dev86 → verifiers-0.2.2.dev91}/verifiers/utils/path_utils.py +0 -0
- {verifiers-0.2.2.dev86 → verifiers-0.2.2.dev91}/verifiers/utils/pricing_utils.py +0 -0
- {verifiers-0.2.2.dev86 → verifiers-0.2.2.dev91}/verifiers/utils/process_utils.py +0 -0
- {verifiers-0.2.2.dev86 → verifiers-0.2.2.dev91}/verifiers/utils/response_utils.py +0 -0
- {verifiers-0.2.2.dev86 → verifiers-0.2.2.dev91}/verifiers/utils/save_utils.py +0 -0
- {verifiers-0.2.2.dev86 → verifiers-0.2.2.dev91}/verifiers/utils/serve_utils.py +0 -0
- {verifiers-0.2.2.dev86 → verifiers-0.2.2.dev91}/verifiers/utils/thread_utils.py +0 -0
- {verifiers-0.2.2.dev86 → verifiers-0.2.2.dev91}/verifiers/utils/threaded_sandbox_client.py +0 -0
- {verifiers-0.2.2.dev86 → verifiers-0.2.2.dev91}/verifiers/utils/tool_utils.py +0 -0
- {verifiers-0.2.2.dev86 → verifiers-0.2.2.dev91}/verifiers/utils/usage_utils.py +0 -0
- {verifiers-0.2.2.dev86 → verifiers-0.2.2.dev91}/verifiers/utils/version_utils.py +0 -0
- {verifiers-0.2.2.dev86 → verifiers-0.2.2.dev91}/verifiers/v1/agent.py +0 -0
- {verifiers-0.2.2.dev86 → verifiers-0.2.2.dev91}/verifiers/v1/cli/__init__.py +0 -0
- {verifiers-0.2.2.dev86 → verifiers-0.2.2.dev91}/verifiers/v1/cli/dashboard/__init__.py +0 -0
- {verifiers-0.2.2.dev86 → verifiers-0.2.2.dev91}/verifiers/v1/cli/dashboard/base.py +0 -0
- {verifiers-0.2.2.dev86 → verifiers-0.2.2.dev91}/verifiers/v1/cli/dashboard/eval.py +0 -0
- {verifiers-0.2.2.dev86 → verifiers-0.2.2.dev91}/verifiers/v1/cli/dashboard/replay.py +0 -0
- {verifiers-0.2.2.dev86 → verifiers-0.2.2.dev91}/verifiers/v1/cli/dashboard/validate.py +0 -0
- {verifiers-0.2.2.dev86 → verifiers-0.2.2.dev91}/verifiers/v1/cli/debug.py +0 -0
- {verifiers-0.2.2.dev86 → verifiers-0.2.2.dev91}/verifiers/v1/cli/eval/__init__.py +0 -0
- {verifiers-0.2.2.dev86 → verifiers-0.2.2.dev91}/verifiers/v1/cli/eval/main.py +0 -0
- {verifiers-0.2.2.dev86 → verifiers-0.2.2.dev91}/verifiers/v1/cli/eval/resume.py +0 -0
- {verifiers-0.2.2.dev86 → verifiers-0.2.2.dev91}/verifiers/v1/cli/eval/runner.py +0 -0
- {verifiers-0.2.2.dev86 → verifiers-0.2.2.dev91}/verifiers/v1/cli/gepa.py +0 -0
- {verifiers-0.2.2.dev86 → verifiers-0.2.2.dev91}/verifiers/v1/cli/init.py +0 -0
- {verifiers-0.2.2.dev86 → verifiers-0.2.2.dev91}/verifiers/v1/cli/output.py +0 -0
- {verifiers-0.2.2.dev86 → verifiers-0.2.2.dev91}/verifiers/v1/cli/replay.py +0 -0
- {verifiers-0.2.2.dev86 → verifiers-0.2.2.dev91}/verifiers/v1/cli/resolve.py +0 -0
- {verifiers-0.2.2.dev86 → verifiers-0.2.2.dev91}/verifiers/v1/cli/validate.py +0 -0
- {verifiers-0.2.2.dev86 → verifiers-0.2.2.dev91}/verifiers/v1/clients/__init__.py +0 -0
- {verifiers-0.2.2.dev86 → verifiers-0.2.2.dev91}/verifiers/v1/clients/base.py +0 -0
- {verifiers-0.2.2.dev86 → verifiers-0.2.2.dev91}/verifiers/v1/clients/client.py +0 -0
- {verifiers-0.2.2.dev86 → verifiers-0.2.2.dev91}/verifiers/v1/clients/eval.py +0 -0
- {verifiers-0.2.2.dev86 → verifiers-0.2.2.dev91}/verifiers/v1/clients/train.py +0 -0
- {verifiers-0.2.2.dev86 → verifiers-0.2.2.dev91}/verifiers/v1/configs/__init__.py +0 -0
- {verifiers-0.2.2.dev86 → verifiers-0.2.2.dev91}/verifiers/v1/configs/agent.py +0 -0
- {verifiers-0.2.2.dev86 → verifiers-0.2.2.dev91}/verifiers/v1/configs/cli/__init__.py +0 -0
- {verifiers-0.2.2.dev86 → verifiers-0.2.2.dev91}/verifiers/v1/configs/cli/debug.py +0 -0
- {verifiers-0.2.2.dev86 → verifiers-0.2.2.dev91}/verifiers/v1/configs/cli/env.py +0 -0
- {verifiers-0.2.2.dev86 → verifiers-0.2.2.dev91}/verifiers/v1/configs/cli/eval.py +0 -0
- {verifiers-0.2.2.dev86 → verifiers-0.2.2.dev91}/verifiers/v1/configs/cli/init.py +0 -0
- {verifiers-0.2.2.dev86 → verifiers-0.2.2.dev91}/verifiers/v1/configs/cli/replay.py +0 -0
- {verifiers-0.2.2.dev86 → verifiers-0.2.2.dev91}/verifiers/v1/configs/cli/validate.py +0 -0
- {verifiers-0.2.2.dev86 → verifiers-0.2.2.dev91}/verifiers/v1/configs/client.py +0 -0
- {verifiers-0.2.2.dev86 → verifiers-0.2.2.dev91}/verifiers/v1/configs/env.py +0 -0
- {verifiers-0.2.2.dev86 → verifiers-0.2.2.dev91}/verifiers/v1/configs/judge.py +0 -0
- {verifiers-0.2.2.dev86 → verifiers-0.2.2.dev91}/verifiers/v1/configs/legacy.py +0 -0
- {verifiers-0.2.2.dev86 → verifiers-0.2.2.dev91}/verifiers/v1/configs/retries.py +0 -0
- {verifiers-0.2.2.dev86 → verifiers-0.2.2.dev91}/verifiers/v1/configs/serve.py +0 -0
- {verifiers-0.2.2.dev86 → verifiers-0.2.2.dev91}/verifiers/v1/configs/task.py +0 -0
- {verifiers-0.2.2.dev86 → verifiers-0.2.2.dev91}/verifiers/v1/configs/taskset.py +0 -0
- {verifiers-0.2.2.dev86 → verifiers-0.2.2.dev91}/verifiers/v1/dialects/__init__.py +0 -0
- {verifiers-0.2.2.dev86 → verifiers-0.2.2.dev91}/verifiers/v1/dialects/anthropic.py +0 -0
- {verifiers-0.2.2.dev86 → verifiers-0.2.2.dev91}/verifiers/v1/dialects/base.py +0 -0
- {verifiers-0.2.2.dev86 → verifiers-0.2.2.dev91}/verifiers/v1/dialects/chat.py +0 -0
- {verifiers-0.2.2.dev86 → verifiers-0.2.2.dev91}/verifiers/v1/dialects/responses.py +0 -0
- {verifiers-0.2.2.dev86 → verifiers-0.2.2.dev91}/verifiers/v1/env.py +0 -0
- {verifiers-0.2.2.dev86 → verifiers-0.2.2.dev91}/verifiers/v1/envs/__init__.py +0 -0
- {verifiers-0.2.2.dev86 → verifiers-0.2.2.dev91}/verifiers/v1/envs/agentic_judge/__init__.py +0 -0
- {verifiers-0.2.2.dev86 → verifiers-0.2.2.dev91}/verifiers/v1/envs/agentic_judge/env.py +0 -0
- {verifiers-0.2.2.dev86 → verifiers-0.2.2.dev91}/verifiers/v1/envs/best_of_n/__init__.py +0 -0
- {verifiers-0.2.2.dev86 → verifiers-0.2.2.dev91}/verifiers/v1/envs/best_of_n/env.py +0 -0
- {verifiers-0.2.2.dev86 → verifiers-0.2.2.dev91}/verifiers/v1/envs/shared_agentic_judge/__init__.py +0 -0
- {verifiers-0.2.2.dev86 → verifiers-0.2.2.dev91}/verifiers/v1/envs/single_agent/__init__.py +0 -0
- {verifiers-0.2.2.dev86 → verifiers-0.2.2.dev91}/verifiers/v1/envs/single_agent/env.py +0 -0
- {verifiers-0.2.2.dev86 → verifiers-0.2.2.dev91}/verifiers/v1/envs/user_sim/__init__.py +0 -0
- {verifiers-0.2.2.dev86 → verifiers-0.2.2.dev91}/verifiers/v1/envs/user_sim/env.py +0 -0
- {verifiers-0.2.2.dev86 → verifiers-0.2.2.dev91}/verifiers/v1/episode.py +0 -0
- {verifiers-0.2.2.dev86 → verifiers-0.2.2.dev91}/verifiers/v1/errors.py +0 -0
- {verifiers-0.2.2.dev86 → verifiers-0.2.2.dev91}/verifiers/v1/gepa/__init__.py +0 -0
- {verifiers-0.2.2.dev86 → verifiers-0.2.2.dev91}/verifiers/v1/gepa/adapter.py +0 -0
- {verifiers-0.2.2.dev86 → verifiers-0.2.2.dev91}/verifiers/v1/gepa/config.py +0 -0
- {verifiers-0.2.2.dev86 → verifiers-0.2.2.dev91}/verifiers/v1/gepa/dataset.py +0 -0
- {verifiers-0.2.2.dev86 → verifiers-0.2.2.dev91}/verifiers/v1/gepa/reflection.py +0 -0
- {verifiers-0.2.2.dev86 → verifiers-0.2.2.dev91}/verifiers/v1/gepa/runner.py +0 -0
- {verifiers-0.2.2.dev86 → verifiers-0.2.2.dev91}/verifiers/v1/graph.py +0 -0
- {verifiers-0.2.2.dev86 → verifiers-0.2.2.dev91}/verifiers/v1/harnesses/__init__.py +0 -0
- {verifiers-0.2.2.dev86 → verifiers-0.2.2.dev91}/verifiers/v1/harnesses/bash/__init__.py +0 -0
- {verifiers-0.2.2.dev86 → verifiers-0.2.2.dev91}/verifiers/v1/harnesses/bash/harness.py +0 -0
- {verifiers-0.2.2.dev86 → verifiers-0.2.2.dev91}/verifiers/v1/harnesses/bash/program.py +0 -0
- {verifiers-0.2.2.dev86 → verifiers-0.2.2.dev91}/verifiers/v1/harnesses/browser_use/__init__.py +0 -0
- {verifiers-0.2.2.dev86 → verifiers-0.2.2.dev91}/verifiers/v1/harnesses/browser_use/harness.py +0 -0
- {verifiers-0.2.2.dev86 → verifiers-0.2.2.dev91}/verifiers/v1/harnesses/browser_use/program.py +0 -0
- {verifiers-0.2.2.dev86 → verifiers-0.2.2.dev91}/verifiers/v1/harnesses/claude_code/__init__.py +0 -0
- {verifiers-0.2.2.dev86 → verifiers-0.2.2.dev91}/verifiers/v1/harnesses/codex/__init__.py +0 -0
- {verifiers-0.2.2.dev86 → verifiers-0.2.2.dev91}/verifiers/v1/harnesses/hermes_agent/__init__.py +0 -0
- {verifiers-0.2.2.dev86 → verifiers-0.2.2.dev91}/verifiers/v1/harnesses/hermes_agent/harness.py +0 -0
- {verifiers-0.2.2.dev86 → verifiers-0.2.2.dev91}/verifiers/v1/harnesses/hermes_agent/program.py +0 -0
- {verifiers-0.2.2.dev86 → verifiers-0.2.2.dev91}/verifiers/v1/harnesses/kimi_code/__init__.py +0 -0
- {verifiers-0.2.2.dev86 → verifiers-0.2.2.dev91}/verifiers/v1/harnesses/kimi_code/harness.py +0 -0
- {verifiers-0.2.2.dev86 → verifiers-0.2.2.dev91}/verifiers/v1/harnesses/mini_swe_agent/__init__.py +0 -0
- {verifiers-0.2.2.dev86 → verifiers-0.2.2.dev91}/verifiers/v1/harnesses/mini_swe_agent/harness.py +0 -0
- {verifiers-0.2.2.dev86 → verifiers-0.2.2.dev91}/verifiers/v1/harnesses/mini_swe_agent/program.py +0 -0
- {verifiers-0.2.2.dev86 → verifiers-0.2.2.dev91}/verifiers/v1/harnesses/null/__init__.py +0 -0
- {verifiers-0.2.2.dev86 → verifiers-0.2.2.dev91}/verifiers/v1/harnesses/null/harness.py +0 -0
- {verifiers-0.2.2.dev86 → verifiers-0.2.2.dev91}/verifiers/v1/harnesses/null/program.py +0 -0
- {verifiers-0.2.2.dev86 → verifiers-0.2.2.dev91}/verifiers/v1/harnesses/openclaw/__init__.py +0 -0
- {verifiers-0.2.2.dev86 → verifiers-0.2.2.dev91}/verifiers/v1/harnesses/openclaw/harness.py +0 -0
- {verifiers-0.2.2.dev86 → verifiers-0.2.2.dev91}/verifiers/v1/harnesses/pi/__init__.py +0 -0
- {verifiers-0.2.2.dev86 → verifiers-0.2.2.dev91}/verifiers/v1/harnesses/pool/__init__.py +0 -0
- {verifiers-0.2.2.dev86 → verifiers-0.2.2.dev91}/verifiers/v1/harnesses/pool/harness.py +0 -0
- {verifiers-0.2.2.dev86 → verifiers-0.2.2.dev91}/verifiers/v1/harnesses/rlm/__init__.py +0 -0
- {verifiers-0.2.2.dev86 → verifiers-0.2.2.dev91}/verifiers/v1/harnesses/terminus_2/__init__.py +0 -0
- {verifiers-0.2.2.dev86 → verifiers-0.2.2.dev91}/verifiers/v1/harnesses/terminus_2/harness.py +0 -0
- {verifiers-0.2.2.dev86 → verifiers-0.2.2.dev91}/verifiers/v1/harnesses/terminus_2/program.py +0 -0
- {verifiers-0.2.2.dev86 → verifiers-0.2.2.dev91}/verifiers/v1/interception/__init__.py +0 -0
- {verifiers-0.2.2.dev86 → verifiers-0.2.2.dev91}/verifiers/v1/interception/base.py +0 -0
- {verifiers-0.2.2.dev86 → verifiers-0.2.2.dev91}/verifiers/v1/interception/pool.py +0 -0
- {verifiers-0.2.2.dev86 → verifiers-0.2.2.dev91}/verifiers/v1/interception/tunnel/__init__.py +0 -0
- {verifiers-0.2.2.dev86 → verifiers-0.2.2.dev91}/verifiers/v1/interception/tunnel/base.py +0 -0
- {verifiers-0.2.2.dev86 → verifiers-0.2.2.dev91}/verifiers/v1/interception/tunnel/custom.py +0 -0
- {verifiers-0.2.2.dev86 → verifiers-0.2.2.dev91}/verifiers/v1/interception/tunnel/prime.py +0 -0
- {verifiers-0.2.2.dev86 → verifiers-0.2.2.dev91}/verifiers/v1/judge.py +0 -0
- {verifiers-0.2.2.dev86 → verifiers-0.2.2.dev91}/verifiers/v1/judges/__init__.py +0 -0
- {verifiers-0.2.2.dev86 → verifiers-0.2.2.dev91}/verifiers/v1/judges/reference.py +0 -0
- {verifiers-0.2.2.dev86 → verifiers-0.2.2.dev91}/verifiers/v1/judges/reference.txt +0 -0
- {verifiers-0.2.2.dev86 → verifiers-0.2.2.dev91}/verifiers/v1/judges/rubric.py +0 -0
- {verifiers-0.2.2.dev86 → verifiers-0.2.2.dev91}/verifiers/v1/judges/rubric.txt +0 -0
- {verifiers-0.2.2.dev86 → verifiers-0.2.2.dev91}/verifiers/v1/legacy.py +0 -0
- {verifiers-0.2.2.dev86 → verifiers-0.2.2.dev91}/verifiers/v1/mcp/__init__.py +0 -0
- {verifiers-0.2.2.dev86 → verifiers-0.2.2.dev91}/verifiers/v1/mcp/server.py +0 -0
- {verifiers-0.2.2.dev86 → verifiers-0.2.2.dev91}/verifiers/v1/mcp/toolset.py +0 -0
- {verifiers-0.2.2.dev86 → verifiers-0.2.2.dev91}/verifiers/v1/runtimes/docker/egress.py +0 -0
- {verifiers-0.2.2.dev86 → verifiers-0.2.2.dev91}/verifiers/v1/runtimes/limiters.py +0 -0
- {verifiers-0.2.2.dev86 → verifiers-0.2.2.dev91}/verifiers/v1/serve/__init__.py +0 -0
- {verifiers-0.2.2.dev86 → verifiers-0.2.2.dev91}/verifiers/v1/serve/client.py +0 -0
- {verifiers-0.2.2.dev86 → verifiers-0.2.2.dev91}/verifiers/v1/serve/pool.py +0 -0
- {verifiers-0.2.2.dev86 → verifiers-0.2.2.dev91}/verifiers/v1/serve/server.py +0 -0
- {verifiers-0.2.2.dev86 → verifiers-0.2.2.dev91}/verifiers/v1/serve/types.py +0 -0
- {verifiers-0.2.2.dev86 → verifiers-0.2.2.dev91}/verifiers/v1/state.py +0 -0
- {verifiers-0.2.2.dev86 → verifiers-0.2.2.dev91}/verifiers/v1/task.py +0 -0
- {verifiers-0.2.2.dev86 → verifiers-0.2.2.dev91}/verifiers/v1/taskset.py +0 -0
- {verifiers-0.2.2.dev86 → verifiers-0.2.2.dev91}/verifiers/v1/tasksets/__init__.py +0 -0
- {verifiers-0.2.2.dev86 → verifiers-0.2.2.dev91}/verifiers/v1/tasksets/harbor/__init__.py +0 -0
- {verifiers-0.2.2.dev86 → verifiers-0.2.2.dev91}/verifiers/v1/tasksets/harbor/env.py +0 -0
- {verifiers-0.2.2.dev86 → verifiers-0.2.2.dev91}/verifiers/v1/tasksets/harbor/taskset.py +0 -0
- {verifiers-0.2.2.dev86 → verifiers-0.2.2.dev91}/verifiers/v1/tasksets/lean/__init__.py +0 -0
- {verifiers-0.2.2.dev86 → verifiers-0.2.2.dev91}/verifiers/v1/tasksets/lean/scoring.py +0 -0
- {verifiers-0.2.2.dev86 → verifiers-0.2.2.dev91}/verifiers/v1/tasksets/lean/taskset.py +0 -0
- {verifiers-0.2.2.dev86 → verifiers-0.2.2.dev91}/verifiers/v1/tasksets/openenv/__init__.py +0 -0
- {verifiers-0.2.2.dev86 → verifiers-0.2.2.dev91}/verifiers/v1/tasksets/openenv/taskset.py +0 -0
- {verifiers-0.2.2.dev86 → verifiers-0.2.2.dev91}/verifiers/v1/tasksets/textarena/__init__.py +0 -0
- {verifiers-0.2.2.dev86 → verifiers-0.2.2.dev91}/verifiers/v1/tasksets/textarena/taskset.py +0 -0
- {verifiers-0.2.2.dev86 → verifiers-0.2.2.dev91}/verifiers/v1/trace.py +0 -0
- {verifiers-0.2.2.dev86 → verifiers-0.2.2.dev91}/verifiers/v1/types.py +0 -0
- {verifiers-0.2.2.dev86 → verifiers-0.2.2.dev91}/verifiers/v1/utils/__init__.py +0 -0
- {verifiers-0.2.2.dev86 → verifiers-0.2.2.dev91}/verifiers/v1/utils/aio.py +0 -0
- {verifiers-0.2.2.dev86 → verifiers-0.2.2.dev91}/verifiers/v1/utils/artifacts.py +0 -0
- {verifiers-0.2.2.dev86 → verifiers-0.2.2.dev91}/verifiers/v1/utils/compile.py +0 -0
- {verifiers-0.2.2.dev86 → verifiers-0.2.2.dev91}/verifiers/v1/utils/decorators.py +0 -0
- {verifiers-0.2.2.dev86 → verifiers-0.2.2.dev91}/verifiers/v1/utils/format.py +0 -0
- {verifiers-0.2.2.dev86 → verifiers-0.2.2.dev91}/verifiers/v1/utils/generic.py +0 -0
- {verifiers-0.2.2.dev86 → verifiers-0.2.2.dev91}/verifiers/v1/utils/git.py +0 -0
- {verifiers-0.2.2.dev86 → verifiers-0.2.2.dev91}/verifiers/v1/utils/image.py +0 -0
- {verifiers-0.2.2.dev86 → verifiers-0.2.2.dev91}/verifiers/v1/utils/install.py +0 -0
- {verifiers-0.2.2.dev86 → verifiers-0.2.2.dev91}/verifiers/v1/utils/interrupt.py +0 -0
- {verifiers-0.2.2.dev86 → verifiers-0.2.2.dev91}/verifiers/v1/utils/loaders.py +0 -0
- {verifiers-0.2.2.dev86 → verifiers-0.2.2.dev91}/verifiers/v1/utils/logging.py +0 -0
- {verifiers-0.2.2.dev86 → verifiers-0.2.2.dev91}/verifiers/v1/utils/memory.py +0 -0
- {verifiers-0.2.2.dev86 → verifiers-0.2.2.dev91}/verifiers/v1/utils/platform.py +0 -0
- {verifiers-0.2.2.dev86 → verifiers-0.2.2.dev91}/verifiers/v1/utils/retries.py +0 -0
- {verifiers-0.2.2.dev86 → verifiers-0.2.2.dev91}/verifiers/v1/utils/score.py +0 -0
- {verifiers-0.2.2.dev86 → verifiers-0.2.2.dev91}/verifiers/v1/utils/version.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: verifiers
|
|
3
|
-
Version: 0.2.2.
|
|
3
|
+
Version: 0.2.2.dev91
|
|
4
4
|
Summary: Verifiers: Environments for LLM Reinforcement Learning
|
|
5
5
|
Project-URL: Homepage, https://github.com/primeintellect-ai/verifiers
|
|
6
6
|
Project-URL: Documentation, https://github.com/primeintellect-ai/verifiers
|
|
@@ -35,7 +35,7 @@ Requires-Dist: numpy>=2.1.0
|
|
|
35
35
|
Requires-Dist: openai-agents>=0.8.2
|
|
36
36
|
Requires-Dist: openai>=2.9.0
|
|
37
37
|
Requires-Dist: prime-pydantic-config[toml]>=0.4.2
|
|
38
|
-
Requires-Dist: prime-sandboxes>=0.2.
|
|
38
|
+
Requires-Dist: prime-sandboxes>=0.2.35
|
|
39
39
|
Requires-Dist: prime-tunnel>=0.1.8
|
|
40
40
|
Requires-Dist: pydantic>=2.12.3
|
|
41
41
|
Requires-Dist: pyzmq>=27.1.0
|
|
@@ -44,6 +44,8 @@ from verifiers.v1.configs.cli.eval import EvalConfig
|
|
|
44
44
|
from verifiers.v1.trace import Trace
|
|
45
45
|
from verifiers.v1.utils.loaders import load_environment
|
|
46
46
|
|
|
47
|
+
CI_MODEL = "openai/gpt-5.6-luna"
|
|
48
|
+
|
|
47
49
|
# Fixture tasksets/envs (echo-v1, echo-agentic-v1, echo-v0, echo-multi-v0) live in
|
|
48
50
|
# tests/v1/fixtures, added to the path via `pythonpath` in pyproject so the v1 loader and the
|
|
49
51
|
# v0 legacy bridge both resolve them by id (no install).
|
|
@@ -129,13 +131,12 @@ def _eval_config(
|
|
|
129
131
|
runtime: dict | None = None,
|
|
130
132
|
env: dict | None = None,
|
|
131
133
|
pool: dict | None = None,
|
|
132
|
-
model: str | None = "deepseek/deepseek-v4-flash-0731",
|
|
133
134
|
reasoning_effort: str | None = None,
|
|
134
135
|
) -> EvalConfig:
|
|
135
136
|
"""Build the smallest `EvalConfig` that still exercises the path, shared by the in-process
|
|
136
137
|
(`run_v1`) and env-server (`run_v1_server`) fixtures. `taskset_overrides` merges onto the
|
|
137
138
|
`{id: ...}` config; `runtime` places the `agent` seat's harness (an agent field, not a
|
|
138
|
-
harness one)
|
|
139
|
+
harness one).
|
|
139
140
|
|
|
140
141
|
`harness=None` leaves every seat on its own story — the multi-agent case: there
|
|
141
142
|
is no run-level harness, so a single-agent test's `harness` lands on the `agent`
|
|
@@ -178,7 +179,7 @@ def _eval_config(
|
|
|
178
179
|
rich=False,
|
|
179
180
|
output_dir=output_dir,
|
|
180
181
|
**({"serve": {"pool": pool}} if pool else {}),
|
|
181
|
-
|
|
182
|
+
model=CI_MODEL,
|
|
182
183
|
)
|
|
183
184
|
|
|
184
185
|
|
|
@@ -223,7 +224,7 @@ async def live_ctx():
|
|
|
223
224
|
|
|
224
225
|
# Endpoint config only — each rollout builds and closes its own client.
|
|
225
226
|
yield ModelContext(
|
|
226
|
-
model=
|
|
227
|
+
model=CI_MODEL,
|
|
227
228
|
client=EvalClientConfig(),
|
|
228
229
|
sampling=SamplingConfig(max_tokens=2048),
|
|
229
230
|
)
|
|
@@ -244,6 +245,7 @@ def run_v0():
|
|
|
244
245
|
) -> list[Trace]:
|
|
245
246
|
config = EvalConfig(
|
|
246
247
|
legacy={"id": env_id, "args": args or {}},
|
|
248
|
+
model=CI_MODEL,
|
|
247
249
|
num_tasks=1,
|
|
248
250
|
num_rollouts=n,
|
|
249
251
|
sampling={"max_tokens": max_tokens},
|
|
@@ -7,11 +7,11 @@ are local-only (their marks are excluded in CI)."""
|
|
|
7
7
|
|
|
8
8
|
import pytest
|
|
9
9
|
|
|
10
|
-
|
|
10
|
+
mark = pytest.mark
|
|
11
11
|
|
|
12
12
|
|
|
13
|
-
def
|
|
14
|
-
marks = [getattr(
|
|
13
|
+
def pair(a: str, b: str, id: str, *extra_marks):
|
|
14
|
+
marks = [getattr(mark, a.replace("-", "_")), getattr(mark, b.replace("-", "_"))]
|
|
15
15
|
return pytest.param(a, b, marks=[*marks, *extra_marks], id=id)
|
|
16
16
|
|
|
17
17
|
|
|
@@ -20,79 +20,83 @@ def _pair(a: str, b: str, id: str, *extra_marks):
|
|
|
20
20
|
# provider. codex/claude-code are excluded here (unreliable on a no-op echo chat
|
|
21
21
|
# task) — test_agentic covers them.
|
|
22
22
|
CHAT_PLACEMENTS = [
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
|
|
23
|
+
pair("null", "subprocess", "null-harness-in-subprocess"),
|
|
24
|
+
pair("bash", "docker", "bash-harness-in-docker"),
|
|
25
|
+
pair("rlm", "docker", "rlm-harness-in-docker"),
|
|
26
|
+
pair("kimi-code", "docker", "kimi-code-harness-in-docker"),
|
|
27
|
+
pair("bash", "prime", "bash-harness-in-prime"),
|
|
28
|
+
pair("bash", "modal", "bash-harness-in-modal"),
|
|
29
29
|
]
|
|
30
30
|
|
|
31
31
|
# harness x harness runtime for the shell task: every coding agent once (null is a chat
|
|
32
32
|
# loop with no shell), both local runtimes hit (subprocess only carries bash), one
|
|
33
33
|
# remote row per provider.
|
|
34
34
|
AGENTIC_PLACEMENTS = [
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
|
|
35
|
+
pair("bash", "subprocess", "bash-harness-in-subprocess"),
|
|
36
|
+
pair("rlm", "docker", "rlm-harness-in-docker"),
|
|
37
|
+
pair("kimi-code", "docker", "kimi-code-harness-in-docker"),
|
|
38
|
+
pair("codex", "docker", "codex-harness-in-docker"),
|
|
39
|
+
pair("claude-code", "docker", "claude-code-harness-in-docker"),
|
|
40
|
+
pair("hermes-agent", "docker", "hermes-agent-harness-in-docker"),
|
|
41
|
+
pair("bash", "prime", "bash-harness-in-prime"),
|
|
42
|
+
pair("bash", "modal", "bash-harness-in-modal"),
|
|
43
43
|
]
|
|
44
44
|
|
|
45
45
|
# The scripted user runs in the eval process itself (no placement axis); the harness
|
|
46
46
|
# runtime is the exchange's only axis.
|
|
47
47
|
USER_RUNTIMES = [
|
|
48
|
-
pytest.param("subprocess", marks=[
|
|
49
|
-
pytest.param("docker", marks=[
|
|
50
|
-
pytest.param("prime", marks=[
|
|
51
|
-
pytest.param("modal", marks=[
|
|
48
|
+
pytest.param("subprocess", marks=[mark.subprocess], id="harness-in-subprocess"),
|
|
49
|
+
pytest.param("docker", marks=[mark.docker], id="harness-in-docker"),
|
|
50
|
+
pytest.param("prime", marks=[mark.prime], id="harness-in-prime"),
|
|
51
|
+
pytest.param("modal", marks=[mark.modal], id="harness-in-modal"),
|
|
52
52
|
]
|
|
53
53
|
|
|
54
|
-
# ACP-backed harnesses: each must preserve an exchange across
|
|
54
|
+
# ACP-backed harnesses: each must preserve an exchange across interaction segments and
|
|
55
55
|
# retain MCP access after resuming. Cover every harness in the local container runtime,
|
|
56
|
-
# plus
|
|
56
|
+
# plus remote placements for the sandbox/tunnel and native-process boundaries.
|
|
57
57
|
ACP_RESUME_PLACEMENTS = [
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
|
|
63
|
-
|
|
58
|
+
pair("codex", "docker", "codex-acp-in-docker"),
|
|
59
|
+
pair("claude-code", "docker", "claude-code-acp-in-docker"),
|
|
60
|
+
pair("hermes-agent", "docker", "hermes-agent-acp-in-docker"),
|
|
61
|
+
pair("rlm", "docker", "rlm-acp-in-docker"),
|
|
62
|
+
pair("kimi-code", "docker", "kimi-code-acp-in-docker"),
|
|
63
|
+
pair("pi", "docker", "pi-acp-in-docker"),
|
|
64
|
+
pair("pool", "docker", "pool-acp-in-docker"),
|
|
65
|
+
pair("openclaw", "docker", "openclaw-acp-in-docker"),
|
|
66
|
+
pair("pool", "prime", "pool-acp-in-prime"),
|
|
67
|
+
pair("rlm", "prime", "rlm-acp-in-prime-vm"),
|
|
64
68
|
]
|
|
65
69
|
|
|
66
70
|
# harness runtime x tool placement: every axis value once plus the two-container case
|
|
67
71
|
# (harness and tool in separate docker boxes) and a prime-colocated row (a tool in its
|
|
68
72
|
# OWN prime sandbox needs port exposure; colocated rides the harness's box).
|
|
69
73
|
TOOL_PLACEMENTS = [
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
|
|
76
|
-
|
|
77
|
-
|
|
74
|
+
pair("subprocess", "colocated", "harness-in-subprocess-with-tool-colocated"),
|
|
75
|
+
pair("docker", "colocated", "harness-in-docker-with-tool-colocated"),
|
|
76
|
+
pair("subprocess", "docker", "harness-in-subprocess-with-tool-in-docker"),
|
|
77
|
+
pair("docker", "subprocess", "harness-in-docker-with-tool-in-subprocess"),
|
|
78
|
+
pair("docker", "docker", "harness-in-docker-with-tool-in-docker"),
|
|
79
|
+
pair("prime", "colocated", "harness-in-prime-with-tool-colocated"),
|
|
80
|
+
pair("modal", "colocated", "harness-in-modal-with-tool-colocated"),
|
|
81
|
+
pair("subprocess", "modal", "harness-in-subprocess-with-tool-in-modal"),
|
|
78
82
|
]
|
|
79
83
|
|
|
80
84
|
# The state channel rides the same reachability as TOOL_PLACEMENTS; cover each axis
|
|
81
85
|
# value once rather than re-running the whole list.
|
|
82
86
|
TOOL_STATE_PLACEMENTS = [
|
|
83
|
-
|
|
84
|
-
|
|
85
|
-
|
|
86
|
-
|
|
87
|
+
pair("subprocess", "colocated", "harness-in-subprocess-with-tool-colocated"),
|
|
88
|
+
pair("docker", "subprocess", "harness-in-docker-with-tool-in-subprocess"),
|
|
89
|
+
pair("subprocess", "docker", "harness-in-subprocess-with-tool-in-docker"),
|
|
90
|
+
pair("modal", "colocated", "harness-in-modal-with-tool-colocated"),
|
|
87
91
|
]
|
|
88
92
|
|
|
89
93
|
# Shared servers always run in their own runtime (colocation is per-rollout, shared is
|
|
90
94
|
# eval-level): same-runtime pairs plus one cross-boundary row.
|
|
91
95
|
SHARED_TOOL_PLACEMENTS = [
|
|
92
|
-
|
|
93
|
-
|
|
94
|
-
|
|
95
|
-
|
|
96
|
+
pair("subprocess", "subprocess", "harness-in-subprocess-with-tool-in-subprocess"),
|
|
97
|
+
pair("docker", "docker", "harness-in-docker-with-tool-in-docker"),
|
|
98
|
+
pair("subprocess", "docker", "harness-in-subprocess-with-tool-in-docker"),
|
|
99
|
+
pair("modal", "modal", "harness-in-modal-with-tool-in-modal"),
|
|
96
100
|
]
|
|
97
101
|
|
|
98
102
|
|
|
@@ -215,7 +219,10 @@ async def test_acp_resume_with_tool(run_v1, harness, harness_runtime, tmp_path):
|
|
|
215
219
|
(trace,) = await run_v1(
|
|
216
220
|
"echo-acp-resume-v1",
|
|
217
221
|
harness=harness,
|
|
218
|
-
runtime={
|
|
222
|
+
runtime={
|
|
223
|
+
"type": harness_runtime,
|
|
224
|
+
**({"vm": True} if harness_runtime == "prime" else {}),
|
|
225
|
+
},
|
|
219
226
|
output_dir=tmp_path,
|
|
220
227
|
max_turns=8,
|
|
221
228
|
max_tokens=8192,
|
|
@@ -231,6 +238,8 @@ async def test_acp_resume_with_tool(run_v1, harness, harness_runtime, tmp_path):
|
|
|
231
238
|
assert segments[1]["terminated"] is False
|
|
232
239
|
assert "tool" in segments[1]["roles"]
|
|
233
240
|
assert segments[1]["tool_outputs"]
|
|
241
|
+
if harness == "rlm":
|
|
242
|
+
assert "turns_since_last_compaction" in trace.metrics
|
|
234
243
|
|
|
235
244
|
|
|
236
245
|
@pytest.mark.e2e
|
|
@@ -314,7 +323,6 @@ async def test_tool_response_image(run_v1, tmp_path):
|
|
|
314
323
|
"tool-response-image-v1",
|
|
315
324
|
harness="null",
|
|
316
325
|
runtime={"type": "subprocess"},
|
|
317
|
-
model="openai/gpt-5.6-luna",
|
|
318
326
|
reasoning_effort="none",
|
|
319
327
|
output_dir=tmp_path,
|
|
320
328
|
max_turns=4,
|
|
@@ -47,7 +47,7 @@ def test_eval(taskset: str):
|
|
|
47
47
|
pytest.skip(f"{taskset} can't run a plain-CI smoke eval")
|
|
48
48
|
if os.getenv("PRIME_API_KEY"):
|
|
49
49
|
model = [
|
|
50
|
-
"-m", "openai/gpt-
|
|
50
|
+
"-m", "openai/gpt-5.6-luna",
|
|
51
51
|
"--client.base-url", "https://api.pinference.ai/api/v1",
|
|
52
52
|
"--client.api-key-var", "PRIME_API_KEY",
|
|
53
53
|
] # fmt: skip
|
|
@@ -43,7 +43,7 @@ from verifiers.v1.errors import (
|
|
|
43
43
|
TunnelError,
|
|
44
44
|
)
|
|
45
45
|
from verifiers.v1.graph import MessageNode
|
|
46
|
-
from verifiers.v1.harness import Harness
|
|
46
|
+
from verifiers.v1.harness import Harness, HarnessSession
|
|
47
47
|
from verifiers.v1.judge import Judge, JudgeResponse, JudgeView
|
|
48
48
|
from verifiers.v1.judges import (
|
|
49
49
|
Criterion,
|
|
@@ -64,6 +64,7 @@ from verifiers.v1.runtimes import (
|
|
|
64
64
|
Runtime,
|
|
65
65
|
RuntimeConfig,
|
|
66
66
|
RuntimeInfo,
|
|
67
|
+
RuntimeProcess,
|
|
67
68
|
SubprocessConfig,
|
|
68
69
|
)
|
|
69
70
|
from verifiers.v1.state import State, StateT
|
|
@@ -241,10 +242,12 @@ __all__ = [ # noqa: RUF022 - grouped by public API area
|
|
|
241
242
|
"TasksetConfig",
|
|
242
243
|
"BaseConfig",
|
|
243
244
|
"Harness",
|
|
245
|
+
"HarnessSession",
|
|
244
246
|
"HarnessConfig",
|
|
245
247
|
"ACP",
|
|
246
248
|
"ModelContext",
|
|
247
249
|
"Runtime",
|
|
250
|
+
"RuntimeProcess",
|
|
248
251
|
"RuntimeConfig",
|
|
249
252
|
"RuntimeInfo",
|
|
250
253
|
"ProgramResult",
|
|
@@ -0,0 +1,312 @@
|
|
|
1
|
+
"""Public Agent Client Protocol support for harness programs."""
|
|
2
|
+
|
|
3
|
+
import asyncio
|
|
4
|
+
import contextlib
|
|
5
|
+
import json
|
|
6
|
+
import secrets
|
|
7
|
+
from collections.abc import AsyncIterator
|
|
8
|
+
from pathlib import Path
|
|
9
|
+
|
|
10
|
+
from verifiers.v1.clients import ModelContext
|
|
11
|
+
from verifiers.v1.dialects.chat import message_to_wire
|
|
12
|
+
from verifiers.v1.errors import HarnessError
|
|
13
|
+
from verifiers.v1.harness import Harness, HarnessSession
|
|
14
|
+
from verifiers.v1.runtimes import ProgramResult, Runtime, RuntimeProcess
|
|
15
|
+
from verifiers.v1.task import TaskData
|
|
16
|
+
from verifiers.v1.trace import Trace
|
|
17
|
+
from verifiers.v1.types import Messages
|
|
18
|
+
from verifiers.v1.utils.aio import run_shielded
|
|
19
|
+
|
|
20
|
+
ACP_SOURCE = (Path(__file__).resolve().parent / "runner.py").read_text()
|
|
21
|
+
MAX_PACKET_BYTES = 128 * 1024 * 1024
|
|
22
|
+
|
|
23
|
+
__all__ = ["ACP"]
|
|
24
|
+
|
|
25
|
+
|
|
26
|
+
class ACP:
|
|
27
|
+
"""Run one-shot ACP agents or create rollout-scoped ACP sessions."""
|
|
28
|
+
|
|
29
|
+
async def setup(self, harness: Harness, runtime: Runtime) -> None:
|
|
30
|
+
await runtime.prepare_uv_script(
|
|
31
|
+
ACP_SOURCE, {**harness.config.resolved_env, "UV_FROZEN": "false"}
|
|
32
|
+
)
|
|
33
|
+
|
|
34
|
+
def session(
|
|
35
|
+
self,
|
|
36
|
+
harness: Harness,
|
|
37
|
+
ctx: ModelContext,
|
|
38
|
+
trace: Trace,
|
|
39
|
+
runtime: Runtime,
|
|
40
|
+
endpoint: str,
|
|
41
|
+
secret: str,
|
|
42
|
+
mcp_urls: dict[str, str],
|
|
43
|
+
data: TaskData,
|
|
44
|
+
*,
|
|
45
|
+
env: dict[str, str],
|
|
46
|
+
command: list[str],
|
|
47
|
+
prompt: str | Messages | None,
|
|
48
|
+
system_prompt: str | None = None,
|
|
49
|
+
session_meta: dict | None = None,
|
|
50
|
+
) -> "ACPHarnessSession":
|
|
51
|
+
"""Create a persistent ACP-backed handle owned by one rollout."""
|
|
52
|
+
return ACPHarnessSession(
|
|
53
|
+
harness,
|
|
54
|
+
ctx,
|
|
55
|
+
trace,
|
|
56
|
+
runtime,
|
|
57
|
+
endpoint,
|
|
58
|
+
secret,
|
|
59
|
+
mcp_urls,
|
|
60
|
+
data,
|
|
61
|
+
env=env,
|
|
62
|
+
command=command,
|
|
63
|
+
prompt=prompt,
|
|
64
|
+
system_prompt=system_prompt,
|
|
65
|
+
session_meta=session_meta,
|
|
66
|
+
)
|
|
67
|
+
|
|
68
|
+
async def run(
|
|
69
|
+
self,
|
|
70
|
+
runtime: Runtime,
|
|
71
|
+
env: dict[str, str],
|
|
72
|
+
command: list[str],
|
|
73
|
+
prompt: str | Messages | None,
|
|
74
|
+
*,
|
|
75
|
+
mcp_urls: dict[str, str] | None = None,
|
|
76
|
+
system_prompt: str | None = None,
|
|
77
|
+
session_path: str | None = None,
|
|
78
|
+
session_meta: dict | None = None,
|
|
79
|
+
allow_empty_tool_reply: bool = False,
|
|
80
|
+
) -> ProgramResult:
|
|
81
|
+
"""Run one ACP segment without retaining its process."""
|
|
82
|
+
return await self._run(
|
|
83
|
+
runtime,
|
|
84
|
+
env,
|
|
85
|
+
command,
|
|
86
|
+
prompt,
|
|
87
|
+
mcp_urls=mcp_urls,
|
|
88
|
+
system_prompt=system_prompt,
|
|
89
|
+
session_path=session_path,
|
|
90
|
+
session_meta=session_meta,
|
|
91
|
+
allow_empty_tool_reply=allow_empty_tool_reply,
|
|
92
|
+
)
|
|
93
|
+
|
|
94
|
+
async def _run(
|
|
95
|
+
self,
|
|
96
|
+
runtime: Runtime,
|
|
97
|
+
env: dict[str, str],
|
|
98
|
+
command: list[str],
|
|
99
|
+
prompt: str | Messages | None,
|
|
100
|
+
*,
|
|
101
|
+
mcp_urls: dict[str, str] | None = None,
|
|
102
|
+
system_prompt: str | None = None,
|
|
103
|
+
session_path: str | None = None,
|
|
104
|
+
session_meta: dict | None = None,
|
|
105
|
+
allow_empty_tool_reply: bool = False,
|
|
106
|
+
) -> ProgramResult:
|
|
107
|
+
if prompt is None:
|
|
108
|
+
raise ValueError("ACP requires a prompt")
|
|
109
|
+
messages = (
|
|
110
|
+
[{"role": "user", "content": prompt}]
|
|
111
|
+
if isinstance(prompt, str)
|
|
112
|
+
else [message_to_wire(message) for message in prompt]
|
|
113
|
+
)
|
|
114
|
+
config = {
|
|
115
|
+
"command": command,
|
|
116
|
+
"messages": messages,
|
|
117
|
+
"mcp_urls": mcp_urls or {},
|
|
118
|
+
"system_prompt": system_prompt or "",
|
|
119
|
+
"session_path": session_path,
|
|
120
|
+
"session_meta": session_meta or {},
|
|
121
|
+
"allow_empty_tool_reply": allow_empty_tool_reply,
|
|
122
|
+
}
|
|
123
|
+
program = await runtime.prepare_uv_script(
|
|
124
|
+
ACP_SOURCE, {**env, "UV_FROZEN": "false"}
|
|
125
|
+
)
|
|
126
|
+
directory = f".vf-acp-{secrets.token_hex(8)}"
|
|
127
|
+
created = await runtime.run(["mkdir", "-m", "700", directory], {})
|
|
128
|
+
if created.exit_code != 0:
|
|
129
|
+
raise RuntimeError(f"ACP config directory failed: {created.stderr.strip()}")
|
|
130
|
+
path = f"{directory}/config.json"
|
|
131
|
+
try:
|
|
132
|
+
await runtime.write(path, json.dumps(config).encode())
|
|
133
|
+
return await runtime.run_program([*program, "once", path], env)
|
|
134
|
+
finally:
|
|
135
|
+
await run_shielded(runtime.run(["rm", "-rf", directory], {}))
|
|
136
|
+
|
|
137
|
+
|
|
138
|
+
def _packet(value: dict) -> bytes:
|
|
139
|
+
data = json.dumps(value, ensure_ascii=False).encode()
|
|
140
|
+
if len(data) > MAX_PACKET_BYTES:
|
|
141
|
+
raise ValueError(f"ACP session packet is too large: {len(data)} bytes")
|
|
142
|
+
return len(data).to_bytes(8, "big") + data
|
|
143
|
+
|
|
144
|
+
|
|
145
|
+
class _PacketReader:
|
|
146
|
+
def __init__(self, source: AsyncIterator[bytes]) -> None:
|
|
147
|
+
self._source = source.__aiter__()
|
|
148
|
+
self._buffer = bytearray()
|
|
149
|
+
|
|
150
|
+
async def _readexactly(self, size: int) -> bytes:
|
|
151
|
+
while len(self._buffer) < size:
|
|
152
|
+
try:
|
|
153
|
+
self._buffer.extend(await anext(self._source))
|
|
154
|
+
except StopAsyncIteration as e:
|
|
155
|
+
raise EOFError("ACP process closed its stdout") from e
|
|
156
|
+
data = bytes(self._buffer[:size])
|
|
157
|
+
del self._buffer[:size]
|
|
158
|
+
return data
|
|
159
|
+
|
|
160
|
+
async def read(self) -> dict:
|
|
161
|
+
size = int.from_bytes(await self._readexactly(8), "big")
|
|
162
|
+
if size > MAX_PACKET_BYTES:
|
|
163
|
+
raise ValueError(f"ACP session packet is too large: {size} bytes")
|
|
164
|
+
return json.loads((await self._readexactly(size)).decode())
|
|
165
|
+
|
|
166
|
+
|
|
167
|
+
class ACPHarnessSession(HarnessSession):
|
|
168
|
+
"""A live ACP process, connection, and native session for one rollout."""
|
|
169
|
+
|
|
170
|
+
def __init__(
|
|
171
|
+
self,
|
|
172
|
+
harness: Harness,
|
|
173
|
+
ctx: ModelContext,
|
|
174
|
+
trace: Trace,
|
|
175
|
+
runtime: Runtime,
|
|
176
|
+
endpoint: str,
|
|
177
|
+
secret: str,
|
|
178
|
+
mcp_urls: dict[str, str],
|
|
179
|
+
data: TaskData,
|
|
180
|
+
env: dict[str, str],
|
|
181
|
+
command: list[str],
|
|
182
|
+
prompt: str | Messages | None,
|
|
183
|
+
system_prompt: str | None,
|
|
184
|
+
session_meta: dict | None,
|
|
185
|
+
) -> None:
|
|
186
|
+
super().__init__(harness, ctx, trace, runtime, endpoint, secret, mcp_urls, data)
|
|
187
|
+
self.env = env
|
|
188
|
+
self.command = command
|
|
189
|
+
self.prompt = prompt
|
|
190
|
+
self.system_prompt = system_prompt
|
|
191
|
+
self.session_meta = session_meta or {}
|
|
192
|
+
self._process: RuntimeProcess | None = None
|
|
193
|
+
self._reader: _PacketReader | None = None
|
|
194
|
+
self._stderr_tail = bytearray()
|
|
195
|
+
self._stderr_task: asyncio.Task[None] | None = None
|
|
196
|
+
self._lock = asyncio.Lock()
|
|
197
|
+
|
|
198
|
+
async def _start(self) -> None:
|
|
199
|
+
self._stderr_tail.clear()
|
|
200
|
+
program = await self.runtime.prepare_uv_script(
|
|
201
|
+
ACP_SOURCE, {**self.env, "UV_FROZEN": "false"}
|
|
202
|
+
)
|
|
203
|
+
process = await self.runtime.open_process([*program, "stream"], self.env)
|
|
204
|
+
self._process = process
|
|
205
|
+
self._reader = _PacketReader(process.stdout)
|
|
206
|
+
self._stderr_task = asyncio.create_task(self._drain_stderr(process.stderr))
|
|
207
|
+
|
|
208
|
+
async def _drain_stderr(self, stream: AsyncIterator[bytes]) -> None:
|
|
209
|
+
async for chunk in stream:
|
|
210
|
+
self._stderr_tail.extend(chunk)
|
|
211
|
+
if len(self._stderr_tail) > 4000:
|
|
212
|
+
del self._stderr_tail[:-4000]
|
|
213
|
+
|
|
214
|
+
def _stderr(self) -> str:
|
|
215
|
+
return self._stderr_tail.decode(errors="replace").strip()
|
|
216
|
+
|
|
217
|
+
async def _run(self, messages: Messages | None) -> ProgramResult:
|
|
218
|
+
prompt = self.prompt if messages is None else messages
|
|
219
|
+
if prompt is None:
|
|
220
|
+
raise ValueError("ACP requires a prompt")
|
|
221
|
+
wire_messages = (
|
|
222
|
+
[{"role": "user", "content": prompt}]
|
|
223
|
+
if isinstance(prompt, str)
|
|
224
|
+
else [message_to_wire(message) for message in prompt]
|
|
225
|
+
)
|
|
226
|
+
config = {
|
|
227
|
+
"command": self.command,
|
|
228
|
+
"messages": wire_messages,
|
|
229
|
+
"mcp_urls": self.mcp_urls,
|
|
230
|
+
"system_prompt": self.system_prompt or "",
|
|
231
|
+
"session_path": None,
|
|
232
|
+
"session_meta": self.session_meta,
|
|
233
|
+
}
|
|
234
|
+
async with self._lock:
|
|
235
|
+
if self._closed:
|
|
236
|
+
raise HarnessError(
|
|
237
|
+
f"harness {self.harness.config.id!r} session is already closed"
|
|
238
|
+
)
|
|
239
|
+
if self._process is None:
|
|
240
|
+
await self._start()
|
|
241
|
+
assert self._process is not None
|
|
242
|
+
assert self._reader is not None
|
|
243
|
+
try:
|
|
244
|
+
await self._process.write(
|
|
245
|
+
_packet({"operation": "prompt", "config": config})
|
|
246
|
+
)
|
|
247
|
+
response = await self._reader.read()
|
|
248
|
+
except BaseException:
|
|
249
|
+
await run_shielded(self._stop(graceful=False))
|
|
250
|
+
raise
|
|
251
|
+
if not response.get("ok"):
|
|
252
|
+
detail = response.get("error") or "ACP session request failed"
|
|
253
|
+
if stderr := self._stderr():
|
|
254
|
+
detail = f"{detail}\n\nACP process stderr:\n{stderr}"
|
|
255
|
+
raise RuntimeError(detail)
|
|
256
|
+
return ProgramResult(exit_code=0, stdout=response.get("reply", ""), stderr="")
|
|
257
|
+
|
|
258
|
+
async def _stop(self, *, graceful: bool) -> None:
|
|
259
|
+
process, self._process = self._process, None
|
|
260
|
+
reader, self._reader = self._reader, None
|
|
261
|
+
stderr_task, self._stderr_task = self._stderr_task, None
|
|
262
|
+
if process is None:
|
|
263
|
+
return
|
|
264
|
+
failure: BaseException | None = None
|
|
265
|
+
try:
|
|
266
|
+
if graceful and reader is not None:
|
|
267
|
+
try:
|
|
268
|
+
await process.write(_packet({"operation": "shutdown"}))
|
|
269
|
+
response = await asyncio.wait_for(reader.read(), timeout=10)
|
|
270
|
+
if not response.get("ok"):
|
|
271
|
+
raise RuntimeError(
|
|
272
|
+
response.get("error") or "ACP session shutdown failed"
|
|
273
|
+
)
|
|
274
|
+
except BaseException as error: # noqa: BLE001 - finish teardown if cancelled
|
|
275
|
+
failure = error
|
|
276
|
+
try:
|
|
277
|
+
await asyncio.wait_for(process.wait(), timeout=10 if graceful else 0.1)
|
|
278
|
+
except BaseException: # noqa: BLE001 - cancellation still requires termination
|
|
279
|
+
with contextlib.suppress(Exception):
|
|
280
|
+
await asyncio.wait_for(process.terminate(), timeout=5)
|
|
281
|
+
try:
|
|
282
|
+
await asyncio.wait_for(process.wait(), timeout=5)
|
|
283
|
+
except BaseException: # noqa: BLE001 - cancellation still requires a kill
|
|
284
|
+
with contextlib.suppress(Exception):
|
|
285
|
+
await asyncio.wait_for(process.kill(), timeout=5)
|
|
286
|
+
with contextlib.suppress(BaseException):
|
|
287
|
+
await asyncio.wait_for(process.wait(), timeout=5)
|
|
288
|
+
finally:
|
|
289
|
+
if stderr_task is not None:
|
|
290
|
+
if not stderr_task.done():
|
|
291
|
+
stderr_task.cancel()
|
|
292
|
+
with contextlib.suppress(BaseException):
|
|
293
|
+
await stderr_task
|
|
294
|
+
if failure is not None:
|
|
295
|
+
detail = str(failure)
|
|
296
|
+
if stderr := self._stderr():
|
|
297
|
+
detail = f"{detail}\n\nACP process stderr:\n{stderr}"
|
|
298
|
+
raise RuntimeError(detail) from failure
|
|
299
|
+
|
|
300
|
+
async def close(self) -> None:
|
|
301
|
+
if self._closed:
|
|
302
|
+
return
|
|
303
|
+
# Publish closure before waiting for the process lock. A turn that
|
|
304
|
+
# already passed HarnessSession.turn()'s fast check rechecks under the
|
|
305
|
+
# same lock in _run(), so it cannot restart after teardown.
|
|
306
|
+
await super().close()
|
|
307
|
+
|
|
308
|
+
async def close_process() -> None:
|
|
309
|
+
async with self._lock:
|
|
310
|
+
await self._stop(graceful=True)
|
|
311
|
+
|
|
312
|
+
await run_shielded(close_process())
|