verifiers 0.1.15.dev380__tar.gz → 0.1.15.dev382__tar.gz

{verifiers-0.1.15.dev380 → verifiers-0.1.15.dev382}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: verifiers
-Version: 0.1.15.dev380
+Version: 0.1.15.dev382
 Summary: Verifiers: Environments for LLM Reinforcement Learning
 Project-URL: Homepage, https://github.com/primeintellect-ai/verifiers
 Project-URL: Documentation, https://github.com/primeintellect-ai/verifiers

{verifiers-0.1.15.dev380 → verifiers-0.1.15.dev382}/pyproject.toml

@@ -78,11 +78,9 @@ dev = [
 # from this repo. The built-in tasksets/harnesses ship inside verifiers (verifiers/v1/...).
 examples = [
     "compact",
-    "gsm8k-v1", "wikispeedia-v1", "glossary-v1", "deepwiki-v1", "wiki-search-v1", "code-golf-v1",
-    "reverse-text-v1", "math-env-v1", "aime24-v1", "color-codeword-v1",
-    "wordle-v1", "terminal-bench-2-v1", "alphabet-sort-v1",
-    "r2e-gym-v1", "scaleswe-v1", "swelego-v1", "scratchpad-v1",
-    "general-agent-v1", "swebench-verified-v1", "tau2-bench-v1",
+    "gsm8k-v1", "glossary-v1", "deepwiki-v1", "wiki-search-v1", "code-golf-v1",
+    "reverse-text-v1", "color-codeword-v1",
+    "wordle-v1", "alphabet-sort-v1", "scratchpad-v1",
 ]
 
 [project.optional-dependencies]
@@ -137,25 +135,15 @@ default = true
 compact = { path = "environments/compact", editable = true }
 glossary-v1 = { path = "environments/glossary_v1", editable = true }
 deepwiki-v1 = { path = "environments/deepwiki_v1", editable = true }
-wikispeedia-v1 = { path = "environments/wikispeedia_v1", editable = true }
 wiki-search-v1 = { path = "environments/wiki_search_v1", editable = true }
 gsm8k-v1 = { path = "environments/gsm8k_v1", editable = true }
 code-golf-v1 = { path = "environments/code_golf_v1", editable = true }
 reverse-text-v1 = { path = "environments/reverse_text_v1", editable = true }
 reverse-text = { path = "environments/reverse_text", editable = true }
 wordle-v1 = { path = "environments/wordle_v1", editable = true }
-terminal-bench-2-v1 = { path = "environments/terminal_bench_2_v1", editable = true }
-math-env-v1 = { path = "environments/math_env_v1", editable = true }
-aime24-v1 = { path = "environments/aime24_v1", editable = true }
 color-codeword-v1 = { path = "environments/color_codeword_v1", editable = true }
-r2e-gym-v1 = { path = "environments/r2e_gym_v1", editable = true }
-scaleswe-v1 = { path = "environments/scaleswe_v1", editable = true }
-swelego-v1 = { path = "environments/swelego_v1", editable = true }
 alphabet-sort-v1 = { path = "environments/alphabet_sort_v1", editable = true }
 scratchpad-v1 = { path = "environments/scratchpad_v1", editable = true }
-general-agent-v1 = { path = "environments/general_agent_v1", editable = true }
-swebench-verified-v1 = { path = "environments/swebench_verified_v1", editable = true }
-tau2-bench-v1 = { path = "environments/tau2_bench_v1", editable = true }
 
 [tool.uv.exclude-newer-package]
 # PrimeIntellect-published on PyPI (trusted publisher)

{verifiers-0.1.15.dev380 → verifiers-0.1.15.dev382}/tests/test_envs.py

@@ -171,10 +171,6 @@ def help_test_can_load_env(tmp_venv_dir: Path, env_dir: Path):
 
 def help_test_can_eval_env(tmp_venv_dir: Path, env_dir: Path):
     """Test that the environment can be run via vf-eval."""
-    if env_dir.name == "tau2_bench_v1" and not os.getenv("PRIME_API_KEY"):
-        pytest.skip(
-            "Skipping tau2 default eval because PRIME_API_KEY is not configured"
-        )
     if os.getenv("PRIME_API_KEY"):
         model_flags = "-m openai/gpt-4.1-mini -b https://api.pinference.ai/api/v1 -k PRIME_API_KEY"
     elif os.getenv("OPENAI_API_KEY"):

{verifiers-0.1.15.dev380 → verifiers-0.1.15.dev382}/tests/v1/test_envs.py

@@ -19,21 +19,10 @@ EVAL_TIMEOUT = 600  # 10 minutes for a capped eval (-n 1 -r 2)
 
 ENVIRONMENTS = Path(__file__).parent.parent.parent / "environments"
 
-# v1 tasksets that need a docker/prime runtime + image-backed sandboxes, so they can't run a
-# smoke eval in plain CI — covered by the dedicated v1 e2e tests.
-NEEDS_CONTAINER = {
-    "r2e_gym_v1",
-    "scaleswe_v1",
-    "swelego_v1",
-    "swebench_verified_v1",
-    "terminal_bench_2_v1",
-}
-
-# v1 tasksets that can't run a plain-CI smoke eval for non-container reasons — e.g. the eval
-# clones a corpus repo that CI has no credentials to read.
-SKIP_SMOKE_EVAL = {
-    "general_agent_v1",  # clones a corpus repo unreadable from CI
-}
+# v1 tasksets that can't run a plain-CI smoke eval — e.g. they need a docker/prime runtime or
+# clone a corpus CI can't read. Empty: the SWE/container and corpus tasksets live in
+# research-environments now.
+SKIP_EVAL: set[str] = set()
 
 
 def v1_tasksets() -> list[str]:
@@ -47,12 +36,8 @@ def v1_tasksets() -> list[str]:
 @pytest.mark.parametrize("taskset", v1_tasksets())
 def test_eval(taskset: str):
     """Run one capped rollout of `taskset`; a taskset that bundles a harness uses it by default."""
-    if taskset in NEEDS_CONTAINER:
-        pytest.skip(f"{taskset} needs a docker/prime runtime (covered by v1 e2e tests)")
-    if taskset in SKIP_SMOKE_EVAL:
-        pytest.skip(
-            f"{taskset} can't run a CI smoke eval (clones a corpus repo unreadable here)"
-        )
+    if taskset in SKIP_EVAL:
+        pytest.skip(f"{taskset} can't run a plain-CI smoke eval")
     if os.getenv("PRIME_API_KEY"):
         model = [
             "-m", "openai/gpt-4.1-mini",

{verifiers-0.1.15.dev380 → verifiers-0.1.15.dev382}/verifiers/v1/GUIDE.md

@@ -478,20 +478,15 @@ The `*_v1` tasksets under `environments/` are the reference library — each sho
 | example | pattern |
 | --- | --- |
 | `reverse-text-v1` | the minimal single-turn taskset |
-| `gsm8k-v1`, `aime24-v1`, `math-env-v1` | single-turn + in-runtime scoring (a `@reward` uv script) |
+| `gsm8k-v1` | single-turn + in-runtime scoring (a `@reward` uv script) |
 | `code-golf-v1` | group rewards (`@group_reward` over a task's N rollouts) |
 | `alphabet-sort-v1` | multi-turn, stateful, driven by a `vf.User` simulator |
 | `glossary-v1` | the simplest tool server (own host runtime) |
-| `wikispeedia-v1` | a stateful tool server (global `setup` + per-task `setup_task`) |
-| `general-agent-v1` | per-task tools loaded dynamically (each task ships its own `tools.py`) + gold tool-call-chain scoring |
 | `wiki-search-v1` | a shared, read-only tool server (built once) + an LLM judge |
 | `scratchpad-v1` | a shared, **writable** tool server — per-rollout state isolated via `self.state` |
 | `deepwiki-v1` | an existing remote tool server, by URL |
-| `tau2-bench-v1` | a taskset that **bundles its own harness** (runs the whole tau2 benchmark sim in a subprocess, result into `trace.info`) |
 | `color-codeword-v1` | a multimodal (image) task |
-| `scaleswe-v1`, `swelego-v1`, `r2e-gym-v1` | containerized SWE tasks (rlm harness, prime runtime) |
-| `swebench-verified-v1` | SWE-bench Verified via `harbor-v1` on prime's prebuilt images (no Dockerfile build) |
-| `wordle-v1`, `terminal-bench-2-v1` | thin configs over the shipped `textarena-v1` / `harbor-v1` integrations |
+| `wordle-v1` | a thin config over the shipped `textarena-v1` integration |
 
 ---
 
@@ -742,7 +737,7 @@ form. In a prime-rl config:
 ```toml
 [[orchestrator.train.env]]
 name    = "gsm8k"
-taskset = { id = "math-env-v1", dataset_name = "..." }              # any v1 taskset id
+taskset = { id = "gsm8k-v1", dataset_name = "..." }                 # any v1 taskset id
 harness = { id = "default", runtime = { type = "subprocess" } }
 timeout = { scoring = 10 }                                          # per-stage cap (default: no limit)
 # pool  = { type = "elastic", max_workers = 8, multiplex = 128 }    # env-server pool (default elastic, self-sizing)

{verifiers-0.1.15.dev380 → verifiers-0.1.15.dev382}/verifiers/v1/README.md

@@ -69,15 +69,13 @@ Taskset examples (the `*_v1` packages under `environments/`):
 | example | pattern it shows |
 | --- | --- |
 | `reverse-text-v1` | the minimal single-turn taskset |
-| `gsm8k-v1`, `aime24-v1`, `math-env-v1` | single-turn + in-runtime scoring |
+| `gsm8k-v1` | single-turn + in-runtime scoring |
 | `code-golf-v1` | group rewards (`@group_reward` over a task's N rollouts) |
 | `alphabet-sort-v1` | a multi-turn, stateful task driven by a `vf.User` simulator |
 | `glossary-v1` | a custom **colocated** tool server |
-| `wikispeedia-v1` | a tool server in its **own per-rollout** runtime |
 | `wiki-search-v1` | a **shared** tool server (built once for the eval) + an LLM judge |
 | `deepwiki-v1` | an **existing remote** tool server, by URL |
 | `wordle-v1` | configuring the vendored `textarena-v1` integration |
-| `terminal-bench-2-v1` | configuring the vendored `harbor-v1` integration |
 
 Harness examples (under `environments/`):
 
@@ -135,7 +133,6 @@ each show one placement:
 
 ```bash
 uv run eval glossary-v1 -n 1     # colocated — in the harness's own runtime, localhost (default)
-uv run eval wikispeedia-v1 -n 1 --taskset.min-dist 2 --taskset.max-dist 2  # its own per-rollout runtime
 uv run eval wiki-search-v1 -n 1  # shared — one instance built once for the whole eval
 uv run eval deepwiki-v1 -n 1     # an existing remote server, by URL
 ```

{verifiers-0.1.15.dev380 → verifiers-0.1.15.dev382}/verifiers/v1/mcp/multiplex.py

@@ -9,8 +9,8 @@ state copy-on-write (in-memory state isolated on write), runs in a private worki
 (relative-path writes isolated), and runs `setup_task` for its rollout's task (fetched from the
 interception server's `/task` channel — a shared server gets no task via env). The parent is a thin
 async reverse proxy that pins each rollout to its child and streams MCP traffic (so SSE works). So an
-ordinary stateful per-rollout server (expensive `setup` + per-rollout `setup_task`, e.g.
-`wikispeedia-v1`) is isolated per rollout with no rollout-aware code.
+ordinary stateful per-rollout server (expensive `setup` + per-rollout `setup_task`) is isolated
+per rollout with no rollout-aware code.
 
 The rollout key is the per-rollout secret the framework tags onto a shared server's URL
 (`STATE_SECRET_PARAM`, see `serve_tools`), alongside the reachable interception base