verifiers 0.1.15.dev382__tar.gz → 0.1.15.dev384__tar.gz

{verifiers-0.1.15.dev382 → verifiers-0.1.15.dev384}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: verifiers
-Version: 0.1.15.dev382
+Version: 0.1.15.dev384
 Summary: Verifiers: Environments for LLM Reinforcement Learning
 Project-URL: Homepage, https://github.com/primeintellect-ai/verifiers
 Project-URL: Documentation, https://github.com/primeintellect-ai/verifiers

{verifiers-0.1.15.dev382 → verifiers-0.1.15.dev384}/tests/v1/conftest.py

@@ -199,6 +199,7 @@ def _eval_config(
         max_output_tokens=max_tokens,
         sampling={"max_tokens": max_tokens, "temperature": 0},
         timeout={"rollout": rollout_timeout, "scoring": 60},
+        retries={"rollout": {"max_retries": 2, "include": ["ProviderError"]}},
         rich=False,
         output_dir=output_dir,
         **({"pool": pool} if pool else {}),

{verifiers-0.1.15.dev382 → verifiers-0.1.15.dev384}/verifiers/v1/GUIDE.md

@@ -486,7 +486,7 @@ The `*_v1` tasksets under `environments/` are the reference library — each sho
 | `scratchpad-v1` | a shared, **writable** tool server — per-rollout state isolated via `self.state` |
 | `deepwiki-v1` | an existing remote tool server, by URL |
 | `color-codeword-v1` | a multimodal (image) task |
-| `wordle-v1` | a thin config over the shipped `textarena-v1` integration |
+| `wordle-v1` | a thin config over the shipped `textarena` integration |
 
 ---
 

{verifiers-0.1.15.dev382 → verifiers-0.1.15.dev384}/verifiers/v1/README.md

@@ -75,7 +75,7 @@ Taskset examples (the `*_v1` packages under `environments/`):
 | `glossary-v1` | a custom **colocated** tool server |
 | `wiki-search-v1` | a **shared** tool server (built once for the eval) + an LLM judge |
 | `deepwiki-v1` | an **existing remote** tool server, by URL |
-| `wordle-v1` | configuring the vendored `textarena-v1` integration |
+| `wordle-v1` | configuring the vendored `textarena` integration |
 
 Harness examples (under `environments/`):
 
@@ -100,10 +100,10 @@ filesystem, so run it under a containerized runtime: `docker` locally, or a remo
 `modal` sandbox (not the default `subprocess`). 
 
 ```bash
-uv run eval harbor-v1 -n 1 --taskset.ignore-dockerfile --harness.runtime.type docker --harness.id bash            # bash-only agent
-uv run eval harbor-v1 -n 1 --taskset.ignore-dockerfile --harness.runtime.type docker --harness.id mini-swe-agent  # the mini-swe-agent CLI
-uv run eval harbor-v1 -n 1 --taskset.ignore-dockerfile --harness.runtime.type docker --harness.id rlm             # the rlm CLI agent
-uv run eval harbor-v1 -n 1 --taskset.ignore-dockerfile --harness.runtime.type docker --harness.id codex           # the codex CLI agent
+uv run eval harbor -n 1 --taskset.ignore-dockerfile --harness.runtime.type docker --harness.id bash            # bash-only agent
+uv run eval harbor -n 1 --taskset.ignore-dockerfile --harness.runtime.type docker --harness.id mini-swe-agent  # the mini-swe-agent CLI
+uv run eval harbor -n 1 --taskset.ignore-dockerfile --harness.runtime.type docker --harness.id rlm             # the rlm CLI agent
+uv run eval harbor -n 1 --taskset.ignore-dockerfile --harness.runtime.type docker --harness.id codex           # the codex CLI agent
 ```
 
 ### Swappable runtime
@@ -227,13 +227,13 @@ uv run eval gsm8k-v1 -n 1 --retries.rollout.max-retries 3 --retries.rollout.incl
 ### Integrations
 
 Some tasksets wrap a whole benchmark family rather than a single task — shipped, installed by
-default. For example, `textarena-v1` (TextArena games) and `harbor-v1` (the agentic-
+default. For example, `textarena` (TextArena games) and `harbor` (the agentic-
 benchmark registry). Harbor is the showcase: it pulls tasks straight from the Harbor registry
 via the `harbor` CLI (`uv tool install harbor`), each in its own declared, pullable container
 image — e.g. Terminal-Bench 2:
 
 ```bash
-uv run eval harbor-v1 --taskset.dataset terminal-bench/terminal-bench-2 -n 10 --harness.id rlm
+uv run eval harbor --taskset.dataset terminal-bench/terminal-bench-2 -n 10 --harness.id rlm
 ```
 
 ## Backwards compatibility

{verifiers-0.1.15.dev382 → verifiers-0.1.15.dev384}/verifiers/v1/clients/train.py

@@ -147,6 +147,7 @@ def response_from_generate(
             completion_ids=completion_ids,
             completion_logprobs=result.get("completion_logprobs") or [],
             message_spans=message_spans,
+            is_content=attribution.is_content if attribution is not None else None,
             multi_modal_data=result.get("multi_modal_data"),
             routed_experts=result.get("routed_experts"),
         ),

{verifiers-0.1.15.dev382 → verifiers-0.1.15.dev384}/verifiers/v1/graph.py

@@ -82,6 +82,15 @@ class MessageNode(StrictBaseModel):
     """Per-token, parallel to `token_ids`: True for trainable, model-sampled tokens (only an
     assistant node's completion span); False for template scaffold and every input-message
     token."""
+    is_content: list[bool] = Field(default_factory=list)
+    """Per-token, parallel to `token_ids` (when populated): True for message-body tokens (the
+    renderer's content), False for template scaffold (role-tag openers/closers, inter-turn
+    separators, tool-response wraps, the generation prompt). Populated from the renderer's
+    `RenderedTokens.is_content`; empty when the renderer doesn't attribute content (e.g. the
+    default Jinja renderer) or for relay (eval) turns that carry no token ids. Distinct from
+    `mask`: `mask` is "did the model sample this?" (assistant completion only); `is_content`
+    is "is this caller/model body vs scaffold?" — meaningful on every role, so observation
+    weighting (prime-rl `echo`) can train a tool/user message's *body* without its scaffold."""
     logprobs: list[float] = Field(default_factory=list)
     """Sampling logprobs for the sampled tokens — length equals the number of True entries in
     `mask`; empty for input messages."""
@@ -433,6 +442,8 @@ def _commit_turn(turn: PendingTurn, response: Response) -> None:
     multi_modal_data = tokens.multi_modal_data if tokens else None
     prompt_ids = tokens.prompt_ids if tokens else []
     spans = tokens.message_spans if tokens else None
+    is_content = tokens.is_content if tokens else None
+    has_is_content = is_content is not None and len(is_content) == len(prompt_ids)
     idx = _head_index(trace)
 
     # Token-based prefix reuse. `prepare_turn` matched the prefix by message hash (content); when
@@ -485,6 +496,7 @@ def _commit_turn(turn: PendingTurn, response: Response) -> None:
                 message=msg,
                 token_ids=node_tokens,
                 mask=[False] * len(node_tokens),
+                is_content=is_content[start:end] if has_is_content else [],
             )
         )
         parent = len(trace.nodes) - 1
@@ -505,6 +517,9 @@ def _commit_turn(turn: PendingTurn, response: Response) -> None:
             sampled=True,
             token_ids=[*gen_prompt, *comp_ids],
             mask=[False] * len(gen_prompt) + [True] * len(comp_ids),
+            is_content=([False] * len(gen_prompt) + [True] * len(comp_ids))
+            if has_is_content
+            else [],
             # TurnTokens is discarded after commit, so transfer its logprobs without copying.
             logprobs=tokens.completion_logprobs if tokens else [],
             finish_reason=response.finish_reason,

{verifiers-0.1.15.dev382 → verifiers-0.1.15.dev384}/verifiers/v1/loaders.py

@@ -2,9 +2,9 @@
 
 A plugin (taskset or harness) is a module that exports its `Taskset` / `Harness` subclass via
 `__all__` — vf walks the exported names and finds the single subclass of the base. An id (an
-`EnvId`) resolves to that module: a built-in id (`default`, `rlm`, `harbor_v1`, `textarena_v1`)
+`EnvId`) resolves to that module: a built-in id (`default`, `rlm`, `harbor`, `textarena`)
 resolves to its namespaced module under the group package (`verifiers.v1.harnesses.rlm`,
-`verifiers.v1.tasksets.harbor_v1`, ...); any other id names a flat module — a local package
+`verifiers.v1.tasksets.harbor`, ...); any other id names a flat module — a local package
 (hyphens → underscores), or an `org/name[@version]` package installed on demand from the
 Environments Hub. Built-ins ship with verifiers under `verifiers/v1/{harnesses,tasksets}`;
 custom ones live under `environments/`, on `sys.path`, or on the hub.

{verifiers-0.1.15.dev382 → verifiers-0.1.15.dev384}/verifiers/v1/tasksets/__init__.py

@@ -1,9 +1,9 @@
 """Built-in tasksets, resolved by id (`--taskset.id <id>`) as `verifiers.v1.tasksets.<id>`.
 
-Re-exports each taskset's class + config off the package. `textarena_v1` is not re-exported
+Re-exports each taskset's class + config off the package. `textarena` is not re-exported
 here — it imports the optional `textarena` dependency at module load — but still resolves as
-`verifiers.v1.tasksets.textarena_v1`."""
+`verifiers.v1.tasksets.textarena`."""
 
-from verifiers.v1.tasksets.harbor_v1 import HarborConfig, HarborTaskset
+from verifiers.v1.tasksets.harbor import HarborConfig, HarborTaskset
 
 __all__ = ["HarborConfig", "HarborTaskset"]

{verifiers-0.1.15.dev382/verifiers/v1/tasksets/harbor_v1 → verifiers-0.1.15.dev384/verifiers/v1/tasksets/harbor}/__init__.py

@@ -1,4 +1,4 @@
-from verifiers.v1.tasksets.harbor_v1.taskset import (
+from verifiers.v1.tasksets.harbor.taskset import (
     HarborConfig,
     HarborTask,
     HarborTaskset,

{verifiers-0.1.15.dev382/verifiers/v1/tasksets/textarena_v1 → verifiers-0.1.15.dev384/verifiers/v1/tasksets/textarena}/__init__.py

@@ -1,4 +1,4 @@
-from verifiers.v1.tasksets.textarena_v1.taskset import (
+from verifiers.v1.tasksets.textarena.taskset import (
     TextArenaConfig,
     TextArenaState,
     TextArenaTask,

{verifiers-0.1.15.dev382/verifiers/v1/tasksets/textarena_v1 → verifiers-0.1.15.dev384/verifiers/v1/tasksets/textarena}/taskset.py

@@ -1,4 +1,4 @@
-"""textarena_v1 — TextArena games as a v1 taskset, driven by a user simulator.
+"""textarena — TextArena games as a v1 taskset, driven by a user simulator.
 
 Each task is one episode of a TextArena game (the working example is Wordle). The model
 plays by emitting moves; the framework's interception server drives a `vf.User` (the game
@@ -27,7 +27,7 @@ try:
     import textarena as ta
 except ImportError as e:
     raise ImportError(
-        "textarena_v1 requires nltk and textarena. Install with: uv add 'verifiers[ta]'"
+        "textarena requires nltk and textarena. Install with: uv add 'verifiers[ta]'"
     ) from e
 
 SYSTEM_PROMPT = (

{verifiers-0.1.15.dev382 → verifiers-0.1.15.dev384}/verifiers/v1/types.py

@@ -219,6 +219,7 @@ class TurnTokens(StrictBaseModel):
     message_spans: list[tuple[int, int] | None] | None = Field(
         default=None, exclude=True
     )
+    is_content: list[bool] | None = Field(default=None, exclude=True)
     # Transient carrier (excluded): the renderer's multimodal sidecar (image tensors + offsets),
     # attributed per node by the turn's `commit`, then dropped — never persisted.
     multi_modal_data: MultiModalData | None = Field(default=None, exclude=True)