verifiers 0.3.2.dev159__tar.gz → 0.3.2.dev160__tar.gz

{verifiers-0.3.2.dev159 → verifiers-0.3.2.dev160}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.5
 Name: verifiers
-Version: 0.3.2.dev159
+Version: 0.3.2.dev160
 Summary: Verifiers: Environments for LLM Reinforcement Learning
 Project-URL: Homepage, https://github.com/primeintellect-ai/verifiers
 Project-URL: Documentation, https://github.com/primeintellect-ai/verifiers

{verifiers-0.3.2.dev159 → verifiers-0.3.2.dev160}/verifiers/v1/cli/validate.py

@@ -30,6 +30,7 @@ from verifiers.v1.cli.resolve import (
 )
 from verifiers.v1.cli.resume import distribute
 from verifiers.v1.configs.cli.validate import ValidateConfig
+from verifiers.v1.errors import TaskError, boundary
 from verifiers.v1.runtimes import make_runtime
 from verifiers.v1.state import state_cls
 from verifiers.v1.task import Task
@@ -47,14 +48,17 @@ SUMMARY_FILE = "summary.json"
 LOG_FILE = "logs/validate.log"
 FINAL_VALUES = {"valid": True, "invalid": False, "unchecked": None}
 REASONS = ("valid", "invalid", "unchecked", "error", "timeout")
+SOLVED = 1.0
+"""An untouched task whose reward reaches this already passes: its noop check is
+invalid. Lower (partial) rewards are recorded but pass the check."""
 
 ResultRow = dict[str, Any]
 
 USAGE = (
-    "usage: uv run vf-validate [<taskset-id>] [--only-setup | --only-gold] "
+    "usage: uv run vf-validate [<taskset-id>] [--only-setup | --only-gold | --only-noop] "
     "[-o <output-dir>] [--runtime.type subprocess] [options] [@ file.toml]\n"
     "       uv run vf-validate @ <run-dir>/configs/validate.json --resume   (re-run missing/errored/timed-out tasks)\n"
-    "       runs persisted gold and setup-only checks per task (no model)"
+    "       runs persisted gold and noop (untouched scoring) checks per task (no agent)"
 )
 
 
@@ -70,6 +74,8 @@ def validation_mode(config: ValidateConfig) -> str:
         return "gold"
     if config.only_setup:
         return "setup"
+    if config.only_noop:
+        return "noop"
     return "all"
 
 
@@ -98,11 +104,14 @@ def _is_final(row: object, key: str, mode: str) -> bool:
     if not isinstance(row, dict):
         return False
     reason = row.get("reason")
+    # A noop check records its untouched scoring; a row without it never scored.
+    noop = row.get("noop") if mode == "all" else row
     return (
         row.get("task_key") == key
         and row.get("mode") == mode
         and reason in FINAL_VALUES
         and row.get("valid") is FINAL_VALUES[reason]
+        and (mode in ("gold", "setup") or isinstance(noop, dict) and "rewards" in noop)
     )
 
 
@@ -173,7 +182,7 @@ def summarize(rows: Sequence[ResultRow], total: int, mode: str) -> dict[str, Any
     }
     if mode == "all":
         checks: dict[str, dict[str, int]] = {}
-        for check in ("gold", "setup"):
+        for check in ("gold", "noop"):
             check_counts = Counter(
                 row.get(check, {}).get("reason")
                 for row in rows
@@ -225,6 +234,36 @@ def _row(
     }
 
 
+async def _setup_only(task: Task, trace: Trace, runtime) -> bool:
+    """Setup completing is the whole check."""
+    return True
+
+
+async def _gold(task: Task, trace: Trace, runtime) -> bool | None:
+    """Apply and check the reference answer."""
+    return await task.validate(runtime)
+
+
+async def _score_untouched(task: Task, trace: Trace, runtime) -> bool | None:
+    """Finalize and score the freshly set-up task exactly as a rollout would, with no
+    agent turn in between. Valid iff it does not already pass; `None` when the task
+    declares no reward to check."""
+    async with (
+        asyncio.timeout(task.data.timeout.finalize),
+        boundary(TaskError, "task finalize"),
+    ):
+        await invoke(task.finalize, {"trace": trace, "runtime": runtime})
+    async with asyncio.timeout(task.data.timeout.scoring):
+        await task.score(trace, runtime)
+    if not trace.rewards:
+        return None
+    return trace.reward < SOLVED
+
+
+CHECKS = {"setup": _setup_only, "gold": _gold, "noop": _score_untouched}
+"""What each check runs after `Task.setup`, in the check's own fresh runtime."""
+
+
 async def _run_check(task: Task, config: ValidateConfig, mode: str) -> ResultRow:
     start = time.time()
     runtime = make_runtime(
@@ -238,6 +277,7 @@ async def _run_check(task: Task, config: ValidateConfig, mode: str) -> ResultRow
     )
     valid: bool | None = False
     exc = None
+    trace: Trace | None = None
     try:
         runtime.env = dict(task.runtime_env())
         trace = Trace(
@@ -257,10 +297,8 @@ async def _run_check(task: Task, config: ValidateConfig, mode: str) -> ResultRow
             invoke(task.setup, {"trace": trace, "runtime": runtime}),
             setup_timeout,
         )
-        valid = (
-            await asyncio.wait_for(task.validate(runtime), config.timeout.total)
-            if mode == "gold"
-            else True
+        valid = await asyncio.wait_for(
+            CHECKS[mode](task, trace, runtime), config.timeout.total
         )
     except Exception as e:  # noqa: BLE001 - validation reports plugin failures per task
         exc = e
@@ -271,7 +309,21 @@ async def _run_check(task: Task, config: ValidateConfig, mode: str) -> ResultRow
             logger.warning(
                 "runtime teardown failed (task %s)", task.data.idx, exc_info=True
             )
-    return _row(task, mode, valid, exc, start)
+    row = _row(task, mode, valid, exc, start)
+    if mode == "noop":
+        rewards = trace.rewards if trace is not None else {}
+        row["reward"] = trace.reward if exc is None and rewards else None
+        row["rewards"] = {
+            name: reward.score if reward is not None else None
+            for name, reward in rewards.items()
+        }
+    return row
+
+
+def _describe(row: ResultRow) -> str | None:
+    if row["mode"] == "noop" and row["reason"] == "invalid":
+        return f"passes untouched (reward={row['reward']:g})"
+    return row["error"]
 
 
 def _all_reason(rows: list[ResultRow]) -> str:
@@ -289,7 +341,7 @@ def _all_reason(rows: list[ResultRow]) -> str:
 
 def _all_error(rows: list[ResultRow]) -> tuple[str | None, str | None]:
     failed = [row for row in rows if row["reason"] not in ("valid", "unchecked")]
-    parts = [f"{row['mode']}: {row['error'] or row['reason']}" for row in failed]
+    parts = [f"{row['mode']}: {_describe(row) or row['reason']}" for row in failed]
     error_types = {str(row["error_type"]) for row in failed if row["error_type"]}
     return ("; ".join(parts) or None, "+".join(sorted(error_types)) or None)
 
@@ -297,8 +349,8 @@ def _all_error(rows: list[ResultRow]) -> tuple[str | None, str | None]:
 async def _run_all(task: Task, config: ValidateConfig) -> ResultRow:
     start = time.time()
     gold = await _run_check(task, config, "gold")
-    setup = await _run_check(task, config, "setup")
-    rows = [gold, setup]
+    noop = await _run_check(task, config, "noop")
+    rows = [gold, noop]
     reason = _all_reason(rows)
     error, error_type = _all_error(rows)
     return {
@@ -311,7 +363,7 @@ async def _run_all(task: Task, config: ValidateConfig) -> ResultRow:
         "error": error,
         "error_type": error_type,
         "gold": gold,
-        "setup": setup,
+        "noop": noop,
     }
 
 
@@ -338,7 +390,7 @@ async def run_validate(config: ValidateConfig) -> list[dict]:
             "taskset needs a container runtime to validate - pass --runtime.type docker (or prime)"
         )
     mode = validation_mode(config)
-    checks = "gold+setup" if mode == "all" else mode
+    checks = "gold+noop" if mode == "all" else mode
     out = output_path(config)
     selected_keys = [task.hash for task in tasks]
     if not config.resume:
@@ -395,7 +447,7 @@ async def run_validate(config: ValidateConfig) -> list[dict]:
         await run_shielded(persist())
         st.end, st.state = time.time(), row["reason"]
         if not config.rich:  # the dashboard shows this live; otherwise log each task
-            detail = f" - {row['error']}" if row["error"] else ""
+            detail = f" - {note}" if (note := _describe(row)) else ""
             logger.info(
                 "idx=%s valid=%s reason=%s (%.1fs)%s",
                 row["index"],

{verifiers-0.3.2.dev159 → verifiers-0.3.2.dev160}/verifiers/v1/configs/cli/validate.py

@@ -16,8 +16,8 @@ class CheckTimeoutConfig(BaseConfig):
     setup: float | None = None
     """Max wall-clock for the task's `setup` hook."""
     total: float | None = None
-    """Max wall-clock for the check itself per task — the `validate` hook, or the debug
-    command/script."""
+    """Max wall-clock for the check itself per task, after setup — the gold check's
+    `validate` hook, or the noop check's `finalize` and scoring."""
 
 
 class ValidateConfig(BaseConfig):
@@ -32,9 +32,13 @@ class ValidateConfig(BaseConfig):
     """Where each task's validation hooks run."""
     timeout: CheckTimeoutConfig = CheckTimeoutConfig()
     only_setup: bool = False
-    """Run only `Task.setup`."""
+    """Run only the setup check: `Task.setup`."""
     only_gold: bool = False
-    """Run only `Task.setup` and `Task.validate`."""
+    """Run only the gold check: `Task.setup`, then `Task.validate`."""
+    only_noop: bool = False
+    """Run only the noop check: `Task.setup`, then `Task.finalize` and scoring on the
+    untouched task (no reference answer, no agent). Scoring is the task's full rewards,
+    configured judges included. Invalid when it already passes."""
     max_concurrent: int | None = Field(
         128, validation_alias=AliasChoices("max_concurrent", "c")
     )
@@ -72,8 +76,10 @@ class ValidateConfig(BaseConfig):
 
     @model_validator(mode="after")
     def _validate_only(self):
-        if self.only_setup and self.only_gold:
-            raise ValueError("pass at most one of `--only-setup` or `--only-gold`")
+        if self.only_setup + self.only_gold + self.only_noop > 1:
+            raise ValueError(
+                "pass at most one of `--only-setup`, `--only-gold` or `--only-noop`"
+            )
         return self
 
     @model_validator(mode="before")