verifiers 0.2.2.dev94__py3-none-any.whl → 0.2.2.dev96__py3-none-any.whl

verifiers/envs/experimental/composable/harnesses/rlm.py

@@ -15,6 +15,7 @@ from verifiers.envs.experimental.utils.git_checkout_cache import (
     validate_git_checkout,
 )
 from verifiers.types import Messages, State, SystemMessage, TrajectoryStep
+from verifiers.utils.path_utils import CACHE_DIR
 
 DEFAULT_RLM_REPO_URL = "github.com/PrimeIntellect-ai/rlm-harness.git"
 DEFAULT_RLM_REF = "main"
@@ -24,9 +25,7 @@ DEFAULT_RLM_MAX_DEPTH = 0
 DEFAULT_APPEND_TO_SYSTEM_PROMPT_PATH = "/task/append_to_system_prompt.txt"
 DEFAULT_RLM_CHECKOUT_PATH = "/tmp/rlm-checkout"
 DEFAULT_RLM_CHECKOUT_UPLOAD_NAME = "rlm_checkout"
-DEFAULT_RLM_LOCAL_CHECKOUT_CACHE_ROOT = (
-    Path.home() / ".cache" / "verifiers" / "rlm-checkouts"
-)
+DEFAULT_RLM_LOCAL_CHECKOUT_CACHE_ROOT = CACHE_DIR / "rlm-checkouts"
 _REQUIRED_CHECKOUT_FILES = ("install.sh", "pyproject.toml")
 
 COMPACTION_BOUNDARY_MARKER = "--- context compacted ---"

verifiers/envs/experimental/utils/git_checkout_cache.py

@@ -16,6 +16,7 @@ from verifiers.envs.experimental.utils.file_locks import (
     exclusive_path_lock,
     sibling_lock_path,
 )
+from verifiers.utils.path_utils import CACHE_DIR
 
 _IN_USE_LOCK_SUFFIX = ".in-use.lock"
 
@@ -23,7 +24,7 @@ _IN_USE_LOCK_SUFFIX = ".in-use.lock"
 # resolved checkout's in-use lock file. See ``_acquire_in_use_lock``.
 _held_in_use_locks: dict[Path, IO] = {}
 
-DEFAULT_GIT_CHECKOUT_CACHE_ROOT = Path.home() / ".cache" / "verifiers" / "git-checkouts"
+DEFAULT_GIT_CHECKOUT_CACHE_ROOT = CACHE_DIR / "git-checkouts"
 _FULL_COMMIT_SHA_RE = re.compile(r"^[0-9a-fA-F]{40}$")
 
 logger = logging.getLogger(__name__)

verifiers/utils/path_utils.py

@@ -9,6 +9,18 @@ from verifiers.types import EvalConfig
 logger = logging.getLogger(__name__)
 
 
+def home_dir() -> Path:
+    """Best-effort home directory; fall back to the temp dir so import never fails."""
+    try:
+        return Path.home()
+    except RuntimeError:
+        return Path(tempfile.gettempdir())
+
+
+CACHE_DIR = home_dir() / ".cache" / "verifiers"
+"""User-local cache root for verifiers-managed state."""
+
+
 def write_temp_file(content: str, suffix: str = ".txt") -> str:
     """Write content to a named temporary file and return its path.
 

verifiers/v1/cli/eval/main.py

@@ -7,7 +7,7 @@ import sys
 from pydantic_config import cli
 
 import verifiers.v1 as vf
-from verifiers.v1.cli.eval.resume import load_resume_config, split_resume
+from verifiers.v1.cli.eval.resume import load_resume_config
 from verifiers.v1.cli.eval.runner import run_eval
 from verifiers.v1.cli.output import output_path, write_config
 from verifiers.v1.cli.resolve import (
@@ -17,6 +17,7 @@ from verifiers.v1.cli.resolve import (
     references_config_file,
     with_positional_taskset,
 )
+from verifiers.v1.cli.resume import split_resume
 from verifiers.v1.configs.cli.eval import EvalConfig
 from verifiers.v1.utils.interrupt import install_interrupt
 from verifiers.v1.utils.logging import setup_logging
@@ -40,7 +41,7 @@ def main(argv: list[str] | None = None) -> None:
                 narrow_config(EvalConfig, argv)
             )  # full option help, narrowed to the given ids
         return
-    resume_dir, rest = split_resume(argv)
+    resume_dir, rest = split_resume(argv, "eval")
     # re-run a previous run's missing/errored rollouts, in place
     if resume_dir is not None:
         if rest:

verifiers/v1/cli/eval/resume.py

@@ -11,7 +11,6 @@ changed since the interrupted run re-runs, and nothing depends on `data.idx`. Th
 legacy (v0) bridge still matches by row index (`key_of`).
 """
 
-import hashlib
 import json
 import tomllib
 from collections import Counter, defaultdict
@@ -22,6 +21,7 @@ from typing import TypeVar
 from pydantic_core import from_json
 
 from verifiers.v1.cli.output import CONFIG_FILE, TRACES_FILE, sniff_episode
+from verifiers.v1.cli.resume import task_key
 from verifiers.v1.configs.cli.eval import EvalConfig
 from verifiers.v1.episode import Episode, WireEpisode
 from verifiers.v1.trace import WireTrace
@@ -29,43 +29,6 @@ from verifiers.v1.trace import WireTrace
 K = TypeVar("K", bound=Hashable)
 
 
-def task_key(data: Mapping) -> str:
-    """Content identity of one task's wire data — an `exclude_none` dump, the shape
-    saved rows already have on disk. `sort_keys` so field order can't split identity."""
-    return hashlib.sha256(json.dumps(data, sort_keys=True).encode()).hexdigest()
-
-
-def distribute(
-    selected_keys: list[K], owed: dict[K, int], num_rollouts: int
-) -> list[int]:
-    """Spread each key's owed rollouts over its selection instances, in order —
-    content-identical tasks are interchangeable, so any instance can absorb the
-    debt (capped at `num_rollouts` each). Returns one count per selection."""
-    remaining = dict(owed)
-    counts: list[int] = []
-    for key in selected_keys:
-        take = min(num_rollouts, remaining.get(key, 0))
-        if take:
-            remaining[key] -= take
-        counts.append(take)
-    return counts
-
-
-def split_resume(argv: list[str]) -> tuple[Path | None, list[str]]:
-    """Pull `--resume <dir>` / `--resume=<dir>` out of argv, returning (dir, the other args).
-    The caller rejects any leftover args, since resume re-runs the saved config verbatim."""
-    for i, arg in enumerate(argv):
-        if arg == "--resume":
-            if i + 1 >= len(argv):
-                raise SystemExit(
-                    "--resume needs an output dir: uv run eval --resume <dir>"
-                )
-            return Path(argv[i + 1]), argv[:i] + argv[i + 2 :]
-        if arg.startswith("--resume="):
-            return Path(arg.split("=", 1)[1]), argv[:i] + argv[i + 1 :]
-    return None, argv
-
-
 def load_resume_config(resume_dir: Path) -> EvalConfig:
     """Rebuild the run's `EvalConfig` from its saved `config.toml`, pointed back at its own
     output dir so the resumed rollouts append to the same `traces.jsonl`."""

verifiers/v1/cli/eval/runner.py

@@ -14,6 +14,7 @@ from verifiers.v1.cli.output import (
     output_path,
     save_config,
 )
+from verifiers.v1.cli.resume import distribute, task_key
 from verifiers.v1.clients import ModelContext
 from verifiers.v1.configs.cli.eval import EvalConfig
 from verifiers.v1.env import Env, RunSlot
@@ -48,14 +49,13 @@ async def run_eval(env: Env, config: EvalConfig) -> list[Episode]:
     finished: list[Episode] = []
     if config.resume is not None:
         keys = [
-            resume.task_key(t.data.model_dump(mode="json", exclude_none=True))
-            for t in tasks
+            task_key(t.data.model_dump(mode="json", exclude_none=True)) for t in tasks
         ]
         finished, owed = resume.load(out, keys, config.num_rollouts, env.complete)
         if not owed:  # already complete - report it and exit successfully
             print(resume.nothing_to_resume_msg(out, len(tasks), config.num_rollouts))
             raise SystemExit(0)
-        counts = resume.distribute(keys, owed, config.num_rollouts)
+        counts = distribute(keys, owed, config.num_rollouts)
         plan = [(task, n) for task, n in zip(tasks, counts) if n]
         logger.info(
             "resuming %s: %d task(s), %d rollout(s) owed",
@@ -179,7 +179,7 @@ async def run_eval_server(config: EvalConfig) -> list[Episode]:
         client = EnvClient(address=address)
         await client.wait_for_server_startup(timeout=600)
         # A v1 run dispatches — and resumes — tasks by content: the client owns them,
-        # and `resume.task_key` is their identity. Only the legacy bridge is addressed
+        # and `task_key` is their identity. Only the legacy bridge is addressed
         # by dataset row (its dataset lives server-side, reported via `info`), and
         # only a legacy env group-scores; a v1 env scores siblings in its own rollout.
         if legacy:
@@ -209,14 +209,14 @@ async def run_eval_server(config: EvalConfig) -> list[Episode]:
                     whole_task=group_scored,
                     key_of=lambda data: data.get("idx"),
                 )
-                counts = resume.distribute(idxs, owed, config.num_rollouts)
+                counts = distribute(idxs, owed, config.num_rollouts)
             else:
                 keys = [
-                    resume.task_key(t.data.model_dump(mode="json", exclude_none=True))
+                    task_key(t.data.model_dump(mode="json", exclude_none=True))
                     for t in tasks
                 ]
                 finished, owed = resume.load(out, keys, config.num_rollouts)
-                counts = resume.distribute(keys, owed, config.num_rollouts)
+                counts = distribute(keys, owed, config.num_rollouts)
             if not owed:  # already complete - report it and exit successfully
                 print(resume.nothing_to_resume_msg(out, len(plan), config.num_rollouts))
                 raise SystemExit(0)

verifiers/v1/cli/resume.py

@@ -0,0 +1,42 @@
+"""Resume primitives shared by eval-like CLIs."""
+
+import hashlib
+import json
+from collections.abc import Hashable, Mapping
+from pathlib import Path
+from typing import TypeVar
+
+K = TypeVar("K", bound=Hashable)
+
+
+def task_key(data: Mapping) -> str:
+    """Content identity for task wire data, independent of field order."""
+    return hashlib.sha256(json.dumps(data, sort_keys=True).encode()).hexdigest()
+
+
+def distribute(
+    selected_keys: list[K], owed: dict[K, int], num_results: int
+) -> list[int]:
+    """Spread each key's owed results over its selected instances, in order."""
+    remaining = dict(owed)
+    counts: list[int] = []
+    for key in selected_keys:
+        take = min(num_results, remaining.get(key, 0))
+        if take:
+            remaining[key] -= take
+        counts.append(take)
+    return counts
+
+
+def split_resume(argv: list[str], command: str) -> tuple[Path | None, list[str]]:
+    """Pull ``--resume <dir>`` from argv, returning the dir and other arguments."""
+    for i, arg in enumerate(argv):
+        if arg == "--resume":
+            if i + 1 >= len(argv):
+                raise SystemExit(
+                    f"--resume needs an output dir: uv run {command} --resume <dir>"
+                )
+            return Path(argv[i + 1]), argv[:i] + argv[i + 2 :]
+        if arg.startswith("--resume="):
+            return Path(arg.split("=", 1)[1]), argv[:i] + argv[i + 1 :]
+    return None, argv

verifiers/v1/cli/validate.py

@@ -2,16 +2,23 @@
 
 import asyncio
 import contextlib
+import json
 import logging
 import sys
 import time
+import tomllib
+from collections import Counter, defaultdict
+from collections.abc import Mapping, Sequence
+from pathlib import Path
 from typing import Any
 from uuid import uuid4
 
 from pydantic_config import cli
+from pydantic_core import from_json
 
 import verifiers.v1 as vf
 from verifiers.v1.cli.dashboard import TaskProgress, validate_dashboard
+from verifiers.v1.cli.output import CONFIG_FILE, write_config
 from verifiers.v1.cli.resolve import (
     extract_id,
     narrow_taskset_config,
@@ -19,11 +26,13 @@ from verifiers.v1.cli.resolve import (
     references_config_file,
     with_positional_taskset,
 )
+from verifiers.v1.cli.resume import distribute, split_resume, task_key
 from verifiers.v1.configs.cli.validate import ValidateConfig
 from verifiers.v1.runtimes import make_runtime
 from verifiers.v1.state import state_cls
 from verifiers.v1.task import Task
 from verifiers.v1.trace import Trace, TraceTask
+from verifiers.v1.utils.aio import run_shielded
 from verifiers.v1.utils.compile import resolve_runtime_config
 from verifiers.v1.utils.decorators import invoke
 from verifiers.v1.utils.interrupt import install_interrupt
@@ -31,10 +40,19 @@ from verifiers.v1.utils.logging import setup_logging
 
 logger = logging.getLogger(__name__)
 
+RESULTS_FILE = "results.jsonl"
+SUMMARY_FILE = "summary.json"
+LOG_FILE = "validate.log"
+FINAL_REASONS = frozenset({"valid", "invalid"})
+REASONS = ("valid", "invalid", "error", "timeout")
+
+ResultRow = dict[str, Any]
+
 USAGE = (
     "usage: uv run validate [<taskset-id>] [--only-setup | --only-gold] "
-    "[--runtime.type subprocess] [options] [@ file.toml]\n"
-    "       runs the gold and setup-only checks per task (no model)"
+    "[-o <output-dir>] [--runtime.type subprocess] [options] [@ file.toml]\n"
+    "       uv run validate --resume <output-dir>\n"
+    "       runs persisted gold and setup-only checks per task (no model)"
 )
 
 
@@ -45,7 +63,149 @@ def _narrow(argv: list[str]) -> type[ValidateConfig]:
     return narrow_taskset_config(ValidateConfig, extract_id(argv, "taskset"))
 
 
-ResultRow = dict[str, Any]
+def validation_mode(config: ValidateConfig) -> str:
+    if config.only_gold:
+        return "gold"
+    if config.only_setup:
+        return "setup"
+    return "all"
+
+
+def output_path(config: ValidateConfig) -> Path:
+    if config.output_dir is not None:
+        return config.output_dir
+    return Path("outputs") / f"{config.name}--validate" / config.uuid
+
+
+def _write_rows(path: Path, rows: Sequence[ResultRow]) -> None:
+    tmp = path.with_suffix(f"{path.suffix}.tmp")
+    with tmp.open("w", encoding="utf-8") as f:
+        for row in rows:
+            f.write(json.dumps(row, sort_keys=True, separators=(",", ":")) + "\n")
+    tmp.replace(path)
+
+
+def append_result(results_dir: Path, row: ResultRow) -> None:
+    data = json.dumps(row, sort_keys=True, separators=(",", ":")).encode()
+    with (results_dir / RESULTS_FILE).open("ab") as f:
+        f.write(data + b"\n")
+
+
+def _is_final(row: object, key: str, mode: str) -> bool:
+    if not isinstance(row, dict):
+        return False
+    reason = row.get("reason")
+    return (
+        row.get("task_key") == key
+        and row.get("mode") == mode
+        and reason in FINAL_REASONS
+        and row.get("valid") is (reason == "valid")
+    )
+
+
+def load_results(
+    results_dir: Path, selected_keys: list[str], mode: str
+) -> tuple[list[ResultRow], dict[str, int]]:
+    """Keep final rows by eval task-content key; return counts owed per key."""
+    path = results_dir / RESULTS_FILE
+    targets = Counter(selected_keys)
+    good: dict[str, list[ResultRow]] = defaultdict(list)
+    if path.exists():
+        with path.open("rb") as f:
+            for line in f:
+                if not line.strip():
+                    continue
+                try:
+                    row = from_json(line)
+                except ValueError:
+                    try:
+                        row = json.loads(line)
+                    except (json.JSONDecodeError, UnicodeDecodeError):
+                        continue
+                if not isinstance(row, dict):
+                    continue
+                key = row.get("task_key")
+                if (
+                    isinstance(key, str)
+                    and key in targets
+                    and len(good[key]) < targets[key]
+                    and _is_final(row, key, mode)
+                ):
+                    good[key].append(row)
+
+    owed = {
+        key: target - len(good.get(key, []))
+        for key, target in targets.items()
+        if len(good.get(key, [])) < target
+    }
+    # Eval spreads debt over content-identical selected tasks in order. Assign the
+    # interchangeable kept rows to the remaining positions so reporting positions
+    # stay unique even when duplicate task content exists.
+    counts = distribute(selected_keys, owed, 1)
+    rows = []
+    used = Counter()
+    for position, (key, count) in enumerate(zip(selected_keys, counts)):
+        if count:
+            continue
+        row = good[key][used[key]]
+        used[key] += 1
+        rows.append({**row, "task_position": position})
+    _write_rows(path, rows)
+    return rows, owed
+
+
+def summarize(rows: Sequence[ResultRow], total: int, mode: str) -> dict[str, Any]:
+    counts = Counter(row.get("reason") for row in rows)
+    missing = max(0, total - len(rows))
+    outcomes = {reason: counts[reason] for reason in REASONS}
+    outcomes["missing"] = missing
+    terminal = outcomes["valid"] + outcomes["invalid"]
+    summary: dict[str, Any] = {
+        "mode": mode,
+        "total": total,
+        "recorded": len(rows),
+        "terminal": terminal,
+        "owed": missing + outcomes["error"] + outcomes["timeout"],
+        "outcomes": outcomes,
+        "valid_rate": round(outcomes["valid"] / total, 6) if total else None,
+    }
+    if mode == "all":
+        checks: dict[str, dict[str, int]] = {}
+        for check in ("gold", "setup"):
+            check_counts = Counter(
+                row.get(check, {}).get("reason")
+                for row in rows
+                if isinstance(row.get(check), dict)
+            )
+            checks[check] = {reason: check_counts[reason] for reason in REASONS}
+            checks[check]["missing"] = missing
+        summary["checks"] = checks
+    return summary
+
+
+def write_summary(results_dir: Path, summary: Mapping[str, Any]) -> None:
+    path = results_dir / SUMMARY_FILE
+    tmp = path.with_suffix(f"{path.suffix}.tmp")
+    tmp.write_text(json.dumps(summary, indent=2, sort_keys=True) + "\n")
+    tmp.replace(path)
+
+
+def save_run(config: ValidateConfig, results_dir: Path, total: int) -> None:
+    write_config(config, results_dir)
+    (results_dir / RESULTS_FILE).write_text("")
+    write_summary(results_dir, summarize([], total, validation_mode(config)))
+
+
+def load_resume_config(resume_dir: Path) -> ValidateConfig:
+    path = resume_dir / CONFIG_FILE
+    if not path.exists():
+        raise SystemExit(
+            f"--resume: no config.toml in {resume_dir} - not a validate output dir"
+        )
+    config = ValidateConfig.model_validate(tomllib.loads(path.read_text()))
+    config.resume = resume_dir
+    config.output_dir = resume_dir
+    return config
 
 
 def _classify(valid: bool, exc: BaseException | None) -> str:
@@ -217,27 +377,64 @@ async def run_validate(config: ValidateConfig) -> list[dict]:
         raise SystemExit(
             "taskset needs a container runtime to validate - pass --runtime.type docker (or prime)"
         )
-    checks = (
-        "gold" if config.only_gold else "setup" if config.only_setup else "gold+setup"
-    )
+    mode = validation_mode(config)
+    checks = "gold+setup" if mode == "all" else mode
+    out = output_path(config)
+    selected_keys = [
+        task_key(task.data.model_dump(mode="json", exclude_none=True)) for task in tasks
+    ]
+    if config.resume is None:
+        save_run(config, out, len(tasks))
+        rows: list[ResultRow] = []
+        counts = [1] * len(tasks)
+    else:
+        rows, owed = load_results(out, selected_keys, mode)
+        counts = distribute(selected_keys, owed, 1)
+        write_summary(out, summarize(rows, len(tasks), mode))
+    plan = [
+        (position, task, key)
+        for position, (task, key, count) in enumerate(zip(tasks, selected_keys, counts))
+        if count
+    ]
     logger.info(
-        "validating %d task(s) from %s on the %s runtime (%s)",
+        "%s %d/%d task(s) from %s on the %s runtime (%s)",
+        "resuming" if config.resume is not None else "validating",
+        len(plan),
         len(tasks),
         config.name,
         config.runtime.type,
         checks,
     )
+    logger.info("results: %s", out)
 
     sem = asyncio.Semaphore(config.max_concurrent) if config.max_concurrent else None
     states = [TaskProgress(idx=t.data.idx, name=t.data.name) for t in tasks]
-    state_by_idx = {s.idx: s for s in states}
+    for row in rows:
+        state = states[row["task_position"]]
+        state.state = row["reason"]
+
+    write_lock = asyncio.Lock()
 
-    async def _one(task) -> dict:
-        st = state_by_idx[task.data.idx]
+    async def _one(position: int, task: Task, key: str) -> ResultRow:
+        st = states[position]
         async with sem or contextlib.nullcontext():
             st.start = time.time()
             st.state = "running"
             row = await _validate_task(task, config)
+        row["task_position"] = position
+        row["task_key"] = key
+
+        async def persist() -> None:
+            async with write_lock:
+                await asyncio.to_thread(append_result, out, row)
+                rows.append(row)
+                await asyncio.to_thread(
+                    write_summary,
+                    out,
+                    summarize(rows, len(tasks), mode),
+                )
+
+        await run_shielded(persist())
         st.end, st.state = time.time(), row["reason"]
         if not config.rich:  # the dashboard shows this live; otherwise log each task
             detail = f" - {row['error']}" if row["error"] else ""
@@ -257,7 +454,17 @@ async def run_validate(config: ValidateConfig) -> list[dict]:
         else contextlib.nullcontext()
     )
     async with display:
-        return await asyncio.gather(*(_one(t) for t in tasks))
+        if not plan:
+            logger.info(
+                "nothing to resume: all %d task(s) are valid or invalid", len(tasks)
+            )
+            return rows
+        await asyncio.gather(
+            *(_one(position, task, key) for position, task, key in plan)
+        )
+    rows.sort(key=lambda row: row["task_position"])
+    write_summary(out, summarize(rows, len(tasks), mode))
+    return rows
 
 
 def main(argv: list[str] | None = None) -> None:
@@ -271,27 +478,46 @@ def main(argv: list[str] | None = None) -> None:
         with plugin_errors():
             cli(_narrow(argv))  # full option help, narrowed to the given taskset
         return
-    if not extract_id(argv, "taskset") and not references_config_file(argv):
-        raise SystemExit(
-            USAGE
-        )  # need a taskset (positional / --taskset.id) or a @ file.toml
-
-    with plugin_errors():
-        config_type = _narrow(argv)
-        sys.argv = [
-            sys.argv[0],
-            *argv,
-        ]  # let prime-pydantic-config render help/errors
-        config = cli(config_type)
-    # Nothing is persisted, so logs are the whole output. Under `--rich` the dashboard owns the
-    # screen, so keep logs off the console (else stray records print over the UI).
-    setup_logging("DEBUG" if config.verbose else "INFO", console=not config.rich)
+    resume_dir, rest = split_resume(argv, "validate")
+    if resume_dir is not None:
+        if rest:
+            raise SystemExit(
+                f"{USAGE}\n--resume replays the saved config and takes no other arguments"
+            )
+        with plugin_errors():
+            config = load_resume_config(resume_dir)
+    else:
+        if not extract_id(argv, "taskset") and not references_config_file(argv):
+            raise SystemExit(
+                USAGE
+            )  # need a taskset (positional / --taskset.id) or a @ file.toml
+
+        with plugin_errors():
+            config_type = _narrow(argv)
+            sys.argv = [
+                sys.argv[0],
+                *argv,
+            ]  # let prime-pydantic-config render help/errors
+            config = cli(config_type)
+    out = output_path(config)
+    setup_logging(
+        "DEBUG" if config.verbose else "INFO",
+        log_file=str(out / LOG_FILE),
+        console=not config.rich,
+    )
     if config.rich:
         logging.lastResort = None  # drop stdlib records that bypass loguru
     # Graceful shutdown: first Ctrl-C/SIGTERM unwinds each task's teardown `finally`
     # (containers/sandboxes); a second is swallowed so it can't orphan them mid-cleanup.
     install_interrupt()
-    asyncio.run(run_validate(config))
+    try:
+        asyncio.run(run_validate(config))
+    except KeyboardInterrupt:
+        print(f"interrupted; partial results: {out}", file=sys.stderr)
+        raise SystemExit(130)
+    summary = json.loads((out / SUMMARY_FILE).read_text())
+    print(f"results: {out}")
+    print(json.dumps(summary, indent=2, sort_keys=True))
 
 
 if __name__ == "__main__":

verifiers/v1/configs/cli/validate.py

@@ -1,5 +1,8 @@
 """Configuration for model-free task validation."""
 
+from pathlib import Path
+from uuid import uuid4
+
 from pydantic import AliasChoices, Field, SerializeAsAny, model_validator
 from pydantic_config import BaseConfig
 
@@ -16,6 +19,9 @@ class CheckTimeoutConfig(BaseConfig):
 
 
 class ValidateConfig(BaseConfig):
+    uuid: str = Field(default_factory=lambda: str(uuid4()), exclude=True)
+    """Auto-generated run id — the default output directory leaf. Excluded from the
+    saved config so re-running it starts a fresh run."""
     taskset: SerializeAsAny[TasksetConfig] = TasksetConfig()
     runtime: RuntimeConfig = DockerConfig()
     """Where each task's validation hooks run."""
@@ -40,6 +46,14 @@ class ValidateConfig(BaseConfig):
     """Log at debug level instead of the default info."""
     rich: bool = True
     """Show a live dashboard (one row per task) instead of per-task log lines."""
+    output_dir: Path | None = Field(
+        None, validation_alias=AliasChoices("output_dir", "o")
+    )
+    """Where to write config.toml, results.jsonl, summary.json, and validate.log. None
+    creates a fresh run under outputs/<taskset>--validate/<uuid>."""
+    resume: Path | None = Field(None, exclude=True)
+    """Set by --resume: re-run missing, errored, and timed-out tasks in this directory.
+    The saved config is replayed verbatim, so resume takes no other arguments."""
 
     @property
     def name(self) -> str:

verifiers/v1/interception/tunnel/prime.py

@@ -13,7 +13,7 @@ from verifiers.v1.runtimes.limiters import creation_limiter
 from verifiers.v1.utils.aio import run_shielded
 
 # The prime_tunnel service caps tunnel starts at 512/min per API token — a property of the
-# tunnel service, shared by every process on the host that opens one. One host-global
+# tunnel service, shared by every process for the user that opens one. One user-global
 # limiter, not a per-runtime config knob.
 _TUNNELS_PER_MIN = 512
 TUNNEL_LIMITER = creation_limiter(_TUNNELS_PER_MIN / 60, "prime-tunnel")
@@ -30,7 +30,7 @@ class PrimeTunnel(Tunnel[PrimeTunnelConfig]):
     @contextlib.asynccontextmanager
     async def expose(self, port: int) -> AsyncIterator[str]:
         """Bridge the host `port` to a public URL via prime_tunnel (frpc). Tunnel creation
-        is network-bound and globally rate-capped (512/min, host-wide via the shared
+        is network-bound and globally rate-capped (512/min, user-wide via the shared
         `TUNNEL_LIMITER`), so transient failures are retried; a terminal one raises
         `TunnelError`. The tunnel is torn down on exit."""
         from prime_tunnel import Tunnel as TunnelClient

verifiers/v1/runtimes/limiters.py

@@ -1,41 +1,45 @@
-"""Host-global creation-rate limiters for the remote runtimes.
+"""User-global creation-rate limiters for the remote runtimes.
 
-A leaky bucket backed by a lock file, so a provider's per-account creation rate (Modal
-sandboxes, Prime tunnels) is enforced across EVERY process on the host — the single-process
-eval and all the elastically-spawned env-server worker processes alike — not just within one
-process. So the configured rate is the actual account-wide rate (assuming one env-server
-host). Keyed by name: one bucket file per name, shared by every process (and run) on the host.
+A leaky bucket backed by a lock file under the user cache (``~/.cache/verifiers``, falling
+back to the temp dir when no home is resolvable), so a provider's per-account creation rate
+(Modal sandboxes, Prime tunnels) is enforced across EVERY process for the user —
+the single-process eval and all the elastically-spawned env-server worker processes alike — not
+just within one process. Keyed by name: one bucket file per name, shared by every process (and
+run) for the user.
 """
 
 import asyncio
 import fcntl
 import os
-import tempfile
 import time
 from typing import Self
 
-_LIMITER_DIR = os.path.join(tempfile.gettempdir(), "vf-rate-limiters")
+from verifiers.utils.path_utils import CACHE_DIR
+
+LIMITER_DIR = CACHE_DIR / "limiter"
 
 
 class CreationLimiter:
     """An async leaky bucket shared across processes via a lock file: each `async with`
     reserves the next `1/per_sec`-spaced slot (advancing the on-disk cursor under an exclusive
-    flock) and sleeps until it, so the aggregate creation rate across all host processes stays
-    at `per_sec`. The reservation runs off the event loop; the wait does not hold the lock."""
+    flock) and sleeps until it, so the aggregate creation rate across all of the user's
+    processes stays at `per_sec`. The reservation runs off the event loop; the wait does not
+    hold the lock."""
 
     def __init__(self, name: str, per_sec: float) -> None:
         self._interval = 1 / per_sec
-        self._path = os.path.join(_LIMITER_DIR, f"{name}.bucket")
+        self._path = LIMITER_DIR / f"{name}.bucket"
 
     def _reserve(self) -> float:
-        os.makedirs(_LIMITER_DIR, exist_ok=True)
-        # monotonic is host-wide on Linux, so the cursor is comparable across processes.
+        os.makedirs(LIMITER_DIR, exist_ok=True)
+        # Wall clock persists across reboots (monotonic does not), so a cursor left in
+        # ~/.cache from a previous boot cannot turn into a huge stale wait.
         with open(self._path, "a+") as f:
             fcntl.flock(f.fileno(), fcntl.LOCK_EX)
             try:
                 f.seek(0)
                 data = f.read().strip()
-                now = time.monotonic()
+                now = time.time()
                 slot = max(now, float(data) if data else 0.0)
                 f.seek(0)
                 f.truncate()
@@ -59,7 +63,7 @@ _creation_limiters: dict[str, CreationLimiter] = {}
 
 
 def creation_limiter(per_sec: float | None, name: str) -> CreationLimiter | None:
-    """A host-global limiter pacing `name`'s creation to `per_sec`/s (None/<= 0 disables).
+    """A user-global limiter pacing `name`'s creation to `per_sec`/s (None/<= 0 disables).
 
     All callers (and processes) sharing a `name` share one bucket, so use one rate per name."""
     if not per_sec or per_sec <= 0:

verifiers/v1/runtimes/modal.py

@@ -53,7 +53,7 @@ class ModalConfig(BaseConfig):
     """Disk in GB. Modal sandboxes have no disk knob, so this is accepted (so a task can
     declare it without a warning) but not enforced."""
     creates_per_sec: float | None = 40.0
-    """Pace sandbox creation to this many per second, enforced host-wide across every
+    """Pace sandbox creation to this many per second, enforced user-wide across every
     env-server worker process (None/<= 0 disables it)."""
 
 

verifiers/v1/runtimes/prime.py

@@ -66,7 +66,7 @@ class PrimeConfig(NetworkPolicyConfig):
     idle_timeout: float | None = 3600
     """Seconds of inactivity before the sandbox self-deletes (None disables)."""
     creates_per_min: int | None = None
-    """Pace sandbox creation to this many per minute, enforced host-wide across every
+    """Pace sandbox creation to this many per minute, enforced user-wide across every
     env-server worker process (None/<= 0 disables it). (Tunnel creation is limited separately
     and globally — see interception.tunnel.prime.TUNNEL_LIMITER.)"""
 

{verifiers-0.2.2.dev94.dist-info → verifiers-0.2.2.dev96.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: verifiers
-Version: 0.2.2.dev94
+Version: 0.2.2.dev96
 Summary: Verifiers: Environments for LLM Reinforcement Learning
 Project-URL: Homepage, https://github.com/primeintellect-ai/verifiers
 Project-URL: Documentation, https://github.com/primeintellect-ai/verifiers

{verifiers-0.2.2.dev94.dist-info → verifiers-0.2.2.dev96.dist-info}/RECORD

@@ -52,7 +52,7 @@ verifiers/envs/experimental/composable/harnesses/__init__.py,sha256=cL29z-h8RY8v
 verifiers/envs/experimental/composable/harnesses/mini_swe_agent.py,sha256=-oJ4Nz2DZOzMEbVtxgcGvnzAZYXt-pdz3B-6hNsoU2A,8882
 verifiers/envs/experimental/composable/harnesses/opencode.py,sha256=u7raPlUrAL2C_nX4WTAt-1dWkEVR5iLeQhvjXJB3Pkk,10338
 verifiers/envs/experimental/composable/harnesses/prompt.txt,sha256=dpQFIXxhad0EFZ0SeYuRvt5upTrhfb-d4BxfRL-i0OY,1633
-verifiers/envs/experimental/composable/harnesses/rlm.py,sha256=aIjYcvjhp4C7RWcJQ2Hb-dhYU5lcZ8V-e4YYZXZnh24,13081
+verifiers/envs/experimental/composable/harnesses/rlm.py,sha256=keRKbNK1zsCqxewlbp-8cCQHgC2L90GyPJxNOcp7EY8,13095
 verifiers/envs/experimental/composable/tasksets/__init__.py,sha256=5VKUn5W11Ekx6Issyg0l5rs1n0kNapJYPAIpVnaNBsg,1995
 verifiers/envs/experimental/composable/tasksets/cp/__init__.py,sha256=Aj5uBcEbzkgmBz5ZX7Wl044fEJcFxS0q0CZqs-SU75s,92
 verifiers/envs/experimental/composable/tasksets/cp/cp_task.py,sha256=aODpFS85pVqArflN0IZ22dA-u_oGH_cB4X-RjvFnuuo,7882
@@ -92,7 +92,7 @@ verifiers/envs/experimental/harbor_env/env.py,sha256=ObexUEWqPRDnjbnDmDi4zo4frWK
 verifiers/envs/experimental/harbor_env/mcp.py,sha256=_XrwnqnkIHUp_GA6sMua5cMTLm0boD-EJulQ8KqVTxw,12343
 verifiers/envs/experimental/utils/__init__.py,sha256=MLspkUZeGYH3TNv8FXIq2knBEnUAS8bvkK4pMXeKs7M,575
 verifiers/envs/experimental/utils/file_locks.py,sha256=S6e4Z8lFSepp-Bbvl3xA4uT9WbcuyI_6ZcV8k0q9DXc,1424
-verifiers/envs/experimental/utils/git_checkout_cache.py,sha256=wt0EqNQGZMVVWr0ERY67dHunpoKfLSaXp09FPj0bhKI,12117
+verifiers/envs/experimental/utils/git_checkout_cache.py,sha256=iEZIJIbYV-6RBleHoPZPHQ8zDIPiZBGcH3RSdYRxjf4,12139
 verifiers/envs/integrations/README.md,sha256=trk8lpwTq7uCQBIbid8aNa4O0Q5YnQWmx-X7xQsZQHM,7226
 verifiers/envs/integrations/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 verifiers/envs/integrations/openenv_env.py,sha256=MrthKsss3QmuphyW4cHLH14yXBGilRiXtZs4Vn5rzIw,35745
@@ -156,7 +156,7 @@ verifiers/utils/interception_utils.py,sha256=JFppjEGmx2bWcX8VOMjd786DcStkt0xhBTY
 verifiers/utils/logging_utils.py,sha256=ORhY77WdhjhExTC8ckXBMubVt-bLwzETYKzZL-zuH9U,8529
 verifiers/utils/message_utils.py,sha256=c1orN5CVFSK4Hk2ai__LhBV4xboZcSyyWtpltAt15w4,17884
 verifiers/utils/metric_utils.py,sha256=sdjoixAwjcRBfRHPKH33RSrPB8HVpLXlGa9CzvB8Z-o,6825
-verifiers/utils/path_utils.py,sha256=DU1xOkGUWuQZ6d4LoDI8Thh5blmPCSa3OQXQP67YKLg,6077
+verifiers/utils/path_utils.py,sha256=s0Nnx2x2sXqDQ5Ml7vVwteLHnXa8ZsFAgVevF2NON28,6401
 verifiers/utils/pricing_utils.py,sha256=V5FaqVUOL02z7JDy4NtPe98UZ9OC-5Sd0oMHSZlwwvg,5156
 verifiers/utils/process_utils.py,sha256=tFizSYGIUFieg4eGP3dYM2EonB2qmlCzl5EZX8qjmak,2873
 verifiers/utils/response_utils.py,sha256=E3CrgZ9N_6pCQGxQEI78xAHH0LUGK6utuI01g5Q1NEk,5579
@@ -192,16 +192,17 @@ verifiers/v1/cli/init.py,sha256=Qj96Kwzfhaejhrc0yYqHNRl3ncoID028RGlHvS1C98Q,8508
 verifiers/v1/cli/output.py,sha256=JiNjMO2kwM4p3zMPg3k4spuJbbE7-jOAtYBttPap7qI,6045
 verifiers/v1/cli/replay.py,sha256=DAh9kB5PB-ppZLGhxdJwrCPch_Yky5HKhplwGFhKEz0,10164
 verifiers/v1/cli/resolve.py,sha256=OsNw4C3r9xOVFa-mminJFcex_bAsJ6EQ8K7A_H8u2kQ,4370
-verifiers/v1/cli/validate.py,sha256=7ax6FqIzNBSYjd3JXK4v7Vbq34Rozyh1OvGpYnGqlSg,10266
+verifiers/v1/cli/resume.py,sha256=eW9EJU-l4pXufyuNA5nLXMdeRPcbdU6_YMDnYr7_R5I,1422
+verifiers/v1/cli/validate.py,sha256=f-6gJPoSvs5psTKirZQ3VjP4NvjJdFxY2EFnV0gkUfk,18206
 verifiers/v1/cli/dashboard/__init__.py,sha256=v-baMxQuWxOCsbU7-p_jj2Q9BUnTN-TWi1q2hK6rU2s,198
 verifiers/v1/cli/dashboard/base.py,sha256=kUP93zJSIVLptSbnWX6MOy8kGg63fpeAzPqakCpPyDc,3547
 verifiers/v1/cli/dashboard/eval.py,sha256=4j2YCPyzNHDT7d8MwqYR3C906lzwDVAd3FxgFYFgsw0,33588
 verifiers/v1/cli/dashboard/replay.py,sha256=CvRVaf0dUum5v0IzPQbFPPikBQmTrkMeaunGfVMTWHc,2755
 verifiers/v1/cli/dashboard/validate.py,sha256=rLsQ_31DjTIZpC_VO7zSG41ncUEBxqCx8mL9nTnGiMc,3650
 verifiers/v1/cli/eval/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
-verifiers/v1/cli/eval/main.py,sha256=xiMMIIbsy77SEOryHXOQyswS9WyrPKTyBwW65W9WTjI,5511
-verifiers/v1/cli/eval/resume.py,sha256=QwXuLPs2lN-aMZk1CyaT12egRPzeI6vfY39DAd0nYLU,7171
-verifiers/v1/cli/eval/runner.py,sha256=iR_Bapy5r0IC43GpzlfKIO9RAzFmKcY6EMcwYbEJupY,12181
+verifiers/v1/cli/eval/main.py,sha256=12eDekY8IsJkwiodHepf8pxqYulO2uSp8_Z29xIPXA4,5554
+verifiers/v1/cli/eval/resume.py,sha256=tFMAUHSlnNgfEbh14vc2uGI6jzWi7pVx0NNK61jmpiQ,5644
+verifiers/v1/cli/eval/runner.py,sha256=ufWTbONCAuIU-c0-DluYU_EnbRoEivYVsJ-kHTuHjf4,12184
 verifiers/v1/clients/__init__.py,sha256=Ysig0tE_0E4Jsfgfes1XHN-fK1s_RCXdqZD6E7lK4PU,507
 verifiers/v1/clients/base.py,sha256=hhtTumKC7lgI6cj7Hks91DCjqNTwAh40DD_j8CWGd6c,1560
 verifiers/v1/clients/client.py,sha256=L4axOl-dF9_MSYagB_xAO5qbeE53jNZJnB39VLdVBwo,3156
@@ -224,7 +225,7 @@ verifiers/v1/configs/cli/env.py,sha256=Qwba1wD2Upbh2W09YbSpktueCJZaWV_DymHRaPgl_
 verifiers/v1/configs/cli/eval.py,sha256=HqIBtt8DuZEHZ07KgtE1jiqZy8OsDMfTahTuBAvslcA,6863
 verifiers/v1/configs/cli/init.py,sha256=tMaTDF_lU1Af3JmQpksk2mwe8CAfpnHsquXhwepZhYo,1019
 verifiers/v1/configs/cli/replay.py,sha256=xvMDZzIkvjgYl-a3ragjXI65DsD6FYl6gEFM5qpdIuA,2913
-verifiers/v1/configs/cli/validate.py,sha256=UdgC6bQ2RYx3t8n9HjsImAHV_ifoOxYOSlbtdZzOnZY,2298
+verifiers/v1/configs/cli/validate.py,sha256=Yy3bJdCHdnLblaG-l84xAFHg9YgHfg2i-gVwH1hdOsg,3048
 verifiers/v1/dialects/__init__.py,sha256=p72C7VYrFQzVgTcfmzpHGSwXlvThfc4pvjBxsqw7kHE,786
 verifiers/v1/dialects/anthropic.py,sha256=nj9J7OOhexmPHa-sWiNP2xlI9_bGhvwM5VWzdh19pBc,13526
 verifiers/v1/dialects/base.py,sha256=YZQDnasQseDXK-GHLfJLZgyHIIygS80YhD_sTjmUbfA,8126
@@ -287,7 +288,7 @@ verifiers/v1/interception/server.py,sha256=jJmAkzQpDOEQggJNwCQlJlboMZFrsoF0YpuL7
 verifiers/v1/interception/tunnel/__init__.py,sha256=eVNZJszj6myrKhx9jmJz7VRd9qlfj0AUvUZwI2bXEj8,893
 verifiers/v1/interception/tunnel/base.py,sha256=DZB6uPLwM4Qy7n7m0vKeyg-Px87MhsmpYTpe2vpPEng,1998
 verifiers/v1/interception/tunnel/custom.py,sha256=yL4UbGf4bAuMsxk42mEP-t1wITQCT8BE9FHQIz0qRMA,1708
-verifiers/v1/interception/tunnel/prime.py,sha256=PP3Be3PfbfsLlhF4FfpsZt7IkiYiC1DBF-EidlKQio4,2530
+verifiers/v1/interception/tunnel/prime.py,sha256=0CjoQ2YoxwRUFRU1BftNgqGvqFlbM5is8KaWGrD7eQ0,2531
 verifiers/v1/judges/__init__.py,sha256=MUIBWcx6c70BykrTDHhB0ITIyJPxXe6xDBou4VM7jDQ,286
 verifiers/v1/judges/reference.py,sha256=iEVHw-iJ6dbwLOOqrvWrEMJOcUEYJ0YspT04rZz7Sd4,3868
 verifiers/v1/judges/reference.txt,sha256=Ej35kGXiT2uJ0LcHIeez49rCS_9uHmCQKoVdn93V6A8,353
@@ -299,9 +300,9 @@ verifiers/v1/mcp/server.py,sha256=dRBxytFgS6rv1BS6t5EgBd3xeF6U1nTyt0ArDCLk3VE,12
 verifiers/v1/mcp/toolset.py,sha256=T8Ioiq4wphxVOTHUDf28ZoQBM2ZYEevGnGgw8TFEMiA,1002
 verifiers/v1/runtimes/__init__.py,sha256=Tt1x_rS08R_5f6vkqW0EmrhXRXQAp23czHyoFrWSPHM,2671
 verifiers/v1/runtimes/base.py,sha256=lqTTnJk-NcN18QTP9lDIqkSKRCw8lK_jZXN1THpGl98,17175
-verifiers/v1/runtimes/limiters.py,sha256=ZjjYUFEt20yH8ggL5uU9w9U500AOB44qSUTQ7bYyh0c,2730
-verifiers/v1/runtimes/modal.py,sha256=QMNVQtiTTN6ZDiS-TA3nJZGfZw_HVlkuxHvPx55Yk8Q,12591
-verifiers/v1/runtimes/prime.py,sha256=QtDtkZITorFdHyWSr-i37RB7HStZ9pxp-I-I6zkkZmA,16240
+verifiers/v1/runtimes/limiters.py,sha256=F8HD3Sbn7XFum9cblUzvtDnyVKA1LnoYwi_J-GHUIzQ,2814
+verifiers/v1/runtimes/modal.py,sha256=XLCXLXYee7-6xY9kg2CFd-o83ztwYZESW1WAa5bJZ3A,12591
+verifiers/v1/runtimes/prime.py,sha256=FZ0ePwcpooWcviUbobbpJjE9bF_61cXGe4BWjCYlMqw,16240
 verifiers/v1/runtimes/subprocess.py,sha256=4WBdLJrDF347FDGqTcazBVb2g_jRw0Q0r3AlCbqy9PE,7552
 verifiers/v1/runtimes/docker/__init__.py,sha256=LvClYPNVchcrnW2wVbOhi8-bzXUu3E4hdwZv2iNMrXQ,20346
 verifiers/v1/runtimes/docker/egress.py,sha256=37UXgNX9gnoLxjMg4KZaJIGoApCAM6MK22eJGNoMi38,14514
@@ -339,8 +340,8 @@ verifiers/v1/utils/platform.py,sha256=56Ixmk1SER6q5LvzyYcA0hgzGpzhxKZL8Hiqp4c_XV
 verifiers/v1/utils/retries.py,sha256=Y2ZgrAjn-qNkRKZP_RVNL_7EK0iaRcZeCa404lpGYi4,5417
 verifiers/v1/utils/score.py,sha256=-R5Cog14r_tJ6Q_oOfGr5VPAm2CCUhofYZB6B9lm4wM,5787
 verifiers/v1/utils/version.py,sha256=-obEo_-l9-D8FLef4hYxncOe-uJpxrM1g2Hig_37Sgs,1607
-verifiers-0.2.2.dev94.dist-info/METADATA,sha256=Apenwhh-Aj7Nzm8EIVuzl1-UHCvpksZZQSOnGmCg4rE,4545
-verifiers-0.2.2.dev94.dist-info/WHEEL,sha256=lCkmxWfQsSc9CfIClYeavTdQeEX2toPqufh9gI35EQA,87
-verifiers-0.2.2.dev94.dist-info/entry_points.txt,sha256=v6v0QT9vVExnfn4br42MostI_o3f3ZbWYlTormz-U3g,515
-verifiers-0.2.2.dev94.dist-info/licenses/LICENSE,sha256=v0RrUsdV3IDoZhrRce297IXS3xMHNJ-_LdLpFAUWb9k,1072
-verifiers-0.2.2.dev94.dist-info/RECORD,,
+verifiers-0.2.2.dev96.dist-info/METADATA,sha256=PcK9Bjal72b0WZkEVAOhTTlLqkJUhtpMvEl2xhEJlPA,4545
+verifiers-0.2.2.dev96.dist-info/WHEEL,sha256=lCkmxWfQsSc9CfIClYeavTdQeEX2toPqufh9gI35EQA,87
+verifiers-0.2.2.dev96.dist-info/entry_points.txt,sha256=v6v0QT9vVExnfn4br42MostI_o3f3ZbWYlTormz-U3g,515
+verifiers-0.2.2.dev96.dist-info/licenses/LICENSE,sha256=v0RrUsdV3IDoZhrRce297IXS3xMHNJ-_LdLpFAUWb9k,1072
+verifiers-0.2.2.dev96.dist-info/RECORD,,