verifiers 0.3.2.dev168__tar.gz → 0.3.2.dev170__tar.gz

{verifiers-0.3.2.dev168 → verifiers-0.3.2.dev170}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.5
 Name: verifiers
-Version: 0.3.2.dev168
+Version: 0.3.2.dev170
 Summary: Verifiers: Environments for LLM Reinforcement Learning
 Project-URL: Homepage, https://github.com/primeintellect-ai/verifiers
 Project-URL: Documentation, https://github.com/primeintellect-ai/verifiers
@@ -36,7 +36,7 @@ Requires-Dist: numpy>=2.1.0
 Requires-Dist: openai<3.0.0,>=2.54.0
 Requires-Dist: prime-pydantic-config[toml]>=0.4.3
 Requires-Dist: prime-runs>=0.1.3
-Requires-Dist: prime-sandboxes>=0.3.0
+Requires-Dist: prime-sandboxes>=0.4.1
 Requires-Dist: prime-tunnel>=0.1.8
 Requires-Dist: pydantic>=2.12.3
 Requires-Dist: pyzmq>=27.1.0

{verifiers-0.3.2.dev168 → verifiers-0.3.2.dev170}/pyproject.toml

@@ -33,7 +33,7 @@ dependencies = [
     "math-verify>=0.8.0",
     "openai>=2.54.0,<3.0.0",
     "prime-tunnel>=0.1.8",
-    "prime-sandboxes>=0.3.0",
+    "prime-sandboxes>=0.4.1",
     "prime-runs>=0.1.3",
     "pydantic>=2.12.3",
     "rich>=11.0.0",

verifiers-0.3.2.dev170/tests/v1/fixtures/agent_user_v1.py

@@ -0,0 +1,82 @@
+"""agent-user-v1: a scripted, model-free agent that runs as a non-root user and tries root actions.
+
+Task setup runs as root: it creates the task's `agent_user` (images normally ship one),
+plants a root-only secret, and hands the workdir to that user. The scripted harness then runs
+as that user, attempts four actions only root may take, and records each outcome in the
+workdir. The reward is 1 only if every root action was denied while the agent still saw the
+secret and could write its workspace.
+Needs a runtime that can switch users (docker or prime).
+"""
+
+import json
+
+import verifiers.v1 as vf
+from verifiers.v1.clients import ModelContext
+from verifiers.v1.configs.harness import HarnessConfig
+from verifiers.v1.harness import Harness
+from verifiers.v1.runtimes import ProgramResult, Runtime
+
+__all__ = ["AgentUserTaskset", "ScriptedAgentUserHarness"]
+
+AGENT_USER = "vf-agent"
+SECRET = "/opt/vf-agent-user-secret"
+REPORT = "agent-user-report.json"
+ROOT_ACTIONS = ("read_secret", "write_etc", "chmod_secret", "signal_init")
+
+SETUP = (
+    f"useradd -m {AGENT_USER} && echo secret > {SECRET} && chmod 600 {SECRET} "
+    f"&& chown -R {AGENT_USER} ."
+)
+
+PROBE = f"""
+r() {{ "$@" >/dev/null 2>&1 && echo ok || echo denied; }}
+printf '{{"user":"%s","secret_exists":"%s","read_secret":"%s","write_etc":"%s","chmod_secret":"%s","signal_init":"%s","write_workdir":"%s"}}' \\
+  "$(id -un)" "$(r test -e {SECRET})" "$(r cat {SECRET})" "$(r touch /etc/vf-agent-user-probe)" \\
+  "$(r chmod 644 {SECRET})" "$(r kill -0 1)" "$(r touch probe)" > {REPORT}
+"""
+
+
+class ScriptedAgentUserHarness(Harness[HarnessConfig]):
+    async def launch(
+        self,
+        ctx: ModelContext,
+        trace: vf.Trace,
+        runtime: Runtime,
+        endpoint: str,
+        secret: str,
+        mcp_urls: dict[str, str],
+        data: vf.TaskData,
+        tool_interception_url: str | None = None,
+    ) -> ProgramResult:
+        return await runtime.run_program(["sh", "-c", PROBE], {})
+
+
+class AgentUserTask(vf.Task):
+    async def setup(self, trace: vf.Trace, runtime: Runtime) -> None:
+        result = await runtime.run(["sh", "-c", SETUP], {})
+        if result.exit_code != 0:
+            raise RuntimeError(f"root setup failed: {result.stderr}")
+
+    @vf.reward(weight=1.0)
+    async def root_actions_denied(self, trace: vf.Trace, runtime: Runtime) -> float:
+        report = json.loads(await runtime.read(REPORT))
+        trace.info["agent_user_report"] = report
+        return float(
+            report["user"] == AGENT_USER
+            and report["secret_exists"] == "ok"
+            and report["write_workdir"] == "ok"
+            and all(report[action] == "denied" for action in ROOT_ACTIONS)
+        )
+
+
+class AgentUserTaskset(vf.Taskset[AgentUserTask, vf.TasksetConfig]):
+    def load(self) -> list[AgentUserTask]:
+        return [
+            AgentUserTask(
+                vf.TaskData(
+                    idx=0,
+                    prompt="Scripted: try root-only actions as a non-root user.",
+                    agent_user=AGENT_USER,
+                )
+            )
+        ]

{verifiers-0.3.2.dev168 → verifiers-0.3.2.dev170}/tests/v1/test_e2e.py

@@ -190,6 +190,12 @@ USER_RUNTIMES = [
     pytest.param("e2b", marks=[mark.e2b], id="harness-in-e2b"),
 ]
 
+# Runtimes that can switch users (`Runtime.with_user`).
+AGENT_USER_RUNTIMES = [
+    pytest.param("docker", marks=[mark.docker], id="harness-in-docker"),
+    pytest.param("prime", marks=[mark.prime], id="harness-in-prime"),
+]
+
 # ACP-backed harnesses: each must preserve an exchange across interaction segments and
 # retain MCP access after resuming. Cover every harness in the local container runtime,
 # plus remote placements for the sandbox/tunnel and native-process boundaries.
@@ -332,6 +338,29 @@ async def test_browser_use(run_v1, tmp_path):
     assert trace.reward == 1.0
 
 
+@pytest.mark.e2e
+@pytest.mark.parametrize("harness_runtime", AGENT_USER_RUNTIMES, indirect=True)
+async def test_agent_user_cannot_act_as_root(run_v1, harness_runtime, tmp_path):
+    """A scripted, model-free agent runs as the task's non-root `agent_user`: it can see
+    the secret the root-run setup planted and write its own workspace, but every
+    root-only action (read the secret, write /etc, chmod the secret, signal PID 1) is
+    denied."""
+    (trace,) = await run_v1(
+        "agent-user-v1",
+        harness="agent-user-v1",
+        runtime={"type": harness_runtime},
+        output_dir=tmp_path,
+    )
+    report = trace.info["agent_user_report"]
+    assert report["user"] == "vf-agent"
+    assert report["secret_exists"] == "ok" and report["write_workdir"] == "ok"
+    root_actions = ("read_secret", "write_etc", "chmod_secret", "signal_init")
+    assert {action: report[action] for action in root_actions} == dict.fromkeys(
+        root_actions, "denied"
+    )
+    assert trace.reward == 1.0
+
+
 @pytest.mark.e2e
 @pytest.mark.parametrize("harness_runtime", USER_RUNTIMES, indirect=True)
 async def test_user(run_v1, harness_runtime, tmp_path):

{verifiers-0.3.2.dev168 → verifiers-0.3.2.dev170}/verifiers/v1/rollout.py

@@ -14,6 +14,7 @@ from verifiers.v1.configs.runtime import NetworkPolicyConfig
 from verifiers.v1.errors import (
     HarnessError,
     RolloutError,
+    SandboxError,
     TaskError,
     ToolsetError,
     boundary,
@@ -52,6 +53,39 @@ class RolloutTimeouts:
     """Timeout (in seconds) for the task + harness metrics + scoring hooks."""
 
 
+_HOME_PROBE = (
+    "home=$(awk -F: -v u=\"$1\" '$1 == u { print $6 }' /etc/passwd); "
+    'echo "$home"; test -n "$home" && test -d "$home"'
+)
+
+
+async def agent_runtime(runtime: Runtime, user: str | None) -> Runtime:
+    """The runtime the harness runs on: `runtime` itself, or a view running as `user`
+    with `HOME` set to its home. `user` must exist with a home directory it can write
+    (task authors ship it in the image), and the view must really run as it, so a
+    runtime ignoring the user never runs the agent as root."""
+    if user is None:
+        return runtime
+    lookup = await runtime.run(["sh", "-c", _HOME_PROBE, "sh", user], {})
+    home = lookup.stdout.strip()
+    if not home:
+        raise TaskError(f"agent_user {user!r} does not exist in the image")
+    if lookup.exit_code != 0:
+        raise TaskError(f"agent_user {user!r} has no home directory ({home})")
+    view = runtime.with_user(user)
+    view = view.with_env({**view.env, "HOME": home})
+    probe = await view.run(["sh", "-c", 'id -un; test -w "$HOME" && echo writable'], {})
+    effective, *rest = probe.stdout.split() or [""]
+    if effective != user:
+        raise SandboxError(
+            f"{type(runtime).__name__} ran as {effective!r} instead of agent_user "
+            f"{user!r}; refusing to run the agent"
+        )
+    if rest != ["writable"]:
+        raise TaskError(f"agent_user {user!r} cannot write its home directory ({home})")
+    return view
+
+
 class Rollout:
     """Manages one rollout's lifecycle (open, step, close)."""
 
@@ -341,7 +375,7 @@ class Rollout:
                     self._harness_session = await self.harness.session(
                         self.ctx,
                         self.trace,
-                        runtime,
+                        await agent_runtime(runtime, harness_data.agent_user),
                         self._endpoint,
                         self._secret,
                         self._urls,

{verifiers-0.3.2.dev168 → verifiers-0.3.2.dev170}/verifiers/v1/runtimes/base.py

@@ -136,13 +136,16 @@ class BaseRuntimeInfo(BaseConfig):
 
 
 class Runtime(ABC):
-    __slots__ = ("env",)
+    __slots__ = ("env", "user")
 
     is_local: bool = True
     """Whether this runtime exchanges host-local URLs without a public tunnel. True for
     subprocess and the local container runtimes; remote runtimes override to False and
     use a host `Tunnel` inward plus `expose` outward."""
 
+    supports_user: ClassVar[bool] = False
+    """Whether `with_user` can run this runtime's commands as another existing user."""
+
     scripts_dir: ClassVar[str] = "/tmp/vf-scripts"
     """Digest-keyed PEP 723 scripts inside the runtime. Sandboxes own their `/tmp`;
     the host subprocess runtime overrides this with its per-user cache."""
@@ -159,6 +162,8 @@ class Runtime(ABC):
         # Per-run task values live on the runtime rather than its serializable config/info.
         # Explicit process values (model credentials, proxy settings, etc.) override these.
         self.env: dict[str, str] = {}
+        self.user: str | None = None
+        """Existing user this view runs commands as; None uses the runtime's default."""
         self._uv_interpreters: dict[str, str] = {}
         self._uv_script_locks: dict[str, asyncio.Lock] = {}
         self._mcp_sources: set[str] = set()
@@ -215,6 +220,19 @@ class Runtime(ABC):
         runtime.env = dict(env)
         return runtime
 
+    def with_user(self, user: str | None) -> "Runtime":
+        """Share this physical runtime through a view that runs commands as `user`, an
+        existing user in the box (None keeps the default). Only the view runs as `user`:
+        setup, collection, and grading keep using the original runtime."""
+        if user is not None and not self.supports_user:
+            raise ValueError(
+                f"{type(self).__name__} cannot run commands as another user"
+            )
+        runtime = copy.copy(self)
+        runtime.__dict__ = self.__dict__
+        runtime.user = user
+        return runtime
+
     async def alive(self) -> bool:
         """Whether the box still executes anything. Not every runtime raises when
         the box is gone — some surface it as `exec`'s own non-zero result,

{verifiers-0.3.2.dev168 → verifiers-0.3.2.dev170}/verifiers/v1/runtimes/docker/__init__.py

@@ -70,6 +70,7 @@ control.sendmsg([b"listener"], [(socket.SOL_SOCKET, socket.SCM_RIGHTS, array.arr
 
 
 class DockerRuntime(ContainerRuntime):
+    supports_user: ClassVar[bool] = True
     engine: ClassVar[str] = "docker"
     """The CLI binary for the shared OCI container operations."""
     info_cls: ClassVar[type[BaseRuntimeInfo]] = DockerRuntimeInfo
@@ -529,6 +530,7 @@ class DockerRuntime(ContainerRuntime):
             "exec",
             *(("-i",) if stdin else ()),
             *(arg for key, value in env.items() for arg in ("--env", f"{key}={value}")),
+            *(("--user", self.user) if self.user is not None else ()),
             "--workdir",
             self.config.workdir,
             self._container,

{verifiers-0.3.2.dev168 → verifiers-0.3.2.dev170}/verifiers/v1/runtimes/prime.py

@@ -144,6 +144,7 @@ class PrimeProcess(RuntimeProcess):
 
 
 class PrimeRuntime(Runtime):
+    supports_user: ClassVar[bool] = True
     is_local: ClassVar[bool] = False
 
     def __init__(self, config: PrimeConfig, name: str | None = None) -> None:
@@ -292,6 +293,7 @@ class PrimeRuntime(Runtime):
                 shlex.join(argv),
                 working_dir=self.config.workdir,
                 env=self.process_env(env),
+                user=self.user,
             )
             delay = 0.1
             output_retries = 0
@@ -335,6 +337,7 @@ class PrimeRuntime(Runtime):
                 shlex.join(argv),
                 working_dir=self.config.workdir,
                 env=self.process_env(env),
+                user=self.user,
             )
         except Exception as e:
             raise SandboxError(f"prime live process failed to start: {e}") from e
@@ -356,6 +359,7 @@ class PrimeRuntime(Runtime):
                 command,
                 working_dir=self.config.workdir,
                 env=self.process_env(env),
+                user=self.user,
             )
         except Exception as e:
             raise SandboxError(f"prime background launch failed: {e}") from e
@@ -370,6 +374,7 @@ class PrimeRuntime(Runtime):
                     f"head -c {max_bytes} -- {shlex.quote(path)}",
                     working_dir=self.config.workdir,
                     env=self.process_env({}),
+                    user=self.user,
                 )
                 async with contextlib.aclosing(process):
                     with io.BytesIO() as data:

{verifiers-0.3.2.dev168 → verifiers-0.3.2.dev170}/verifiers/v1/task.py

@@ -99,6 +99,10 @@ class TaskData(BaseModel):
     """Optional Docker image to use for the task. Only relevant for tasks that run in a container."""
     workdir: str | None = None
     """Optional working directory to use for the task. Only relevant for tasks that run in a container."""
+    agent_user: str | None = None
+    """Existing user in the task's box that the harness, and so every command the agent
+    runs, executes as. None uses the runtime's default user (usually root). Task setup,
+    harness setup, artifact collection, and grading keep running as the default user."""
 
     skills: list[SkillSource] = Field(default_factory=list)
     """Skill sources installed before the harness's configured skills for this task."""

{verifiers-0.3.2.dev168 → verifiers-0.3.2.dev170}/verifiers/v1/tasksets/harbor/taskset.py

@@ -676,6 +676,7 @@ def parse_task(task_dir: Path, idx: int, harbor_config: HarborConfig) -> HarborD
         prompt=harbor_task.instruction.strip(),
         image=image,
         workdir=environment.workdir,
+        agent_user=None if parsed.agent.user is None else str(parsed.agent.user),
         network_allow=(
             ["*"]
             if network.network_mode == NetworkMode.PUBLIC

{verifiers-0.3.2.dev168 → verifiers-0.3.2.dev170}/verifiers/v1/utils/artifacts.py

@@ -190,7 +190,17 @@ async def _tar_out(runtime: Runtime, artifact: Artifact, budget: int) -> bytes:
         )
         # The runtime enforces the remaining collection budget while transferring the
         # bytes, so replacing or growing the archive cannot race a separate size probe.
-        return await runtime.read(path, max_bytes=budget)
+        archive, dropped = _drop_special_files(
+            await runtime.read(path, max_bytes=budget)
+        )
+        if dropped:
+            logger.warning(
+                "artifact %r: dropped %d FIFO or device node(s), e.g. %s",
+                artifact.source,
+                len(dropped),
+                dropped[:3],
+            )
+        return archive
     finally:
         # Best-effort: the box is about to be destroyed and the name is unique per call.
         try:
@@ -199,6 +209,32 @@ async def _tar_out(runtime: Runtime, artifact: Artifact, budget: int) -> bytes:
             logger.debug("failed to remove %s", path, exc_info=True)
 
 
+def _drop_special_files(archive: bytes) -> tuple[bytes, list[str]]:
+    """Remove FIFOs and device nodes, plus hard links to them, from a collected archive.
+
+    They carry no content, and `restore` refuses them, so one an agent leaves in its
+    workspace (e.g. a named pipe it made to test streaming input) would otherwise fail
+    the whole restore. Returns the archive and the dropped member names."""
+    with tarfile.open(fileobj=io.BytesIO(archive), mode="r:") as source:
+        members = source.getmembers()
+        special = {m.name for m in members if m.isfifo() or m.ischr() or m.isblk()}
+        if not special:
+            return archive, []
+        dropped = [
+            m.name
+            for m in members
+            if m.name in special or (m.islnk() and m.linkname in special)
+        ]
+        out = io.BytesIO()
+        with tarfile.open(fileobj=out, mode="w:", format=source.format) as target:
+            for member in members:
+                if member.name in dropped:
+                    continue
+                content = source.extractfile(member) if member.isfile() else None
+                target.addfile(member, content)
+    return out.getvalue(), dropped
+
+
 def _validate_restore(root: str, archive: bytes | None) -> None:
     """Reject archive content that could restore outside its declared root."""
     root_path = PurePosixPath(root)