verifiers 0.2.1.dev8__tar.gz → 0.2.1.dev11__tar.gz

{verifiers-0.2.1.dev8 → verifiers-0.2.1.dev11}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: verifiers
-Version: 0.2.1.dev8
+Version: 0.2.1.dev11
 Summary: Verifiers: Environments for LLM Reinforcement Learning
 Project-URL: Homepage, https://github.com/primeintellect-ai/verifiers
 Project-URL: Documentation, https://github.com/primeintellect-ai/verifiers

{verifiers-0.2.1.dev8 → verifiers-0.2.1.dev11}/pyproject.toml

@@ -80,7 +80,7 @@ examples = [
     "compact",
     "gsm8k-v1", "glossary-v1", "deepwiki-v1", "wiki-search-v1", "code-golf-v1",
     "reverse-text-v1", "color-codeword-v1",
-    "wordle-v1", "alphabet-sort-v1", "scratchpad-v1",
+    "wordle-v1", "alphabet-sort-v1", "scratchpad-v1", "mmmu-v1",
 ]
 
 [project.optional-dependencies]
@@ -148,6 +148,7 @@ wordle-v1 = { path = "environments/wordle_v1", editable = true }
 color-codeword-v1 = { path = "environments/color_codeword_v1", editable = true }
 alphabet-sort-v1 = { path = "environments/alphabet_sort_v1", editable = true }
 scratchpad-v1 = { path = "environments/scratchpad_v1", editable = true }
+mmmu-v1 = { path = "environments/mmmu_v1", editable = true }
 
 [tool.uv.exclude-newer-package]
 # PrimeIntellect-published on PyPI (trusted publisher)

{verifiers-0.2.1.dev8 → verifiers-0.2.1.dev11}/verifiers/v1/harnesses/codex/harness.py

@@ -3,13 +3,16 @@
 The static musl binary runs in Linux containers without additional runtime dependencies.
 """
 
+import base64
 import logging
+import re
 import shlex
 
 from verifiers.v1.clients import ModelContext
 from verifiers.v1.harness import Harness, HarnessConfig
 from verifiers.v1.runtimes import ProgramResult, Runtime
 from verifiers.v1.trace import Trace
+from verifiers.v1.types import TextContentPart
 
 logger = logging.getLogger(__name__)
 
@@ -42,6 +45,7 @@ class CodexHarnessConfig(HarnessConfig):
 class CodexHarness(Harness[CodexHarnessConfig]):
     APPENDS_SYSTEM_PROMPT = False  # TODO
     SUPPORTS_MCP = False  # TODO
+    SUPPORTS_MESSAGE_PROMPT = True
 
     async def setup(self, runtime: Runtime) -> None:
         logger.info("codex: ensuring codex %s is installed", self.config.version)
@@ -68,7 +72,53 @@ class CodexHarness(Harness[CodexHarnessConfig]):
         secret: str,
         mcp_urls: dict[str, str],
     ) -> ProgramResult:
-        _, prompt = self.resolve_prompt(trace.task.data)
+        task = trace.task.data
+        if (
+            task.system_prompt is not None
+            and task.prompt is not None
+            and not isinstance(task.prompt, str)
+        ):
+            system_prompt, prompt = task.system_prompt, task.prompt
+        else:
+            system_prompt, prompt = self.resolve_prompt(task)
+        image_args: list[str] = []
+        image_dir = f".vf-codex-images-{trace.id}"
+        if prompt is not None and not isinstance(prompt, str):
+            # Codex seeds one initial turn, so Messages system text joins its prompt.
+            texts = [system_prompt] if system_prompt else []
+            image_index = 0
+            for message in prompt:
+                if message.role not in ("system", "user"):
+                    raise ValueError(
+                        "codex exec only supports system and user initial messages"
+                    )
+                parts = (
+                    [TextContentPart(text=message.content)]
+                    if isinstance(message.content, str)
+                    else message.content
+                )
+                for part in parts:
+                    if isinstance(part, TextContentPart):
+                        texts.append(part.text)
+                        continue
+                    metadata, separator, encoded = part.image_url.url.partition(",")
+                    media_type, *parameters = metadata.removeprefix("data:").split(";")
+                    if (
+                        not separator
+                        or not metadata.startswith("data:image/")
+                        or not any(p.lower() == "base64" for p in parameters)
+                    ):
+                        raise ValueError(
+                            "codex image prompts require base64 data:image URLs"
+                        )
+                    extension = re.sub(
+                        r"[^a-zA-Z0-9]+", "_", media_type.removeprefix("image/")
+                    ).strip("_")
+                    path = f"{image_dir}/image_{image_index}.{extension or 'image'}"
+                    await runtime.write(path, base64.b64decode(encoded))
+                    image_args += ["-i", path]
+                    image_index += 1
+            prompt = "\n\n".join(texts)
         # codex authenticates to the interception server with the session secret (its provider
         # api key) and posts Responses calls to `{endpoint}/responses`.
         env = {**self.config.resolved_env, KEY_VAR: secret}
@@ -111,7 +161,18 @@ class CodexHarness(Harness[CodexHarnessConfig]):
             "-c",
             f"model_providers.{PROVIDER}.requires_openai_auth=false",
             *tool_config,
+            *image_args,
             "--",
             prompt,
         ]
-        return await runtime.run_program(argv, env)
+        try:
+            return await runtime.run_program(argv, env)
+        finally:
+            if image_args:
+                try:
+                    await runtime.run(["rm", "-rf", image_dir], {})
+                except Exception:
+                    # Runtime teardown is the fallback; preserve the rollout result.
+                    logger.warning(
+                        "failed to clean up Codex prompt images", exc_info=True
+                    )

{verifiers-0.2.1.dev8 → verifiers-0.2.1.dev11}/verifiers/v1/harnesses/default/harness.py

@@ -103,13 +103,16 @@ class DefaultHarness(Harness[DefaultHarnessConfig]):
                     }
                 )
             )
-        # A Messages prompt (e.g. an image-bearing prompt) seeds the chat loop directly via env
-        # (it can be large multimodal content that overflows argv); a plain string is the single
-        # first user message; None means the task has no prompt and the user simulator opens it.
         if isinstance(prompt, str):
             args.append(f"--prompt={prompt}")
         elif prompt is not None:
-            env["INITIAL_MESSAGES"] = json.dumps([message_to_wire(m) for m in prompt])
+            # Base64 images can exceed exec limits, so hand Messages off through a file.
+            path = f".vf-initial-messages-{trace.id}.json"
+            await runtime.write(
+                path,
+                json.dumps([message_to_wire(m) for m in prompt]).encode(),
+            )
+            args.append(f"--initial-messages-file={path}")
         program = await runtime.prepare_uv_script(
             PROGRAM_SOURCE, self.config.resolved_env
         )

{verifiers-0.2.1.dev8 → verifiers-0.2.1.dev11}/verifiers/v1/harnesses/default/program.py

@@ -7,7 +7,6 @@
 import argparse
 import asyncio
 import json
-import os
 import subprocess
 from contextlib import AsyncExitStack
 from pathlib import Path
@@ -244,6 +243,7 @@ def parse_args() -> argparse.Namespace:
     parser.add_argument("--model", required=True)
     parser.add_argument("--system-prompt", default="")
     parser.add_argument("--prompt", default="")
+    parser.add_argument("--initial-messages-file", default="")
     parser.add_argument("--mcp-config", default="")
     parser.add_argument("--edit", action="store_true")
     parser.add_argument("--search", action="store_true")
@@ -253,6 +253,12 @@ def parse_args() -> argparse.Namespace:
 
 async def main() -> None:
     args = parse_args()
+    initial = []
+    if args.initial_messages_file:
+        path = Path(args.initial_messages_file)
+        payload = path.read_bytes()
+        path.unlink()
+        initial = json.loads(payload)
     # No client-side retries: re-sending a request whose turn the interception already committed
     # re-samples it and forks the trace into an extra dead-end branch. A generous timeout lets a
     # slow turn finish instead of timing out and re-sending; genuine failures surface as error rows.
@@ -275,11 +281,6 @@ async def main() -> None:
             if args.system_prompt
             else []
         )
-        # A Messages prompt (e.g. an image-bearing prompt) arrives pre-built as OpenAI wire dicts
-        # via INITIAL_MESSAGES (kept in env: it can be large multimodal content that overflows
-        # argv, and it's prompt content, not a credential); otherwise --prompt is the opening
-        # message. Both empty means the task has no prompt — the user simulator seeds the opening.
-        initial = json.loads(os.environ.get("INITIAL_MESSAGES", "[]"))
         if initial:
             messages.extend(initial)
         elif args.prompt:

{verifiers-0.2.1.dev8 → verifiers-0.2.1.dev11}/verifiers/v1/harnesses/null/harness.py

@@ -54,13 +54,16 @@ class NullHarness(Harness[NullHarnessConfig]):
                     }
                 )
             )
-        # A Messages prompt (e.g. an image-bearing prompt) seeds the chat loop directly via env
-        # (it can be large multimodal content that overflows argv); a plain string is the single
-        # first user message; None means the task has no prompt and the user simulator opens it.
         if isinstance(prompt, str):
             args.append(f"--prompt={prompt}")
         elif prompt is not None:
-            env["INITIAL_MESSAGES"] = json.dumps([message_to_wire(m) for m in prompt])
+            # Base64 images can exceed exec limits, so hand Messages off through a file.
+            path = f".vf-initial-messages-{trace.id}.json"
+            await runtime.write(
+                path,
+                json.dumps([message_to_wire(m) for m in prompt]).encode(),
+            )
+            args.append(f"--initial-messages-file={path}")
         program = await runtime.prepare_uv_script(
             PROGRAM_SOURCE, self.config.resolved_env
         )

{verifiers-0.2.1.dev8 → verifiers-0.2.1.dev11}/verifiers/v1/harnesses/null/program.py

@@ -7,8 +7,8 @@
 import argparse
 import asyncio
 import json
-import os
 from contextlib import AsyncExitStack
+from pathlib import Path
 
 from openai import AsyncOpenAI
 
@@ -88,12 +88,19 @@ def parse_args() -> argparse.Namespace:
     parser.add_argument("--model", required=True)
     parser.add_argument("--system-prompt", default="")
     parser.add_argument("--prompt", default="")
+    parser.add_argument("--initial-messages-file", default="")
     parser.add_argument("--mcp-config", default="")
     return parser.parse_args()
 
 
 async def main() -> None:
     args = parse_args()
+    initial = []
+    if args.initial_messages_file:
+        path = Path(args.initial_messages_file)
+        payload = path.read_bytes()
+        path.unlink()
+        initial = json.loads(payload)
     client = AsyncOpenAI(base_url=args.base_url, api_key=args.api_key)
     config = json.loads(args.mcp_config or "{}")
     async with AsyncExitStack() as stack:
@@ -105,11 +112,6 @@ async def main() -> None:
             if args.system_prompt
             else []
         )
-        # A Messages prompt (e.g. an image-bearing prompt) arrives pre-built as OpenAI wire dicts
-        # via INITIAL_MESSAGES (kept in env: it can be large multimodal content that overflows
-        # argv, and it's prompt content, not a credential); otherwise --prompt is the opening
-        # message. Both empty means the task has no prompt — the user simulator seeds the opening.
-        initial = json.loads(os.environ.get("INITIAL_MESSAGES", "[]"))
         if initial:
             messages.extend(initial)
         elif args.prompt:

{verifiers-0.2.1.dev8 → verifiers-0.2.1.dev11}/verifiers/v1/serve/pool.py

@@ -69,27 +69,6 @@ def _arm_teardown(death_pipe=None) -> None:
     threading.Thread(target=_watch, daemon=True).start()
 
 
-def _worker_entry(
-    *, server_kwargs: dict, address: str, death_pipe, legacy: bool, log_setup=None
-) -> None:
-    """Spawned worker: an ordinary EnvServer/LegacyEnvServer bound to `address` (ipc).
-    A native config arrives as a dict (`config_data`): the eval/serve CLI's narrowed
-    config type is dynamic and unpicklable, so we rebuild it here via EnvConfig's
-    id-resolving validator. `log_setup` (if given) configures this fresh process's
-    logging so per-rollout logs surface — a spawned worker inherits no handlers."""
-    from verifiers.v1.legacy import LegacyEnvServer
-
-    _arm_teardown(death_pipe)
-    if log_setup is not None:
-        log_setup()
-    if "config_data" in server_kwargs:
-        server_kwargs = {
-            "config": EnvConfig.model_validate(server_kwargs["config_data"])
-        }
-    cls = LegacyEnvServer if legacy else EnvServer
-    cls.run_server(address=address, **server_kwargs)
-
-
 class EnvServerPool:
     """ROUTER broker that elastically scales worker processes (least-busy dispatch).
 
@@ -138,13 +117,14 @@ class EnvServerPool:
         address = f"ipc://{self._worker_path(i)}"
         parent_conn, child_conn = self._mpctx.Pipe()
         proc = self._mpctx.Process(
-            target=_worker_entry,
+            target=serve_env,
             kwargs=dict(
-                server_kwargs=self.server_kwargs,
+                max_workers=1,
                 address=address,
                 death_pipe=child_conn,
                 legacy=self.legacy,
                 log_setup=self.log_setup,
+                **self.server_kwargs,
             ),
             daemon=False,
         )
@@ -363,3 +343,6 @@ def serve_env(
             )
     except KeyboardInterrupt:
         pass
+    except Exception:
+        logger.exception("Env server failed")
+        raise