verifiers 0.3.2.dev55__py3-none-any.whl → 0.3.2.dev57__py3-none-any.whl

verifiers/v1/envs/agentic_judge/env.py

@@ -5,7 +5,9 @@ a fresh box from the solver's runtime policy and restores only the task's collec
 artifacts; `--env.id shared-agentic-judge` explicitly runs the judge in the
 solver's box. The judge grades rubric criteria (`[env.task]`: policy prompt,
 criteria file) and writes its verdicts to `/tmp/verdict.json`, with the solver's
-full trace record uploaded at `/tmp/trace.json`. `finalize()` validates them
+observable trace record uploaded at `/tmp/trace.json`. Hidden reasoning and opaque
+provider state are omitted by default and may be explicitly included through the
+judge task config. `finalize()` validates the verdicts
 strictly onto the solver's trace — `judge/<name>` metrics plus a weighted-mean
 `judge` reward, composed with the taskset's own rewards via `[env.score]`
 (judge-only by default).
@@ -82,15 +84,16 @@ def _render(template: str, **fields: str) -> str:
 
 
 _RECORD_NOTE = f"""\
-The agent's raw trace record (JSON: messages, tool calls, and its `info`
+The agent's observable trace record (JSON: messages, tool calls, and its `info`
 artifacts) is written by the harness — not the agent — at `{TRACE_FILE}`. The
 record can be very large — never dump it whole; peek selectively (list its
 keys, then slice out specific fields with python or jq) and pull only what you
-need. It is complete — it may also carry the task's own scores/metrics and
-reference material (a gold answer, a reference solution, held-out tests). Those
-are context, not your standard: recorded scores can be wrong and references can
-be narrower than the task; do not over-index on how a reference solves it. Your
-verdict is what YOU verified by execution."""
+need. Hidden reasoning and opaque provider state are not included unless the run
+explicitly opts in. The record may also carry the task's own scores/metrics and
+reference material (a gold answer, a reference solution, held-out tests). Those are
+context, not your standard: recorded scores can be wrong and references can be
+narrower than the task; do not over-index on how a reference solves it. Your verdict
+is what YOU verified by execution."""
 
 SHARED_WORKSPACE_NOTE = f"""\
 ## Your workspace
@@ -146,7 +149,13 @@ class JudgeTask(vf.Task):
         paths they had.
         """
         solved = solution.task.data
-        files = {TRACE_FILE: json.dumps(solution.to_record()).encode()}
+        record = solution.to_record()
+        if not config.include_hidden_reasoning:
+            for node in record["nodes"]:
+                message = node["message"]
+                message.pop("reasoning_content", None)
+                message.pop("provider_state", None)
+        files = {TRACE_FILE: json.dumps(record).encode()}
         template = config.build_prompt()
         body = _render(template, prompt=solved.prompt_text)
         if "{prompt}" not in template:
@@ -224,6 +233,9 @@ class JudgeTaskConfig(vf.BaseConfig):
     """Criteria the judge grades against: a `.toml`/`.json` file with a
     `criteria` list — the plugged rubric judge's format, so the same rubric
     files work for both. None grades the single built-in `solved` criterion."""
+    include_hidden_reasoning: bool = False
+    """Include assistant reasoning and opaque provider state in `/tmp/trace.json`.
+    Disabled by default so the judge grades only observable agent behavior."""
 
     @staticmethod
     def _resolve(value: TextSource) -> str:

verifiers/v1/harnesses/codex/harness.py

@@ -120,12 +120,6 @@ class CodexHarness(ACPHarness[CodexHarnessConfig]):
         mcp_urls: dict[str, str],
     ) -> dict[str, str]:
         home = self.trace_home(trace)
-        created = await runtime.run(["mkdir", "-p", home], {})
-        if created.exit_code != 0:
-            raise RuntimeError(
-                f"failed to create Codex home: {created.stderr.strip()[-500:]}"
-            )
-
         mcp_config = "features={mcp_2026_07_28=true}\n" + (
             "mcp_servers={"
             + ",".join(

{verifiers-0.3.2.dev55.dist-info → verifiers-0.3.2.dev57.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.5
 Name: verifiers
-Version: 0.3.2.dev55
+Version: 0.3.2.dev57
 Summary: Verifiers: Environments for LLM Reinforcement Learning
 Project-URL: Homepage, https://github.com/primeintellect-ai/verifiers
 Project-URL: Documentation, https://github.com/primeintellect-ai/verifiers

{verifiers-0.3.2.dev55.dist-info → verifiers-0.3.2.dev57.dist-info}/RECORD

@@ -66,7 +66,7 @@ verifiers/v1/dialects/chat.py,sha256=_pMsNpbffiXiK0_0tboAKtuj7-smUnkS9H0LPK4j1mU
 verifiers/v1/dialects/responses.py,sha256=KRuS9TKw2uoNz4NsG7E-H6eyhm_EJQwVtZ5_XW9iB3c,29090
 verifiers/v1/envs/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
 verifiers/v1/envs/agentic_judge/__init__.py,sha256=LfEcRHLG_siKzhzWmODDXWSLpLgfum3UFdQDK9XG0gk,325
-verifiers/v1/envs/agentic_judge/env.py,sha256=IZ8G8OgMI4P5JKJhXM0rrsBloWKgqUSzYiKHpqhEPDY,14639
+verifiers/v1/envs/agentic_judge/env.py,sha256=GovnHqgRm8nQLAM7wMx57-A_wJBRYopsfiy2Bgf32nA,15339
 verifiers/v1/envs/best_of_n/__init__.py,sha256=mubASiwXMNIhnQFDccNAc7yZ0OFsOVK8hH8WIx9b2A4,119
 verifiers/v1/envs/best_of_n/env.py,sha256=KEUJBCVqblnXP_lEwm3mjGdaikBMkecz8UjOFPBhxc0,1987
 verifiers/v1/envs/isolated_verifier/__init__.py,sha256=HZxc65f87pSIPRslwTZzs0I17wGa-UMM-c-CHliwPss,214
@@ -92,7 +92,7 @@ verifiers/v1/harnesses/browser_use/program.py,sha256=3dSFHSDIW3R5ZejV23sXReEKnGQ
 verifiers/v1/harnesses/claude_code/__init__.py,sha256=JkZQylMSA3o1fvBx_NM41y_umuahmOSqFjTrJ2hryTw,171
 verifiers/v1/harnesses/claude_code/harness.py,sha256=Racxbs1EAH3gRwSeUptAIa3kB-EUm_nreUnHMwRUHuE,4119
 verifiers/v1/harnesses/codex/__init__.py,sha256=ocyBvlpSO8c3pT6D1ebrQohoXYESQcCyPT5kN3j1-N4,132
-verifiers/v1/harnesses/codex/harness.py,sha256=c8Vht0LOba7xTxCXwFH2W9rRTRdAv8Eypc1e-uSxFvg,7050
+verifiers/v1/harnesses/codex/harness.py,sha256=trrOj9qMLkPWxnh-Iz87uGYuwEg97cwrt67PxHrayrk,6825
 verifiers/v1/harnesses/hermes_agent/__init__.py,sha256=k84Y8UTTf1KOw5LGqPb2dk-hYFagPevDJBOF7qsR_vo,176
 verifiers/v1/harnesses/hermes_agent/harness.py,sha256=dSsE2VZ8jaM-U14QXO3J7P-U6NMpSuOz3DUMtDWpWk0,3622
 verifiers/v1/harnesses/hermes_agent/program.py,sha256=RaIYm6NFD8KLXERGXp6rOE4QrQjlcTnm1Fu33xYsiHw,570
@@ -189,8 +189,8 @@ verifiers/v1/utils/prime.py,sha256=UTYRjp9cbjNb6CVmBHda-1wWZAIIfxNOmTyNuT7_wL4,9
 verifiers/v1/utils/retries.py,sha256=Y2ZgrAjn-qNkRKZP_RVNL_7EK0iaRcZeCa404lpGYi4,5417
 verifiers/v1/utils/score.py,sha256=493yJVMw8teCu9JxapxMFPFzI0hNUqdo0Y2nGW4kckk,6200
 verifiers/v1/utils/version.py,sha256=-obEo_-l9-D8FLef4hYxncOe-uJpxrM1g2Hig_37Sgs,1607
-verifiers-0.3.2.dev55.dist-info/METADATA,sha256=OtvXOPjwWbDjpndEQBpMA1sITQIWJYUVkkzNo0-3_X4,4204
-verifiers-0.3.2.dev55.dist-info/WHEEL,sha256=zOwg4jB6zX2kU910N-cMawjivD6tO8NEWvE12je1bVk,87
-verifiers-0.3.2.dev55.dist-info/entry_points.txt,sha256=uqQje0TMsr7k6nXlWxlPs13Si0evb3pVCEDF6c-YL80,241
-verifiers-0.3.2.dev55.dist-info/licenses/LICENSE,sha256=v0RrUsdV3IDoZhrRce297IXS3xMHNJ-_LdLpFAUWb9k,1072
-verifiers-0.3.2.dev55.dist-info/RECORD,,
+verifiers-0.3.2.dev57.dist-info/METADATA,sha256=0TbLIm4QKGNQoSm54MoeFlzZV4esh0OxCatJga_9d74,4204
+verifiers-0.3.2.dev57.dist-info/WHEEL,sha256=zOwg4jB6zX2kU910N-cMawjivD6tO8NEWvE12je1bVk,87
+verifiers-0.3.2.dev57.dist-info/entry_points.txt,sha256=uqQje0TMsr7k6nXlWxlPs13Si0evb3pVCEDF6c-YL80,241
+verifiers-0.3.2.dev57.dist-info/licenses/LICENSE,sha256=v0RrUsdV3IDoZhrRce297IXS3xMHNJ-_LdLpFAUWb9k,1072
+verifiers-0.3.2.dev57.dist-info/RECORD,,