verifiers 0.2.2.dev4__py3-none-any.whl → 0.2.2.dev6__py3-none-any.whl

verifiers/v1/__init__.py

@@ -72,6 +72,11 @@ from verifiers.v1.scoring import (
 )
 from verifiers.v1.retries import RetryConfig, RolloutRetryConfig
 from verifiers.v1.rollout import Rollout
+from verifiers.v1.utils.git import (
+    PATCH_CAP_BYTES as PATCH_CAP_BYTES,
+    capture_patch as capture_patch,
+    resolve_head as resolve_head,
+)
 from verifiers.v1.runtimes import (
     DockerConfig,
     PrimeConfig,
@@ -260,6 +265,10 @@ __all__ = [
     "RubricJudge",
     "RubricJudgeConfig",
     "Criterion",
+    # git patch capture
+    "PATCH_CAP_BYTES",
+    "capture_patch",
+    "resolve_head",
     # scoring
     "compare_stdout_results",
     "extract_boxed_answer",

verifiers/v1/utils/git.py

@@ -0,0 +1,113 @@
+"""Persist an agent's final git patch into `trace.info` at finalize time.
+
+SWE-style tasksets call `capture_patch` from `Task.finalize` — after the harness
+finishes, while the runtime is live, before scoring mutates the repo (restoring
+test files, switching commits) — so the diff is exactly what the agent produced,
+including edits to test files (intentional: they reveal reward hacking).
+
+The diff is taken against `base_commit` when the caller has one — a dataset row
+field, or a SHA recorded with `resolve_head` at setup time and kept in host
+memory (never the sandbox, where an agent could tamper with it) — so commits the
+agent made are included. Bare `HEAD` is the fallback of last resort and misses
+agent commits.
+"""
+
+from __future__ import annotations
+
+import uuid
+from typing import TYPE_CHECKING
+
+if TYPE_CHECKING:
+    from verifiers.v1.runtimes import Runtime
+    from verifiers.v1.trace import Trace
+
+PATCH_CAP_BYTES = 2_000_000
+"""Truncate captured patches beyond this size; `info["patch_truncated"]` marks it."""
+
+# Temp paths are suffixed per invocation with a host-generated nonce: fixed names
+# would let concurrent rollouts on a shared-filesystem runtime (subprocess) read
+# each other's patches between the write and the read, and would let an agent
+# pre-create the predictable path as a FIFO so the shell's redirect blocks the
+# rollout forever.
+_FULL = "/tmp/vf_agent_patch_full"
+_CAPPED = "/tmp/vf_agent_patch"
+
+# `git reset -q` must run even when staging or diffing fails, or the error path
+# leaves the tree staged and can break scoring's later checkouts. Every step
+# reports: a failure in add, diff, reset, or head fails the capture. A failed
+# add (e.g. a stale index.lock from a killed agent git command) leaves a stale
+# index, so letting the diff's clean exit stand would record a silently
+# incomplete patch; a failed reset (e.g. ENOSPC rewriting .git/index) leaves the
+# tree staged, so reporting success would hide a state later scoring may trip
+# on; a failed head leaves an empty {capped} (the redirect truncates it before
+# head runs), which would read back as a silently empty patch.
+_DIFF = (
+    "rm -f {full} {capped}; "
+    "git add -A; "
+    "add_rc=$?; "
+    'git -c core.quotepath=off diff --cached --binary "$VF_DIFF_BASE" > {full}; '
+    "diff_rc=$?; "
+    "git reset -q; "
+    "reset_rc=$?; "
+    "head -c {cap} {full} > {capped}; "
+    "head_rc=$?; "
+    "rm -f {full}; "
+    "rc=$head_rc; "
+    '[ "$diff_rc" -ne 0 ] && rc=$diff_rc; '
+    '[ "$reset_rc" -ne 0 ] && rc=$reset_rc; '
+    '[ "$add_rc" -ne 0 ] && rc=$add_rc; '
+    'exit "$rc"'
+)
+
+
+async def resolve_head(runtime: Runtime, env: dict | None = None) -> str:
+    """The repo's current commit SHA, or "" when unresolvable.
+
+    Call at the end of `setup`, before the agent runs, and keep the result in
+    host memory (e.g. a dict on the task keyed by `id(runtime)`) for `finalize`
+    to pass as `base_commit` — diffing against a pre-agent SHA is what keeps
+    commits the agent makes inside the captured patch.
+    """
+    result = await runtime.run(["git", "rev-parse", "HEAD"], env or {})
+    if result.exit_code != 0:
+        return ""
+    return (result.stdout or "").strip()
+
+
+async def capture_patch(
+    trace: Trace, runtime: Runtime, base_commit: str = "", env: dict | None = None
+) -> None:
+    """Snapshot the agent's cumulative diff into `trace.info["patch"]`.
+
+    Best-effort by design: a rollout whose sandbox died or whose repo state is
+    broken records `info["patch_error"]` instead of failing the rollout —
+    scoring still runs and the error stays visible in results.
+    """
+    nonce = uuid.uuid4().hex
+    full, capped = f"{_FULL}_{nonce}", f"{_CAPPED}_{nonce}"
+    cmd = _DIFF.format(full=full, capped=capped, cap=PATCH_CAP_BYTES + 1)
+    try:
+        result = await runtime.run(
+            ["sh", "-c", cmd],
+            {**(env or {}), "VF_DIFF_BASE": base_commit or "HEAD"},
+        )
+        if result.exit_code != 0:
+            trace.info["patch_error"] = (
+                f"exit={result.exit_code} {(result.stderr or '').strip()[-500:]}"
+            )
+            return
+        raw = await runtime.read(capped)
+    except Exception as exc:  # noqa: BLE001 - capture must never fail the rollout.
+        trace.info["patch_error"] = f"{type(exc).__name__}: {exc}"
+        return
+    finally:
+        # Unique names don't overwrite each other, so leftovers would accumulate
+        # on shared-filesystem runtimes; removal is best-effort by design.
+        try:
+            await runtime.run(["rm", "-f", full, capped], env or {})
+        except Exception:  # noqa: BLE001,S110 - cleanup must never fail the rollout.
+            pass
+    if len(raw) > PATCH_CAP_BYTES:
+        raw = raw[:PATCH_CAP_BYTES]
+        trace.info["patch_truncated"] = True
+    trace.info["patch"] = raw.decode("utf-8", errors="replace")

{verifiers-0.2.2.dev4.dist-info → verifiers-0.2.2.dev6.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: verifiers
-Version: 0.2.2.dev4
+Version: 0.2.2.dev6
 Summary: Verifiers: Environments for LLM Reinforcement Learning
 Project-URL: Homepage, https://github.com/primeintellect-ai/verifiers
 Project-URL: Documentation, https://github.com/primeintellect-ai/verifiers

{verifiers-0.2.2.dev4.dist-info → verifiers-0.2.2.dev6.dist-info}/RECORD

@@ -168,7 +168,7 @@ verifiers/utils/threaded_sandbox_client.py,sha256=Pbr8MA4FDPEitL6z88S8T1qLJOmtXN
 verifiers/utils/tool_utils.py,sha256=gInWZODWQUZN4TyEMuuITyOUm9qlHJxtcvr1zZl_zJs,1020
 verifiers/utils/usage_utils.py,sha256=GPLC0xGY_Obrr8X7huWY-2iODZ7tgX-MtO8WSDN4rXI,3904
 verifiers/utils/version_utils.py,sha256=am3hZLnlaUWTFdllGZR1VMN91hEhiiE8FDmPf1cOG1k,2642
-verifiers/v1/__init__.py,sha256=hjN9xg5SVAqjNHhoCVvqr3qpGfiQQ57FTpiqfwnbfTU,5863
+verifiers/v1/__init__.py,sha256=1crIo9cOEd05cuziZqwYr6CoviuR5tMenb8nYi6GhXk,6100
 verifiers/v1/decorators.py,sha256=tCgmFCta0gnEqoZArHnvLw97_vlg8AH35NlwBNEqeWM,4073
 verifiers/v1/env.py,sha256=zU3_zTYaVwJKFeg05mgYCKWEgySPKs4dhLbuIhd2WpQ,19292
 verifiers/v1/episode.py,sha256=emcXB_yD2KHNFRLyk8VFK2zZAAD_2difuyDeJJ046Yw,2153
@@ -298,6 +298,7 @@ verifiers/v1/utils/__init__.py,sha256=1RDUX9pjAuQHKYEDevehifHd3IB03y9LAUsln3exPQ
 verifiers/v1/utils/aio.py,sha256=ZKTHeURNbWhTpHMyYuqMLsdPWVHyn5anfG1IJs5y-Zg,1481
 verifiers/v1/utils/format.py,sha256=NfQo9M5KMzWCZpQaet5YtsJx56vCKAZPfbjZZJLJhhM,2187
 verifiers/v1/utils/generic.py,sha256=cpiyt_GIhFcHVK3dh7eBDqQJ6tQFXSQyHUuPKRLZfj4,647
+verifiers/v1/utils/git.py,sha256=5VX5B3011yhclPex8Mc79ZmZ5ZT4hGfKaXoqL12QMdc,4872
 verifiers/v1/utils/image.py,sha256=OFw_wdwVtbdwa8uJ3X3vYfhpUAi0CTH5HMBupjVsRRQ,282
 verifiers/v1/utils/install.py,sha256=fWNsyKrw_PyhC0Qhqv5Ri0adFm5Ddx4AVy_hbsra1GU,1390
 verifiers/v1/utils/interrupt.py,sha256=F-KKhc5ndPJJfhd3SuMqyqhXhA32FhCRy5KWFJpEoM4,1179
@@ -305,8 +306,8 @@ verifiers/v1/utils/logging.py,sha256=OcMHA6NsYux3oIzjPuI95rDWmFBHNcHDjZeNIhXTX-Y
 verifiers/v1/utils/memory.py,sha256=ZkIvGk6uITAH5sKon65LifKPbvZr8mJ__PVc23FZpOQ,1835
 verifiers/v1/utils/sampling.py,sha256=JczGzBn6s3wsIrSY2Hy3hE8m6NreNgsNzhSjDikQqX0,1037
 verifiers/v1/utils/version.py,sha256=75ZtI2NHBmlb52KpcKLUpiSXp8q4bASr7uKeXoCKlT8,1582
-verifiers-0.2.2.dev4.dist-info/METADATA,sha256=ZmUyYSoNBajAItnypg3IUt4GdNNfS-4T_Wtx5zsKJIU,4540
-verifiers-0.2.2.dev4.dist-info/WHEEL,sha256=lCkmxWfQsSc9CfIClYeavTdQeEX2toPqufh9gI35EQA,87
-verifiers-0.2.2.dev4.dist-info/entry_points.txt,sha256=dF82JUYEFslR1AOW8LZfuBWeZeQoyX4wCRrduklg8-I,551
-verifiers-0.2.2.dev4.dist-info/licenses/LICENSE,sha256=v0RrUsdV3IDoZhrRce297IXS3xMHNJ-_LdLpFAUWb9k,1072
-verifiers-0.2.2.dev4.dist-info/RECORD,,
+verifiers-0.2.2.dev6.dist-info/METADATA,sha256=tmKW4Ax2c5YNJDMpMyHodgJuE_x7SNSnp60ozWIOhvo,4540
+verifiers-0.2.2.dev6.dist-info/WHEEL,sha256=lCkmxWfQsSc9CfIClYeavTdQeEX2toPqufh9gI35EQA,87
+verifiers-0.2.2.dev6.dist-info/entry_points.txt,sha256=dF82JUYEFslR1AOW8LZfuBWeZeQoyX4wCRrduklg8-I,551
+verifiers-0.2.2.dev6.dist-info/licenses/LICENSE,sha256=v0RrUsdV3IDoZhrRce297IXS3xMHNJ-_LdLpFAUWb9k,1072
+verifiers-0.2.2.dev6.dist-info/RECORD,,