verifiers 0.1.15.dev401__tar.gz → 0.1.15.dev402__tar.gz

{verifiers-0.1.15.dev401 → verifiers-0.1.15.dev402}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: verifiers
-Version: 0.1.15.dev401
+Version: 0.1.15.dev402
 Summary: Verifiers: Environments for LLM Reinforcement Learning
 Project-URL: Homepage, https://github.com/primeintellect-ai/verifiers
 Project-URL: Documentation, https://github.com/primeintellect-ai/verifiers

{verifiers-0.1.15.dev401 → verifiers-0.1.15.dev402}/verifiers/envs/experimental/composable/tasksets/__init__.py

@@ -17,10 +17,6 @@ from verifiers.envs.experimental.composable.tasksets.swe.swe_tasksets import (
     make_swesmith_taskset,
     make_swesmith_ts_taskset,
 )
-from verifiers.envs.experimental.composable.tasksets.lean.lean_task import (
-    LEAN_SYSTEM_PROMPT,
-    LeanTaskSet,
-)
 from verifiers.envs.experimental.composable.tasksets.math.math_task import MathTaskSet
 from verifiers.envs.experimental.composable.tasksets.cp.cp_task import (
     CPRubric,
@@ -59,8 +55,6 @@ __all__ = [
     "make_swesmith_rs_taskset",
     "make_swesmith_cpp_taskset",
     "make_swesmith_php_taskset",
-    "LeanTaskSet",
-    "LEAN_SYSTEM_PROMPT",
     "MathTaskSet",
     "CPTaskSet",
     "CPRubric",

{verifiers-0.1.15.dev401 → verifiers-0.1.15.dev402}/verifiers/v1/tasksets/__init__.py

@@ -5,5 +5,18 @@ here — it imports the optional `textarena` dependency at module load — but s
 `verifiers.v1.tasksets.textarena`."""
 
 from verifiers.v1.tasksets.harbor import HarborConfig, HarborTaskset
+from verifiers.v1.tasksets.lean import (
+    LeanConfig,
+    LeanDatasetConfig,
+    LeanTask,
+    LeanTaskset,
+)
 
-__all__ = ["HarborConfig", "HarborTaskset"]
+__all__ = [
+    "HarborConfig",
+    "HarborTaskset",
+    "LeanConfig",
+    "LeanDatasetConfig",
+    "LeanTask",
+    "LeanTaskset",
+]

verifiers-0.1.15.dev402/verifiers/v1/tasksets/lean/__init__.py

@@ -0,0 +1,31 @@
+from verifiers.v1.tasksets.lean.scoring import (
+    build_starter_file,
+    expected_protected_signature,
+    parse_compile_output,
+    protected_signature_substring_present,
+    strip_lean_comments,
+)
+from verifiers.v1.tasksets.lean.taskset import (
+    DEFAULT_DOCKER_IMAGE,
+    LEAN_PROJECT_PATH,
+    PROOF_FILE_PATH,
+    LeanConfig,
+    LeanDatasetConfig,
+    LeanTask,
+    LeanTaskset,
+)
+
+__all__ = [
+    "DEFAULT_DOCKER_IMAGE",
+    "LEAN_PROJECT_PATH",
+    "PROOF_FILE_PATH",
+    "LeanConfig",
+    "LeanDatasetConfig",
+    "LeanTask",
+    "LeanTaskset",
+    "build_starter_file",
+    "expected_protected_signature",
+    "parse_compile_output",
+    "protected_signature_substring_present",
+    "strip_lean_comments",
+]

verifiers-0.1.15.dev402/verifiers/v1/tasksets/lean/scoring.py

@@ -0,0 +1,262 @@
+"""Pure helpers for the Lean taskset: starter-file construction, theorem-signature
+canonicalization, the reward-hacking signature guard, and compile-output parsing.
+
+Everything here is a pure function over plain strings — no runtime, no sandbox, no
+verifiers types — so it's trivially unit-testable and shared between ``setup``
+(building the starter file), the reward (the signature guard + compile parsing),
+and ``validate`` (planting the gold proof).
+"""
+
+from __future__ import annotations
+
+import re
+
+PROTECTED_HEADER_COMMENT = (
+    "-- DO NOT MODIFY the theorem statement below. The grader checks\n"
+    "-- that the original `theorem ... := by` text still appears in\n"
+    "-- this file. Only edit the proof body (currently `sorry`) and\n"
+    "-- lines after it."
+)
+
+
+# ── Imports / preamble normalization ─────────────────────────────────────────
+
+
+def normalize_imports(text: str) -> str:
+    """Collapse every ``import Mathlib*`` line to a single ``import Mathlib``.
+
+    Some datasets (miniF2F) ship fine-grained Mathlib imports that don't resolve
+    against the monolithic ``import Mathlib`` the sandbox image provides.
+    """
+    lines = text.split("\n")
+    out: list[str] = []
+    inserted = False
+    for line in lines:
+        if line.strip().startswith("import Mathlib"):
+            if not inserted:
+                out.append("import Mathlib")
+                inserted = True
+        else:
+            out.append(line)
+    return "\n".join(out)
+
+
+def _build_preamble(imports_str: str, header: str, normalize: bool) -> str:
+    if header and header.strip().startswith("import"):
+        preamble = header.strip()
+        return normalize_imports(preamble) if normalize else preamble
+    parts = [imports_str.strip()]
+    if header and header.strip():
+        parts.append(header.strip())
+    preamble = "\n\n".join(parts)
+    return normalize_imports(preamble) if normalize else preamble
+
+
+# ── Theorem signature canonicalization ───────────────────────────────────────
+
+
+def _normalize_signature(stmt: str) -> str:
+    """Canonicalize a Lean theorem statement to end with ``:= by``.
+
+    Strips trailing ``sorry``/``admit`` placeholders and any trailing ``by`` /
+    ``:=`` tokens, then re-appends `` := by``. Places the appended token on a new
+    indented line when the last line of the stripped signature already contains a
+    ``--`` comment (otherwise the ``:= by`` would land inside the line comment and
+    Lean would silently ignore it).
+    """
+    s = stmt.rstrip()
+    s = re.sub(r"\s*\b(?:sorry|admit)\b\s*$", "", s)
+    s = re.sub(r"\s*\bby\b\s*$", "", s)
+    s = re.sub(r"\s*:=\s*$", "", s)
+    s = s.rstrip()
+    last_newline = s.rfind("\n")
+    last_line = s[last_newline + 1 :] if last_newline != -1 else s
+    sep = "\n    " if "--" in last_line else " "
+    return s + sep + ":= by"
+
+
+def _split_imports_and_signature(stmt: str) -> tuple[str, str]:
+    decl_match = re.search(r"^(?:theorem|lemma|example)\s", stmt, flags=re.MULTILINE)
+    if not decl_match:
+        return "", stmt
+    return stmt[: decl_match.start()].rstrip(), stmt[decl_match.start() :]
+
+
+def expected_protected_signature(formal_statement: str) -> str:
+    """Return the canonical ``theorem ... := by`` block the reward pins as ground truth.
+
+    Pure function over ``formal_statement``; the imports/header portion (if the
+    statement carries one inline) is stripped first.
+    """
+    stmt = formal_statement or ""
+    if stmt.strip().startswith("import "):
+        _, signature_raw = _split_imports_and_signature(stmt)
+    else:
+        signature_raw = stmt
+    return _normalize_signature(signature_raw).strip()
+
+
+# ── Comment / string stripping for the signature guard ───────────────────────
+
+
+def strip_lean_comments(text: str) -> str:
+    """Remove Lean line/block comments **and string literals** from ``text``.
+
+    Lean comments come in two forms: ``-- ...`` line comments (to end of line) and
+    ``/- ... -/`` block comments (nestable; ``/-- ... -/`` doc comments are a
+    special case). String literals must also be stripped: a Lean
+    ``"theorem ... := by"`` constant or doc-string would otherwise let a model hide
+    the pinned signature inside a string while rewriting the live declaration to a
+    trivial one, defeating the substring guard. We handle both regular
+    double-quoted strings (with backslash escapes) and triple-quoted raw strings.
+
+    Block comments nest, so we count depth; line comments end at the next newline.
+    Outside comments and strings, newlines are preserved so the result keeps
+    roughly the right shape for substring matching downstream.
+    """
+    out: list[str] = []
+    i = 0
+    n = len(text)
+    block_depth = 0
+    in_line_comment = False
+    while i < n:
+        ch = text[i]
+        if in_line_comment:
+            if ch == "\n":
+                in_line_comment = False
+                out.append(ch)
+            i += 1
+            continue
+        if block_depth > 0:
+            if i + 1 < n and text[i : i + 2] == "-/":
+                block_depth -= 1
+                i += 2
+                continue
+            if i + 1 < n and text[i : i + 2] == "/-":
+                block_depth += 1
+                i += 2
+                continue
+            if ch == "\n":
+                out.append(ch)
+            i += 1
+            continue
+        if i + 1 < n and text[i : i + 2] == "/-":
+            block_depth = 1
+            i += 2
+            continue
+        if i + 1 < n and text[i : i + 2] == "--":
+            in_line_comment = True
+            i += 2
+            continue
+        # Triple-quoted raw string ``"""..."""`` — skip until the closing
+        # triple-quote, preserving newlines.
+        if i + 2 < n and text[i : i + 3] == '"""':
+            i += 3
+            while i < n:
+                if i + 2 < n and text[i : i + 3] == '"""':
+                    i += 3
+                    break
+                if text[i] == "\n":
+                    out.append("\n")
+                i += 1
+            continue
+        # Regular string ``"..."`` — handle ``\\`` escapes, stop at the next
+        # unescaped quote or newline (Lean strings are single-line).
+        if ch == '"':
+            i += 1
+            while i < n and text[i] != '"' and text[i] != "\n":
+                if text[i] == "\\" and i + 1 < n:
+                    i += 2
+                else:
+                    i += 1
+            if i < n and text[i] == '"':
+                i += 1
+            continue
+        out.append(ch)
+        i += 1
+    return "".join(out)
+
+
+def protected_signature_substring_present(
+    content: str, expected_signature: str
+) -> bool:
+    """True when the locked signature text still appears in the file.
+
+    Strips Lean comments from BOTH sides — without that a model could paste the
+    pinned signature into a ``--`` or ``/- ... -/`` block while rewriting the live
+    declaration to something trivial, and the asymmetric variant would also misfire
+    if ``expected_signature`` itself contains a comment. Tries an exact substring
+    match first, then a whitespace-flexible match (each side collapsed to
+    single-spaced tokens) so the model can re-indent or reflow whitespace freely.
+    """
+    if not expected_signature:
+        return True
+    decommented_content = strip_lean_comments(content)
+    decommented_expected = strip_lean_comments(expected_signature)
+    if not decommented_expected.strip():
+        return True
+    if decommented_expected in decommented_content:
+        return True
+    flat_signature = " ".join(decommented_expected.split())
+    flat_content = " ".join(decommented_content.split())
+    return flat_signature in flat_content
+
+
+# ── Starter-file construction ────────────────────────────────────────────────
+
+
+def build_starter_file(
+    formal_statement: str,
+    header: str = "",
+    imports: str = "import Mathlib",
+    normalize: bool = False,
+    proof_body: str | None = None,
+) -> str:
+    """Construct the starter proof file.
+
+    Layout: preamble (imports / header) + a brief ``-- DO NOT MODIFY`` comment
+    block + the normalized theorem signature + the proof body. If ``proof_body`` is
+    None (the default) the body is the placeholder ``  sorry`` — what's planted at
+    rollout start. A supplied gold ``proof_body`` (e.g. by ``validate``) replaces
+    the placeholder so the file is the full reference solution.
+    """
+    stmt = formal_statement or ""
+    if stmt.strip().startswith("import "):
+        imports_block, signature_raw = _split_imports_and_signature(stmt)
+        preamble = normalize_imports(imports_block) if normalize else imports_block
+    else:
+        preamble = _build_preamble(imports or "import Mathlib", header, normalize)
+        signature_raw = stmt
+
+    signature = _normalize_signature(signature_raw)
+    body = "  sorry" if proof_body is None else proof_body.rstrip()
+    wrapped = f"{PROTECTED_HEADER_COMMENT}\n{signature}\n{body}\n"
+    if preamble:
+        return preamble.rstrip() + "\n\n" + wrapped
+    return wrapped
+
+
+# ── Compile-output parsing ───────────────────────────────────────────────────
+
+
+def parse_compile_output(output: str) -> tuple[bool, str, int]:
+    """Parse a ``lake env lean ...; echo EXIT_CODE:$?`` transcript.
+
+    Returns ``(compiled, cleaned_output, exit_code)`` where ``compiled`` is True iff
+    the compiler exited 0 with no ``declaration uses 'sorry'`` diagnostic.
+
+    Matches the LAST ``EXIT_CODE:N`` — that's the one our shell appends at the end
+    of the command. Matching the first occurrence would let a model inject
+    ``#eval IO.println "EXIT_CODE:0"`` into the proof file to bypass the
+    sorry/exit-code checks: the regex would hit the injected marker, truncate
+    everything after it (hiding the real ``declaration uses 'sorry'`` diagnostic and
+    the real EXIT_CODE), and report success.
+    """
+    exit_code = 1
+    matches = list(re.finditer(r"EXIT_CODE:(\d+)", output))
+    if matches:
+        last = matches[-1]
+        exit_code = int(last.group(1))
+        output = output[: last.start()].strip()
+    has_sorry = bool(re.search(r"declaration uses 'sorry'", output))
+    return (exit_code == 0 and not has_sorry), output, exit_code

verifiers-0.1.15.dev402/verifiers/v1/tasksets/lean/taskset.py

@@ -0,0 +1,265 @@
+"""lean: a reusable Lean 4 theorem-proving taskset base.
+
+Loads a Lean dataset (any HuggingFace dataset with a formal-statement column),
+plants a ``sorry`` starter file in a Mathlib sandbox at ``setup``, and rewards a
+clean ``lake env lean`` compile — with a reward-hacking guard that pins the
+original theorem signature — at scoring time. It exposes **no tools**: the agent
+reads, edits, and compiles ``proof.lean`` through whichever shell-capable harness
+drives the rollout (``mini-swe-agent`` / ``rlm`` / ``bash``), so a lean task runs
+under ANY harness.
+
+This is a *base*: point it at a dataset via ``--taskset.dataset-name`` (+ column
+config), or subclass it in a thin per-dataset package that just sets the config
+defaults — the way ``swebench-verified-v1`` etc. wrap ``HarborTaskset``. The
+per-dataset packages live in research-environments.
+"""
+
+from __future__ import annotations
+
+import shlex
+
+from pydantic_config import BaseConfig
+
+from verifiers.v1.decorators import reward
+from verifiers.v1.runtimes import Runtime
+from verifiers.v1.task import Task, TaskResources
+from verifiers.v1.taskset import Taskset, TasksetConfig
+from verifiers.v1.tasksets.lean.scoring import (
+    build_starter_file,
+    expected_protected_signature,
+    parse_compile_output,
+    protected_signature_substring_present,
+)
+from verifiers.v1.trace import Trace
+
+# Default sandbox image: Lean v4.27 + Mathlib v4.27 (PI Research team registry).
+# A per-dataset package overrides ``docker_image`` with its version-matched image.
+DEFAULT_DOCKER_IMAGE = "team-clyvldofb0000gg1kx39rgzjq/lean-tactic:mathlib-v4.27.0-v3"
+LEAN_PROJECT_PATH = "/workspace/mathlib4"
+PROOF_FILE_PATH = "/tmp/proof.lean"
+
+DEFAULT_SYSTEM_PROMPT = "You are an expert Lean 4 theorem prover working with Mathlib."
+
+
+class LeanTask(Task):
+    formal_statement: str
+    header: str = ""
+    imports: str = "import Mathlib"
+    normalize_mathlib_imports: bool = False
+    # Canonical ``theorem ... := by`` text pinned at load; the reward checks it
+    # still appears in the final file (the only edit the reward cares about).
+    protected_signature: str = ""
+    # Gold proof body (replaces ``  sorry``); "" when the dataset ships no gold.
+    formal_proof: str = ""
+
+
+class LeanDatasetConfig(BaseConfig):
+    """Which HuggingFace dataset to load and how to read its columns."""
+
+    name: str
+    """HuggingFace dataset id (required; each per-dataset package sets it)."""
+    split: str = "train"
+    subset: str | None = None
+    statement_column: str = "formal_statement"
+    header_column: str | None = None
+    imports_column: str | None = None
+    name_column: str | None = None
+    proof_column: str | None = None
+    """Column holding the gold proof body (used by ``validate``); None = no gold."""
+    normalize_mathlib_imports: bool = False
+
+
+class LeanConfig(TasksetConfig):
+    dataset: LeanDatasetConfig
+    docker_image: str = DEFAULT_DOCKER_IMAGE
+    lean_project_path: str = LEAN_PROJECT_PATH
+    proof_file_path: str = PROOF_FILE_PATH
+    compile_timeout: int = 300
+    """Per-compile ``timeout`` wrapper (seconds), bounding each ``lake env lean``."""
+    system_prompt: str = DEFAULT_SYSTEM_PROMPT
+
+
+class LeanTaskset(Taskset[LeanTask, LeanConfig]):
+    NEEDS_CONTAINER = True
+
+    # ---- task loading -------------------------------------------------------
+
+    def load_tasks(self) -> list[LeanTask]:
+        from datasets import load_dataset
+
+        config = self.config
+        ds = config.dataset
+        raw = load_dataset(
+            ds.name,
+            ds.subset,
+            split=ds.split,
+            keep_in_memory=True,
+            num_proc=8,
+        )
+
+        # Fail loud on a misconfigured column name — otherwise a typo is silently
+        # treated as a missing/empty field (e.g. a wrong proof_column makes every
+        # gold check vacuously fail), or a statement_column typo raises a raw
+        # KeyError on the first row instead of this clear error.
+        for label, col in (
+            ("statement_column", ds.statement_column),
+            ("header_column", ds.header_column),
+            ("imports_column", ds.imports_column),
+            ("name_column", ds.name_column),
+            ("proof_column", ds.proof_column),
+        ):
+            if col is not None and col not in raw.column_names:
+                raise ValueError(
+                    f"dataset.{label}={col!r} not found in {ds.name!r}; columns={raw.column_names}"
+                )
+
+        resources = TaskResources(cpu=4, memory=4, disk=10)
+        tasks: list[LeanTask] = []
+        for index, row in enumerate(raw):
+            # Skip degenerate rows with no statement: there's nothing to prove, and
+            # an empty statement collapses the pinned signature to just ``:= by``,
+            # which every proof contains — so the reward-hacking guard would pass on
+            # a rewritten trivial theorem. Drop them rather than ship a free-reward task.
+            formal_statement = row[ds.statement_column]
+            if not isinstance(formal_statement, str) or not formal_statement.strip():
+                continue
+            # Unset columns are None, and row.get(None) is None, so the `or`
+            # fallbacks cover both an unset column and an empty value.
+            header = row.get(ds.header_column) or ""
+            imports = row.get(ds.imports_column) or "import Mathlib"
+            gold = row.get(ds.proof_column) or ""
+            name = row.get(ds.name_column)
+            tasks.append(
+                LeanTask(
+                    idx=index,
+                    name=str(name) if name else f"task_{index:05d}",
+                    prompt=self._build_prompt(formal_statement, header),
+                    system_prompt=config.system_prompt,
+                    image=config.docker_image,
+                    workdir=config.lean_project_path,
+                    resources=resources,
+                    formal_statement=formal_statement,
+                    header=header,
+                    imports=imports,
+                    normalize_mathlib_imports=ds.normalize_mathlib_imports,
+                    protected_signature=expected_protected_signature(formal_statement),
+                    formal_proof=gold,
+                )
+            )
+        return tasks
+
+    def _build_prompt(self, formal_statement: str, header: str) -> str:
+        cfg = self.config
+        block = (
+            "Prove the following Lean 4 theorem. A starter proof file is at "
+            f"`{cfg.proof_file_path}` with the theorem statement and a `sorry` "
+            "placeholder already in place. Edit it and compile with "
+            f"`cd {cfg.lean_project_path} && lake env lean {cfg.proof_file_path}`.\n\n"
+            f"```lean\n{formal_statement}\n```"
+        )
+        if header:
+            block += f"\n\nThe file header (imports/namespaces) is already set up:\n```lean\n{header}\n```"
+        block += (
+            "\n\nDo NOT modify the theorem statement (the lines from `theorem ...` "
+            "through `:= by`) — the grader checks the original statement still "
+            "appears and gives zero reward if you rewrote it. Write your proof "
+            "tactics in place of `sorry`; the final proof must not contain `sorry` "
+            "or `admit`. A clean compile prints nothing and exits 0."
+        )
+        return block
+
+    # ---- sandbox helpers ----------------------------------------------------
+
+    async def _compile(self, runtime: Runtime) -> tuple[bool, str, int]:
+        """Run ``lake env lean`` on the proof file; returns (compiled, output, exit_code)."""
+        cfg = self.config
+        cmd = (
+            f"cd {shlex.quote(cfg.lean_project_path)} && "
+            f"timeout {cfg.compile_timeout} lake env lean {shlex.quote(cfg.proof_file_path)} 2>&1; "
+            "echo EXIT_CODE:$?"
+        )
+        result = await runtime.run(["bash", "-lc", cmd], {})
+        return parse_compile_output((result.stdout or "") + (result.stderr or ""))
+
+    # ---- lifecycle ----------------------------------------------------------
+
+    async def setup(self, task: LeanTask, runtime: Runtime) -> None:
+        """Plant the ``sorry`` starter file in the sandbox before the agent runs."""
+        content = build_starter_file(
+            task.formal_statement,
+            header=task.header,
+            imports=task.imports,
+            normalize=task.normalize_mathlib_imports,
+        )
+        await runtime.write(self.config.proof_file_path, content.encode())
+
+    # ---- scoring ------------------------------------------------------------
+
+    @reward(weight=1.0)
+    async def lean_compiled(
+        self, task: LeanTask, trace: Trace, runtime: Runtime
+    ) -> float:
+        """1.0 iff the final proof compiles cleanly (exit 0, no ``sorry``) and the
+        protected theorem signature is intact; 0.0 otherwise.
+
+        Reads the final ``proof.lean`` back through the live runtime, runs the
+        host-side signature guard (with comment/string stripping), then re-runs
+        ``lake env lean``. Diagnostics land on ``trace.info`` for inspection.
+        """
+        if trace.has_error:
+            return 0.0
+
+        # Read the final proof back. A read failure here is genuinely exceptional
+        # (setup planted the file; the agent edits it in-sandbox), so let it
+        # propagate as a scoring error rather than swallow an infra/sandbox failure
+        # into a false-negative 0. (The prime runtime collapses every read error
+        # into SandboxError, so there's no file-not-found type to narrow to.)
+        current = (await runtime.read(self.config.proof_file_path)).decode(
+            "utf-8", "replace"
+        )
+
+        expected_sig = task.protected_signature or expected_protected_signature(
+            task.formal_statement
+        )
+        if expected_sig and not protected_signature_substring_present(
+            current, expected_sig
+        ):
+            trace.info["lean_tampered"] = True
+            trace.info["compile_output"] = "signature rewritten or hidden in a comment"
+            return 0.0
+        trace.info["lean_tampered"] = False
+
+        compiled, output, exit_code = await self._compile(runtime)
+        trace.info["lean_compiled"] = compiled
+        trace.info["compile_exit_code"] = exit_code
+        trace.info["compile_output"] = output[-4000:]
+        return 1.0 if compiled else 0.0
+
+    # ---- validation (model-free gold check) ---------------------------------
+
+    async def validate(self, task: LeanTask, runtime: Runtime) -> bool:
+        """Compile the gold proof: substitute it for ``sorry`` and check it type-checks.
+
+        ``False`` is reserved for a row whose gold proof exists but **fails to
+        compile** (a genuinely bad/unprovable row). A row with **no** gold proof
+        (statement-only datasets, or an empty proof column) returns ``True`` — there
+        is nothing to refute, matching the base ``Taskset.validate`` no-op; flagging
+        it ``invalid`` would both swamp the report on statement-only datasets and
+        mask the rows whose gold actually fails on a sparse-gold dataset.
+        """
+        gold = (task.formal_proof or "").rstrip()
+        if not gold:
+            return True
+        content = build_starter_file(
+            task.formal_statement,
+            header=task.header,
+            imports=task.imports,
+            normalize=task.normalize_mathlib_imports,
+            proof_body=gold,
+        )
+        await runtime.write(self.config.proof_file_path, content.encode())
+        compiled, _, _ = await self._compile(runtime)
+        return compiled
+
+
+__all__ = ["LeanTask", "LeanConfig", "LeanTaskset"]