verifiers 0.3.2.dev127__tar.gz → 0.3.2.dev128__tar.gz

{verifiers-0.3.2.dev127 → verifiers-0.3.2.dev128}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.5
 Name: verifiers
-Version: 0.3.2.dev127
+Version: 0.3.2.dev128
 Summary: Verifiers: Environments for LLM Reinforcement Learning
 Project-URL: Homepage, https://github.com/primeintellect-ai/verifiers
 Project-URL: Documentation, https://github.com/primeintellect-ai/verifiers
@@ -48,8 +48,6 @@ Requires-Dist: typing-extensions>=4.12.2
 Requires-Dist: uvicorn>=0.52.0
 Provides-Extra: harbor
 Requires-Dist: harbor==0.21.0; (python_full_version >= '3.12') and extra == 'harbor'
-Provides-Extra: lean
-Requires-Dist: datasets<6.0.0,>=3.3.0; extra == 'lean'
 Provides-Extra: modal
 Requires-Dist: modal>=1.5.4; extra == 'modal'
 Provides-Extra: openenv

{verifiers-0.3.2.dev127 → verifiers-0.3.2.dev128}/pyproject.toml

@@ -89,9 +89,6 @@ examples = [
 ]
 
 [project.optional-dependencies]
-lean = [
-    "datasets>=3.3.0,<6.0.0",
-]
 ta = [
     "textarena==0.7.4",
     "nltk>=3.9.2",

{verifiers-0.3.2.dev127 → verifiers-0.3.2.dev128}/verifiers/v1/gepa/dataset.py

@@ -40,5 +40,5 @@ def resolve_gepa_seed_prompt(tasks: list[Task], initial_prompt: str | None) -> s
         "no task in this taskset sets Task.system_prompt — some tasksets bake instructions "
         "directly into `prompt` instead (e.g. gsm8k) and can't be optimized this way. Pass "
         "--initial-prompt to seed one explicitly, or pick a taskset whose load() sets "
-        "system_prompt on its task data (e.g. reverse-text, lean, textarena)."
+        "system_prompt on its task data (e.g. reverse-text, textarena)."
     )

{verifiers-0.3.2.dev127 → verifiers-0.3.2.dev128}/verifiers/v1/tasksets/__init__.py

@@ -1,10 +1,4 @@
 from verifiers.v1.tasksets.harbor import HarborConfig, HarborTaskset
-from verifiers.v1.tasksets.lean import (
-    LeanConfig,
-    LeanDatasetConfig,
-    LeanTask,
-    LeanTaskset,
-)
 from verifiers.v1.tasksets.nemo_gym import NeMoGymConfig, NeMoGymTaskset
 from verifiers.v1.tasksets.openenv import (
     OpenEnvConfig,
@@ -18,10 +12,6 @@ from verifiers.v1.tasksets.openenv import (
 __all__ = [
     "HarborConfig",
     "HarborTaskset",
-    "LeanConfig",
-    "LeanDatasetConfig",
-    "LeanTask",
-    "LeanTaskset",
     "NeMoGymConfig",
     "NeMoGymTaskset",
     "OpenEnvConfig",

verifiers-0.3.2.dev127/verifiers/v1/tasksets/lean/__init__.py

@@ -1,33 +0,0 @@
-from verifiers.v1.tasksets.lean.scoring import (
-    build_starter_file,
-    expected_protected_signature,
-    parse_compile_output,
-    protected_signature_substring_present,
-    strip_lean_comments,
-)
-from verifiers.v1.tasksets.lean.taskset import (
-    DEFAULT_DOCKER_IMAGE,
-    LEAN_PROJECT_PATH,
-    PROOF_FILE_PATH,
-    LeanConfig,
-    LeanDatasetConfig,
-    LeanTask,
-    LeanTaskConfig,
-    LeanTaskset,
-)
-
-__all__ = [
-    "DEFAULT_DOCKER_IMAGE",
-    "LEAN_PROJECT_PATH",
-    "PROOF_FILE_PATH",
-    "LeanConfig",
-    "LeanDatasetConfig",
-    "LeanTask",
-    "LeanTaskConfig",
-    "LeanTaskset",
-    "build_starter_file",
-    "expected_protected_signature",
-    "parse_compile_output",
-    "protected_signature_substring_present",
-    "strip_lean_comments",
-]

verifiers-0.3.2.dev127/verifiers/v1/tasksets/lean/scoring.py

@@ -1,262 +0,0 @@
-"""Pure helpers for the Lean taskset: starter-file construction, theorem-signature
-canonicalization, the reward-hacking signature guard, and compile-output parsing.
-
-Everything here is a pure function over plain strings — no runtime, no sandbox, no
-verifiers types — so it's trivially unit-testable and shared between ``setup``
-(building the starter file), the reward (the signature guard + compile parsing),
-and ``validate`` (planting the gold proof).
-"""
-
-from __future__ import annotations
-
-import re
-
-PROTECTED_HEADER_COMMENT = (
-    "-- DO NOT MODIFY the theorem statement below. The grader checks\n"
-    "-- that the original `theorem ... := by` text still appears in\n"
-    "-- this file. Only edit the proof body (currently `sorry`) and\n"
-    "-- lines after it."
-)
-
-
-# ── Imports / preamble normalization ─────────────────────────────────────────
-
-
-def normalize_imports(text: str) -> str:
-    """Collapse every ``import Mathlib*`` line to a single ``import Mathlib``.
-
-    Some datasets (miniF2F) ship fine-grained Mathlib imports that don't resolve
-    against the monolithic ``import Mathlib`` the sandbox image provides.
-    """
-    lines = text.split("\n")
-    out: list[str] = []
-    inserted = False
-    for line in lines:
-        if line.strip().startswith("import Mathlib"):
-            if not inserted:
-                out.append("import Mathlib")
-                inserted = True
-        else:
-            out.append(line)
-    return "\n".join(out)
-
-
-def _build_preamble(imports_str: str, header: str, normalize: bool) -> str:
-    if header and header.strip().startswith("import"):
-        preamble = header.strip()
-        return normalize_imports(preamble) if normalize else preamble
-    parts = [imports_str.strip()]
-    if header and header.strip():
-        parts.append(header.strip())
-    preamble = "\n\n".join(parts)
-    return normalize_imports(preamble) if normalize else preamble
-
-
-# ── Theorem signature canonicalization ───────────────────────────────────────
-
-
-def _normalize_signature(stmt: str) -> str:
-    """Canonicalize a Lean theorem statement to end with ``:= by``.
-
-    Strips trailing ``sorry``/``admit`` placeholders and any trailing ``by`` /
-    ``:=`` tokens, then re-appends `` := by``. Places the appended token on a new
-    indented line when the last line of the stripped signature already contains a
-    ``--`` comment (otherwise the ``:= by`` would land inside the line comment and
-    Lean would silently ignore it).
-    """
-    s = stmt.rstrip()
-    s = re.sub(r"\s*\b(?:sorry|admit)\b\s*$", "", s)
-    s = re.sub(r"\s*\bby\b\s*$", "", s)
-    s = re.sub(r"\s*:=\s*$", "", s)
-    s = s.rstrip()
-    last_newline = s.rfind("\n")
-    last_line = s[last_newline + 1 :] if last_newline != -1 else s
-    sep = "\n    " if "--" in last_line else " "
-    return s + sep + ":= by"
-
-
-def _split_imports_and_signature(stmt: str) -> tuple[str, str]:
-    decl_match = re.search(r"^(?:theorem|lemma|example)\s", stmt, flags=re.MULTILINE)
-    if not decl_match:
-        return "", stmt
-    return stmt[: decl_match.start()].rstrip(), stmt[decl_match.start() :]
-
-
-def expected_protected_signature(formal_statement: str) -> str:
-    """Return the canonical ``theorem ... := by`` block the reward pins as ground truth.
-
-    Pure function over ``formal_statement``; the imports/header portion (if the
-    statement carries one inline) is stripped first.
-    """
-    stmt = formal_statement or ""
-    if stmt.strip().startswith("import "):
-        _, signature_raw = _split_imports_and_signature(stmt)
-    else:
-        signature_raw = stmt
-    return _normalize_signature(signature_raw).strip()
-
-
-# ── Comment / string stripping for the signature guard ───────────────────────
-
-
-def strip_lean_comments(text: str) -> str:
-    """Remove Lean line/block comments **and string literals** from ``text``.
-
-    Lean comments come in two forms: ``-- ...`` line comments (to end of line) and
-    ``/- ... -/`` block comments (nestable; ``/-- ... -/`` doc comments are a
-    special case). String literals must also be stripped: a Lean
-    ``"theorem ... := by"`` constant or doc-string would otherwise let a model hide
-    the pinned signature inside a string while rewriting the live declaration to a
-    trivial one, defeating the substring guard. We handle both regular
-    double-quoted strings (with backslash escapes) and triple-quoted raw strings.
-
-    Block comments nest, so we count depth; line comments end at the next newline.
-    Outside comments and strings, newlines are preserved so the result keeps
-    roughly the right shape for substring matching downstream.
-    """
-    out: list[str] = []
-    i = 0
-    n = len(text)
-    block_depth = 0
-    in_line_comment = False
-    while i < n:
-        ch = text[i]
-        if in_line_comment:
-            if ch == "\n":
-                in_line_comment = False
-                out.append(ch)
-            i += 1
-            continue
-        if block_depth > 0:
-            if i + 1 < n and text[i : i + 2] == "-/":
-                block_depth -= 1
-                i += 2
-                continue
-            if i + 1 < n and text[i : i + 2] == "/-":
-                block_depth += 1
-                i += 2
-                continue
-            if ch == "\n":
-                out.append(ch)
-            i += 1
-            continue
-        if i + 1 < n and text[i : i + 2] == "/-":
-            block_depth = 1
-            i += 2
-            continue
-        if i + 1 < n and text[i : i + 2] == "--":
-            in_line_comment = True
-            i += 2
-            continue
-        # Triple-quoted raw string ``"""..."""`` — skip until the closing
-        # triple-quote, preserving newlines.
-        if i + 2 < n and text[i : i + 3] == '"""':
-            i += 3
-            while i < n:
-                if i + 2 < n and text[i : i + 3] == '"""':
-                    i += 3
-                    break
-                if text[i] == "\n":
-                    out.append("\n")
-                i += 1
-            continue
-        # Regular string ``"..."`` — handle ``\\`` escapes, stop at the next
-        # unescaped quote or newline (Lean strings are single-line).
-        if ch == '"':
-            i += 1
-            while i < n and text[i] != '"' and text[i] != "\n":
-                if text[i] == "\\" and i + 1 < n:
-                    i += 2
-                else:
-                    i += 1
-            if i < n and text[i] == '"':
-                i += 1
-            continue
-        out.append(ch)
-        i += 1
-    return "".join(out)
-
-
-def protected_signature_substring_present(
-    content: str, expected_signature: str
-) -> bool:
-    """True when the locked signature text still appears in the file.
-
-    Strips Lean comments from BOTH sides — without that a model could paste the
-    pinned signature into a ``--`` or ``/- ... -/`` block while rewriting the live
-    declaration to something trivial, and the asymmetric variant would also misfire
-    if ``expected_signature`` itself contains a comment. Tries an exact substring
-    match first, then a whitespace-flexible match (each side collapsed to
-    single-spaced tokens) so the model can re-indent or reflow whitespace freely.
-    """
-    if not expected_signature:
-        return True
-    decommented_content = strip_lean_comments(content)
-    decommented_expected = strip_lean_comments(expected_signature)
-    if not decommented_expected.strip():
-        return True
-    if decommented_expected in decommented_content:
-        return True
-    flat_signature = " ".join(decommented_expected.split())
-    flat_content = " ".join(decommented_content.split())
-    return flat_signature in flat_content
-
-
-# ── Starter-file construction ────────────────────────────────────────────────
-
-
-def build_starter_file(
-    formal_statement: str,
-    header: str = "",
-    imports: str = "import Mathlib",
-    normalize: bool = False,
-    proof_body: str | None = None,
-) -> str:
-    """Construct the starter proof file.
-
-    Layout: preamble (imports / header) + a brief ``-- DO NOT MODIFY`` comment
-    block + the normalized theorem signature + the proof body. If ``proof_body`` is
-    None (the default) the body is the placeholder ``  sorry`` — what's planted at
-    rollout start. A supplied gold ``proof_body`` (e.g. by ``validate``) replaces
-    the placeholder so the file is the full reference solution.
-    """
-    stmt = formal_statement or ""
-    if stmt.strip().startswith("import "):
-        imports_block, signature_raw = _split_imports_and_signature(stmt)
-        preamble = normalize_imports(imports_block) if normalize else imports_block
-    else:
-        preamble = _build_preamble(imports or "import Mathlib", header, normalize)
-        signature_raw = stmt
-
-    signature = _normalize_signature(signature_raw)
-    body = "  sorry" if proof_body is None else proof_body.rstrip()
-    wrapped = f"{PROTECTED_HEADER_COMMENT}\n{signature}\n{body}\n"
-    if preamble:
-        return preamble.rstrip() + "\n\n" + wrapped
-    return wrapped
-
-
-# ── Compile-output parsing ───────────────────────────────────────────────────
-
-
-def parse_compile_output(output: str) -> tuple[bool, str, int]:
-    """Parse a ``lake env lean ...; echo EXIT_CODE:$?`` transcript.
-
-    Returns ``(compiled, cleaned_output, exit_code)`` where ``compiled`` is True iff
-    the compiler exited 0 with no ``declaration uses 'sorry'`` diagnostic.
-
-    Matches the LAST ``EXIT_CODE:N`` — that's the one our shell appends at the end
-    of the command. Matching the first occurrence would let a model inject
-    ``#eval IO.println "EXIT_CODE:0"`` into the proof file to bypass the
-    sorry/exit-code checks: the regex would hit the injected marker, truncate
-    everything after it (hiding the real ``declaration uses 'sorry'`` diagnostic and
-    the real EXIT_CODE), and report success.
-    """
-    exit_code = 1
-    matches = list(re.finditer(r"EXIT_CODE:(\d+)", output))
-    if matches:
-        last = matches[-1]
-        exit_code = int(last.group(1))
-        output = output[: last.start()].strip()
-    has_sorry = bool(re.search(r"declaration uses 'sorry'", output))
-    return (exit_code == 0 and not has_sorry), output, exit_code

verifiers-0.3.2.dev127/verifiers/v1/tasksets/lean/taskset.py

@@ -1,229 +0,0 @@
-"""Lean 4 theorem-proving tasks backed by Hugging Face datasets.
-
-Each task plants a ``sorry``-based starter file, lets any container-capable
-harness edit it, and rewards a clean ``lake env lean`` compile. The original
-theorem signature must remain present, which prevents replacing the assigned
-statement with an easier theorem. Dataset-specific packages can subclass this
-taskset and only supply column mappings.
-"""
-
-from __future__ import annotations
-
-import shlex
-from collections.abc import Iterator
-
-from pydantic_config import BaseConfig
-
-from verifiers.v1.configs.task import TaskConfig
-from verifiers.v1.configs.taskset import TasksetConfig
-from verifiers.v1.runtimes import Runtime
-from verifiers.v1.state import State
-from verifiers.v1.task import Task, TaskData, TaskResources
-from verifiers.v1.taskset import Taskset
-from verifiers.v1.tasksets.lean.scoring import (
-    build_starter_file,
-    expected_protected_signature,
-    parse_compile_output,
-    protected_signature_substring_present,
-)
-from verifiers.v1.trace import Trace
-from verifiers.v1.utils.decorators import reward
-
-# Lean v4.27 with Mathlib v4.27.
-DEFAULT_DOCKER_IMAGE = "team-clyvldofb0000gg1kx39rgzjq/lean-tactic:mathlib-v4.27.0-v3"
-LEAN_PROJECT_PATH = "/workspace/mathlib4"
-PROOF_FILE_PATH = "/tmp/proof.lean"
-
-DEFAULT_SYSTEM_PROMPT = "You are an expert Lean 4 theorem prover working with Mathlib."
-
-
-class LeanDatasetConfig(BaseConfig):
-    name: str
-    """HuggingFace dataset id (required; each per-dataset package sets it)."""
-    split: str = "train"
-    subset: str | None = None
-    statement_column: str = "formal_statement"
-    header_column: str | None = None
-    imports_column: str | None = None
-    name_column: str | None = None
-    proof_column: str | None = None
-    """Column holding the gold proof body (used by ``validate``); None = no gold."""
-    normalize_mathlib_imports: bool = False
-
-
-class LeanTaskConfig(TaskConfig):
-    lean_project_path: str = LEAN_PROJECT_PATH
-    proof_file_path: str = PROOF_FILE_PATH
-    compile_timeout: int = 300
-    """Per-compile ``timeout`` wrapper (seconds), bounding each ``lake env lean``."""
-
-
-class LeanConfig(TasksetConfig):
-    dataset: LeanDatasetConfig
-    docker_image: str = DEFAULT_DOCKER_IMAGE
-    task: LeanTaskConfig = LeanTaskConfig()
-
-
-class LeanData(TaskData):
-    formal_statement: str
-    header: str = ""
-    imports: str = "import Mathlib"
-    normalize_mathlib_imports: bool = False
-    # Canonical ``theorem ... := by`` text pinned at load; the reward checks it
-    # still appears in the final file (the only edit the reward cares about).
-    protected_signature: str = ""
-    # Gold proof body (replaces ``  sorry``); "" when the dataset ships no gold.
-    formal_proof: str = ""
-
-
-class LeanTask(Task[LeanData, State, LeanTaskConfig]):
-    NEEDS_CONTAINER = True
-
-    async def _compile(self, runtime: Runtime) -> tuple[bool, str, int]:
-        cmd = (
-            f"cd {shlex.quote(self.config.lean_project_path)} && "
-            f"timeout {self.config.compile_timeout} lake env lean "
-            f"{shlex.quote(self.config.proof_file_path)} 2>&1; "
-            "echo EXIT_CODE:$?"
-        )
-        result = await runtime.run(["bash", "-lc", cmd], {})
-        return parse_compile_output((result.stdout or "") + (result.stderr or ""))
-
-    async def setup(self, runtime: Runtime) -> None:
-        content = build_starter_file(
-            self.data.formal_statement,
-            header=self.data.header,
-            imports=self.data.imports,
-            normalize=self.data.normalize_mathlib_imports,
-        )
-        await runtime.write(self.config.proof_file_path, content.encode())
-
-    @reward(weight=1.0)
-    async def lean_compiled(self, trace: Trace, runtime: Runtime) -> float:
-        """Require both the assigned signature and a clean Lean compile."""
-        if trace.has_error:
-            return 0.0
-
-        # Setup created this file, so a read failure is an infrastructure error.
-        current = (await runtime.read(self.config.proof_file_path)).decode(
-            "utf-8", "replace"
-        )
-
-        expected_sig = self.data.protected_signature or expected_protected_signature(
-            self.data.formal_statement
-        )
-        if expected_sig and not protected_signature_substring_present(
-            current, expected_sig
-        ):
-            trace.info["lean_tampered"] = True
-            trace.info["compile_output"] = "signature rewritten or hidden in a comment"
-            return 0.0
-        trace.info["lean_tampered"] = False
-
-        compiled, output, exit_code = await self._compile(runtime)
-        trace.info["lean_compiled"] = compiled
-        trace.info["compile_exit_code"] = exit_code
-        trace.info["compile_output"] = output[-4000:]
-        return 1.0 if compiled else 0.0
-
-    async def validate(self, runtime: Runtime) -> bool | None:
-        """Compile the gold proof; rows without one have nothing to preflight."""
-        gold = (self.data.formal_proof or "").rstrip()
-        if not gold:
-            return None
-        content = build_starter_file(
-            self.data.formal_statement,
-            header=self.data.header,
-            imports=self.data.imports,
-            normalize=self.data.normalize_mathlib_imports,
-            proof_body=gold,
-        )
-        await runtime.write(self.config.proof_file_path, content.encode())
-        compiled, _, _ = await self._compile(runtime)
-        return compiled
-
-
-class LeanTaskset(Taskset[LeanTask, LeanConfig]):
-    def load(self) -> Iterator[LeanTask]:
-        try:
-            from datasets import load_dataset
-        except ModuleNotFoundError as e:
-            raise ModuleNotFoundError(
-                "the Lean taskset requires the `lean` extra; install `verifiers[lean]`"
-            ) from e
-
-        config = self.config
-        ds = config.dataset
-        raw = load_dataset(
-            ds.name,
-            ds.subset,
-            split=ds.split,
-            num_proc=8,
-        )
-
-        # Validate optional columns before empty-value fallbacks hide a typo.
-        for label, col in (
-            ("statement_column", ds.statement_column),
-            ("header_column", ds.header_column),
-            ("imports_column", ds.imports_column),
-            ("name_column", ds.name_column),
-            ("proof_column", ds.proof_column),
-        ):
-            if col is not None and col not in raw.column_names:
-                raise ValueError(
-                    f"dataset.{label}={col!r} not found in {ds.name!r}; columns={raw.column_names}"
-                )
-
-        resources = TaskResources(cpu=4, memory=4, disk=10)
-        for index, row in enumerate(raw):
-            # An empty statement would reduce the signature guard to `:= by`.
-            formal_statement = row[ds.statement_column]
-            if not isinstance(formal_statement, str) or not formal_statement.strip():
-                continue
-            # A disabled optional column is None; row.get(None) yields the same
-            # empty fallback as a present-but-empty column.
-            header = row.get(ds.header_column) or ""
-            imports = row.get(ds.imports_column) or "import Mathlib"
-            gold = row.get(ds.proof_column) or ""
-            name = row.get(ds.name_column)
-            yield LeanTask(
-                LeanData(
-                    idx=index,
-                    name=str(name) if name else f"task_{index:05d}",
-                    prompt=self._build_prompt(formal_statement, header),
-                    system_prompt=DEFAULT_SYSTEM_PROMPT,
-                    image=config.docker_image,
-                    workdir=config.task.lean_project_path,
-                    resources=resources,
-                    formal_statement=formal_statement,
-                    header=header,
-                    imports=imports,
-                    normalize_mathlib_imports=ds.normalize_mathlib_imports,
-                    protected_signature=expected_protected_signature(formal_statement),
-                    formal_proof=gold,
-                ),
-                self.config.task,
-            )
-
-    def _build_prompt(self, formal_statement: str, header: str) -> str:
-        cfg = self.config
-        block = (
-            "Prove the following Lean 4 theorem. A starter proof file is at "
-            f"`{cfg.task.proof_file_path}` with the theorem statement and a `sorry` "
-            "placeholder already in place. Edit it and compile with "
-            f"`cd {cfg.task.lean_project_path} && lake env lean {cfg.task.proof_file_path}`.\n\n"
-            f"```lean\n{formal_statement}\n```"
-        )
-        if header:
-            block += f"\n\nThe file header (imports/namespaces) is already set up:\n```lean\n{header}\n```"
-        block += (
-            "\n\nDo NOT modify the theorem statement (the lines from `theorem ...` "
-            "through `:= by`) — the grader checks the original statement still "
-            "appears and gives zero reward if you rewrote it. Write your proof "
-            "tactics in place of `sorry`; the final proof must not contain `sorry` "
-            "or `admit`. A clean compile prints nothing and exits 0."
-        )
-        return block
-
-
-__all__ = ["LeanConfig", "LeanTask", "LeanTaskset"]