verifiers 0.1.15.dev400__tar.gz → 0.1.15.dev402__tar.gz

{verifiers-0.1.15.dev400 → verifiers-0.1.15.dev402}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: verifiers
-Version: 0.1.15.dev400
+Version: 0.1.15.dev402
 Summary: Verifiers: Environments for LLM Reinforcement Learning
 Project-URL: Homepage, https://github.com/primeintellect-ai/verifiers
 Project-URL: Documentation, https://github.com/primeintellect-ai/verifiers

{verifiers-0.1.15.dev400 → verifiers-0.1.15.dev402}/verifiers/envs/experimental/composable/tasksets/__init__.py

@@ -17,10 +17,6 @@ from verifiers.envs.experimental.composable.tasksets.swe.swe_tasksets import (
     make_swesmith_taskset,
     make_swesmith_ts_taskset,
 )
-from verifiers.envs.experimental.composable.tasksets.lean.lean_task import (
-    LEAN_SYSTEM_PROMPT,
-    LeanTaskSet,
-)
 from verifiers.envs.experimental.composable.tasksets.math.math_task import MathTaskSet
 from verifiers.envs.experimental.composable.tasksets.cp.cp_task import (
     CPRubric,
@@ -59,8 +55,6 @@ __all__ = [
     "make_swesmith_rs_taskset",
     "make_swesmith_cpp_taskset",
     "make_swesmith_php_taskset",
-    "LeanTaskSet",
-    "LEAN_SYSTEM_PROMPT",
     "MathTaskSet",
     "CPTaskSet",
     "CPRubric",

{verifiers-0.1.15.dev400 → verifiers-0.1.15.dev402}/verifiers/v1/README.md

@@ -241,6 +241,20 @@ each in its own declared, pullable container image — e.g. Terminal-Bench 2:
 uv run eval harbor --taskset.dataset terminal-bench/terminal-bench-2 -n 10 --harness.id rlm
 ```
 
+Harbor registry selectors pass through as taskset config. For the
+`research-environments` registry:
+
+```bash
+uv run eval harbor --taskset.dataset general-agent@2026-06-25 \
+  --taskset.repo PrimeIntellect-ai/research-environments@REF_OR_SHA
+
+uv run eval harbor --taskset.dataset general-agent@2026-06-25 \
+  --taskset.registry-path ./registry.json
+
+uv run eval harbor --taskset.dataset general-agent@2026-06-25 \
+  --taskset.registry-url https://raw.githubusercontent.com/PrimeIntellect-ai/research-environments/REF_OR_SHA/registry.json
+```
+
 ## Backwards compatibility
 
 The v0 framework is untouched — the classic `verifiers` API and its entrypoints (`vf-eval`,

{verifiers-0.1.15.dev400 → verifiers-0.1.15.dev402}/verifiers/v1/tasksets/__init__.py

@@ -5,5 +5,18 @@ here — it imports the optional `textarena` dependency at module load — but s
 `verifiers.v1.tasksets.textarena`."""
 
 from verifiers.v1.tasksets.harbor import HarborConfig, HarborTaskset
+from verifiers.v1.tasksets.lean import (
+    LeanConfig,
+    LeanDatasetConfig,
+    LeanTask,
+    LeanTaskset,
+)
 
-__all__ = ["HarborConfig", "HarborTaskset"]
+__all__ = [
+    "HarborConfig",
+    "HarborTaskset",
+    "LeanConfig",
+    "LeanDatasetConfig",
+    "LeanTask",
+    "LeanTaskset",
+]

{verifiers-0.1.15.dev400 → verifiers-0.1.15.dev402}/verifiers/v1/tasksets/harbor/taskset.py

@@ -1,9 +1,9 @@
 """harbor: run a Harbor (Terminal-Bench) dataset.
 
 `dataset` is a Harbor Hub package id (e.g. "org/name" or "org/name@ref"),
-downloaded directly from the registry and cached on first use. Each task dir
-ships task.toml + instruction.md (+ tests/, solution/, environment/). Defaults
-to the registry `harbor/hello-world` dataset.
+downloaded through the installed Harbor CLI and cached on first use. Each task
+dir ships task.toml + instruction.md (+ tests/, solution/, environment/).
+Defaults to the registry `harbor/hello-world` dataset.
 
 The harness runs in a container and edits /app; then the task's verifier
 (tests/test.sh) runs in the SAME container and the reward it writes to
@@ -18,17 +18,16 @@ environment is only a `Dockerfile` has no pullable image — we don't build Dock
 no environment at all also runs on that image, unless `require_image`.
 """
 
+import hashlib
 import io
-import json
-import os
+import shutil
+import subprocess
+import sys
 import tarfile
 import tempfile
 import tomllib
-from concurrent.futures import ThreadPoolExecutor
 from functools import lru_cache
 from pathlib import Path
-from urllib.parse import urlencode
-from urllib.request import Request, urlopen
 
 from pydantic import Field
 
@@ -41,14 +40,20 @@ from verifiers.v1.trace import Trace
 from verifiers.v1.types import StrictBaseModel
 
 CACHE = Path.home() / ".cache" / "harbor"
-HARBOR_SUPABASE_URL = "https://ofhuhcpkvzjlejydnvyd.supabase.co"
-HARBOR_SUPABASE_PUBLISHABLE_KEY = "sb_publishable_Z-vuQbpvpG-PStjbh4yE0Q_e-d3MTIH"
+HARBOR_INSTALL_HINT = "uv sync --python 3.12 --extra harbor"
 
 
 class HarborConfig(TasksetConfig):
     dataset: str = "harbor/hello-world"
     """A Harbor Hub package id ("org/name" or "org/name@ref"), where ref is a
-    tag, integer revision, or sha256 digest. Downloaded directly and cached."""
+    tag, integer revision, or sha256 digest. Legacy registries selected with `repo`,
+    `registry_path`, or `registry_url` use a bare dataset name ("name" or "name@version")."""
+    repo: str | None = None
+    """Optional Harbor `--repo` registry selector, e.g. "org/repo@ref"."""
+    registry_path: Path | None = None
+    """Optional Harbor `--registry-path` selector. Local unless `repo` is also set."""
+    registry_url: str | None = None
+    """Optional Harbor `--registry-url` selector for a raw registry.json URL."""
     tasks: list[str] | None = None
     """Optional subset of task names to load (None = all)."""
     timeout_multiplier: float = Field(1.0, gt=0)
@@ -87,127 +92,95 @@ class HarborTask(Task):
     """Host path to the task dir; used to stage tests/ to verify, not serialized."""
 
 
-def dataset_dir(dataset: str) -> Path:
-    """Download a Harbor Hub task or dataset package, cached on first use.
+def harbor_cli() -> str:
+    """Return the Harbor CLI installed in the active Python environment."""
+    scripts_dir = Path(sys.executable).parent
+    harbor_bin = shutil.which("harbor", path=str(scripts_dir))
+    if harbor_bin is None:
+        raise RuntimeError(
+            "Harbor tasksets require the Harbor CLI from the `harbor` extra. "
+            f"Install it with: `{HARBOR_INSTALL_HINT}`"
+        )
+    return harbor_bin
+
+
+def cache_dir(config: HarborConfig) -> Path:
+    selector_parts = [config.dataset]
+    if config.repo is not None:
+        selector_parts.extend(("repo", config.repo))
+    if config.registry_path is not None:
+        registry_path = (
+            config.registry_path
+            if config.repo is not None
+            else config.registry_path.expanduser().resolve()
+        )
+        selector_parts.extend(("registry_path", str(registry_path)))
+    if config.registry_url is not None:
+        selector_parts.extend(("registry_url", config.registry_url))
+
+    name = config.dataset.replace("/", "_").replace("@", "_")
+    if len(selector_parts) > 1:
+        digest = hashlib.sha256("\0".join(selector_parts).encode()).hexdigest()[:12]
+        name = f"{name}_{digest}"
+    return CACHE / name
+
+
+def download_command(config: HarborConfig, output_dir: Path) -> list[str]:
+    command = [
+        harbor_cli(),
+        "download",
+        config.dataset,
+        "--export",
+        "-o",
+        str(output_dir),
+    ]
+    if config.repo is not None:
+        command.extend(["--repo", config.repo])
+    if config.registry_path is not None:
+        registry_path = (
+            config.registry_path
+            if config.repo is not None
+            else config.registry_path.expanduser()
+        )
+        command.extend(["--registry-path", str(registry_path)])
+    if config.registry_url is not None:
+        command.extend(["--registry-url", config.registry_url])
+    return command
+
 
-    Harbor refs are tags, integer revisions, or ``sha256:`` digests.
+def dataset_dir(config: HarborConfig) -> Path:
+    """Download a Harbor task or dataset package, cached on first use.
+
+    Harbor Hub refs are tags, integer revisions, or ``sha256:`` digests. Legacy registry
+    refs are selected through Harbor's CLI selectors (`repo`, `registry_path`,
+    `registry_url`).
     """
-    out = CACHE / dataset.replace("/", "_").replace("@", "_")
+    out = cache_dir(config)
     if out.is_dir():
         return out
 
-    package, _, ref = dataset.partition("@")
-    org, name = package.split("/", 1)
-    ref = ref or "latest"
-    url = os.getenv("HARBOR_SUPABASE_URL", HARBOR_SUPABASE_URL).rstrip("/")
-    key = os.getenv("HARBOR_SUPABASE_PUBLISHABLE_KEY", HARBOR_SUPABASE_PUBLISHABLE_KEY)
-    headers = {
-        "apikey": key,
-        "Authorization": f"Bearer {key}",
-        "Content-Type": "application/json",
-    }
-    # Harbor stores datasets in version tables; standalone tasks use the RPC fallback below.
-    version_query = {
-        "package.name": f"eq.{name}",
-        "package.type": "eq.dataset",
-        "package.org.name": f"eq.{org}",
-        "limit": 1,
-    }
-    if ref.isdigit():
-        table = "dataset_version"
-        version_query |= {
-            "select": "id,package:package_id!inner(name,type,org:org_id!inner(name))",
-            "revision": f"eq.{ref}",
-        }
-    elif ref.startswith("sha256:"):
-        table = "dataset_version"
-        version_query |= {
-            "select": "id,package:package_id!inner(name,type,org:org_id!inner(name))",
-            "content_hash": f"eq.{ref.removeprefix('sha256:')}",
-        }
-    else:
-        table = "dataset_version_tag"
-        version_query |= {
-            "select": "dataset_version:dataset_version_id(id),package:package_id!inner(name,type,org:org_id!inner(name))",
-            "tag": f"eq.{ref}",
-        }
-    request = Request(
-        f"{url}/rest/v1/{table}?{urlencode(version_query)}", headers=headers
-    )
-    with urlopen(request) as response:
-        versions = json.load(response)
-
-    if versions:
-        version = versions[0]
-        version_id = version.get("id") or version["dataset_version"]["id"]
-        task_query = {
-            "select": "task_version:task_version_id(archive_path,package:package_id(name))",
-            "dataset_version_id": f"eq.{version_id}",
-            "order": "task_version_id",
-            "limit": 1000,
-            "offset": 0,
-        }
-        request = Request(
-            f"{url}/rest/v1/dataset_version_task?{urlencode(task_query)}",
-            headers={**headers, "Prefer": "count=exact"},
-        )
-        with urlopen(request) as response:
-            pages = [json.load(response)]
-            total = int(response.headers["Content-Range"].rsplit("/", 1)[1])
-
-        for offset in range(1000, total, 1000):
-            task_query["offset"] = offset
-            request = Request(
-                f"{url}/rest/v1/dataset_version_task?{urlencode(task_query)}",
-                headers=headers,
-            )
-            with urlopen(request) as response:
-                pages.append(json.load(response))
-        downloads = [
-            (
-                Path(name) / row["task_version"]["package"]["name"],
-                row["task_version"]["archive_path"],
-            )
-            for page in pages
-            for row in page
-        ]
-    else:
-        body = json.dumps({"p_org": org, "p_name": name, "p_ref": ref}).encode()
-        request = Request(
-            f"{url}/rest/v1/rpc/resolve_task_version", body, headers=headers
-        )
-        with urlopen(request) as response:
-            task = json.load(response)
-        if task is None:
-            raise ValueError(f"Harbor package not found: {dataset}")
-        downloads = [(Path(name), task["archive_path"])]
-
-    if not downloads:
-        raise ValueError(f"Harbor dataset has no tasks: {dataset}")
-
     CACHE.mkdir(parents=True, exist_ok=True)
-    # Stream archives concurrently into a temporary directory; publish only a complete cache.
+    # Publish only a complete CLI export to the cache.
     with tempfile.TemporaryDirectory(dir=CACHE) as temp:
-
-        def extract(item: tuple[Path, str]) -> None:
-            relative_path, archive_path = item
-            target = Path(temp) / relative_path
-            target.mkdir(parents=True, exist_ok=True)
-            request = Request(
-                f"{url}/storage/v1/object/authenticated/packages/{archive_path}",
-                headers=headers,
+        export_dir = Path(temp) / "export"
+        command = download_command(config, export_dir)
+        try:
+            subprocess.run(command, check=True, capture_output=True, text=True)
+        except subprocess.CalledProcessError as exc:
+            message = (
+                f"Harbor download failed for {config.dataset!r} with exit code "
+                f"{exc.returncode}"
             )
-            with urlopen(request) as response:
-                with tarfile.open(fileobj=response, mode="r|gz") as tar:
-                    kwargs = (
-                        {"filter": "data"} if hasattr(tarfile, "data_filter") else {}
-                    )
-                    tar.extractall(target, **kwargs)
-
-        with ThreadPoolExecutor(max_workers=100) as pool:
-            list(pool.map(extract, downloads))
+            outputs = [
+                output.strip()
+                for output in (exc.stdout, exc.stderr)
+                if isinstance(output, str) and output.strip()
+            ]
+            if output := "\n".join(outputs):
+                message = f"{message}:\n{output}"
+            raise RuntimeError(message) from exc
         try:
-            Path(temp).rename(out)
+            export_dir.rename(out)
         except OSError:
             if out.is_dir():
                 return out
@@ -312,7 +285,7 @@ def make_tar(directory: Path) -> bytes:
 
 class HarborTaskset(Taskset[HarborTask, HarborConfig]):
     def load_tasks(self) -> list[HarborTask]:
-        root = dataset_dir(self.config.dataset)
+        root = dataset_dir(self.config)
         task_dirs = [
             toml_path.parent
             for toml_path in sorted(root.rglob("task.toml"))

verifiers-0.1.15.dev402/verifiers/v1/tasksets/lean/__init__.py

@@ -0,0 +1,31 @@
+from verifiers.v1.tasksets.lean.scoring import (
+    build_starter_file,
+    expected_protected_signature,
+    parse_compile_output,
+    protected_signature_substring_present,
+    strip_lean_comments,
+)
+from verifiers.v1.tasksets.lean.taskset import (
+    DEFAULT_DOCKER_IMAGE,
+    LEAN_PROJECT_PATH,
+    PROOF_FILE_PATH,
+    LeanConfig,
+    LeanDatasetConfig,
+    LeanTask,
+    LeanTaskset,
+)
+
+__all__ = [
+    "DEFAULT_DOCKER_IMAGE",
+    "LEAN_PROJECT_PATH",
+    "PROOF_FILE_PATH",
+    "LeanConfig",
+    "LeanDatasetConfig",
+    "LeanTask",
+    "LeanTaskset",
+    "build_starter_file",
+    "expected_protected_signature",
+    "parse_compile_output",
+    "protected_signature_substring_present",
+    "strip_lean_comments",
+]

verifiers-0.1.15.dev402/verifiers/v1/tasksets/lean/scoring.py

@@ -0,0 +1,262 @@
+"""Pure helpers for the Lean taskset: starter-file construction, theorem-signature
+canonicalization, the reward-hacking signature guard, and compile-output parsing.
+
+Everything here is a pure function over plain strings — no runtime, no sandbox, no
+verifiers types — so it's trivially unit-testable and shared between ``setup``
+(building the starter file), the reward (the signature guard + compile parsing),
+and ``validate`` (planting the gold proof).
+"""
+
+from __future__ import annotations
+
+import re
+
+PROTECTED_HEADER_COMMENT = (
+    "-- DO NOT MODIFY the theorem statement below. The grader checks\n"
+    "-- that the original `theorem ... := by` text still appears in\n"
+    "-- this file. Only edit the proof body (currently `sorry`) and\n"
+    "-- lines after it."
+)
+
+
+# ── Imports / preamble normalization ─────────────────────────────────────────
+
+
+def normalize_imports(text: str) -> str:
+    """Collapse every ``import Mathlib*`` line to a single ``import Mathlib``.
+
+    Some datasets (miniF2F) ship fine-grained Mathlib imports that don't resolve
+    against the monolithic ``import Mathlib`` the sandbox image provides.
+    """
+    lines = text.split("\n")
+    out: list[str] = []
+    inserted = False
+    for line in lines:
+        if line.strip().startswith("import Mathlib"):
+            if not inserted:
+                out.append("import Mathlib")
+                inserted = True
+        else:
+            out.append(line)
+    return "\n".join(out)
+
+
+def _build_preamble(imports_str: str, header: str, normalize: bool) -> str:
+    if header and header.strip().startswith("import"):
+        preamble = header.strip()
+        return normalize_imports(preamble) if normalize else preamble
+    parts = [imports_str.strip()]
+    if header and header.strip():
+        parts.append(header.strip())
+    preamble = "\n\n".join(parts)
+    return normalize_imports(preamble) if normalize else preamble
+
+
+# ── Theorem signature canonicalization ───────────────────────────────────────
+
+
+def _normalize_signature(stmt: str) -> str:
+    """Canonicalize a Lean theorem statement to end with ``:= by``.
+
+    Strips trailing ``sorry``/``admit`` placeholders and any trailing ``by`` /
+    ``:=`` tokens, then re-appends `` := by``. Places the appended token on a new
+    indented line when the last line of the stripped signature already contains a
+    ``--`` comment (otherwise the ``:= by`` would land inside the line comment and
+    Lean would silently ignore it).
+    """
+    s = stmt.rstrip()
+    s = re.sub(r"\s*\b(?:sorry|admit)\b\s*$", "", s)
+    s = re.sub(r"\s*\bby\b\s*$", "", s)
+    s = re.sub(r"\s*:=\s*$", "", s)
+    s = s.rstrip()
+    last_newline = s.rfind("\n")
+    last_line = s[last_newline + 1 :] if last_newline != -1 else s
+    sep = "\n    " if "--" in last_line else " "
+    return s + sep + ":= by"
+
+
+def _split_imports_and_signature(stmt: str) -> tuple[str, str]:
+    decl_match = re.search(r"^(?:theorem|lemma|example)\s", stmt, flags=re.MULTILINE)
+    if not decl_match:
+        return "", stmt
+    return stmt[: decl_match.start()].rstrip(), stmt[decl_match.start() :]
+
+
+def expected_protected_signature(formal_statement: str) -> str:
+    """Return the canonical ``theorem ... := by`` block the reward pins as ground truth.
+
+    Pure function over ``formal_statement``; the imports/header portion (if the
+    statement carries one inline) is stripped first.
+    """
+    stmt = formal_statement or ""
+    if stmt.strip().startswith("import "):
+        _, signature_raw = _split_imports_and_signature(stmt)
+    else:
+        signature_raw = stmt
+    return _normalize_signature(signature_raw).strip()
+
+
+# ── Comment / string stripping for the signature guard ───────────────────────
+
+
+def strip_lean_comments(text: str) -> str:
+    """Remove Lean line/block comments **and string literals** from ``text``.
+
+    Lean comments come in two forms: ``-- ...`` line comments (to end of line) and
+    ``/- ... -/`` block comments (nestable; ``/-- ... -/`` doc comments are a
+    special case). String literals must also be stripped: a Lean
+    ``"theorem ... := by"`` constant or doc-string would otherwise let a model hide
+    the pinned signature inside a string while rewriting the live declaration to a
+    trivial one, defeating the substring guard. We handle both regular
+    double-quoted strings (with backslash escapes) and triple-quoted raw strings.
+
+    Block comments nest, so we count depth; line comments end at the next newline.
+    Outside comments and strings, newlines are preserved so the result keeps
+    roughly the right shape for substring matching downstream.
+    """
+    out: list[str] = []
+    i = 0
+    n = len(text)
+    block_depth = 0
+    in_line_comment = False
+    while i < n:
+        ch = text[i]
+        if in_line_comment:
+            if ch == "\n":
+                in_line_comment = False
+                out.append(ch)
+            i += 1
+            continue
+        if block_depth > 0:
+            if i + 1 < n and text[i : i + 2] == "-/":
+                block_depth -= 1
+                i += 2
+                continue
+            if i + 1 < n and text[i : i + 2] == "/-":
+                block_depth += 1
+                i += 2
+                continue
+            if ch == "\n":
+                out.append(ch)
+            i += 1
+            continue
+        if i + 1 < n and text[i : i + 2] == "/-":
+            block_depth = 1
+            i += 2
+            continue
+        if i + 1 < n and text[i : i + 2] == "--":
+            in_line_comment = True
+            i += 2
+            continue
+        # Triple-quoted raw string ``"""..."""`` — skip until the closing
+        # triple-quote, preserving newlines.
+        if i + 2 < n and text[i : i + 3] == '"""':
+            i += 3
+            while i < n:
+                if i + 2 < n and text[i : i + 3] == '"""':
+                    i += 3
+                    break
+                if text[i] == "\n":
+                    out.append("\n")
+                i += 1
+            continue
+        # Regular string ``"..."`` — handle ``\\`` escapes, stop at the next
+        # unescaped quote or newline (Lean strings are single-line).
+        if ch == '"':
+            i += 1
+            while i < n and text[i] != '"' and text[i] != "\n":
+                if text[i] == "\\" and i + 1 < n:
+                    i += 2
+                else:
+                    i += 1
+            if i < n and text[i] == '"':
+                i += 1
+            continue
+        out.append(ch)
+        i += 1
+    return "".join(out)
+
+
+def protected_signature_substring_present(
+    content: str, expected_signature: str
+) -> bool:
+    """True when the locked signature text still appears in the file.
+
+    Strips Lean comments from BOTH sides — without that a model could paste the
+    pinned signature into a ``--`` or ``/- ... -/`` block while rewriting the live
+    declaration to something trivial, and the asymmetric variant would also misfire
+    if ``expected_signature`` itself contains a comment. Tries an exact substring
+    match first, then a whitespace-flexible match (each side collapsed to
+    single-spaced tokens) so the model can re-indent or reflow whitespace freely.
+    """
+    if not expected_signature:
+        return True
+    decommented_content = strip_lean_comments(content)
+    decommented_expected = strip_lean_comments(expected_signature)
+    if not decommented_expected.strip():
+        return True
+    if decommented_expected in decommented_content:
+        return True
+    flat_signature = " ".join(decommented_expected.split())
+    flat_content = " ".join(decommented_content.split())
+    return flat_signature in flat_content
+
+
+# ── Starter-file construction ────────────────────────────────────────────────
+
+
+def build_starter_file(
+    formal_statement: str,
+    header: str = "",
+    imports: str = "import Mathlib",
+    normalize: bool = False,
+    proof_body: str | None = None,
+) -> str:
+    """Construct the starter proof file.
+
+    Layout: preamble (imports / header) + a brief ``-- DO NOT MODIFY`` comment
+    block + the normalized theorem signature + the proof body. If ``proof_body`` is
+    None (the default) the body is the placeholder ``  sorry`` — what's planted at
+    rollout start. A supplied gold ``proof_body`` (e.g. by ``validate``) replaces
+    the placeholder so the file is the full reference solution.
+    """
+    stmt = formal_statement or ""
+    if stmt.strip().startswith("import "):
+        imports_block, signature_raw = _split_imports_and_signature(stmt)
+        preamble = normalize_imports(imports_block) if normalize else imports_block
+    else:
+        preamble = _build_preamble(imports or "import Mathlib", header, normalize)
+        signature_raw = stmt
+
+    signature = _normalize_signature(signature_raw)
+    body = "  sorry" if proof_body is None else proof_body.rstrip()
+    wrapped = f"{PROTECTED_HEADER_COMMENT}\n{signature}\n{body}\n"
+    if preamble:
+        return preamble.rstrip() + "\n\n" + wrapped
+    return wrapped
+
+
+# ── Compile-output parsing ───────────────────────────────────────────────────
+
+
+def parse_compile_output(output: str) -> tuple[bool, str, int]:
+    """Parse a ``lake env lean ...; echo EXIT_CODE:$?`` transcript.
+
+    Returns ``(compiled, cleaned_output, exit_code)`` where ``compiled`` is True iff
+    the compiler exited 0 with no ``declaration uses 'sorry'`` diagnostic.
+
+    Matches the LAST ``EXIT_CODE:N`` — that's the one our shell appends at the end
+    of the command. Matching the first occurrence would let a model inject
+    ``#eval IO.println "EXIT_CODE:0"`` into the proof file to bypass the
+    sorry/exit-code checks: the regex would hit the injected marker, truncate
+    everything after it (hiding the real ``declaration uses 'sorry'`` diagnostic and
+    the real EXIT_CODE), and report success.
+    """
+    exit_code = 1
+    matches = list(re.finditer(r"EXIT_CODE:(\d+)", output))
+    if matches:
+        last = matches[-1]
+        exit_code = int(last.group(1))
+        output = output[: last.start()].strip()
+    has_sorry = bool(re.search(r"declaration uses 'sorry'", output))
+    return (exit_code == 0 and not has_sorry), output, exit_code