leanscreen 0.1.0__py3-none-any.whl

leanscreen/__init__.py

@@ -0,0 +1,16 @@
+"""leanscreen — a calibrated faithfulness screen for informal↔Lean 4 pairs.
+
+The screen may only reject; it never certifies. See
+:data:`leanscreen.score.CALIBRATION_DISCLOSURE` for exactly what a verdict
+is worth.
+"""
+
+from leanscreen.score import (
+    CALIBRATION_DISCLOSURE,
+    ExternalPair,
+    PairScore,
+    score_pair,
+)
+
+__all__ = ["CALIBRATION_DISCLOSURE", "ExternalPair", "PairScore", "score_pair"]
+__version__ = "0.1.0"

leanscreen/backtranslate.py

@@ -0,0 +1,205 @@
+"""Step 7: back-translation review aid for faithfulness checking.
+
+A draft that *typechecks* and *passes the vacuity filter* is only "clean", not
+necessarily faithful: it may silently drop a hypothesis, weaken the conclusion,
+or add an assumption. The cheapest way to surface that for a human reviewer is to
+translate the Lean statement BACK to plain English (blind to the original) and
+put the two side by side — mismatches jump out.
+
+We make ONE model call per draft that returns a literal back-translation plus a
+flagged verdict. This is an AID, never a gate: the human still decides. It exists
+to make the review bottleneck (the real moat) fast, and to validate that our
+"clean" proxy actually tracks faithfulness.
+"""
+
+from __future__ import annotations
+
+import re
+from dataclasses import dataclass
+from typing import Protocol
+
+from leanscreen.llm import LLMClient, UsageTotals
+
+_SYSTEM_PROMPT = (
+    "You are a meticulous Lean 4 / mathlib proof engineer doing FAITHFULNESS "
+    "review. You are given an informal mathematical statement and a Lean 4 "
+    "formalization of it (proof omitted). Do two things:\n"
+    "1. Translate the LEAN statement back into plain English LITERALLY — say "
+    "exactly what the Lean says, including every hypothesis and the precise "
+    "conclusion. Do NOT consult the informal version while doing this; describe "
+    "the Lean on its own terms.\n"
+    "2. Compare your back-translation to the informal statement and decide whether "
+    "the Lean faithfully captures it. Watch for: dropped or weakened hypotheses, a "
+    "weakened or trivialized conclusion, extra assumptions the informal didn't "
+    "state, free parameters standing in for specific defined objects, and "
+    "quantifiers that changed scope.\n"
+    "Respond in EXACTLY this format:\n"
+    "BACK-TRANSLATION: <your literal English rendering of the Lean>\n"
+    "VERDICT: FAITHFUL | MISMATCH | UNCLEAR\n"
+    "ISSUES: <none, or a terse list of the specific discrepancies>"
+)
+
+# DEFINITION variant: same blind two-step and the SAME output format, but the
+# comparison rules are definition-shaped (defining conditions, type/arity,
+# degenerate bodies) instead of hypothesis/conclusion-shaped. Calibrate on the
+# human def verdicts (scripts/eval_judge.py --definitions) BEFORE wiring it as
+# a gate anywhere — the aid contract (reject-only, human decides) is unchanged.
+_SYSTEM_PROMPT_DEFINITION = (
+    "You are a meticulous Lean 4 / mathlib proof engineer doing FAITHFULNESS "
+    "review of a DEFINITION. You are given an informal mathematical definition "
+    "and a Lean 4 `def`/`abbrev`/`structure` intended to formalize it. Do two "
+    "things:\n"
+    "1. Translate the LEAN definition back into plain English LITERALLY — say "
+    "exactly what object or predicate it defines, over what arguments and "
+    "types, and what its body requires. Do NOT consult the informal version "
+    "while doing this; describe the Lean on its own terms.\n"
+    "2. Compare your back-translation to the informal definition and decide "
+    "whether the Lean defines the SAME concept. Watch for: a defining "
+    "condition from the informal that is missing, weakened, or replaced; a "
+    "substantive condition the informal never stated; the wrong type, arity, "
+    "or domain (a property of one object where the informal defines a "
+    "relation, a global constant where the informal defines a family); a "
+    "degenerate body (a literal True/False leaf standing in for a condition, "
+    "a constant ignoring its arguments); and paper-specific components left "
+    "as free unconstrained parameters instead of being defined.\n"
+    "Do NOT flag faithful modeling choices: Finset vs Set, a structure vs a "
+    "predicate, curried vs tupled arguments, meaning-preserving "
+    "generalizations, inlined notation, or an equivalent re-encoding of the "
+    "same condition.\n"
+    "Respond in EXACTLY this format:\n"
+    "BACK-TRANSLATION: <your literal English rendering of the Lean definition>\n"
+    "VERDICT: FAITHFUL | MISMATCH | UNCLEAR\n"
+    "ISSUES: <none, or a terse list of the specific discrepancies>"
+)
+
+# v2, gated behind ARXIV_FORMALIZE_SELFCHECK_CHECKLIST ('+sc2' cohorts): the
+# docs/faithfulness-patterns.md triage run clause-by-clause. Same blind
+# back-translation step and the SAME output format (everything downstream
+# parses it); only the comparison instructions change. Adopt only if the
+# judge-agreement eval beats the v1 baseline without raising false rejection.
+_SYSTEM_PROMPT_CHECKLIST = (
+    "You are a meticulous Lean 4 / mathlib proof engineer doing FAITHFULNESS "
+    "review. You are given an informal mathematical statement and a Lean 4 "
+    "formalization of it (proof omitted). Do two things:\n"
+    "1. Translate the LEAN statement back into plain English LITERALLY — say "
+    "exactly what the Lean says, including every hypothesis and the precise "
+    "conclusion. Do NOT consult the informal version while doing this; describe "
+    "the Lean on its own terms.\n"
+    "2. Compare against the informal statement by auditing EACH hypothesis and "
+    "EACH conclusion conjunct independently — most clauses being right says "
+    "nothing about the rest. Checks:\n"
+    "a. Does any hypothesis restate, contain, or one-step-yield the conclusion? "
+    "Assuming a premise the informal itself states is fine; assuming the "
+    "consequence the lemma exists to derive is a MISMATCH.\n"
+    "b. Is any conclusion conjunct true for every object of its type? Watch junk "
+    "values: Real.log 0 = 0, division by zero = 0, an ENNReal constant satisfied "
+    "by infinity, nat subtraction truncating, nat division flooring.\n"
+    "c. Did an informal decoration or index vanish — a torus/quotient norm as "
+    "bare abs, a one-sided derivative as deriv, a superscript-n sequence as an "
+    "n-free object, a ceiling as nat division? Judge operators by actual mathlib "
+    "semantics on that exact type.\n"
+    "d. Does the type lack structure the paper's regime uses (ZMod q is a field "
+    "only for prime q; commutative renderings of noncommutative claims)?\n"
+    "e. Is every stated hypothesis and constraint set present? Dropped "
+    "regularity/integrability/continuity or quantifying over everything where "
+    "the informal restricts to a constrained set usually makes it false.\n"
+    "f. Is a named subject declared but never used (dead parameter)?\n"
+    "g. Must a named specific ('all voters', 'the identity') be a concrete term "
+    "(univ, 1) rather than a fresh variable? Does every refinement ('depending "
+    "only on', 'in particular', multiplicity, 'is exactly') map to a Lean "
+    "clause? Any extra Lean conjunct not forced by the informal's wording?\n"
+    "h. Is every parameter pinned by a characterizing hypothesis or genuinely "
+    "ambient? Mentally instantiate free parameters with something trivial "
+    "(empty set, zero function, infinity): if the statement collapses or "
+    "falsifies, it is a MISMATCH.\n"
+    "NEVER flag these (they are faithful): pinned parameters (a defining "
+    "equation or iff); hypothesis strengthening unless the informal explicitly "
+    "names the weaker class; added implicit regularity (measurability, "
+    "positivity); provably equivalent re-encodings (preimage for image, cleared "
+    "denominators, sup-bound as forall); meaning-preserving generalizations; "
+    "inlined definitions; consistent reindexing; a faithful render of a claim "
+    "that is itself false (faithfulness is not truth).\n"
+    "Verdict: a demonstrated defect is MISMATCH; a load-bearing object you "
+    "cannot verify from what is shown (truncated informal, off-screen "
+    "definition, a convention whose two readings differ) is UNCLEAR even when "
+    "everything else is clean; all clauses passing including the never-flag "
+    "guards is FAITHFUL.\n"
+    "Respond in EXACTLY this format:\n"
+    "BACK-TRANSLATION: <your literal English rendering of the Lean>\n"
+    "VERDICT: FAITHFUL | MISMATCH | UNCLEAR\n"
+    "ISSUES: <none, or a terse list of the specific discrepancies>"
+)
+
+_BACK_RE = re.compile(
+    r"BACK-TRANSLATION:\s*(?P<v>.+?)(?=\n\s*VERDICT:|\Z)", re.DOTALL | re.IGNORECASE
+)
+_VERDICT_RE = re.compile(r"VERDICT:\s*(?P<v>FAITHFUL|MISMATCH|UNCLEAR)", re.IGNORECASE)
+_ISSUES_RE = re.compile(r"ISSUES:\s*(?P<v>.+?)\Z", re.DOTALL | re.IGNORECASE)
+
+_VALID_VERDICTS = {"faithful", "mismatch", "unclear"}
+
+
+@dataclass(frozen=True, slots=True)
+class BackReview:
+    """A back-translation review of one draft (an aid, not a verdict of record)."""
+
+    back_english: str
+    verdict: str  # "faithful" | "mismatch" | "unclear"
+    issues: str
+    raw: str
+
+
+class ReviewAid(Protocol):
+    """A faithfulness-review aid, so the pre-screen can inject a fake in tests."""
+
+    def review(self, informal: str, lean_code: str) -> BackReview:
+        """Return a back-translation review for one informal/Lean pair."""
+        ...
+
+
+def build_user_prompt(informal: str, lean_code: str) -> str:
+    """Compose the review prompt for one informal/Lean pair."""
+    return f"Informal statement:\n{informal.strip()}\n\nLean formalization:\n{lean_code.strip()}"
+
+
+def parse_review(response: str) -> BackReview:
+    """Parse the structured response; degrade gracefully if the format slips."""
+    back = _BACK_RE.search(response)
+    verdict = _VERDICT_RE.search(response)
+    issues = _ISSUES_RE.search(response)
+    v = verdict.group("v").strip().lower() if verdict else "unclear"
+    if v not in _VALID_VERDICTS:
+        v = "unclear"
+    return BackReview(
+        back_english=back.group("v").strip() if back else response.strip(),
+        verdict=v,
+        issues=issues.group("v").strip() if issues else "",
+        raw=response,
+    )
+
+
+class BackTranslator:
+    """Produces a back-translation faithfulness aid via an injected LLM client.
+
+    ``checklist=True`` selects the clause-by-clause v2 prompt (identical output
+    format, '+sc2' cohorts); the default stays the calibrated v1 prompt.
+    """
+
+    def __init__(
+        self, client: LLMClient, *, checklist: bool = False, definition: bool = False
+    ) -> None:
+        self._client = client
+        if definition:
+            self._system = _SYSTEM_PROMPT_DEFINITION
+        else:
+            self._system = _SYSTEM_PROMPT_CHECKLIST if checklist else _SYSTEM_PROMPT
+
+    @property
+    def usage(self) -> UsageTotals:
+        """Tokens spent by the aid's client (zero for test fakes)."""
+        return getattr(self._client, "usage", UsageTotals())
+
+    def review(self, informal: str, lean_code: str) -> BackReview:
+        """Return a back-translation review for one informal/Lean pair."""
+        response = self._client.complete(self._system, build_user_prompt(informal, lean_code))
+        return parse_review(response)

leanscreen/config.py

@@ -0,0 +1,69 @@
+"""Typed settings, populated from ``LEANSCREEN_``-prefixed env vars (or `.env`).
+
+Only what the screen needs: where Lean lives, which models judge, and the
+HTTP budget. No database, no storage, no pipeline knobs.
+"""
+
+from __future__ import annotations
+
+from pathlib import Path
+
+from pydantic import Field
+from pydantic_settings import BaseSettings, SettingsConfigDict
+
+
+class Settings(BaseSettings):
+    """Application settings (env prefix ``LEANSCREEN_``)."""
+
+    model_config = SettingsConfigDict(
+        env_prefix="LEANSCREEN_",
+        env_file=".env",
+        env_file_encoding="utf-8",
+        extra="ignore",
+    )
+
+    # --- Lean toolchain -----------------------------------------------------
+    lean_project_path: Path | None = Field(
+        default=None,
+        description="A Lean 4 project with mathlib as a dependency (built with `lake build`).",
+    )
+    lean_repl_path: Path | None = Field(
+        default=None,
+        description=(
+            "Path to the community REPL binary (repl/.lake/build/bin/repl). "
+            "With it, mathlib loads once (~100s) and each check takes ~0.1s; "
+            "without it, every check pays a full `lake env lean` invocation."
+        ),
+    )
+    lean_timeout_seconds: float = Field(default=180.0, gt=0.0)
+
+    # --- Anthropic API (check_deep only) ------------------------------------
+    anthropic_base_url: str = Field(default="https://api.anthropic.com/v1/messages")
+    anthropic_version: str = Field(default="2023-06-01")
+    anthropic_model: str = Field(
+        default="claude-opus-4-8",
+        description="Model for judge A and the counterexample probe (the calibrated default).",
+    )
+    judge_b_model: str = Field(
+        default="claude-fable-5",
+        description=(
+            "Model for the checklist judge (judge B). The default matches the "
+            "calibrated configuration. Fable/Mythos are locked-surface models: "
+            "thinking is always on and billed as output tokens, so judge B "
+            "gets a 32k max_tokens budget automatically."
+        ),
+    )
+    anthropic_timeout_seconds: float = Field(
+        default=600.0,
+        gt=0.0,
+        description=(
+            "HTTP read timeout. Always-thinking models can spend minutes on one "
+            "response, and a timeout that fires mid-generation still bills the "
+            "full response server-side — keep this generous."
+        ),
+    )
+    max_tokens: int = Field(
+        default=4096,
+        ge=1,
+        description="Response budget for judge A (judge B is sized automatically).",
+    )

leanscreen/errors.py

@@ -0,0 +1,47 @@
+"""Typed error hierarchy.
+
+Ground rule: no bare excepts, no silent failures. Every failure path raises a
+typed error from this module so callers can branch on *why* something failed.
+"""
+
+from __future__ import annotations
+
+
+class LeanScreenError(Exception):
+    """Base class for every error raised by this package."""
+
+    #: Stable machine-readable code.
+    code: str = "leanscreen_error"
+
+
+class ConfigError(LeanScreenError):
+    """Invalid or missing configuration."""
+
+    code = "config_error"
+
+
+class LLMError(LeanScreenError):
+    """The language-model API failed or returned an unusable response."""
+
+    code = "llm_error"
+
+
+class VerifierError(LeanScreenError):
+    """The Lean verifier could not be invoked (toolchain/config problem).
+
+    Note: a Lean *rejection* of a statement is a normal result, not this error;
+    this is raised only when the verifier itself cannot run.
+    """
+
+    code = "verifier_error"
+
+
+class CreditsExhaustedError(LeanScreenError):
+    """The API account has no credit — a GLOBAL condition, not a per-call one.
+
+    Deliberately NOT an LLMError subclass: per-call handlers catch LLMError
+    and continue, but an empty account dooms every subsequent call, so this
+    must propagate instead of being absorbed.
+    """
+
+    code = "credits_exhausted"

leanscreen/falsify.py

@@ -0,0 +1,116 @@
+"""Counterexample falsification probe — shared by the adversarial sweep script
+and the external scoring harness.
+
+A statement is FALSIFIED only when the model exhibits concrete values that
+satisfy every hypothesis while the conclusion fails, under real Lean/mathlib
+semantics (junk values included). Objective evidence, not judgment: the probe
+may only ever add a REJECT signal (invariant #4 — nothing here certifies).
+"""
+
+from __future__ import annotations
+
+import json
+from dataclasses import dataclass
+from typing import Any
+
+from leanscreen.llm import LLMClient
+
+FALSIFY_SYSTEM_PROMPT = (
+    "You attempt to FALSIFY a Lean 4 theorem statement by constructing an explicit "
+    "counterexample. This is a narrow, objective task — NOT a faithfulness judgment, "
+    "NOT a style review, NOT a provability assessment. A statement is FALSIFIED only "
+    "if you exhibit concrete values for its variables and free parameters such that "
+    "every hypothesis is satisfied and the conclusion is false, all under actual "
+    "Lean/mathlib semantics.\n"
+    "SEARCH ORDER — try each in turn:\n"
+    "1. Degenerate instantiations of unpinned parameters: the empty set, the zero "
+    "function, a constant function, Set.univ, t = 0, n = 0 or 1, the trivial "
+    "group/space. A parameter with no hypothesis linking it to the conclusion's "
+    "objects is free — instantiate it adversarially.\n"
+    "2. Junk values — these are REAL mathlib semantics and count: Real.log 0 = 0; "
+    "x / 0 = 0; an `∃ C : ENNReal` may pick C = ∞ (and ∞ * 0 = 0); nat subtraction "
+    "truncates at 0; nat division floors; Real.sqrt of a negative = 0; `deriv` of a "
+    "non-differentiable function = 0.\n"
+    "3. Boundary cases the hypotheses permit: x = -1 when the hypothesis is -1 ≤ x; "
+    "equality cases of ≤; empty index ranges (empty product = 1, empty sum = 0); "
+    "k = 0 exponents.\n"
+    "4. Extreme but legal values: a constant large enough or small enough to break "
+    "an unparameterized inequality.\n"
+    "RULES:\n"
+    "- The counterexample must be CHECKABLE: state the value of every relevant "
+    "variable, verify EACH hypothesis holds (including typeclass assumptions like "
+    "[Fact p.Prime] — pick a concrete instance), and show the conclusion fails.\n"
+    "- If you cannot satisfy all hypotheses simultaneously, the statement is "
+    "vacuously true — that is NOT falsified.\n"
+    "- 'I cannot prove it' is NOT falsified. Generality mismatches with the "
+    "informal statement are out of scope.\n"
+    "- Do not claim falsified on an uncertain computation: if the conclusion's "
+    "failure rests on arithmetic you are not sure of, report not falsified.\n"
+    "- Prefer the SIMPLEST counterexample that works.\n"
+    "- BUDGET your search: brief scratch work is fine, but keep it under ~300 words. "
+    "If the search is inconclusive by then, stop and report not falsified. You MUST "
+    "end your reply with the JSON verdict object — a reply without it is discarded "
+    "and wasted.\n"
+    "OUTPUT: end with exactly one JSON object:\n"
+    '{"falsified": true|false, '
+    '"counterexample": "<explicit assignment of every relevant variable, or empty>", '
+    '"check": "<hypotheses hold because ...; conclusion fails because ...>", '
+    '"informal_also_false": true|false|null}\n'
+    "Set informal_also_false = true ONLY when falsified AND the informal statement "
+    "plainly asserts the same false claim (then this may be a faithful render of a "
+    "false claim — a different problem than a translation error). Use null when "
+    "not falsified."
+)
+
+
+@dataclass(frozen=True, slots=True)
+class FalsifyResult:
+    """One falsification attempt's verdict."""
+
+    falsified: bool
+    counterexample: str
+    check: str
+    informal_also_false: bool | None
+
+
+def extract_json(raw: str) -> dict[str, Any] | None:
+    """Last balanced ``{...}`` in ``raw`` that parses as a JSON object.
+
+    A backward balanced-brace scan, not a regex: the model's prose routinely
+    contains Lean braces, and the verdict object is always last.
+    """
+    end = raw.rfind("}")
+    while end != -1:
+        depth = 0
+        for i in range(end, -1, -1):
+            if raw[i] == "}":
+                depth += 1
+            elif raw[i] == "{":
+                depth -= 1
+                if depth == 0:
+                    try:
+                        obj = json.loads(raw[i : end + 1])
+                    except json.JSONDecodeError:
+                        break
+                    return obj if isinstance(obj, dict) else None
+        end = raw.rfind("}", 0, end)
+    return None
+
+
+def falsify(client: LLMClient, informal: str, lean_code: str) -> FalsifyResult:
+    """Run one falsification attempt; raises ValueError on an unparseable reply."""
+    user = f"Informal statement:\n{informal}\n\nLean statement to falsify:\n{lean_code}"
+    raw = client.complete(FALSIFY_SYSTEM_PROMPT, user)
+    obj = extract_json(raw)
+    if obj is None:
+        raise ValueError(f"no JSON object in response: {raw[:120]!r}")
+    falsified = obj.get("falsified")
+    if not isinstance(falsified, bool):
+        raise ValueError(f"missing/invalid falsified field: {raw[:120]!r}")
+    also = obj.get("informal_also_false")
+    return FalsifyResult(
+        falsified=falsified,
+        counterexample=str(obj.get("counterexample", "")).strip()[:600],
+        check=str(obj.get("check", "")).strip()[:800],
+        informal_also_false=also if isinstance(also, bool) else None,
+    )