machinechecked 0.1.0__py3-none-any.whl

prooflens/pdfclaims.py

@@ -0,0 +1,129 @@
+"""Claims of a paper that is only a PDF: its numbered statements, found by their bold heads.
+
+A typeset theorem starts with its head in bold ("Lemma 8.6."), which distinguishes it from a
+reference to it in the text ("by Lemma 8.6"). The PDF is read with poppler's `pdftohtml -xml`,
+which keeps bold runs and positions. Each claim is placed from its head to the next head or
+proof on the same page; the end is an estimate (a PDF does not record where a statement ends),
+so its anchor is `approximate`.
+"""
+
+from __future__ import annotations
+
+import html
+import re
+import subprocess
+import tempfile
+import xml.etree.ElementTree as ET
+from pathlib import Path
+
+from .util import norm_ws, sha256_text
+
+KINDS = ("Theorem", "Lemma", "Proposition", "Corollary", "Definition", "Remark", "Example",
+         "Assumption", "Conjecture", "Claim", "Hypothesis", "Convention")
+HEAD = re.compile(r"^(" + "|".join(KINDS) + r")\s+([A-Z]?\d+(?:\.\d+)*[a-z]?)\s*(\.|\(|$)")
+STOP = re.compile(r"^(Proof\b|\d+(\.\d+)*\s+[A-Z])")
+
+
+BOLD_FONT = re.compile(r"bold|black|heavy|semibold|demi|cmbx|-bd\b|bx\d", re.I)
+
+
+def _lines(xml_text: str) -> list[dict]:
+    """Every text run, in reading order: page, top and bottom in PDF points, text, bold. A run
+    is bold when it is marked so, or when its font's name says so (Computer Modern's bold,
+    CMBX, is not marked)."""
+    root = ET.fromstring(xml_text)
+    bold_fonts = {f.get("id") for f in root.iter("fontspec")
+                  if BOLD_FONT.search(f.get("family", ""))}
+    out = []
+    for page in root.iter("page"):
+        pno = int(page.get("number"))
+        h = float(page.get("height"))
+        scale = 1.0
+        if page.get("pt_height"):
+            scale = float(page.get("pt_height")) / h
+        for t in page.iter("text"):
+            raw = ET.tostring(t, encoding="unicode", method="xml")
+            whole = t.get("font") in bold_fonts
+            bold = "<b>" in raw or whole
+            txt = html.unescape(re.sub(r"<[^>]+>", "", raw))
+            top, height = float(t.get("top")), float(t.get("height"))
+            out.append({"page": pno, "y0": top * scale, "y1": (top + height) * scale,
+                        "text": txt.strip(), "bold": bold,
+                        "head": txt.strip() if whole else html.unescape(re.sub(
+                            r"<[^>]+>", "", "".join(ET.tostring(b, encoding="unicode")
+                                                    for b in t.iter("b")))).strip()})
+    return out
+
+
+def page_heights(pdf: Path) -> dict[int, float]:
+    """Each page's height in points, from `pdfinfo`."""
+    r = subprocess.run(["pdfinfo", "-f", "1", "-l", "100000", str(pdf)], capture_output=True,
+                       text=True)
+    out = {}
+    for m in re.finditer(r"Page\s+(\d+)\s+size:\s+[\d.]+\s+x\s+([\d.]+)", r.stdout):
+        out[int(m.group(1))] = float(m.group(2))
+    return out
+
+
+def claims_from_pdf(pdf: Path) -> tuple[list[dict], tuple[float, float] | None]:
+    """The paper's numbered statements as anchors (the shape LaTeX anchors have), and its page
+    size in points."""
+    with tempfile.TemporaryDirectory(prefix="mc-pdf-") as tmp:
+        base = Path(tmp) / "doc"
+        r = subprocess.run(["pdftohtml", "-xml", "-i", "-q", "-nodrm", "-noframes", str(pdf),
+                            str(base)], capture_output=True, text=True, timeout=600)
+        x = base.with_suffix(".xml")
+        if r.returncode != 0 or not x.exists():
+            return [], None
+        xml_text = x.read_text(encoding="utf-8", errors="replace")
+    heights = page_heights(pdf)
+    root = ET.fromstring(xml_text)
+    xml_h = {int(p.get("number")): float(p.get("height")) for p in root.iter("page")}
+    xml_w = {int(p.get("number")): float(p.get("width")) for p in root.iter("page")}
+    lines = _lines(xml_text)
+    for l in lines:
+        s = heights.get(l["page"], 0) / xml_h[l["page"]] if heights.get(l["page"]) else 1.0
+        l["y0"] *= s
+        l["y1"] *= s
+    size = None
+    if heights:
+        p1 = min(heights)
+        size = (round(xml_w[p1] * heights[p1] / xml_h[p1], 2), round(heights[p1], 2))
+
+    heads = []
+    for i, l in enumerate(lines):
+        m = HEAD.match(l["head"]) if l["bold"] else None
+        if m:
+            heads.append((i, m))
+    anchors, seen = [], set()
+    for k, (i, m) in enumerate(heads):
+        kind, number = m.group(1).lower(), m.group(2)
+        if (kind, number) in seen:          # a statement restated later keeps its first place
+            continue
+        seen.add((kind, number))
+        nxt = heads[k + 1][0] if k + 1 < len(heads) else len(lines)
+        body, end = [], lines[i]
+        for j in range(i, nxt):
+            l = lines[j]
+            if j > i and (re.match(r"Proof\b", l["text"]) or STOP.match(l["text"]) and l["bold"]
+                          or l["page"] != lines[i]["page"] and len(body) > 3):
+                break
+            body.append(l["text"])
+            if l["page"] == lines[i]["page"]:
+                end = l
+        text = norm_ws(" ".join(body))
+        rest = text[len(m.group(0)):].lstrip() if text.startswith(m.group(0)) else text
+        title = None
+        tm = re.match(r"\(([^)]{1,120})\)\s*\.?", rest) if m.group(3) == "(" or \
+            rest.startswith("(") else None
+        if tm:
+            title = tm.group(1).strip()
+        statement = text[:4000]
+        anchors.append({
+            "id": len(anchors) + 1, "env": kind, "file": None, "line": None, "number": number,
+            "label": None, "labels": [], "title": title,
+            "begin": [lines[i]["page"], 0, round(lines[i]["y0"], 2)],
+            "head": None,
+            "end": [lines[i]["page"], 0, round(end["y1"], 2)],
+            "statement": statement, "source_sha256": sha256_text(statement), "kind": "pdf"})
+    return anchors, size

prooflens/pipeline.py

@@ -0,0 +1,331 @@
+"""The build: acquire the papers, run the probe, compile with anchors, assess, store the
+evidence, and write the private preview site."""
+
+from __future__ import annotations
+
+import os
+import shutil
+import subprocess
+from importlib import resources
+from importlib.metadata import PackageNotFoundError, version
+from pathlib import Path
+
+from . import evidence as ev
+from . import manifest as mf
+from . import review_queue as rq
+from .assess import Assessor, line_location
+from .config import Config
+from .probe import FINGERPRINT_VERSION, requests_from_inventory, run_probe
+from .reviews import load_dismissals, load_reviews
+from .schemas import validate
+from .sources import acquire
+from .tex import DEFAULT_ENVS, TexError, anchors_for, theorem_envs
+from .util import (canonical_json, git_dirty, git_head, norm_ws, read_json, sha256_file,
+                   sha256_text,
+                   write_json)
+
+
+def tool_version() -> str:
+    try:
+        return version("prooflens")
+    except PackageNotFoundError:
+        return "0.1.0+src"
+
+
+def project_provenance(cfg: Config) -> dict:
+    lr = cfg.lean_root
+    tc = lr / "lean-toolchain"
+    lm = lr / "lake-manifest.json"
+    prov = {"name": cfg.name, "commit": git_head(cfg.root), "dirty": git_dirty(cfg.root),
+            "toolchain": tc.read_text().strip() if tc.exists() else None,
+            "lake_manifest_sha256": sha256_file(lm) if lm.exists() else None,
+            "dependencies": {}}
+    if lm.exists():
+        for p in read_json(lm).get("packages", []):
+            prov["dependencies"][p.get("name")] = p.get("rev")
+    return prov
+
+
+def build(cfg: Config, release: str | None = None, skip_lean: bool = False,
+          skip_tex: bool = False, lake: str = "lake", web_dist: Path | None = None,
+          lean_build: bool = True, reuse_lean: bool = False, log=print) -> dict:
+    inv = read_json(cfg.inventory)
+    validate(inv, "inventory")
+    reviews, review_hashes = load_reviews(cfg.reviews, cfg.ai_reviewers, cfg.root)
+    dismissals = load_dismissals(cfg.dismissals)
+
+    lean = None
+    lean_digest = None
+    probe_used = probe_digest()  # the probe this build runs, read before it runs
+    if reuse_lean:
+        lean = reusable_lean(cfg)
+        missing = [r["name"] for r in requests_from_inventory(inv)
+                   if (r["kind"] == "spec" and r["name"] not in lean["specs"])
+                   or (r["kind"] == "decl" and r["name"] not in lean["defs"])
+                   or (r["kind"] in ("accept", "refute", "witness") and
+                       f"{r['kind']}:{r['name']}:{r.get('spec') or ''}" not in lean["proofs"])]
+        if missing:
+            raise RuntimeError(f"the stored probe does not cover {missing[:5]}; run the probe")
+        lean_digest = lean_inputs_digest(cfg)
+        log("probe: reusing the stored probe records (no Lean input changed)")
+    elif not skip_lean:
+        lean_digest = lean_inputs_digest(cfg)
+        log(f"probe: {len(requests_from_inventory(inv))} requests in {cfg.lean_root}")
+        lean = run_probe(cfg, requests_from_inventory(inv), lake=lake, build=lean_build)
+
+    tex, papers_prov, pdfs = {}, {}, {}
+    for paper in cfg.papers:
+        snap = acquire(cfg, paper, cfg.out_dir / "sources" / paper.id)
+        prov = {"title": paper.title, "kind": snap.kind, "revision": snap.revision,
+                "mode": snap.mode, "root": snap.root, "note": snap.note,
+                "committed_pdf": snap.pdf,
+                "committed_pdf_sha256": sha256_file(snap.dir / snap.pdf) if snap.pdf else None}
+        t = None
+        if snap.mode == "latex" and not skip_tex:
+            envs = paper.envs or sorted(set(DEFAULT_ENVS) | set(theorem_envs(snap.dir)))
+            log(f"tex: compiling {paper.id} ({snap.root})")
+            try:
+                t = anchors_for(snap, cfg.out_dir / "build" / paper.id, envs)
+            except TexError as e:
+                # The sandbox is not weakened to make a document compile: the source's own
+                # PDF is used instead, and the paper is marked as not compiled.
+                prov["compile_error"] = str(e)[-2000:]
+                log(f"tex: {paper.id} did not compile; using its committed PDF if any")
+        if t is not None:
+            tex[paper.id] = {"mode": "latex", "envs": envs, "page_size": t["page_size"],
+                             "inputs": t["inputs"],
+                             "anchors": [a.to_json() for a in t["anchors"]]}
+            prov["pdf_sha256"] = t["pdf_sha256"]
+            prov["pdf_origin"] = "compiled"
+            pdfs[paper.id] = t["pdf"]
+        elif snap.pdf:
+            tex[paper.id] = {"mode": "pdf", "anchors": [], "page_size": None, "inputs": []}
+            prov["mode"] = "pdf"
+            prov["pdf_sha256"] = prov["committed_pdf_sha256"]
+            prov["pdf_origin"] = "committed"
+            pdfs[paper.id] = snap.dir / snap.pdf
+        if paper.id in tex:
+            tex[paper.id]["lines"] = line_hashes(inv, paper.id, snap.dir)
+            if t is not None:
+                locs = [c["location"] for c in inv["claims"]
+                        if c["paper"] == paper.id and line_location(c) is not None]
+                tex[paper.id]["line_anchors"] = t["line_anchors"](locs)
+            tex[paper.id]["manual_anchors"] = manual_anchors(cfg, paper.id)
+        papers_prov[paper.id] = prov
+
+    comp = cfg.comparator_log
+    comp_text = comp.read_text(errors="replace") if comp and comp.exists() else None
+    assessment = Assessor(cfg, inv, lean, tex, reviews, dismissals, comp_text,
+                          git_head(cfg.lean_root)).run()
+    body = {
+        "provenance": {"tool": {"name": "prooflens", "version": tool_version(),
+                                "fingerprint": FINGERPRINT_VERSION},
+                       "probe_sha256": probe_used if lean else None,
+                       "lean_inputs_sha256": lean_digest,
+                       "lean_reused": reuse_lean,
+                       "lean": (lean or {}).get("env"), "project": project_provenance(cfg),
+                       "papers": papers_prov,
+                       "comparator_log_sha256": sha256_file(comp) if comp_text else None},
+        "inventory": {"file": os.path.relpath(cfg.inventory, cfg.root), "claims": inv["claims"]},
+        "reviews": {"files": review_hashes, "records": [r.to_json() for r in reviews]},
+        "dismissals": dismissals,
+        "lean": lean, "tex": tex, "assessment": assessment}
+    eid = ev.store(cfg.out_dir, body)
+    rel = release or f"r-{eid[:12]}"
+    ev.record_release(cfg.out_dir, rel, eid)
+    record = ev.load(cfg.out_dir, eid)
+    index, chunks, store = mf.build(record, rel, load_display(cfg))
+    index.update(issues=cfg.issues, site_url=cfg.site_url)
+    index["project"].update(source_info(cfg))
+    site = cfg.out_dir / "site"
+    write_preview(site, index, chunks, store, pdfs, web_dist,
+                  rq.assurance_queue(index, chunks, store, cfg.statement_authors))
+    log(f"evidence {eid[:12]}  release {rel}  site {site}")
+    return {"evidence": eid, "release": rel, "site": site, "index": index}
+
+
+LEAN_INPUTS = ["*.lean", "lakefile.toml", "lakefile.lean", "lake-manifest.json", "lean-toolchain"]
+
+
+def lean_input_roots(cfg: Config) -> list[Path]:
+    """The project and every path dependency of its lake manifest: their sources are inputs of
+    the probe; remote dependencies are pinned by the manifest itself."""
+    roots = [cfg.lean_root]
+    lm = cfg.lean_root / "lake-manifest.json"
+    if lm.exists():
+        for p in read_json(lm).get("packages", []):
+            if p.get("type") == "path":
+                roots.append((cfg.lean_root / p["dir"]).resolve())
+    return roots
+
+
+def lean_inputs_digest(cfg: Config) -> str:
+    """SHA-256 over the content of every Lean input of the project and its path dependencies
+    (sources, lakefiles, manifest, toolchain), committed or not."""
+    import hashlib
+    h = hashlib.sha256()
+    for root in lean_input_roots(cfg):
+        files = []
+        for pat in LEAN_INPUTS:
+            files += [f for f in root.rglob(pat) if ".lake" not in f.parts and ".git" not in
+                      f.parts and ".prooflens" not in f.parts]
+        h.update(f"root {root.name}\n".encode())
+        for f in sorted(set(files)):
+            h.update(f"{f.relative_to(root)}\n".encode())
+            h.update(hashlib.sha256(f.read_bytes()).digest())
+    return h.hexdigest()
+
+
+def probe_digest() -> str:
+    return sha256_text(resources.files("prooflens").joinpath("data/ProofLensProbe.lean")
+                       .read_text())
+
+
+def source_info(cfg: Config) -> dict:
+    """The project's source repository, for the reader's Lean source links."""
+    from . import audit
+    return audit.source_info(cfg.lean_root, cfg.scope_roots, cfg.repository,
+                             cfg.repository_private)
+
+
+def reusable_lean(cfg: Config) -> dict:
+    """The probe records of the latest evidence, if they still describe this checkout: the same
+    content of every Lean input (here and in every path dependency) and the same probe."""
+    rels = sorted((cfg.out_dir / "releases").glob("*.json"), key=lambda p: p.stat().st_mtime)
+    if not rels:
+        raise RuntimeError("no stored evidence to reuse")
+    body = ev.load(cfg.out_dir, read_json(rels[-1])["evidence"])["body"]
+    lean, prov = body.get("lean"), body["provenance"]
+    if not lean or not prov.get("lean_inputs_sha256") or not prov.get("probe_sha256"):
+        raise RuntimeError("the stored evidence does not record its Lean inputs; run the probe")
+    if prov["lean_inputs_sha256"] != lean_inputs_digest(cfg):
+        raise RuntimeError("Lean inputs changed since the stored probe (here or in a path "
+                           "dependency); run the probe")
+    if prov["probe_sha256"] != probe_digest():
+        raise RuntimeError("the probe changed since the stored records; run the probe")
+    return lean
+
+
+def work_dir(cfg: Config, paper) -> Path:
+    return cfg.out_dir / "build" / paper.id
+
+
+def manual_anchors(cfg: Config, pid: str) -> dict:
+    """Anchors placed by hand (`anchors.json` beside the configuration, versioned with the
+    project): `{claim id: {page, y0, y1, note, by}}` for the claims of paper `pid`."""
+    path = cfg.root / "anchors.json"
+    if not path.exists():
+        return {}
+    return {cid: a for cid, a in read_json(path).items() if cid.split(":")[0] == pid}
+
+
+def line_hashes(inv: dict, pid: str, src: Path) -> dict:
+    """The hash of every source line a claim of the paper is recorded at."""
+    out = {}
+    for c in inv["claims"]:
+        if c["paper"] != pid or line_location(c) is None:
+            continue
+        path, _, n = c["location"].rpartition(":")
+        try:
+            lines = (src / path).read_text(encoding="utf-8").split("\n")
+            out[c["location"]] = sha256_text(norm_ws(lines[int(n) - 1]))
+        except (OSError, IndexError):
+            pass
+    return out
+
+
+def reassess(cfg: Config, web_dist: Path | None = None, release: str | None = None,
+             log=print) -> dict:
+    """Assess the latest evidence again with the current rules and the current review files,
+    without probing or compiling, and store the result as a new evidence record and release."""
+    rels = sorted((cfg.out_dir / "releases").glob("*.json"), key=lambda p: p.stat().st_mtime)
+    if not rels:
+        raise RuntimeError("no release: run `prooflens build` first")
+    old = ev.load(cfg.out_dir, read_json(rels[-1])["evidence"])["body"]
+    inv = read_json(cfg.inventory)
+    validate(inv, "inventory")
+    if canonical_json(inv["claims"]) != canonical_json(old["inventory"]["claims"]):
+        raise RuntimeError("the claim inventory changed since the stored evidence: its Lean and "
+                           "LaTeX results may not cover it; run `prooflens build` (with "
+                           "--reuse-lean if no Lean input changed)")
+    reviews, review_hashes = load_reviews(cfg.reviews, cfg.ai_reviewers, cfg.root)
+    dismissals = load_dismissals(cfg.dismissals)
+    comp = cfg.comparator_log
+    comp_text = comp.read_text(errors="replace") if comp and comp.exists() else None
+    assessment = Assessor(cfg, inv, old["lean"], old["tex"], reviews, dismissals, comp_text,
+                          old["provenance"]["project"].get("commit")).run()
+    body = {**old, "inventory": {"file": os.path.relpath(cfg.inventory, cfg.root),
+                                 "claims": inv["claims"]},
+            "reviews": {"files": review_hashes, "records": [r.to_json() for r in reviews]},
+            "dismissals": dismissals, "assessment": assessment,
+            "provenance": {**old["provenance"],
+                           "tool": {**old["provenance"]["tool"], "version": tool_version()},
+                           "comparator_log_sha256": sha256_file(comp) if comp_text else None,
+                           "reassessed_from": read_json(rels[-1])["evidence"]}}
+    body.pop("schema", None)
+    eid = ev.store(cfg.out_dir, body)
+    rel = release or f"r-{eid[:12]}"
+    ev.record_release(cfg.out_dir, rel, eid)
+    log(f"evidence {eid[:12]}  release {rel} (reassessed)")
+    return rebuild_site(cfg, web_dist, rel)
+
+
+def pdf_paths(cfg: Config, body: dict) -> dict:
+    """The PDF each paper's evidence was built with: the compiled one, or the committed one."""
+    out = {}
+    for pid, prov in body["provenance"]["papers"].items():
+        if prov.get("pdf_origin") == "compiled" and prov.get("root"):
+            p = cfg.out_dir / "build" / pid / Path(prov["root"]).with_suffix(".pdf")
+        elif prov.get("committed_pdf"):
+            p = cfg.out_dir / "sources" / pid / prov["committed_pdf"]
+        else:
+            continue
+        if p.exists() and sha256_file(p) == prov.get("pdf_sha256"):
+            out[pid] = p
+    return out
+
+
+def rebuild_site(cfg: Config, web_dist: Path | None = None, release: str | None = None) -> dict:
+    """The preview site again from a stored release, without probing or compiling. A PDF is
+    included only if the one on disk is the one the evidence records."""
+    rels = sorted((cfg.out_dir / "releases").glob("*.json"), key=lambda p: p.stat().st_mtime)
+    if not rels:
+        raise RuntimeError("no release: run `prooflens build` first")
+    rel = read_json(cfg.out_dir / "releases" / f"{release}.json") if release else \
+        read_json(rels[-1])
+    record = ev.load(cfg.out_dir, rel["evidence"])
+    index, chunks, store = mf.build(record, rel["release"], load_display(cfg))
+    index.update(issues=cfg.issues, site_url=cfg.site_url)
+    index["project"].update(source_info(cfg))
+    site = cfg.out_dir / "site"
+    write_preview(site, index, chunks, store, pdf_paths(cfg, record["body"]), web_dist,
+                  rq.assurance_queue(index, chunks, store, cfg.statement_authors))
+    return {"release": rel["release"], "site": site, "index": index}
+
+
+def load_display(cfg: Config) -> dict | None:
+    return read_json(cfg.display) if cfg.display and cfg.display.exists() else None
+
+
+def write_preview(site: Path, index: dict, chunks: dict, store: dict, pdfs: dict,
+                  web_dist: Path | None, queue: dict | None = None):
+    if queue is not None:
+        index = {**index, "review_queue": "data/review-queue.json"}
+    if site.exists():
+        shutil.rmtree(site)
+    if web_dist and web_dist.exists():
+        shutil.copytree(web_dist, site)
+    data = site / "data"
+    validate(index, "manifest-index")
+    write_json(data / "index.json", index, indent=None)
+    for name, chunk in chunks.items():
+        validate(chunk, "manifest-claim")
+        write_json(data / "claims" / name, chunk, indent=None)
+    for key, record in store.items():
+        write_json(data / "decls" / f"{key}.json", record, indent=None)
+    write_json(data / "search.json", mf.search_index(index, chunks), indent=None)
+    if queue is not None:
+        write_json(data / "review-queue.json", queue, indent=None)
+    for pid, pdf in pdfs.items():
+        (data / "papers").mkdir(parents=True, exist_ok=True)
+        shutil.copyfile(pdf, data / "papers" / f"{pid}.pdf")

prooflens/probe.py

@@ -0,0 +1,126 @@
+"""Run the Lean probe inside a project and collect its records.
+
+The probe body (`data/ProofLensProbe.lean`) is prefixed with `import Lean` and the project's
+modules and run with `lake env lean` in the project root, with the project's own toolchain. The
+project is not modified and needs no dependency on ProofLens.
+"""
+
+from __future__ import annotations
+
+import json
+import os
+import tempfile
+from importlib import resources
+from pathlib import Path
+
+from .config import Config
+from .util import run, sha256_text
+
+FINGERPRINT_VERSION = "pl-fp-v1"
+
+
+class ProbeError(RuntimeError):
+    pass
+
+
+def requests_from_inventory(inv: dict) -> list[dict]:
+    """One request per specification, proof, refutation, witness and definition named by the
+    inventory, without duplicates and in a stable order."""
+    reqs, seen = [], set()
+
+    def add(r):
+        key = (r["kind"], r["name"], r.get("spec"))
+        if key not in seen:
+            seen.add(key)
+            reqs.append(r)
+
+    for c in inv["claims"]:
+        for cl in c.get("clauses", []):
+            for d in cl.get("defs", []):
+                add({"kind": "decl", "name": d})
+            if cl.get("spec"):
+                add({"kind": "spec", "name": cl["spec"]})
+                if cl.get("accept"):
+                    add({"kind": "accept", "name": cl["accept"], "spec": cl["spec"]})
+                if cl.get("refute"):
+                    add({"kind": "refute", "name": cl["refute"], "spec": cl["spec"]})
+            for w in [cl.get("witness")] + [r.get("witness") for r in cl.get("regimes", [])]:
+                if w:
+                    add({"kind": "witness", "name": w})
+    return reqs
+
+
+def fingerprint(canon: str) -> str:
+    return f"{FINGERPRINT_VERSION}:{sha256_text(canon)}"
+
+
+def run_probe(cfg: Config, requests: list[dict], lake: str = "lake",
+              timeout: int = 3600, build: bool = True) -> dict:
+    """Build the project's modules (unless `build` is false), run the probe, and return its
+    records grouped by kind, with every canonical serialisation replaced by its SHA-256
+    fingerprint."""
+    if build:
+        r = run([lake, "build", *cfg.modules], cwd=cfg.lean_root, timeout=timeout)
+        if r.returncode != 0:
+            raise ProbeError(f"lake build failed:\n{r.stdout[-4000:]}\n{r.stderr[-4000:]}")
+    body = resources.files("prooflens").joinpath("data/ProofLensProbe.lean").read_text()
+    header = "import Lean\n" + "".join(f"import {m}\n" for m in cfg.modules) + "\n"
+    with tempfile.TemporaryDirectory(prefix="prooflens-probe-") as tmp:
+        tmp = Path(tmp)
+        (tmp / "Probe.lean").write_text(header + body)
+        (tmp / "query.json").write_text(json.dumps({
+            "scope_roots": cfg.scope_roots, "spec_prefix": cfg.spec_prefix,
+            "accept_prefix": cfg.accept_prefix, "allowed_axioms": cfg.allowed_axioms,
+            "requests": requests}))
+        env = dict(os.environ, PROOFLENS_QUERY=str(tmp / "query.json"),
+                   PROOFLENS_OUT=str(tmp / "out.jsonl"))
+        r = run([lake, "env", "lean", str(tmp / "Probe.lean")], cwd=cfg.lean_root, env=env,
+                timeout=timeout)
+        out = tmp / "out.jsonl"
+        if r.returncode != 0 or not out.exists():
+            raise ProbeError(f"probe failed ({r.returncode}):\n{r.stdout[-4000:]}\n{r.stderr[-4000:]}")
+        return parse_records(out.read_text(encoding="utf-8").splitlines())
+
+
+def _strip_decl(rec: dict) -> dict:
+    rec = dict(rec)
+    rec["fingerprint"] = fingerprint(rec.pop("canon"))
+    rec.pop("record", None)
+    return rec
+
+
+def parse_records(lines: list[str]) -> dict:
+    res = {"env": None, "layering": [], "specs": {}, "proofs": {}, "decls": {}, "defs": {},
+           "complete": False}
+    for line in lines:
+        if not line.strip():
+            continue
+        rec = json.loads(line)
+        kind = rec.get("record")
+        if kind == "env":
+            res["env"] = {"lean": rec["lean"], "githash": rec["githash"]}
+        elif kind == "layering":
+            res["layering"] = rec["violations"]
+        elif kind == "decl":
+            res["decls"][rec["name"]] = _strip_decl(rec)
+        elif kind == "defclosure":
+            res["defs"][rec["name"]] = {"closure": rec.get("closure", []),
+                                        "errors": rec.get("errors", [])}
+        elif kind == "spec":
+            rec = dict(rec)
+            rec.pop("record")
+            if "decl" in rec:
+                d = _strip_decl(rec.pop("decl"))
+                res["decls"][d["name"]] = d
+                rec["legacy_hash"] = d["legacy_hash"]
+                rec["fingerprint"] = d["fingerprint"]
+            res["specs"][rec["name"]] = rec
+        elif kind in ("accept", "refute", "witness"):
+            rec = dict(rec)
+            rec["kind"] = rec.pop("record")
+            res["proofs"][f"{rec['kind']}:{rec['name']}:{rec.get('spec') or ''}"] = rec
+        elif kind == "done":
+            res["complete"] = True
+    if not res["complete"]:
+        raise ProbeError("probe output ends before its `done` record")
+    return res