machinechecked 0.1.0__py3-none-any.whl

prooflens/export.py

@@ -0,0 +1,227 @@
+"""The public export.
+
+The export is assembled from fields each paper's `publish` settings allow; it is never a copy of
+the private preview with things removed. After assembly every output file is scanned against the
+private inputs (manuscript sources, Lean sources, PDFs not approved), and any match blocks the
+export.
+"""
+
+from __future__ import annotations
+
+import json
+import shutil
+from pathlib import Path
+
+from . import evidence as ev
+from . import manifest as mf
+from .config import Config
+from .util import read_json, sha256_bytes, sha256_file, write_json
+
+MIN_LINE = 30  # a private source line at least this long must not occur in the export
+
+
+class ExportError(RuntimeError):
+    pass
+
+
+def _pick(d: dict | None, keys: list[str]) -> dict | None:
+    if d is None:
+        return None
+    return {k: d[k] for k in keys if k in d}
+
+
+SEG_KEYS = ["text", "segments"]
+DECL_KEYS_LEAN = ["name", "kind", "module", "range", "doc", "signature", "value", "uses",
+                  "fingerprint", "legacy_hash"]
+DECL_KEYS_MIN = ["name", "kind", "module", "fingerprint", "legacy_hash"]
+
+
+def public_claim(chunk: dict, pub) -> dict:
+    """The fields of a claim passport that `pub` allows."""
+    out = _pick(chunk, ["schema", "release", "id", "paper", "number", "title", "kind", "scope",
+                        "scope_note", "scope_reviewed", "status", "class", "source_matches",
+                        "source_sha256", "blockers"])
+    out["location"] = chunk.get("location") if pub.source_text else None
+    out["statement_tex"] = chunk.get("statement_tex") if pub.source_text else None
+    a = chunk.get("anchor")
+    out["anchor"] = _pick(a, ["id", "env", "number", "label", "begin", "end", "source_sha256"] +
+                          (["file", "line"] if pub.source_text else [])) if a else None
+    clauses = []
+    for cl in chunk["clauses"]:
+        c = _pick(cl, ["name", "spec", "accept", "refute", "corrects", "witness", "defs", "state",
+                       "review", "checks", "ladder"])
+        c["regimes"] = [_pick(r, ["name", "description", "witness", "discharged"] +
+                              (["type"] if pub.lean else [])) for r in cl.get("regimes", [])]
+        if pub.lean and cl.get("statement"):
+            c["statement"] = cl["statement"]
+            c["proofs"] = cl.get("proofs")
+            c["witness_record"] = cl.get("witness_record")
+        else:
+            c["statement"] = None
+        if "spec_review" in cl:
+            c["spec_review"] = _reviews(cl["spec_review"], pub.reviews)
+        if "def_reviews" in cl:
+            c["def_reviews"] = {n: _pick(r, ["state", "by", "kind"])
+                                for n, r in cl["def_reviews"].items()}
+        if not pub.source_text:
+            for rung in c["ladder"]:
+                if rung["rung"] == "source" and rung["state"] == "pass":
+                    rung["detail"] = "located in the compiled paper"
+        clauses.append(c)
+    out["clauses"] = clauses
+    out["decls"] = dict(chunk.get("decls", {}))
+    out["findings"] = [_pick(f, ["id", "tier", "code", "subject", "message", "trigger",
+                                 "signal", "dismissed", "dismissal_stale"])
+                       for f in chunk["findings"]]
+    return out
+
+
+def _reviews(r: dict | None, with_text: bool) -> dict | None:
+    if r is None:
+        return None
+    keys = ["target", "name", "hash", "by", "kind", "provenance"] + (["text"] if with_text else [])
+    return {"state": r["state"], "current": [_pick(x, keys) for x in r.get("current", [])],
+            "stale": [_pick(x, keys) for x in r.get("stale", [])]}
+
+
+def private_inputs(cfg: Config) -> tuple[list[Path], set[str]]:
+    """Text files whose lines must not appear, and file hashes that must not appear."""
+    texts, hashes = [], set()
+    for paper in cfg.papers:
+        src = cfg.out_dir / "sources" / paper.id
+        if not paper.publish.source_text:
+            texts += [p for p in src.rglob("*") if p.suffix in (".tex", ".bib", ".sty", ".cls",
+                                                                  ".bbl", ".md")]
+        if not paper.publish.pdf:
+            hashes |= {sha256_file(p) for p in src.rglob("*.pdf")}
+            built = cfg.out_dir / "site" / "data" / "papers" / f"{paper.id}.pdf"
+            if built.exists():
+                hashes.add(sha256_file(built))
+    if not any(p.publish.lean for p in cfg.papers):
+        texts += [p for p in cfg.lean_root.rglob("*.lean") if ".lake" not in p.parts]
+    return texts, hashes
+
+
+def scan(out: Path, texts: list[Path], hashes: set[str]) -> list[str]:
+    """Every leak of a private input into the export."""
+    lines = {}
+    for t in texts:
+        for ln in t.read_text(encoding="utf-8", errors="replace").splitlines():
+            s = ln.strip()
+            if len(s) >= MIN_LINE and not s.startswith(("%", "--", "/-", "import ")):
+                lines.setdefault(s, t)
+    leaks = []
+    for f in sorted(out.rglob("*")):
+        if not f.is_file():
+            continue
+        b = f.read_bytes()
+        if sha256_bytes(b) in hashes:
+            leaks.append(f"{f.relative_to(out)}: is a private PDF")
+            continue
+        if f.suffix in (".json", ".html", ".js", ".css", ".txt", ".md", ".svg"):
+            text = b.decode("utf-8", errors="replace")
+            # JSON escapes; compare against the decoded strings as well.
+            hay = text
+            if f.suffix == ".json":
+                try:
+                    hay = text + "\n" + "\n".join(_strings(json.loads(text)))
+                except json.JSONDecodeError:
+                    pass
+            for s, src in lines.items():
+                if s in hay:
+                    leaks.append(f"{f.relative_to(out)}: contains a line of {src.name}")
+                    break
+    return leaks
+
+
+def _strings(obj):
+    if isinstance(obj, str):
+        yield obj
+    elif isinstance(obj, dict):
+        for v in obj.values():
+            yield from _strings(v)
+    elif isinstance(obj, list):
+        for v in obj:
+            yield from _strings(v)
+
+
+def export(cfg: Config, out: Path, release: str | None = None,
+           web_dist: Path | None = None) -> dict:
+    rels = sorted((cfg.out_dir / "releases").glob("*.json"), key=lambda p: p.stat().st_mtime)
+    if not rels:
+        raise ExportError("no release: run `prooflens build` first")
+    relrec = read_json(cfg.out_dir / "releases" / f"{release}.json") if release \
+        else read_json(rels[-1])
+    record = ev.load(cfg.out_dir, relrec["evidence"])
+    index, chunks, store = mf.build(record, relrec["release"])
+    pubs = {p.id: p.publish for p in cfg.papers}
+
+    if out.exists():
+        shutil.rmtree(out)
+    if web_dist and web_dist.exists():
+        shutil.copytree(web_dist, out)
+    data = out / "data"
+    pidx = _pick(index, ["schema", "release", "evidence", "created", "tool", "lean", "summary",
+                         "classes", "gates", "search"])
+    pidx["issues"], pidx["site_url"] = cfg.issues, cfg.site_url
+    pidx["project"] = _pick(index["project"], ["name", "commit", "dirty", "toolchain",
+                                               "repository", "repository_private",
+                                               "source_prefix", "source_roots",
+                                               "lake_manifest_sha256", "dependencies"])
+    from .pipeline import source_info
+    pidx["project"].update(source_info(cfg))
+    pidx["papers"] = []
+    for p in index["papers"]:
+        pub = pubs[p["id"]]
+        q = _pick(p, ["id", "title", "mode", "revision", "pdf_sha256", "page_size", "note"])
+        q["root"] = p.get("root") if pub.source_text else None
+        q["pdf"] = p["pdf"] if pub.pdf else None
+        pidx["papers"].append(q)
+        if pub.pdf and p["pdf"]:
+            src = cfg.out_dir / "site" / "data" / p["pdf"]
+            (data / "papers").mkdir(parents=True, exist_ok=True)
+            shutil.copyfile(src, data / p["pdf"])
+    pidx["findings"] = {"total": index["findings"]["total"],
+                        "general": [_pick(f, ["id", "tier", "code", "subject", "message"])
+                                    for f in index["findings"]["general"]]}
+    pidx["claims"] = [_pick(c, ["id", "paper", "number", "title", "kind", "scope", "status",
+                                "class", "anchor", "counts", "file"]) for c in index["claims"]]
+    write_json(data / "index.json", pidx, indent=None)
+    for name, chunk in chunks.items():
+        write_json(data / "claims" / name, public_claim(chunk, pubs[chunk["paper"]]),
+                   indent=None)
+    # Declarations are shared between papers: their Lean text and review texts are exported only
+    # when every paper allows it. A redacted record is a different record, so it gets its own key.
+    lean_ok = all(p.publish.lean for p in cfg.papers)
+    rev_ok = all(p.publish.reviews for p in cfg.papers)
+    rekey = {}
+    for key, d in store.items():
+        q = _pick(d, DECL_KEYS_LEAN if lean_ok else DECL_KEYS_MIN)
+        q["review"] = _reviews(d.get("review"), rev_ok)
+        new = mf.record_key(q)
+        rekey[key] = new
+        write_json(data / "decls" / f"{new}.json", q, indent=None)
+    for name in chunks:
+        path = data / "claims" / name
+        c = json.loads(path.read_text(encoding="utf-8"))
+        c["decls"] = {n: rekey[k] for n, k in c["decls"].items()}
+        write_json(path, c, indent=None)
+    # The search index, with only what each paper allows.
+    srch = mf.search_index(index, chunks)
+    pub_search = {"claims": [], "decls": srch["decls"] if lean_ok else []}
+    for c in srch["claims"]:
+        pub = pubs[c["paper"]]
+        pub_search["claims"].append({**c, "statement": c["statement"] if pub.source_text else "",
+                                     "lean": c["lean"] if pub.lean else []})
+    write_json(data / "search.json", pub_search, indent=None)
+    from . import audit
+    arx = [p.source["arxiv"] for p in cfg.papers if p.source.get("arxiv")]
+    write_json(out / audit.NAME, audit.descriptor(
+        pidx, cfg.lean_root, author=cfg.author,
+        paper_url=f"https://arxiv.org/abs/{arx[0]}" if len(cfg.papers) == 1 and arx else None))
+    texts, hashes = private_inputs(cfg)
+    leaks = scan(out, texts, hashes)
+    if leaks:
+        shutil.rmtree(out)
+        raise ExportError("export blocked; private material found:\n  " + "\n  ".join(leaks))
+    return {"release": relrec["release"], "evidence": relrec["evidence"], "out": out}

prooflens/formalization.py

@@ -0,0 +1,106 @@
+"""Reading a project's `formalization.yaml` (the mathlib-initiative standard).
+
+The file is the project's own account of itself: its sources, its main results, how it was made,
+its review status, and an `alignment` between the source's statements and Lean declarations. The
+alignment is free-form in the standard; the shape read here is its suggested one,
+`statements: [{source, lean, module, status}]`, with `lean` a name or a list of names.
+
+A mapping read from the file is a link the project's authors declared, shown as such; the
+file's review status is the project's own description of its review, shown as such. Neither
+changes what Lean is checked to establish.
+"""
+
+from __future__ import annotations
+
+import re
+from pathlib import Path
+
+NAMES = ("formalization.yaml", "formalization.yml")
+KINDS = ("theorem", "lemma", "proposition", "corollary", "definition", "remark", "example",
+         "conjecture", "claim", "assumption", "hypothesis", "convention")
+REF = re.compile(r"\b(" + "|".join(KINDS) + r")\s+([A-Z]?\d+(?:\.\d+)*[a-z]?)", re.I)
+
+
+def find(root: Path) -> Path | None:
+    return next((root / n for n in NAMES if (root / n).is_file()), None)
+
+
+def load(root: Path) -> dict | None:
+    """The project's formalization.yaml, parsed, or None. A file that cannot be read is
+    reported as `{"error": ...}`."""
+    p = find(root)
+    if p is None:
+        return None
+    try:
+        import yaml
+    except ImportError:
+        return {"error": "formalization.yaml found, but PyYAML is not installed"}
+    try:
+        data = yaml.safe_load(p.read_text(encoding="utf-8"))
+    except Exception as e:  # malformed YAML is the file's problem, not a crash
+        return {"error": f"formalization.yaml could not be read: {e}"}
+    return data if isinstance(data, dict) else {"error": "formalization.yaml is not a mapping"}
+
+
+def _tokens(s: str) -> set[str]:
+    return {t for t in re.split(r"[^\w]+", (s or "").lower()) if len(t) > 2}
+
+
+def statements(fy: dict) -> list[dict]:
+    """The alignment's statements in the suggested shape: each with the claim it names
+    (`kind`, `number`), the words that say which source it is in, and its Lean names."""
+    al = fy.get("alignment") if isinstance(fy, dict) else None
+    rows = al.get("statements") if isinstance(al, dict) else al if isinstance(al, list) else None
+    out = []
+    for r in rows or []:
+        if not isinstance(r, dict):
+            continue
+        lean = r.get("lean") or r.get("declaration") or r.get("declarations")
+        names = [lean] if isinstance(lean, str) else [n for n in lean or [] if isinstance(n, str)]
+        src = str(r.get("source") or "")
+        m = REF.search(src)
+        if not names or not m:
+            continue
+        rest = src[m.end():]
+        out.append({"source": src, "kind": m.group(1).lower(), "number": m.group(2),
+                    "hint": _tokens(rest), "lean": names, "status": r.get("status")})
+    return out
+
+
+def formalized_sources(fy: dict) -> list[dict]:
+    return [s for s in fy.get("sources") or [] if isinstance(s, dict)
+            and s.get("relationship", "formalizes") == "formalizes"]
+
+
+def links_for(fy: dict, claim_kind: str, claim_number: str, paper_title: str) -> list[dict]:
+    """The alignment's statements that name this claim of this paper. When the file formalizes
+    several sources, the paper is the source whose title it shares most words with, and a
+    statement must name it by a word only that source's title has ("(Euler)")."""
+    num = claim_number.split()[-1] if claim_number else ""
+    rows = [st for st in statements(fy) if st["kind"] == claim_kind.lower()
+            and st["number"] == num]
+    srcs = [_tokens(s.get("title", "")) for s in formalized_sources(fy)]
+    if len(srcs) <= 1 or not rows:
+        return rows
+    title = _tokens(paper_title)
+    best = max(range(len(srcs)), key=lambda i: len(srcs[i] & title))
+    if not srcs[best] & title:
+        return []
+    own = srcs[best] - set().union(*(t for i, t in enumerate(srcs) if i != best))
+    return [st for st in rows if st["hint"] & own]
+
+
+def summary(fy: dict | None) -> dict | None:
+    """What the reader shows about the file: the project's own review status and methods."""
+    if not fy:
+        return None
+    if "error" in fy:
+        return {"error": fy["error"]}
+    rv = fy.get("review") if isinstance(fy.get("review"), dict) else {}
+    auto = fy.get("automation") if isinstance(fy.get("automation"), dict) else {}
+    return {"review_status": rv.get("status"), "reviewers": rv.get("reviewers") or [],
+            "methods": [m.get("method") for m in auto.get("methods") or []
+                        if isinstance(m, dict) and m.get("method")],
+            "sources": [{"title": s.get("title"), "id": s.get("id")}
+                        for s in formalized_sources(fy)],
+            "version": fy.get("version")}

prooflens/manifest.py

@@ -0,0 +1,296 @@
+"""The presentation manifest, derived from one evidence record.
+
+`index.json` lists the papers and claims; `claims/<file>.json` holds one claim with everything
+its passport shows. Nothing here decides a status: statuses and findings are copied from the
+assessment in the evidence, and the evidence ladder is a fixed reading of the clause checks.
+"""
+
+from __future__ import annotations
+
+import re
+from collections import Counter
+
+from .assess import line_location, match_anchor
+from .util import content_id, sha256_text
+
+SCHEMA = "prooflens/manifest@0"
+
+
+def claim_file(cid: str) -> str:
+    return re.sub(r"[^A-Za-z0-9._-]", "_", cid) + "-" + sha256_text(cid)[:8] + ".json"
+
+
+def status_class(status: str) -> str:
+    """A coarse class for colouring; the status text is always shown verbatim beside it."""
+    if status in ("formal: accepted", "definition: reviewed"):
+        return "accepted"
+    if status.startswith("formal: proved") or status == "definition: author-reviewed":
+        return "proved"
+    if status.startswith("refuted"):
+        return "refuted"
+    if status in ("formal: open", "definition: review pending"):
+        return "open"
+    if status.endswith("malformed"):
+        return "malformed"
+    if status.startswith("out of scope"):
+        return "out-of-scope"
+    return "unspecified"
+
+
+def top_of(anchor: dict) -> tuple[int, float]:
+    """Page and top of a located claim: a line height above the baseline of its first line when
+    that is recorded, else the position where the environment opened."""
+    if anchor.get("head"):
+        return anchor["head"][0], round(anchor["head"][2] - 11, 2)
+    return anchor["begin"][0], anchor["begin"][2]
+
+
+def ladder(cl: dict, anchor: dict | None, mode: str | None, src_ok: bool | None) -> list[dict]:
+    """The rungs of the evidence ladder of one clause, each with its own state."""
+    ch = cl.get("checks", {})
+    proof = ch.get("accept") or ch.get("refute")
+    rungs = []
+    if cl["state"] == "unchecked":
+        rungs.append({"rung": "Lean", "state": "none",
+                      "detail": "the Lean project was not checked in this build"})
+    elif cl.get("defs") and not cl.get("spec"):
+        rungs.append({"rung": "stated", "state": "pass" if cl["state"] == "stated" else "fail",
+                      "detail": ", ".join(cl["defs"])})
+    else:
+        spec = ch.get("spec")
+        rungs.append({"rung": "locked statement",
+                      "state": "none" if not cl.get("spec") else
+                      ("pass" if spec and spec["ok"] else "fail"),
+                      "detail": cl.get("spec") or "no proposition written"})
+        if proof is None:
+            rungs.append({"rung": "exact statement", "state": "none", "detail": "no proof"})
+        else:
+            type_errs = [e for e in proof["errors"] if not e.startswith("axioms=")]
+            rungs.append({"rung": "exact statement",
+                          "state": "pass" if not type_errs and not proof["sorry"] else "fail",
+                          "detail": ("a theorem has exactly the negation as its type"
+                                     if ch.get("refute") else
+                                     "a theorem has exactly this proposition as its type"),
+                          "errors": proof["errors"]})
+            rungs.append({"rung": "axioms", "state": "pass" if not any(
+                e.startswith("axioms=") for e in proof["errors"]) and not proof["sorry"]
+                else "fail", "detail": ", ".join(proof["axioms"]) or "none"})
+            rp = ch.get("replay") or {"state": "not-run", "detail": "no Comparator log"}
+            rungs.append({"rung": "kernel replay", "state": rp["state"], "detail": rp["detail"]})
+        w = ch.get("witness")
+        rungs.append({"rung": "witness",
+                      "state": "n/a" if w is None else ("pass" if w["ok"] else "open"),
+                      "detail": cl.get("witness") or ("not needed" if w is None else "missing")})
+        regs = cl.get("regimes", [])
+        rungs.append({"rung": "regimes",
+                      "state": "n/a" if not regs else
+                      ("pass" if all(r["discharged"] for r in regs) else "open"),
+                      "detail": f"{sum(r['discharged'] for r in regs)}/{len(regs)} covered"
+                      if regs else "none declared"})
+    rev = cl.get("review", "unknown")
+    kinds = sorted({r["kind"] for r in (cl.get("spec_review") or {}).get("current", [])})
+    rungs.append({"rung": "review",
+                  "state": {"reviewed": "pass", "author-reviewed": "author",
+                            "pending": "open"}.get(rev, "none"),
+                  "detail": rev + (f" ({', '.join(kinds)})" if kinds else "")})
+    if anchor and anchor.get("kind") == "ambiguous":
+        rungs.append({"rung": "source", "state": "open" if src_ok is not False else "fail",
+                      "detail": f"{anchor['file']}:{anchor['line']} is typeset on pages "
+                                f"{', '.join(map(str, anchor['pages']))}: ambiguous, not placed "
+                                "(place it by hand in anchors.json)"})
+        return rungs
+    rungs.append({"rung": "source",
+                  "state": "pdf-only" if mode == "pdf" else
+                  ("none" if anchor is None else ("pass" if src_ok is not False else "fail")),
+                  "detail": "PDF only: statement not hashed from source" if mode == "pdf" else
+                  ("not located in the compiled paper" if anchor is None else
+                   f"{anchor.get('file', '')}:{anchor.get('line', '')}" +
+                   {"approximate": " (located approximately, by SyncTeX)",
+                    "manual": " (placed by hand)"}.get(anchor.get("kind"), ""))})
+    return rungs
+
+
+def build(evidence: dict, release: str, display: dict | None = None
+          ) -> tuple[dict, dict[str, dict]]:
+    """The presentation manifest of `evidence`. `display` (the project's display overlay) gives
+    per claim a TeX `title_tex` and a printed `number` to show in place of the inventory's; it
+    changes what is shown, not what is assessed."""
+    body = evidence["body"]
+    shown = (display or {}).get("claims", {})
+    clause_tex = {n: v["title_tex"] for n, v in (display or {}).get("clauses", {}).items()
+                  if v.get("title_tex")}
+    inv = body["inventory"]["claims"]
+    lean = body.get("lean") or {}
+    tex = body.get("tex") or {}
+    assessed = {c["id"]: c for c in body["assessment"]["claims"]}
+    findings = body["assessment"]["findings"]
+    by_subject: dict[str, list] = {}
+    for f in findings:
+        by_subject.setdefault(f["subject"].split("#")[0], []).append(f)
+
+    papers = []
+    for pid, prov in body["provenance"]["papers"].items():
+        t = tex.get(pid, {})
+        papers.append({"id": pid, "title": prov.get("title", pid), "mode": prov["mode"],
+                       "revision": prov.get("revision"), "root": prov.get("root"),
+                       "pdf": f"papers/{pid}.pdf" if prov.get("pdf_sha256") else None,
+                       "pdf_sha256": prov.get("pdf_sha256"), "page_size": t.get("page_size"),
+                       "note": prov.get("note", "")})
+
+    store = decl_records(body, lean)
+    keys = {n: record_key(r) for n, r in store.items()}
+    index_claims, chunks, summary = [], {}, {}
+    for c in inv:
+        a = assessed[c["id"]]
+        t = tex.get(c["paper"], {})
+        anchor = match_anchor(c, t.get("anchors", [])) if t.get("mode") == "latex" else None
+        src_ok = None
+        if anchor:
+            anchor = {**anchor, "kind": "exact"}
+            if c.get("source_sha256") and anchor.get("source_sha256"):
+                src_ok = anchor["source_sha256"] == c["source_sha256"]
+        elif line_location(c) is not None and t.get("mode") == "latex":
+            got = (t.get("lines") or {}).get(c["location"])
+            if c.get("source_sha256") and got:
+                src_ok = got == c["source_sha256"]
+            la = (t.get("line_anchors") or {}).get(c["location"])
+            if la and la.get("kind") == "ambiguous":
+                path, _, line = c["location"].rpartition(":")
+                anchor = {"kind": "ambiguous", "file": path, "line": int(line),
+                          "pages": la["pages"]}
+            elif la and la.get("kind", "approximate") == "approximate":
+                path, _, line = c["location"].rpartition(":")
+                anchor = {"kind": "approximate", "file": path, "line": int(line),
+                          "begin": [la["page"], 0, la["y0"]], "end": [la["page"], 0, la["y1"]]}
+        manual = (t.get("manual_anchors") or {}).get(c["id"])
+        if manual:
+            anchor = {**(anchor or {}), "kind": "manual", "note": manual.get("note", ""),
+                      "by": manual.get("by", ""), "head": None,
+                      "begin": [manual["page"], 0, manual["y0"]],
+                      "end": [manual["page"], 0, manual["y1"]]}
+        # The title the paper prints (`\\begin{theorem}[Efficiency]`), as TeX; the inventory's
+        # own description is kept as `title` and shown only as such.
+        heading = (anchor or {}).get("title") if (anchor or {}).get("kind") == "exact" else None
+        over = shown.get(c["id"], {})
+        number = over.get("number") or c.get("number", "")
+        description = over["title_tex"] if "title_tex" in over else c.get("title", "")
+        fs = by_subject.get(c["id"], [])
+        counts = Counter(f["tier"] for f in fs if not f.get("dismissed"))
+        summary.setdefault(c["paper"], Counter())[a["status"]] += 1
+        fname = claim_file(c["id"])
+        index_claims.append({
+            "id": c["id"], "paper": c["paper"], "number": number,
+            "title": description, "heading": heading, "kind": c.get("kind", ""),
+            "scope": c["scope"], "status": a["status"], "class": status_class(a["status"]),
+            "anchor": {"kind": anchor["kind"], "page": top_of(anchor)[0], "y0": top_of(anchor)[1],
+                       "y1": anchor["end"][2] if anchor.get("end") and
+                       anchor["end"][0] == anchor["begin"][0] else None,
+                       "page_end": anchor["end"][0] if anchor.get("end") else None}
+            if anchor and anchor.get("begin") else None,
+            "unplaced": anchor.get("kind") if anchor and not anchor.get("begin") else None,
+            "counts": dict(counts), "file": fname})
+        clauses = []
+        for cl in a["clauses"]:
+            spec = lean.get("specs", {}).get(cl.get("spec") or "")
+            proofs = {}
+            for kind in ("accept", "refute"):
+                if cl.get(kind):
+                    proofs[kind] = lean.get("proofs", {}).get(f"{kind}:{cl[kind]}:{cl['spec']}")
+            wit = lean.get("proofs", {}).get(f"witness:{cl.get('witness')}:") \
+                if cl.get("witness") else None
+            clauses.append({**cl, "label": clause_tex.get(cl["name"], cl["name"]),
+                            "corrects_label": clause_tex.get(cl.get("corrects") or "",
+                                                             cl.get("corrects")),
+                            "statement": {
+                "binders": spec.get("binders", []), "conclusion": spec.get("conclusion"),
+                "axioms": spec.get("axioms", []), "closure": spec.get("closure", []),
+                "direct": spec.get("direct", []), "legacy_hash": spec.get("legacy_hash"),
+                "fingerprint": spec.get("fingerprint"),
+                "decl": lean.get("decls", {}).get(cl["spec"])} if spec else None,
+                "proofs": proofs, "witness_record": wit,
+                "ladder": ladder(cl, anchor, t.get("mode"), src_ok)})
+        names = set()
+        for cl in clauses:
+            st = cl.get("statement") or {}
+            names.update(st.get("closure", []))
+            names.update(cl.get("defs", []))
+            names.update((cl.get("def_reviews") or {}).keys())
+            # A definition's review belongs to the definition (shared store); the clause keeps
+            # its state and reviewer.
+            cl["def_reviews"] = {n: {"state": r["state"],
+                                     "by": [x["by"] for x in r.get("current", [])],
+                                     "kind": [x["kind"] for x in r.get("current", [])]}
+                                 for n, r in (cl.get("def_reviews") or {}).items()}
+            if st:
+                st.pop("decl", None)
+        decls = {n: keys[n] for n in sorted(names) if n in keys}
+        chunks[fname] = {
+            "schema": SCHEMA, "release": release, "id": c["id"], "paper": c["paper"],
+            "number": number, "number_note": over.get("number_note"), "title": description,
+            "heading": heading,
+            "kind": c.get("kind", ""), "scope": c["scope"], "scope_note": c.get("scope_note", ""),
+            "scope_reviewed": c.get("scope_reviewed"), "location": c.get("location"),
+            "source_sha256": c.get("source_sha256"), "status": a["status"],
+            "class": status_class(a["status"]), "anchor": anchor, "source_matches": src_ok,
+            "statement_tex": (anchor or {}).get("statement"),
+            "blockers": [shown_names(b, clause_tex) for b in a.get("blockers", [])],
+            "clauses": clauses, "decls": decls, "findings": fs}
+
+    prov = body["provenance"]
+    index = {"schema": SCHEMA, "release": release, "evidence": evidence["id"],
+             "created": evidence["created"], "project": prov["project"],
+             "lean": prov.get("lean"), "tool": prov["tool"], "papers": papers,
+             "summary": {p: dict(s) for p, s in summary.items()},
+             "classes": {st: status_class(st) for s in summary.values() for st in s},
+             "gates": body["assessment"].get("gates", {}),
+             "findings": {"total": dict(Counter(f["tier"] for f in findings
+                                                if not f.get("dismissed"))),
+                          "general": [f for f in findings
+                                      if f["subject"].split("#")[0] not in
+                                      {c["id"] for c in inv}]},
+             "claims": index_claims}
+    index["search"] = "data/search.json"
+    return index, chunks, {keys[n]: r for n, r in store.items()}
+
+
+def shown_names(text: str, clause_tex: dict[str, str]) -> str:
+    """`text` with the clause names it quotes in their displayed (TeX) form."""
+    for n in sorted(clause_tex, key=len, reverse=True):
+        if n in text:
+            text = text.replace(n, clause_tex[n])
+    return text
+
+
+def search_index(index: dict, chunks: dict) -> dict:
+    """What the reader searches: every claim (number, title, id, status, printed statement, Lean
+    names of its clauses) and every declaration (with the claims whose closure holds it)."""
+    claims, decls = [], {}
+    for c in index["claims"]:
+        ch = chunks[c["file"]]
+        lean = [n for cl in ch["clauses"] for n in (cl.get("spec"), cl.get("accept"),
+                                                    cl.get("refute"), *cl.get("defs", [])) if n]
+        claims.append({"id": c["id"], "paper": c["paper"], "number": c.get("number", ""),
+                       "title": c.get("title", ""), "heading": c.get("heading"),
+                       "status": c["status"],
+                       "statement": ch.get("statement_tex") or "", "lean": lean})
+        for n in ch.get("decls", {}):
+            decls.setdefault(n, []).append(c["id"])
+    return {"claims": claims, "decls": [{"name": n, "claims": ids} for n, ids in
+                                        sorted(decls.items())]}
+
+
+def record_key(record: dict) -> str:
+    """The key of a declaration record: the first 24 hex digits of the SHA-256 of its canonical
+    JSON. The record holds the declaration's name, kind and fingerprint, so different
+    declarations never share a key, and a record never changes under its key."""
+    return content_id(record)[:24]
+
+
+def decl_records(body: dict, lean: dict) -> dict[str, dict]:
+    """Every declaration once, with its current review."""
+    reviews: dict[str, dict] = {}
+    for c in body["assessment"]["claims"]:
+        for cl in c["clauses"]:
+            for n, r in (cl.get("def_reviews") or {}).items():
+                reviews.setdefault(n, r)
+    return {n: {**d, "review": reviews.get(n)} for n, d in lean.get("decls", {}).items()}