machinechecked 0.1.3__tar.gz → 0.1.5__tar.gz

{machinechecked-0.1.3 → machinechecked-0.1.5}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: machinechecked
-Version: 0.1.3
+Version: 0.1.5
 Summary: Read a mathematical paper beside its Lean formalisation, and see claim by claim what Lean establishes.
 License: Apache-2.0
 Project-URL: Homepage, https://machinechecked.org

{machinechecked-0.1.3 → machinechecked-0.1.5}/engine/python/machinechecked.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: machinechecked
-Version: 0.1.3
+Version: 0.1.5
 Summary: Read a mathematical paper beside its Lean formalisation, and see claim by claim what Lean establishes.
 License: Apache-2.0
 Project-URL: Homepage, https://machinechecked.org

{machinechecked-0.1.3 → machinechecked-0.1.5}/engine/python/machinechecked.egg-info/SOURCES.txt

@@ -18,6 +18,7 @@ engine/python/prooflens/evidence.py
 engine/python/prooflens/export.py
 engine/python/prooflens/formalization.py
 engine/python/prooflens/manifest.py
+engine/python/prooflens/paper.py
 engine/python/prooflens/pdfclaims.py
 engine/python/prooflens/pipeline.py
 engine/python/prooflens/probe.py
@@ -37,6 +38,7 @@ engine/python/prooflens/data/schema/inventory.schema.json
 engine/python/prooflens/data/schema/manifest-claim.schema.json
 engine/python/prooflens/data/schema/manifest-index.schema.json
 engine/python/prooflens/web/index.html
+engine/python/prooflens/web/theme.css
 engine/python/prooflens/web/assets/KaTeX_AMS-Regular-BQhdFMY1.woff2
 engine/python/prooflens/web/assets/KaTeX_Caligraphic-Bold-Dq_IR9rO.woff2
 engine/python/prooflens/web/assets/KaTeX_Caligraphic-Regular-Di6jR-x-.woff2
@@ -56,6 +58,6 @@ engine/python/prooflens/web/assets/KaTeX_Size1-Regular-mCD8mA8B.woff2
 engine/python/prooflens/web/assets/KaTeX_Size2-Regular-Dy4dx90m.woff2
 engine/python/prooflens/web/assets/KaTeX_Size4-Regular-Dl5lxZxV.woff2
 engine/python/prooflens/web/assets/KaTeX_Typewriter-Regular-CO6r4hn1.woff2
-engine/python/prooflens/web/assets/index-CEQVnsCC.js
+engine/python/prooflens/web/assets/index-D0aKe720.js
 engine/python/prooflens/web/assets/index-T9UdK0vf.css
 engine/python/prooflens/web/assets/pdf.worker.min-yatZIOMy.mjs

{machinechecked-0.1.3 → machinechecked-0.1.5}/engine/python/prooflens/cli.py

@@ -8,7 +8,8 @@ Everyday commands, run from anywhere inside a Lean project:
 
 A project with a `machinechecked.toml` (an acceptance layer: locked statements, reviews) gets
 the full assessment; any other project gets the zero-configuration view, whose only file in the
-repository is `machinechecked.links.json`. The other commands are for that full assessment.
+repository is `machinechecked.links.json` (its working files, a downloaded paper among them, are
+in `.machinechecked/`, which Git ignores). The other commands are for that full assessment.
 `prooflens` is another name for the same command.
 """
 
@@ -149,7 +150,8 @@ def _fetch_paper(root: Path, url: str, title: str | None = None) -> Path:
     repository), with its address and title beside it."""
     import hashlib
     import urllib.request
-    d = root / cfgmod.work_dir_name(root) / "papers"
+    from .util import prepare_work_dir
+    d = prepare_work_dir(root / cfgmod.work_dir_name(root)) / "papers"
     name = re.sub(r"[^\w.-]+", "_", url.rstrip("/").rsplit("/", 1)[-1]) or "paper"
     if not name.lower().endswith(".pdf"):
         name += ".pdf"
@@ -357,7 +359,8 @@ def cmd_export(a):
     from .export import ExportError, export
     cfg = cfgmod.load(_config_arg(a))
     try:
-        res = export(cfg, Path(a.out).resolve(), a.release, _web_dist(a.web))
+        res = export(cfg, Path(a.out).resolve(), a.release,
+                     None if a.data_only else _web_dist(a.web))
     except ExportError as e:
         sys.exit(str(e))
     print(f"exported release {res['release']} to {res['out']}")
@@ -471,7 +474,8 @@ def main(argv=None):
     b.set_defaults(func=cmd_build)
 
     k = sub.add_parser("check", help="report what the Lean side establishes (exit 1 if a "
-                                     "linked declaration is missing or unproved)")
+                                     "linked declaration is missing or unproved, 2 if the check "
+                                     "could not run)")
     k.add_argument("paper", nargs="?")
     k.add_argument("--json", action="store_true",
                    help="print every claim's status and links as JSON (for scripts and agents)")
@@ -504,6 +508,9 @@ def main(argv=None):
     e.add_argument("--release")
     e.add_argument("--out", required=True)
     e.add_argument("--web")
+    e.add_argument("--data-only", action="store_true",
+                   help="write the audit's data only, without the reader (for a site that "
+                   "reads it with its own reader, such as machinechecked.org)")
     e.set_defaults(func=cmd_export)
 
     s = sub.add_parser("serve")
@@ -531,11 +538,49 @@ def main(argv=None):
     a = p.parse_args(argv)
     try:
         a.func(a)
+    except SystemExit as e:
+        # `sys.exit("message")` is a command that could not do its job; an integer is its result.
+        if isinstance(e.code, str):
+            _could_not_run(a, "error", e.code.removeprefix("machinechecked: "))
+        raise
     except FileNotFoundError as e:
-        hint = MISSING.get(Path(str(e.filename or "")).name)
-        if not hint:
-            raise
-        sys.exit(f"machinechecked: `{Path(e.filename).name}` not found: {hint}")
+        tool = Path(str(e.filename or "")).name
+        if tool in MISSING:
+            _could_not_run(a, "missing-tool", f"`{tool}` not found: {MISSING[tool]}", tool=tool)
+        _could_not_run(a, "internal", _crash(e))
+    except (RuntimeError, ValueError, OSError) as e:
+        # The engine's own errors (the Lean probe, TeX, the paper's sources, the configuration,
+        # stored evidence) say what went wrong in their message.
+        if type(e).__module__.startswith("prooflens"):
+            _could_not_run(a, "error", str(e))
+        _could_not_run(a, "internal", _crash(e))
+    except Exception as e:
+        _could_not_run(a, "internal", _crash(e))
+
+
+# Exit codes: 0 the command did its job (and `check` found nothing wrong), 1 `check` (or
+# `build --fail-on-violation`) found a problem, 2 the command could not run.
+COULD_NOT_RUN = 2
+
+
+def _crash(e: Exception) -> str:
+    """An unexpected error: its traceback on stderr, for a bug report, and a one-line message."""
+    import traceback
+    traceback.print_exc()
+    return f"unexpected error ({type(e).__name__}: {e}); please report it with the traceback above"
+
+
+def _could_not_run(a, kind: str, message: str, **extra):
+    """Stop with exit 2: the message on stderr and, for `check --json`, one JSON document on
+    stdout saying the check did not run."""
+    print(f"machinechecked: {message}", file=sys.stderr)
+    if getattr(a, "func", None) is cmd_check and getattr(a, "json", False):
+        import json
+        print(json.dumps({"schema": "machinechecked/check@1", "ok": False,
+                          "error": {"kind": kind, "message": message, **extra},
+                          "note": "The check did not run; nothing is known about the claims."},
+                         indent=1))
+    raise SystemExit(COULD_NOT_RUN)
 
 
 if __name__ == "__main__":

{machinechecked-0.1.3 → machinechecked-0.1.5}/engine/python/prooflens/data/schema/manifest-index.schema.json

@@ -103,7 +103,8 @@
               "open",
               "malformed",
               "out-of-scope",
-              "unspecified"
+              "unspecified",
+              "declared"
             ]
           },
           "anchor": {

{machinechecked-0.1.3 → machinechecked-0.1.5}/engine/python/prooflens/formalization.py

@@ -18,7 +18,8 @@ from pathlib import Path
 NAMES = ("formalization.yaml", "formalization.yml")
 KINDS = ("theorem", "lemma", "proposition", "corollary", "definition", "remark", "example",
          "conjecture", "claim", "assumption", "hypothesis", "convention")
-REF = re.compile(r"\b(" + "|".join(KINDS) + r")\s+([A-Z]?\d+(?:\.\d+)*[a-z]?)", re.I)
+REF = re.compile(r"\b(" + "|".join(KINDS) + r")\s+((?:[A-Z](?:\.\d+)+|[A-Z]?\d+(?:\.\d+)*)[a-z]?)",
+                 re.I)
 
 
 def find(root: Path) -> Path | None:

{machinechecked-0.1.3 → machinechecked-0.1.5}/engine/python/prooflens/manifest.py

@@ -45,6 +45,18 @@ def top_of(anchor: dict) -> tuple[int, float]:
     return anchor["begin"][0], anchor["begin"][2]
 
 
+def index_anchor(anchor: dict | None, kind: str) -> dict | None:
+    """A located claim's place as the index lists it (for the contents and the margin): its
+    page and top, where it ends on that page, and the page it ends on."""
+    if not anchor or not anchor.get("begin"):
+        return None
+    page, y0 = top_of({**anchor, "kind": kind})
+    end = anchor.get("end")
+    return {"kind": kind, "page": page, "y0": y0,
+            "y1": end[2] if end and end[0] == anchor["begin"][0] else None,
+            "page_end": end[0] if end else None}
+
+
 def ladder(cl: dict, anchor: dict | None, mode: str | None, src_ok: bool | None) -> list[dict]:
     """The rungs of the evidence ladder of one clause, each with its own state."""
     ch = cl.get("checks", {})

machinechecked-0.1.5/engine/python/prooflens/paper.py

@@ -0,0 +1,103 @@
+"""A paper's page: the paper's PDF, the statements found in it, and for each Lean project that
+states it formalises the paper, the declarations its `formalization.yaml` maps them to.
+
+It is written in the reader's standard data format (`data/index.json`, `data/claims/`), so a
+paper page is read by the same code as a view or an audit. Nothing here runs Lean, so a mapped
+statement reads "linked: Lean not checked here (no audit published)": the link says which
+declaration the project names, not what Lean establishes. Each project's links and statuses are
+kept apart (`by_project`); the reader shows one project at a time and never combines them.
+
+No statement's text is written: only its number, heading title and place on the page.
+"""
+
+from __future__ import annotations
+
+import re
+from pathlib import Path
+
+from . import formalization as fz
+from . import manifest as mf
+from .util import content_id
+
+SCHEMA = "prooflens/manifest@0"
+NOT_CHECKED = "linked: Lean not checked here (no audit published)"
+NOT_LINKED = "not linked"
+NAME = re.compile(r"[\w.'«»]+")
+
+
+def paper_claims(anchors: list[dict], title: str,
+                 alignments: list[tuple[dict, dict | None]]) -> list[dict]:
+    """The statements found in the paper (anchors from `pdfclaims.claims_from_pdf`), each with
+    the links the projects' `formalization.yaml` files declare for it. `alignments` pairs each
+    project (`repository`, `commit`) with its parsed `formalization.yaml`, or None when it has
+    none or it could not be read."""
+    out = []
+    for a in anchors:
+        number = f"{a['env'].capitalize()} {a['number']}"
+        links = []
+        for f, fy in alignments:
+            for row in fz.links_for(fy, a["env"], number, title) if fy else []:
+                links += [{"name": n, "repository": f["repository"], "commit": f["commit"],
+                           "source": "formalization.yaml"}
+                          for n in row["lean"] if NAME.fullmatch(n)]
+        out.append({"key": f"{a['env']}-{a['number']}", "number": number, "kind": a["env"],
+                    "title": a.get("title"),
+                    "anchor": {k: a[k] for k in ("begin", "end", "head") if a.get(k)},
+                    "links": links})
+    return out
+
+
+def _state(links: list[dict]) -> tuple[str, str]:
+    return (NOT_CHECKED, "declared") if links else (NOT_LINKED, "unspecified")
+
+
+def write_paper_site(site: Path, paper: dict, claims: list[dict],
+                     page_size: list | tuple | None, pdf_url: str | None,
+                     web_dist: Path | None = None) -> dict:
+    """Write the paper page's data under `site/data/` (and the reader, given its built
+    directory). `paper` is the paper's registry entry (its identifier, title, authors, links and
+    formalisations), which the reader's panel beside the paper shows; `claims` are from
+    `paper_claims`. Replaces whatever is at `site`."""
+    from .pipeline import write_preview
+    projects = [f["repository"] for f in paper.get("formalisations", [])]
+    index_claims, chunks = [], {}
+    for c in claims:
+        cid = f"P:{c['key']}"
+        fname = mf.claim_file(cid)
+        anchor = {**c["anchor"], "kind": "approximate"} if c.get("anchor") else None
+        by_project = {}
+        for r in projects:
+            status, cls = _state([l for l in c["links"] if l["repository"] == r])
+            by_project[r] = {"status": status, "class": cls}
+        status, cls = _state([l for l in c["links"]
+                              if not projects or l["repository"] == projects[0]])
+        common = {"id": cid, "paper": "P", "number": c["number"], "title": c.get("title"),
+                  "heading": c.get("title") or None, "kind": c["kind"], "scope": "formal",
+                  "status": status, "class": cls, "by_project": by_project}
+        index_claims.append({**common, "anchor": mf.index_anchor(anchor, "approximate"),
+                             "counts": {}, "file": fname, "linked": len(c["links"]),
+                             "suggested": 0})
+        chunks[fname] = {
+            **common, "schema": SCHEMA, "mode": "simple", "release": "paper", "key": c["key"],
+            "scope_note": "", "scope_reviewed": True, "anchor": anchor,
+            "source_matches": None, "statement_tex": None, "blockers": [],
+            "links": [{**l, "status": NOT_CHECKED, "class": "declared", "why": None,
+                       "statement": None, "path": None} for l in c["links"]],
+            "suggestions": [], "clauses": [], "decls": {}, "findings": []}
+    summary = {}
+    for c in index_claims:
+        summary[c["status"]] = summary.get(c["status"], 0) + 1
+    # The page has no Lean evidence; its identifier is that of what it shows.
+    eid = content_id({"paper": paper, "claims": claims, "pdf": pdf_url})
+    index = {"schema": SCHEMA, "mode": "paper", "release": "paper", "evidence": eid,
+             "tool": {"name": "machinechecked"}, "project": {}, "projects": projects,
+             "paper": paper, "notes": [],
+             "papers": [{"id": "P", "title": paper.get("title", ""), "mode": "pdf", "pdf": None,
+                         "pdf_url": pdf_url, "page_size": list(page_size) if page_size else None,
+                         "note": ""}],
+             "summary": {"P": summary},
+             "classes": {c["status"]: c["class"] for c in index_claims},
+             "findings": {"total": {}, "general": []}, "gates": {}, "claims": index_claims,
+             "search": "data/search.json"}
+    write_preview(site, index, chunks, {}, {}, web_dist)
+    return index

{machinechecked-0.1.3 → machinechecked-0.1.5}/engine/python/prooflens/pdfclaims.py

@@ -20,7 +20,9 @@ from .util import norm_ws, sha256_text
 
 KINDS = ("Theorem", "Lemma", "Proposition", "Corollary", "Definition", "Remark", "Example",
          "Assumption", "Conjecture", "Claim", "Hypothesis", "Convention")
-HEAD = re.compile(r"^(" + "|".join(KINDS) + r")\s+([A-Z]?\d+(?:\.\d+)*[a-z]?)\s*(\.|\(|$)")
+# A statement's number: "3", "8.6", "A1", "4.2a", or an appendix's "A.1", "B.10".
+NUMBER = r"(?:[A-Z](?:\.\d+)+|[A-Z]?\d+(?:\.\d+)*)[a-z]?"
+HEAD = re.compile(r"^(" + "|".join(KINDS) + r")\s+(" + NUMBER + r")\s*(\.|\(|$)")
 STOP = re.compile(r"^(Proof\b|\d+(\.\d+)*\s+[A-Z])")
 
 

{machinechecked-0.1.3 → machinechecked-0.1.5}/engine/python/prooflens/pipeline.py

@@ -20,9 +20,8 @@ from .reviews import load_dismissals, load_reviews
 from .schemas import validate
 from .sources import acquire
 from .tex import DEFAULT_ENVS, TexError, anchors_for, theorem_envs
-from .util import (canonical_json, git_dirty, git_head, norm_ws, read_json, sha256_file,
-                   sha256_text,
-                   write_json)
+from .util import (canonical_json, git_dirty, git_head, link_or_copy, norm_ws, prepare_work_dir,
+                   read_json, sha256_file, sha256_text, write_json)
 
 
 def tool_version() -> str:
@@ -49,6 +48,7 @@ def project_provenance(cfg: Config) -> dict:
 def build(cfg: Config, release: str | None = None, skip_lean: bool = False,
           skip_tex: bool = False, lake: str = "lake", web_dist: Path | None = None,
           lean_build: bool = True, reuse_lean: bool = False, log=print) -> dict:
+    prepare_work_dir(cfg.out_dir)
     inv = read_json(cfg.inventory)
     validate(inv, "inventory")
     reviews, review_hashes = load_reviews(cfg.reviews, cfg.ai_reviewers, cfg.root)
@@ -328,4 +328,4 @@ def write_preview(site: Path, index: dict, chunks: dict, store: dict, pdfs: dict
         write_json(data / "review-queue.json", queue, indent=None)
     for pid, pdf in pdfs.items():
         (data / "papers").mkdir(parents=True, exist_ok=True)
-        shutil.copyfile(pdf, data / "papers" / f"{pid}.pdf")
+        link_or_copy(pdf, data / "papers" / f"{pid}.pdf")

{machinechecked-0.1.3 → machinechecked-0.1.5}/engine/python/prooflens/simple.py

@@ -29,7 +29,7 @@ from .probe import fingerprint
 from .sources import SourceError, acquire
 from .tex import DEFAULT_ENVS, TexError, anchors_for, theorem_envs
 from . import formalization as fz
-from .util import content_id, git_head, read_json, sha256_file, write_json
+from .util import content_id, git_head, prepare_work_dir, read_json, sha256_file, write_json
 
 LINKS = "machinechecked.links.json"
 OLD_LINKS = "prooflens.links.json"
@@ -275,11 +275,36 @@ def save_settings(root: Path, issues: str | None = None, site_url: str | None =
 
 # -- the Lean side ---------------------------------------------------------------------------
 
+# What an index record keeps: the reader lists a declaration's statement only to this length,
+# and suggestions read a docstring only for the statement numbers it cites.
+INDEX_TYPE_CHARS, INDEX_DOC_CHARS = 300, 4000
+
+
+def _record(line: str) -> dict:
+    """One probe record, as kept in memory: an index record trimmed to what is used, any other
+    with its canonical form replaced by its fingerprint."""
+    rec = json.loads(line)
+    if rec.get("record") == "index":
+        return {"record": "index", "name": rec["name"], "kind": rec["kind"],
+                "module": rec.get("module"), "type": (rec.get("type") or "")[:INDEX_TYPE_CHARS],
+                "doc": (rec.get("doc") or "")[:INDEX_DOC_CHARS] or None}
+    for d in [rec, rec.get("decl")]:
+        if isinstance(d, dict) and "canon" in d:
+            d["fingerprint"] = fingerprint(d.pop("canon"))
+    return rec
+
+
 def run_simple_probe(root: Path, modules: list[str], libs: list[str], requests: list[dict],
-                     lake: str = "lake", timeout: int = 3600) -> list[dict]:
+                     lake: str = "lake", timeout: int = 3600,
+                     work: Path | None = None) -> list[dict]:
+    """Run the Lean probe on `modules` in one session. Its output is read one record at a time
+    (it can be gigabytes for a large project) from a file in `work` (the working directory's
+    build folder, rather than a system temporary folder that may be in memory)."""
     body = resources.files("prooflens").joinpath("data/ProofLensProbe.lean").read_text()
     header = "import Lean\n" + "".join(f"import {m}\n" for m in modules) + "\n"
-    with tempfile.TemporaryDirectory(prefix="prooflens-view-") as tmp:
+    if work is not None:
+        work.mkdir(parents=True, exist_ok=True)
+    with tempfile.TemporaryDirectory(prefix="prooflens-view-", dir=work) as tmp:
         tmp = Path(tmp)
         (tmp / "Probe.lean").write_text(header + body)
         (tmp / "query.json").write_text(json.dumps({
@@ -292,12 +317,8 @@ def run_simple_probe(root: Path, modules: list[str], libs: list[str], requests:
         out = tmp / "out.jsonl"
         if r.returncode != 0 or not out.exists():
             raise ViewError(f"the Lean probe failed:\n{r.stdout[-3000:]}\n{r.stderr[-3000:]}")
-        recs = [json.loads(l) for l in out.read_text(encoding="utf-8").splitlines() if l.strip()]
-    for rec in recs:
-        for d in [rec, rec.get("decl")]:
-            if isinstance(d, dict) and "canon" in d:
-                d["fingerprint"] = fingerprint(d.pop("canon"))
-    return recs
+        with out.open(encoding="utf-8") as f:
+            return [_record(l) for l in f if l.strip()]
 
 
 def library_of(module: str, libs: list[str]) -> str:
@@ -307,7 +328,8 @@ def library_of(module: str, libs: list[str]) -> str:
 
 
 def probe_modules(root: Path, modules: list[str], libs: list[str], requests: list[dict],
-                  lake: str = "lake", log=print) -> tuple[list[dict], dict[str, str]]:
+                  lake: str = "lake", log=print,
+                  work: Path | None = None) -> tuple[list[dict], dict[str, str]]:
     """Probe the built modules without assuming they can all be imported into one Lean session.
 
     A project may build modules that do not belong together: two libraries declaring the same
@@ -320,7 +342,7 @@ def probe_modules(root: Path, modules: list[str], libs: list[str], requests: lis
 
     def run(group: list[str], label: str):
         try:
-            out = run_simple_probe(root, group, libs, requests, lake)
+            out = run_simple_probe(root, group, libs, requests, lake, work=work)
         except ViewError as e:
             if len(group) == 1:
                 failed[group[0]] = str(e)[-600:]
@@ -497,6 +519,7 @@ def paper_title(p: Path) -> str:
 def build_view(root: Path, paper_path: Path, out_dir: Path, lake: str = "lake",
                log=print) -> dict:
     """Everything the reader shows for `paper_path` in the Lean project at `root`."""
+    prepare_work_dir(out_dir)
     libs = lib_names(root)
     built, unbuilt = built_modules(root, libs) if libs else ([], [])
     cfg = view_config(root, paper_path, out_dir, libs)
@@ -569,41 +592,69 @@ def build_view(root: Path, paper_path: Path, out_dir: Path, lake: str = "lake",
         if unbuilt:
             notes.append(f"{len(unbuilt)} module(s) of the project are not built and are not "
                          f"indexed")
-        # The index is reused while no Lean input of the project changed.
-        icache = out_dir / "build" / "index-cache.json"
+        # The names linked before any suggestion (confirmed links, \lean tags, formalization.yaml),
+        # less those whose module is known not to be built: Lean cannot answer for those.
+        for c in claims:
+            c["link_names"] = linked_names(c, links.get(c["key"], {}))
+        unbuilt_set = set(unbuilt)
+        linked = {n for c in claims for n in c["link_names"]
+                  if c.get("declared_modules", {}).get(n) not in unbuilt_set}
+        work = out_dir / "build"
+
+        def take(recs):
+            for r in recs:
+                if r["record"] == "link":
+                    linkrecs.setdefault(r["name"], []).append(r)
+                elif r["record"] == "decl":
+                    decls.setdefault(r["name"], r)
+
+        # The index is reused while no Lean input of the project changed. Built afresh, it is
+        # made in the same sessions as the linked names' records, so the modules load once.
+        icache = work / "index-cache.json"
         idigest = lean_digest(root)
         cached = read_json(icache) if icache.exists() else {}
         failed = {}
         if cached.get("digest") == idigest:
             index, failed = cached["index"], cached.get("failed", {})
         else:
-            log(f"lean: indexing {len(built)} built module(s)")
-            recs, failed = probe_modules(root, built, libs, [{"kind": "index", "name": ""}],
-                                         lake, log)
+            log(f"lean: indexing {len(built)} built module(s)"
+                + (f" and checking {len(linked)} linked declaration(s)" if linked else ""))
+            recs, failed = probe_modules(root, built, libs, [{"kind": "index", "name": ""}]
+                                         + [{"kind": "link", "name": n} for n in sorted(linked)],
+                                         lake, log, work)
             index = [r for r in recs if r["record"] == "index"]
+            take(recs)
             write_json(icache, {"digest": idigest, "index": index, "failed": failed},
                        indent=None)
-        wanted = set()
+        wanted = set(linked)
         for c in claims:
             entry = links.get(c["key"], {})
-            c["link_names"] = linked_names(c, entry)
             c["suggestions"] = [s for s in suggest(c, index)
                                 if s["name"] not in entry.get("rejected", [])
                                 and s["name"] not in c["link_names"]]
-            wanted |= set(c["link_names"])
             wanted |= {s["name"] for s in c["suggestions"][:1] if s.get("kind")}
+        wanted -= set(linkrecs)
         if wanted:
+            # Only the modules that declare the names are loaded (with what they import). A name
+            # the index does not hold (from a dependency, say) is looked for in every loadable
+            # module, unless its source shows it in a module that is not built.
             log(f"lean: checking {len(wanted)} linked or suggested declaration(s)")
             loadable = [m for m in built if m not in failed]
-            recs, more = probe_modules(root, loadable, libs,
-                                       [{"kind": "link", "name": n} for n in sorted(wanted)],
-                                       lake, log)
-            failed.update(more)
-            for r in recs:
-                if r["record"] == "link":
-                    linkrecs.setdefault(r["name"], []).append(r)
-                elif r["record"] == "decl":
-                    decls.setdefault(r["name"], r)
+            where: dict[str, set[str]] = {}
+            for r in index:
+                if r["name"] in wanted and r.get("module") in loadable:
+                    where.setdefault(r["name"], set()).add(r["module"])
+            elsewhere = sorted(n for n in wanted if n not in where
+                               and not declaring_modules(root, unbuilt, n))
+            groups = [(sorted(set().union(*where.values())), sorted(where))] if where else []
+            if elsewhere:
+                groups.append((loadable, elsewhere))
+            for mods, names in groups:
+                recs, more = probe_modules(root, mods, libs,
+                                           [{"kind": "link", "name": n} for n in names],
+                                           lake, log, work)
+                failed.update(more)
+                take(recs)
         if failed:
             notes.append(f"{len(failed)} built module(s) could not be loaded by MachineChecked "
                          f"and are not indexed: {', '.join(sorted(failed)[:5])}"
@@ -679,16 +730,11 @@ def write_view_site(view: dict, site: Path, web_dist: Path | None,
                        if (a.get("label") or f"{a['env']}-{a.get('number')}") == c["key"]), None)
         fname = mf.claim_file(c["id"])
         akind = "approximate" if anchor and anchor.get("kind") == "pdf" else "exact"
-        top = mf.top_of({**anchor, "kind": akind}) if anchor and anchor.get("begin") else None
         index_claims.append({
             "id": c["id"], "paper": c["paper"], "number": c["number"], "title": c["title"],
             "heading": c["title"] or None, "kind": c["kind"], "scope": c["scope"],
             "status": c["status"], "class": c["class"],
-            "anchor": {"kind": akind, "page": top[0], "y0": top[1],
-                       "y1": anchor["end"][2] if anchor.get("end") and
-                       anchor["end"][0] == anchor["begin"][0] else None,
-                       "page_end": anchor["end"][0] if anchor.get("end") else None}
-            if top else None,
+            "anchor": mf.index_anchor(anchor, akind),
             "counts": {}, "file": fname, "linked": len(c["links"]),
             "suggested": len(c.get("suggestions", []))})
         names = set()

{machinechecked-0.1.3 → machinechecked-0.1.5}/engine/python/prooflens/sources.py

@@ -16,6 +16,7 @@ from dataclasses import dataclass
 from pathlib import Path
 
 from .config import Config, Paper
+from .util import link_or_copy
 
 
 class SourceError(RuntimeError):
@@ -49,6 +50,24 @@ def _latex_roots(d: Path) -> list[str]:
     return roots
 
 
+SKIP = (".git", ".lake", ".machinechecked", ".prooflens")
+
+
+def _has_latex_root(d: Path) -> bool:
+    """Whether a copy of `d` (without the directories a snapshot leaves out) has a LaTeX root."""
+    for top, dirs, files in os.walk(d):
+        dirs[:] = [x for x in dirs if x not in SKIP]
+        for f in files:
+            if f.endswith(".tex"):
+                try:
+                    if DOCCLASS.search((Path(top) / f).read_text(encoding="utf-8",
+                                                                errors="replace")):
+                        return True
+                except OSError:
+                    continue
+    return False
+
+
 def _pdfs(d: Path) -> list[str]:
     return sorted(str(p.relative_to(d)) for p in d.rglob("*.pdf"))
 
@@ -105,10 +124,15 @@ def acquire(cfg: Config, paper: Paper, dest: Path) -> Snapshot:
         base = (cfg.root / src["path"]).resolve()
         if not base.is_dir():
             raise SourceError(f"paper {paper.id}: {base} is not a directory")
-        shutil.copytree(base, dest, dirs_exist_ok=True, copy_function=shutil.copy2,
-                        ignore_dangling_symlinks=True,
-                        ignore=shutil.ignore_patterns(".git", ".lake", ".machinechecked", ".prooflens",
-                                                      "*.aux", "*.log", "*.fls"))
+        if paper.pdf and not paper.root and (base / paper.pdf).is_file() \
+                and not _has_latex_root(base):
+            # A paper that is only a PDF needs only that file, not the directory it is in (the
+            # source would be read as the same PDF either way).
+            link_or_copy(base / paper.pdf, dest / paper.pdf)
+        else:
+            shutil.copytree(base, dest, dirs_exist_ok=True, copy_function=shutil.copy2,
+                            ignore_dangling_symlinks=True,
+                            ignore=shutil.ignore_patterns(*SKIP, "*.aux", "*.log", "*.fls"))
         kind, rev, commit_time = "path", None, None
     elif "git" in src:
         rev = src.get("rev")

{machinechecked-0.1.3 → machinechecked-0.1.5}/engine/python/prooflens/util.py

@@ -4,7 +4,9 @@ from __future__ import annotations
 
 import hashlib
 import json
+import os
 import re
+import shutil
 import subprocess
 from pathlib import Path
 
@@ -64,3 +66,33 @@ def write_json(path: str | Path, obj, indent: int | None = 1) -> None:
 
 def read_json(path: str | Path):
     return json.loads(Path(path).read_text(encoding="utf-8"))
+
+
+def link_or_copy(src: str | Path, dst: str | Path) -> None:
+    """Put `src` at `dst` as a hard link (the file is then stored once), else as a copy."""
+    dst = Path(dst)
+    dst.parent.mkdir(parents=True, exist_ok=True)
+    dst.unlink(missing_ok=True)
+    try:
+        os.link(src, dst)
+    except OSError:
+        shutil.copy2(src, dst)
+
+
+def prepare_work_dir(d: str | Path) -> Path:
+    """Create the working directory (`.machinechecked/`) and have Git ignore it, as Lake does
+    `.lake/`: a `.gitignore` of `*` inside it, unless it has one or Git already tracks files in
+    it."""
+    d = Path(d)
+    d.mkdir(parents=True, exist_ok=True)
+    ignore = d / ".gitignore"
+    if not ignore.exists():
+        try:
+            tracked = subprocess.run(["git", "ls-files", "--", "."], cwd=d, capture_output=True,
+                                     text=True).stdout.strip()
+        except OSError:  # no Git: nothing can be tracked
+            tracked = ""
+        if not tracked:
+            ignore.write_text("# MachineChecked's working files, never committed.\n*\n")
+    return d
+