machinechecked 0.1.4__tar.gz → 0.1.5__tar.gz

{machinechecked-0.1.4 → machinechecked-0.1.5}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: machinechecked
-Version: 0.1.4
+Version: 0.1.5
 Summary: Read a mathematical paper beside its Lean formalisation, and see claim by claim what Lean establishes.
 License: Apache-2.0
 Project-URL: Homepage, https://machinechecked.org

{machinechecked-0.1.4 → machinechecked-0.1.5}/engine/python/machinechecked.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: machinechecked
-Version: 0.1.4
+Version: 0.1.5
 Summary: Read a mathematical paper beside its Lean formalisation, and see claim by claim what Lean establishes.
 License: Apache-2.0
 Project-URL: Homepage, https://machinechecked.org

{machinechecked-0.1.4 → machinechecked-0.1.5}/engine/python/machinechecked.egg-info/SOURCES.txt

@@ -38,6 +38,7 @@ engine/python/prooflens/data/schema/inventory.schema.json
 engine/python/prooflens/data/schema/manifest-claim.schema.json
 engine/python/prooflens/data/schema/manifest-index.schema.json
 engine/python/prooflens/web/index.html
+engine/python/prooflens/web/theme.css
 engine/python/prooflens/web/assets/KaTeX_AMS-Regular-BQhdFMY1.woff2
 engine/python/prooflens/web/assets/KaTeX_Caligraphic-Bold-Dq_IR9rO.woff2
 engine/python/prooflens/web/assets/KaTeX_Caligraphic-Regular-Di6jR-x-.woff2
@@ -57,6 +58,6 @@ engine/python/prooflens/web/assets/KaTeX_Size1-Regular-mCD8mA8B.woff2
 engine/python/prooflens/web/assets/KaTeX_Size2-Regular-Dy4dx90m.woff2
 engine/python/prooflens/web/assets/KaTeX_Size4-Regular-Dl5lxZxV.woff2
 engine/python/prooflens/web/assets/KaTeX_Typewriter-Regular-CO6r4hn1.woff2
-engine/python/prooflens/web/assets/index-Crg6e6Ow.js
+engine/python/prooflens/web/assets/index-D0aKe720.js
 engine/python/prooflens/web/assets/index-T9UdK0vf.css
 engine/python/prooflens/web/assets/pdf.worker.min-yatZIOMy.mjs

{machinechecked-0.1.4 → machinechecked-0.1.5}/engine/python/prooflens/cli.py

@@ -8,7 +8,8 @@ Everyday commands, run from anywhere inside a Lean project:
 
 A project with a `machinechecked.toml` (an acceptance layer: locked statements, reviews) gets
 the full assessment; any other project gets the zero-configuration view, whose only file in the
-repository is `machinechecked.links.json`. The other commands are for that full assessment.
+repository is `machinechecked.links.json` (its working files, a downloaded paper among them, are
+in `.machinechecked/`, which Git ignores). The other commands are for that full assessment.
 `prooflens` is another name for the same command.
 """
 
@@ -149,7 +150,8 @@ def _fetch_paper(root: Path, url: str, title: str | None = None) -> Path:
     repository), with its address and title beside it."""
     import hashlib
     import urllib.request
-    d = root / cfgmod.work_dir_name(root) / "papers"
+    from .util import prepare_work_dir
+    d = prepare_work_dir(root / cfgmod.work_dir_name(root)) / "papers"
     name = re.sub(r"[^\w.-]+", "_", url.rstrip("/").rsplit("/", 1)[-1]) or "paper"
     if not name.lower().endswith(".pdf"):
         name += ".pdf"
@@ -357,7 +359,8 @@ def cmd_export(a):
     from .export import ExportError, export
     cfg = cfgmod.load(_config_arg(a))
     try:
-        res = export(cfg, Path(a.out).resolve(), a.release, _web_dist(a.web))
+        res = export(cfg, Path(a.out).resolve(), a.release,
+                     None if a.data_only else _web_dist(a.web))
     except ExportError as e:
         sys.exit(str(e))
     print(f"exported release {res['release']} to {res['out']}")
@@ -471,7 +474,8 @@ def main(argv=None):
     b.set_defaults(func=cmd_build)
 
     k = sub.add_parser("check", help="report what the Lean side establishes (exit 1 if a "
-                                     "linked declaration is missing or unproved)")
+                                     "linked declaration is missing or unproved, 2 if the check "
+                                     "could not run)")
     k.add_argument("paper", nargs="?")
     k.add_argument("--json", action="store_true",
                    help="print every claim's status and links as JSON (for scripts and agents)")
@@ -504,6 +508,9 @@ def main(argv=None):
     e.add_argument("--release")
     e.add_argument("--out", required=True)
     e.add_argument("--web")
+    e.add_argument("--data-only", action="store_true",
+                   help="write the audit's data only, without the reader (for a site that "
+                   "reads it with its own reader, such as machinechecked.org)")
     e.set_defaults(func=cmd_export)
 
     s = sub.add_parser("serve")
@@ -531,11 +538,49 @@ def main(argv=None):
     a = p.parse_args(argv)
     try:
         a.func(a)
+    except SystemExit as e:
+        # `sys.exit("message")` is a command that could not do its job; an integer is its result.
+        if isinstance(e.code, str):
+            _could_not_run(a, "error", e.code.removeprefix("machinechecked: "))
+        raise
     except FileNotFoundError as e:
-        hint = MISSING.get(Path(str(e.filename or "")).name)
-        if not hint:
-            raise
-        sys.exit(f"machinechecked: `{Path(e.filename).name}` not found: {hint}")
+        tool = Path(str(e.filename or "")).name
+        if tool in MISSING:
+            _could_not_run(a, "missing-tool", f"`{tool}` not found: {MISSING[tool]}", tool=tool)
+        _could_not_run(a, "internal", _crash(e))
+    except (RuntimeError, ValueError, OSError) as e:
+        # The engine's own errors (the Lean probe, TeX, the paper's sources, the configuration,
+        # stored evidence) say what went wrong in their message.
+        if type(e).__module__.startswith("prooflens"):
+            _could_not_run(a, "error", str(e))
+        _could_not_run(a, "internal", _crash(e))
+    except Exception as e:
+        _could_not_run(a, "internal", _crash(e))
+
+
+# Exit codes: 0 the command did its job (and `check` found nothing wrong), 1 `check` (or
+# `build --fail-on-violation`) found a problem, 2 the command could not run.
+COULD_NOT_RUN = 2
+
+
+def _crash(e: Exception) -> str:
+    """An unexpected error: its traceback on stderr, for a bug report, and a one-line message."""
+    import traceback
+    traceback.print_exc()
+    return f"unexpected error ({type(e).__name__}: {e}); please report it with the traceback above"
+
+
+def _could_not_run(a, kind: str, message: str, **extra):
+    """Stop with exit 2: the message on stderr and, for `check --json`, one JSON document on
+    stdout saying the check did not run."""
+    print(f"machinechecked: {message}", file=sys.stderr)
+    if getattr(a, "func", None) is cmd_check and getattr(a, "json", False):
+        import json
+        print(json.dumps({"schema": "machinechecked/check@1", "ok": False,
+                          "error": {"kind": kind, "message": message, **extra},
+                          "note": "The check did not run; nothing is known about the claims."},
+                         indent=1))
+    raise SystemExit(COULD_NOT_RUN)
 
 
 if __name__ == "__main__":

{machinechecked-0.1.4 → machinechecked-0.1.5}/engine/python/prooflens/formalization.py

@@ -18,7 +18,8 @@ from pathlib import Path
 NAMES = ("formalization.yaml", "formalization.yml")
 KINDS = ("theorem", "lemma", "proposition", "corollary", "definition", "remark", "example",
          "conjecture", "claim", "assumption", "hypothesis", "convention")
-REF = re.compile(r"\b(" + "|".join(KINDS) + r")\s+([A-Z]?\d+(?:\.\d+)*[a-z]?)", re.I)
+REF = re.compile(r"\b(" + "|".join(KINDS) + r")\s+((?:[A-Z](?:\.\d+)+|[A-Z]?\d+(?:\.\d+)*)[a-z]?)",
+                 re.I)
 
 
 def find(root: Path) -> Path | None:

{machinechecked-0.1.4 → machinechecked-0.1.5}/engine/python/prooflens/pdfclaims.py

@@ -20,7 +20,9 @@ from .util import norm_ws, sha256_text
 
 KINDS = ("Theorem", "Lemma", "Proposition", "Corollary", "Definition", "Remark", "Example",
          "Assumption", "Conjecture", "Claim", "Hypothesis", "Convention")
-HEAD = re.compile(r"^(" + "|".join(KINDS) + r")\s+([A-Z]?\d+(?:\.\d+)*[a-z]?)\s*(\.|\(|$)")
+# A statement's number: "3", "8.6", "A1", "4.2a", or an appendix's "A.1", "B.10".
+NUMBER = r"(?:[A-Z](?:\.\d+)+|[A-Z]?\d+(?:\.\d+)*)[a-z]?"
+HEAD = re.compile(r"^(" + "|".join(KINDS) + r")\s+(" + NUMBER + r")\s*(\.|\(|$)")
 STOP = re.compile(r"^(Proof\b|\d+(\.\d+)*\s+[A-Z])")
 
 

{machinechecked-0.1.4 → machinechecked-0.1.5}/engine/python/prooflens/pipeline.py

@@ -20,9 +20,8 @@ from .reviews import load_dismissals, load_reviews
 from .schemas import validate
 from .sources import acquire
 from .tex import DEFAULT_ENVS, TexError, anchors_for, theorem_envs
-from .util import (canonical_json, git_dirty, git_head, norm_ws, read_json, sha256_file,
-                   sha256_text,
-                   write_json)
+from .util import (canonical_json, git_dirty, git_head, link_or_copy, norm_ws, prepare_work_dir,
+                   read_json, sha256_file, sha256_text, write_json)
 
 
 def tool_version() -> str:
@@ -49,6 +48,7 @@ def project_provenance(cfg: Config) -> dict:
 def build(cfg: Config, release: str | None = None, skip_lean: bool = False,
           skip_tex: bool = False, lake: str = "lake", web_dist: Path | None = None,
           lean_build: bool = True, reuse_lean: bool = False, log=print) -> dict:
+    prepare_work_dir(cfg.out_dir)
     inv = read_json(cfg.inventory)
     validate(inv, "inventory")
     reviews, review_hashes = load_reviews(cfg.reviews, cfg.ai_reviewers, cfg.root)
@@ -328,4 +328,4 @@ def write_preview(site: Path, index: dict, chunks: dict, store: dict, pdfs: dict
         write_json(data / "review-queue.json", queue, indent=None)
     for pid, pdf in pdfs.items():
         (data / "papers").mkdir(parents=True, exist_ok=True)
-        shutil.copyfile(pdf, data / "papers" / f"{pid}.pdf")
+        link_or_copy(pdf, data / "papers" / f"{pid}.pdf")

{machinechecked-0.1.4 → machinechecked-0.1.5}/engine/python/prooflens/simple.py

@@ -29,7 +29,7 @@ from .probe import fingerprint
 from .sources import SourceError, acquire
 from .tex import DEFAULT_ENVS, TexError, anchors_for, theorem_envs
 from . import formalization as fz
-from .util import content_id, git_head, read_json, sha256_file, write_json
+from .util import content_id, git_head, prepare_work_dir, read_json, sha256_file, write_json
 
 LINKS = "machinechecked.links.json"
 OLD_LINKS = "prooflens.links.json"
@@ -275,11 +275,36 @@ def save_settings(root: Path, issues: str | None = None, site_url: str | None =
 
 # -- the Lean side ---------------------------------------------------------------------------
 
+# What an index record keeps: the reader lists a declaration's statement only to this length,
+# and suggestions read a docstring only for the statement numbers it cites.
+INDEX_TYPE_CHARS, INDEX_DOC_CHARS = 300, 4000
+
+
+def _record(line: str) -> dict:
+    """One probe record, as kept in memory: an index record trimmed to what is used, any other
+    with its canonical form replaced by its fingerprint."""
+    rec = json.loads(line)
+    if rec.get("record") == "index":
+        return {"record": "index", "name": rec["name"], "kind": rec["kind"],
+                "module": rec.get("module"), "type": (rec.get("type") or "")[:INDEX_TYPE_CHARS],
+                "doc": (rec.get("doc") or "")[:INDEX_DOC_CHARS] or None}
+    for d in [rec, rec.get("decl")]:
+        if isinstance(d, dict) and "canon" in d:
+            d["fingerprint"] = fingerprint(d.pop("canon"))
+    return rec
+
+
 def run_simple_probe(root: Path, modules: list[str], libs: list[str], requests: list[dict],
-                     lake: str = "lake", timeout: int = 3600) -> list[dict]:
+                     lake: str = "lake", timeout: int = 3600,
+                     work: Path | None = None) -> list[dict]:
+    """Run the Lean probe on `modules` in one session. Its output is read one record at a time
+    (it can be gigabytes for a large project) from a file in `work` (the working directory's
+    build folder, rather than a system temporary folder that may be in memory)."""
     body = resources.files("prooflens").joinpath("data/ProofLensProbe.lean").read_text()
     header = "import Lean\n" + "".join(f"import {m}\n" for m in modules) + "\n"
-    with tempfile.TemporaryDirectory(prefix="prooflens-view-") as tmp:
+    if work is not None:
+        work.mkdir(parents=True, exist_ok=True)
+    with tempfile.TemporaryDirectory(prefix="prooflens-view-", dir=work) as tmp:
         tmp = Path(tmp)
         (tmp / "Probe.lean").write_text(header + body)
         (tmp / "query.json").write_text(json.dumps({
@@ -292,12 +317,8 @@ def run_simple_probe(root: Path, modules: list[str], libs: list[str], requests:
         out = tmp / "out.jsonl"
         if r.returncode != 0 or not out.exists():
             raise ViewError(f"the Lean probe failed:\n{r.stdout[-3000:]}\n{r.stderr[-3000:]}")
-        recs = [json.loads(l) for l in out.read_text(encoding="utf-8").splitlines() if l.strip()]
-    for rec in recs:
-        for d in [rec, rec.get("decl")]:
-            if isinstance(d, dict) and "canon" in d:
-                d["fingerprint"] = fingerprint(d.pop("canon"))
-    return recs
+        with out.open(encoding="utf-8") as f:
+            return [_record(l) for l in f if l.strip()]
 
 
 def library_of(module: str, libs: list[str]) -> str:
@@ -307,7 +328,8 @@ def library_of(module: str, libs: list[str]) -> str:
 
 
 def probe_modules(root: Path, modules: list[str], libs: list[str], requests: list[dict],
-                  lake: str = "lake", log=print) -> tuple[list[dict], dict[str, str]]:
+                  lake: str = "lake", log=print,
+                  work: Path | None = None) -> tuple[list[dict], dict[str, str]]:
     """Probe the built modules without assuming they can all be imported into one Lean session.
 
     A project may build modules that do not belong together: two libraries declaring the same
@@ -320,7 +342,7 @@ def probe_modules(root: Path, modules: list[str], libs: list[str], requests: lis
 
     def run(group: list[str], label: str):
         try:
-            out = run_simple_probe(root, group, libs, requests, lake)
+            out = run_simple_probe(root, group, libs, requests, lake, work=work)
         except ViewError as e:
             if len(group) == 1:
                 failed[group[0]] = str(e)[-600:]
@@ -497,6 +519,7 @@ def paper_title(p: Path) -> str:
 def build_view(root: Path, paper_path: Path, out_dir: Path, lake: str = "lake",
                log=print) -> dict:
     """Everything the reader shows for `paper_path` in the Lean project at `root`."""
+    prepare_work_dir(out_dir)
     libs = lib_names(root)
     built, unbuilt = built_modules(root, libs) if libs else ([], [])
     cfg = view_config(root, paper_path, out_dir, libs)
@@ -569,41 +592,69 @@ def build_view(root: Path, paper_path: Path, out_dir: Path, lake: str = "lake",
         if unbuilt:
             notes.append(f"{len(unbuilt)} module(s) of the project are not built and are not "
                          f"indexed")
-        # The index is reused while no Lean input of the project changed.
-        icache = out_dir / "build" / "index-cache.json"
+        # The names linked before any suggestion (confirmed links, \lean tags, formalization.yaml),
+        # less those whose module is known not to be built: Lean cannot answer for those.
+        for c in claims:
+            c["link_names"] = linked_names(c, links.get(c["key"], {}))
+        unbuilt_set = set(unbuilt)
+        linked = {n for c in claims for n in c["link_names"]
+                  if c.get("declared_modules", {}).get(n) not in unbuilt_set}
+        work = out_dir / "build"
+
+        def take(recs):
+            for r in recs:
+                if r["record"] == "link":
+                    linkrecs.setdefault(r["name"], []).append(r)
+                elif r["record"] == "decl":
+                    decls.setdefault(r["name"], r)
+
+        # The index is reused while no Lean input of the project changed. Built afresh, it is
+        # made in the same sessions as the linked names' records, so the modules load once.
+        icache = work / "index-cache.json"
         idigest = lean_digest(root)
         cached = read_json(icache) if icache.exists() else {}
         failed = {}
         if cached.get("digest") == idigest:
             index, failed = cached["index"], cached.get("failed", {})
         else:
-            log(f"lean: indexing {len(built)} built module(s)")
-            recs, failed = probe_modules(root, built, libs, [{"kind": "index", "name": ""}],
-                                         lake, log)
+            log(f"lean: indexing {len(built)} built module(s)"
+                + (f" and checking {len(linked)} linked declaration(s)" if linked else ""))
+            recs, failed = probe_modules(root, built, libs, [{"kind": "index", "name": ""}]
+                                         + [{"kind": "link", "name": n} for n in sorted(linked)],
+                                         lake, log, work)
             index = [r for r in recs if r["record"] == "index"]
+            take(recs)
             write_json(icache, {"digest": idigest, "index": index, "failed": failed},
                        indent=None)
-        wanted = set()
+        wanted = set(linked)
         for c in claims:
             entry = links.get(c["key"], {})
-            c["link_names"] = linked_names(c, entry)
             c["suggestions"] = [s for s in suggest(c, index)
                                 if s["name"] not in entry.get("rejected", [])
                                 and s["name"] not in c["link_names"]]
-            wanted |= set(c["link_names"])
             wanted |= {s["name"] for s in c["suggestions"][:1] if s.get("kind")}
+        wanted -= set(linkrecs)
         if wanted:
+            # Only the modules that declare the names are loaded (with what they import). A name
+            # the index does not hold (from a dependency, say) is looked for in every loadable
+            # module, unless its source shows it in a module that is not built.
             log(f"lean: checking {len(wanted)} linked or suggested declaration(s)")
             loadable = [m for m in built if m not in failed]
-            recs, more = probe_modules(root, loadable, libs,
-                                       [{"kind": "link", "name": n} for n in sorted(wanted)],
-                                       lake, log)
-            failed.update(more)
-            for r in recs:
-                if r["record"] == "link":
-                    linkrecs.setdefault(r["name"], []).append(r)
-                elif r["record"] == "decl":
-                    decls.setdefault(r["name"], r)
+            where: dict[str, set[str]] = {}
+            for r in index:
+                if r["name"] in wanted and r.get("module") in loadable:
+                    where.setdefault(r["name"], set()).add(r["module"])
+            elsewhere = sorted(n for n in wanted if n not in where
+                               and not declaring_modules(root, unbuilt, n))
+            groups = [(sorted(set().union(*where.values())), sorted(where))] if where else []
+            if elsewhere:
+                groups.append((loadable, elsewhere))
+            for mods, names in groups:
+                recs, more = probe_modules(root, mods, libs,
+                                           [{"kind": "link", "name": n} for n in names],
+                                           lake, log, work)
+                failed.update(more)
+                take(recs)
         if failed:
             notes.append(f"{len(failed)} built module(s) could not be loaded by MachineChecked "
                          f"and are not indexed: {', '.join(sorted(failed)[:5])}"

{machinechecked-0.1.4 → machinechecked-0.1.5}/engine/python/prooflens/sources.py

@@ -16,6 +16,7 @@ from dataclasses import dataclass
 from pathlib import Path
 
 from .config import Config, Paper
+from .util import link_or_copy
 
 
 class SourceError(RuntimeError):
@@ -49,6 +50,24 @@ def _latex_roots(d: Path) -> list[str]:
     return roots
 
 
+SKIP = (".git", ".lake", ".machinechecked", ".prooflens")
+
+
+def _has_latex_root(d: Path) -> bool:
+    """Whether a copy of `d` (without the directories a snapshot leaves out) has a LaTeX root."""
+    for top, dirs, files in os.walk(d):
+        dirs[:] = [x for x in dirs if x not in SKIP]
+        for f in files:
+            if f.endswith(".tex"):
+                try:
+                    if DOCCLASS.search((Path(top) / f).read_text(encoding="utf-8",
+                                                                errors="replace")):
+                        return True
+                except OSError:
+                    continue
+    return False
+
+
 def _pdfs(d: Path) -> list[str]:
     return sorted(str(p.relative_to(d)) for p in d.rglob("*.pdf"))
 
@@ -105,10 +124,15 @@ def acquire(cfg: Config, paper: Paper, dest: Path) -> Snapshot:
         base = (cfg.root / src["path"]).resolve()
         if not base.is_dir():
             raise SourceError(f"paper {paper.id}: {base} is not a directory")
-        shutil.copytree(base, dest, dirs_exist_ok=True, copy_function=shutil.copy2,
-                        ignore_dangling_symlinks=True,
-                        ignore=shutil.ignore_patterns(".git", ".lake", ".machinechecked", ".prooflens",
-                                                      "*.aux", "*.log", "*.fls"))
+        if paper.pdf and not paper.root and (base / paper.pdf).is_file() \
+                and not _has_latex_root(base):
+            # A paper that is only a PDF needs only that file, not the directory it is in (the
+            # source would be read as the same PDF either way).
+            link_or_copy(base / paper.pdf, dest / paper.pdf)
+        else:
+            shutil.copytree(base, dest, dirs_exist_ok=True, copy_function=shutil.copy2,
+                            ignore_dangling_symlinks=True,
+                            ignore=shutil.ignore_patterns(*SKIP, "*.aux", "*.log", "*.fls"))
         kind, rev, commit_time = "path", None, None
     elif "git" in src:
         rev = src.get("rev")

{machinechecked-0.1.4 → machinechecked-0.1.5}/engine/python/prooflens/util.py

@@ -4,7 +4,9 @@ from __future__ import annotations
 
 import hashlib
 import json
+import os
 import re
+import shutil
 import subprocess
 from pathlib import Path
 
@@ -64,3 +66,33 @@ def write_json(path: str | Path, obj, indent: int | None = 1) -> None:
 
 def read_json(path: str | Path):
     return json.loads(Path(path).read_text(encoding="utf-8"))
+
+
+def link_or_copy(src: str | Path, dst: str | Path) -> None:
+    """Put `src` at `dst` as a hard link (the file is then stored once), else as a copy."""
+    dst = Path(dst)
+    dst.parent.mkdir(parents=True, exist_ok=True)
+    dst.unlink(missing_ok=True)
+    try:
+        os.link(src, dst)
+    except OSError:
+        shutil.copy2(src, dst)
+
+
+def prepare_work_dir(d: str | Path) -> Path:
+    """Create the working directory (`.machinechecked/`) and have Git ignore it, as Lake does
+    `.lake/`: a `.gitignore` of `*` inside it, unless it has one or Git already tracks files in
+    it."""
+    d = Path(d)
+    d.mkdir(parents=True, exist_ok=True)
+    ignore = d / ".gitignore"
+    if not ignore.exists():
+        try:
+            tracked = subprocess.run(["git", "ls-files", "--", "."], cwd=d, capture_output=True,
+                                     text=True).stdout.strip()
+        except OSError:  # no Git: nothing can be tracked
+            tracked = ""
+        if not tracked:
+            ignore.write_text("# MachineChecked's working files, never committed.\n*\n")
+    return d
+