machinechecked 0.1.12__tar.gz → 0.1.14__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (76) hide show
  1. {machinechecked-0.1.12 → machinechecked-0.1.14}/PKG-INFO +1 -1
  2. {machinechecked-0.1.12 → machinechecked-0.1.14}/engine/python/machinechecked.egg-info/PKG-INFO +1 -1
  3. {machinechecked-0.1.12 → machinechecked-0.1.14}/engine/python/machinechecked.egg-info/SOURCES.txt +2 -2
  4. {machinechecked-0.1.12 → machinechecked-0.1.14}/engine/python/prooflens/assess.py +4 -2
  5. {machinechecked-0.1.12 → machinechecked-0.1.14}/engine/python/prooflens/cache.py +72 -16
  6. {machinechecked-0.1.12 → machinechecked-0.1.14}/engine/python/prooflens/cli.py +3 -1
  7. {machinechecked-0.1.12 → machinechecked-0.1.14}/engine/python/prooflens/data/ProofLensProbe.lean +98 -43
  8. {machinechecked-0.1.12 → machinechecked-0.1.14}/engine/python/prooflens/manifest.py +8 -1
  9. {machinechecked-0.1.12 → machinechecked-0.1.14}/engine/python/prooflens/source_regions.py +61 -9
  10. {machinechecked-0.1.12 → machinechecked-0.1.14}/engine/python/prooflens/symbols.py +96 -15
  11. machinechecked-0.1.12/engine/python/prooflens/web/assets/index-C8DRs1uO.css → machinechecked-0.1.14/engine/python/prooflens/web/assets/index-GfZV3aEH.css +1 -1
  12. machinechecked-0.1.14/engine/python/prooflens/web/assets/index-YAI0NIYT.js +278 -0
  13. {machinechecked-0.1.12 → machinechecked-0.1.14}/engine/python/prooflens/web/index.html +2 -2
  14. {machinechecked-0.1.12 → machinechecked-0.1.14}/pyproject.toml +1 -1
  15. machinechecked-0.1.12/engine/python/prooflens/web/assets/index-DXica3gb.js +0 -278
  16. {machinechecked-0.1.12 → machinechecked-0.1.14}/LICENSE +0 -0
  17. {machinechecked-0.1.12 → machinechecked-0.1.14}/README.md +0 -0
  18. {machinechecked-0.1.12 → machinechecked-0.1.14}/engine/python/machinechecked.egg-info/dependency_links.txt +0 -0
  19. {machinechecked-0.1.12 → machinechecked-0.1.14}/engine/python/machinechecked.egg-info/entry_points.txt +0 -0
  20. {machinechecked-0.1.12 → machinechecked-0.1.14}/engine/python/machinechecked.egg-info/requires.txt +0 -0
  21. {machinechecked-0.1.12 → machinechecked-0.1.14}/engine/python/machinechecked.egg-info/top_level.txt +0 -0
  22. {machinechecked-0.1.12 → machinechecked-0.1.14}/engine/python/prooflens/__init__.py +0 -0
  23. {machinechecked-0.1.12 → machinechecked-0.1.14}/engine/python/prooflens/__main__.py +0 -0
  24. {machinechecked-0.1.12 → machinechecked-0.1.14}/engine/python/prooflens/audit.py +0 -0
  25. {machinechecked-0.1.12 → machinechecked-0.1.14}/engine/python/prooflens/bundle.py +0 -0
  26. {machinechecked-0.1.12 → machinechecked-0.1.14}/engine/python/prooflens/config.py +0 -0
  27. {machinechecked-0.1.12 → machinechecked-0.1.14}/engine/python/prooflens/data/prooflens.sty +0 -0
  28. {machinechecked-0.1.12 → machinechecked-0.1.14}/engine/python/prooflens/data/schema/evidence.schema.json +0 -0
  29. {machinechecked-0.1.12 → machinechecked-0.1.14}/engine/python/prooflens/data/schema/inventory.schema.json +0 -0
  30. {machinechecked-0.1.12 → machinechecked-0.1.14}/engine/python/prooflens/data/schema/manifest-claim.schema.json +0 -0
  31. {machinechecked-0.1.12 → machinechecked-0.1.14}/engine/python/prooflens/data/schema/manifest-index.schema.json +0 -0
  32. {machinechecked-0.1.12 → machinechecked-0.1.14}/engine/python/prooflens/data/statuses.json +0 -0
  33. {machinechecked-0.1.12 → machinechecked-0.1.14}/engine/python/prooflens/deps.py +0 -0
  34. {machinechecked-0.1.12 → machinechecked-0.1.14}/engine/python/prooflens/evidence.py +0 -0
  35. {machinechecked-0.1.12 → machinechecked-0.1.14}/engine/python/prooflens/export.py +0 -0
  36. {machinechecked-0.1.12 → machinechecked-0.1.14}/engine/python/prooflens/formalization.py +0 -0
  37. {machinechecked-0.1.12 → machinechecked-0.1.14}/engine/python/prooflens/paper.py +0 -0
  38. {machinechecked-0.1.12 → machinechecked-0.1.14}/engine/python/prooflens/pdfclaims.py +0 -0
  39. {machinechecked-0.1.12 → machinechecked-0.1.14}/engine/python/prooflens/pipeline.py +0 -0
  40. {machinechecked-0.1.12 → machinechecked-0.1.14}/engine/python/prooflens/probe.py +0 -0
  41. {machinechecked-0.1.12 → machinechecked-0.1.14}/engine/python/prooflens/promote.py +0 -0
  42. {machinechecked-0.1.12 → machinechecked-0.1.14}/engine/python/prooflens/published.py +0 -0
  43. {machinechecked-0.1.12 → machinechecked-0.1.14}/engine/python/prooflens/replay.py +0 -0
  44. {machinechecked-0.1.12 → machinechecked-0.1.14}/engine/python/prooflens/review_cli.py +0 -0
  45. {machinechecked-0.1.12 → machinechecked-0.1.14}/engine/python/prooflens/review_queue.py +0 -0
  46. {machinechecked-0.1.12 → machinechecked-0.1.14}/engine/python/prooflens/reviews.py +0 -0
  47. {machinechecked-0.1.12 → machinechecked-0.1.14}/engine/python/prooflens/schemas.py +0 -0
  48. {machinechecked-0.1.12 → machinechecked-0.1.14}/engine/python/prooflens/server.py +0 -0
  49. {machinechecked-0.1.12 → machinechecked-0.1.14}/engine/python/prooflens/simple.py +0 -0
  50. {machinechecked-0.1.12 → machinechecked-0.1.14}/engine/python/prooflens/sources.py +0 -0
  51. {machinechecked-0.1.12 → machinechecked-0.1.14}/engine/python/prooflens/status.py +0 -0
  52. {machinechecked-0.1.12 → machinechecked-0.1.14}/engine/python/prooflens/tex.py +0 -0
  53. {machinechecked-0.1.12 → machinechecked-0.1.14}/engine/python/prooflens/util.py +0 -0
  54. {machinechecked-0.1.12 → machinechecked-0.1.14}/engine/python/prooflens/web/assets/KaTeX_AMS-Regular-BQhdFMY1.woff2 +0 -0
  55. {machinechecked-0.1.12 → machinechecked-0.1.14}/engine/python/prooflens/web/assets/KaTeX_Caligraphic-Bold-Dq_IR9rO.woff2 +0 -0
  56. {machinechecked-0.1.12 → machinechecked-0.1.14}/engine/python/prooflens/web/assets/KaTeX_Caligraphic-Regular-Di6jR-x-.woff2 +0 -0
  57. {machinechecked-0.1.12 → machinechecked-0.1.14}/engine/python/prooflens/web/assets/KaTeX_Fraktur-Bold-CL6g_b3V.woff2 +0 -0
  58. {machinechecked-0.1.12 → machinechecked-0.1.14}/engine/python/prooflens/web/assets/KaTeX_Fraktur-Regular-CTYiF6lA.woff2 +0 -0
  59. {machinechecked-0.1.12 → machinechecked-0.1.14}/engine/python/prooflens/web/assets/KaTeX_Main-Bold-Cx986IdX.woff2 +0 -0
  60. {machinechecked-0.1.12 → machinechecked-0.1.14}/engine/python/prooflens/web/assets/KaTeX_Main-BoldItalic-DxDJ3AOS.woff2 +0 -0
  61. {machinechecked-0.1.12 → machinechecked-0.1.14}/engine/python/prooflens/web/assets/KaTeX_Main-Italic-NWA7e6Wa.woff2 +0 -0
  62. {machinechecked-0.1.12 → machinechecked-0.1.14}/engine/python/prooflens/web/assets/KaTeX_Main-Regular-B22Nviop.woff2 +0 -0
  63. {machinechecked-0.1.12 → machinechecked-0.1.14}/engine/python/prooflens/web/assets/KaTeX_Math-BoldItalic-CZnvNsCZ.woff2 +0 -0
  64. {machinechecked-0.1.12 → machinechecked-0.1.14}/engine/python/prooflens/web/assets/KaTeX_Math-Italic-t53AETM-.woff2 +0 -0
  65. {machinechecked-0.1.12 → machinechecked-0.1.14}/engine/python/prooflens/web/assets/KaTeX_SansSerif-Bold-D1sUS0GD.woff2 +0 -0
  66. {machinechecked-0.1.12 → machinechecked-0.1.14}/engine/python/prooflens/web/assets/KaTeX_SansSerif-Italic-C3H0VqGB.woff2 +0 -0
  67. {machinechecked-0.1.12 → machinechecked-0.1.14}/engine/python/prooflens/web/assets/KaTeX_SansSerif-Regular-DDBCnlJ7.woff2 +0 -0
  68. {machinechecked-0.1.12 → machinechecked-0.1.14}/engine/python/prooflens/web/assets/KaTeX_Script-Regular-D3wIWfF6.woff2 +0 -0
  69. {machinechecked-0.1.12 → machinechecked-0.1.14}/engine/python/prooflens/web/assets/KaTeX_Size1-Regular-mCD8mA8B.woff2 +0 -0
  70. {machinechecked-0.1.12 → machinechecked-0.1.14}/engine/python/prooflens/web/assets/KaTeX_Size2-Regular-Dy4dx90m.woff2 +0 -0
  71. {machinechecked-0.1.12 → machinechecked-0.1.14}/engine/python/prooflens/web/assets/KaTeX_Size4-Regular-Dl5lxZxV.woff2 +0 -0
  72. {machinechecked-0.1.12 → machinechecked-0.1.14}/engine/python/prooflens/web/assets/KaTeX_Typewriter-Regular-CO6r4hn1.woff2 +0 -0
  73. {machinechecked-0.1.12 → machinechecked-0.1.14}/engine/python/prooflens/web/assets/cursor-editorTextHighlight-kG3uUOwL.svg +0 -0
  74. {machinechecked-0.1.12 → machinechecked-0.1.14}/engine/python/prooflens/web/assets/pdf.worker.min-yatZIOMy.mjs +0 -0
  75. {machinechecked-0.1.12 → machinechecked-0.1.14}/engine/python/prooflens/web/theme.css +0 -0
  76. {machinechecked-0.1.12 → machinechecked-0.1.14}/setup.cfg +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: machinechecked
3
- Version: 0.1.12
3
+ Version: 0.1.14
4
4
  Summary: Read a mathematical paper beside its Lean formalisation, and see claim by claim what Lean establishes.
5
5
  License: Apache-2.0
6
6
  Project-URL: Homepage, https://machinechecked.org
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: machinechecked
3
- Version: 0.1.12
3
+ Version: 0.1.14
4
4
  Summary: Read a mathematical paper beside its Lean formalisation, and see claim by claim what Lean establishes.
5
5
  License: Apache-2.0
6
6
  Project-URL: Homepage, https://machinechecked.org
@@ -68,6 +68,6 @@ engine/python/prooflens/web/assets/KaTeX_Size2-Regular-Dy4dx90m.woff2
68
68
  engine/python/prooflens/web/assets/KaTeX_Size4-Regular-Dl5lxZxV.woff2
69
69
  engine/python/prooflens/web/assets/KaTeX_Typewriter-Regular-CO6r4hn1.woff2
70
70
  engine/python/prooflens/web/assets/cursor-editorTextHighlight-kG3uUOwL.svg
71
- engine/python/prooflens/web/assets/index-C8DRs1uO.css
72
- engine/python/prooflens/web/assets/index-DXica3gb.js
71
+ engine/python/prooflens/web/assets/index-GfZV3aEH.css
72
+ engine/python/prooflens/web/assets/index-YAI0NIYT.js
73
73
  engine/python/prooflens/web/assets/pdf.worker.min-yatZIOMy.mjs
@@ -25,7 +25,7 @@ from __future__ import annotations
25
25
  from .status import fields
26
26
 
27
27
  from .reviews import Review, history
28
- from .source_regions import SCHEME as SOURCE_SCHEME, matches as source_matches
28
+ from .source_regions import SCHEME as SOURCE_SCHEME, changed_line, matches as source_matches
29
29
 
30
30
  OUT_OF_SCOPE = {"expository", "literature", "empirical", "proof-step", "part-of", "duplicate"}
31
31
 
@@ -541,9 +541,11 @@ class Assessor:
541
541
  "revision (python -m prooflens.source_regions --config FILE "
542
542
  f"--paper {pid}); then rebuild"))
543
543
  elif not source_matches(c, t):
544
+ where = changed_line(c, region) if region else None
544
545
  detail = region.get("error") or (
545
546
  "is missing; rebuild the source evidence" if not region else
546
- "differs from the frozen source region")
547
+ "differs from the frozen source region"
548
+ + (f": {where}" if where else ""))
547
549
  self.findings.append(_finding(
548
550
  "violation", "source-changed", c["id"],
549
551
  f"the source region at {c['location']} {detail}"))
@@ -184,25 +184,79 @@ def _pack(arc: Path, members: list[tuple[Path, str]]) -> None:
184
184
  raise CacheError("zstd failed while writing the archive")
185
185
 
186
186
 
187
+ def _unpacked_size(members: list[tuple[Path, str]]) -> int:
188
+ """The bytes of the files the archive holds, unpacked."""
189
+ total = 0
190
+ for src, _ in members:
191
+ for f in src.rglob("*"):
192
+ if f.is_file() and not f.is_symlink():
193
+ total += f.stat().st_size
194
+ return total
195
+
196
+
197
+ def _gb(n: float) -> str:
198
+ return f"{n / 1e9:.1f} GB"
199
+
200
+
201
+ def check_space(project: Path, tmp: Path, man: dict) -> None:
202
+ """Refuse a restore that cannot fit, before anything is downloaded: the archive needs its
203
+ size in `tmp`, and its unpacked build (recorded by `save`; four times the archive for an
204
+ older manifest) next to the project."""
205
+ size = int(man.get("size") or 0)
206
+ unpacked = int(man.get("unpacked_size") or 4 * size)
207
+ lake = project / ".lake"
208
+ where = lake if lake.exists() else project
209
+ need = {where: unpacked}
210
+ if os.stat(tmp).st_dev == os.stat(where).st_dev:
211
+ need[where] += size
212
+ else:
213
+ need[tmp] = size
214
+ for path, n in need.items():
215
+ free = shutil.disk_usage(path).free
216
+ if free < n:
217
+ raise CacheError(
218
+ f"not enough disk space to restore: {_gb(n)} needed under {path}, {_gb(free)} "
219
+ "free. Free some space (for example a second copy of a dependency at the same "
220
+ "revision, or old build output) and restore again; nothing was changed")
221
+
222
+
187
223
  def _unpack(arc: Path, project: Path) -> None:
188
224
  """Unpack an archive written by `_pack` into the project and its path dependencies, with the
189
- tar `data` filter (no absolute paths, no `..`, no links out of the destination)."""
225
+ tar `data` filter (no absolute paths, no `..`, no links out of the destination). Everything
226
+ is unpacked into a staging folder first and moved into place only once the whole archive
227
+ has been read, so a failure (a full disk, a corrupt archive) leaves the build as it was."""
190
228
  import tarfile
191
229
  deps = {d.name: d for d in path_deps(project)}
192
- z = subprocess.Popen(["zstd", "-d", "-q", "-c", str(arc)], stdout=subprocess.PIPE)
193
- with tarfile.open(fileobj=z.stdout, mode="r|") as tf:
194
- for m in tf:
195
- head, _, rest = m.name.partition("/")
196
- if head == "project" and rest:
197
- base, m.name = project, rest
198
- elif head == "path" and rest.partition("/")[0] in deps:
199
- name, _, m.name = rest.partition("/")
200
- base = deps[name]
201
- else:
202
- raise CacheError(f"unexpected path in the archive: {m.name}")
203
- tf.extract(m, base, filter="data")
204
- if z.wait() != 0:
205
- raise CacheError("zstd failed while reading the archive")
230
+ stage = Path(tempfile.mkdtemp(prefix=".lean-cache-restore-", dir=project))
231
+ try:
232
+ bases: dict[str, Path] = {}
233
+ z = subprocess.Popen(["zstd", "-d", "-q", "-c", str(arc)], stdout=subprocess.PIPE)
234
+ with tarfile.open(fileobj=z.stdout, mode="r|") as tf:
235
+ for m in tf:
236
+ head, _, rest = m.name.partition("/")
237
+ if head == "project" and rest:
238
+ key, base, m.name = "project", project, rest
239
+ elif head == "path" and rest.partition("/")[0] in deps:
240
+ name, _, m.name = rest.partition("/")
241
+ key, base = f"path-{name}", deps[name]
242
+ else:
243
+ raise CacheError(f"unexpected path in the archive: {m.name}")
244
+ bases[key] = base
245
+ tf.extract(m, stage / key, filter="data")
246
+ if z.wait() != 0:
247
+ raise CacheError("zstd failed while reading the archive")
248
+ for key, base in bases.items():
249
+ root = stage / key
250
+ for f in sorted(root.rglob("*")):
251
+ if f.is_dir() and not f.is_symlink():
252
+ continue
253
+ dest = base / f.relative_to(root)
254
+ dest.parent.mkdir(parents=True, exist_ok=True)
255
+ shutil.move(str(f), str(dest))
256
+ except OSError as e:
257
+ raise CacheError(f"the restore failed ({e}); the build was left as it was") from e
258
+ finally:
259
+ shutil.rmtree(stage, ignore_errors=True)
206
260
 
207
261
 
208
262
  def repository(project: Path) -> str | None:
@@ -263,7 +317,8 @@ def save(project: Path, out: Path, targets: list[str], pin_file: str | None = No
263
317
  manifest = {"commit": commit, "project": package_name(project), "layout": 2,
264
318
  "inputs_digest": inputs_digest(project), "repository": repository(project),
265
319
  "targets": targets or ["(default)"], "dirs": dirs, "sha256": digest,
266
- "size": size, "parts": parts, "licence_files": licences,
320
+ "size": size, "unpacked_size": _unpacked_size(_members(project)),
321
+ "parts": parts, "licence_files": licences,
267
322
  "created": time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()),
268
323
  **fingerprint(project)}
269
324
  (out / f"{name}.json").write_text(json.dumps(manifest, indent=1) + "\n")
@@ -403,6 +458,7 @@ def restore(project: Path, repo: str | None = None, local: Path | None = None,
403
458
  if no_history else ""))
404
459
  key, stem, man = best
405
460
  dist = key[1]
461
+ check_space(project, tmp, man)
406
462
  t0 = time.time()
407
463
  arc = tmp / f"{stem}.tar.zst"
408
464
  pieces = man["parts"] if man.get("parts") else [{"name": arc.name, "sha256": man["sha256"]}]
@@ -535,7 +535,7 @@ def cmd_symbol(a):
535
535
  try:
536
536
  if a.action == "add":
537
537
  ln = sy.add(cfg, a.claim, a.tex, a.decl, a.reviewer, var=a.var, type_of=a.type_of,
538
- term=a.term)
538
+ term=a.term, at=a.at)
539
539
  print(f"linked {a.tex} in {a.claim} to {ln['lean']} (by {ln['by']}); "
540
540
  "rebuild the site to show it")
541
541
  elif a.action == "remove":
@@ -808,6 +808,8 @@ def main(argv=None):
808
808
  p_.add_argument("--type-of", dest="type_of", help="the binder whose type it is")
809
809
  p_.add_argument("--term", help="the sub-term it is, as WHERE#K: b:<binder>, c (the "
810
810
  "conclusion) or l:<part>, and the sub-term's number")
811
+ p_.add_argument("--at", help="the sub-term it is, by its identity in the proposition "
812
+ "(the `id` the reader's data gives it)")
811
813
  p_.add_argument("--as", dest="reviewer",
812
814
  default=os.environ.get("MACHINECHECKED_REVIEWER"),
813
815
  help="human:NAME or llm:MODEL (default: $MACHINECHECKED_REVIEWER)")
@@ -169,16 +169,19 @@ def constPrint (ci : ConstantInfo) : MetaM (String × String × Array Name) := d
169
169
  index of the statement's binder it refers to (`var`, a position in `vars`), when the innermost
170
170
  subexpression around it is one. Each tagged subexpression is also recorded as the range of
171
171
  segments it covers (`terms`, in printing order, an outer one before those inside it), so that the
172
- reader can select a whole sub-term. -/
173
- partial def segments (vars : Array FVarId) (t : Widget.CodeWithInfos) (ref : Option (String × Json))
174
- (acc : Array Json) (terms : Array (Nat × Nat)) : Array Json × Array (Nat × Nat) :=
172
+ reader can select a whole sub-term, with its position in the whole proposition (`root`, the
173
+ position of the printed expression, extended by the tag's own `subexprPos`): the identity of a
174
+ sub-term, which depends on the proposition only, not on how it is printed or split. -/
175
+ partial def segments (vars : Array FVarId) (root : SubExpr.Pos) (t : Widget.CodeWithInfos)
176
+ (ref : Option (String × Json)) (acc : Array Json) (terms : Array (Nat × Nat × String)) :
177
+ Array Json × Array (Nat × Nat × String) :=
175
178
  match t with
176
179
  | Widget.TaggedText.text s =>
177
180
  match ref with
178
181
  | some (k, v) => (acc.push (Json.mkObj [("t", s), (k, v)]), terms)
179
182
  | none => (acc.push (Json.mkObj [("t", s)]), terms)
180
183
  | Widget.TaggedText.append xs =>
181
- xs.foldl (fun (a, ts) x => segments vars x ref a ts) (acc, terms)
184
+ xs.foldl (fun (a, ts) x => segments vars root x ref a ts) (acc, terms)
182
185
  | Widget.TaggedText.tag info sub =>
183
186
  let r := match info.info.val.info with
184
187
  | .ofTermInfo ti => match ti.expr with
@@ -188,30 +191,37 @@ partial def segments (vars : Array FVarId) (t : Widget.CodeWithInfos) (ref : Opt
188
191
  | _ => none
189
192
  let start := acc.size
190
193
  let slot := terms.size
191
- let (acc', terms') := segments vars sub r acc (terms.push (start, start))
192
- (acc', terms'.set! slot (start, acc'.size))
194
+ let id := toString (root.append info.subexprPos)
195
+ let (acc', terms') := segments vars root sub r acc (terms.push (start, start, id))
196
+ (acc', terms'.set! slot (start, acc'.size, id))
193
197
 
194
198
  /-- The sub-terms worth selecting: those covering two segments or more, each range once (the
195
199
  outermost tag of a range keeps it), and none for a very large term. -/
196
- def termsJson (terms : Array (Nat × Nat)) : Json := Id.run do
200
+ def termsJson (terms : Array (Nat × Nat × String)) : Json := Id.run do
197
201
  let mut out : Array Json := #[]
198
202
  let mut seen : Array (Nat × Nat) := #[]
199
- for (a, b) in terms do
203
+ for (a, b, id) in terms do
200
204
  if b - a ≥ 2 && !seen.contains (a, b) then
201
205
  seen := seen.push (a, b)
202
- out := out.push (Json.mkObj [("from", toJson a), ("to", toJson b)])
206
+ out := out.push (Json.mkObj [("from", toJson a), ("to", toJson b), ("id", Json.str id)])
203
207
  return Json.arr (if out.size > 3000 then #[] else out)
204
208
 
205
209
  /-- The fields of a pretty-printed expression with navigable constants and binders; plain text
206
- if tagging fails, and a marker if printing fails, so that one large term cannot stop the probe. -/
207
- def ppFields (e : Expr) (vars : Array FVarId := #[]) : MetaM (List (String × Json)) := do
210
+ if tagging fails, and a marker if printing fails, so that one large term cannot stop the probe.
211
+ `root` is the position of `e` in the whole proposition (`id`). -/
212
+ def ppFields (e : Expr) (vars : Array FVarId := #[]) (root : SubExpr.Pos := .root) :
213
+ MetaM (List (String × Json)) := do
208
214
  try
209
- -- The head of every application is tagged too (`pp.tagAppFns`), so that the function a
210
- -- term applies is a constant the reader can name; the text is the same.
211
- let code ← withOptions (fun o => o.setBool `pp.tagAppFns true) (Widget.ppExprTagged e)
212
- let text := toString (← ppExpr e)
213
- let (segs, terms) := segments vars code none #[] #[]
214
- return [("text", Json.str text), ("segments", Json.arr segs), ("terms", termsJson terms)]
215
+ -- The type of a bound variable is printed (`pp.funBinderTypes`): `∃ Φ : T, …` says what Φ
216
+ -- is. The head of every application is tagged too (`pp.tagAppFns`), so that the function a
217
+ -- term applies is a constant the reader can name. The text and the tagged code are printed
218
+ -- with the same options.
219
+ let typed := fun (o : Options) => o.setBool `pp.funBinderTypes true
220
+ let code ← withOptions (fun o => (typed o).setBool `pp.tagAppFns true) (Widget.ppExprTagged e)
221
+ let text := toString (← withOptions typed (ppExpr e))
222
+ let (segs, terms) := segments vars root code none #[] #[]
223
+ return [("text", Json.str text), ("segments", Json.arr segs), ("terms", termsJson terms),
224
+ ("id", Json.str (toString root))]
215
225
  catch _ =>
216
226
  try
217
227
  return [("text", Json.str (toString (← ppExpr e)))]
@@ -219,21 +229,8 @@ def ppFields (e : Expr) (vars : Array FVarId := #[]) : MetaM (List (String × Js
219
229
  return [("text", Json.str "(not printable)"),
220
230
  ("error", Json.str (toString (← err.toMessageData.toString)))]
221
231
 
222
- def ppJson (e : Expr) (vars : Array FVarId := #[]) : MetaM Json :=
223
- return Json.mkObj (← ppFields e vars)
224
-
225
- /-- The logical shape of a conclusion: a conjunction, disjunction or equivalence as a tree whose
226
- leaves are pretty-printed propositions. Display only; it changes no record that is assessed. -/
227
- partial def logicJson (vars : Array FVarId) (e : Expr) : MetaM Json := do
228
- let op := if e.isAppOfArity ``And 2 then some "and"
229
- else if e.isAppOfArity ``Or 2 then some "or"
230
- else if e.isAppOfArity ``Iff 2 then some "iff" else none
231
- match op with
232
- | some o => return Json.mkObj [("op", o),
233
- ("args", Json.arr #[← logicJson vars e.appFn!.appArg!, ← logicJson vars e.appArg!])]
234
- | none => return Json.mkObj [("leaf", ← ppJson e vars)]
235
-
236
- /-! ## The inspection representation -/
232
+ def ppJson (e : Expr) (vars : Array FVarId := #[]) (root : SubExpr.Pos := .root) : MetaM Json :=
233
+ return Json.mkObj (← ppFields e vars root)
237
234
 
238
235
  def isExistsApp (e : Expr) : Option (Name × Expr) :=
239
236
  if e.isAppOfArity ``Exists 2 then
@@ -242,36 +239,94 @@ def isExistsApp (e : Expr) : Option (Name × Expr) :=
242
239
  | _ => none
243
240
  else none
244
241
 
242
+ /-- The logical shape of a conclusion: a conjunction, disjunction, equivalence, negation,
243
+ implication, "for every" or "there is" as a tree whose leaves are pretty-printed propositions. A
244
+ quantifier inside the conclusion is a node (`forall`, `exists`) with its variable's name, type and
245
+ number, and its body as its one part. `vars` is
246
+ threaded through the tree, so that every witness has a variable of its own: two witnesses in
247
+ sibling parts never share one. `pos` is the position of `e` in the whole proposition; each
248
+ node and variable carries its own (`id`). Display only; it changes no record that is assessed. -/
249
+ partial def logicJson (vars : Array FVarId) (e : Expr) (pos : SubExpr.Pos) :
250
+ MetaM (Json × Array FVarId) := do
251
+ let op := if e.isAppOfArity ``And 2 then some "and"
252
+ else if e.isAppOfArity ``Or 2 then some "or"
253
+ else if e.isAppOfArity ``Iff 2 then some "iff" else none
254
+ match op with
255
+ | some o =>
256
+ let (a, vars) ← logicJson vars e.appFn!.appArg! pos.pushAppFn.pushAppArg
257
+ let (b, vars) ← logicJson vars e.appArg! pos.pushAppArg
258
+ return (Json.mkObj [("op", o), ("id", Json.str (toString pos)),
259
+ ("args", Json.arr #[a, b])], vars)
260
+ | none =>
261
+ if e.isAppOfArity ``Not 1 then
262
+ let (a, vars) ← logicJson vars e.appArg! pos.pushAppArg
263
+ return (Json.mkObj [("op", "not"), ("id", Json.str (toString pos)),
264
+ ("args", Json.arr #[a])], vars)
265
+ if let .forallE n t body bi := e then
266
+ if (← isProp t) && !body.hasLooseBVars then
267
+ -- An implication inside a part: its premise is a part of its own, not an assumption
268
+ -- of the whole statement.
269
+ let (a, vars) ← logicJson vars t pos.pushBindingDomain
270
+ let (b, vars) ← logicJson vars body pos.pushBindingBody
271
+ return (Json.mkObj [("op", "implies"), ("id", Json.str (toString pos)),
272
+ ("args", Json.arr #[a, b])], vars)
273
+ let tj ← ppJson t vars pos.pushBindingDomain
274
+ return ← withLocalDecl n bi t fun x => do
275
+ let idx := vars.size
276
+ let (b, vars') ←
277
+ logicJson (vars.push x.fvarId!) (body.instantiate1 x) pos.pushBindingBody
278
+ return (Json.mkObj [("op", "forall"), ("name", n.eraseMacroScopes.toString),
279
+ ("var", toJson idx), ("id", Json.str (toString pos)), ("type", tj),
280
+ ("args", Json.arr #[b])], vars')
281
+ match isExistsApp e with
282
+ | some (_, .lam n t body bi) =>
283
+ -- The witness is bound by the predicate `fun x => …`, the second argument of `Exists`.
284
+ let lam := pos.pushAppArg
285
+ let tj ← ppJson t vars lam.pushBindingDomain
286
+ withLocalDecl n bi t fun x => do
287
+ let idx := vars.size
288
+ let (b, vars') ←
289
+ logicJson (vars.push x.fvarId!) (body.instantiate1 x) lam.pushBindingBody
290
+ return (Json.mkObj [("op", "exists"), ("name", n.eraseMacroScopes.toString),
291
+ ("var", toJson idx), ("id", Json.str (toString lam)), ("type", tj),
292
+ ("args", Json.arr #[b])], vars')
293
+ | _ => return (Json.mkObj [("leaf", ← ppJson e vars pos)], vars)
294
+
295
+ /-! ## The inspection representation -/
296
+
245
297
  /-- Split a proposition into its quantifier prefix and its conclusion. Universal binders are
246
298
  `forall` (data), `hypothesis` (a proposition) or `instance`; existential binders are `exists`.
247
299
  Definitions are not unfolded: the prefix is the one written in the specification. -/
248
- partial def telescope (e : Expr) (acc : Array Json) (vars : Array FVarId := #[]) :
249
- MetaM (Array Json × Json) := do
300
+ partial def telescope (e : Expr) (acc : Array Json) (vars : Array FVarId := #[])
301
+ (pos : SubExpr.Pos := .root) : MetaM (Array Json × Json) := do
250
302
  match e with
251
303
  | .forallE n t body bi =>
252
304
  let isHyp ← isProp t
253
305
  let role :=
254
306
  if bi.isInstImplicit then "instance" else if isHyp then "hypothesis" else "forall"
255
- let tj ← ppJson t vars
307
+ let tj ← ppJson t vars pos.pushBindingDomain
256
308
  withLocalDecl n bi t fun x => do
257
309
  let hyps := (acc.filter fun j => j.getObjValD "role" == Json.str "hypothesis").size
258
310
  let nm := if n.hasMacroScopes || n.isAnonymous then s!"h{hyps + 1}"
259
311
  else n.toString
260
- let j := Json.mkObj [("role", role), ("name", nm),
261
- ("type", tj), ("binder", binderStr bi)]
262
- telescope (body.instantiate1 x) (acc.push j) (vars.push x.fvarId!)
312
+ -- What an instance supplies: a proposition (`[Nonempty E]`) or data (`[Inhabited E]`).
313
+ let supplies := if isHyp then "proposition" else "data"
314
+ let j := Json.mkObj [("role", role), ("name", nm), ("id", Json.str (toString pos)),
315
+ ("type", tj), ("binder", binderStr bi), ("supplies", supplies)]
316
+ telescope (body.instantiate1 x) (acc.push j) (vars.push x.fvarId!) pos.pushBindingBody
263
317
  | _ =>
264
318
  match isExistsApp e with
265
319
  | some (_, .lam n t body bi) =>
266
- let tj ← ppJson t vars
320
+ let lam := pos.pushAppArg
321
+ let tj ← ppJson t vars lam.pushBindingDomain
267
322
  withLocalDecl n bi t fun x => do
268
323
  let j := Json.mkObj [("role", "exists"), ("name", n.eraseMacroScopes.toString),
269
- ("type", tj), ("binder", binderStr bi)]
270
- telescope (body.instantiate1 x) (acc.push j) (vars.push x.fvarId!)
324
+ ("id", Json.str (toString lam)), ("type", tj), ("binder", binderStr bi)]
325
+ telescope (body.instantiate1 x) (acc.push j) (vars.push x.fvarId!) lam.pushBindingBody
271
326
  | _ =>
272
327
  -- The conclusion, with its logical shape when it has one (`logic`, display only).
273
- let fields ← ppFields e vars
274
- let logic ← logicJson vars e
328
+ let fields ← ppFields e vars pos
329
+ let (logic, _) ← logicJson vars e pos
275
330
  let shaped := (logic.getObjVal? "op").toOption.isSome
276
331
  return (acc, Json.mkObj (if shaped then fields ++ [("logic", logic)] else fields))
277
332
  where
@@ -102,8 +102,15 @@ def symbol_links(display: dict | None, cid: str, tex: str | None, lean: dict) ->
102
102
  target = ln.get("lean") or {}
103
103
  spec = (lean.get("specs") or {}).get(target.get("decl")) or {}
104
104
  pins = ln.get("pins") or {}
105
+ # A link to a sub-term is located by numbers that depend on how the statement is
106
+ # printed, so it also needs the text it was made on (older links have none).
107
+ from .symbols import target_text
108
+ shown = target_text(spec, target)
105
109
  stale = (pins.get("statement") != spec.get("fingerprint")
106
- or not tex or pins.get("source") != sha256_text(tex))
110
+ or not tex or pins.get("source") != sha256_text(tex)
111
+ or shown is None
112
+ or ("target" in pins and pins["target"] != shown)
113
+ or (("term" in target or "at" in target) and "target" not in pins))
107
114
  out.append({"symbol": ln.get("symbol") or {}, "lean": target, "by": ln.get("by"),
108
115
  "stale": bool(stale)})
109
116
  return out
@@ -82,16 +82,25 @@ class Region:
82
82
  end_column: int
83
83
  kind: str
84
84
  text: str
85
+ lines: tuple[str, ...] = ()
85
86
 
86
87
  @property
87
88
  def sha256(self) -> str:
88
89
  return hashlib.sha256(self.text.encode("utf-8")).hexdigest()
89
90
 
91
+ @property
92
+ def line_sha256(self) -> list[str]:
93
+ """One digest per source line the passage spans (the whole line, raw, without its line
94
+ terminator), so that a change can be located, a passage that only moved can be told
95
+ from a changed one, and any checker can verify the pins from the file's lines alone."""
96
+ return [hashlib.sha256(line.encode("utf-8")).hexdigest() for line in self.lines]
97
+
90
98
  def record(self) -> dict:
91
- # The evidence needs bounds and a digest, not an additional public copy of the text.
99
+ # The evidence needs bounds and digests, not an additional public copy of the text.
92
100
  return {"file": self.file, "start_line": self.start_line, "end_line": self.end_line,
93
101
  "start_column": self.start_column, "end_column": self.end_column,
94
- "kind": self.kind, "scheme": SCHEME, "sha256": self.sha256}
102
+ "kind": self.kind, "scheme": SCHEME, "sha256": self.sha256,
103
+ "line_sha256": self.line_sha256}
95
104
 
96
105
 
97
106
  def positive(value, name: str) -> int:
@@ -100,6 +109,10 @@ def positive(value, name: str) -> int:
100
109
  return value
101
110
 
102
111
 
112
+ def _span(lines: list[str], first: int, last: int) -> tuple[str, ...]:
113
+ return tuple(l.rstrip("\r\n") for l in lines[first - 1:last])
114
+
115
+
103
116
  def extract(root: Path, claim: dict) -> Region:
104
117
  loc = location(claim)
105
118
  if loc is None:
@@ -129,7 +142,8 @@ def extract(root: Path, claim: dict) -> Region:
129
142
  end = starts[end_line - 1] + end_col - 1
130
143
  if start >= end or not text[start:end].strip():
131
144
  raise RegionError("source region is empty or reversed")
132
- return Region(name, line, end_line, start_col, end_col, "span", text[start:end])
145
+ return Region(name, line, end_line, start_col, end_col, "span", text[start:end],
146
+ _span(lines, line, end_line))
133
147
  start = starts[line - 1] + len(first) - len(first.lstrip())
134
148
  opening = ENV.match(text, start)
135
149
  if opening and opening.group(1) == "begin" and opening.group(2).rstrip("*") in DISPLAYS:
@@ -143,13 +157,14 @@ def extract(root: Path, claim: dict) -> Region:
143
157
  end_line = text.count("\n", 0, end) + 1
144
158
  end_column = end - (text.rfind("\n", 0, end) + 1) + 1
145
159
  return Region(name, line, end_line, start - starts[line - 1] + 1,
146
- end_column, "environment", text[start:end])
160
+ end_column, "environment", text[start:end],
161
+ _span(lines, line, end_line))
147
162
  raise RegionError("source environment has no matching end")
148
163
  if first.lstrip().startswith(r"\begin"):
149
164
  raise RegionError("unsupported or malformed environment; select an explicit source region")
150
165
  if not first.strip() or first.lstrip().startswith("%"):
151
166
  raise RegionError("source line is blank or a comment; select the actual passage")
152
- return Region(name, line, line, 1, len(first) + 1, "line", first)
167
+ return Region(name, line, line, 1, len(first) + 1, "line", first, (first,))
153
168
 
154
169
 
155
170
  def collect(inventory: dict, paper: str, root: Path) -> dict:
@@ -174,6 +189,30 @@ def matches(claim: dict, tex: dict) -> bool:
174
189
  and claim["source_sha256"] == rec.get("sha256"))
175
190
 
176
191
 
192
+ def same_revision(pin: str, revision: str) -> bool:
193
+ """Whether an inventory's pin names the snapshot's revision: the same value, or an
194
+ abbreviated commit hash (at least 7 hexadecimal digits) of it."""
195
+ return pin == revision or bool(re.fullmatch(r"[0-9a-f]{7,40}", pin)
196
+ and re.fullmatch(r"[0-9a-f]{40}", revision)
197
+ and revision.startswith(pin))
198
+
199
+
200
+ def changed_line(claim: dict, rec: dict) -> str | None:
201
+ """Where a passage differs from its pins, from the line digests both sides keep: the first
202
+ changed line, by its place in the passage and in the file, or a change in its length."""
203
+ old, new = claim.get("source_line_sha256"), rec.get("line_sha256")
204
+ if not old or not new:
205
+ return None
206
+ for i, (a, b) in enumerate(zip(old, new)):
207
+ if a != b:
208
+ return (f"line {i + 1} of the passage (line {rec['start_line'] + i} of "
209
+ f"{rec['file']}) changed")
210
+ if len(old) != len(new):
211
+ return (f"the passage has {len(new)} lines, not {len(old)}: lines were "
212
+ + ("added" if len(new) > len(old) else "removed") + " at its end")
213
+ return None
214
+
215
+
177
216
  def migration_plan(inventory: dict, paper: str, root: Path, revision: str) -> tuple[dict, list]:
178
217
  """Rehash only the original frozen snapshot; never edit a review or a paper revision."""
179
218
  result = deepcopy(inventory)
@@ -182,7 +221,7 @@ def migration_plan(inventory: dict, paper: str, root: Path, revision: str) -> tu
182
221
  if claim["paper"] != paper or location(claim) is None:
183
222
  continue
184
223
  pin = claim.get("commit") or inventory.get("papers", {}).get(paper, {}).get("commit")
185
- if not pin or str(pin).removeprefix("arxiv:") != revision:
224
+ if not pin or not same_revision(str(pin).removeprefix("arxiv:"), revision):
186
225
  raise RegionError(f"{claim['id']}: original paper revision must match {revision}")
187
226
  scheme = claim.get("source_hash_scheme")
188
227
  if scheme not in (None, SCHEME):
@@ -192,27 +231,40 @@ def migration_plan(inventory: dict, paper: str, root: Path, revision: str) -> tu
192
231
  if scheme == SCHEME:
193
232
  if expected != region.sha256:
194
233
  raise RegionError(f"{claim['id']}: region differs from its frozen hash")
234
+ if claim.get("source_line_sha256") != region.line_sha256:
235
+ # The same passage: add the line digests it was pinned without.
236
+ claim["source_line_sha256"] = region.line_sha256
237
+ changes.append({"claim": claim["id"], "old_sha256": expected,
238
+ "lines_added": True, **region.record()})
195
239
  continue
196
240
  name, line = location(claim)
197
241
  original = source_file(root, name).read_text(encoding="utf-8").split("\n")[line - 1]
198
242
  legacy = hashlib.sha256(" ".join(original.split()).encode("utf-8")).hexdigest()
199
243
  if not expected or expected != legacy:
200
244
  raise RegionError(f"{claim['id']}: legacy hash does not match the original snapshot")
201
- claim.update(source_sha256=region.sha256, source_hash_scheme=SCHEME)
245
+ claim.update(source_sha256=region.sha256, source_hash_scheme=SCHEME,
246
+ source_line_sha256=region.line_sha256)
202
247
  changes.append({"claim": claim["id"], "old_sha256": expected, **region.record()})
203
248
  if not any(c["paper"] == paper for c in inventory["claims"]):
204
249
  raise RegionError(f"no claims for paper {paper}")
205
250
  return result, changes
206
251
 
207
252
 
253
+ def _indent(raw: bytes) -> int:
254
+ """The indentation of a JSON file, so that a rewrite changes only the fields it updates."""
255
+ m = re.search(rb"\n( +)\S", raw)
256
+ return len(m.group(1)) if m else 2
257
+
258
+
208
259
  def write_plan(path: Path, before: bytes, result: dict) -> None:
209
- """Atomically replace the inventory only if it is still the one that was planned."""
260
+ """Atomically replace the inventory only if it is still the one that was planned, in its
261
+ own indentation."""
210
262
  temporary = None
211
263
  try:
212
264
  with tempfile.NamedTemporaryFile(mode="w", encoding="utf-8", dir=path.parent,
213
265
  prefix=".source-regions-", delete=False) as handle:
214
266
  temporary = Path(handle.name)
215
- json.dump(result, handle, ensure_ascii=False, indent=2)
267
+ json.dump(result, handle, ensure_ascii=False, indent=_indent(before))
216
268
  handle.write("\n")
217
269
  handle.flush()
218
270
  os.fsync(handle.fileno())