lit-reader 1.2.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
lit_reader/core.py ADDED
@@ -0,0 +1,3377 @@
1
+ #!/usr/bin/env python3
2
+ """lit_reader.py: reading versions of academic papers as local HTML pages.
3
+
4
+ Each paper becomes one HTML page, styled for long reading: about 65 characters
5
+ a line, a choice of fonts, light, sepia or dark, a contents sidebar with time
6
+ left, citations that open as cards, notes and page numbers in the margins,
7
+ and a saved reading position.
8
+
9
+ The lit.py cache (~/.cache/lit) is never written to: the ledger's claims and
10
+ reads store character offsets into it. Sources fetched for reading live in
11
+ ~/.cache/lit-reader/src/<slug>/.
12
+
13
+ Routes, best first:
14
+ arxiv arXiv's own HTML (LaTeXML), or ar5iv's when arXiv has none: real headings, linked citations.
15
+ jats Europe PMC full-text XML (JATS): the same, for PMC papers.
16
+ pdf Docling's JSON from the cached PDF, with figures and tables cropped
17
+ from the PDF page by pdftoppm. Run Docling first (see `pdf-convert`).
18
+
19
+ Standalone use needs no ledger:
20
+ lit-reader PDF | ARXIV-ID-OR-URL | DOI | PMCID [-o DIR] [--route R] [--pages A-B]
21
+ Metadata comes from arXiv, Crossref or Europe PMC, or failing those the PDF's
22
+ first page. The page is written to ./<slug>/index.html. A local PDF is always
23
+ the source unless --route online asks for arXiv's or Europe PMC's version.
24
+
25
+ Ledger commands read their paths from a settings file, by default
26
+ ~/.config/lit-reader/config.toml (LIT_READER_CONFIG points elsewhere):
27
+ output_dir = "~/papers/reader" required
28
+ list_file = "~/papers/list.json" default: list.json in output_dir
29
+ lit_cache = "~/.cache/lit" optional
30
+ index_bar, index_kicker, index_note optional wording for the index page
31
+ [ledgers]
32
+ main = "~/papers/ledger.sqlite" names usable as --ledger or in the list
33
+ [email] for --email, in standalone mode too; see mail.py
34
+ to = "you@example.com"
35
+ smtp_host = "smtp.gmail.com"
36
+
37
+ fetch --route arxiv|jats --pid PID --ledger DB download the structured source (online)
38
+ pdf-convert --pid PID --ledger DB [--pages A-B] run Docling on the cached PDF (offline)
39
+ build --pid PID --ledger DB --route R [--pages A-B] write the page (offline)
40
+ index write index.html from list.json
41
+ all build every list.json entry that has a route
42
+
43
+ Every build prints a coverage check: the share of the cached pdftotext words
44
+ that reached the page, and the longest runs that did not. Those are the places
45
+ to look for dropped text.
46
+
47
+ SECURITY: fetched HTML, XML and PDFs are untrusted. Pages are built only from
48
+ whitelisted tags with escaped text; images are embedded only after checking
49
+ their magic bytes; PDFs are read only by Docling and pdftoppm, given paths as
50
+ arguments, never through a shell.
51
+ """
52
+ import argparse
53
+ import base64
54
+ import datetime
55
+ import functools
56
+ import hashlib
57
+ import html
58
+ import io
59
+ import json
60
+ import os
61
+ import re
62
+ import shutil
63
+ import sqlite3
64
+ import subprocess
65
+ import sys
66
+ import tempfile
67
+ import time
68
+ import unicodedata
69
+ import urllib.error
70
+ import urllib.parse
71
+ import urllib.request
72
+ import xml.etree.ElementTree as ET
73
+ import zipfile
74
+ from collections import Counter
75
+ from html.parser import HTMLParser
76
+ from pathlib import Path
77
+
78
+ from . import __version__
79
+
80
+ HERE = Path(__file__).resolve().parent
81
+ HOME = Path.home()
82
+ LIT_CACHE = Path(os.environ.get("LIT_CACHE", "~/.cache/lit")).expanduser()
83
+ CONFIG = Path(os.environ.get("LIT_READER_CONFIG") or
84
+ Path(os.environ.get("XDG_CONFIG_HOME") or HOME / ".config") / "lit-reader" / "config.toml").expanduser()
85
+ WORK = HOME / ".cache" / "lit-reader"
86
+ SRC = WORK / "src"
87
+ FONTS = HERE / "fonts"
88
+ OUT = None # the settings below fill these in for the ledger commands
89
+ LIST_FILE = None
90
+ LEDGERS = {}
91
+ INDEX_TEXT = {"index_bar": "Reading copies", "index_kicker": "Papers", "index_note": ""}
92
+ WPM = 200 # must match reader.js
93
+ UA = "lit-reader/%s (offline reading tool)" % __version__
94
+
95
+
96
+ class ReaderError(Exception):
97
+ status = None # the HTTP status, when a server answered with an error
98
+
99
+
100
+ def read_settings(path):
101
+ """The small part of TOML the settings file uses: comments, [sections] and
102
+ key = "string" lines. (tomllib needs Python 3.11; this tool supports 3.9.)"""
103
+ out = {}
104
+ section = out
105
+ for no, raw in enumerate(path.read_text(encoding="utf-8").splitlines(), 1):
106
+ line = raw.strip()
107
+ if not line or line.startswith("#"):
108
+ continue
109
+ m = re.fullmatch(r"\[([A-Za-z0-9_-]+)\]", line)
110
+ if m:
111
+ section = out.setdefault(m.group(1), {})
112
+ continue
113
+ m = re.fullmatch(r"""([A-Za-z0-9_-]+)\s*=\s*(?:"([^"\\]*)"|'([^']*)')\s*(?:#.*)?""", line)
114
+ if not m:
115
+ raise ReaderError("%s line %d: expected key = \"value\"" % (path, no))
116
+ section[m.group(1)] = m.group(2) if m.group(2) is not None else m.group(3)
117
+ return out
118
+
119
+
120
+ def apply_settings():
121
+ """Load the paths the ledger commands need. Standalone builds never call this."""
122
+ global OUT, LIST_FILE, LIT_CACHE
123
+ conf = read_settings(CONFIG) if CONFIG.is_file() else {}
124
+ out = os.environ.get("LIT_READER_OUT") or conf.get("output_dir")
125
+ if not out:
126
+ raise ReaderError("ledger commands need a settings file with output_dir: %s" % CONFIG)
127
+ OUT = Path(out).expanduser()
128
+ LIST_FILE = Path(conf["list_file"]).expanduser() if conf.get("list_file") else OUT / "list.json"
129
+ if conf.get("lit_cache") and not os.environ.get("LIT_CACHE"):
130
+ LIT_CACHE = Path(conf["lit_cache"]).expanduser()
131
+ LEDGERS.update(conf.get("ledgers") or {})
132
+ INDEX_TEXT.update({k: conf[k] for k in INDEX_TEXT if k in conf})
133
+
134
+
135
+ def esc(s):
136
+ return html.escape(s, quote=False)
137
+
138
+
139
+ def attr(s):
140
+ return html.escape(s, quote=True)
141
+
142
+
143
+ def safe_id(s):
144
+ return re.sub(r"[^A-Za-z0-9_-]", "-", s)
145
+
146
+
147
+ def words(text):
148
+ return len(re.findall(r"\w+", text))
149
+
150
+
151
+ def strip_tags(h):
152
+ return html.unescape(re.sub(r"<[^>]+>", " ", h))
153
+
154
+
155
+ # --------------------------------------------------------------------------
156
+ # A small DOM, shared by every route
157
+ # --------------------------------------------------------------------------
158
+
159
+ VOID = {"area", "base", "br", "col", "embed", "hr", "img", "input", "link", "meta",
160
+ "source", "track", "wbr", "mspace", "none", "mprescripts"}
161
+
162
+
163
+ class Node(object):
164
+ __slots__ = ("tag", "attrs", "kids", "parent")
165
+
166
+ def __init__(self, tag, attrs=None, parent=None):
167
+ self.tag = tag
168
+ self.attrs = dict(attrs or {})
169
+ self.kids = []
170
+ self.parent = parent
171
+
172
+ def cls(self):
173
+ return set((self.attrs.get("class") or "").split())
174
+
175
+ def text(self):
176
+ return "".join(k if isinstance(k, str) else k.text() for k in self.kids)
177
+
178
+ def iter(self):
179
+ yield self
180
+ for k in self.kids:
181
+ if isinstance(k, Node):
182
+ for d in k.iter():
183
+ yield d
184
+
185
+ def find(self, pred):
186
+ for d in self.iter():
187
+ if d is not self and pred(d):
188
+ return d
189
+ return None
190
+
191
+ def findall(self, pred):
192
+ return [d for d in self.iter() if d is not self and pred(d)]
193
+
194
+ def children(self):
195
+ return [k for k in self.kids if isinstance(k, Node)]
196
+
197
+
198
+ class DomBuilder(HTMLParser):
199
+ def __init__(self):
200
+ HTMLParser.__init__(self, convert_charrefs=True)
201
+ self.root = Node("#root")
202
+ self.cur = self.root
203
+
204
+ def handle_starttag(self, tag, attrs):
205
+ n = Node(tag, [(k, v or "") for k, v in attrs], self.cur)
206
+ self.cur.kids.append(n)
207
+ if tag not in VOID:
208
+ self.cur = n
209
+
210
+ def handle_startendtag(self, tag, attrs):
211
+ self.cur.kids.append(Node(tag, [(k, v or "") for k, v in attrs], self.cur))
212
+
213
+ def handle_endtag(self, tag):
214
+ c = self.cur
215
+ while c is not self.root and c.tag != tag:
216
+ c = c.parent
217
+ if c is not self.root:
218
+ self.cur = c.parent
219
+
220
+ def handle_data(self, data):
221
+ self.cur.kids.append(data)
222
+
223
+
224
+ def parse_html(text):
225
+ b = DomBuilder()
226
+ b.feed(text)
227
+ b.close()
228
+ return b.root
229
+
230
+
231
+ def local(name):
232
+ return name.rsplit("}", 1)[-1].rsplit(":", 1)[-1]
233
+
234
+
235
+ def et_to_node(e, parent=None):
236
+ n = Node(local(e.tag), {local(k): v for k, v in e.attrib.items()}, parent)
237
+ if e.text:
238
+ n.kids.append(e.text)
239
+ for c in e:
240
+ n.kids.append(et_to_node(c, n))
241
+ if c.tail:
242
+ n.kids.append(c.tail)
243
+ return n
244
+
245
+
246
+ # --------------------------------------------------------------------------
247
+ # Safe output: only these tags and attributes ever reach a page
248
+ # --------------------------------------------------------------------------
249
+
250
+ DROP = {"script", "style", "annotation", "annotation-xml", "svg", "iframe", "object", "embed",
251
+ "noscript", "template", "button", "form", "input", "textarea", "select", "link", "meta",
252
+ "img", "video", "audio", "canvas", "tex-math", "inline-graphic", "head", "title"}
253
+ SIMPLE = {"em": "em", "i": "em", "italic": "em", "strong": "strong", "b": "strong", "bold": "strong",
254
+ "sup": "sup", "sub": "sub", "code": "code", "monospace": "code", "sc": "span", "u": "u",
255
+ "underline": "u", "small": "small"}
256
+ MATHML = {
257
+ "math": ("display",), "mi": ("mathvariant",), "mo": ("stretchy", "fence", "separator", "form", "lspace", "rspace", "largeop", "movablelimits"),
258
+ "mn": (), "mrow": (), "msub": (), "msup": (), "msubsup": (), "mfrac": ("linethickness",),
259
+ "msqrt": (), "mroot": (), "mover": ("accent",), "munder": ("accentunder",), "munderover": (),
260
+ "mtext": (), "mspace": ("width",), "mtable": (), "mtr": (), "mtd": ("columnspan",), "semantics": (),
261
+ "mstyle": ("displaystyle", "scriptlevel"), "mpadded": (), "mphantom": (), "mmultiscripts": (),
262
+ "mprescripts": (), "none": (), "menclose": ("notation",), "ms": (),
263
+ }
264
+ TABLE = {"table": (), "thead": (), "tbody": (), "tfoot": (), "tr": (), "th": ("colspan", "rowspan"),
265
+ "td": ("colspan", "rowspan"), "caption": ()}
266
+ ATTR_OK = re.compile(r"^[\w .%#:-]{0,40}$")
267
+
268
+
269
+ def math_html(n):
270
+ """MathML, whitelisted; annotations (LaTeX source) dropped."""
271
+ if isinstance(n, str):
272
+ return esc(n)
273
+ if n.tag in DROP:
274
+ return ""
275
+ inner = "".join(math_html(k) for k in n.kids)
276
+ if n.tag not in MATHML:
277
+ return inner
278
+ attrs = "".join(' %s="%s"' % (a, attr(n.attrs[a])) for a in MATHML[n.tag]
279
+ if a in n.attrs and ATTR_OK.match(n.attrs[a]))
280
+ if n.tag == "math":
281
+ return '<math%s>%s</math>' % (attrs, inner)
282
+ return "<%s%s>%s</%s>" % (n.tag, attrs, inner, n.tag)
283
+
284
+
285
+ def safe_href(h):
286
+ h = (h or "").strip()
287
+ if h.startswith("#"):
288
+ return "#" + safe_id(h[1:])
289
+ try:
290
+ p = urllib.parse.urlparse(h)
291
+ except ValueError: # a malformed link in the paper is dropped, not fatal
292
+ return None
293
+ if p.scheme in ("http", "https") and p.netloc:
294
+ return h
295
+ return None
296
+
297
+
298
+ class Inline(object):
299
+ """Turns a node's children into safe inline HTML. A route subclasses
300
+ `rule` to handle its own citation, note and link markup."""
301
+
302
+ def __init__(self, doc):
303
+ self.doc = doc
304
+
305
+ def run(self, node):
306
+ return "".join(self.one(k) for k in node.kids)
307
+
308
+ def rule(self, n):
309
+ return None
310
+
311
+ def one(self, k):
312
+ if isinstance(k, str):
313
+ return esc(k)
314
+ r = self.rule(k)
315
+ if r is not None:
316
+ return r
317
+ t = k.tag
318
+ if t in DROP:
319
+ return ""
320
+ if t in SIMPLE:
321
+ inner = self.run(k)
322
+ if not inner.strip():
323
+ return inner
324
+ tag = SIMPLE[t]
325
+ if t == "sc":
326
+ return '<span style="font-variant:small-caps">%s</span>' % inner
327
+ return "<%s>%s</%s>" % (tag, inner, tag)
328
+ if t in ("br", "break"):
329
+ return "<br>"
330
+ if t == "math":
331
+ return math_html(k)
332
+ if t in ("a", "ext-link", "uri"):
333
+ href = safe_href(k.attrs.get("href") or k.attrs.get("xlink:href") or k.attrs.get("href", ""))
334
+ inner = self.run(k)
335
+ if href and not href.startswith("#"):
336
+ return '<a href="%s" rel="noopener noreferrer">%s</a>' % (attr(href), inner)
337
+ if href:
338
+ return '<a href="%s">%s</a>' % (attr(href), inner)
339
+ return inner
340
+ if t in TABLE:
341
+ attrs = "".join(' %s="%s"' % (a, attr(k.attrs[a])) for a in TABLE[t]
342
+ if a in k.attrs and re.match(r"^\d{1,2}$", k.attrs[a]))
343
+ cls = ' class="data"' if t == "table" else ""
344
+ return "<%s%s%s>%s</%s>" % (t, cls, attrs, self.run(k), t)
345
+ if t in ("ul", "ol", "li", "list", "list-item"):
346
+ tag = {"list": "ul", "list-item": "li"}.get(t, t)
347
+ return "<%s>%s</%s>" % (tag, self.run(k), tag)
348
+ if t == "p" or (t == "div" and k.parent is not None and k.parent.tag in ("td", "th", "li", "list-item")):
349
+ return self.run(k) + " "
350
+ return self.run(k)
351
+
352
+
353
+ # --------------------------------------------------------------------------
354
+ # The document model and the page renderer
355
+ # --------------------------------------------------------------------------
356
+
357
+ class Doc(object):
358
+ def __init__(self, meta):
359
+ self.meta = meta # title, authors, year, venue, doi, pid, url
360
+ self.abstracts = [] # (heading, html)
361
+ self.blocks = [] # dicts, see render()
362
+ self.refs = [] # (id, html)
363
+ self.ref_ids = set()
364
+ self.nnotes = 0
365
+ self.source = "" # one line: where the text came from
366
+ self.pdf = None # Path of the cached PDF, for page links
367
+ self.page_images = {} # PDF page -> image of the printed page, offered where text may be missing
368
+
369
+ def note(self, inner_html, shown=None):
370
+ """A sidenote. `shown` keeps the number the paper itself gives the note."""
371
+ self.nnotes += 1
372
+ n = self.nnotes
373
+ shown = shown or n
374
+ return ('<span class="sn-mark" data-for="sn-%d" role="button" tabindex="0" aria-label="note %d">%d</span>'
375
+ '<span class="sidenote" id="sn-%d"><span class="n">%d</span>%s</span>') % (n, shown, shown, n, shown, inner_html)
376
+
377
+ def add(self, **b):
378
+ self.blocks.append(b)
379
+
380
+ def cite(self, ids, inner):
381
+ ids = [safe_id(i) for i in ids if i]
382
+ if not ids:
383
+ return inner
384
+ return '<a class="cite" href="#ref-%s" data-ref="%s">%s</a>' % (ids[0], attr(" ".join(ids)), inner)
385
+
386
+
387
+ AUTHORS_SHOWN = 3 # a list longer than AUTHORS_MAX opens with this many names
388
+ AUTHORS_MAX = 5
389
+
390
+
391
+ def surname(name):
392
+ name = name.strip()
393
+ if "," in name:
394
+ return name.split(",", 1)[0].strip()
395
+ parts = [p for p in name.split() if p.rstrip(".,").lower() not in ("jr", "sr", "ii", "iii", "iv")]
396
+ if not parts:
397
+ return name
398
+ i = len(parts) - 1
399
+ while i > 1 and parts[i - 1].lower() in ("van", "von", "der", "den", "de", "del", "della", "di", "da", "du", "la", "le", "dos", "das", "ter"):
400
+ i -= 1 # "Johannes van der Waals" is cited as "van der Waals"
401
+ return " ".join(parts[i:])
402
+
403
+
404
+ def short_citation(m):
405
+ """Author-year, as a quote from this paper would be cited in the text."""
406
+ a = [surname(x) for x in (m.get("authors") or []) if x.strip()]
407
+ if a:
408
+ who = a[0] if len(a) == 1 else "%s and %s" % (a[0], a[1]) if len(a) == 2 else "%s et al." % a[0]
409
+ else: # no authors on record (a local PDF with no DOI): cite by a short title, not as anonymous
410
+ t = (m.get("title") or "Untitled").split()
411
+ who = "“%s”" % (" ".join(t[:5]) + ("…" if len(t) > 5 else ""))
412
+ return "%s, %s" % (who, m.get("year") or "n.d.")
413
+
414
+
415
+ def full_reference(m, title):
416
+ """One plain line for a reference list: authors (year). Title. Venue. Link."""
417
+ a = [x.strip() for x in (m.get("authors") or []) if x.strip()]
418
+ who = ", ".join(a[:20]) + (" et al." if len(a) > 20 else "") if a else ""
419
+ end = lambda t: t if t.endswith(("?", "!")) else t.rstrip(".") + "." # "Lonely?" keeps its own mark
420
+ bits = ["%s (%s)." % (who, m.get("year") or "n.d.")] if who else []
421
+ bits.append(end(title.strip()) + ("" if who else " (%s)." % (m.get("year") or "n.d.")))
422
+ if m.get("venue"):
423
+ bits.append(end(m["venue"].strip()))
424
+ if m.get("doi"):
425
+ bits.append("https://doi.org/" + m["doi"])
426
+ elif m.get("arxiv"):
427
+ bits.append("https://arxiv.org/abs/" + m["arxiv"])
428
+ elif m.get("url"):
429
+ bits.append(m["url"])
430
+ return " ".join(bits)
431
+
432
+
433
+ def authors_html(authors):
434
+ """The byline. A long author list starts collapsed, so the paper comes first."""
435
+ if len(authors) <= AUTHORS_MAX:
436
+ return '<p class="authors">%s</p>' % esc(", ".join(authors))
437
+ rest = len(authors) - AUTHORS_SHOWN
438
+ return ('<p class="authors">%s<span class="au-rest" id="au-rest" hidden>, %s</span> '
439
+ '<button class="au-toggle" id="au-toggle" type="button" aria-expanded="false" aria-controls="au-rest" data-more="Show %d more">'
440
+ '<span class="au-label">Show %d more</span><span class="au-chevron" aria-hidden="true"></span></button></p>') % (
441
+ esc(", ".join(authors[:AUTHORS_SHOWN])), esc(", ".join(authors[AUTHORS_SHOWN:])), rest, rest)
442
+
443
+
444
+ def image_size(src):
445
+ """(width, height) of an embedded image, so the page doesn't shift as images load."""
446
+ try:
447
+ d = base64.b64decode(src.split(",", 1)[1])
448
+ except (IndexError, ValueError):
449
+ return None
450
+ if d[:8] == b"\x89PNG\r\n\x1a\n" and len(d) >= 24:
451
+ return int.from_bytes(d[16:20], "big"), int.from_bytes(d[20:24], "big")
452
+ if d[:3] == b"GIF" and len(d) >= 10:
453
+ return int.from_bytes(d[6:8], "little"), int.from_bytes(d[8:10], "little")
454
+ i = 2
455
+ while d[:2] == b"\xff\xd8" and i + 9 < len(d):
456
+ if d[i] != 0xFF:
457
+ return None
458
+ m = d[i + 1]
459
+ if 0xC0 <= m <= 0xCF and m not in (0xC4, 0xC8, 0xCC):
460
+ return int.from_bytes(d[i + 7:i + 9], "big"), int.from_bytes(d[i + 5:i + 7], "big")
461
+ i += 2 + int.from_bytes(d[i + 2:i + 4], "big")
462
+ return None
463
+
464
+
465
+ def img_tag(src, alt):
466
+ alt = "" if (alt or "").strip().lower() == "refer to caption" else (alt or "") # arXiv's placeholder; the caption follows
467
+ size = image_size(src)
468
+ dims = ' width="%d" height="%d"' % size if size else ""
469
+ return '<img src="%s" alt="%s"%s loading="lazy">' % (src, attr(alt), dims)
470
+
471
+
472
+ def heading_text(h):
473
+ """A heading's words, without any footnote attached to it."""
474
+ def words_of(n):
475
+ if isinstance(n, str):
476
+ return n
477
+ return "" if n.cls() & {"sn-mark", "sidenote"} else "".join(words_of(k) for k in n.kids)
478
+ return words_of(parse_html(h))
479
+
480
+
481
+ def pdf_href(doc):
482
+ """Standalone pages link to their own copy of the PDF, so no local path is written into the page.
483
+ A single-file copy travels without the PDF, so it links only to a PDF online (None otherwise)."""
484
+ href = doc.meta.get("pdf_href") or doc.pdf.as_uri()
485
+ if doc.meta.get("single_file") and not href.startswith(("https://", "http://")):
486
+ return None
487
+ return href
488
+
489
+
490
+ def pdf_link(doc, page):
491
+ if not doc.pdf or not page or not pdf_href(doc):
492
+ return ""
493
+ return '<a class="pdf-link" href="%s#page=%d" target="_blank" rel="noopener">open page %d of the PDF</a>' % (attr(pdf_href(doc)), page, page)
494
+
495
+
496
+ def render(doc):
497
+ """Doc -> page HTML."""
498
+ m = doc.meta
499
+ out, toc = [], []
500
+ used = set()
501
+ sec_id = ""
502
+ body = True
503
+ bid = 0
504
+ # words per heading, for the contents sidebar
505
+ heads = [i for i, b in enumerate(doc.blocks) if b["t"] == "h"]
506
+ sec_words = {}
507
+ for n, i in enumerate(heads):
508
+ lvl = doc.blocks[i]["level"]
509
+ w = 0
510
+ for b in doc.blocks[i + 1:]:
511
+ if b["t"] == "h" and b["level"] <= lvl:
512
+ break
513
+ w += b.get("_w", 0) or words(strip_tags(block_text(b)))
514
+ sec_words[i] = w
515
+
516
+ labels = dict((b["pdf"], b["label"]) for b in doc.blocks if b["t"] == "page" and b.get("pdf"))
517
+ shown = set()
518
+
519
+ def original_pages(upto):
520
+ """The printed page, behind a toggle, for each flagged page up to `upto`."""
521
+ res = []
522
+ for p in sorted(doc.page_images):
523
+ if p in shown or (upto is not None and p > upto):
524
+ continue
525
+ shown.add(p)
526
+ name = ("p. %s" % labels[p]) if labels.get(p) else ("page %d of the PDF" % p)
527
+ res.append('<details class="orig-page"><summary>Show the original page (%s)</summary>'
528
+ '<p class="orig-note">Some text on this page may not have come through. This is the page as printed.</p>%s</details>'
529
+ % (esc(name), img_tag(doc.page_images[p], "The original %s" % name)))
530
+ return "".join(res)
531
+
532
+ for i, b in enumerate(doc.blocks):
533
+ t = b["t"]
534
+ bid += 1
535
+ if t == "h":
536
+ hid = b.get("id") or "s-" + safe_id(heading_text(b["html"]).strip().lower())[:40]
537
+ while hid in used:
538
+ hid += "-"
539
+ used.add(hid)
540
+ sec_id = hid
541
+ if b.get("appendix") and body:
542
+ body = False
543
+ out.append(DONE_SLOT) # the main text ends here, and so does a reader
544
+ out.append('<div class="appendix-start"></div>')
545
+ lvl = min(max(b["level"], 2), 4)
546
+ h_html = re.sub(r"\.\s*$", "", b["html"].strip()) # run-in heads: "Control Variables." (the doc stays as it was: it may be rendered twice)
547
+ title = heading_text(h_html).strip()
548
+ no = '<span class="secno">%s</span>' % esc(b["no"]) if b.get("no") else ""
549
+ out.append('<h%d id="%s" data-sec="1" data-title="%s">%s%s</h%d>' % (
550
+ lvl, hid, attr(((b.get("no") or "") + " " + title).strip()), no, h_html, lvl))
551
+ if lvl <= 3:
552
+ mins = max(1, round(sec_words.get(i, 0) / float(WPM)))
553
+ toc.append((lvl, hid, ((b.get("no") or "") + " " + title).strip(), mins))
554
+ continue
555
+ w = words(strip_tags(block_text(b)))
556
+ common = 'id="b%d" data-w="%d" data-in="%s" data-body="%d"' % (bid, w, sec_id, 1 if body else 0)
557
+ if t == "p":
558
+ out.append("<p %s>%s</p>" % (common, chunked(b["html"])))
559
+ elif t == "page":
560
+ if doc.page_images and b.get("pdf"):
561
+ out.append(original_pages(b["pdf"] - 1)) # flagged pages that have no marker of their own
562
+ out.append('<span class="pg" id="pg-%s">%s</span>' % (
563
+ safe_id(str(b["label"])), (pdf_anchor(doc, b) or "p. " + esc(str(b["label"])))))
564
+ if doc.page_images and b.get("pdf"):
565
+ out.append(original_pages(b["pdf"]))
566
+ elif t == "list":
567
+ tag = "ol" if b.get("ordered") else "ul"
568
+ out.append('<%s %s%s>%s</%s>' % (tag, common, ' class="plain"' if b.get("plain") else "", "".join("<li>%s</li>" % x for x in b["items"]), tag))
569
+ elif t == "eq":
570
+ out.append('<div class="eq" %s>%s</div>' % (common, b["html"]))
571
+ elif t == "pre":
572
+ out.append('<pre class="listing" %s>%s</pre>' % (common, esc(b["text"])))
573
+ elif t == "fig":
574
+ parts = []
575
+ for src, alt in b.get("imgs") or []:
576
+ parts.append(img_tag(src, alt))
577
+ if b.get("table"):
578
+ parts.append('<div class="tablewrap" role="region" tabindex="0" aria-label="%s">%s</div>' % (attr((b.get("label") or "Table").rstrip(":. ")), b["table"]))
579
+ if not parts:
580
+ parts.append('<div class="missing-figure">This %s did not come through in the source. %s</div>' % (
581
+ "table" if b.get("kind") == "table" else "figure", pdf_link(doc, b.get("page")) or ""))
582
+ cap = ""
583
+ if b.get("label") or b.get("caption"):
584
+ link = pdf_link(doc, b.get("page")) if (b.get("imgs") or b.get("table")) else ""
585
+ cap = "<figcaption>%s%s%s</figcaption>" % (
586
+ '<span class="label">%s</span>' % esc(b["label"]) if b.get("label") else "",
587
+ b.get("caption") or "", (" " + link) if link else "")
588
+ after = ""
589
+ if b.get("table_text"): # Docling's reading of a table shown as an image; after the figure, so the caption stays its last child
590
+ after = ('<details class="table-text"><summary>%s as text</summary>'
591
+ '<p class="orig-note">From the conversion and not checked by hand: compare it with the image above before relying on it.</p>'
592
+ '<div class="tablewrap" role="region" tabindex="0" aria-label="%s as text">%s</div></details>') % (
593
+ esc((b.get("label") or "Table").rstrip(":. ")), attr((b.get("label") or "Table").rstrip(":. ")), b["table_text"])
594
+ fid = ' data-fig="%s"' % attr(b["id"]) if b.get("id") else ""
595
+ out.append('<figure %s%s>%s%s</figure>%s' % (common.replace('id="b%d"' % bid, 'id="%s"' % (b.get("id") or "b%d" % bid)), fid, "".join(parts), cap, after))
596
+
597
+ if doc.page_images:
598
+ out.append(original_pages(None))
599
+
600
+ # references
601
+ refs_html = ""
602
+ if doc.refs:
603
+ linked = {} if m.get("single_file") else bundle_links(doc.refs, m) # a single file travels without the other pages
604
+ items = "".join('<li id="ref-%s"%s>%s%s</li>' % (safe_id(i), ' value="%s"' % i if str(i).isdigit() else "", h, linked.get(i, ""))
605
+ for i, h in doc.refs)
606
+ refs_html = ('<section class="references" id="references" data-title="References">'
607
+ '<h2 id="references-start" data-sec="1" data-title="References" data-w="0" data-in="references" data-body="0">References</h2>'
608
+ '<details open><summary>%d references (citations in the text open these as cards)</summary><ol>%s</ol></details></section>') % (len(doc.refs), items)
609
+ toc.append((2, "references", "References", 0))
610
+
611
+ def chunk_ps(a):
612
+ return re.sub(r"<p>(.*?)</p>", lambda m: "<p>%s</p>" % chunked(m.group(1), 40), a, flags=re.S)
613
+ abstracts = "".join('<section class="abstract" id="abstract-%d"><h2>%s</h2>%s</section>' % (n + 1, esc(h), chunk_ps(a))
614
+ for n, (h, a) in enumerate(doc.abstracts))
615
+ total_words = sum(words(strip_tags(block_text(b))) for b in doc.blocks if b["t"] != "h")
616
+ authors = m.get("authors") or []
617
+ meta_bits = [x for x in [esc(m.get("venue") or ""), str(m.get("year") or "")] if x]
618
+ doi = m.get("doi")
619
+ if doi:
620
+ meta_bits.append('<a href="https://doi.org/%s" target="_blank" rel="noopener">doi:%s</a>' % (attr(doi), esc(doi)))
621
+ elif m.get("url"):
622
+ meta_bits.append('<a href="%s" target="_blank" rel="noopener">%s</a>' % (attr(m["url"]), esc(m["url"])))
623
+ main_mins = max(1, round(main_words(doc) / float(WPM)))
624
+ meta_bits.append("about %d min of reading in the main text" % main_mins)
625
+
626
+ alone = bool((m.get("standalone") and not m.get("bundle")) or m.get("single_file")) # a single file has no list of papers beside it
627
+ back = '<ol class="back"><li><a href="#top">↑ Back to the start</a></li>%s</ol>' % (
628
+ "" if alone else '<li><a href="index.html">← All papers</a></li>')
629
+ # on a phone the top bar holds only the contents button and where you are; the rest waits here
630
+ tools = '<div class="toc-tools">%s<button type="button" data-proxy="clips">Quotes<span class="clip-count-n"></span></button>' \
631
+ '<button type="button" data-proxy="aa" aria-label="Aa, reading settings">Aa</button></div>' % (
632
+ "" if alone else all_link(m.get("place")).replace('class="btn"', 'class="btn tools-all"', 1))
633
+ toc_html = tools + '<div class="toc-head">Contents</div><ol>%s</ol>%s' % ("".join(
634
+ '<li class="l%d"><a href="#%s"><span>%s</span><span class="mins">%s</span></a></li>' % (
635
+ l, h, esc(t), ("%d min" % mins) if mins else "")
636
+ for l, h, t, mins in toc), back)
637
+
638
+ title_text = strip_tags(m.get("title_html") or esc(m.get("title", ""))).strip()
639
+ page = PAGE.format(
640
+ title=esc(title_text), pid=attr(m.get("pid", "")),
641
+ where="About %d min of reading" % main_mins, # the script keeps this up to date as you read
642
+ cite=attr(short_citation(m)), reference=attr(full_reference(m, title_text)),
643
+ kicker=esc(" · ".join(x for x in [m.get("kicker")] if x) or "Reading copy"),
644
+ title_html=m.get("title_html") or esc(m.get("title", "")),
645
+ part=('<p class="part">%s</p>' % esc(m["part"])) if m.get("part") else "",
646
+ authors=authors_html(authors), meta=" · ".join(meta_bits),
647
+ abstracts=abstracts, body="\n".join(out), refs=refs_html, toc=toc_html,
648
+ source=esc(doc.source), built=datetime.date.today().isoformat(),
649
+ panel=PANEL, all_link="" if alone else all_link(m.get("place")), done="" if alone else DONE_SLOT)
650
+ # the finished prompt goes where the main text ends: before any appendix, or else before the references
651
+ page = page.replace(DONE_SLOT, "" if alone else DONE_BOX, 1).replace(DONE_SLOT, "")
652
+ return page, total_words
653
+
654
+
655
+ ABBR = {"al", "e.g", "i.e", "eg", "ie", "fig", "figs", "eq", "eqs", "vs", "cf", "approx", "no", "nos", "pp", "p",
656
+ "st", "dr", "mr", "ms", "mrs", "prof", "inc", "ltd", "jr", "sr", "etc", "ca", "vol", "ch", "sec", "tab", "ref", "refs"}
657
+
658
+
659
+ def chunks(h, target):
660
+ """Split one paragraph of safe HTML into runs of whole sentences of about
661
+ `target` words, cutting only between tags (never inside a citation, note
662
+ or emphasis). Returns a list of HTML strings; joined, they are `h` again."""
663
+ parts = re.split(r"(<[^>]+>)", h)
664
+ cuts, depth, pos = [], 0, 0
665
+ for seg in parts:
666
+ if seg.startswith("<"):
667
+ if not re.match(r"<(br|img)\b", seg):
668
+ depth += -1 if seg.startswith("</") else (0 if seg.endswith("/>") else 1)
669
+ elif depth == 0:
670
+ for m in re.finditer(r"([.?!])([”\"’)]*)(\s+)", seg):
671
+ before = seg[:m.start()]
672
+ word = re.search(r"([\w.]+)$", before)
673
+ w = word.group(1).lower() if word else ""
674
+ if w in ABBR or re.fullmatch(r"[a-z]", w) or re.fullmatch(r"(?:[a-z]\.)+[a-z]", w):
675
+ continue # et al. / e.g. / Fig. / an initial
676
+ nxt = seg[m.end():m.end() + 1] or "<"
677
+ if not (nxt.isupper() or nxt.isdigit() or nxt in "“\"‘'(<["):
678
+ continue
679
+ cuts.append(pos + m.end())
680
+ pos += len(seg)
681
+ if not cuts:
682
+ return [h]
683
+ out, start, acc = [], 0, 0
684
+ total = words(strip_tags(h))
685
+ for c in cuts:
686
+ piece = h[start:c]
687
+ n = words(strip_tags(piece))
688
+ rest = total - acc - n
689
+ if n >= target * 0.6 and rest >= target * 0.5:
690
+ out.append(piece)
691
+ acc += n
692
+ start = c
693
+ out.append(h[start:])
694
+ return out
695
+
696
+
697
+ def chunked(h, target=70):
698
+ cs = chunks(h, target)
699
+ if len(cs) == 1:
700
+ return h
701
+ return "".join('<span class="chunk">%s</span>' % c for c in cs)
702
+
703
+
704
+ def pdf_anchor(doc, b):
705
+ if doc.pdf and b.get("pdf") and pdf_href(doc):
706
+ return '<a href="%s#page=%d" target="_blank" rel="noopener" title="Printed page %s. Opens page %d of the PDF.">p. %s</a>' % (
707
+ attr(pdf_href(doc)), b["pdf"], attr(str(b["label"])), b["pdf"], esc(str(b["label"])))
708
+ return None
709
+
710
+
711
+ def block_text(b):
712
+ t = b["t"]
713
+ if t in ("p", "eq", "h"):
714
+ return b["html"]
715
+ if t == "list":
716
+ return " ".join(b["items"])
717
+ if t == "pre":
718
+ return b["text"]
719
+ if t == "fig":
720
+ return (b.get("caption") or "") + " " + (b.get("table") or "")
721
+ return ""
722
+
723
+
724
+ def main_words(doc):
725
+ n = 0
726
+ for b in doc.blocks:
727
+ if b["t"] == "h" and b.get("appendix"):
728
+ break
729
+ if b["t"] != "h":
730
+ n += words(strip_tags(block_text(b)))
731
+ return n
732
+
733
+
734
+ PANEL = """<div class="panel" id="panel" role="group" aria-label="Reading settings" hidden>
735
+ <div class="row" role="group" aria-label="Theme"><span>Theme</span><span class="seg">
736
+ <button data-pref="theme" data-val="auto">Auto</button><button data-pref="theme" data-val="light">Light</button>
737
+ <button data-pref="theme" data-val="sepia">Sepia</button><button data-pref="theme" data-val="dark">Dark</button></span></div>
738
+ <div class="row" role="group" aria-label="Font"><span>Font</span><span class="seg">
739
+ <button data-pref="font" data-val="literata" style="font-family:Literata">Literata</button>
740
+ <button data-pref="font" data-val="iowan" style="font-family:'Iowan Old Style'">Iowan</button>
741
+ <button data-pref="font" data-val="charter" style="font-family:Charter">Charter</button>
742
+ <button data-pref="font" data-val="sans">Sans</button></span></div>
743
+ <div class="row" role="group" aria-label="Size"><span>Size</span><span class="seg">
744
+ <button id="size-down" aria-label="A− smaller text">A−</button><span id="size-now" aria-live="polite" style="padding:.2rem .3rem">20px</span>
745
+ <button id="size-up" aria-label="A+ larger text">A+</button></span></div>
746
+ <div class="row row-width" role="group" aria-label="Line length"><span>Line length</span><span class="seg">
747
+ <button data-pref="width" data-val="narrow">Narrow</button><button data-pref="width" data-val="medium">Medium</button>
748
+ <button data-pref="width" data-val="wide">Wide</button></span></div>
749
+ <div class="row" role="group" aria-label="Paragraphs"><span>Paragraphs</span><span class="seg">
750
+ <button data-pref="para" data-val="short">Shorter</button><button data-pref="para" data-val="orig">As published</button></span></div>
751
+ <div class="row" role="group" aria-label="Line spacing"><span>Line spacing</span><span class="seg">
752
+ <button data-pref="lh" data-val="tight">1.4</button><button data-pref="lh" data-val="normal">1.5</button>
753
+ <button data-pref="lh" data-val="loose">1.7</button></span></div>
754
+ <div class="note">People read fastest in different fonts, and liking a font doesn't predict it (Wallace et al., 2022). It's worth trying each one for a page.</div>
755
+ </div>"""
756
+
757
+ def all_link(place=None):
758
+ """The way back to the list, with the paper's place in it ("3 of 7") when it has one."""
759
+ if not place:
760
+ return '<a class="btn" href="index.html" aria-label="All papers"><span aria-hidden="true">←</span><span class="lbl"> All papers</span></a>'
761
+ n, total = place
762
+ return ('<a class="btn" href="index.html" aria-label="All papers. This is paper %d of %d">'
763
+ '<span aria-hidden="true">←</span><span class="lbl"> All papers</span>'
764
+ '<span class="place" aria-hidden="true">%d of %d</span></a>') % (n, total, n, total)
765
+
766
+
767
+ def norm_words(s):
768
+ return re.sub(r"[^a-z0-9]", "", unicodedata.normalize("NFKD", s or "").encode("ascii", "ignore").decode().lower())
769
+
770
+
771
+ def same_paper(ref_html, other):
772
+ """Whether a reference-list entry is `other`: by DOI or arXiv ID where the
773
+ entry gives one, otherwise by the other paper's title appearing in it."""
774
+ text = html.unescape(strip_tags(ref_html))
775
+ dois = set(d.rstrip(".,;)]").lower() for d in re.findall(DOI_RE, text))
776
+ if other.get("doi") and other["doi"].lower() in dois:
777
+ return True
778
+ if other.get("arxiv"):
779
+ aid = re.sub(r"v\d+$", "", other["arxiv"])
780
+ if re.search(r"(?<![\d.])%s(?:v\d+)?(?![\d])" % re.escape(aid), text):
781
+ return True
782
+ title = norm_words(other.get("title"))
783
+ return len(title) >= 20 and title[:60] in norm_words(text) # short titles ("Trust") would match by accident
784
+
785
+
786
+ def bundle_links(refs, m):
787
+ """ref id -> a link to another paper's reading copy in the same bundle."""
788
+ out = {}
789
+ for other in m.get("bundle_members") or []:
790
+ if other.get("pid") == m.get("pid"):
791
+ continue
792
+ for i, h in refs:
793
+ if i not in out and same_paper(h, other):
794
+ out[i] = (' <span class="in-bundle">In this bundle (%d of %d): <a href="%s">open its reading copy</a></span>'
795
+ % (other["n"], other["total"], attr(other["href"])))
796
+ return out
797
+
798
+
799
+
800
+ DONE_BOX = '<div class="done-box">Reached the end? <button id="mark-done" type="button">Mark as finished</button> It shows on the list of papers.</div>'
801
+ DONE_SLOT = "<!-- lit-reader: end of the main text -->"
802
+
803
+ PAGE = """<!doctype html>
804
+ <html lang="en">
805
+ <head>
806
+ <meta charset="utf-8">
807
+ <meta name="viewport" content="width=device-width, initial-scale=1">
808
+ <meta name="generator" content="lit-reader">
809
+ <meta http-equiv="Content-Security-Policy" content="default-src 'none'; img-src data: file:; style-src 'self' 'unsafe-inline' file:; font-src data:; script-src 'self' file:">
810
+ <title>{title}</title>
811
+ <link rel="stylesheet" href="assets/fonts.css">
812
+ <link rel="stylesheet" href="assets/reader.css">
813
+ </head>
814
+ <body data-pid="{pid}" data-cite="{cite}" data-reference="{reference}">
815
+ <a class="skip" href="#text-start">Skip to the text</a>
816
+ <div class="bar">
817
+ {all_link}
818
+ <button id="toc-toggle" type="button" aria-expanded="false" aria-controls="toc" aria-label="Contents"><span aria-hidden="true">☰</span><span class="lbl"> Contents</span></button>
819
+ <a class="nojs-toc" href="#toc" aria-label="Contents"><span aria-hidden="true">☰</span><span class="lbl"> Contents</span></a>
820
+ <span class="where" id="where">{where}</span>
821
+ <button id="clips" type="button" aria-expanded="false" aria-controls="clips-panel">Quotes <span id="clip-count"></span></button>
822
+ <button id="aa" type="button" aria-label="Aa, reading settings" aria-expanded="false" aria-controls="panel">Aa</button>
823
+ </div>
824
+ <div class="panel clips-panel" id="clips-panel" role="group" aria-label="Saved quotes" hidden></div>
825
+ <div class="progress" id="progress"></div>
826
+ {panel}
827
+ <div class="layout">
828
+ <nav class="toc" id="toc" aria-label="Contents">{toc}</nav>
829
+ <main id="text-start" tabindex="-1">
830
+ <header class="paper" id="top">
831
+ <div class="kicker">{kicker}</div>
832
+ <h1>{title_html}</h1>
833
+ {part}
834
+ {authors}
835
+ <div class="meta">{meta}</div>
836
+ </header>
837
+ {abstracts}
838
+ {body}
839
+ <div id="end-of-body"></div>
840
+ {done}
841
+ {refs}
842
+ <footer class="source">{source} Built {built}.</footer>
843
+ </main>
844
+ </div>
845
+ <script src="assets/reader.js"></script>
846
+ </body>
847
+ </html>
848
+ """
849
+
850
+
851
+ # --------------------------------------------------------------------------
852
+ # Ledger, cache and list
853
+ # --------------------------------------------------------------------------
854
+
855
+ def ledger_meta(ledger, pid):
856
+ ledger = LEDGERS.get(ledger, ledger)
857
+ try:
858
+ con = sqlite3.connect("file:%s?mode=ro" % Path(ledger).expanduser(), uri=True)
859
+ con.row_factory = sqlite3.Row
860
+ r = con.execute("select * from papers where pid=?", (pid,)).fetchone()
861
+ con.close()
862
+ except sqlite3.Error as e:
863
+ raise ReaderError("can't read the ledger %s: %s" % (ledger, e))
864
+ if not r:
865
+ raise ReaderError("%s is not in %s" % (pid, ledger))
866
+ return {"pid": pid, "title": r["title"], "authors": [a.strip() for a in (r["authors"] or "").split(";") if a.strip()],
867
+ "year": r["year"], "venue": r["venue"], "doi": r["doi"], "arxiv": r["arxiv"], "pmcid": r["pmcid"]}
868
+
869
+
870
+ def stem(pid):
871
+ return re.sub(r"[^A-Za-z0-9._-]", "_", pid)
872
+
873
+
874
+ def cache_file(sub, pid, ext):
875
+ d = LIT_CACHE / sub
876
+ exact = d / (stem(pid) + ext)
877
+ if exact.exists():
878
+ return exact
879
+ hits = sorted(d.glob(stem(pid) + "~*" + ext))
880
+ return hits[0] if hits else None
881
+
882
+
883
+ def source_pdf(meta):
884
+ """The PDF behind a page. Standalone builds never look in the lit.py cache."""
885
+ if meta.get("standalone"):
886
+ return meta.get("pdf_path")
887
+ return cache_file("pdf", meta["pid"], ".pdf")
888
+
889
+
890
+ def source_text(meta):
891
+ """The plain text a build is checked against."""
892
+ if meta.get("standalone"):
893
+ return meta.get("text_path")
894
+ return cache_file("text", meta.get("pid", ""), ".txt")
895
+
896
+
897
+ def slug(meta):
898
+ first = (meta["authors"][0].split()[-1] if meta["authors"] else "paper").lower()
899
+ first = unicodedata.normalize("NFKD", first).encode("ascii", "ignore").decode() or "paper"
900
+ title = re.sub(r"[^a-z0-9]+", "-", unicodedata.normalize("NFKD", meta["title"] or "").encode("ascii", "ignore").decode().lower())
901
+ title = "-".join([w for w in title.split("-") if w][:4])
902
+ return "%s-%s-%s" % (re.sub(r"[^a-z0-9]", "", first), meta.get("year") or "", title)
903
+
904
+
905
+
906
+ # --------------------------------------------------------------------------
907
+ # Fetching (online)
908
+ # --------------------------------------------------------------------------
909
+
910
+ class NamedHostsOnly(urllib.request.HTTPRedirectHandler):
911
+ """A redirect may not leave the hosts the fetch was allowed to reach."""
912
+
913
+ def __init__(self, allowed_hosts):
914
+ self.allowed_hosts = allowed_hosts
915
+
916
+ def redirect_request(self, req, fp, code, msg, headers, newurl):
917
+ p = urllib.parse.urlparse(newurl)
918
+ if p.scheme != "https" or p.netloc not in self.allowed_hosts:
919
+ raise ReaderError("refusing a redirect to %s" % p.netloc)
920
+ return urllib.request.HTTPRedirectHandler.redirect_request(self, req, fp, code, msg, headers, newurl)
921
+
922
+
923
+ def http_get(url, allowed_hosts):
924
+ p = urllib.parse.urlparse(url)
925
+ if p.scheme != "https" or p.netloc not in allowed_hosts:
926
+ raise ReaderError("refusing to fetch from %s" % p.netloc)
927
+ req = urllib.request.Request(url, headers={"User-Agent": UA})
928
+ for attempt in range(3):
929
+ try:
930
+ with urllib.request.build_opener(NamedHostsOnly(allowed_hosts)).open(req, timeout=60) as r:
931
+ return r.read(), r.headers.get("Content-Type", "")
932
+ except urllib.error.HTTPError as e:
933
+ if e.code not in (429, 500, 502, 503, 504) or attempt == 2:
934
+ err = ReaderError("could not fetch %s: %s" % (url, e))
935
+ err.status = e.code
936
+ raise err
937
+ time.sleep(2 * (attempt + 1)) # a busy service often answers on the next try
938
+ except (urllib.error.URLError, OSError) as e:
939
+ raise ReaderError("could not fetch %s: %s" % (url, e))
940
+
941
+
942
+ def image_kind(data):
943
+ if data[:8] == b"\x89PNG\r\n\x1a\n":
944
+ return "image/png"
945
+ if data[:3] == b"\xff\xd8\xff":
946
+ return "image/jpeg"
947
+ if data[:6] in (b"GIF87a", b"GIF89a"):
948
+ return "image/gif"
949
+ return None
950
+
951
+
952
+ def data_uri(data):
953
+ k = image_kind(data)
954
+ if not k:
955
+ return None
956
+ return "data:%s;base64,%s" % (k, base64.b64encode(data).decode())
957
+
958
+
959
+ AR5IV = "ar5iv.labs.arxiv.org"
960
+
961
+
962
+ def fetch_arxiv(aid, d, sources=("arxiv", "ar5iv")):
963
+ """arXiv's own HTML, or ar5iv's conversion when arXiv has none. ar5iv
964
+ sends a paper it lacks back to arxiv.org/abs, so that host is allowed too."""
965
+ d.mkdir(parents=True, exist_ok=True)
966
+ q = urllib.parse.quote(aid, safe="/.")
967
+ body = None
968
+ tries = {"arxiv": ("https://arxiv.org/html/%s" % q, {"arxiv.org"}),
969
+ "ar5iv": ("https://%s/html/%s" % (AR5IV, q), {AR5IV, "arxiv.org"})}
970
+ for url, hosts in (tries[s_] for s_ in sources):
971
+ try:
972
+ got, _ = http_get(url, hosts)
973
+ except ReaderError as e:
974
+ if e.status != 404:
975
+ raise # an outage or a network error is reported, not mistaken for "no HTML version"
976
+ continue
977
+ if b"ltx_document" in got:
978
+ body = got
979
+ break
980
+ if body is None:
981
+ err = ReaderError("%s of %s" % ("neither arXiv nor ar5iv has an HTML version" if len(sources) > 1 else
982
+ "%s has no HTML version" % ("arXiv" if sources[0] == "arxiv" else "ar5iv"), aid))
983
+ err.no_html = True
984
+ raise err
985
+ if AR5IV in url:
986
+ print(" arXiv has no HTML version of %s, so using ar5iv's conversion" % aid)
987
+ (d / "arxiv.html").write_bytes(body)
988
+ (d / "arxiv.url").write_text(url)
989
+ root = parse_html(body.decode("utf-8", "replace"))
990
+ imgs = d / "img"
991
+ imgs.mkdir(exist_ok=True)
992
+ n = 0
993
+ for img in root.findall(lambda x: x.tag == "img"):
994
+ src = img.attrs.get("src", "")
995
+ if not src or src.startswith("data:") or src.startswith("http"):
996
+ continue
997
+ name = safe_id(src)
998
+ target = imgs / name
999
+ if target.exists():
1000
+ continue
1001
+ try:
1002
+ data, _ = http_get(urllib.parse.urljoin(url, src), {urllib.parse.urlparse(url).netloc})
1003
+ except (ReaderError, ValueError) as e:
1004
+ print(" image failed: %s (%s)" % (src, e))
1005
+ continue
1006
+ if image_kind(data):
1007
+ target.write_bytes(data)
1008
+ n += 1
1009
+ print("fetched arXiv HTML for %s and %d images into %s" % (aid, n, d))
1010
+
1011
+
1012
+ def fetch_jats(pmcid, d):
1013
+ d.mkdir(parents=True, exist_ok=True)
1014
+ body, _ = http_get("https://www.ebi.ac.uk/europepmc/webservices/rest/%s/fullTextXML" % pmcid, {"www.ebi.ac.uk"})
1015
+ (d / "jats.xml").write_bytes(body)
1016
+ imgs = d / "img"
1017
+ imgs.mkdir(exist_ok=True)
1018
+ n = 0
1019
+ try:
1020
+ z, _ = http_get("https://www.ebi.ac.uk/europepmc/webservices/rest/%s/supplementaryFiles?includeInlineImage=yes" % pmcid, {"www.ebi.ac.uk"})
1021
+ with zipfile.ZipFile(io.BytesIO(z)) as zf:
1022
+ for info in zf.infolist():
1023
+ name = os.path.basename(info.filename)
1024
+ if info.file_size > 30 * 1024 * 1024 or not re.match(r"^[\w.-]+\.(jpe?g|png|gif)$", name, re.I):
1025
+ continue
1026
+ data = zf.read(info)
1027
+ if image_kind(data):
1028
+ (imgs / name).write_bytes(data)
1029
+ n += 1
1030
+ except (ReaderError, zipfile.BadZipFile) as e:
1031
+ print(" no figure bundle for %s (%s)" % (pmcid, e))
1032
+ print("fetched JATS XML for %s and %d images into %s" % (pmcid, n, d))
1033
+
1034
+
1035
+ def cmd_fetch(a):
1036
+ meta = ledger_meta(a.ledger, a.pid)
1037
+ d = SRC / slug(meta)
1038
+ if a.route == "arxiv":
1039
+ fetch_arxiv(meta["arxiv"] or a.pid.split(":", 1)[-1], d)
1040
+ elif a.route == "jats":
1041
+ pmcid = meta["pmcid"] or a.pmcid
1042
+ if not pmcid:
1043
+ body, _ = http_get("https://www.ebi.ac.uk/europepmc/webservices/rest/search?format=json&resultType=lite&query=DOI:%s"
1044
+ % urllib.parse.quote(meta["doi"] or "", safe="/"), {"www.ebi.ac.uk"})
1045
+ res = json.loads(body.decode("utf-8"))["resultList"]["result"]
1046
+ pmcid = next((r.get("pmcid") for r in res if r.get("pmcid")), None)
1047
+ if not pmcid or not re.match(r"^PMC\d+$", pmcid):
1048
+ raise ReaderError("no PMC id for %s" % a.pid)
1049
+ fetch_jats(pmcid, d)
1050
+
1051
+
1052
+ # --------------------------------------------------------------------------
1053
+ # Route: arXiv HTML
1054
+ # --------------------------------------------------------------------------
1055
+
1056
+ def visible_text(n):
1057
+ """A node's text without the TeX and MathML annotations hidden inside formulas."""
1058
+ if isinstance(n, str):
1059
+ return n
1060
+ return "" if n.tag in ("annotation", "annotation-xml") else "".join(visible_text(k) for k in n.kids)
1061
+
1062
+
1063
+ ARXIV_BLOCKS = {"h1", "h2", "h3", "h4", "h5", "h6", "p", "div", "section", "figure", "table", "ul", "ol", "dl"}
1064
+
1065
+
1066
+ def span_table(ix, n):
1067
+ """LaTeXML sometimes builds a table from spans, for example when it has been resized."""
1068
+ def owner(tr):
1069
+ p = tr.parent
1070
+ while p is not None and "ltx_tabular" not in p.cls():
1071
+ p = p.parent
1072
+ return p
1073
+ rows = []
1074
+ for tr in n.findall(lambda x: "ltx_tr" in x.cls()):
1075
+ if owner(tr) is not n:
1076
+ continue
1077
+ cells = []
1078
+ for td in tr.children():
1079
+ if td.cls() & {"ltx_td", "ltx_th"}:
1080
+ tag = "th" if "ltx_th" in td.cls() else "td"
1081
+ spans = "".join(' %s="%s"' % (a, td.attrs[a]) for a in ("colspan", "rowspan") if re.fullmatch(r"\d{1,2}", td.attrs.get(a, "")))
1082
+ cells.append("<%s%s>%s</%s>" % (tag, spans, ix.run(td).strip(), tag))
1083
+ rows.append("<tr>%s</tr>" % "".join(cells))
1084
+ return '<table class="data">%s</table>' % "".join(rows)
1085
+
1086
+
1087
+ class ArxivInline(Inline):
1088
+ def rule(self, n):
1089
+ c = n.cls()
1090
+ if n.tag == "span" and "ltx_tabular" in c:
1091
+ return span_table(self, n)
1092
+ if n.tag == "span" and "ltx_note" in c:
1093
+ if c & {"ltx_role_footnote", "ltx_role_footnotetext"}:
1094
+ content = n.find(lambda x: "ltx_note_content" in x.cls())
1095
+ if content is None:
1096
+ return ""
1097
+ for junk in content.findall(lambda x: x.cls() & {"ltx_note_mark", "ltx_note_type", "ltx_tag_note"}):
1098
+ junk.kids = []
1099
+ return self.doc.note(self.run(content).strip())
1100
+ return ""
1101
+ if n.tag == "a" and "ltx_ref" in c:
1102
+ href = n.attrs.get("href", "")
1103
+ if href.startswith("#bib."):
1104
+ return self.doc.cite([href[1:].replace("bib.", "", 1)], self.run(n))
1105
+ if href.startswith("#"):
1106
+ return '<a href="#x-%s">%s</a>' % (safe_id(href[1:]), self.run(n))
1107
+ if n.tag == "span" and "ltx_text" in c:
1108
+ inner = self.run(n)
1109
+ if "ltx_font_italic" in c:
1110
+ return "<em>%s</em>" % inner
1111
+ if "ltx_font_bold" in c:
1112
+ return "<strong>%s</strong>" % inner
1113
+ if "ltx_font_typewriter" in c:
1114
+ return "<code>%s</code>" % inner
1115
+ return inner
1116
+ if c & {"ltx_acm_description", "ltx_bib_cited", "ltx_tag_bibitem"}:
1117
+ return ""
1118
+ return None
1119
+
1120
+
1121
+ def arxiv_doc(meta, d):
1122
+ path = d / "arxiv.html"
1123
+ if not path.exists():
1124
+ raise ReaderError("no arXiv HTML; run fetch --route arxiv first")
1125
+ root = parse_html(path.read_text(encoding="utf-8"))
1126
+ art = root.find(lambda x: x.tag == "article" and "ltx_document" in x.cls()) or root
1127
+ doc = Doc(dict(meta))
1128
+ url = (d / "arxiv.url").read_text().strip() if (d / "arxiv.url").exists() else ""
1129
+ doc.meta["url"] = url
1130
+ doc.meta["kicker"] = "arXiv " + (meta.get("arxiv") or "")
1131
+ if AR5IV in url:
1132
+ doc.source = "Text from ar5iv (%s), arXiv Labs' HTML conversion, because arXiv has no HTML version of this paper." % url
1133
+ else:
1134
+ doc.source = "Text from arXiv's HTML version (%s), which arXiv labels experimental." % url
1135
+ ix = ArxivInline(doc)
1136
+ imgdir = d / "img"
1137
+ doc.pdf = source_pdf(meta)
1138
+ doc._crops = docling_crops(d, doc.pdf)
1139
+ t = art.find(lambda x: "ltx_title_document" in x.cls())
1140
+ if t is not None:
1141
+ for n in t.findall(lambda x: "ltx_note" in x.cls()):
1142
+ n.kids = [] # a note on the title has no place in the heading or the browser tab
1143
+ doc.meta["title_html"] = ArxivInline(Doc({})).run(t).strip()
1144
+
1145
+ ab = art.find(lambda x: "ltx_abstract" in x.cls())
1146
+ if ab is not None:
1147
+ paras = [ix.run(p).strip() for p in ab.findall(lambda x: x.tag == "p")]
1148
+ doc.abstracts.append(("Abstract", "".join("<p>%s</p>" % p for p in paras if p)))
1149
+
1150
+ def heading(n, level, appendix):
1151
+ tag = n.find(lambda x: "ltx_tag" in x.cls())
1152
+ no = tag.text().strip() if tag is not None else ""
1153
+ if tag is not None:
1154
+ tag.kids = []
1155
+ txt = ix.run(n).strip()
1156
+ if no and not no.endswith("."):
1157
+ no = no # "Appendix A" style tags keep their own form
1158
+ return dict(t="h", level=level, html=txt, no=no, appendix=appendix)
1159
+
1160
+ def walk(n, depth, appendix):
1161
+ for k in n.children():
1162
+ c = k.cls()
1163
+ if k.tag in ("nav", "header", "footer") or c & {"ltx_page_navbar", "ltx_authors", "ltx_abstract", "ltx_dates",
1164
+ "ltx_keywords", "ltx_pubnotes", "ltx_classification", "ltx_title_document",
1165
+ "ltx_acknowledgements_x"}:
1166
+ continue
1167
+ if "ltx_bibliography" in c:
1168
+ for li in k.findall(lambda x: x.tag == "li" and "ltx_bibitem" in x.cls()):
1169
+ rid = li.attrs.get("id", "").replace("bib.", "", 1)
1170
+ blocks = [ix.run(b).strip() for b in li.findall(lambda x: "ltx_bibblock" in x.cls() and "ltx_bib_cited" not in x.cls())]
1171
+ doc.refs.append((rid, " ".join(x for x in blocks if x)))
1172
+ continue
1173
+ if k.tag == "section":
1174
+ app = appendix or "ltx_appendix" in c
1175
+ h = k.find(lambda x: x.tag in ("h2", "h3", "h4", "h5", "h6") and x.parent is k)
1176
+ if h is not None:
1177
+ level = {"h2": 2, "h3": 3, "h4": 4}.get(h.tag, 4)
1178
+ blk = heading(h, level, app and not appendix)
1179
+ blk["id"] = "x-" + safe_id(k.attrs.get("id", "")) if k.attrs.get("id") else None
1180
+ if app and not appendix:
1181
+ blk["appendix"] = True
1182
+ doc.blocks.append(blk)
1183
+ walk(k, depth + 1, app)
1184
+ continue
1185
+ if k.tag in ("h2", "h3", "h4", "h5", "h6"):
1186
+ if k.parent is not None and k.parent.tag == "section":
1187
+ continue # handled with its section
1188
+ doc.blocks.append(heading(k, 4, False))
1189
+ continue
1190
+ if k.tag == "p":
1191
+ h = ix.run(k).strip()
1192
+ if h:
1193
+ doc.add(t="p", html=h)
1194
+ for img in k.findall(lambda x: x.tag == "img"): # a picture placed without a figure around it
1195
+ f = imgdir / safe_id(img.attrs.get("src", ""))
1196
+ u = data_uri(f.read_bytes()) if f.exists() else None
1197
+ if u:
1198
+ doc.add(t="fig", kind="figure", label="", caption="", table="", imgs=[(u, img.attrs.get("alt") or "")])
1199
+ continue
1200
+ if k.tag == "figure" and k.parent is not None and k.parent.tag == "figure":
1201
+ continue
1202
+ if k.tag == "figure":
1203
+ arxiv_figure(doc, ix, k, imgdir)
1204
+ continue
1205
+ if k.tag in ("ul", "ol"):
1206
+ items = []
1207
+ marks = [tg.text().strip() for tg in k.findall(lambda x: "ltx_tag_item" in x.cls())]
1208
+ custom = any(not re.fullmatch(r"\d+[.)]?|[•∙·*–—-]", m) for m in marks) # "(iv)" must survive: the text may cite it
1209
+ for li in k.children():
1210
+ if li.tag != "li":
1211
+ continue
1212
+ mark = ""
1213
+ for tg in li.findall(lambda x: "ltx_tag_item" in x.cls()):
1214
+ mark = mark or tg.text().strip()
1215
+ tg.kids = []
1216
+ items.append((esc(mark) + " " if custom and mark else "") + ix.run(li).strip())
1217
+ doc.add(t="list", ordered=k.tag == "ol" and not custom, plain=custom, items=items)
1218
+ continue
1219
+ if k.tag == "table" and c & {"ltx_equation", "ltx_equationgroup"}:
1220
+ rows = [r for r in k.findall(lambda x: x.tag == "tr") if r.find(lambda x: x.tag == "math") is not None] or [k]
1221
+ parts = []
1222
+ for r in rows:
1223
+ tag = r.find(lambda x: "ltx_tag_equation" in x.cls())
1224
+ parts.append(" ".join(math_html(mm) for mm in r.findall(lambda x: x.tag == "math")) + ("&emsp;" + esc(tag.text().strip()) if tag is not None else ""))
1225
+ targets = [x.attrs["id"] for x in [k] + k.findall(lambda x: re.search(r"\.E\d+$", x.attrs.get("id", "")) is not None) if x.attrs.get("id")]
1226
+ doc.add(t="eq", html="".join('<span id="x-%s"></span>' % safe_id(i) for i in dict.fromkeys(targets)) + "<br>".join(parts))
1227
+ continue
1228
+ if k.tag == "table" and "ltx_tabular" in c:
1229
+ doc.add(t="fig", kind="table", label="", caption="", imgs=[], table=ix.one(k))
1230
+ continue
1231
+ if "ltx_listing" in c:
1232
+ lines = [visible_text(ln).rstrip() for ln in k.findall(lambda x: "ltx_listingline" in x.cls())]
1233
+ doc.add(t="pre", text="\n".join(lines))
1234
+ continue
1235
+ if k.tag == "dl":
1236
+ terms = [c for c in k.children() if c.tag in ("dt", "dd")]
1237
+ items = ["<strong>%s</strong> %s" % (ix.run(a).strip(), ix.run(b).strip()) for a, b in zip(terms[::2], terms[1::2])]
1238
+ doc.add(t="list", ordered=False, plain=True, items=items)
1239
+ continue
1240
+ if "ltx_note" in c:
1241
+ continue # notes are placed where they are cited
1242
+ if k.tag in ("div", "section") and any( # text sitting directly in a container
1243
+ (isinstance(x, str) and x.strip()) or
1244
+ (isinstance(x, Node) and x.tag in ("span", "a", "em", "strong", "i", "b") and "ltx_note" not in x.cls() and x.text().strip())
1245
+ for x in k.kids):
1246
+ run = Node("p", parent=k)
1247
+ for x in k.kids + [None]:
1248
+ if x is None or (isinstance(x, Node) and x.tag in ARXIV_BLOCKS):
1249
+ h = ix.run(run).strip()
1250
+ if h:
1251
+ doc.add(t="p", html=h)
1252
+ run.kids = []
1253
+ if x is not None:
1254
+ one = Node(k.tag, parent=k.parent)
1255
+ one.kids = [x]
1256
+ walk(one, depth, appendix)
1257
+ else:
1258
+ run.kids.append(x)
1259
+ continue
1260
+ walk(k, depth, appendix)
1261
+
1262
+ walk(art, 0, False)
1263
+ return doc
1264
+
1265
+
1266
+ def arxiv_figure(doc, ix, k, imgdir):
1267
+ def owner(c):
1268
+ p = c.parent
1269
+ while p is not None and p.tag != "figure":
1270
+ p = p.parent
1271
+ return p
1272
+ own = [c for c in k.findall(lambda x: x.tag == "figcaption") if owner(c) is k]
1273
+ panels = [c for c in k.findall(lambda x: x.tag == "figcaption") if owner(c) is not k]
1274
+ cap = own[-1] if own else (panels[0] if panels else None) # with panels, the figure's own caption comes last
1275
+ panels = [c for c in panels if c is not cap]
1276
+ label, caption, alts = "", "", []
1277
+ if cap is not None:
1278
+ for desc in cap.findall(lambda x: "ltx_acm_description" in x.cls()):
1279
+ alts.append(desc.text().strip())
1280
+ tg = cap.find(lambda x: "ltx_tag" in x.cls())
1281
+ if tg is not None:
1282
+ label = tg.text().strip()
1283
+ tg.kids = []
1284
+ caption = " ".join(x for x in [ix.run(cap).strip()] + [ix.run(c).strip() for c in panels] if x)
1285
+ imgs = []
1286
+ for img in k.findall(lambda x: x.tag == "img"):
1287
+ f = imgdir / safe_id(img.attrs.get("src", ""))
1288
+ if f.exists():
1289
+ u = data_uri(f.read_bytes())
1290
+ if u:
1291
+ alt = img.attrs.get("alt") or (alts[0] if alts else "")
1292
+ imgs.append((u, label.rstrip(":. ") + ", shown as an image" if label and alt.strip().lower() == "refer to caption" else alt))
1293
+ table = ""
1294
+ def outer_table(t):
1295
+ p = t.parent
1296
+ while p is not None and p is not k:
1297
+ if p.tag == "table":
1298
+ return False
1299
+ p = p.parent
1300
+ return True
1301
+ table = "".join(ix.one(t) for t in k.findall(lambda x: x.tag == "table") # a table holding only pictures is layout
1302
+ if outer_table(t) and t.find(lambda x: x.tag == "img") is None and t.text().strip())
1303
+
1304
+ def outer_span_table(t):
1305
+ p = t.parent
1306
+ while p is not None and p is not k:
1307
+ if p.tag == "table" or "ltx_tabular" in p.cls():
1308
+ return False
1309
+ p = p.parent
1310
+ return t.find(lambda x: x.tag == "img") is None and bool(t.text().strip())
1311
+ table += "".join(span_table(ix, t) for t in k.findall(lambda x: x.tag == "span" and "ltx_tabular" in x.cls()) if outer_span_table(t))
1312
+ kind = "table" if "ltx_table" in k.cls() else "figure"
1313
+ fid = "x-" + safe_id(k.attrs.get("id", "")) if k.attrs.get("id") else None
1314
+ page = find_pdf_page(doc, label)
1315
+ if not imgs and not table:
1316
+ # arXiv's converter lost this figure: use Docling's crop of it from the PDF
1317
+ crop = doc._crops.get(float_key(label)) if hasattr(doc, "_crops") else None
1318
+ if crop:
1319
+ imgs, page = [(crop[0], alts[0] if alts else "")], crop[1]
1320
+ if not imgs and not table: # an algorithm or other float made of text
1321
+ lines = [visible_text(ln).rstrip() for ln in k.findall(lambda x: "ltx_listingline" in x.cls())]
1322
+
1323
+ def spoken_only(p): # captions and the accessibility descriptions of a lost image
1324
+ while p is not None and p is not k:
1325
+ if p.tag == "figcaption" or "ltx_acm_description" in p.cls():
1326
+ return True
1327
+ p = p.parent
1328
+ return False
1329
+ body = [ix.run(p).strip() for p in k.findall(lambda x: x.tag == "p") if not spoken_only(p)]
1330
+ if lines:
1331
+ table = '<pre class="listing">%s</pre>' % esc("\n".join(lines))
1332
+ elif not float_key(label): # a lost figure or table keeps its "did not come through" notice
1333
+ table = "".join("<p>%s</p>" % b for b in body if b)
1334
+ if not label and not caption and not imgs and not table:
1335
+ return
1336
+ doc.add(t="fig", kind=kind, id=fid, label=label, caption=caption, imgs=imgs, table=table, page=page)
1337
+
1338
+
1339
+ def float_key(label):
1340
+ m = re.match(r"^\s*(Fig(?:ure)?|Table|TABLE|FIGURE)\.?\s*([\dA-Z]+)", label or "", re.I)
1341
+ if not m:
1342
+ return None
1343
+ return ("table" if m.group(1).lower().startswith("tab") else "figure", m.group(2))
1344
+
1345
+
1346
+ def docling_crops(d, pdf):
1347
+ """{("figure", "3"): (data uri, pdf page)} for every captioned picture or
1348
+ table in this paper's Docling output, cropped from the PDF page."""
1349
+ out = {}
1350
+ js = sorted((d / "docling").glob("*.json"))
1351
+ if not js or not pdf:
1352
+ return out
1353
+ j = json.loads(js[0].read_text(encoding="utf-8"))
1354
+ page_h = {int(k): v["size"]["height"] for k, v in j.get("pages", {}).items()}
1355
+ for kind in ("pictures", "tables"):
1356
+ for item in j.get(kind, []):
1357
+ caps = " ".join(j["texts"][int(c["$ref"].split("/")[2])].get("text", "") for c in item.get("captions", []))
1358
+ key = float_key(caps)
1359
+ prov = (item.get("prov") or [{}])[0]
1360
+ if not key or key in out or not prov.get("page_no"):
1361
+ continue
1362
+ png = crop_png(pdf, prov["page_no"], docling_box(prov["bbox"], page_h.get(prov["page_no"], 792)), page_h.get(prov["page_no"], 792))
1363
+ u = data_uri(png) if png else None
1364
+ if u:
1365
+ out[key] = (u, prov["page_no"])
1366
+ return out
1367
+
1368
+
1369
+ def docling_box(bb, page_h):
1370
+ """Docling bbox -> (l, t, r, b) in PDF points, origin bottom left."""
1371
+ if bb.get("coord_origin", "BOTTOMLEFT") == "BOTTOMLEFT":
1372
+ return (bb["l"], bb["t"], bb["r"], bb["b"])
1373
+ return (bb["l"], page_h - bb["t"], bb["r"], page_h - bb["b"])
1374
+
1375
+
1376
+ def find_pdf_page(doc, label):
1377
+ """The PDF page whose text starts a line with this figure label."""
1378
+ if not label:
1379
+ return None
1380
+ txt = source_text(doc.meta)
1381
+ if not txt:
1382
+ return None
1383
+ pages = txt.read_text(encoding="utf-8", errors="replace").split("\f")
1384
+ pat = re.compile(r"^\s*" + re.escape(label.rstrip(".:")) + r"[.:]", re.M)
1385
+ for i, p in enumerate(pages):
1386
+ if pat.search(p):
1387
+ return i + 1
1388
+ return None
1389
+
1390
+
1391
+ # --------------------------------------------------------------------------
1392
+ # Route: JATS (Europe PMC)
1393
+ # --------------------------------------------------------------------------
1394
+
1395
+ def parse_xml(path):
1396
+ try:
1397
+ return ET.parse(str(path)).getroot()
1398
+ except ET.ParseError as e:
1399
+ raise ReaderError("the article's XML can't be read: %s" % e)
1400
+
1401
+
1402
+ def jats_image(doc, href):
1403
+ base = os.path.basename(href or "")
1404
+ f = next((doc._imgdir / (base + e) for e in ("", ".jpg", ".jpeg", ".png", ".gif") if base and (doc._imgdir / (base + e)).is_file()), None)
1405
+ return data_uri(f.read_bytes()) if f is not None else None
1406
+
1407
+
1408
+ def formula_html(n, doc):
1409
+ """MathML if the article has it, then its picture of the equation, then its TeX source."""
1410
+ m = n.find(lambda x: x.tag == "math")
1411
+ if m is not None:
1412
+ return math_html(m)
1413
+ g = n.find(lambda x: x.tag in ("graphic", "inline-graphic"))
1414
+ u = jats_image(doc, g.attrs.get("href")) if g is not None else None
1415
+ if u:
1416
+ return img_tag(u, "Equation")
1417
+ tex = n.find(lambda x: x.tag == "tex-math")
1418
+ return "<code>%s</code>" % esc(tex.text().strip()) if tex is not None else ""
1419
+
1420
+
1421
+ class JatsInline(Inline):
1422
+ def rule(self, n):
1423
+ if n.tag in ("name", "string-name") and n.find(lambda x: x.tag == "surname") is not None:
1424
+ sur = n.find(lambda x: x.tag == "surname").text().strip()
1425
+ giv = n.find(lambda x: x.tag == "given-names")
1426
+ return esc(sur + (" " + giv.text().strip() if giv is not None else ""))
1427
+ if n.tag == "xref":
1428
+ kind = n.attrs.get("ref-type")
1429
+ rid = n.attrs.get("rid", "")
1430
+ if kind == "bibr":
1431
+ ids = rid.split()
1432
+ r = re.fullmatch(r"\s*(\d{1,3})\s*[–—-]\s*(\d{1,3})\s*", n.text())
1433
+ m = re.fullmatch(r"(.*?)0*%s" % r.group(1), ids[0]) if r and len(ids) == 1 else None
1434
+ if m and int(r.group(1)) < int(r.group(2)) <= int(r.group(1)) + 50: # "1–3" with only the first reference tagged
1435
+ width = len(ids[0]) - len(m.group(1))
1436
+ wanted = [m.group(1) + str(k).zfill(width) for k in range(int(r.group(1)), int(r.group(2)) + 1)]
1437
+ ids = [x for x in wanted if x in self.doc._jats_refs] or ids
1438
+ return self.doc.cite(ids, self.run(n))
1439
+ if kind in ("fig", "table", "sec", "disp-formula", "supplementary-material"):
1440
+ return '<a href="#j-%s">%s</a>' % (safe_id(rid.split()[0]) if rid else "", self.run(n))
1441
+ if kind == "fn":
1442
+ fn = self.doc._jats_fns.get(rid)
1443
+ if fn is not None:
1444
+ self.doc._jats_cited.add(rid)
1445
+ return self.doc.note(self.run(fn).strip())
1446
+ return self.run(n)
1447
+ if n.tag in ("inline-formula", "disp-formula", "alternatives"):
1448
+ return formula_html(n, self.doc) or self.run(n)
1449
+ if n.tag in ("label",) and n.parent is not None and n.parent.tag in ("fig", "table-wrap", "ref"):
1450
+ return ""
1451
+ if n.tag == "label" and n.parent is not None and n.parent.tag == "fn": # a table's notes need their marks
1452
+ in_table = n.parent.parent is not None and n.parent.parent.tag == "table-wrap-foot"
1453
+ return self.run(n).strip() + " " if in_table else ""
1454
+ if n.tag == "label" and n.parent is not None and n.parent.tag == "list-item":
1455
+ return self.run(n).strip() + " "
1456
+ if n.tag == "title" and n.parent is not None and n.parent.tag == "caption":
1457
+ return self.run(n).strip() + " " # the shared whitelist drops <title>, but here it is the caption's first line
1458
+ if n.tag == "ext-link":
1459
+ href = safe_href(n.attrs.get("href", ""))
1460
+ inner = self.run(n)
1461
+ return '<a href="%s" rel="noopener noreferrer">%s</a>' % (attr(href), inner) if href else inner
1462
+ return None
1463
+
1464
+
1465
+ def structured_ref(ix, cit):
1466
+ """An element-citation has fields but no punctuation between them."""
1467
+ names = [ix.one(n).strip() for n in cit.findall(lambda x: x.tag in ("name", "string-name", "collab"))]
1468
+ fields = [ix.run(c).strip() for c in cit.children() if c.tag not in ("person-group", "name", "string-name", "collab")]
1469
+ return ". ".join(x for x in [", ".join(names)] + fields if x) + "."
1470
+
1471
+
1472
+ def jats_doc(meta, d):
1473
+ path = d / "jats.xml"
1474
+ if not path.exists():
1475
+ raise ReaderError("no JATS XML; run fetch --route jats first")
1476
+ root = et_to_node(parse_xml(path))
1477
+ doc = Doc(dict(meta))
1478
+ doc._jats_fns = {n.attrs.get("id"): n for n in root.findall(lambda x: x.tag == "fn") if n.attrs.get("id")}
1479
+ doc._jats_refs = {n.attrs.get("id") for n in root.findall(lambda x: x.tag == "ref")}
1480
+ doc._jats_cited = set()
1481
+ ix = JatsInline(doc)
1482
+ imgdir = d / "img"
1483
+ doc._imgdir = imgdir
1484
+ doc.pdf = source_pdf(meta)
1485
+ doc.meta["kicker"] = meta.get("venue") or "Article"
1486
+ doc.source = "Text from Europe PMC's full-text XML for this article; figures from Europe PMC's file bundle."
1487
+ art_meta = root.find(lambda x: x.tag == "article-meta")
1488
+ if art_meta is not None:
1489
+ t = art_meta.find(lambda x: x.tag == "article-title")
1490
+ if t is not None:
1491
+ for x in t.findall(lambda x: x.tag == "xref" and x.attrs.get("ref-type") == "fn"):
1492
+ x.kids, x.attrs["ref-type"] = [], "none" # a note on the title has no place in the heading or the browser tab
1493
+ doc.meta["title_html"] = ix.run(t).strip()
1494
+ all_abstracts = art_meta.findall(lambda x: x.tag == "abstract")
1495
+ full = [a for a in all_abstracts if a.attrs.get("abstract-type") not in ("toc", "teaser", "precis", "web-summary", "graphical")]
1496
+ for ab in full or all_abstracts:
1497
+ title = ab.find(lambda x: x.tag == "title" and x.parent is ab)
1498
+ head = title.text().strip() if title is not None else "Abstract"
1499
+ paras = []
1500
+ for p in ab.findall(lambda x: x.tag == "p"):
1501
+ sec = p.parent if p.parent is not None and p.parent.tag == "sec" else None
1502
+ label = sec.find(lambda x: x.tag == "title" and x.parent is sec) if sec is not None and sec.find(lambda x: x.tag == "p") is p else None
1503
+ paras.append(("<strong>%s</strong> " % ix.run(label).strip().rstrip(":.") if label is not None else "") + ix.run(p).strip())
1504
+ doc.abstracts.append((head, "".join("<p>%s</p>" % p for p in paras if p)))
1505
+
1506
+ def walk(n, depth, appendix=False):
1507
+ for k in n.children():
1508
+ t = k.tag
1509
+ if t == "sec" and k.find(lambda x: x.tag == "ref-list") is not None and k.find(lambda x: x.tag == "p") is None:
1510
+ continue # a section holding only the reference list, which has its own place on the page
1511
+ if t == "sec":
1512
+ title = k.find(lambda x: x.tag == "title" and x.parent is k)
1513
+ if title is not None:
1514
+ number = k.find(lambda x: x.tag == "label" and x.parent is k)
1515
+ doc.add(t="h", level=min(2 + depth, 4), html=ix.run(title).strip(), id="j-" + safe_id(k.attrs.get("id", "")) if k.attrs.get("id") else None,
1516
+ no=number.text().strip().rstrip(".") if number is not None else "", appendix=appendix)
1517
+ walk(k, depth + 1)
1518
+ elif t == "p":
1519
+ # a <p> can hold a figure or table; pull those out after the text
1520
+ # equations and lists stay where they fall in the sentence; figures and tables follow the paragraph
1521
+ floats, cur = [c for c in k.children() if c.tag in ("fig", "table-wrap")], Node("p", parent=k.parent)
1522
+
1523
+ def emit():
1524
+ h = ix.run(cur).strip()
1525
+ if h:
1526
+ doc.add(t="p", html=h)
1527
+ cur.kids = []
1528
+ for kid in k.kids:
1529
+ if isinstance(kid, Node) and kid.tag in ("disp-formula", "list"):
1530
+ emit()
1531
+ walk_one(kid)
1532
+ elif not (isinstance(kid, Node) and kid.tag in ("fig", "table-wrap")):
1533
+ cur.kids.append(kid)
1534
+ emit()
1535
+ for f in floats:
1536
+ walk_one(f)
1537
+ elif t in ("fig", "table-wrap", "disp-formula", "list"):
1538
+ walk_one(k)
1539
+ elif t in ("supplementary-material",):
1540
+ cap = k.find(lambda x: x.tag == "caption")
1541
+ if cap is not None:
1542
+ doc.add(t="p", html="<em>Supplementary material (not held locally):</em> " + ix.run(cap).strip())
1543
+ elif t == "def-list":
1544
+ items = []
1545
+ for di in k.findall(lambda x: x.tag == "def-item"):
1546
+ term, meaning = di.find(lambda x: x.tag == "term"), di.find(lambda x: x.tag == "def")
1547
+ items.append(" ".join(x for x in ["<strong>%s</strong>" % ix.run(term).strip() if term is not None else "",
1548
+ ix.run(meaning).strip() if meaning is not None else ""] if x))
1549
+ doc.add(t="list", ordered=False, plain=True, items=items)
1550
+ elif t in ("title", "label", "ref-list"):
1551
+ continue
1552
+ else:
1553
+ walk(k, depth, appendix)
1554
+
1555
+ def walk_one(k):
1556
+ t = k.tag
1557
+ if t == "fig" or t == "table-wrap":
1558
+ label = k.find(lambda x: x.tag == "label" and x.parent is k)
1559
+ cap = k.find(lambda x: x.tag == "caption")
1560
+ imgs = []
1561
+ for g in k.findall(lambda x: x.tag == "graphic"):
1562
+ href = g.attrs.get("href", "")
1563
+ base = os.path.basename(href)
1564
+ f = next((imgdir / (base + e) for e in ("", ".jpg", ".jpeg", ".png", ".gif") if base and (imgdir / (base + e)).is_file()), None)
1565
+ if f is not None:
1566
+ u = data_uri(f.read_bytes())
1567
+ if u:
1568
+ imgs.append((u, "%s, shown as an image" % (label.text().strip().rstrip(".:") if label is not None else ("Table" if t == "table-wrap" else "Figure"))))
1569
+ if len(imgs) > 1: # publishers often add a small preview of the same figure
1570
+ biggest = max(len(u) for u, _ in imgs)
1571
+ imgs = [x for x in imgs if len(x[0]) >= biggest * 0.2]
1572
+ table = ""
1573
+ tb = k.find(lambda x: x.tag == "table")
1574
+ if tb is not None:
1575
+ table = ix.one(tb)
1576
+ if table:
1577
+ imgs = [] # the same table supplied both as data and as a picture
1578
+ caption = ix.run(cap).strip() if cap is not None else ""
1579
+ foot = k.find(lambda x: x.tag == "table-wrap-foot")
1580
+ foot_text = " ".join(x for x in (ix.run(c).strip() for c in foot.children()) if x) if foot is not None else ""
1581
+ if foot_text:
1582
+ caption += ' <small class="tnote">%s</small>' % foot_text
1583
+ doc.add(t="fig", kind="table" if t == "table-wrap" else "figure",
1584
+ id="j-" + safe_id(k.attrs.get("id", "")) if k.attrs.get("id") else None,
1585
+ label=label.text().strip() if label is not None else "",
1586
+ caption=caption, imgs=imgs, table=table)
1587
+ elif t == "disp-formula":
1588
+ lab = k.find(lambda x: x.tag == "label" and x.parent is k)
1589
+ html_ = formula_html(k, doc)
1590
+ target = '<span id="j-%s"></span>' % safe_id(k.attrs["id"]) if k.attrs.get("id") else ""
1591
+ doc.add(t="eq", html=target + (html_ + ("&emsp;" + esc(lab.text().strip()) if lab is not None else "") if html_ else ix.run(k)))
1592
+ elif t == "list":
1593
+ items = [ix.run(li).strip() for li in k.children() if li.tag == "list-item"]
1594
+ labelled = any(c.tag == "label" for li in k.children() if li.tag == "list-item" for c in li.children())
1595
+ doc.add(t="list", ordered=not labelled and k.attrs.get("list-type") in ("order", "arabic", "alpha-lower", "roman-lower"),
1596
+ plain=labelled, items=items)
1597
+
1598
+ body = root.find(lambda x: x.tag == "body")
1599
+ if body is not None:
1600
+ if body.children() and body.children()[0].tag == "p":
1601
+ doc.add(t="h", level=2, html="Introduction", id="j-intro")
1602
+ walk(body, 0)
1603
+ back = root.find(lambda x: x.tag == "back")
1604
+ if back is not None:
1605
+ ack = back.find(lambda x: x.tag == "ack")
1606
+ if ack is not None:
1607
+ doc.add(t="h", level=2, html="Acknowledgments", appendix=True)
1608
+ for p in ack.findall(lambda x: x.tag == "p"):
1609
+ doc.add(t="p", html=ix.run(p).strip())
1610
+ for k in back.children(): # appendices and sections such as data availability
1611
+ for sec in ([k] if k.tag in ("sec", "notes") else k.findall(lambda x: x.tag == "app") if k.tag == "app-group" else [k] if k.tag == "app" else []):
1612
+ title = sec.find(lambda x: x.tag == "title" and x.parent is sec)
1613
+ doc.add(t="h", level=2, html=ix.run(title).strip() if title is not None else ("Notes" if sec.tag == "notes" else "Appendix"),
1614
+ appendix=not any(b.get("appendix") for b in doc.blocks))
1615
+ walk(sec, 1)
1616
+ for ref in root.findall(lambda x: x.tag == "ref"): # usually in <back>, but some articles keep them in the body
1617
+ cit = ref.find(lambda x: x.tag in ("mixed-citation", "element-citation", "citation"))
1618
+ if cit is not None:
1619
+ doc.refs.append((ref.attrs.get("id", ""), structured_ref(ix, cit) if cit.tag == "element-citation" else ix.run(cit).strip()))
1620
+ fns = [f for f in root.findall(lambda x: x.tag == "fn" and x.parent is not None and x.parent.tag == "fn-group")
1621
+ if f.attrs.get("id") not in doc._jats_cited] # notes already shown where they are cited
1622
+ notes = root.findall(lambda x: x.tag == "author-notes")
1623
+ extra = [ix.run(f).strip() for f in fns] + [ix.run(fn).strip() for an in notes for fn in an.findall(lambda x: x.tag == "fn")]
1624
+ extra = [e for e in extra if e]
1625
+ if extra:
1626
+ first_app = not any(b.get("appendix") for b in doc.blocks)
1627
+ doc.add(t="h", level=2, html="Article notes", appendix=first_app)
1628
+ for e in extra:
1629
+ doc.add(t="p", html=e)
1630
+ return doc
1631
+
1632
+
1633
+ # --------------------------------------------------------------------------
1634
+ # Route: PDF via Docling
1635
+ # --------------------------------------------------------------------------
1636
+
1637
+ def run_docling(pdf, d, pages=None):
1638
+ """Convert a private copy of the PDF, so Docling never touches the original."""
1639
+ (d / "in").mkdir(parents=True, exist_ok=True)
1640
+ (d / "docling").mkdir(exist_ok=True)
1641
+ src = d / "in" / "paper.pdf"
1642
+ if Path(pdf).resolve() != src.resolve():
1643
+ shutil.copyfile(str(pdf), str(src))
1644
+ exe = shutil.which("docling") or str(HOME / ".local" / "bin" / "docling")
1645
+ if not Path(exe).exists():
1646
+ raise ReaderError("PDFs need Docling: uv tool install docling --python 3.12")
1647
+ cmd = [exe, str(src), "--to", "json", "--image-export-mode", "placeholder", "--output", str(d / "docling")]
1648
+ if pages:
1649
+ cmd += ["--page-range", pages] # one argument, "253-314"
1650
+ print("running Docling on %s ..." % Path(pdf).name)
1651
+ r = subprocess.run(cmd, cwd=str(d), stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
1652
+ if r.returncode != 0:
1653
+ raise ReaderError("Docling failed:\n" + r.stdout.decode("utf-8", "replace")[-1500:])
1654
+ print("Docling JSON in %s" % (d / "docling"))
1655
+
1656
+
1657
+ def cmd_pdf_convert(a):
1658
+ meta = ledger_meta(a.ledger, a.pid)
1659
+ pdf = cache_file("pdf", meta["pid"], ".pdf")
1660
+ if not pdf:
1661
+ raise ReaderError("no cached PDF for %s" % a.pid)
1662
+ run_docling(pdf, SRC / slug(meta), a.pages)
1663
+
1664
+
1665
+ def crop_png(pdf, page, bbox, page_h, dpi=170):
1666
+ """Render one region of one PDF page with pdftoppm. bbox is
1667
+ (l, t, r, b) in PDF points with the origin at the bottom left."""
1668
+ l, t, r, b = bbox
1669
+ scale = dpi / 72.0
1670
+ pad = 4
1671
+ x, y = max(0, int((l - pad) * scale)), max(0, int((page_h - t - pad) * scale))
1672
+ w, h = int((r - l + 2 * pad) * scale), int((t - b + 2 * pad) * scale)
1673
+ if w < 10 or h < 10:
1674
+ return None
1675
+ WORK.mkdir(parents=True, exist_ok=True)
1676
+ with tempfile.TemporaryDirectory(dir=str(WORK)) as tmp: # its own folder, so simultaneous builds can't mix up crops
1677
+ root = Path(tmp) / "crop"
1678
+ png = Path(tmp) / "crop.png"
1679
+ # pdftoppm 26 writes nothing when the output root is "-", so go through a file
1680
+ try:
1681
+ r_ = subprocess.run(["pdftoppm", "-f", str(page), "-l", str(page), "-r", str(dpi), "-x", str(x), "-y", str(y),
1682
+ "-W", str(w), "-H", str(h), "-png", "-singlefile", "--", str(pdf), str(root)],
1683
+ stdout=subprocess.PIPE, stderr=subprocess.PIPE, timeout=60)
1684
+ except subprocess.TimeoutExpired:
1685
+ return None # one very slow page shouldn't stop the build
1686
+ return png.read_bytes() if r_.returncode == 0 and png.exists() else None
1687
+
1688
+
1689
+ WATERMARK = re.compile(r"^(Downloaded from |Downloaded by |This article is protected by copyright|Copyright ©)", re.I)
1690
+ DICT = None
1691
+
1692
+
1693
+ def word_list():
1694
+ global DICT
1695
+ if DICT is None:
1696
+ p = Path("/usr/share/dict/words")
1697
+ DICT = set(w.strip().lower() for w in p.read_text(errors="ignore").splitlines()) if p.exists() else set()
1698
+ return DICT
1699
+
1700
+
1701
+ def join_drop_cap(text):
1702
+ """'H uman' is a drop cap read as its own word; 'T cells' is not."""
1703
+ m = re.match(r"^([B-HJ-Z]) ([a-z]+)", text)
1704
+ if m and m.group(2) not in ("is", "are", "was", "were", "be", "in", "of", "and", "or", "to", "at", "on", "as", "has", "can", "may", "cells"):
1705
+ w = (m.group(1) + m.group(2)).lower()
1706
+ stems = [w, w[:-1] if w.endswith("s") else w, w[:-2] if w.endswith("ed") else w, w[:-1] if w.endswith("ed") else w,
1707
+ w[:-3] if w.endswith("ing") else w, w[:-3] + "e" if w.endswith("ing") else w]
1708
+ if any(x in word_list() for x in stems):
1709
+ return m.group(1) + text[2:]
1710
+ return text
1711
+
1712
+
1713
+ def dehyphen(a, b):
1714
+ """Join 'perspec-' + 'tives' when the result is a word."""
1715
+ word_list()
1716
+ m1, m2 = re.search(r"(\w+)-$", a), re.match(r"([a-z]+)", b)
1717
+ if m1 and m2:
1718
+ joined = (m1.group(1) + m2.group(1)).lower()
1719
+ if joined in DICT or (joined.rstrip("s") in DICT) or (joined[:-2] in DICT and joined.endswith("ed")):
1720
+ return a[:-1] + b
1721
+ return a + b
1722
+
1723
+
1724
+ def docling_table_html(item):
1725
+ """Docling's reading of a table as an HTML table, or "". It goes under the
1726
+ table's image, marked unchecked, so screen readers and phones get the data."""
1727
+ data = item.get("data") or {}
1728
+ cells = [c for c in data.get("table_cells") or [] if isinstance(c, dict)]
1729
+ try: # from the cells themselves too, so a missing or low count can't empty or cut the table
1730
+ nrows = max([data.get("num_rows") or 0] + [int(c.get("end_row_offset_idx") or 0) for c in cells])
1731
+ ncols = max([data.get("num_cols") or 0] + [int(c.get("end_col_offset_idx") or 0) for c in cells])
1732
+ except (TypeError, ValueError):
1733
+ return ""
1734
+ if nrows < 2 or not any((c.get("text") or "").strip() for c in cells):
1735
+ return ""
1736
+ if ncols < 1:
1737
+ return ""
1738
+ starts, covered = {}, set() # Docling leaves empty cells out, so each cell is placed by its offsets
1739
+ for c in cells:
1740
+ try:
1741
+ r0, r1 = int(c["start_row_offset_idx"]), int(c["end_row_offset_idx"])
1742
+ c0, c1 = int(c["start_col_offset_idx"]), int(c["end_col_offset_idx"])
1743
+ except (KeyError, TypeError, ValueError):
1744
+ return "" # not the shape we know: offer nothing rather than a garbled table
1745
+ r1, c1 = min(max(r1, r0 + 1), nrows), min(max(c1, c0 + 1), ncols)
1746
+ span = set((r, k) for r in range(r0, r1) for k in range(c0, c1))
1747
+ if not (0 <= r0 < nrows and 0 <= c0 < ncols) or span & covered:
1748
+ continue # overlaps a cell already placed: keep the first, so every row keeps its width
1749
+ starts[(r0, c0)] = (r1 - r0, c1 - c0, c)
1750
+ covered |= span
1751
+ out = []
1752
+ for r in range(nrows):
1753
+ tds, k = [], 0
1754
+ while k < ncols:
1755
+ if (r, k) in starts:
1756
+ rs, cs, c = starts[(r, k)]
1757
+ head = c.get("column_header") or c.get("row_header")
1758
+ tag = "th" if head else "td"
1759
+ scope = (' scope="col"' if c.get("column_header") else ' scope="row"') if head else "" # col and row already cover every column or row spanned
1760
+ span = (' rowspan="%d"' % rs if rs > 1 else "") + (' colspan="%d"' % cs if cs > 1 else "")
1761
+ tds.append("<%s%s%s>%s</%s>" % (tag, scope, span, esc((c.get("text") or "").strip()), tag))
1762
+ k += cs
1763
+ else:
1764
+ if (r, k) not in covered:
1765
+ tds.append("<td></td>") # an empty cell, or one Docling didn't read
1766
+ k += 1
1767
+ out.append("<tr>%s</tr>" % "".join(tds))
1768
+ return '<table class="data">%s</table>' % "".join(out)
1769
+
1770
+
1771
+ def docling_doc(meta, d, pages=None):
1772
+ js = sorted((d / "docling").glob("*.json"))
1773
+ if not js:
1774
+ raise ReaderError("no Docling output; run pdf-convert first")
1775
+ j = json.loads(js[0].read_text(encoding="utf-8"))
1776
+ doc = Doc(dict(meta))
1777
+ doc.pdf = source_pdf(meta)
1778
+ doc.meta["kicker"] = meta.get("venue") or "Article"
1779
+ doc.source = "Text converted from the PDF by Docling; figures and tables are images cropped from the PDF pages."
1780
+ page_h = {int(k): v["size"]["height"] for k, v in j.get("pages", {}).items()}
1781
+ offset = 0
1782
+ if pages and page_h and min(page_h) == 1:
1783
+ offset = int(pages.split("-")[0]) - 1 # only if Docling renumbered the range from 1
1784
+
1785
+ def resolve(ref):
1786
+ kind, idx = ref["$ref"].split("/")[1], int(ref["$ref"].split("/")[2])
1787
+ return kind, j[kind][idx]
1788
+
1789
+ # one figure can arrive as several pictures, with its labels read as text: gather them under the caption
1790
+ float_box, absorbed, figure_areas = {}, set(), {}
1791
+ pictures = [x for x in j.get("pictures", []) if x.get("prov")]
1792
+
1793
+ def area(it):
1794
+ pg = it["prov"][0]["page_no"]
1795
+ return docling_box(it["prov"][0]["bbox"], page_h.get(pg, 792))
1796
+ for pic in pictures:
1797
+ if not pic.get("captions"):
1798
+ continue
1799
+ pg = pic["prov"][0]["page_no"]
1800
+ l, t, r, b = area(pic)
1801
+ cap = resolve(pic["captions"][0])[1]
1802
+ cl, cr = (area(cap)[0], area(cap)[2]) if cap.get("prov") else (l, r)
1803
+ for other in pictures:
1804
+ if other is pic or other.get("captions") or other["prov"][0]["page_no"] != pg:
1805
+ continue
1806
+ ol, ot, orr, ob = area(other)
1807
+ if ob <= t + 30 and ot >= b - 30 and cl <= (ol + orr) / 2 <= cr: # under this caption, not in the next column
1808
+ l, t, r, b = min(l, ol), max(t, ot), max(r, orr), min(b, ob)
1809
+ absorbed.add(other.get("self_ref"))
1810
+ if cr - cl > 1.25 * (r - l): # the caption spans the whole figure; the picture found is only part of it
1811
+ l, r = min(l, cl), max(r, cr)
1812
+ float_box[pic.get("self_ref")] = (l, t, r, b)
1813
+ figure_areas.setdefault(pg, []).append((l, t + 12, r, b))
1814
+ head_counts = Counter((t.get("text") or "").strip() for t in j.get("texts", []) if t.get("label") == "section_header")
1815
+
1816
+ def figure_text(item, text):
1817
+ """A label drawn inside a figure, which Docling reads as text or a heading."""
1818
+ if not item.get("prov") or len(text) >= 80:
1819
+ return False
1820
+ pg = item["prov"][0]["page_no"]
1821
+ il, it, ir, ib = area(item)
1822
+ cx, cy = (il + ir) / 2, (it + ib) / 2
1823
+ for l, t, r, b in figure_areas.get(pg, []):
1824
+ if l <= cx <= r and (b <= cy <= t or (head_counts[text] >= 2 and re.search(r"[^\W\d_]\d\b", text) and 0 <= ib - t <= 55)):
1825
+ return True
1826
+ return False
1827
+
1828
+ # reading order: the body tree, flattening groups
1829
+ order = []
1830
+
1831
+ def flatten(node):
1832
+ for ch in node.get("children", []):
1833
+ kind, item = resolve(ch)
1834
+ if kind == "groups":
1835
+ order.append(("group-start", item))
1836
+ flatten(item)
1837
+ order.append(("group-end", item))
1838
+ else:
1839
+ order.append((kind, item))
1840
+ if kind in ("pictures", "tables"):
1841
+ continue # captions are children; handled with the float
1842
+ flatten(item)
1843
+ flatten(j["body"])
1844
+
1845
+ # a repository's cover sheet ("Please cite the original version ... Take down policy") is not the paper
1846
+ cover = set()
1847
+ by_page = {}
1848
+ for t in j.get("texts", []):
1849
+ pg = (t.get("prov") or [{}])[0].get("page_no")
1850
+ by_page.setdefault(pg, []).append(t.get("text") or "")
1851
+ for pg, txts in by_page.items():
1852
+ blob = " ".join(txts).lower()
1853
+ if pg and pg <= 2 and sum(k in blob for k in ("please cite the original version", "take down policy",
1854
+ "general rights", "this is an electronic reprint",
1855
+ "self-archived", "to cite this article", "submit your article to this journal",
1856
+ "view related articles", "citing articles")) >= 2:
1857
+ cover.add(pg)
1858
+ names = set(re.sub(r"\W", "", a.lower()) for a in meta.get("authors") or [])
1859
+ venue = re.sub(r"\W", "", (meta.get("venue") or "").lower())
1860
+
1861
+ printed = printed_page_numbers(j)
1862
+ in_refs, refs_raw = False, []
1863
+ last_page = None
1864
+ pending = None # paragraph text waiting for its continuation
1865
+ list_items = None
1866
+ list_depth = 0 # a list inside a list joins the outer one
1867
+ list_numbered = False
1868
+ caption_ids = set()
1869
+ for kind, item in order:
1870
+ if kind in ("pictures", "tables"):
1871
+ for c in item.get("captions", []):
1872
+ caption_ids.add(c["$ref"])
1873
+
1874
+ total_words = sum(words(t.get("text") or "") for t in j.get("texts", []) if t.get("label") not in ("page_header", "page_footer"))
1875
+ deferred = []
1876
+ last_fn = [None] # the latest footnote, which may run on to the next page
1877
+ front_dropped = []
1878
+ seen_head = [False]
1879
+ summary = [None] # None, then "open" under a page-one summary, then "closed"
1880
+ pending_label = [None] # the run-in label of the held summary paragraph
1881
+ chapter_pre = [None]
1882
+
1883
+ def flush():
1884
+ nonlocal pending
1885
+ if pending is not None:
1886
+ doc.add(t="p", html=("<b>%s:</b> " % esc(pending_label[0]) if pending_label[0] else "") + esc(pending))
1887
+ pending = None
1888
+ pending_label[0] = None
1889
+ while deferred:
1890
+ fn = deferred.pop(0)
1891
+ fn["block"] = {"t": "p", "html": "<small>%s</small>" % esc(fn["text"])}
1892
+ doc.blocks.append(fn["block"])
1893
+
1894
+ def page_marker(p):
1895
+ nonlocal last_page
1896
+ if p and p != last_page:
1897
+ last_page = p
1898
+ if printed.get(p) == "":
1899
+ return
1900
+ label = printed.get(p) or str(p + offset)
1901
+ doc.add(t="page", label=label, pdf=p + offset)
1902
+
1903
+ for kind, item in order:
1904
+ if kind == "group-start":
1905
+ if item.get("label") in ("list", "ordered_list"):
1906
+ if list_depth == 0:
1907
+ flush()
1908
+ list_items = []
1909
+ list_numbered = False
1910
+ list_depth += 1
1911
+ continue
1912
+ if kind == "group-end":
1913
+ if list_depth and item.get("label") in ("list", "ordered_list"):
1914
+ list_depth -= 1
1915
+ if list_depth == 0:
1916
+ if list_items:
1917
+ doc.add(t="list", ordered=item.get("label") == "ordered_list" or list_numbered, items=list_items)
1918
+ list_items = None
1919
+ continue
1920
+ if item.get("self_ref") in caption_ids:
1921
+ continue
1922
+ label = item.get("label")
1923
+ prov = item.get("prov") or [{}]
1924
+ p = prov[0].get("page_no")
1925
+ if label in ("page_header", "page_footer") or p in cover:
1926
+ continue
1927
+ text = STAMP_IN_TEXT.sub("", item.get("text") or "").strip() # Docling can fuse a page's publisher line into a sentence
1928
+ if label in ("text", "paragraph", "list_item", "footnote", "caption", "section_header", "title", "formula", "code", "checkbox_selected", "checkbox_unselected", "reference"):
1929
+ if WATERMARK.match(text):
1930
+ continue
1931
+ if kind in ("pictures", "tables") and item.get("self_ref") in absorbed:
1932
+ continue
1933
+ if kind in ("pictures", "tables"):
1934
+ pnum = p
1935
+ caps = [resolve(c)[1].get("text", "") for c in item.get("captions", [])]
1936
+ cap_text = " ".join(caps).strip()
1937
+ lab = ""
1938
+ mm = re.match(r"^((?:Fig(?:ure)?|Table|TABLE|FIGURE)\.?\s*[\dA-Z]+[.:]?)\s*(.*)$", cap_text, re.S)
1939
+ if mm:
1940
+ lab, cap_text = mm.group(1), mm.group(2)
1941
+ imgs = []
1942
+ table_text = docling_table_html(item) if kind == "tables" else ""
1943
+ if pnum and doc.pdf:
1944
+ bb = prov[0]["bbox"]
1945
+ box = float_box.get(item.get("self_ref")) or docling_box(bb, page_h.get(pnum, 792))
1946
+ png = crop_png(doc.pdf, pnum + offset, box, page_h.get(pnum, 792))
1947
+ u = data_uri(png) if png else None
1948
+ if u:
1949
+ alt = "%s, shown as an image" % (lab.rstrip(".: ") or ("Table" if kind == "tables" else "Figure"))
1950
+ imgs.append((u, alt + (". The table as text follows it" if table_text else "")))
1951
+ if not imgs and not cap_text:
1952
+ continue
1953
+ bb = prov[0].get("bbox") or {}
1954
+ if not cap_text and (abs(bb.get("r", 0) - bb.get("l", 0)) < 120 or abs(bb.get("t", 0) - bb.get("b", 0)) < 80):
1955
+ continue # logos and icons
1956
+ held = None
1957
+ if not cap_text and pending and len(pending) < 400 and CAPTION_LIKE.match(pending):
1958
+ cm = CAPTION_LIKE.match(pending) # the caption above the float, left as a paragraph
1959
+ lab, cap_text, pending = cm.group(1), cm.group(2).strip(), None
1960
+ pending_label[0] = None # the held text was a caption, not a labelled summary part
1961
+ caption = esc(cap_text)
1962
+ gm = TABLE_NOTE_GLUED.search(pending) if (pending and kind == "tables") else None
1963
+ if gm: # Docling sometimes glues a table note onto the sentence the table cuts
1964
+ pending = pending[:gm.start()]
1965
+ caption += ' <small class="tnote">%s</small>' % esc(gm.group(1).strip())
1966
+ if pending is not None and not re.search(r"[.!?:;”\"')\]]$", pending):
1967
+ held, pending = pending, None # a sentence cut by the float: finish it after the float
1968
+ flush()
1969
+ page_marker(pnum)
1970
+ doc.add(t="fig", kind="table" if kind == "tables" else "figure", label=lab, caption=caption,
1971
+ imgs=imgs, table="", table_text=table_text if imgs else "", page=(pnum + offset) if pnum else None)
1972
+ pending = held
1973
+ continue
1974
+ if label == "formula" and not text: # Docling found an equation but couldn't read it: show it as printed
1975
+ png = crop_png(doc.pdf, p + offset, docling_box(prov[0]["bbox"], page_h.get(p, 792)), page_h.get(p, 792)) if doc.pdf and p else None
1976
+ u = data_uri(png) if png else None
1977
+ if u:
1978
+ held = None
1979
+ if pending is not None and not re.search(r"[.!?:;”\"')\]]$", pending):
1980
+ held, pending = pending, None # the sentence carries on after the equation
1981
+ flush()
1982
+ page_marker(p)
1983
+ doc.add(t="eq", html=img_tag(u, "Equation"))
1984
+ pending = held
1985
+ continue
1986
+ real_heading = label == "section_header" and p and p > 2 and re.fullmatch(r"perspectives?|reviews?|reports?|artificial intelligence", text, re.I)
1987
+ if not text or figure_text(item, text) or (furniture(text, doc.meta.get("title")) and not real_heading):
1988
+ continue
1989
+ last = doc.blocks[-1] if doc.blocks else None
1990
+ cm = CAPTION_LIKE.match(text)
1991
+ if cm and len(text) < 400 and last is not None and last["t"] == "fig" and not last.get("label") and not last.get("caption"):
1992
+ last["label"], last["caption"] = cm.group(1), esc(cm.group(2).strip())
1993
+ continue
1994
+ if last is not None and last["t"] == "fig" and len(text) < 600 and (
1995
+ re.match(r"(Notes?[:.]|Sources?:)\s", text) or (last.get("kind") == "table" and TABLE_NOTE.match(text))):
1996
+ last["caption"] += ' <small class="tnote">%s</small>' % esc(text) # "NS: Not significant ...", "Note: ..."
1997
+ continue
1998
+ if (pending is None and last is not None and last["t"] == "fig" and last.get("caption")
1999
+ and not re.search(r"[.!?)]$", strip_tags(last["caption"]).strip()) and re.match(r"^[a-z(]", text)):
2000
+ last["caption"] = esc(dehyphen(strip_tags(last["caption"]).strip(), text)) if last["caption"].endswith("-") \
2001
+ else last["caption"] + " " + esc(text) # a caption continued in the next column
2002
+ continue
2003
+ if label in ("section_header", "title") and re.sub(r"\W", "", text.lower()) in names | {venue}:
2004
+ continue # an author's name or the journal's name set as a heading
2005
+ if label in ("section_header", "title"):
2006
+ flush()
2007
+ seen_head[0] = True
2008
+ if REFS_HEAD.match(text):
2009
+ in_refs = True
2010
+ continue
2011
+ if in_refs and refs_raw:
2012
+ in_refs = False
2013
+ page_marker(p)
2014
+ if re.fullmatch(r"(Chapter|CHAPTER|Part|PART) [\dIVX]+", text):
2015
+ chapter_pre[0] = text # "Chapter 9" then "Restoring Trust": one heading
2016
+ continue
2017
+ if chapter_pre[0]:
2018
+ text, chapter_pre[0] = chapter_pre[0] + ": " + text, None
2019
+ lvl = 2 if (item.get("level") or 1) <= 1 else 3
2020
+ numbered = re.match(r"^\d{1,2}((?:\.\d{1,2})*)\.?\s", text)
2021
+ if numbered: # Docling often flattens levels; "3.2" says where it belongs
2022
+ lvl = 3 if numbered.group(1) else 2
2023
+ if text.isupper() and len(text) > 3:
2024
+ text = re.sub(r"[^\W\d_]", lambda c: c.group(0).upper(), text.lower(), count=1)
2025
+ # acknowledgments near the front of a book or thesis don't end the main text
2026
+ end_matter = bool(END_MATTER.match(text)) and not any(b.get("appendix") for b in doc.blocks) and (
2027
+ sum(words(strip_tags(block_text(b))) for b in doc.blocks) >= FRONT_MATTER_SHARE * total_words)
2028
+ doc.add(t="h", level=lvl, html=esc(text), appendix=end_matter)
2029
+ continue
2030
+ surnames = [a.split()[-1] for a in doc.meta.get("authors") or [] if a.split()]
2031
+ if not in_refs and p and p <= 2 and len(text) < 800 and len(surnames) >= 3 and sum(1 for s in surnames if s in text) >= 3:
2032
+ continue # the author line, with affiliation marks; the header shows the authors
2033
+ if not in_refs and not seen_head[0] and p and p <= 2 and len(text) < 1200 and len(re.findall(r"(?:^|\.\s+)\d{1,2}\s+[A-Z]", text)) >= 2 \
2034
+ and re.match(r"1\s", text) and re.search(r"\b(Universit|Institut|College|School|Department|Laborator|Centre|Center)", text):
2035
+ front_dropped.append(text) # a numbered affiliation list ("1 Google DeepMind, London, UK. 2 …"), set mid-column
2036
+ continue
2037
+ if not seen_head[0] and len(text.split()) < 25 and not re.match(r"keywords\b", text, re.I) and (
2038
+ not re.search(r"[.?!]$", text) or "@" in text or re.search(r"10\.\d{4,}/", text)):
2039
+ front_dropped.append(text) # front matter before the first heading: affiliations, article type, DOI lines
2040
+ continue
2041
+ if in_refs:
2042
+ mk = (item.get("marker") or "").strip()
2043
+ refs_raw.append((mk + " " + text) if re.fullmatch(r"\[?\d{1,3}[.\]]?", mk) else text)
2044
+ continue
2045
+ if label == "list_item" and list_items is not None:
2046
+ flush()
2047
+ list_numbered = list_numbered or bool(item.get("enumerated"))
2048
+ list_items.append(esc(re.sub(r"^[•·\-–]\s*", "", text)))
2049
+ continue
2050
+ if label == "footnote" and last is not None and last["t"] == "fig" and last.get("kind") == "table" and (
2051
+ TABLE_NOTE.match(text) or re.search(r"\bp\s?[<=≤]\s?0?\.\d", text)):
2052
+ last["caption"] += ' <small class="tnote">%s</small>' % esc(text)
2053
+ continue
2054
+ if label == "footnote":
2055
+ fn = last_fn[0]
2056
+ if (fn and p and fn["page"] and p > fn["page"] and re.match(r"^[a-z(]", text)
2057
+ and not re.search(r"[.!?:;”\"')\]]$", fn["text"])):
2058
+ # the rest of the previous page's last footnote, carried over the page break
2059
+ fn["text"] = dehyphen(fn["text"], text) if fn["text"].endswith("-") else fn["text"] + " " + text
2060
+ fn["page"] = p
2061
+ if fn.get("block"):
2062
+ fn["block"]["html"] = "<small>%s</small>" % esc(fn["text"])
2063
+ continue
2064
+ last_fn[0] = {"text": text, "page": p}
2065
+ deferred.append(last_fn[0]) # after the paragraph it interrupts, not inside it
2066
+ continue
2067
+ lab = SUMMARY_LABEL.match(text) if not seen_head[0] and label in ("text", "paragraph") else None
2068
+ if lab and re.fullmatch(r"KEY ?WORDS", lab.group(1).strip()):
2069
+ lab = None
2070
+ if lab: # a page-one summary in run-in labels: Science's "INTRODUCTION: …", "RATIONALE: …", with no heading over it
2071
+ flush()
2072
+ if not summary[0]:
2073
+ page_marker(p)
2074
+ doc.add(t="h", level=2, html="Abstract" if lab.group(1).strip() == "ABSTRACT" else "Summary")
2075
+ summary[0] = "open"
2076
+ pending = text[lab.end():] # held, so a part split at a column can still be joined
2077
+ pending_label[0] = None if lab.group(1).strip() == "ABSTRACT" else lab.group(1).strip().capitalize()
2078
+ continue
2079
+ if summary[0] == "open" and pending is not None and pending_label[0] and label in ("text", "paragraph") \
2080
+ and not re.search(r"[.!?:;”\"')\]]$", pending) and re.match(r"^[a-z(]", text):
2081
+ pending = dehyphen(pending, text) if pending.endswith("-") else pending + " " + text
2082
+ if p != last_page:
2083
+ page_marker(p)
2084
+ continue
2085
+ if summary[0] == "open" and not seen_head[0] and label in ("text", "paragraph") and len(text.split()) >= 40:
2086
+ flush() # the article proper starts here, untitled; without a heading it would sit inside the summary
2087
+ page_marker(p)
2088
+ doc.add(t="h", level=2, html="Introduction")
2089
+ summary[0] = "closed"
2090
+ if label == "formula":
2091
+ flush()
2092
+ doc.add(t="eq", html=esc(text))
2093
+ continue
2094
+ if label == "code":
2095
+ flush()
2096
+ doc.add(t="pre", text=text)
2097
+ continue
2098
+ # body text: join a paragraph broken by a column or page
2099
+ if pending is not None and (pending.endswith("-") or not re.search(r"[.!?:;”\"')\]]$", pending)) and re.match(r"^[a-z(]", text):
2100
+ pending = dehyphen(pending, text) if pending.endswith("-") else pending + " " + text
2101
+ if p != last_page:
2102
+ page_marker(p) # the marker lands before the joined paragraph; close enough
2103
+ continue
2104
+ flush()
2105
+ page_marker(p)
2106
+ pending = join_drop_cap(text)
2107
+ flush()
2108
+ link_endnotes(doc)
2109
+ link_references(doc, refs_raw)
2110
+ if front_dropped:
2111
+ print(" dropped %d short lines before the first heading: %s" % (
2112
+ len(front_dropped), " | ".join(t[:40] for t in front_dropped)[:400]))
2113
+ return doc
2114
+
2115
+
2116
+ SUMMARY_LABEL = re.compile(r"^((?:[A-Z]{3,} ?){1,3}):\s+(?=\S)") # "INTRODUCTION: ", "RATIONALE: "
2117
+ FRONT_MATTER_SHARE = 0.25 # end matter can't begin before this share of the document's words
2118
+ TABLE_NOTE = re.compile(r"^(Notes?[:.]\s|Sources?:\s|[A-Z][A-Za-z]{0,5}: [A-Z]?[a-z]|[*∗†‡]+\s?p\s?[<=≤])")
2119
+ CAPTION_LIKE = re.compile(r"^((?:Fig(?:ure)?|Table|TABLE|FIGURE)\.?\s*[A-Z]?\d+(?:\.\d+)?[.:])\s*(.+)$", re.S)
2120
+ TABLE_NOTE_GLUED = re.compile(r"(?<=[a-z,]) ((?:[A-Z]{1,5}: [a-z][a-z ,()-]{2,80}\.\s?)+)$")
2121
+
2122
+ FURNITURE = re.compile(
2123
+ r"^(research article( summary)?|research articles?|reports?|reviews?|perspectives?|artificial intelligence|"
2124
+ r"read the full article at|view the article online|permissions|use of this article is subject to.*|"
2125
+ r"cite this article as.*|the list of author affiliations is available in the full article online.*|downloaded from .*|this article is protected by copyright.*|"
2126
+ r"routledge|taylor & francis group|check for updates|open n? ?access)$", re.I)
2127
+
2128
+
2129
+ STAMP_IN_TEXT = re.compile(r"(?:[^©]{0,40}©\s*the author\(s\)\s+\d{4}\s*)?(?:https?://doi\.org/\S+\s+)?article reuse guidelines:.{0,160}?journals\.sagepub\.com/home/\w+\s*", re.I | re.S)
2130
+ PUBLISHER_STAMP = re.compile(r"provided proper attribution is provided|article reuse guidelines|journals-permissions|©\s*the author\(s\)\s+\d{4}", re.I)
2131
+
2132
+
2133
+ def furniture(text, title):
2134
+ """Publisher page furniture: kicker labels, bare URLs, stray glyphs, and the
2135
+ title repeated on later pages."""
2136
+ t = text.strip()
2137
+ if len(t) < 400 and PUBLISHER_STAMP.search(t):
2138
+ return True
2139
+ if len(t) <= 2 or FURNITURE.match(t) or re.fullmatch(r"(https?://|www\.|doi\.org/|science\.org/)\S+", t):
2140
+ return True
2141
+ if title:
2142
+ norm = lambda s: re.sub(r"[^a-z0-9]", "", s.lower())
2143
+ if norm(t) and norm(t) == norm(title):
2144
+ return True
2145
+ return False
2146
+
2147
+
2148
+ END_MATTER = re.compile(r"^\s*(acknowledg|supplementary|funding|author contributions|competing interests|"
2149
+ r"declaration|data (and materials )?availability|disclosure|conflicts? of interest|notes on contributors|editor'?s summary)", re.I)
2150
+
2151
+ REFS_HEAD = re.compile(r"^\s*(references(\s+and\s+notes)?|bibliography|works cited|literature cited)\s*$", re.I)
2152
+
2153
+
2154
+ NOTES_HEAD = re.compile(r"^\s*(end)?notes\s*$", re.I)
2155
+
2156
+
2157
+ def link_endnotes(doc):
2158
+ """Numbered endnotes whose markers lost their superscript. A marker is only
2159
+ recognised straight after punctuation ("in Korsholm. 2 The"), in order. One
2160
+ that can't be told from an ordinary number stays as plain text."""
2161
+ start = next((i for i, b in enumerate(doc.blocks) if b["t"] == "h" and NOTES_HEAD.match(strip_tags(b["html"]))), None)
2162
+ if start is None:
2163
+ return
2164
+ notes, count = {}, 0
2165
+ for b in doc.blocks[start + 1:]:
2166
+ if b["t"] == "h":
2167
+ break
2168
+ for h in (b["items"] if b["t"] == "list" else [b.get("html", "")] if b["t"] == "p" else []):
2169
+ count += 1
2170
+ m = re.match(r"^(?:<small>)?\s*(\d{1,3})[.)]?\s+(.*?)(?:</small>)?$", h, re.S)
2171
+ if m:
2172
+ notes[int(m.group(1))] = m.group(2)
2173
+ elif b["t"] == "list": # Docling keeps a list's numbers apart from its text
2174
+ notes[count] = h
2175
+ at, pos = 0, 0
2176
+ for n in sorted(notes):
2177
+ pat = re.compile(r"(?<=[^\W\d_][.,;:?!”’\")]) (%d)(?=\s|$)" % n)
2178
+ for i in range(at, start):
2179
+ b = doc.blocks[i]
2180
+ m = pat.search(b["html"], pos if i == at else 0) if b["t"] == "p" else None
2181
+ if m:
2182
+ note = doc.note(notes[n], n)
2183
+ b["html"] = b["html"][:m.start()] + note + b["html"][m.end():]
2184
+ at, pos = i, m.start() + len(note)
2185
+ break
2186
+
2187
+
2188
+ def link_references(doc, raw):
2189
+ """Turn the PDF's reference list into doc.refs and link the citations in
2190
+ the text: numbered "(19)" / "(5, 8-10)" style, or author-year style."""
2191
+ if not raw:
2192
+ return
2193
+ numbered = sum(1 for r in raw if re.match(r"^\s*\[?\d{1,3}[.\])]?\s", r)) > len(raw) / 2
2194
+ if numbered:
2195
+ cur = None
2196
+ for r in raw:
2197
+ m = re.match(r"^\s*\[?(\d{1,3})[.\])]?\s+(.*)$", r, re.S)
2198
+ if m:
2199
+ cur = [m.group(1), m.group(2)]
2200
+ doc.refs.append(cur)
2201
+ elif cur is not None:
2202
+ cur[1] = dehyphen(cur[1], r) if cur[1].endswith("-") else cur[1] + " " + r
2203
+ doc.refs = [(n, esc(t)) for n, t in doc.refs]
2204
+ known = set(n for n, _ in doc.refs)
2205
+
2206
+ def numfix(m):
2207
+ before, after = m.string[max(0, m.start() - 14):m.start()], m.string[m.end():m.end() + 3]
2208
+ if re.search(r"(?i)\b(eqs?\.?|equations?|figs?\.?|figures?|tables?|sections?|steps?|models?|hypothes[ie]s)\s*$", before):
2209
+ return m.group(0) # "Eq. (3)" is not a citation
2210
+ if m.group(1) == "(" and re.fullmatch(r"\d", m.group(2)) and re.match(r"\s[a-z]", after) and re.search(r"(^|[:;,]|\b(?:and|or))\s*$", before):
2211
+ return m.group(0) # "two aims: (1) trust and (2) recall"
2212
+ out, linked = [], False
2213
+ for x in re.split(r"(\s*,\s*)", m.group(2)):
2214
+ r = re.fullmatch(r"(\d{1,3})\s*[–-]\s*(\d{1,3})", x)
2215
+ ids = [x] if x in known else []
2216
+ if r and int(r.group(1)) < int(r.group(2)) <= int(r.group(1)) + 50:
2217
+ ids = [str(k) for k in range(int(r.group(1)), int(r.group(2)) + 1) if str(k) in known]
2218
+ linked = linked or bool(ids)
2219
+ out.append(doc.cite(ids, x) if ids else x)
2220
+ return m.group(1) + "".join(out) + m.group(3) if linked else m.group(0)
2221
+ pat = re.compile(r"([(\[])(\d{1,3}(?:\s*[,–-]\s*\d{1,3})*)([)\]])")
2222
+ else:
2223
+ # author-year: a reference starts with a capitalised surname and contains a year
2224
+ refs, cur = [], None
2225
+ for r in raw:
2226
+ # "Abbott, A. (1988)", "Ackerman B and Fishkin JS (2002)", "van der Meer TWG (2017)", "Yle News (2019)"
2227
+ starts = re.match(r"^(?:(?:[Vv]an|[Vv]on|[Dd]e|[Dd]er|[Dd]en|[Dd]u|[Ll]a|[Ll]e)\s+)*[A-ZÀ-Þ][\w'’-]*(?:,| [A-ZÀ-Þ]{1,4}\b| \(| and | [A-ZÀ-Þ][a-zà-ÿ]+ \()", r) \
2228
+ and re.search(r"\b(1[89]\d\d|20\d\d)[a-z]?\b", r[:200])
2229
+ if starts or cur is None:
2230
+ cur = [r]
2231
+ refs.append(cur)
2232
+ else:
2233
+ cur.append(r)
2234
+ keys = {}
2235
+ for i, parts in enumerate(refs):
2236
+ t = " ".join(parts)
2237
+ rid = "r%d" % (i + 1)
2238
+ doc.refs.append((rid, esc(t)))
2239
+ sur = re.match(r"^(?:(?:[Vv]an|[Vv]on|[Dd]e|[Dd]er|[Dd]en|[Dd]u|[Ll]a|[Ll]e)\s+)*([A-ZÀ-Þ][\w'’-]+)", t)
2240
+ yr = re.search(r"(1[89]\d\d|20\d\d)([a-z]?)", t)
2241
+ if sur and yr:
2242
+ keys.setdefault((sur.group(1).lower(), yr.group(1) + yr.group(2)), rid)
2243
+
2244
+ def numfix(m):
2245
+ rid = keys.get((m.group(1).lower(), m.group(2) or m.group(3)))
2246
+ return doc.cite([rid], m.group(0)) if rid else m.group(0)
2247
+ pat = re.compile(r"\b(?:(?:[Vv]an|[Vv]on|[Dd]e|[Dd]er|[Dd]en|[Dd]u|[Ll]a|[Ll]e)\s+)*([A-ZÀ-Ý][\w'’-]+)(?:\s+et al\.?|\s+(?:and|&amp;)\s+[A-ZÀ-Ý][\w'’-]+)?,?\s+(?:\(((?:1[89]|20)\d\d[a-z]?)\)|\(?((?:1[89]|20)\d\d[a-z]?)\b)")
2248
+ for b in doc.blocks:
2249
+ if b["t"] == "p":
2250
+ b["html"] = pat.sub(numfix, b["html"])
2251
+ elif b["t"] == "list":
2252
+ b["items"] = [pat.sub(numfix, x) for x in b["items"]]
2253
+
2254
+
2255
+ def printed_page_numbers(j):
2256
+ """Printed page numbers from headers and footers, where they are bare numbers."""
2257
+ out = {}
2258
+ for t in j.get("texts", []):
2259
+ if t.get("label") in ("page_header", "page_footer"):
2260
+ s = (t.get("text") or "").strip()
2261
+ m = re.fullmatch(r"(?:Page\s+)?(\d{1,4})(?:\s+of\s+\d+)?", s)
2262
+ p = (t.get("prov") or [{}])[0].get("page_no")
2263
+ if m and p and p not in out:
2264
+ out[p] = m.group(1)
2265
+ # most pages share one gap between PDF page and printed page; use it to fill misses and overrule misreadings
2266
+ gaps = Counter(p - int(n) for p, n in out.items())
2267
+ if gaps:
2268
+ gap, seen = gaps.most_common(1)[0]
2269
+ if seen >= 2 and seen * 2 >= len(out):
2270
+ pages = [int(k) for k in j.get("pages", {})] or list(out)
2271
+ out = {p: str(p - gap) if p - gap >= 1 else "" for p in pages} # "": a cover page with no printed number
2272
+ return out
2273
+
2274
+
2275
+ # --------------------------------------------------------------------------
2276
+ # Coverage check
2277
+ # --------------------------------------------------------------------------
2278
+
2279
+ def tokens(text, pages_of=None):
2280
+ """Lower-case words. With a list `pages_of`, also appends each word's page
2281
+ index, counted by the form feeds pdftotext puts between pages."""
2282
+ text = unicodedata.normalize("NFKC", text).lower()
2283
+ if pages_of is None:
2284
+ return re.findall(r"[a-z0-9]+", re.sub(r"(\w)-\s*\n\s*(\w)", r"\1\2", text))
2285
+ # a word hyphenated across a page break keeps the break as \x00 inside it, so the count stays right
2286
+ text = re.sub(r"(\w)-\s*\n\s*(\w)", lambda m: m.group(1) + "\x00" * m.group(0).count("\f") + m.group(2), text)
2287
+ out, page = [], 0
2288
+ for m in re.finditer(r"\f|[a-z0-9\x00]+", text):
2289
+ w = m.group()
2290
+ if w == "\f":
2291
+ page += 1
2292
+ continue
2293
+ breaks = w.count("\x00")
2294
+ w = w.replace("\x00", "")
2295
+ if w:
2296
+ out.append(w)
2297
+ pages_of.append(page)
2298
+ page += breaks
2299
+ return out
2300
+
2301
+
2302
+ def coverage(page_html, meta, pages=None, report=True):
2303
+ """How much of the source text reached the page. Returns the counts and the
2304
+ missing runs, each with the PDF pages its words came from."""
2305
+ txt = source_text(meta)
2306
+ if not txt:
2307
+ if report:
2308
+ print(" coverage: no source text to compare against")
2309
+ return None
2310
+ raw = txt.read_text(encoding="utf-8", errors="replace")
2311
+ first = 1
2312
+ if pages: # compare against the converted pages only (pdftotext separates pages with form feeds)
2313
+ lo, hi = (int(x) for x in pages.split("-"))
2314
+ raw = "\f".join(raw.split("\f")[lo - 1:hi])
2315
+ first = lo
2316
+ pg = []
2317
+ src = tokens(raw, pg)
2318
+ body = re.sub(r"<(script|style)[^>]*>.*?</\1>", " ", page_html, flags=re.S)
2319
+ body = re.sub(r"<details class=\"(?:orig-page|table-text)\">.*?</details>", " ", body, flags=re.S) # aids, not the reading text
2320
+ body = re.sub(r'<img[^>]*>', " ", body)
2321
+ got = tokens(strip_tags(body))
2322
+ cs, cg = Counter(src), Counter(got)
2323
+ hit = sum(min(n, cg[w]) for w, n in cs.items())
2324
+ cov = {"hit": hit, "n": len(src), "label": meta.get("text_label", "cached text"), "runs": [], "first": first}
2325
+ grams = set(tuple(got[i:i + 5]) for i in range(len(got) - 4))
2326
+ miss = [tuple(src[i:i + 5]) not in grams for i in range(len(src) - 4)]
2327
+ runs, i = [], 0
2328
+ while i < len(miss):
2329
+ if miss[i]:
2330
+ k = i
2331
+ while k < len(miss) and miss[k]:
2332
+ k += 1
2333
+ if k - i >= 25:
2334
+ words_ = src[i:k + 4]
2335
+ cov["runs"].append({"n": k - i, "words": words_,
2336
+ "pages": Counter(first + p for p in pg[i:k + 4])})
2337
+ i = k
2338
+ else:
2339
+ i += 1
2340
+ cov["runs"].sort(key=lambda r: -r["n"])
2341
+ if report:
2342
+ print_coverage(cov)
2343
+ return cov
2344
+
2345
+
2346
+ def print_coverage(cov):
2347
+ print(" coverage: %.1f%% of the %d words in the %s reached the page" % (
2348
+ 100.0 * cov["hit"] / max(1, cov["n"]), cov["n"], cov["label"]))
2349
+ for r in cov["runs"][:8]:
2350
+ print(" missing run of %d words: %s ..." % (r["n"], " ".join(r["words"][:18])))
2351
+
2352
+
2353
+ FRONT_MATTER_WORDS = {"copyright", "https", "http", "doi", "licence", "license", "licensed", "corresponding", "permissions"}
2354
+
2355
+
2356
+ def weak_pages(cov, limit=10):
2357
+ """PDF pages where a long run of ordinary prose went missing: the pages worth
2358
+ showing as printed. Runs of numbers and labels (charts, tables) don't count,
2359
+ and nor do the licence and contact lines the conversion drops on purpose."""
2360
+ known = word_list()
2361
+ missing = Counter()
2362
+ for r in cov["runs"]:
2363
+ ws = r["words"]
2364
+ prose = sum(1 for w in ws if w.isalpha() and len(w) > 2 and (not known or w in known))
2365
+ if prose < 0.4 * len(ws): # sentences score about 0.5 or more; tables and charts 0.25 or less
2366
+ continue
2367
+ hits = sum(1 for w in ws if w in FRONT_MATTER_WORDS)
2368
+ if hits and (min(r["pages"]) <= cov["first"] + 1 or 30 * hits >= len(ws)):
2369
+ continue # licence and contact blocks: on the opening pages, or dense with these words. A page footer adds one or two.
2370
+ for p, n in r["pages"].items():
2371
+ if n >= 10:
2372
+ missing[p] += n
2373
+ return sorted(p for p, _ in missing.most_common(limit))
2374
+
2375
+
2376
+ def page_image(pdf, page):
2377
+ """The whole printed page as a JPEG, for showing where text may be missing."""
2378
+ WORK.mkdir(parents=True, exist_ok=True)
2379
+ with tempfile.TemporaryDirectory(dir=str(WORK)) as tmp:
2380
+ root = Path(tmp) / "page"
2381
+ try:
2382
+ r = subprocess.run(["pdftoppm", "-f", str(page), "-l", str(page), "-scale-to", "1400", "-jpeg",
2383
+ "-jpegopt", "quality=75", "-singlefile", "--", str(pdf), str(root)],
2384
+ stdout=subprocess.PIPE, stderr=subprocess.PIPE, timeout=60)
2385
+ except subprocess.TimeoutExpired:
2386
+ return None
2387
+ out = Path(tmp) / "page.jpg"
2388
+ return out.read_bytes() if r.returncode == 0 and out.exists() else None
2389
+
2390
+
2391
+ def finish_page(doc, meta, pages=None):
2392
+ """Render the page, check its coverage, and render again with the printed
2393
+ page offered wherever prose went missing. Returns (html, words, coverage)."""
2394
+ page, total = render(doc)
2395
+ cov = coverage(page, meta, pages, report=False)
2396
+ if cov and doc.pdf and shutil.which("pdftoppm") and any(b["t"] == "page" for b in doc.blocks):
2397
+ for p in weak_pages(cov):
2398
+ u = data_uri(page_image(doc.pdf, p) or b"")
2399
+ if u:
2400
+ doc.page_images[p] = u
2401
+ if doc.page_images:
2402
+ page, total = render(doc)
2403
+ return page, total, cov
2404
+
2405
+
2406
+ def report_coverage(cov, doc):
2407
+ if cov is None:
2408
+ print(" coverage: no source text to compare against")
2409
+ return
2410
+ print_coverage(cov)
2411
+ if doc.page_images:
2412
+ print(" original page offered for PDF pages %s, where text may be missing" % ", ".join(str(p) for p in sorted(doc.page_images)))
2413
+
2414
+
2415
+ # --------------------------------------------------------------------------
2416
+ # Commands
2417
+ # --------------------------------------------------------------------------
2418
+
2419
+ @functools.lru_cache(maxsize=1) # the same fonts for every page in a build
2420
+ def fonts_css():
2421
+ faces = []
2422
+ for style, f in (("normal", "literata-opsz-normal.woff2"), ("italic", "literata-opsz-italic.woff2")):
2423
+ p = FONTS / f
2424
+ if p.exists():
2425
+ faces.append("@font-face{font-family:'Literata';font-style:%s;font-weight:200 900;font-display:swap;"
2426
+ "src:url(data:font/woff2;base64,%s) format('woff2');}" % (style, base64.b64encode(p.read_bytes()).decode()))
2427
+ return "/* Literata, SIL Open Font License, via @fontsource-variable/literata@5.3.0 */\n" + "\n".join(faces)
2428
+
2429
+
2430
+ def write_assets(out=None):
2431
+ a = (out or OUT) / "assets"
2432
+ a.mkdir(parents=True, exist_ok=True)
2433
+ shutil.copyfile(str(HERE / "reader.css"), str(a / "reader.css"))
2434
+ shutil.copyfile(str(HERE / "reader.js"), str(a / "reader.js"))
2435
+ (a / "fonts.css").write_text(fonts_css())
2436
+
2437
+
2438
+ def inline_assets(page):
2439
+ """The page with its styles, script and fonts inside it, so it works as one file.
2440
+ The script stays allowed only by its hash, so the page still runs no other script."""
2441
+ # Plain ASCII throughout: a file sent by email can be re-encoded on the way
2442
+ # (Gmail turned one into Latin-1 on 19 November 2026), which garbles every
2443
+ # accent and breaks the script's hash. Escapes survive that.
2444
+ def js_ascii(s): # reader.js keeps its non-ASCII inside strings, where \u escapes mean the same
2445
+ return re.sub(r"[^\x00-\x7f]", lambda m: json.dumps(m.group())[1:-1], s) # \uXXXX, surrogate pairs included
2446
+ def css_ascii(s):
2447
+ return re.sub(r"[^\x00-\x7f]", lambda m: "\\%x " % ord(m.group()), s)
2448
+ page = page.encode("ascii", "xmlcharrefreplace").decode("ascii")
2449
+ js = js_ascii((HERE / "reader.js").read_text(encoding="utf-8"))
2450
+ css = css_ascii((HERE / "reader.css").read_text(encoding="utf-8"))
2451
+ fonts = css_ascii(fonts_css())
2452
+ for text in (js, css, fonts):
2453
+ if re.search(r"</(script|style)", text, re.I): # would end the inline block early
2454
+ raise ReaderError("an asset contains a closing script or style tag and can't be inlined")
2455
+ digest = base64.b64encode(hashlib.sha256(js.encode("utf-8")).digest()).decode()
2456
+ swaps = [
2457
+ ("script-src 'self' file:", "script-src 'sha256-%s'" % digest),
2458
+ ('<link rel="stylesheet" href="assets/fonts.css">', "<style>%s</style>" % fonts),
2459
+ ('<link rel="stylesheet" href="assets/reader.css">', "<style>%s</style>" % css),
2460
+ ('<script src="assets/reader.js"></script>', "<script>%s</script>" % js),
2461
+ ]
2462
+ for old, new in swaps:
2463
+ if page.count(old) != 1:
2464
+ raise ReaderError("page template changed; can't inline %s" % old)
2465
+ page = page.replace(old, new)
2466
+ return page
2467
+
2468
+
2469
+ def single_file_copy(doc):
2470
+ """A copy of the page that needs nothing beside it, to send to a phone."""
2471
+ doc.meta["single_file"] = True
2472
+ try:
2473
+ page, _ = render(doc)
2474
+ finally:
2475
+ doc.meta.pop("single_file", None)
2476
+ return inline_assets(page)
2477
+
2478
+
2479
+ def build_one(pid, ledger, route, pages=None, part=None, epub=False, single=False, place=None, members=None):
2480
+ """Returns ({"epub": path, "html": path, "page": html}, meta): the copies written this run, None where not written."""
2481
+ meta = ledger_meta(ledger, pid)
2482
+ d = SRC / slug(meta)
2483
+ if route == "arxiv":
2484
+ doc = arxiv_doc(meta, d)
2485
+ elif route == "jats":
2486
+ doc = jats_doc(meta, d)
2487
+ elif route == "pdf":
2488
+ doc = docling_doc(meta, d, pages)
2489
+ else:
2490
+ raise ReaderError("unknown route %s" % route)
2491
+ doc.meta["part"] = part
2492
+ doc.meta["place"] = place
2493
+ doc.meta["bundle_members"] = members
2494
+ page, total, cov = finish_page(doc, meta, pages)
2495
+ write_assets()
2496
+ name = slug(meta) + ".html"
2497
+ (OUT / name).write_text(page, encoding="utf-8")
2498
+ print("wrote %s (%d words, %d references, %d notes)" % (OUT / name, total, len(doc.refs), doc.nnotes))
2499
+ report_coverage(cov, doc)
2500
+ epub_path = write_epub_copy(page, meta, OUT / (slug(meta) + ".epub")) if epub else None
2501
+ single_path = write_single_copy(doc, OUT / "single-file" / name) if single else None
2502
+ return {"epub": epub_path, "html": single_path, "page": page}, doc.meta
2503
+
2504
+
2505
+ def load_list():
2506
+ if not LIST_FILE.exists():
2507
+ raise ReaderError("no %s" % LIST_FILE)
2508
+ return json.loads(LIST_FILE.read_text(encoding="utf-8"))
2509
+
2510
+
2511
+ def list_members(L):
2512
+ """The papers of one list that have a reading copy, for links between them."""
2513
+ out = []
2514
+ for k, it in enumerate(L["items"], 1):
2515
+ if not it.get("route"):
2516
+ continue
2517
+ try:
2518
+ meta = ledger_meta(it["ledger"], it["pid"])
2519
+ except ReaderError:
2520
+ continue
2521
+ out.append({"pid": it["pid"], "title": meta.get("title"), "doi": meta.get("doi"), "arxiv": meta.get("arxiv"),
2522
+ "href": slug(meta) + ".html", "n": it.get("n") or k, "total": len(L["items"])})
2523
+ return out
2524
+
2525
+
2526
+ def list_place(L, it, k):
2527
+ return (it.get("n") or k, len(L["items"]))
2528
+
2529
+
2530
+ EPUB_MODES = ("papers", "bundle", "both")
2531
+
2532
+
2533
+ def epub_choice(v):
2534
+ if v not in EPUB_MODES:
2535
+ raise argparse.ArgumentTypeError("choose from papers, bundle, both")
2536
+ return v
2537
+
2538
+
2539
+ def epub_mode(given):
2540
+ """Which EPUBs to write: None for none, or "papers", "bundle" or "both".
2541
+ A bare --epub (given as True: argparse would run a string const through
2542
+ the type check) means the settings file's `epub = "..."`, or else "papers"."""
2543
+ if given is not True:
2544
+ return given
2545
+ conf = read_settings(CONFIG) if CONFIG.is_file() else {}
2546
+ v = conf.get("epub") or "papers"
2547
+ if v not in EPUB_MODES:
2548
+ raise ReaderError('%s: epub must be "papers", "bundle" or "both", not "%s"' % (CONFIG, v))
2549
+ return v
2550
+
2551
+
2552
+ def cmd_build(a):
2553
+ lst = load_list() if LIST_FILE.exists() else {"lists": []}
2554
+ part = place = members = None
2555
+ for L in lst["lists"]:
2556
+ for k, it in enumerate(L["items"], 1):
2557
+ if it["pid"] == a.pid and place is None: # the first list it's on
2558
+ a.pages = a.pages or it.get("pages")
2559
+ part = it.get("part")
2560
+ place, members = list_place(L, it, k), list_members(L)
2561
+ mail_to = email_settings(a.email)
2562
+ copies, meta = build_one(a.pid, a.ledger, a.route, a.pages, part,
2563
+ a.epub or a.email == "epub", a.single_file or a.email == "html", place, members)
2564
+ index_error = None
2565
+ if LIST_FILE.exists():
2566
+ try:
2567
+ cmd_index(a)
2568
+ except ReaderError as e: # the paper is built, so still send it, then report this
2569
+ index_error = e
2570
+ if mail_to: # after the index, so a failed send leaves nothing else undone
2571
+ email_copy(mail_to, copies[a.email], meta)
2572
+ if index_error:
2573
+ raise index_error
2574
+
2575
+
2576
+ def cmd_all(a):
2577
+ mode = epub_mode(a.epub)
2578
+ lst = load_list()
2579
+ for L in lst["lists"]:
2580
+ members, built, missing = list_members(L), [], []
2581
+ for k, it in enumerate(L["items"], 1):
2582
+ if it.get("route"):
2583
+ try:
2584
+ copies, meta = build_one(it["pid"], it["ledger"], it["route"], it.get("pages"), it.get("part"),
2585
+ mode in ("papers", "both"), a.single_file, list_place(L, it, k), members)
2586
+ built.append((copies["page"], meta, slug(meta) + ".html"))
2587
+ except ReaderError as e:
2588
+ print("skipped %s: %s" % (it["pid"], e))
2589
+ try:
2590
+ missing.append(ledger_meta(it["ledger"], it["pid"]).get("title") or it["pid"])
2591
+ except ReaderError:
2592
+ missing.append(it["pid"])
2593
+ if mode in ("bundle", "both") and built:
2594
+ write_bundle_epub(built, L.get("name") or "Reading list", L.get("note") or "",
2595
+ OUT / (folder_name(L.get("name") or "reading-list") + ".epub"), missing)
2596
+ cmd_index(a)
2597
+
2598
+
2599
+ def paper_row(out, n, pid, meta, name, part=None, why_not=None):
2600
+ """One paper on an index page. `name` is its page's file name in `out`."""
2601
+ exists = bool(name) and (out / name).exists()
2602
+ authors = meta.get("authors") or []
2603
+ who = authors[0].split()[-1] + (" et al." if len(authors) > 2 else (" and " + authors[1].split()[-1] if len(authors) == 2 else "")) if authors else ""
2604
+ desc = " · ".join(esc(x) for x in [who, str(meta.get("year") or ""), meta.get("venue") or "", (part or "").split(":")[0]] if x)
2605
+ quotes = ""
2606
+ if exists:
2607
+ mm = re.search(r"about (\d+) min of reading", (out / name).read_text(encoding="utf-8"))
2608
+ if mm:
2609
+ desc += " · about %s min" % mm.group(1)
2610
+ title = '<a href="%s">%s</a>' % (attr(name), esc(meta["title"]))
2611
+ t = meta.get("title") or ""
2612
+ quotes = ' data-title="%s" data-cite="%s" data-reference="%s"' % (attr(t), attr(short_citation(meta)), attr(full_reference(meta, t)))
2613
+ else:
2614
+ title = esc(meta["title"])
2615
+ desc += (" · " if desc else "") + esc(why_not or "no reading copy yet")
2616
+ return '<li class="%s" data-pid="%s"%s><span class="n">%s</span><span class="t">%s</span><span class="s"></span><span class="d">%s</span></li>' % (
2617
+ "" if exists else "nofile", attr(pid), quotes, esc(str(n)), title, desc)
2618
+
2619
+
2620
+ def write_index(out, heading, sections, kicker, bar, note=""):
2621
+ """sections: (name, note, rows). A section without a name has no heading of its own."""
2622
+ write_assets(out)
2623
+ lists = "\n".join('<section class="list">%s%s<ol>%s</ol></section>' % (
2624
+ "<h2>%s</h2>" % esc(name) if name else "", '<p class="listnote">%s</p>' % esc(lnote) if lnote else "", "".join(rows))
2625
+ for name, lnote, rows in sections)
2626
+ page = INDEX.format(lists=lists, built=datetime.date.today().isoformat(), panel=PANEL, heading=esc(heading),
2627
+ bar=esc(bar), kicker=esc(kicker), note=esc(note + " ") if note else "")
2628
+ (out / "index.html").write_text(page, encoding="utf-8")
2629
+ print("wrote %s" % (out / "index.html"))
2630
+
2631
+
2632
+ def cmd_index(a):
2633
+ lst = load_list()
2634
+ sections = []
2635
+ for L in lst["lists"]:
2636
+ rows = []
2637
+ for it in L["items"]:
2638
+ try:
2639
+ meta = ledger_meta(it["ledger"], it["pid"])
2640
+ except ReaderError:
2641
+ meta = {"title": it["pid"], "authors": [], "year": "", "venue": "", "pid": it["pid"]}
2642
+ name = slug(meta) + ".html" if meta.get("authors") is not None else ""
2643
+ rows.append(paper_row(OUT, it.get("n", ""), it["pid"], meta, name, it.get("part"), it.get("why_not")))
2644
+ sections.append((L["name"], L.get("note", ""), rows))
2645
+ write_index(OUT, "Reading list", sections, INDEX_TEXT["index_kicker"], INDEX_TEXT["index_bar"], INDEX_TEXT["index_note"])
2646
+
2647
+
2648
+ INDEX = """<!doctype html>
2649
+ <html lang="en">
2650
+ <head>
2651
+ <meta charset="utf-8">
2652
+ <meta name="viewport" content="width=device-width, initial-scale=1">
2653
+ <meta name="generator" content="lit-reader index">
2654
+ <meta http-equiv="Content-Security-Policy" content="default-src 'none'; img-src data:; style-src 'self' 'unsafe-inline' file:; font-src data:; script-src 'self' file:">
2655
+ <title>{heading}</title>
2656
+ <link rel="stylesheet" href="assets/fonts.css">
2657
+ <link rel="stylesheet" href="assets/reader.css">
2658
+ </head>
2659
+ <body class="index">
2660
+ <div class="bar"><span class="where">{bar}</span><button id="aa" type="button" aria-label="Aa, reading settings" aria-expanded="false" aria-controls="panel">Aa</button></div>
2661
+ {panel}
2662
+ <div class="layout"><main>
2663
+ <header class="paper"><div class="kicker">{kicker}</div><h1>{heading}</h1>
2664
+ <div class="meta">{note}Built {built}.</div></header>
2665
+ {lists}
2666
+ <section class="bundle-quotes" id="bundle-quotes" aria-labelledby="bundle-quotes-h" hidden></section>
2667
+ </main></div>
2668
+ <script src="assets/reader.js"></script>
2669
+ </body>
2670
+ </html>
2671
+ """
2672
+
2673
+ # Pages also have to work where no script runs: an iPhone's preview of an email
2674
+ # attachment runs none. This small script marks the page as scripted before it
2675
+ # is drawn, so the styles can offer plain links instead of dead buttons when it
2676
+ # doesn't run. It takes the mark back if reader.js never ran (a page opened
2677
+ # without its assets folder), since the buttons would then be dead too.
2678
+ JS_FLAG = ('document.documentElement.classList.add("js");'
2679
+ 'document.addEventListener("DOMContentLoaded",function(){'
2680
+ 'if(!window.litReaderRan)document.documentElement.classList.remove("js")})')
2681
+
2682
+
2683
+ def with_js_flag(tpl):
2684
+ digest = base64.b64encode(hashlib.sha256(JS_FLAG.encode("ascii")).digest()).decode()
2685
+ for old, new in (("script-src 'self' file:", "script-src 'self' file: 'sha256-%s'" % digest),
2686
+ ("<title>", "<script>%s</script>\n<title>" % JS_FLAG.replace("{", "{{").replace("}", "}}"))): # the templates go through str.format
2687
+ if tpl.count(old) != 1: # not an assert: python -O would drop it, and pages would lose their script
2688
+ raise RuntimeError("page template changed; can't add the script flag at %r" % old)
2689
+ tpl = tpl.replace(old, new)
2690
+ return tpl
2691
+
2692
+
2693
+ PAGE = with_js_flag(PAGE)
2694
+ INDEX = with_js_flag(INDEX)
2695
+
2696
+
2697
+ # --------------------------------------------------------------------------
2698
+ # Standalone mode: one paper, no ledger
2699
+ # --------------------------------------------------------------------------
2700
+
2701
+ ARXIV_ID = r"(?:\d{4}\.\d{4,5}|[a-z-]+(?:\.[A-Z]{2})?/\d{7})(?:v\d+)?"
2702
+ DOI_RE = r"10\.\d{4,9}/[^\s\"<>]+"
2703
+ ATOM = "{http://www.w3.org/2005/Atom}"
2704
+ ARXIV_NS = "{http://arxiv.org/schemas/atom}"
2705
+ EPMC = "https://www.ebi.ac.uk/europepmc/webservices/rest/search?format=json&resultType=core&pageSize=5&query="
2706
+
2707
+
2708
+ def parse_input(text):
2709
+ """(kind, value) for a PDF path, arXiv ID or URL, DOI or DOI URL, PMCID or PMC URL."""
2710
+ t = text.strip()
2711
+ p = Path(t).expanduser()
2712
+ if p.is_file():
2713
+ return "pdf", p.resolve()
2714
+ if t.lower().endswith(".pdf") and not re.match(r"https?://", t, re.I):
2715
+ raise ReaderError("no such file: %s" % t)
2716
+ m = re.match(r"^(?:https?://)?(?:www\.|export\.)?arxiv\.org/(?:abs|pdf|html)/(%s)(?:\.pdf)?/?(?:[?#].*)?$" % ARXIV_ID, t, re.I)
2717
+ if m:
2718
+ return "arxiv", m.group(1)
2719
+ m = re.match(r"^arxiv:\s*(%s)$" % ARXIV_ID, t, re.I)
2720
+ if m or re.match(r"^%s$" % ARXIV_ID, t):
2721
+ return "arxiv", m.group(1) if m else t
2722
+ m = re.search(r"\b(PMC\d+)\b", t, re.I)
2723
+ if m and (t.upper() == m.group(1).upper() or re.match(r"^https?://([\w-]+\.)*(nih\.gov|europepmc\.org)/", t, re.I)):
2724
+ return "pmcid", m.group(1).upper()
2725
+ m = re.match(r"^(?:https?://(?:dx\.)?doi\.org/|doi:\s*)?(%s)$" % DOI_RE, t, re.I)
2726
+ if m:
2727
+ doi = urllib.parse.unquote(m.group(1))
2728
+ a = re.match(r"^10\.48550/arxiv\.(%s)$" % ARXIV_ID, doi, re.I)
2729
+ return ("arxiv", a.group(1)) if a else ("doi", doi)
2730
+ raise ReaderError("not a PDF, arXiv ID, DOI or PMCID: %s" % text)
2731
+
2732
+
2733
+ def read_json(body, service):
2734
+ try:
2735
+ return json.loads(body.decode("utf-8"))
2736
+ except ValueError:
2737
+ raise ReaderError("%s sent an unreadable reply" % service)
2738
+
2739
+
2740
+ def clean_title(t):
2741
+ return re.sub(r"\s+", " ", html.unescape(re.sub(r"<[^>]+>", "", t or ""))).strip().rstrip(".")
2742
+
2743
+
2744
+ def arxiv_meta(aid):
2745
+ body, _ = http_get("https://export.arxiv.org/api/query?id_list=%s" % urllib.parse.quote(aid, safe="/.v"), {"export.arxiv.org"})
2746
+ try:
2747
+ e = ET.fromstring(body).find(ATOM + "entry")
2748
+ except ET.ParseError:
2749
+ raise ReaderError("arXiv sent an unreadable reply for %s" % aid)
2750
+ if e is None or e.find(ATOM + "published") is None:
2751
+ raise ReaderError("arXiv has no paper %s" % aid)
2752
+ return {"pid": "arxiv:" + aid, "title": clean_title(e.findtext(ATOM + "title")),
2753
+ "authors": [a.findtext(ATOM + "name").strip() for a in e.findall(ATOM + "author")],
2754
+ "year": int(e.findtext(ATOM + "published")[:4]), "venue": (e.findtext(ARXIV_NS + "journal_ref") or "").strip() or None,
2755
+ "doi": (e.findtext(ARXIV_NS + "doi") or "").strip() or None, "arxiv": aid, "pmcid": None}
2756
+
2757
+
2758
+ def crossref_meta(doi):
2759
+ body, _ = http_get("https://api.crossref.org/works/%s" % urllib.parse.quote(doi, safe=""), {"api.crossref.org"})
2760
+ m = read_json(body, "Crossref")["message"]
2761
+ authors = [" ".join(x for x in (a.get("given"), a.get("family")) if x) or a.get("name", "") for a in m.get("author", [])]
2762
+ parts = ((m.get("issued") or {}).get("date-parts") or [[None]])[0]
2763
+ return {"pid": "doi:" + doi, "title": clean_title((m.get("title") or [""])[0]), "authors": [a for a in authors if a],
2764
+ "year": parts[0] if parts else None, "venue": clean_title((m.get("container-title") or [""])[0]) or None,
2765
+ "doi": doi, "arxiv": None, "pmcid": None}
2766
+
2767
+
2768
+ def epmc_record(query):
2769
+ body, _ = http_get(EPMC + urllib.parse.quote(query, safe=""), {"www.ebi.ac.uk"})
2770
+ res = read_json(body, "Europe PMC").get("resultList", {}).get("result", [])
2771
+ return res[0] if res else None
2772
+
2773
+
2774
+ def epmc_meta(r):
2775
+ authors = []
2776
+ for a in (r.get("authorList") or {}).get("author", []):
2777
+ name = " ".join(x for x in (a.get("firstName"), a.get("lastName")) if x) or a.get("collectiveName") or a.get("fullName")
2778
+ if name:
2779
+ authors.append(name)
2780
+ year = r.get("pubYear")
2781
+ return {"pid": "pmc:" + (r.get("pmcid") or r.get("id", "")), "title": clean_title(r.get("title")), "authors": authors,
2782
+ "year": int(year) if year and year.isdigit() else None,
2783
+ "venue": ((r.get("journalInfo") or {}).get("journal") or {}).get("title"),
2784
+ "doi": r.get("doi"), "arxiv": None, "pmcid": r.get("pmcid")}
2785
+
2786
+
2787
+ def need_tool(name, why):
2788
+ if not shutil.which(name):
2789
+ raise ReaderError("%s needs poppler's %s (macOS: brew install poppler)" % (why, name))
2790
+
2791
+
2792
+ def pdf_to_text(pdf, out):
2793
+ """pdftotext's reading of the PDF, or None without poppler."""
2794
+ if not shutil.which("pdftotext"):
2795
+ return None
2796
+ try:
2797
+ r = subprocess.run(["pdftotext", "-enc", "UTF-8", str(pdf), str(out)], stdout=subprocess.PIPE, stderr=subprocess.PIPE, timeout=180)
2798
+ except subprocess.TimeoutExpired:
2799
+ return None
2800
+ return out if r.returncode == 0 and out.exists() else None
2801
+
2802
+
2803
+ def on_first_page(title, first):
2804
+ norm = lambda x: re.sub(r"[^a-z0-9]", "", x.lower())
2805
+ return bool(norm(title)) and norm(title)[:60] in norm(first)
2806
+
2807
+
2808
+ def guess_title(pdf, first):
2809
+ try:
2810
+ r = subprocess.run(["pdfinfo", str(pdf)], stdout=subprocess.PIPE, stderr=subprocess.PIPE, timeout=60) if shutil.which("pdfinfo") else None
2811
+ except subprocess.TimeoutExpired:
2812
+ r = None
2813
+ m = re.search(r"^Title:\s*(.+)$", r.stdout.decode("utf-8", "replace"), re.M) if r and r.returncode == 0 else None
2814
+ t = m.group(1).strip() if m else ""
2815
+ if len(t.split()) >= 3 and not re.search(r"\.(pdf|docx?|tex|indd)$|microsoft word|untitled", t, re.I):
2816
+ return t
2817
+ for line in first.splitlines():
2818
+ line = line.strip()
2819
+ if len(line.split()) >= 4 and not re.search(r"https?:|doi|©|copyright|journal|vol\.|arxiv", line, re.I):
2820
+ return line
2821
+ return "Untitled paper"
2822
+
2823
+
2824
+ def pdf_meta(pdf, text, key):
2825
+ """Metadata for a local PDF. An identifier printed on its first page is
2826
+ trusted only if the record it leads to has this PDF's title."""
2827
+ first = text.read_text(encoding="utf-8", errors="replace").split("\f")[0] if text else ""
2828
+ m = re.search(r"arXiv:\s*(%s)" % ARXIV_ID, first)
2829
+ if m:
2830
+ try:
2831
+ meta = arxiv_meta(m.group(1))
2832
+ if on_first_page(meta["title"], first):
2833
+ return meta
2834
+ except ReaderError:
2835
+ pass
2836
+ for raw in re.findall(DOI_RE, first):
2837
+ doi = raw.rstrip(".,;)]")
2838
+ try:
2839
+ meta = crossref_meta(doi)
2840
+ except (ReaderError, KeyError, ValueError):
2841
+ continue
2842
+ if on_first_page(meta["title"], first):
2843
+ return meta
2844
+ year = re.search(r"\b(19[5-9]\d|20[0-4]\d)\b", first)
2845
+ print(" no record found for this PDF: title guessed from its first page, authors left blank")
2846
+ return {"pid": "pdf:" + key, "title": guess_title(pdf, first), "authors": [], "year": int(year.group(1)) if year else None,
2847
+ "venue": None, "doi": None, "arxiv": None, "pmcid": None}
2848
+
2849
+
2850
+ def jats_text(xml, out):
2851
+ """The article's own words, for the coverage check of a page built from XML."""
2852
+ skip = {"tex-math", "annotation", "annotation-xml", "graphic", "object-id", "label"}
2853
+ parts = []
2854
+
2855
+ def walk(e):
2856
+ if local(e.tag) in skip:
2857
+ return
2858
+ parts.append(e.text or "")
2859
+ for c in e:
2860
+ walk(c)
2861
+ parts.append(c.tail or "")
2862
+
2863
+ def outermost(e): # a reference list inside the body must not be counted twice
2864
+ if local(e.tag) in ("article-title", "abstract", "body", "ref-list", "ack", "app-group", "app") or (
2865
+ local(e.tag) in ("sec", "notes") and in_back[0]):
2866
+ walk(e)
2867
+ parts.append("\n")
2868
+ else:
2869
+ in_back[0] = in_back[0] or local(e.tag) == "back"
2870
+ for c in e:
2871
+ outermost(c)
2872
+ in_back = [False]
2873
+ outermost(parse_xml(xml))
2874
+ out.write_text(" ".join(parts), encoding="utf-8")
2875
+ return out
2876
+
2877
+
2878
+ def identify(kind, value):
2879
+ """(metadata, work folder, local PDF or None) for one input."""
2880
+ pdf = None
2881
+ if kind == "pdf":
2882
+ data = value.read_bytes()
2883
+ if b"%PDF-" not in data[:1024]:
2884
+ raise ReaderError("%s is not a PDF" % value)
2885
+ key = hashlib.sha256(data).hexdigest()[:16]
2886
+ d = WORK / "standalone" / ("pdf-" + key)
2887
+ (d / "in").mkdir(parents=True, exist_ok=True)
2888
+ pdf = d / "in" / "paper.pdf"
2889
+ pdf.write_bytes(data)
2890
+ meta = pdf_meta(pdf, pdf_to_text(pdf, d / "paper.txt"), key)
2891
+ elif kind == "arxiv":
2892
+ meta = arxiv_meta(value)
2893
+ elif kind == "pmcid":
2894
+ r = epmc_record("PMCID:%s" % value)
2895
+ if not r:
2896
+ raise ReaderError("Europe PMC has no record of %s" % value)
2897
+ meta = epmc_meta(r)
2898
+ meta["pmcid"] = value
2899
+ else:
2900
+ try:
2901
+ meta = crossref_meta(value)
2902
+ except (ReaderError, KeyError, ValueError):
2903
+ r = epmc_record('DOI:"%s"' % value)
2904
+ if not r:
2905
+ raise ReaderError("neither Crossref nor Europe PMC has a record of %s" % value)
2906
+ meta = epmc_meta(r)
2907
+ meta["doi"] = value
2908
+ if kind != "pdf":
2909
+ d = WORK / "standalone" / safe_id(meta["pid"])
2910
+ d.mkdir(parents=True, exist_ok=True)
2911
+ meta["standalone"] = True
2912
+ return meta, d, pdf
2913
+
2914
+
2915
+ def open_pmcid(meta):
2916
+ """The PMCID to fetch, if Europe PMC holds this paper's full text openly."""
2917
+ if meta.get("pmcid"):
2918
+ return meta["pmcid"]
2919
+ if not meta.get("doi"):
2920
+ return None
2921
+ r = epmc_record('DOI:"%s"' % meta["doi"]) # an outage is reported, not mistaken for "no open text"
2922
+ if r and r.get("pmcid") and r.get("isOpenAccess") == "Y" and r.get("inEPMC") == "Y":
2923
+ return r["pmcid"]
2924
+ return None
2925
+
2926
+
2927
+ def arxiv_pdf(aid, d):
2928
+ (d / "in").mkdir(parents=True, exist_ok=True)
2929
+ data, _ = http_get("https://arxiv.org/pdf/%s" % urllib.parse.quote(aid, safe="/.v"), {"arxiv.org"})
2930
+ if b"%PDF-" not in data[:1024]:
2931
+ raise ReaderError("arXiv did not return a PDF for %s" % aid)
2932
+ out = d / "in" / "paper.pdf"
2933
+ out.write_bytes(data)
2934
+ return out
2935
+
2936
+
2937
+ def take_route(want, meta, d, pdf, pages):
2938
+ """A local PDF is used as given, because it may be a particular version of
2939
+ the paper. Otherwise, or with --route online, the best source comes first:
2940
+ arXiv HTML, then Europe PMC XML, then ar5iv's conversion, then the PDF."""
2941
+ tried = []
2942
+ arxiv_has_none = False # ar5iv is tried only when arXiv answered that it has no HTML, never after an outage
2943
+ if want == "auto" and pdf is not None:
2944
+ order = ["pdf"]
2945
+ if meta.get("arxiv") or meta.get("doi"):
2946
+ print(" using your PDF. To try the online version of this paper instead, add --route online")
2947
+ elif want in ("auto", "online"):
2948
+ order = ["arxiv", "jats", "ar5iv", "pdf"] # the publisher's XML is a better copy than ar5iv's older conversion
2949
+ else:
2950
+ order = [want]
2951
+ for route in order:
2952
+ try:
2953
+ if route == "ar5iv" and not arxiv_has_none:
2954
+ continue
2955
+ if route in ("arxiv", "ar5iv") and meta.get("arxiv"):
2956
+ fetch_arxiv(meta["arxiv"], d, ("arxiv", "ar5iv") if want == "arxiv" else (route,))
2957
+ try:
2958
+ copy = pdf or arxiv_pdf(meta["arxiv"], d)
2959
+ meta["text_path"], meta["text_label"] = pdf_to_text(copy, d / "paper.txt"), "PDF's text"
2960
+ meta["pdf_path"], meta["pdf_href"] = copy, "https://arxiv.org/pdf/%s" % meta["arxiv"]
2961
+ except ReaderError as e:
2962
+ print(" coverage check unavailable: %s" % e)
2963
+ return "arxiv", arxiv_doc(meta, d)
2964
+ if route == "jats":
2965
+ pmcid = open_pmcid(meta)
2966
+ if pmcid:
2967
+ fetch_jats(pmcid, d)
2968
+ meta["text_path"], meta["text_label"] = jats_text(d / "jats.xml", d / "jats.txt"), "article's XML"
2969
+ return "jats", jats_doc(meta, d)
2970
+ if route == "pdf":
2971
+ if pdf is None and meta.get("arxiv"):
2972
+ pdf = arxiv_pdf(meta["arxiv"], d)
2973
+ if pdf is None:
2974
+ continue
2975
+ need_tool("pdftoppm", "The PDF route")
2976
+ need_tool("pdftotext", "The PDF route")
2977
+ meta["pdf_path"], meta["pdf_href"] = pdf, "paper.pdf"
2978
+ if meta.get("arxiv"):
2979
+ meta["url"] = "https://arxiv.org/abs/%s" % meta["arxiv"]
2980
+ meta["text_path"], meta["text_label"] = pdf_to_text(pdf, d / "paper.txt"), "PDF's text"
2981
+ if not sorted((d / "docling").glob("*.json")):
2982
+ run_docling(pdf, d, pages)
2983
+ return "pdf", docling_doc(meta, d, pages)
2984
+ except ReaderError as e:
2985
+ arxiv_has_none = arxiv_has_none or (route == "arxiv" and getattr(e, "no_html", False))
2986
+ tried.append("%s: %s" % (route, e))
2987
+ print(" %s route unavailable: %s" % (route, e))
2988
+ if want not in ("auto", "online") and not tried:
2989
+ raise ReaderError("the %s route doesn't apply to this input" % want)
2990
+ if tried:
2991
+ raise ReaderError("couldn't get the full text:\n %s\nTry again later, or download the PDF and run: lit-reader paper.pdf" % "\n ".join(tried))
2992
+ raise ReaderError("no open full text found. Download the PDF and run: lit-reader paper.pdf")
2993
+
2994
+
2995
+ def cmd_standalone(a):
2996
+ if a.pages and not re.fullmatch(r"\d+-\d+", a.pages):
2997
+ raise ReaderError("--pages takes a range such as 12-30")
2998
+ mail_to = email_settings(a.email)
2999
+ a.epub = a.epub or a.email == "epub" # the copy to send has to be written first
3000
+ a.single_file = a.single_file or a.email == "html"
3001
+ kind, value = parse_input(a.input)
3002
+ meta, d, pdf = identify(kind, value)
3003
+ if a.pages: # a page range is a different conversion of the same PDF
3004
+ d = d / ("pages-" + a.pages)
3005
+ d.mkdir(parents=True, exist_ok=True)
3006
+ name = re.sub(r"-+", "-", slug(meta)).strip("-") or "paper"
3007
+ out = Path(a.output).expanduser() if a.output else Path.cwd() / name
3008
+ target = out / "index.html"
3009
+ ours = target.is_file() and 'content="lit-reader"' in target.read_text(encoding="utf-8", errors="replace")[:600]
3010
+ if out.exists() and (not out.is_dir() or (any(out.iterdir()) and not ours)):
3011
+ raise ReaderError("%s already holds other files; choose an empty folder with -o" % out)
3012
+ route, doc = take_route(a.route, meta, d, pdf, a.pages)
3013
+ page, total, cov = finish_page(doc, doc.meta, a.pages)
3014
+ out.mkdir(parents=True, exist_ok=True)
3015
+ write_assets(out)
3016
+ if route == "pdf":
3017
+ shutil.copyfile(str(meta["pdf_path"]), str(out / "paper.pdf"))
3018
+ target.write_text(page, encoding="utf-8")
3019
+ print("wrote %s (route %s, %d words, %d references, %d notes)" % (target, route, total, len(doc.refs), doc.nnotes))
3020
+ report_coverage(cov, doc)
3021
+ copies = {"epub": write_epub_copy(page, doc.meta, out / (name + ".epub")) if a.epub else None,
3022
+ "html": write_single_copy(doc, out / (name + ".html")) if a.single_file else None}
3023
+ if mail_to:
3024
+ email_copy(mail_to, copies[a.email], doc.meta)
3025
+
3026
+
3027
+ # --------------------------------------------------------------------------
3028
+ # Bundles: several papers from a plain text file, with an index page
3029
+ # --------------------------------------------------------------------------
3030
+
3031
+ BUNDLE_OPTS = re.compile(r"(?:\s+(?:pages|route)=\S+)+\s*$")
3032
+ ROUTES = ("auto", "online", "arxiv", "jats", "pdf")
3033
+
3034
+
3035
+ def read_bundle(path):
3036
+ """A bundle file: `#` lines before the first paper give the title (the
3037
+ first) and a note (the rest); then one paper per line, as standalone mode
3038
+ takes it, optionally followed by pages=A-B and route=R. Relative PDF paths
3039
+ are read from the file's own folder."""
3040
+ try:
3041
+ lines = path.read_text(encoding="utf-8").splitlines()
3042
+ except (OSError, UnicodeDecodeError) as e:
3043
+ raise ReaderError("can't read the bundle file %s: %s" % (path, e))
3044
+ title, note, items = None, [], []
3045
+ for no, raw in enumerate(lines, 1):
3046
+ line = raw.strip()
3047
+ if not line:
3048
+ continue
3049
+ if line.startswith("#"):
3050
+ text = line.lstrip("#").strip()
3051
+ if not items and text: # comments among the papers are just comments
3052
+ if title is None:
3053
+ title = text
3054
+ else:
3055
+ note.append(text)
3056
+ continue
3057
+ opts = {}
3058
+ m = BUNDLE_OPTS.search(line)
3059
+ if m:
3060
+ for tok in m.group(0).split():
3061
+ k, v = tok.split("=", 1)
3062
+ opts[k] = v
3063
+ line = line[:m.start()].strip()
3064
+ if opts.get("pages") and not re.fullmatch(r"\d+-\d+", opts["pages"]):
3065
+ raise ReaderError("%s line %d: pages= takes a range such as 12-30" % (path, no))
3066
+ if opts.get("route") and opts["route"] not in ROUTES:
3067
+ raise ReaderError("%s line %d: route= must be one of %s" % (path, no, ", ".join(ROUTES)))
3068
+ p = Path(line).expanduser()
3069
+ if (not re.match(r"[a-z][a-z0-9+.-]*://", line, re.I) and not p.is_absolute()
3070
+ and ((path.parent / p).is_file() or line.lower().endswith(".pdf"))):
3071
+ line = str(path.parent / p)
3072
+ items.append({"input": line, "line": no, "pages": opts.get("pages"), "route": opts.get("route")})
3073
+ if not items:
3074
+ raise ReaderError("%s lists no papers" % path)
3075
+ return {"title": title or path.stem.replace("-", " ").replace("_", " "), "note": " ".join(note), "items": items}
3076
+
3077
+
3078
+ def folder_name(title):
3079
+ words_ = re.sub(r"[^a-z0-9]+", " ", unicodedata.normalize("NFKD", title).encode("ascii", "ignore").decode().lower()).split()
3080
+ return "-".join(words_[:6]) or "bundle"
3081
+
3082
+
3083
+ def write_bundle_epub(built, title, note, path, missing=()):
3084
+ """One EPUB holding every paper in `built`, a list of (page html, meta, page file name).
3085
+ `missing` names the papers that weren't converted, so the book can say so."""
3086
+ from .epub import write_bundle
3087
+ try:
3088
+ write_bundle(built, title, note, path, missing)
3089
+ except Exception as e: # the reading copies are already written
3090
+ print(" bundle EPUB not written (%s): %s" % (type(e).__name__, e))
3091
+ return None
3092
+ print("wrote %s (%d papers in one EPUB)" % (path, len(built)))
3093
+ return path
3094
+
3095
+
3096
+ WHAT_IT_IS = {"doi": "the DOI", "arxiv": "the arXiv ID", "pmcid": "the PMCID"}
3097
+
3098
+
3099
+ def bundle_failure(e, p, bundle_file):
3100
+ """(reason, what to do) for a paper of a bundle that wasn't converted. The
3101
+ advice is about the bundle file: a reading copy made on its own wouldn't
3102
+ join the bundle."""
3103
+ msg = str(e)
3104
+ where = "line %d of %s" % (p["line"], bundle_file.name)
3105
+ if p.get("kind") == "pdf": # the PDF is already there: what went wrong is on this machine or in the file
3106
+ said = next((l.strip()[5:].rstrip(":") for l in msg.splitlines() if l.strip().startswith("pdf: ")), None)
3107
+ if said and "needs poppler" in said:
3108
+ return said, "Install it, then run the command again."
3109
+ if said:
3110
+ return said, "Check that the PDF on %s opens, then run the command again." % where
3111
+ lines = [l.strip() for l in msg.splitlines() if l.strip()] or [msg]
3112
+ first = lines[1] if lines[0].endswith(":") and len(lines) > 1 else lines[0] # the first route's reason, not the header
3113
+ return first, "Check %s." % where
3114
+ instead = WHAT_IT_IS.get(p.get("kind"))
3115
+ swap = ("Download the PDF, put its path on %s in place of %s, and run the command again." % (where, instead) if instead
3116
+ else "Download the PDF, put its path on %s, and run the command again." % where)
3117
+ if msg.startswith("no open full text"):
3118
+ return "no open copy online", swap
3119
+ if msg.startswith("couldn't get the full text"):
3120
+ return "the online sources didn't give its full text", "Try again later. If it still fails: " + swap[0].lower() + swap[1:]
3121
+ return msg.splitlines()[0], "Check %s." % where
3122
+
3123
+
3124
+ def cmd_bundle(a):
3125
+ bundle_file = Path(a.file).expanduser().resolve()
3126
+ spec = read_bundle(bundle_file)
3127
+ mode = epub_mode(a.epub)
3128
+ out = Path(a.output).expanduser() if a.output else Path.cwd() / folder_name(spec["title"])
3129
+ index = out / "index.html"
3130
+ ours = index.is_file() and 'content="lit-reader index"' in index.read_text(encoding="utf-8", errors="replace")[:600]
3131
+ if out.exists() and (not out.is_dir() or (any(out.iterdir()) and not ours)):
3132
+ raise ReaderError("%s already holds other files; choose an empty folder with -o" % out)
3133
+ total = len(spec["items"])
3134
+
3135
+ # First every paper's record, so each page can link to the others
3136
+ papers, used = [], set()
3137
+ for n, it in enumerate(spec["items"], 1):
3138
+ print("[%d/%d] %s" % (n, total, it["input"]))
3139
+ p = dict(it, n=n)
3140
+ try:
3141
+ kind, value = parse_input(it["input"])
3142
+ p["kind"] = kind
3143
+ meta, d, pdf = identify(kind, value)
3144
+ except ReaderError as e:
3145
+ p["error"], p["fix"] = bundle_failure(e, p, bundle_file)
3146
+ print(" not converted: %s. %s" % (p["error"], p["fix"]))
3147
+ papers.append(p)
3148
+ continue
3149
+ if it["pages"]: # a page range is a different conversion of the same PDF
3150
+ d = d / ("pages-" + it["pages"])
3151
+ d.mkdir(parents=True, exist_ok=True)
3152
+ name = re.sub(r"-+", "-", slug(meta)).strip("-") or "paper"
3153
+ base, k = name, 2
3154
+ while name in used: # the same paper twice, say for two page ranges
3155
+ name, k = "%s-%d" % (base, k), k + 1
3156
+ used.add(name)
3157
+ # its own reading state: the same PDF's other ranges are other copies
3158
+ pid = "%s-pages-%s" % (meta["pid"], it["pages"]) if it["pages"] else meta["pid"]
3159
+ p.update(meta=meta, d=d, pdf=pdf, name=name, pid=pid)
3160
+ papers.append(p)
3161
+
3162
+ # Then every conversion, so the pages link only to papers that were converted
3163
+ for p in papers:
3164
+ if not p.get("meta"):
3165
+ continue
3166
+ print("[%d/%d] converting %s" % (p["n"], total, p["meta"].get("title") or p["input"]))
3167
+ meta = p["meta"]
3168
+ meta.update(bundle=True, place=(p["n"], total))
3169
+ try:
3170
+ p["route"], p["doc"] = take_route(p["route"] or a.route, meta, p["d"], p["pdf"], p["pages"])
3171
+ except ReaderError as e:
3172
+ p["error"], p["fix"] = bundle_failure(e, p, bundle_file)
3173
+ print(" not converted: %s. %s" % (p["error"], p["fix"]))
3174
+ continue
3175
+ p["doc"].meta["pid"] = p["pid"]
3176
+ done = [p for p in papers if p.get("doc")]
3177
+ members = [{"pid": p["doc"].meta["pid"], "title": p["meta"].get("title"), "doi": p["meta"].get("doi"),
3178
+ "arxiv": p["meta"].get("arxiv"), "href": p["name"] + ".html", "n": p["n"], "total": total}
3179
+ for p in done]
3180
+
3181
+ out.mkdir(parents=True, exist_ok=True)
3182
+ write_assets(out)
3183
+ built = []
3184
+ for p in done:
3185
+ doc, meta, route = p["doc"], p["meta"], p["route"]
3186
+ doc.meta["bundle_members"] = members
3187
+ if route == "pdf": # the page links to its own copy of the PDF, beside it
3188
+ doc.meta["pdf_href"] = p["name"] + ".pdf"
3189
+ shutil.copyfile(str(meta["pdf_path"]), str(out / (p["name"] + ".pdf")))
3190
+ page, words_, cov = finish_page(doc, doc.meta, p["pages"])
3191
+ (out / (p["name"] + ".html")).write_text(page, encoding="utf-8")
3192
+ print("wrote %s (route %s, %d words, %d references, %d notes)" % (out / (p["name"] + ".html"), route, words_, len(doc.refs), doc.nnotes))
3193
+ report_coverage(cov, doc)
3194
+ if mode in ("papers", "both"):
3195
+ write_epub_copy(page, doc.meta, out / (p["name"] + ".epub"))
3196
+ if a.single_file:
3197
+ write_single_copy(doc, out / "single-file" / (p["name"] + ".html"))
3198
+ built.append((page, doc.meta, p["name"] + ".html"))
3199
+ if mode in ("bundle", "both") and built:
3200
+ missing = [(p.get("meta") or {}).get("title") or p["input"] for p in papers if p.get("error")]
3201
+ write_bundle_epub(built, spec["title"], spec["note"], out / (folder_name(spec["title"]) + ".epub"), missing)
3202
+
3203
+ rows = []
3204
+ for p in papers:
3205
+ meta = (p["doc"].meta if p.get("doc") else p.get("meta")) or {"title": p["input"], "authors": [], "year": "", "venue": ""}
3206
+ why = ("not converted: %s. %s" % (p["error"], p["fix"])) if p.get("error") else None
3207
+ rows.append(paper_row(out, p["n"], p.get("pid") or "line-%d" % p["line"], meta,
3208
+ p["name"] + ".html" if p.get("name") and not p.get("error") else "", why_not=why))
3209
+ write_index(out, spec["title"], [("", spec["note"], rows)], "Bundle", "%d papers" % total)
3210
+ failed = [p for p in papers if p.get("error")]
3211
+ if failed:
3212
+ print("\n%d of %d papers weren't converted:" % (len(failed), total))
3213
+ for p in failed:
3214
+ print(" \u201c%s\u201d: %s. %s" % ((p.get("meta") or {}).get("title") or p["input"], p["error"], p["fix"]))
3215
+
3216
+
3217
+ def bundle_parser():
3218
+ ap = argparse.ArgumentParser(prog="lit-reader bundle",
3219
+ description="Build reading copies of several papers, with an index page, from a text file: "
3220
+ "one paper per line (a PDF path, arXiv ID or URL, DOI, or PMCID). "
3221
+ "Lines starting with # before the first paper give the bundle's title and a note.")
3222
+ ap.add_argument("file", help="the bundle file")
3223
+ ap.add_argument("-o", "--output", help="output folder (default: ./<title>/)")
3224
+ ap.add_argument("--route", choices=list(ROUTES), default="auto", help="source to use for every paper; a line's own route= wins")
3225
+ ap.add_argument("--epub", nargs="?", const=True, type=epub_choice, metavar="papers|bundle|both", help=EPUB_HELP)
3226
+ ap.add_argument("--single-file", action="store_true", help=SINGLE_HELP + ", in single-file/")
3227
+ return ap
3228
+
3229
+
3230
+ LEDGER_COMMANDS = ("fetch", "pdf-convert", "build", "index", "all")
3231
+
3232
+
3233
+ def write_epub_copy(page, meta, path):
3234
+ from .epub import write_epub # only needed when asked for
3235
+ try:
3236
+ write_epub(page, meta, path)
3237
+ except Exception as e: # the HTML copy is already written; one failed EPUB shouldn't stop a batch
3238
+ print(" EPUB not written (%s): %s" % (type(e).__name__, e))
3239
+ return None
3240
+ print("wrote %s" % path)
3241
+ return path
3242
+
3243
+
3244
+ def write_single_copy(doc, path):
3245
+ try:
3246
+ page = single_file_copy(doc)
3247
+ path.parent.mkdir(parents=True, exist_ok=True)
3248
+ path.write_text(page, encoding="utf-8")
3249
+ except Exception as e: # as with the EPUB: the main copy is already written, and one failure shouldn't stop a batch
3250
+ print(" single-file copy not written (%s): %s" % (type(e).__name__, e))
3251
+ return None
3252
+ print("wrote %s (one file, %.1f MB, for a phone)" % (path, len(page.encode("utf-8")) / 1e6))
3253
+ return path
3254
+
3255
+
3256
+ SINGLE_HELP = "also write a copy with everything inside one HTML file, to send to a phone"
3257
+ EPUB_HELP = ("also write EPUBs for an e-reader: papers (the default) writes one per paper, bundle one holding every paper, "
3258
+ "both writes both. A settings-file line epub = \"...\" sets the default")
3259
+ EMAIL_HELP = ("email a copy to yourself: the EPUB (the default, for Apple Books or a Kindle) or the single HTML file. "
3260
+ "Set it up first with: lit-reader setup-email")
3261
+
3262
+
3263
+ def email_settings(kind):
3264
+ """Checked before the build starts, so a long conversion never ends in a missing password."""
3265
+ if not kind:
3266
+ return None
3267
+ from . import mail
3268
+ try:
3269
+ return mail.settings(read_settings(CONFIG) if CONFIG.is_file() else {}, CONFIG)
3270
+ except mail.MailError as e:
3271
+ raise ReaderError(str(e))
3272
+
3273
+
3274
+ def email_copy(s, path, meta):
3275
+ from . import mail
3276
+ if path is None: # never fall back to a copy left by an earlier run
3277
+ raise ReaderError("the copy to email wasn't written this time, so nothing was emailed")
3278
+ title = re.sub(r"\s+", " ", strip_tags(meta.get("title_html") or esc(meta.get("title", "")))).strip() or path.stem
3279
+ try:
3280
+ mail.send(s, mail.message(s, path, title, full_reference(meta, title), __version__))
3281
+ except mail.MailError as e:
3282
+ raise ReaderError("the copies are written, but %s wasn't emailed: %s" % (path.name, e))
3283
+ print("emailed %s to %s" % (path.name, s["to"]))
3284
+
3285
+
3286
+ def standalone_parser():
3287
+ ap = argparse.ArgumentParser(prog="lit-reader", description="Turn one academic paper into an offline HTML reading copy.",
3288
+ epilog="Ledger commands: %s (each has its own --help). "
3289
+ "To set up --email: lit-reader setup-email" % ", ".join(LEDGER_COMMANDS))
3290
+ ap.add_argument("--version", action="version", version="lit-reader " + __version__)
3291
+ ap.add_argument("input", help="a PDF path, arXiv ID or URL, DOI, or PMCID")
3292
+ ap.add_argument("-o", "--output", help="output folder (default: ./<slug>/)")
3293
+ ap.add_argument("--route", choices=["auto", "online", "arxiv", "jats", "pdf"], default="auto",
3294
+ help="source to use. auto (default) keeps a local PDF as given and otherwise picks the best source; "
3295
+ "online prefers arXiv or Europe PMC even when you supply a PDF")
3296
+ ap.add_argument("--pages", help="PDF route only: convert these pages, for example 12-30")
3297
+ ap.add_argument("--epub", action="store_true", help="also write an EPUB copy for an e-reader")
3298
+ ap.add_argument("--single-file", action="store_true", help=SINGLE_HELP)
3299
+ ap.add_argument("--email", nargs="?", const="epub", choices=["epub", "html"], help=EMAIL_HELP)
3300
+ return ap
3301
+
3302
+
3303
+ def email_first(argv):
3304
+ """argv with a bare --email followed by something other than its own value
3305
+ written as --email=epub, so `lit-reader --email paper.pdf` doesn't take the
3306
+ paper for the copy type. A bare --epub of `bundle` or `all` moves to the end,
3307
+ where it keeps meaning "the default kind"."""
3308
+ out = list(argv)
3309
+ for i, v in enumerate(out[:-1]):
3310
+ if v == "--email" and out[i + 1] not in ("epub", "html"):
3311
+ out[i] = "--email=epub"
3312
+ if out[:1] in (["bundle"], ["all"]) and "--epub" in out[:-1] and out[out.index("--epub") + 1] not in EPUB_MODES:
3313
+ out.append(out.pop(out.index("--epub"))) # last, so it can't take the bundle file for its value
3314
+ return out
3315
+
3316
+
3317
+ def main(argv=None):
3318
+ argv = email_first(sys.argv[1:] if argv is None else argv)
3319
+ if argv[:1] == ["setup-email"]:
3320
+ argparse.ArgumentParser(prog="lit-reader setup-email",
3321
+ description="Set up --email: asks for your address and mail server, writes them to "
3322
+ "the settings file, stores the password in Keychain and sends a test.").parse_args(argv[1:])
3323
+ from . import mail
3324
+ try:
3325
+ return mail.setup(CONFIG, read_settings, __version__)
3326
+ except ReaderError as e: # an existing settings file that doesn't read
3327
+ print("error: %s" % e, file=sys.stderr)
3328
+ return 1
3329
+ if argv[:1] == ["bundle"]:
3330
+ try:
3331
+ cmd_bundle(bundle_parser().parse_args(argv[1:]))
3332
+ except ReaderError as e:
3333
+ print("error: %s" % e, file=sys.stderr)
3334
+ return 1
3335
+ return 0
3336
+ if not argv or argv[0] not in LEDGER_COMMANDS:
3337
+ try:
3338
+ cmd_standalone(standalone_parser().parse_args(argv))
3339
+ except ReaderError as e:
3340
+ print("error: %s" % e, file=sys.stderr)
3341
+ return 1
3342
+ return 0
3343
+ ap = argparse.ArgumentParser(prog="lit-reader", description=__doc__.split("\n\n")[0])
3344
+ sub = ap.add_subparsers(dest="cmd", required=True)
3345
+ f = sub.add_parser("fetch")
3346
+ f.add_argument("--pid", required=True)
3347
+ f.add_argument("--ledger", required=True)
3348
+ f.add_argument("--route", choices=["arxiv", "jats"], required=True)
3349
+ f.add_argument("--pmcid")
3350
+ c = sub.add_parser("pdf-convert")
3351
+ c.add_argument("--pid", required=True)
3352
+ c.add_argument("--ledger", required=True)
3353
+ c.add_argument("--pages")
3354
+ b = sub.add_parser("build")
3355
+ b.add_argument("--pid", required=True)
3356
+ b.add_argument("--ledger", required=True)
3357
+ b.add_argument("--route", choices=["arxiv", "jats", "pdf"], required=True)
3358
+ b.add_argument("--pages")
3359
+ b.add_argument("--epub", action="store_true")
3360
+ b.add_argument("--single-file", action="store_true", help=SINGLE_HELP + ", in single-file/")
3361
+ b.add_argument("--email", nargs="?", const="epub", choices=["epub", "html"], help=EMAIL_HELP)
3362
+ sub.add_parser("index")
3363
+ al = sub.add_parser("all")
3364
+ al.add_argument("--epub", nargs="?", const=True, type=epub_choice, metavar="papers|bundle|both", help=EPUB_HELP)
3365
+ al.add_argument("--single-file", action="store_true", help=SINGLE_HELP + ", in single-file/")
3366
+ a = ap.parse_args(argv)
3367
+ try:
3368
+ apply_settings()
3369
+ {"fetch": cmd_fetch, "pdf-convert": cmd_pdf_convert, "build": cmd_build, "index": cmd_index, "all": cmd_all}[a.cmd](a)
3370
+ except ReaderError as e:
3371
+ print("error: %s" % e, file=sys.stderr)
3372
+ return 1
3373
+ return 0
3374
+
3375
+
3376
+ if __name__ == "__main__":
3377
+ sys.exit(main())