lit-reader 1.2.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- lit_reader/__init__.py +3 -0
- lit_reader/__main__.py +5 -0
- lit_reader/core.py +3377 -0
- lit_reader/epub.py +419 -0
- lit_reader/fonts/OFL.txt +93 -0
- lit_reader/fonts/literata-opsz-italic.woff2 +0 -0
- lit_reader/fonts/literata-opsz-normal.woff2 +0 -0
- lit_reader/mail.py +240 -0
- lit_reader/reader.css +443 -0
- lit_reader/reader.js +899 -0
- lit_reader-1.2.0.dist-info/METADATA +269 -0
- lit_reader-1.2.0.dist-info/RECORD +16 -0
- lit_reader-1.2.0.dist-info/WHEEL +4 -0
- lit_reader-1.2.0.dist-info/entry_points.txt +2 -0
- lit_reader-1.2.0.dist-info/licenses/LICENSE +21 -0
- lit_reader-1.2.0.dist-info/licenses/src/lit_reader/fonts/OFL.txt +93 -0
lit_reader/core.py
ADDED
|
@@ -0,0 +1,3377 @@
|
|
|
1
|
+
#!/usr/bin/env python3
|
|
2
|
+
"""lit_reader.py: reading versions of academic papers as local HTML pages.
|
|
3
|
+
|
|
4
|
+
Each paper becomes one HTML page, styled for long reading: about 65 characters
|
|
5
|
+
a line, a choice of fonts, light, sepia or dark, a contents sidebar with time
|
|
6
|
+
left, citations that open as cards, notes and page numbers in the margins,
|
|
7
|
+
and a saved reading position.
|
|
8
|
+
|
|
9
|
+
The lit.py cache (~/.cache/lit) is never written to: the ledger's claims and
|
|
10
|
+
reads store character offsets into it. Sources fetched for reading live in
|
|
11
|
+
~/.cache/lit-reader/src/<slug>/.
|
|
12
|
+
|
|
13
|
+
Routes, best first:
|
|
14
|
+
arxiv arXiv's own HTML (LaTeXML), or ar5iv's when arXiv has none: real headings, linked citations.
|
|
15
|
+
jats Europe PMC full-text XML (JATS): the same, for PMC papers.
|
|
16
|
+
pdf Docling's JSON from the cached PDF, with figures and tables cropped
|
|
17
|
+
from the PDF page by pdftoppm. Run Docling first (see `pdf-convert`).
|
|
18
|
+
|
|
19
|
+
Standalone use needs no ledger:
|
|
20
|
+
lit-reader PDF | ARXIV-ID-OR-URL | DOI | PMCID [-o DIR] [--route R] [--pages A-B]
|
|
21
|
+
Metadata comes from arXiv, Crossref or Europe PMC, or failing those the PDF's
|
|
22
|
+
first page. The page is written to ./<slug>/index.html. A local PDF is always
|
|
23
|
+
the source unless --route online asks for arXiv's or Europe PMC's version.
|
|
24
|
+
|
|
25
|
+
Ledger commands read their paths from a settings file, by default
|
|
26
|
+
~/.config/lit-reader/config.toml (LIT_READER_CONFIG points elsewhere):
|
|
27
|
+
output_dir = "~/papers/reader" required
|
|
28
|
+
list_file = "~/papers/list.json" default: list.json in output_dir
|
|
29
|
+
lit_cache = "~/.cache/lit" optional
|
|
30
|
+
index_bar, index_kicker, index_note optional wording for the index page
|
|
31
|
+
[ledgers]
|
|
32
|
+
main = "~/papers/ledger.sqlite" names usable as --ledger or in the list
|
|
33
|
+
[email] for --email, in standalone mode too; see mail.py
|
|
34
|
+
to = "you@example.com"
|
|
35
|
+
smtp_host = "smtp.gmail.com"
|
|
36
|
+
|
|
37
|
+
fetch --route arxiv|jats --pid PID --ledger DB download the structured source (online)
|
|
38
|
+
pdf-convert --pid PID --ledger DB [--pages A-B] run Docling on the cached PDF (offline)
|
|
39
|
+
build --pid PID --ledger DB --route R [--pages A-B] write the page (offline)
|
|
40
|
+
index write index.html from list.json
|
|
41
|
+
all build every list.json entry that has a route
|
|
42
|
+
|
|
43
|
+
Every build prints a coverage check: the share of the cached pdftotext words
|
|
44
|
+
that reached the page, and the longest runs that did not. Those are the places
|
|
45
|
+
to look for dropped text.
|
|
46
|
+
|
|
47
|
+
SECURITY: fetched HTML, XML and PDFs are untrusted. Pages are built only from
|
|
48
|
+
whitelisted tags with escaped text; images are embedded only after checking
|
|
49
|
+
their magic bytes; PDFs are read only by Docling and pdftoppm, given paths as
|
|
50
|
+
arguments, never through a shell.
|
|
51
|
+
"""
|
|
52
|
+
import argparse
|
|
53
|
+
import base64
|
|
54
|
+
import datetime
|
|
55
|
+
import functools
|
|
56
|
+
import hashlib
|
|
57
|
+
import html
|
|
58
|
+
import io
|
|
59
|
+
import json
|
|
60
|
+
import os
|
|
61
|
+
import re
|
|
62
|
+
import shutil
|
|
63
|
+
import sqlite3
|
|
64
|
+
import subprocess
|
|
65
|
+
import sys
|
|
66
|
+
import tempfile
|
|
67
|
+
import time
|
|
68
|
+
import unicodedata
|
|
69
|
+
import urllib.error
|
|
70
|
+
import urllib.parse
|
|
71
|
+
import urllib.request
|
|
72
|
+
import xml.etree.ElementTree as ET
|
|
73
|
+
import zipfile
|
|
74
|
+
from collections import Counter
|
|
75
|
+
from html.parser import HTMLParser
|
|
76
|
+
from pathlib import Path
|
|
77
|
+
|
|
78
|
+
from . import __version__
|
|
79
|
+
|
|
80
|
+
HERE = Path(__file__).resolve().parent
|
|
81
|
+
HOME = Path.home()
|
|
82
|
+
LIT_CACHE = Path(os.environ.get("LIT_CACHE", "~/.cache/lit")).expanduser()
|
|
83
|
+
CONFIG = Path(os.environ.get("LIT_READER_CONFIG") or
|
|
84
|
+
Path(os.environ.get("XDG_CONFIG_HOME") or HOME / ".config") / "lit-reader" / "config.toml").expanduser()
|
|
85
|
+
WORK = HOME / ".cache" / "lit-reader"
|
|
86
|
+
SRC = WORK / "src"
|
|
87
|
+
FONTS = HERE / "fonts"
|
|
88
|
+
OUT = None # the settings below fill these in for the ledger commands
|
|
89
|
+
LIST_FILE = None
|
|
90
|
+
LEDGERS = {}
|
|
91
|
+
INDEX_TEXT = {"index_bar": "Reading copies", "index_kicker": "Papers", "index_note": ""}
|
|
92
|
+
WPM = 200 # must match reader.js
|
|
93
|
+
UA = "lit-reader/%s (offline reading tool)" % __version__
|
|
94
|
+
|
|
95
|
+
|
|
96
|
+
class ReaderError(Exception):
|
|
97
|
+
status = None # the HTTP status, when a server answered with an error
|
|
98
|
+
|
|
99
|
+
|
|
100
|
+
def read_settings(path):
|
|
101
|
+
"""The small part of TOML the settings file uses: comments, [sections] and
|
|
102
|
+
key = "string" lines. (tomllib needs Python 3.11; this tool supports 3.9.)"""
|
|
103
|
+
out = {}
|
|
104
|
+
section = out
|
|
105
|
+
for no, raw in enumerate(path.read_text(encoding="utf-8").splitlines(), 1):
|
|
106
|
+
line = raw.strip()
|
|
107
|
+
if not line or line.startswith("#"):
|
|
108
|
+
continue
|
|
109
|
+
m = re.fullmatch(r"\[([A-Za-z0-9_-]+)\]", line)
|
|
110
|
+
if m:
|
|
111
|
+
section = out.setdefault(m.group(1), {})
|
|
112
|
+
continue
|
|
113
|
+
m = re.fullmatch(r"""([A-Za-z0-9_-]+)\s*=\s*(?:"([^"\\]*)"|'([^']*)')\s*(?:#.*)?""", line)
|
|
114
|
+
if not m:
|
|
115
|
+
raise ReaderError("%s line %d: expected key = \"value\"" % (path, no))
|
|
116
|
+
section[m.group(1)] = m.group(2) if m.group(2) is not None else m.group(3)
|
|
117
|
+
return out
|
|
118
|
+
|
|
119
|
+
|
|
120
|
+
def apply_settings():
|
|
121
|
+
"""Load the paths the ledger commands need. Standalone builds never call this."""
|
|
122
|
+
global OUT, LIST_FILE, LIT_CACHE
|
|
123
|
+
conf = read_settings(CONFIG) if CONFIG.is_file() else {}
|
|
124
|
+
out = os.environ.get("LIT_READER_OUT") or conf.get("output_dir")
|
|
125
|
+
if not out:
|
|
126
|
+
raise ReaderError("ledger commands need a settings file with output_dir: %s" % CONFIG)
|
|
127
|
+
OUT = Path(out).expanduser()
|
|
128
|
+
LIST_FILE = Path(conf["list_file"]).expanduser() if conf.get("list_file") else OUT / "list.json"
|
|
129
|
+
if conf.get("lit_cache") and not os.environ.get("LIT_CACHE"):
|
|
130
|
+
LIT_CACHE = Path(conf["lit_cache"]).expanduser()
|
|
131
|
+
LEDGERS.update(conf.get("ledgers") or {})
|
|
132
|
+
INDEX_TEXT.update({k: conf[k] for k in INDEX_TEXT if k in conf})
|
|
133
|
+
|
|
134
|
+
|
|
135
|
+
def esc(s):
|
|
136
|
+
return html.escape(s, quote=False)
|
|
137
|
+
|
|
138
|
+
|
|
139
|
+
def attr(s):
|
|
140
|
+
return html.escape(s, quote=True)
|
|
141
|
+
|
|
142
|
+
|
|
143
|
+
def safe_id(s):
|
|
144
|
+
return re.sub(r"[^A-Za-z0-9_-]", "-", s)
|
|
145
|
+
|
|
146
|
+
|
|
147
|
+
def words(text):
|
|
148
|
+
return len(re.findall(r"\w+", text))
|
|
149
|
+
|
|
150
|
+
|
|
151
|
+
def strip_tags(h):
|
|
152
|
+
return html.unescape(re.sub(r"<[^>]+>", " ", h))
|
|
153
|
+
|
|
154
|
+
|
|
155
|
+
# --------------------------------------------------------------------------
|
|
156
|
+
# A small DOM, shared by every route
|
|
157
|
+
# --------------------------------------------------------------------------
|
|
158
|
+
|
|
159
|
+
VOID = {"area", "base", "br", "col", "embed", "hr", "img", "input", "link", "meta",
|
|
160
|
+
"source", "track", "wbr", "mspace", "none", "mprescripts"}
|
|
161
|
+
|
|
162
|
+
|
|
163
|
+
class Node(object):
|
|
164
|
+
__slots__ = ("tag", "attrs", "kids", "parent")
|
|
165
|
+
|
|
166
|
+
def __init__(self, tag, attrs=None, parent=None):
|
|
167
|
+
self.tag = tag
|
|
168
|
+
self.attrs = dict(attrs or {})
|
|
169
|
+
self.kids = []
|
|
170
|
+
self.parent = parent
|
|
171
|
+
|
|
172
|
+
def cls(self):
|
|
173
|
+
return set((self.attrs.get("class") or "").split())
|
|
174
|
+
|
|
175
|
+
def text(self):
|
|
176
|
+
return "".join(k if isinstance(k, str) else k.text() for k in self.kids)
|
|
177
|
+
|
|
178
|
+
def iter(self):
|
|
179
|
+
yield self
|
|
180
|
+
for k in self.kids:
|
|
181
|
+
if isinstance(k, Node):
|
|
182
|
+
for d in k.iter():
|
|
183
|
+
yield d
|
|
184
|
+
|
|
185
|
+
def find(self, pred):
|
|
186
|
+
for d in self.iter():
|
|
187
|
+
if d is not self and pred(d):
|
|
188
|
+
return d
|
|
189
|
+
return None
|
|
190
|
+
|
|
191
|
+
def findall(self, pred):
|
|
192
|
+
return [d for d in self.iter() if d is not self and pred(d)]
|
|
193
|
+
|
|
194
|
+
def children(self):
|
|
195
|
+
return [k for k in self.kids if isinstance(k, Node)]
|
|
196
|
+
|
|
197
|
+
|
|
198
|
+
class DomBuilder(HTMLParser):
|
|
199
|
+
def __init__(self):
|
|
200
|
+
HTMLParser.__init__(self, convert_charrefs=True)
|
|
201
|
+
self.root = Node("#root")
|
|
202
|
+
self.cur = self.root
|
|
203
|
+
|
|
204
|
+
def handle_starttag(self, tag, attrs):
|
|
205
|
+
n = Node(tag, [(k, v or "") for k, v in attrs], self.cur)
|
|
206
|
+
self.cur.kids.append(n)
|
|
207
|
+
if tag not in VOID:
|
|
208
|
+
self.cur = n
|
|
209
|
+
|
|
210
|
+
def handle_startendtag(self, tag, attrs):
|
|
211
|
+
self.cur.kids.append(Node(tag, [(k, v or "") for k, v in attrs], self.cur))
|
|
212
|
+
|
|
213
|
+
def handle_endtag(self, tag):
|
|
214
|
+
c = self.cur
|
|
215
|
+
while c is not self.root and c.tag != tag:
|
|
216
|
+
c = c.parent
|
|
217
|
+
if c is not self.root:
|
|
218
|
+
self.cur = c.parent
|
|
219
|
+
|
|
220
|
+
def handle_data(self, data):
|
|
221
|
+
self.cur.kids.append(data)
|
|
222
|
+
|
|
223
|
+
|
|
224
|
+
def parse_html(text):
|
|
225
|
+
b = DomBuilder()
|
|
226
|
+
b.feed(text)
|
|
227
|
+
b.close()
|
|
228
|
+
return b.root
|
|
229
|
+
|
|
230
|
+
|
|
231
|
+
def local(name):
|
|
232
|
+
return name.rsplit("}", 1)[-1].rsplit(":", 1)[-1]
|
|
233
|
+
|
|
234
|
+
|
|
235
|
+
def et_to_node(e, parent=None):
|
|
236
|
+
n = Node(local(e.tag), {local(k): v for k, v in e.attrib.items()}, parent)
|
|
237
|
+
if e.text:
|
|
238
|
+
n.kids.append(e.text)
|
|
239
|
+
for c in e:
|
|
240
|
+
n.kids.append(et_to_node(c, n))
|
|
241
|
+
if c.tail:
|
|
242
|
+
n.kids.append(c.tail)
|
|
243
|
+
return n
|
|
244
|
+
|
|
245
|
+
|
|
246
|
+
# --------------------------------------------------------------------------
|
|
247
|
+
# Safe output: only these tags and attributes ever reach a page
|
|
248
|
+
# --------------------------------------------------------------------------
|
|
249
|
+
|
|
250
|
+
DROP = {"script", "style", "annotation", "annotation-xml", "svg", "iframe", "object", "embed",
|
|
251
|
+
"noscript", "template", "button", "form", "input", "textarea", "select", "link", "meta",
|
|
252
|
+
"img", "video", "audio", "canvas", "tex-math", "inline-graphic", "head", "title"}
|
|
253
|
+
SIMPLE = {"em": "em", "i": "em", "italic": "em", "strong": "strong", "b": "strong", "bold": "strong",
|
|
254
|
+
"sup": "sup", "sub": "sub", "code": "code", "monospace": "code", "sc": "span", "u": "u",
|
|
255
|
+
"underline": "u", "small": "small"}
|
|
256
|
+
MATHML = {
|
|
257
|
+
"math": ("display",), "mi": ("mathvariant",), "mo": ("stretchy", "fence", "separator", "form", "lspace", "rspace", "largeop", "movablelimits"),
|
|
258
|
+
"mn": (), "mrow": (), "msub": (), "msup": (), "msubsup": (), "mfrac": ("linethickness",),
|
|
259
|
+
"msqrt": (), "mroot": (), "mover": ("accent",), "munder": ("accentunder",), "munderover": (),
|
|
260
|
+
"mtext": (), "mspace": ("width",), "mtable": (), "mtr": (), "mtd": ("columnspan",), "semantics": (),
|
|
261
|
+
"mstyle": ("displaystyle", "scriptlevel"), "mpadded": (), "mphantom": (), "mmultiscripts": (),
|
|
262
|
+
"mprescripts": (), "none": (), "menclose": ("notation",), "ms": (),
|
|
263
|
+
}
|
|
264
|
+
TABLE = {"table": (), "thead": (), "tbody": (), "tfoot": (), "tr": (), "th": ("colspan", "rowspan"),
|
|
265
|
+
"td": ("colspan", "rowspan"), "caption": ()}
|
|
266
|
+
ATTR_OK = re.compile(r"^[\w .%#:-]{0,40}$")
|
|
267
|
+
|
|
268
|
+
|
|
269
|
+
def math_html(n):
|
|
270
|
+
"""MathML, whitelisted; annotations (LaTeX source) dropped."""
|
|
271
|
+
if isinstance(n, str):
|
|
272
|
+
return esc(n)
|
|
273
|
+
if n.tag in DROP:
|
|
274
|
+
return ""
|
|
275
|
+
inner = "".join(math_html(k) for k in n.kids)
|
|
276
|
+
if n.tag not in MATHML:
|
|
277
|
+
return inner
|
|
278
|
+
attrs = "".join(' %s="%s"' % (a, attr(n.attrs[a])) for a in MATHML[n.tag]
|
|
279
|
+
if a in n.attrs and ATTR_OK.match(n.attrs[a]))
|
|
280
|
+
if n.tag == "math":
|
|
281
|
+
return '<math%s>%s</math>' % (attrs, inner)
|
|
282
|
+
return "<%s%s>%s</%s>" % (n.tag, attrs, inner, n.tag)
|
|
283
|
+
|
|
284
|
+
|
|
285
|
+
def safe_href(h):
|
|
286
|
+
h = (h or "").strip()
|
|
287
|
+
if h.startswith("#"):
|
|
288
|
+
return "#" + safe_id(h[1:])
|
|
289
|
+
try:
|
|
290
|
+
p = urllib.parse.urlparse(h)
|
|
291
|
+
except ValueError: # a malformed link in the paper is dropped, not fatal
|
|
292
|
+
return None
|
|
293
|
+
if p.scheme in ("http", "https") and p.netloc:
|
|
294
|
+
return h
|
|
295
|
+
return None
|
|
296
|
+
|
|
297
|
+
|
|
298
|
+
class Inline(object):
|
|
299
|
+
"""Turns a node's children into safe inline HTML. A route subclasses
|
|
300
|
+
`rule` to handle its own citation, note and link markup."""
|
|
301
|
+
|
|
302
|
+
def __init__(self, doc):
|
|
303
|
+
self.doc = doc
|
|
304
|
+
|
|
305
|
+
def run(self, node):
|
|
306
|
+
return "".join(self.one(k) for k in node.kids)
|
|
307
|
+
|
|
308
|
+
def rule(self, n):
|
|
309
|
+
return None
|
|
310
|
+
|
|
311
|
+
def one(self, k):
|
|
312
|
+
if isinstance(k, str):
|
|
313
|
+
return esc(k)
|
|
314
|
+
r = self.rule(k)
|
|
315
|
+
if r is not None:
|
|
316
|
+
return r
|
|
317
|
+
t = k.tag
|
|
318
|
+
if t in DROP:
|
|
319
|
+
return ""
|
|
320
|
+
if t in SIMPLE:
|
|
321
|
+
inner = self.run(k)
|
|
322
|
+
if not inner.strip():
|
|
323
|
+
return inner
|
|
324
|
+
tag = SIMPLE[t]
|
|
325
|
+
if t == "sc":
|
|
326
|
+
return '<span style="font-variant:small-caps">%s</span>' % inner
|
|
327
|
+
return "<%s>%s</%s>" % (tag, inner, tag)
|
|
328
|
+
if t in ("br", "break"):
|
|
329
|
+
return "<br>"
|
|
330
|
+
if t == "math":
|
|
331
|
+
return math_html(k)
|
|
332
|
+
if t in ("a", "ext-link", "uri"):
|
|
333
|
+
href = safe_href(k.attrs.get("href") or k.attrs.get("xlink:href") or k.attrs.get("href", ""))
|
|
334
|
+
inner = self.run(k)
|
|
335
|
+
if href and not href.startswith("#"):
|
|
336
|
+
return '<a href="%s" rel="noopener noreferrer">%s</a>' % (attr(href), inner)
|
|
337
|
+
if href:
|
|
338
|
+
return '<a href="%s">%s</a>' % (attr(href), inner)
|
|
339
|
+
return inner
|
|
340
|
+
if t in TABLE:
|
|
341
|
+
attrs = "".join(' %s="%s"' % (a, attr(k.attrs[a])) for a in TABLE[t]
|
|
342
|
+
if a in k.attrs and re.match(r"^\d{1,2}$", k.attrs[a]))
|
|
343
|
+
cls = ' class="data"' if t == "table" else ""
|
|
344
|
+
return "<%s%s%s>%s</%s>" % (t, cls, attrs, self.run(k), t)
|
|
345
|
+
if t in ("ul", "ol", "li", "list", "list-item"):
|
|
346
|
+
tag = {"list": "ul", "list-item": "li"}.get(t, t)
|
|
347
|
+
return "<%s>%s</%s>" % (tag, self.run(k), tag)
|
|
348
|
+
if t == "p" or (t == "div" and k.parent is not None and k.parent.tag in ("td", "th", "li", "list-item")):
|
|
349
|
+
return self.run(k) + " "
|
|
350
|
+
return self.run(k)
|
|
351
|
+
|
|
352
|
+
|
|
353
|
+
# --------------------------------------------------------------------------
|
|
354
|
+
# The document model and the page renderer
|
|
355
|
+
# --------------------------------------------------------------------------
|
|
356
|
+
|
|
357
|
+
class Doc(object):
|
|
358
|
+
def __init__(self, meta):
|
|
359
|
+
self.meta = meta # title, authors, year, venue, doi, pid, url
|
|
360
|
+
self.abstracts = [] # (heading, html)
|
|
361
|
+
self.blocks = [] # dicts, see render()
|
|
362
|
+
self.refs = [] # (id, html)
|
|
363
|
+
self.ref_ids = set()
|
|
364
|
+
self.nnotes = 0
|
|
365
|
+
self.source = "" # one line: where the text came from
|
|
366
|
+
self.pdf = None # Path of the cached PDF, for page links
|
|
367
|
+
self.page_images = {} # PDF page -> image of the printed page, offered where text may be missing
|
|
368
|
+
|
|
369
|
+
def note(self, inner_html, shown=None):
|
|
370
|
+
"""A sidenote. `shown` keeps the number the paper itself gives the note."""
|
|
371
|
+
self.nnotes += 1
|
|
372
|
+
n = self.nnotes
|
|
373
|
+
shown = shown or n
|
|
374
|
+
return ('<span class="sn-mark" data-for="sn-%d" role="button" tabindex="0" aria-label="note %d">%d</span>'
|
|
375
|
+
'<span class="sidenote" id="sn-%d"><span class="n">%d</span>%s</span>') % (n, shown, shown, n, shown, inner_html)
|
|
376
|
+
|
|
377
|
+
def add(self, **b):
|
|
378
|
+
self.blocks.append(b)
|
|
379
|
+
|
|
380
|
+
def cite(self, ids, inner):
|
|
381
|
+
ids = [safe_id(i) for i in ids if i]
|
|
382
|
+
if not ids:
|
|
383
|
+
return inner
|
|
384
|
+
return '<a class="cite" href="#ref-%s" data-ref="%s">%s</a>' % (ids[0], attr(" ".join(ids)), inner)
|
|
385
|
+
|
|
386
|
+
|
|
387
|
+
AUTHORS_SHOWN = 3 # a list longer than AUTHORS_MAX opens with this many names
|
|
388
|
+
AUTHORS_MAX = 5
|
|
389
|
+
|
|
390
|
+
|
|
391
|
+
def surname(name):
|
|
392
|
+
name = name.strip()
|
|
393
|
+
if "," in name:
|
|
394
|
+
return name.split(",", 1)[0].strip()
|
|
395
|
+
parts = [p for p in name.split() if p.rstrip(".,").lower() not in ("jr", "sr", "ii", "iii", "iv")]
|
|
396
|
+
if not parts:
|
|
397
|
+
return name
|
|
398
|
+
i = len(parts) - 1
|
|
399
|
+
while i > 1 and parts[i - 1].lower() in ("van", "von", "der", "den", "de", "del", "della", "di", "da", "du", "la", "le", "dos", "das", "ter"):
|
|
400
|
+
i -= 1 # "Johannes van der Waals" is cited as "van der Waals"
|
|
401
|
+
return " ".join(parts[i:])
|
|
402
|
+
|
|
403
|
+
|
|
404
|
+
def short_citation(m):
|
|
405
|
+
"""Author-year, as a quote from this paper would be cited in the text."""
|
|
406
|
+
a = [surname(x) for x in (m.get("authors") or []) if x.strip()]
|
|
407
|
+
if a:
|
|
408
|
+
who = a[0] if len(a) == 1 else "%s and %s" % (a[0], a[1]) if len(a) == 2 else "%s et al." % a[0]
|
|
409
|
+
else: # no authors on record (a local PDF with no DOI): cite by a short title, not as anonymous
|
|
410
|
+
t = (m.get("title") or "Untitled").split()
|
|
411
|
+
who = "“%s”" % (" ".join(t[:5]) + ("…" if len(t) > 5 else ""))
|
|
412
|
+
return "%s, %s" % (who, m.get("year") or "n.d.")
|
|
413
|
+
|
|
414
|
+
|
|
415
|
+
def full_reference(m, title):
|
|
416
|
+
"""One plain line for a reference list: authors (year). Title. Venue. Link."""
|
|
417
|
+
a = [x.strip() for x in (m.get("authors") or []) if x.strip()]
|
|
418
|
+
who = ", ".join(a[:20]) + (" et al." if len(a) > 20 else "") if a else ""
|
|
419
|
+
end = lambda t: t if t.endswith(("?", "!")) else t.rstrip(".") + "." # "Lonely?" keeps its own mark
|
|
420
|
+
bits = ["%s (%s)." % (who, m.get("year") or "n.d.")] if who else []
|
|
421
|
+
bits.append(end(title.strip()) + ("" if who else " (%s)." % (m.get("year") or "n.d.")))
|
|
422
|
+
if m.get("venue"):
|
|
423
|
+
bits.append(end(m["venue"].strip()))
|
|
424
|
+
if m.get("doi"):
|
|
425
|
+
bits.append("https://doi.org/" + m["doi"])
|
|
426
|
+
elif m.get("arxiv"):
|
|
427
|
+
bits.append("https://arxiv.org/abs/" + m["arxiv"])
|
|
428
|
+
elif m.get("url"):
|
|
429
|
+
bits.append(m["url"])
|
|
430
|
+
return " ".join(bits)
|
|
431
|
+
|
|
432
|
+
|
|
433
|
+
def authors_html(authors):
|
|
434
|
+
"""The byline. A long author list starts collapsed, so the paper comes first."""
|
|
435
|
+
if len(authors) <= AUTHORS_MAX:
|
|
436
|
+
return '<p class="authors">%s</p>' % esc(", ".join(authors))
|
|
437
|
+
rest = len(authors) - AUTHORS_SHOWN
|
|
438
|
+
return ('<p class="authors">%s<span class="au-rest" id="au-rest" hidden>, %s</span> '
|
|
439
|
+
'<button class="au-toggle" id="au-toggle" type="button" aria-expanded="false" aria-controls="au-rest" data-more="Show %d more">'
|
|
440
|
+
'<span class="au-label">Show %d more</span><span class="au-chevron" aria-hidden="true"></span></button></p>') % (
|
|
441
|
+
esc(", ".join(authors[:AUTHORS_SHOWN])), esc(", ".join(authors[AUTHORS_SHOWN:])), rest, rest)
|
|
442
|
+
|
|
443
|
+
|
|
444
|
+
def image_size(src):
|
|
445
|
+
"""(width, height) of an embedded image, so the page doesn't shift as images load."""
|
|
446
|
+
try:
|
|
447
|
+
d = base64.b64decode(src.split(",", 1)[1])
|
|
448
|
+
except (IndexError, ValueError):
|
|
449
|
+
return None
|
|
450
|
+
if d[:8] == b"\x89PNG\r\n\x1a\n" and len(d) >= 24:
|
|
451
|
+
return int.from_bytes(d[16:20], "big"), int.from_bytes(d[20:24], "big")
|
|
452
|
+
if d[:3] == b"GIF" and len(d) >= 10:
|
|
453
|
+
return int.from_bytes(d[6:8], "little"), int.from_bytes(d[8:10], "little")
|
|
454
|
+
i = 2
|
|
455
|
+
while d[:2] == b"\xff\xd8" and i + 9 < len(d):
|
|
456
|
+
if d[i] != 0xFF:
|
|
457
|
+
return None
|
|
458
|
+
m = d[i + 1]
|
|
459
|
+
if 0xC0 <= m <= 0xCF and m not in (0xC4, 0xC8, 0xCC):
|
|
460
|
+
return int.from_bytes(d[i + 7:i + 9], "big"), int.from_bytes(d[i + 5:i + 7], "big")
|
|
461
|
+
i += 2 + int.from_bytes(d[i + 2:i + 4], "big")
|
|
462
|
+
return None
|
|
463
|
+
|
|
464
|
+
|
|
465
|
+
def img_tag(src, alt):
|
|
466
|
+
alt = "" if (alt or "").strip().lower() == "refer to caption" else (alt or "") # arXiv's placeholder; the caption follows
|
|
467
|
+
size = image_size(src)
|
|
468
|
+
dims = ' width="%d" height="%d"' % size if size else ""
|
|
469
|
+
return '<img src="%s" alt="%s"%s loading="lazy">' % (src, attr(alt), dims)
|
|
470
|
+
|
|
471
|
+
|
|
472
|
+
def heading_text(h):
|
|
473
|
+
"""A heading's words, without any footnote attached to it."""
|
|
474
|
+
def words_of(n):
|
|
475
|
+
if isinstance(n, str):
|
|
476
|
+
return n
|
|
477
|
+
return "" if n.cls() & {"sn-mark", "sidenote"} else "".join(words_of(k) for k in n.kids)
|
|
478
|
+
return words_of(parse_html(h))
|
|
479
|
+
|
|
480
|
+
|
|
481
|
+
def pdf_href(doc):
|
|
482
|
+
"""Standalone pages link to their own copy of the PDF, so no local path is written into the page.
|
|
483
|
+
A single-file copy travels without the PDF, so it links only to a PDF online (None otherwise)."""
|
|
484
|
+
href = doc.meta.get("pdf_href") or doc.pdf.as_uri()
|
|
485
|
+
if doc.meta.get("single_file") and not href.startswith(("https://", "http://")):
|
|
486
|
+
return None
|
|
487
|
+
return href
|
|
488
|
+
|
|
489
|
+
|
|
490
|
+
def pdf_link(doc, page):
|
|
491
|
+
if not doc.pdf or not page or not pdf_href(doc):
|
|
492
|
+
return ""
|
|
493
|
+
return '<a class="pdf-link" href="%s#page=%d" target="_blank" rel="noopener">open page %d of the PDF</a>' % (attr(pdf_href(doc)), page, page)
|
|
494
|
+
|
|
495
|
+
|
|
496
|
+
def render(doc):
|
|
497
|
+
"""Doc -> page HTML."""
|
|
498
|
+
m = doc.meta
|
|
499
|
+
out, toc = [], []
|
|
500
|
+
used = set()
|
|
501
|
+
sec_id = ""
|
|
502
|
+
body = True
|
|
503
|
+
bid = 0
|
|
504
|
+
# words per heading, for the contents sidebar
|
|
505
|
+
heads = [i for i, b in enumerate(doc.blocks) if b["t"] == "h"]
|
|
506
|
+
sec_words = {}
|
|
507
|
+
for n, i in enumerate(heads):
|
|
508
|
+
lvl = doc.blocks[i]["level"]
|
|
509
|
+
w = 0
|
|
510
|
+
for b in doc.blocks[i + 1:]:
|
|
511
|
+
if b["t"] == "h" and b["level"] <= lvl:
|
|
512
|
+
break
|
|
513
|
+
w += b.get("_w", 0) or words(strip_tags(block_text(b)))
|
|
514
|
+
sec_words[i] = w
|
|
515
|
+
|
|
516
|
+
labels = dict((b["pdf"], b["label"]) for b in doc.blocks if b["t"] == "page" and b.get("pdf"))
|
|
517
|
+
shown = set()
|
|
518
|
+
|
|
519
|
+
def original_pages(upto):
|
|
520
|
+
"""The printed page, behind a toggle, for each flagged page up to `upto`."""
|
|
521
|
+
res = []
|
|
522
|
+
for p in sorted(doc.page_images):
|
|
523
|
+
if p in shown or (upto is not None and p > upto):
|
|
524
|
+
continue
|
|
525
|
+
shown.add(p)
|
|
526
|
+
name = ("p. %s" % labels[p]) if labels.get(p) else ("page %d of the PDF" % p)
|
|
527
|
+
res.append('<details class="orig-page"><summary>Show the original page (%s)</summary>'
|
|
528
|
+
'<p class="orig-note">Some text on this page may not have come through. This is the page as printed.</p>%s</details>'
|
|
529
|
+
% (esc(name), img_tag(doc.page_images[p], "The original %s" % name)))
|
|
530
|
+
return "".join(res)
|
|
531
|
+
|
|
532
|
+
for i, b in enumerate(doc.blocks):
|
|
533
|
+
t = b["t"]
|
|
534
|
+
bid += 1
|
|
535
|
+
if t == "h":
|
|
536
|
+
hid = b.get("id") or "s-" + safe_id(heading_text(b["html"]).strip().lower())[:40]
|
|
537
|
+
while hid in used:
|
|
538
|
+
hid += "-"
|
|
539
|
+
used.add(hid)
|
|
540
|
+
sec_id = hid
|
|
541
|
+
if b.get("appendix") and body:
|
|
542
|
+
body = False
|
|
543
|
+
out.append(DONE_SLOT) # the main text ends here, and so does a reader
|
|
544
|
+
out.append('<div class="appendix-start"></div>')
|
|
545
|
+
lvl = min(max(b["level"], 2), 4)
|
|
546
|
+
h_html = re.sub(r"\.\s*$", "", b["html"].strip()) # run-in heads: "Control Variables." (the doc stays as it was: it may be rendered twice)
|
|
547
|
+
title = heading_text(h_html).strip()
|
|
548
|
+
no = '<span class="secno">%s</span>' % esc(b["no"]) if b.get("no") else ""
|
|
549
|
+
out.append('<h%d id="%s" data-sec="1" data-title="%s">%s%s</h%d>' % (
|
|
550
|
+
lvl, hid, attr(((b.get("no") or "") + " " + title).strip()), no, h_html, lvl))
|
|
551
|
+
if lvl <= 3:
|
|
552
|
+
mins = max(1, round(sec_words.get(i, 0) / float(WPM)))
|
|
553
|
+
toc.append((lvl, hid, ((b.get("no") or "") + " " + title).strip(), mins))
|
|
554
|
+
continue
|
|
555
|
+
w = words(strip_tags(block_text(b)))
|
|
556
|
+
common = 'id="b%d" data-w="%d" data-in="%s" data-body="%d"' % (bid, w, sec_id, 1 if body else 0)
|
|
557
|
+
if t == "p":
|
|
558
|
+
out.append("<p %s>%s</p>" % (common, chunked(b["html"])))
|
|
559
|
+
elif t == "page":
|
|
560
|
+
if doc.page_images and b.get("pdf"):
|
|
561
|
+
out.append(original_pages(b["pdf"] - 1)) # flagged pages that have no marker of their own
|
|
562
|
+
out.append('<span class="pg" id="pg-%s">%s</span>' % (
|
|
563
|
+
safe_id(str(b["label"])), (pdf_anchor(doc, b) or "p. " + esc(str(b["label"])))))
|
|
564
|
+
if doc.page_images and b.get("pdf"):
|
|
565
|
+
out.append(original_pages(b["pdf"]))
|
|
566
|
+
elif t == "list":
|
|
567
|
+
tag = "ol" if b.get("ordered") else "ul"
|
|
568
|
+
out.append('<%s %s%s>%s</%s>' % (tag, common, ' class="plain"' if b.get("plain") else "", "".join("<li>%s</li>" % x for x in b["items"]), tag))
|
|
569
|
+
elif t == "eq":
|
|
570
|
+
out.append('<div class="eq" %s>%s</div>' % (common, b["html"]))
|
|
571
|
+
elif t == "pre":
|
|
572
|
+
out.append('<pre class="listing" %s>%s</pre>' % (common, esc(b["text"])))
|
|
573
|
+
elif t == "fig":
|
|
574
|
+
parts = []
|
|
575
|
+
for src, alt in b.get("imgs") or []:
|
|
576
|
+
parts.append(img_tag(src, alt))
|
|
577
|
+
if b.get("table"):
|
|
578
|
+
parts.append('<div class="tablewrap" role="region" tabindex="0" aria-label="%s">%s</div>' % (attr((b.get("label") or "Table").rstrip(":. ")), b["table"]))
|
|
579
|
+
if not parts:
|
|
580
|
+
parts.append('<div class="missing-figure">This %s did not come through in the source. %s</div>' % (
|
|
581
|
+
"table" if b.get("kind") == "table" else "figure", pdf_link(doc, b.get("page")) or ""))
|
|
582
|
+
cap = ""
|
|
583
|
+
if b.get("label") or b.get("caption"):
|
|
584
|
+
link = pdf_link(doc, b.get("page")) if (b.get("imgs") or b.get("table")) else ""
|
|
585
|
+
cap = "<figcaption>%s%s%s</figcaption>" % (
|
|
586
|
+
'<span class="label">%s</span>' % esc(b["label"]) if b.get("label") else "",
|
|
587
|
+
b.get("caption") or "", (" " + link) if link else "")
|
|
588
|
+
after = ""
|
|
589
|
+
if b.get("table_text"): # Docling's reading of a table shown as an image; after the figure, so the caption stays its last child
|
|
590
|
+
after = ('<details class="table-text"><summary>%s as text</summary>'
|
|
591
|
+
'<p class="orig-note">From the conversion and not checked by hand: compare it with the image above before relying on it.</p>'
|
|
592
|
+
'<div class="tablewrap" role="region" tabindex="0" aria-label="%s as text">%s</div></details>') % (
|
|
593
|
+
esc((b.get("label") or "Table").rstrip(":. ")), attr((b.get("label") or "Table").rstrip(":. ")), b["table_text"])
|
|
594
|
+
fid = ' data-fig="%s"' % attr(b["id"]) if b.get("id") else ""
|
|
595
|
+
out.append('<figure %s%s>%s%s</figure>%s' % (common.replace('id="b%d"' % bid, 'id="%s"' % (b.get("id") or "b%d" % bid)), fid, "".join(parts), cap, after))
|
|
596
|
+
|
|
597
|
+
if doc.page_images:
|
|
598
|
+
out.append(original_pages(None))
|
|
599
|
+
|
|
600
|
+
# references
|
|
601
|
+
refs_html = ""
|
|
602
|
+
if doc.refs:
|
|
603
|
+
linked = {} if m.get("single_file") else bundle_links(doc.refs, m) # a single file travels without the other pages
|
|
604
|
+
items = "".join('<li id="ref-%s"%s>%s%s</li>' % (safe_id(i), ' value="%s"' % i if str(i).isdigit() else "", h, linked.get(i, ""))
|
|
605
|
+
for i, h in doc.refs)
|
|
606
|
+
refs_html = ('<section class="references" id="references" data-title="References">'
|
|
607
|
+
'<h2 id="references-start" data-sec="1" data-title="References" data-w="0" data-in="references" data-body="0">References</h2>'
|
|
608
|
+
'<details open><summary>%d references (citations in the text open these as cards)</summary><ol>%s</ol></details></section>') % (len(doc.refs), items)
|
|
609
|
+
toc.append((2, "references", "References", 0))
|
|
610
|
+
|
|
611
|
+
def chunk_ps(a):
|
|
612
|
+
return re.sub(r"<p>(.*?)</p>", lambda m: "<p>%s</p>" % chunked(m.group(1), 40), a, flags=re.S)
|
|
613
|
+
abstracts = "".join('<section class="abstract" id="abstract-%d"><h2>%s</h2>%s</section>' % (n + 1, esc(h), chunk_ps(a))
|
|
614
|
+
for n, (h, a) in enumerate(doc.abstracts))
|
|
615
|
+
total_words = sum(words(strip_tags(block_text(b))) for b in doc.blocks if b["t"] != "h")
|
|
616
|
+
authors = m.get("authors") or []
|
|
617
|
+
meta_bits = [x for x in [esc(m.get("venue") or ""), str(m.get("year") or "")] if x]
|
|
618
|
+
doi = m.get("doi")
|
|
619
|
+
if doi:
|
|
620
|
+
meta_bits.append('<a href="https://doi.org/%s" target="_blank" rel="noopener">doi:%s</a>' % (attr(doi), esc(doi)))
|
|
621
|
+
elif m.get("url"):
|
|
622
|
+
meta_bits.append('<a href="%s" target="_blank" rel="noopener">%s</a>' % (attr(m["url"]), esc(m["url"])))
|
|
623
|
+
main_mins = max(1, round(main_words(doc) / float(WPM)))
|
|
624
|
+
meta_bits.append("about %d min of reading in the main text" % main_mins)
|
|
625
|
+
|
|
626
|
+
alone = bool((m.get("standalone") and not m.get("bundle")) or m.get("single_file")) # a single file has no list of papers beside it
|
|
627
|
+
back = '<ol class="back"><li><a href="#top">↑ Back to the start</a></li>%s</ol>' % (
|
|
628
|
+
"" if alone else '<li><a href="index.html">← All papers</a></li>')
|
|
629
|
+
# on a phone the top bar holds only the contents button and where you are; the rest waits here
|
|
630
|
+
tools = '<div class="toc-tools">%s<button type="button" data-proxy="clips">Quotes<span class="clip-count-n"></span></button>' \
|
|
631
|
+
'<button type="button" data-proxy="aa" aria-label="Aa, reading settings">Aa</button></div>' % (
|
|
632
|
+
"" if alone else all_link(m.get("place")).replace('class="btn"', 'class="btn tools-all"', 1))
|
|
633
|
+
toc_html = tools + '<div class="toc-head">Contents</div><ol>%s</ol>%s' % ("".join(
|
|
634
|
+
'<li class="l%d"><a href="#%s"><span>%s</span><span class="mins">%s</span></a></li>' % (
|
|
635
|
+
l, h, esc(t), ("%d min" % mins) if mins else "")
|
|
636
|
+
for l, h, t, mins in toc), back)
|
|
637
|
+
|
|
638
|
+
title_text = strip_tags(m.get("title_html") or esc(m.get("title", ""))).strip()
|
|
639
|
+
page = PAGE.format(
|
|
640
|
+
title=esc(title_text), pid=attr(m.get("pid", "")),
|
|
641
|
+
where="About %d min of reading" % main_mins, # the script keeps this up to date as you read
|
|
642
|
+
cite=attr(short_citation(m)), reference=attr(full_reference(m, title_text)),
|
|
643
|
+
kicker=esc(" · ".join(x for x in [m.get("kicker")] if x) or "Reading copy"),
|
|
644
|
+
title_html=m.get("title_html") or esc(m.get("title", "")),
|
|
645
|
+
part=('<p class="part">%s</p>' % esc(m["part"])) if m.get("part") else "",
|
|
646
|
+
authors=authors_html(authors), meta=" · ".join(meta_bits),
|
|
647
|
+
abstracts=abstracts, body="\n".join(out), refs=refs_html, toc=toc_html,
|
|
648
|
+
source=esc(doc.source), built=datetime.date.today().isoformat(),
|
|
649
|
+
panel=PANEL, all_link="" if alone else all_link(m.get("place")), done="" if alone else DONE_SLOT)
|
|
650
|
+
# the finished prompt goes where the main text ends: before any appendix, or else before the references
|
|
651
|
+
page = page.replace(DONE_SLOT, "" if alone else DONE_BOX, 1).replace(DONE_SLOT, "")
|
|
652
|
+
return page, total_words
|
|
653
|
+
|
|
654
|
+
|
|
655
|
+
ABBR = {"al", "e.g", "i.e", "eg", "ie", "fig", "figs", "eq", "eqs", "vs", "cf", "approx", "no", "nos", "pp", "p",
|
|
656
|
+
"st", "dr", "mr", "ms", "mrs", "prof", "inc", "ltd", "jr", "sr", "etc", "ca", "vol", "ch", "sec", "tab", "ref", "refs"}
|
|
657
|
+
|
|
658
|
+
|
|
659
|
+
def chunks(h, target):
|
|
660
|
+
"""Split one paragraph of safe HTML into runs of whole sentences of about
|
|
661
|
+
`target` words, cutting only between tags (never inside a citation, note
|
|
662
|
+
or emphasis). Returns a list of HTML strings; joined, they are `h` again."""
|
|
663
|
+
parts = re.split(r"(<[^>]+>)", h)
|
|
664
|
+
cuts, depth, pos = [], 0, 0
|
|
665
|
+
for seg in parts:
|
|
666
|
+
if seg.startswith("<"):
|
|
667
|
+
if not re.match(r"<(br|img)\b", seg):
|
|
668
|
+
depth += -1 if seg.startswith("</") else (0 if seg.endswith("/>") else 1)
|
|
669
|
+
elif depth == 0:
|
|
670
|
+
for m in re.finditer(r"([.?!])([”\"’)]*)(\s+)", seg):
|
|
671
|
+
before = seg[:m.start()]
|
|
672
|
+
word = re.search(r"([\w.]+)$", before)
|
|
673
|
+
w = word.group(1).lower() if word else ""
|
|
674
|
+
if w in ABBR or re.fullmatch(r"[a-z]", w) or re.fullmatch(r"(?:[a-z]\.)+[a-z]", w):
|
|
675
|
+
continue # et al. / e.g. / Fig. / an initial
|
|
676
|
+
nxt = seg[m.end():m.end() + 1] or "<"
|
|
677
|
+
if not (nxt.isupper() or nxt.isdigit() or nxt in "“\"‘'(<["):
|
|
678
|
+
continue
|
|
679
|
+
cuts.append(pos + m.end())
|
|
680
|
+
pos += len(seg)
|
|
681
|
+
if not cuts:
|
|
682
|
+
return [h]
|
|
683
|
+
out, start, acc = [], 0, 0
|
|
684
|
+
total = words(strip_tags(h))
|
|
685
|
+
for c in cuts:
|
|
686
|
+
piece = h[start:c]
|
|
687
|
+
n = words(strip_tags(piece))
|
|
688
|
+
rest = total - acc - n
|
|
689
|
+
if n >= target * 0.6 and rest >= target * 0.5:
|
|
690
|
+
out.append(piece)
|
|
691
|
+
acc += n
|
|
692
|
+
start = c
|
|
693
|
+
out.append(h[start:])
|
|
694
|
+
return out
|
|
695
|
+
|
|
696
|
+
|
|
697
|
+
def chunked(h, target=70):
|
|
698
|
+
cs = chunks(h, target)
|
|
699
|
+
if len(cs) == 1:
|
|
700
|
+
return h
|
|
701
|
+
return "".join('<span class="chunk">%s</span>' % c for c in cs)
|
|
702
|
+
|
|
703
|
+
|
|
704
|
+
def pdf_anchor(doc, b):
|
|
705
|
+
if doc.pdf and b.get("pdf") and pdf_href(doc):
|
|
706
|
+
return '<a href="%s#page=%d" target="_blank" rel="noopener" title="Printed page %s. Opens page %d of the PDF.">p. %s</a>' % (
|
|
707
|
+
attr(pdf_href(doc)), b["pdf"], attr(str(b["label"])), b["pdf"], esc(str(b["label"])))
|
|
708
|
+
return None
|
|
709
|
+
|
|
710
|
+
|
|
711
|
+
def block_text(b):
|
|
712
|
+
t = b["t"]
|
|
713
|
+
if t in ("p", "eq", "h"):
|
|
714
|
+
return b["html"]
|
|
715
|
+
if t == "list":
|
|
716
|
+
return " ".join(b["items"])
|
|
717
|
+
if t == "pre":
|
|
718
|
+
return b["text"]
|
|
719
|
+
if t == "fig":
|
|
720
|
+
return (b.get("caption") or "") + " " + (b.get("table") or "")
|
|
721
|
+
return ""
|
|
722
|
+
|
|
723
|
+
|
|
724
|
+
def main_words(doc):
|
|
725
|
+
n = 0
|
|
726
|
+
for b in doc.blocks:
|
|
727
|
+
if b["t"] == "h" and b.get("appendix"):
|
|
728
|
+
break
|
|
729
|
+
if b["t"] != "h":
|
|
730
|
+
n += words(strip_tags(block_text(b)))
|
|
731
|
+
return n
|
|
732
|
+
|
|
733
|
+
|
|
734
|
+
PANEL = """<div class="panel" id="panel" role="group" aria-label="Reading settings" hidden>
|
|
735
|
+
<div class="row" role="group" aria-label="Theme"><span>Theme</span><span class="seg">
|
|
736
|
+
<button data-pref="theme" data-val="auto">Auto</button><button data-pref="theme" data-val="light">Light</button>
|
|
737
|
+
<button data-pref="theme" data-val="sepia">Sepia</button><button data-pref="theme" data-val="dark">Dark</button></span></div>
|
|
738
|
+
<div class="row" role="group" aria-label="Font"><span>Font</span><span class="seg">
|
|
739
|
+
<button data-pref="font" data-val="literata" style="font-family:Literata">Literata</button>
|
|
740
|
+
<button data-pref="font" data-val="iowan" style="font-family:'Iowan Old Style'">Iowan</button>
|
|
741
|
+
<button data-pref="font" data-val="charter" style="font-family:Charter">Charter</button>
|
|
742
|
+
<button data-pref="font" data-val="sans">Sans</button></span></div>
|
|
743
|
+
<div class="row" role="group" aria-label="Size"><span>Size</span><span class="seg">
|
|
744
|
+
<button id="size-down" aria-label="A− smaller text">A−</button><span id="size-now" aria-live="polite" style="padding:.2rem .3rem">20px</span>
|
|
745
|
+
<button id="size-up" aria-label="A+ larger text">A+</button></span></div>
|
|
746
|
+
<div class="row row-width" role="group" aria-label="Line length"><span>Line length</span><span class="seg">
|
|
747
|
+
<button data-pref="width" data-val="narrow">Narrow</button><button data-pref="width" data-val="medium">Medium</button>
|
|
748
|
+
<button data-pref="width" data-val="wide">Wide</button></span></div>
|
|
749
|
+
<div class="row" role="group" aria-label="Paragraphs"><span>Paragraphs</span><span class="seg">
|
|
750
|
+
<button data-pref="para" data-val="short">Shorter</button><button data-pref="para" data-val="orig">As published</button></span></div>
|
|
751
|
+
<div class="row" role="group" aria-label="Line spacing"><span>Line spacing</span><span class="seg">
|
|
752
|
+
<button data-pref="lh" data-val="tight">1.4</button><button data-pref="lh" data-val="normal">1.5</button>
|
|
753
|
+
<button data-pref="lh" data-val="loose">1.7</button></span></div>
|
|
754
|
+
<div class="note">People read fastest in different fonts, and liking a font doesn't predict it (Wallace et al., 2022). It's worth trying each one for a page.</div>
|
|
755
|
+
</div>"""
|
|
756
|
+
|
|
757
|
+
def all_link(place=None):
|
|
758
|
+
"""The way back to the list, with the paper's place in it ("3 of 7") when it has one."""
|
|
759
|
+
if not place:
|
|
760
|
+
return '<a class="btn" href="index.html" aria-label="All papers"><span aria-hidden="true">←</span><span class="lbl"> All papers</span></a>'
|
|
761
|
+
n, total = place
|
|
762
|
+
return ('<a class="btn" href="index.html" aria-label="All papers. This is paper %d of %d">'
|
|
763
|
+
'<span aria-hidden="true">←</span><span class="lbl"> All papers</span>'
|
|
764
|
+
'<span class="place" aria-hidden="true">%d of %d</span></a>') % (n, total, n, total)
|
|
765
|
+
|
|
766
|
+
|
|
767
|
+
def norm_words(s):
|
|
768
|
+
return re.sub(r"[^a-z0-9]", "", unicodedata.normalize("NFKD", s or "").encode("ascii", "ignore").decode().lower())
|
|
769
|
+
|
|
770
|
+
|
|
771
|
+
def same_paper(ref_html, other):
|
|
772
|
+
"""Whether a reference-list entry is `other`: by DOI or arXiv ID where the
|
|
773
|
+
entry gives one, otherwise by the other paper's title appearing in it."""
|
|
774
|
+
text = html.unescape(strip_tags(ref_html))
|
|
775
|
+
dois = set(d.rstrip(".,;)]").lower() for d in re.findall(DOI_RE, text))
|
|
776
|
+
if other.get("doi") and other["doi"].lower() in dois:
|
|
777
|
+
return True
|
|
778
|
+
if other.get("arxiv"):
|
|
779
|
+
aid = re.sub(r"v\d+$", "", other["arxiv"])
|
|
780
|
+
if re.search(r"(?<![\d.])%s(?:v\d+)?(?![\d])" % re.escape(aid), text):
|
|
781
|
+
return True
|
|
782
|
+
title = norm_words(other.get("title"))
|
|
783
|
+
return len(title) >= 20 and title[:60] in norm_words(text) # short titles ("Trust") would match by accident
|
|
784
|
+
|
|
785
|
+
|
|
786
|
+
def bundle_links(refs, m):
|
|
787
|
+
"""ref id -> a link to another paper's reading copy in the same bundle."""
|
|
788
|
+
out = {}
|
|
789
|
+
for other in m.get("bundle_members") or []:
|
|
790
|
+
if other.get("pid") == m.get("pid"):
|
|
791
|
+
continue
|
|
792
|
+
for i, h in refs:
|
|
793
|
+
if i not in out and same_paper(h, other):
|
|
794
|
+
out[i] = (' <span class="in-bundle">In this bundle (%d of %d): <a href="%s">open its reading copy</a></span>'
|
|
795
|
+
% (other["n"], other["total"], attr(other["href"])))
|
|
796
|
+
return out
|
|
797
|
+
|
|
798
|
+
|
|
799
|
+
|
|
800
|
+
DONE_BOX = '<div class="done-box">Reached the end? <button id="mark-done" type="button">Mark as finished</button> It shows on the list of papers.</div>'
|
|
801
|
+
DONE_SLOT = "<!-- lit-reader: end of the main text -->"
|
|
802
|
+
|
|
803
|
+
PAGE = """<!doctype html>
|
|
804
|
+
<html lang="en">
|
|
805
|
+
<head>
|
|
806
|
+
<meta charset="utf-8">
|
|
807
|
+
<meta name="viewport" content="width=device-width, initial-scale=1">
|
|
808
|
+
<meta name="generator" content="lit-reader">
|
|
809
|
+
<meta http-equiv="Content-Security-Policy" content="default-src 'none'; img-src data: file:; style-src 'self' 'unsafe-inline' file:; font-src data:; script-src 'self' file:">
|
|
810
|
+
<title>{title}</title>
|
|
811
|
+
<link rel="stylesheet" href="assets/fonts.css">
|
|
812
|
+
<link rel="stylesheet" href="assets/reader.css">
|
|
813
|
+
</head>
|
|
814
|
+
<body data-pid="{pid}" data-cite="{cite}" data-reference="{reference}">
|
|
815
|
+
<a class="skip" href="#text-start">Skip to the text</a>
|
|
816
|
+
<div class="bar">
|
|
817
|
+
{all_link}
|
|
818
|
+
<button id="toc-toggle" type="button" aria-expanded="false" aria-controls="toc" aria-label="Contents"><span aria-hidden="true">☰</span><span class="lbl"> Contents</span></button>
|
|
819
|
+
<a class="nojs-toc" href="#toc" aria-label="Contents"><span aria-hidden="true">☰</span><span class="lbl"> Contents</span></a>
|
|
820
|
+
<span class="where" id="where">{where}</span>
|
|
821
|
+
<button id="clips" type="button" aria-expanded="false" aria-controls="clips-panel">Quotes <span id="clip-count"></span></button>
|
|
822
|
+
<button id="aa" type="button" aria-label="Aa, reading settings" aria-expanded="false" aria-controls="panel">Aa</button>
|
|
823
|
+
</div>
|
|
824
|
+
<div class="panel clips-panel" id="clips-panel" role="group" aria-label="Saved quotes" hidden></div>
|
|
825
|
+
<div class="progress" id="progress"></div>
|
|
826
|
+
{panel}
|
|
827
|
+
<div class="layout">
|
|
828
|
+
<nav class="toc" id="toc" aria-label="Contents">{toc}</nav>
|
|
829
|
+
<main id="text-start" tabindex="-1">
|
|
830
|
+
<header class="paper" id="top">
|
|
831
|
+
<div class="kicker">{kicker}</div>
|
|
832
|
+
<h1>{title_html}</h1>
|
|
833
|
+
{part}
|
|
834
|
+
{authors}
|
|
835
|
+
<div class="meta">{meta}</div>
|
|
836
|
+
</header>
|
|
837
|
+
{abstracts}
|
|
838
|
+
{body}
|
|
839
|
+
<div id="end-of-body"></div>
|
|
840
|
+
{done}
|
|
841
|
+
{refs}
|
|
842
|
+
<footer class="source">{source} Built {built}.</footer>
|
|
843
|
+
</main>
|
|
844
|
+
</div>
|
|
845
|
+
<script src="assets/reader.js"></script>
|
|
846
|
+
</body>
|
|
847
|
+
</html>
|
|
848
|
+
"""
|
|
849
|
+
|
|
850
|
+
|
|
851
|
+
# --------------------------------------------------------------------------
|
|
852
|
+
# Ledger, cache and list
|
|
853
|
+
# --------------------------------------------------------------------------
|
|
854
|
+
|
|
855
|
+
def ledger_meta(ledger, pid):
|
|
856
|
+
ledger = LEDGERS.get(ledger, ledger)
|
|
857
|
+
try:
|
|
858
|
+
con = sqlite3.connect("file:%s?mode=ro" % Path(ledger).expanduser(), uri=True)
|
|
859
|
+
con.row_factory = sqlite3.Row
|
|
860
|
+
r = con.execute("select * from papers where pid=?", (pid,)).fetchone()
|
|
861
|
+
con.close()
|
|
862
|
+
except sqlite3.Error as e:
|
|
863
|
+
raise ReaderError("can't read the ledger %s: %s" % (ledger, e))
|
|
864
|
+
if not r:
|
|
865
|
+
raise ReaderError("%s is not in %s" % (pid, ledger))
|
|
866
|
+
return {"pid": pid, "title": r["title"], "authors": [a.strip() for a in (r["authors"] or "").split(";") if a.strip()],
|
|
867
|
+
"year": r["year"], "venue": r["venue"], "doi": r["doi"], "arxiv": r["arxiv"], "pmcid": r["pmcid"]}
|
|
868
|
+
|
|
869
|
+
|
|
870
|
+
def stem(pid):
|
|
871
|
+
return re.sub(r"[^A-Za-z0-9._-]", "_", pid)
|
|
872
|
+
|
|
873
|
+
|
|
874
|
+
def cache_file(sub, pid, ext):
|
|
875
|
+
d = LIT_CACHE / sub
|
|
876
|
+
exact = d / (stem(pid) + ext)
|
|
877
|
+
if exact.exists():
|
|
878
|
+
return exact
|
|
879
|
+
hits = sorted(d.glob(stem(pid) + "~*" + ext))
|
|
880
|
+
return hits[0] if hits else None
|
|
881
|
+
|
|
882
|
+
|
|
883
|
+
def source_pdf(meta):
|
|
884
|
+
"""The PDF behind a page. Standalone builds never look in the lit.py cache."""
|
|
885
|
+
if meta.get("standalone"):
|
|
886
|
+
return meta.get("pdf_path")
|
|
887
|
+
return cache_file("pdf", meta["pid"], ".pdf")
|
|
888
|
+
|
|
889
|
+
|
|
890
|
+
def source_text(meta):
|
|
891
|
+
"""The plain text a build is checked against."""
|
|
892
|
+
if meta.get("standalone"):
|
|
893
|
+
return meta.get("text_path")
|
|
894
|
+
return cache_file("text", meta.get("pid", ""), ".txt")
|
|
895
|
+
|
|
896
|
+
|
|
897
|
+
def slug(meta):
|
|
898
|
+
first = (meta["authors"][0].split()[-1] if meta["authors"] else "paper").lower()
|
|
899
|
+
first = unicodedata.normalize("NFKD", first).encode("ascii", "ignore").decode() or "paper"
|
|
900
|
+
title = re.sub(r"[^a-z0-9]+", "-", unicodedata.normalize("NFKD", meta["title"] or "").encode("ascii", "ignore").decode().lower())
|
|
901
|
+
title = "-".join([w for w in title.split("-") if w][:4])
|
|
902
|
+
return "%s-%s-%s" % (re.sub(r"[^a-z0-9]", "", first), meta.get("year") or "", title)
|
|
903
|
+
|
|
904
|
+
|
|
905
|
+
|
|
906
|
+
# --------------------------------------------------------------------------
|
|
907
|
+
# Fetching (online)
|
|
908
|
+
# --------------------------------------------------------------------------
|
|
909
|
+
|
|
910
|
+
class NamedHostsOnly(urllib.request.HTTPRedirectHandler):
|
|
911
|
+
"""A redirect may not leave the hosts the fetch was allowed to reach."""
|
|
912
|
+
|
|
913
|
+
def __init__(self, allowed_hosts):
|
|
914
|
+
self.allowed_hosts = allowed_hosts
|
|
915
|
+
|
|
916
|
+
def redirect_request(self, req, fp, code, msg, headers, newurl):
|
|
917
|
+
p = urllib.parse.urlparse(newurl)
|
|
918
|
+
if p.scheme != "https" or p.netloc not in self.allowed_hosts:
|
|
919
|
+
raise ReaderError("refusing a redirect to %s" % p.netloc)
|
|
920
|
+
return urllib.request.HTTPRedirectHandler.redirect_request(self, req, fp, code, msg, headers, newurl)
|
|
921
|
+
|
|
922
|
+
|
|
923
|
+
def http_get(url, allowed_hosts):
|
|
924
|
+
p = urllib.parse.urlparse(url)
|
|
925
|
+
if p.scheme != "https" or p.netloc not in allowed_hosts:
|
|
926
|
+
raise ReaderError("refusing to fetch from %s" % p.netloc)
|
|
927
|
+
req = urllib.request.Request(url, headers={"User-Agent": UA})
|
|
928
|
+
for attempt in range(3):
|
|
929
|
+
try:
|
|
930
|
+
with urllib.request.build_opener(NamedHostsOnly(allowed_hosts)).open(req, timeout=60) as r:
|
|
931
|
+
return r.read(), r.headers.get("Content-Type", "")
|
|
932
|
+
except urllib.error.HTTPError as e:
|
|
933
|
+
if e.code not in (429, 500, 502, 503, 504) or attempt == 2:
|
|
934
|
+
err = ReaderError("could not fetch %s: %s" % (url, e))
|
|
935
|
+
err.status = e.code
|
|
936
|
+
raise err
|
|
937
|
+
time.sleep(2 * (attempt + 1)) # a busy service often answers on the next try
|
|
938
|
+
except (urllib.error.URLError, OSError) as e:
|
|
939
|
+
raise ReaderError("could not fetch %s: %s" % (url, e))
|
|
940
|
+
|
|
941
|
+
|
|
942
|
+
def image_kind(data):
|
|
943
|
+
if data[:8] == b"\x89PNG\r\n\x1a\n":
|
|
944
|
+
return "image/png"
|
|
945
|
+
if data[:3] == b"\xff\xd8\xff":
|
|
946
|
+
return "image/jpeg"
|
|
947
|
+
if data[:6] in (b"GIF87a", b"GIF89a"):
|
|
948
|
+
return "image/gif"
|
|
949
|
+
return None
|
|
950
|
+
|
|
951
|
+
|
|
952
|
+
def data_uri(data):
|
|
953
|
+
k = image_kind(data)
|
|
954
|
+
if not k:
|
|
955
|
+
return None
|
|
956
|
+
return "data:%s;base64,%s" % (k, base64.b64encode(data).decode())
|
|
957
|
+
|
|
958
|
+
|
|
959
|
+
AR5IV = "ar5iv.labs.arxiv.org"
|
|
960
|
+
|
|
961
|
+
|
|
962
|
+
def fetch_arxiv(aid, d, sources=("arxiv", "ar5iv")):
|
|
963
|
+
"""arXiv's own HTML, or ar5iv's conversion when arXiv has none. ar5iv
|
|
964
|
+
sends a paper it lacks back to arxiv.org/abs, so that host is allowed too."""
|
|
965
|
+
d.mkdir(parents=True, exist_ok=True)
|
|
966
|
+
q = urllib.parse.quote(aid, safe="/.")
|
|
967
|
+
body = None
|
|
968
|
+
tries = {"arxiv": ("https://arxiv.org/html/%s" % q, {"arxiv.org"}),
|
|
969
|
+
"ar5iv": ("https://%s/html/%s" % (AR5IV, q), {AR5IV, "arxiv.org"})}
|
|
970
|
+
for url, hosts in (tries[s_] for s_ in sources):
|
|
971
|
+
try:
|
|
972
|
+
got, _ = http_get(url, hosts)
|
|
973
|
+
except ReaderError as e:
|
|
974
|
+
if e.status != 404:
|
|
975
|
+
raise # an outage or a network error is reported, not mistaken for "no HTML version"
|
|
976
|
+
continue
|
|
977
|
+
if b"ltx_document" in got:
|
|
978
|
+
body = got
|
|
979
|
+
break
|
|
980
|
+
if body is None:
|
|
981
|
+
err = ReaderError("%s of %s" % ("neither arXiv nor ar5iv has an HTML version" if len(sources) > 1 else
|
|
982
|
+
"%s has no HTML version" % ("arXiv" if sources[0] == "arxiv" else "ar5iv"), aid))
|
|
983
|
+
err.no_html = True
|
|
984
|
+
raise err
|
|
985
|
+
if AR5IV in url:
|
|
986
|
+
print(" arXiv has no HTML version of %s, so using ar5iv's conversion" % aid)
|
|
987
|
+
(d / "arxiv.html").write_bytes(body)
|
|
988
|
+
(d / "arxiv.url").write_text(url)
|
|
989
|
+
root = parse_html(body.decode("utf-8", "replace"))
|
|
990
|
+
imgs = d / "img"
|
|
991
|
+
imgs.mkdir(exist_ok=True)
|
|
992
|
+
n = 0
|
|
993
|
+
for img in root.findall(lambda x: x.tag == "img"):
|
|
994
|
+
src = img.attrs.get("src", "")
|
|
995
|
+
if not src or src.startswith("data:") or src.startswith("http"):
|
|
996
|
+
continue
|
|
997
|
+
name = safe_id(src)
|
|
998
|
+
target = imgs / name
|
|
999
|
+
if target.exists():
|
|
1000
|
+
continue
|
|
1001
|
+
try:
|
|
1002
|
+
data, _ = http_get(urllib.parse.urljoin(url, src), {urllib.parse.urlparse(url).netloc})
|
|
1003
|
+
except (ReaderError, ValueError) as e:
|
|
1004
|
+
print(" image failed: %s (%s)" % (src, e))
|
|
1005
|
+
continue
|
|
1006
|
+
if image_kind(data):
|
|
1007
|
+
target.write_bytes(data)
|
|
1008
|
+
n += 1
|
|
1009
|
+
print("fetched arXiv HTML for %s and %d images into %s" % (aid, n, d))
|
|
1010
|
+
|
|
1011
|
+
|
|
1012
|
+
def fetch_jats(pmcid, d):
|
|
1013
|
+
d.mkdir(parents=True, exist_ok=True)
|
|
1014
|
+
body, _ = http_get("https://www.ebi.ac.uk/europepmc/webservices/rest/%s/fullTextXML" % pmcid, {"www.ebi.ac.uk"})
|
|
1015
|
+
(d / "jats.xml").write_bytes(body)
|
|
1016
|
+
imgs = d / "img"
|
|
1017
|
+
imgs.mkdir(exist_ok=True)
|
|
1018
|
+
n = 0
|
|
1019
|
+
try:
|
|
1020
|
+
z, _ = http_get("https://www.ebi.ac.uk/europepmc/webservices/rest/%s/supplementaryFiles?includeInlineImage=yes" % pmcid, {"www.ebi.ac.uk"})
|
|
1021
|
+
with zipfile.ZipFile(io.BytesIO(z)) as zf:
|
|
1022
|
+
for info in zf.infolist():
|
|
1023
|
+
name = os.path.basename(info.filename)
|
|
1024
|
+
if info.file_size > 30 * 1024 * 1024 or not re.match(r"^[\w.-]+\.(jpe?g|png|gif)$", name, re.I):
|
|
1025
|
+
continue
|
|
1026
|
+
data = zf.read(info)
|
|
1027
|
+
if image_kind(data):
|
|
1028
|
+
(imgs / name).write_bytes(data)
|
|
1029
|
+
n += 1
|
|
1030
|
+
except (ReaderError, zipfile.BadZipFile) as e:
|
|
1031
|
+
print(" no figure bundle for %s (%s)" % (pmcid, e))
|
|
1032
|
+
print("fetched JATS XML for %s and %d images into %s" % (pmcid, n, d))
|
|
1033
|
+
|
|
1034
|
+
|
|
1035
|
+
def cmd_fetch(a):
|
|
1036
|
+
meta = ledger_meta(a.ledger, a.pid)
|
|
1037
|
+
d = SRC / slug(meta)
|
|
1038
|
+
if a.route == "arxiv":
|
|
1039
|
+
fetch_arxiv(meta["arxiv"] or a.pid.split(":", 1)[-1], d)
|
|
1040
|
+
elif a.route == "jats":
|
|
1041
|
+
pmcid = meta["pmcid"] or a.pmcid
|
|
1042
|
+
if not pmcid:
|
|
1043
|
+
body, _ = http_get("https://www.ebi.ac.uk/europepmc/webservices/rest/search?format=json&resultType=lite&query=DOI:%s"
|
|
1044
|
+
% urllib.parse.quote(meta["doi"] or "", safe="/"), {"www.ebi.ac.uk"})
|
|
1045
|
+
res = json.loads(body.decode("utf-8"))["resultList"]["result"]
|
|
1046
|
+
pmcid = next((r.get("pmcid") for r in res if r.get("pmcid")), None)
|
|
1047
|
+
if not pmcid or not re.match(r"^PMC\d+$", pmcid):
|
|
1048
|
+
raise ReaderError("no PMC id for %s" % a.pid)
|
|
1049
|
+
fetch_jats(pmcid, d)
|
|
1050
|
+
|
|
1051
|
+
|
|
1052
|
+
# --------------------------------------------------------------------------
|
|
1053
|
+
# Route: arXiv HTML
|
|
1054
|
+
# --------------------------------------------------------------------------
|
|
1055
|
+
|
|
1056
|
+
def visible_text(n):
|
|
1057
|
+
"""A node's text without the TeX and MathML annotations hidden inside formulas."""
|
|
1058
|
+
if isinstance(n, str):
|
|
1059
|
+
return n
|
|
1060
|
+
return "" if n.tag in ("annotation", "annotation-xml") else "".join(visible_text(k) for k in n.kids)
|
|
1061
|
+
|
|
1062
|
+
|
|
1063
|
+
ARXIV_BLOCKS = {"h1", "h2", "h3", "h4", "h5", "h6", "p", "div", "section", "figure", "table", "ul", "ol", "dl"}
|
|
1064
|
+
|
|
1065
|
+
|
|
1066
|
+
def span_table(ix, n):
|
|
1067
|
+
"""LaTeXML sometimes builds a table from spans, for example when it has been resized."""
|
|
1068
|
+
def owner(tr):
|
|
1069
|
+
p = tr.parent
|
|
1070
|
+
while p is not None and "ltx_tabular" not in p.cls():
|
|
1071
|
+
p = p.parent
|
|
1072
|
+
return p
|
|
1073
|
+
rows = []
|
|
1074
|
+
for tr in n.findall(lambda x: "ltx_tr" in x.cls()):
|
|
1075
|
+
if owner(tr) is not n:
|
|
1076
|
+
continue
|
|
1077
|
+
cells = []
|
|
1078
|
+
for td in tr.children():
|
|
1079
|
+
if td.cls() & {"ltx_td", "ltx_th"}:
|
|
1080
|
+
tag = "th" if "ltx_th" in td.cls() else "td"
|
|
1081
|
+
spans = "".join(' %s="%s"' % (a, td.attrs[a]) for a in ("colspan", "rowspan") if re.fullmatch(r"\d{1,2}", td.attrs.get(a, "")))
|
|
1082
|
+
cells.append("<%s%s>%s</%s>" % (tag, spans, ix.run(td).strip(), tag))
|
|
1083
|
+
rows.append("<tr>%s</tr>" % "".join(cells))
|
|
1084
|
+
return '<table class="data">%s</table>' % "".join(rows)
|
|
1085
|
+
|
|
1086
|
+
|
|
1087
|
+
class ArxivInline(Inline):
|
|
1088
|
+
def rule(self, n):
|
|
1089
|
+
c = n.cls()
|
|
1090
|
+
if n.tag == "span" and "ltx_tabular" in c:
|
|
1091
|
+
return span_table(self, n)
|
|
1092
|
+
if n.tag == "span" and "ltx_note" in c:
|
|
1093
|
+
if c & {"ltx_role_footnote", "ltx_role_footnotetext"}:
|
|
1094
|
+
content = n.find(lambda x: "ltx_note_content" in x.cls())
|
|
1095
|
+
if content is None:
|
|
1096
|
+
return ""
|
|
1097
|
+
for junk in content.findall(lambda x: x.cls() & {"ltx_note_mark", "ltx_note_type", "ltx_tag_note"}):
|
|
1098
|
+
junk.kids = []
|
|
1099
|
+
return self.doc.note(self.run(content).strip())
|
|
1100
|
+
return ""
|
|
1101
|
+
if n.tag == "a" and "ltx_ref" in c:
|
|
1102
|
+
href = n.attrs.get("href", "")
|
|
1103
|
+
if href.startswith("#bib."):
|
|
1104
|
+
return self.doc.cite([href[1:].replace("bib.", "", 1)], self.run(n))
|
|
1105
|
+
if href.startswith("#"):
|
|
1106
|
+
return '<a href="#x-%s">%s</a>' % (safe_id(href[1:]), self.run(n))
|
|
1107
|
+
if n.tag == "span" and "ltx_text" in c:
|
|
1108
|
+
inner = self.run(n)
|
|
1109
|
+
if "ltx_font_italic" in c:
|
|
1110
|
+
return "<em>%s</em>" % inner
|
|
1111
|
+
if "ltx_font_bold" in c:
|
|
1112
|
+
return "<strong>%s</strong>" % inner
|
|
1113
|
+
if "ltx_font_typewriter" in c:
|
|
1114
|
+
return "<code>%s</code>" % inner
|
|
1115
|
+
return inner
|
|
1116
|
+
if c & {"ltx_acm_description", "ltx_bib_cited", "ltx_tag_bibitem"}:
|
|
1117
|
+
return ""
|
|
1118
|
+
return None
|
|
1119
|
+
|
|
1120
|
+
|
|
1121
|
+
def arxiv_doc(meta, d):
|
|
1122
|
+
path = d / "arxiv.html"
|
|
1123
|
+
if not path.exists():
|
|
1124
|
+
raise ReaderError("no arXiv HTML; run fetch --route arxiv first")
|
|
1125
|
+
root = parse_html(path.read_text(encoding="utf-8"))
|
|
1126
|
+
art = root.find(lambda x: x.tag == "article" and "ltx_document" in x.cls()) or root
|
|
1127
|
+
doc = Doc(dict(meta))
|
|
1128
|
+
url = (d / "arxiv.url").read_text().strip() if (d / "arxiv.url").exists() else ""
|
|
1129
|
+
doc.meta["url"] = url
|
|
1130
|
+
doc.meta["kicker"] = "arXiv " + (meta.get("arxiv") or "")
|
|
1131
|
+
if AR5IV in url:
|
|
1132
|
+
doc.source = "Text from ar5iv (%s), arXiv Labs' HTML conversion, because arXiv has no HTML version of this paper." % url
|
|
1133
|
+
else:
|
|
1134
|
+
doc.source = "Text from arXiv's HTML version (%s), which arXiv labels experimental." % url
|
|
1135
|
+
ix = ArxivInline(doc)
|
|
1136
|
+
imgdir = d / "img"
|
|
1137
|
+
doc.pdf = source_pdf(meta)
|
|
1138
|
+
doc._crops = docling_crops(d, doc.pdf)
|
|
1139
|
+
t = art.find(lambda x: "ltx_title_document" in x.cls())
|
|
1140
|
+
if t is not None:
|
|
1141
|
+
for n in t.findall(lambda x: "ltx_note" in x.cls()):
|
|
1142
|
+
n.kids = [] # a note on the title has no place in the heading or the browser tab
|
|
1143
|
+
doc.meta["title_html"] = ArxivInline(Doc({})).run(t).strip()
|
|
1144
|
+
|
|
1145
|
+
ab = art.find(lambda x: "ltx_abstract" in x.cls())
|
|
1146
|
+
if ab is not None:
|
|
1147
|
+
paras = [ix.run(p).strip() for p in ab.findall(lambda x: x.tag == "p")]
|
|
1148
|
+
doc.abstracts.append(("Abstract", "".join("<p>%s</p>" % p for p in paras if p)))
|
|
1149
|
+
|
|
1150
|
+
def heading(n, level, appendix):
|
|
1151
|
+
tag = n.find(lambda x: "ltx_tag" in x.cls())
|
|
1152
|
+
no = tag.text().strip() if tag is not None else ""
|
|
1153
|
+
if tag is not None:
|
|
1154
|
+
tag.kids = []
|
|
1155
|
+
txt = ix.run(n).strip()
|
|
1156
|
+
if no and not no.endswith("."):
|
|
1157
|
+
no = no # "Appendix A" style tags keep their own form
|
|
1158
|
+
return dict(t="h", level=level, html=txt, no=no, appendix=appendix)
|
|
1159
|
+
|
|
1160
|
+
def walk(n, depth, appendix):
|
|
1161
|
+
for k in n.children():
|
|
1162
|
+
c = k.cls()
|
|
1163
|
+
if k.tag in ("nav", "header", "footer") or c & {"ltx_page_navbar", "ltx_authors", "ltx_abstract", "ltx_dates",
|
|
1164
|
+
"ltx_keywords", "ltx_pubnotes", "ltx_classification", "ltx_title_document",
|
|
1165
|
+
"ltx_acknowledgements_x"}:
|
|
1166
|
+
continue
|
|
1167
|
+
if "ltx_bibliography" in c:
|
|
1168
|
+
for li in k.findall(lambda x: x.tag == "li" and "ltx_bibitem" in x.cls()):
|
|
1169
|
+
rid = li.attrs.get("id", "").replace("bib.", "", 1)
|
|
1170
|
+
blocks = [ix.run(b).strip() for b in li.findall(lambda x: "ltx_bibblock" in x.cls() and "ltx_bib_cited" not in x.cls())]
|
|
1171
|
+
doc.refs.append((rid, " ".join(x for x in blocks if x)))
|
|
1172
|
+
continue
|
|
1173
|
+
if k.tag == "section":
|
|
1174
|
+
app = appendix or "ltx_appendix" in c
|
|
1175
|
+
h = k.find(lambda x: x.tag in ("h2", "h3", "h4", "h5", "h6") and x.parent is k)
|
|
1176
|
+
if h is not None:
|
|
1177
|
+
level = {"h2": 2, "h3": 3, "h4": 4}.get(h.tag, 4)
|
|
1178
|
+
blk = heading(h, level, app and not appendix)
|
|
1179
|
+
blk["id"] = "x-" + safe_id(k.attrs.get("id", "")) if k.attrs.get("id") else None
|
|
1180
|
+
if app and not appendix:
|
|
1181
|
+
blk["appendix"] = True
|
|
1182
|
+
doc.blocks.append(blk)
|
|
1183
|
+
walk(k, depth + 1, app)
|
|
1184
|
+
continue
|
|
1185
|
+
if k.tag in ("h2", "h3", "h4", "h5", "h6"):
|
|
1186
|
+
if k.parent is not None and k.parent.tag == "section":
|
|
1187
|
+
continue # handled with its section
|
|
1188
|
+
doc.blocks.append(heading(k, 4, False))
|
|
1189
|
+
continue
|
|
1190
|
+
if k.tag == "p":
|
|
1191
|
+
h = ix.run(k).strip()
|
|
1192
|
+
if h:
|
|
1193
|
+
doc.add(t="p", html=h)
|
|
1194
|
+
for img in k.findall(lambda x: x.tag == "img"): # a picture placed without a figure around it
|
|
1195
|
+
f = imgdir / safe_id(img.attrs.get("src", ""))
|
|
1196
|
+
u = data_uri(f.read_bytes()) if f.exists() else None
|
|
1197
|
+
if u:
|
|
1198
|
+
doc.add(t="fig", kind="figure", label="", caption="", table="", imgs=[(u, img.attrs.get("alt") or "")])
|
|
1199
|
+
continue
|
|
1200
|
+
if k.tag == "figure" and k.parent is not None and k.parent.tag == "figure":
|
|
1201
|
+
continue
|
|
1202
|
+
if k.tag == "figure":
|
|
1203
|
+
arxiv_figure(doc, ix, k, imgdir)
|
|
1204
|
+
continue
|
|
1205
|
+
if k.tag in ("ul", "ol"):
|
|
1206
|
+
items = []
|
|
1207
|
+
marks = [tg.text().strip() for tg in k.findall(lambda x: "ltx_tag_item" in x.cls())]
|
|
1208
|
+
custom = any(not re.fullmatch(r"\d+[.)]?|[•∙·*–—-]", m) for m in marks) # "(iv)" must survive: the text may cite it
|
|
1209
|
+
for li in k.children():
|
|
1210
|
+
if li.tag != "li":
|
|
1211
|
+
continue
|
|
1212
|
+
mark = ""
|
|
1213
|
+
for tg in li.findall(lambda x: "ltx_tag_item" in x.cls()):
|
|
1214
|
+
mark = mark or tg.text().strip()
|
|
1215
|
+
tg.kids = []
|
|
1216
|
+
items.append((esc(mark) + " " if custom and mark else "") + ix.run(li).strip())
|
|
1217
|
+
doc.add(t="list", ordered=k.tag == "ol" and not custom, plain=custom, items=items)
|
|
1218
|
+
continue
|
|
1219
|
+
if k.tag == "table" and c & {"ltx_equation", "ltx_equationgroup"}:
|
|
1220
|
+
rows = [r for r in k.findall(lambda x: x.tag == "tr") if r.find(lambda x: x.tag == "math") is not None] or [k]
|
|
1221
|
+
parts = []
|
|
1222
|
+
for r in rows:
|
|
1223
|
+
tag = r.find(lambda x: "ltx_tag_equation" in x.cls())
|
|
1224
|
+
parts.append(" ".join(math_html(mm) for mm in r.findall(lambda x: x.tag == "math")) + (" " + esc(tag.text().strip()) if tag is not None else ""))
|
|
1225
|
+
targets = [x.attrs["id"] for x in [k] + k.findall(lambda x: re.search(r"\.E\d+$", x.attrs.get("id", "")) is not None) if x.attrs.get("id")]
|
|
1226
|
+
doc.add(t="eq", html="".join('<span id="x-%s"></span>' % safe_id(i) for i in dict.fromkeys(targets)) + "<br>".join(parts))
|
|
1227
|
+
continue
|
|
1228
|
+
if k.tag == "table" and "ltx_tabular" in c:
|
|
1229
|
+
doc.add(t="fig", kind="table", label="", caption="", imgs=[], table=ix.one(k))
|
|
1230
|
+
continue
|
|
1231
|
+
if "ltx_listing" in c:
|
|
1232
|
+
lines = [visible_text(ln).rstrip() for ln in k.findall(lambda x: "ltx_listingline" in x.cls())]
|
|
1233
|
+
doc.add(t="pre", text="\n".join(lines))
|
|
1234
|
+
continue
|
|
1235
|
+
if k.tag == "dl":
|
|
1236
|
+
terms = [c for c in k.children() if c.tag in ("dt", "dd")]
|
|
1237
|
+
items = ["<strong>%s</strong> %s" % (ix.run(a).strip(), ix.run(b).strip()) for a, b in zip(terms[::2], terms[1::2])]
|
|
1238
|
+
doc.add(t="list", ordered=False, plain=True, items=items)
|
|
1239
|
+
continue
|
|
1240
|
+
if "ltx_note" in c:
|
|
1241
|
+
continue # notes are placed where they are cited
|
|
1242
|
+
if k.tag in ("div", "section") and any( # text sitting directly in a container
|
|
1243
|
+
(isinstance(x, str) and x.strip()) or
|
|
1244
|
+
(isinstance(x, Node) and x.tag in ("span", "a", "em", "strong", "i", "b") and "ltx_note" not in x.cls() and x.text().strip())
|
|
1245
|
+
for x in k.kids):
|
|
1246
|
+
run = Node("p", parent=k)
|
|
1247
|
+
for x in k.kids + [None]:
|
|
1248
|
+
if x is None or (isinstance(x, Node) and x.tag in ARXIV_BLOCKS):
|
|
1249
|
+
h = ix.run(run).strip()
|
|
1250
|
+
if h:
|
|
1251
|
+
doc.add(t="p", html=h)
|
|
1252
|
+
run.kids = []
|
|
1253
|
+
if x is not None:
|
|
1254
|
+
one = Node(k.tag, parent=k.parent)
|
|
1255
|
+
one.kids = [x]
|
|
1256
|
+
walk(one, depth, appendix)
|
|
1257
|
+
else:
|
|
1258
|
+
run.kids.append(x)
|
|
1259
|
+
continue
|
|
1260
|
+
walk(k, depth, appendix)
|
|
1261
|
+
|
|
1262
|
+
walk(art, 0, False)
|
|
1263
|
+
return doc
|
|
1264
|
+
|
|
1265
|
+
|
|
1266
|
+
def arxiv_figure(doc, ix, k, imgdir):
|
|
1267
|
+
def owner(c):
|
|
1268
|
+
p = c.parent
|
|
1269
|
+
while p is not None and p.tag != "figure":
|
|
1270
|
+
p = p.parent
|
|
1271
|
+
return p
|
|
1272
|
+
own = [c for c in k.findall(lambda x: x.tag == "figcaption") if owner(c) is k]
|
|
1273
|
+
panels = [c for c in k.findall(lambda x: x.tag == "figcaption") if owner(c) is not k]
|
|
1274
|
+
cap = own[-1] if own else (panels[0] if panels else None) # with panels, the figure's own caption comes last
|
|
1275
|
+
panels = [c for c in panels if c is not cap]
|
|
1276
|
+
label, caption, alts = "", "", []
|
|
1277
|
+
if cap is not None:
|
|
1278
|
+
for desc in cap.findall(lambda x: "ltx_acm_description" in x.cls()):
|
|
1279
|
+
alts.append(desc.text().strip())
|
|
1280
|
+
tg = cap.find(lambda x: "ltx_tag" in x.cls())
|
|
1281
|
+
if tg is not None:
|
|
1282
|
+
label = tg.text().strip()
|
|
1283
|
+
tg.kids = []
|
|
1284
|
+
caption = " ".join(x for x in [ix.run(cap).strip()] + [ix.run(c).strip() for c in panels] if x)
|
|
1285
|
+
imgs = []
|
|
1286
|
+
for img in k.findall(lambda x: x.tag == "img"):
|
|
1287
|
+
f = imgdir / safe_id(img.attrs.get("src", ""))
|
|
1288
|
+
if f.exists():
|
|
1289
|
+
u = data_uri(f.read_bytes())
|
|
1290
|
+
if u:
|
|
1291
|
+
alt = img.attrs.get("alt") or (alts[0] if alts else "")
|
|
1292
|
+
imgs.append((u, label.rstrip(":. ") + ", shown as an image" if label and alt.strip().lower() == "refer to caption" else alt))
|
|
1293
|
+
table = ""
|
|
1294
|
+
def outer_table(t):
|
|
1295
|
+
p = t.parent
|
|
1296
|
+
while p is not None and p is not k:
|
|
1297
|
+
if p.tag == "table":
|
|
1298
|
+
return False
|
|
1299
|
+
p = p.parent
|
|
1300
|
+
return True
|
|
1301
|
+
table = "".join(ix.one(t) for t in k.findall(lambda x: x.tag == "table") # a table holding only pictures is layout
|
|
1302
|
+
if outer_table(t) and t.find(lambda x: x.tag == "img") is None and t.text().strip())
|
|
1303
|
+
|
|
1304
|
+
def outer_span_table(t):
|
|
1305
|
+
p = t.parent
|
|
1306
|
+
while p is not None and p is not k:
|
|
1307
|
+
if p.tag == "table" or "ltx_tabular" in p.cls():
|
|
1308
|
+
return False
|
|
1309
|
+
p = p.parent
|
|
1310
|
+
return t.find(lambda x: x.tag == "img") is None and bool(t.text().strip())
|
|
1311
|
+
table += "".join(span_table(ix, t) for t in k.findall(lambda x: x.tag == "span" and "ltx_tabular" in x.cls()) if outer_span_table(t))
|
|
1312
|
+
kind = "table" if "ltx_table" in k.cls() else "figure"
|
|
1313
|
+
fid = "x-" + safe_id(k.attrs.get("id", "")) if k.attrs.get("id") else None
|
|
1314
|
+
page = find_pdf_page(doc, label)
|
|
1315
|
+
if not imgs and not table:
|
|
1316
|
+
# arXiv's converter lost this figure: use Docling's crop of it from the PDF
|
|
1317
|
+
crop = doc._crops.get(float_key(label)) if hasattr(doc, "_crops") else None
|
|
1318
|
+
if crop:
|
|
1319
|
+
imgs, page = [(crop[0], alts[0] if alts else "")], crop[1]
|
|
1320
|
+
if not imgs and not table: # an algorithm or other float made of text
|
|
1321
|
+
lines = [visible_text(ln).rstrip() for ln in k.findall(lambda x: "ltx_listingline" in x.cls())]
|
|
1322
|
+
|
|
1323
|
+
def spoken_only(p): # captions and the accessibility descriptions of a lost image
|
|
1324
|
+
while p is not None and p is not k:
|
|
1325
|
+
if p.tag == "figcaption" or "ltx_acm_description" in p.cls():
|
|
1326
|
+
return True
|
|
1327
|
+
p = p.parent
|
|
1328
|
+
return False
|
|
1329
|
+
body = [ix.run(p).strip() for p in k.findall(lambda x: x.tag == "p") if not spoken_only(p)]
|
|
1330
|
+
if lines:
|
|
1331
|
+
table = '<pre class="listing">%s</pre>' % esc("\n".join(lines))
|
|
1332
|
+
elif not float_key(label): # a lost figure or table keeps its "did not come through" notice
|
|
1333
|
+
table = "".join("<p>%s</p>" % b for b in body if b)
|
|
1334
|
+
if not label and not caption and not imgs and not table:
|
|
1335
|
+
return
|
|
1336
|
+
doc.add(t="fig", kind=kind, id=fid, label=label, caption=caption, imgs=imgs, table=table, page=page)
|
|
1337
|
+
|
|
1338
|
+
|
|
1339
|
+
def float_key(label):
|
|
1340
|
+
m = re.match(r"^\s*(Fig(?:ure)?|Table|TABLE|FIGURE)\.?\s*([\dA-Z]+)", label or "", re.I)
|
|
1341
|
+
if not m:
|
|
1342
|
+
return None
|
|
1343
|
+
return ("table" if m.group(1).lower().startswith("tab") else "figure", m.group(2))
|
|
1344
|
+
|
|
1345
|
+
|
|
1346
|
+
def docling_crops(d, pdf):
|
|
1347
|
+
"""{("figure", "3"): (data uri, pdf page)} for every captioned picture or
|
|
1348
|
+
table in this paper's Docling output, cropped from the PDF page."""
|
|
1349
|
+
out = {}
|
|
1350
|
+
js = sorted((d / "docling").glob("*.json"))
|
|
1351
|
+
if not js or not pdf:
|
|
1352
|
+
return out
|
|
1353
|
+
j = json.loads(js[0].read_text(encoding="utf-8"))
|
|
1354
|
+
page_h = {int(k): v["size"]["height"] for k, v in j.get("pages", {}).items()}
|
|
1355
|
+
for kind in ("pictures", "tables"):
|
|
1356
|
+
for item in j.get(kind, []):
|
|
1357
|
+
caps = " ".join(j["texts"][int(c["$ref"].split("/")[2])].get("text", "") for c in item.get("captions", []))
|
|
1358
|
+
key = float_key(caps)
|
|
1359
|
+
prov = (item.get("prov") or [{}])[0]
|
|
1360
|
+
if not key or key in out or not prov.get("page_no"):
|
|
1361
|
+
continue
|
|
1362
|
+
png = crop_png(pdf, prov["page_no"], docling_box(prov["bbox"], page_h.get(prov["page_no"], 792)), page_h.get(prov["page_no"], 792))
|
|
1363
|
+
u = data_uri(png) if png else None
|
|
1364
|
+
if u:
|
|
1365
|
+
out[key] = (u, prov["page_no"])
|
|
1366
|
+
return out
|
|
1367
|
+
|
|
1368
|
+
|
|
1369
|
+
def docling_box(bb, page_h):
|
|
1370
|
+
"""Docling bbox -> (l, t, r, b) in PDF points, origin bottom left."""
|
|
1371
|
+
if bb.get("coord_origin", "BOTTOMLEFT") == "BOTTOMLEFT":
|
|
1372
|
+
return (bb["l"], bb["t"], bb["r"], bb["b"])
|
|
1373
|
+
return (bb["l"], page_h - bb["t"], bb["r"], page_h - bb["b"])
|
|
1374
|
+
|
|
1375
|
+
|
|
1376
|
+
def find_pdf_page(doc, label):
|
|
1377
|
+
"""The PDF page whose text starts a line with this figure label."""
|
|
1378
|
+
if not label:
|
|
1379
|
+
return None
|
|
1380
|
+
txt = source_text(doc.meta)
|
|
1381
|
+
if not txt:
|
|
1382
|
+
return None
|
|
1383
|
+
pages = txt.read_text(encoding="utf-8", errors="replace").split("\f")
|
|
1384
|
+
pat = re.compile(r"^\s*" + re.escape(label.rstrip(".:")) + r"[.:]", re.M)
|
|
1385
|
+
for i, p in enumerate(pages):
|
|
1386
|
+
if pat.search(p):
|
|
1387
|
+
return i + 1
|
|
1388
|
+
return None
|
|
1389
|
+
|
|
1390
|
+
|
|
1391
|
+
# --------------------------------------------------------------------------
|
|
1392
|
+
# Route: JATS (Europe PMC)
|
|
1393
|
+
# --------------------------------------------------------------------------
|
|
1394
|
+
|
|
1395
|
+
def parse_xml(path):
|
|
1396
|
+
try:
|
|
1397
|
+
return ET.parse(str(path)).getroot()
|
|
1398
|
+
except ET.ParseError as e:
|
|
1399
|
+
raise ReaderError("the article's XML can't be read: %s" % e)
|
|
1400
|
+
|
|
1401
|
+
|
|
1402
|
+
def jats_image(doc, href):
|
|
1403
|
+
base = os.path.basename(href or "")
|
|
1404
|
+
f = next((doc._imgdir / (base + e) for e in ("", ".jpg", ".jpeg", ".png", ".gif") if base and (doc._imgdir / (base + e)).is_file()), None)
|
|
1405
|
+
return data_uri(f.read_bytes()) if f is not None else None
|
|
1406
|
+
|
|
1407
|
+
|
|
1408
|
+
def formula_html(n, doc):
|
|
1409
|
+
"""MathML if the article has it, then its picture of the equation, then its TeX source."""
|
|
1410
|
+
m = n.find(lambda x: x.tag == "math")
|
|
1411
|
+
if m is not None:
|
|
1412
|
+
return math_html(m)
|
|
1413
|
+
g = n.find(lambda x: x.tag in ("graphic", "inline-graphic"))
|
|
1414
|
+
u = jats_image(doc, g.attrs.get("href")) if g is not None else None
|
|
1415
|
+
if u:
|
|
1416
|
+
return img_tag(u, "Equation")
|
|
1417
|
+
tex = n.find(lambda x: x.tag == "tex-math")
|
|
1418
|
+
return "<code>%s</code>" % esc(tex.text().strip()) if tex is not None else ""
|
|
1419
|
+
|
|
1420
|
+
|
|
1421
|
+
class JatsInline(Inline):
|
|
1422
|
+
def rule(self, n):
|
|
1423
|
+
if n.tag in ("name", "string-name") and n.find(lambda x: x.tag == "surname") is not None:
|
|
1424
|
+
sur = n.find(lambda x: x.tag == "surname").text().strip()
|
|
1425
|
+
giv = n.find(lambda x: x.tag == "given-names")
|
|
1426
|
+
return esc(sur + (" " + giv.text().strip() if giv is not None else ""))
|
|
1427
|
+
if n.tag == "xref":
|
|
1428
|
+
kind = n.attrs.get("ref-type")
|
|
1429
|
+
rid = n.attrs.get("rid", "")
|
|
1430
|
+
if kind == "bibr":
|
|
1431
|
+
ids = rid.split()
|
|
1432
|
+
r = re.fullmatch(r"\s*(\d{1,3})\s*[–—-]\s*(\d{1,3})\s*", n.text())
|
|
1433
|
+
m = re.fullmatch(r"(.*?)0*%s" % r.group(1), ids[0]) if r and len(ids) == 1 else None
|
|
1434
|
+
if m and int(r.group(1)) < int(r.group(2)) <= int(r.group(1)) + 50: # "1–3" with only the first reference tagged
|
|
1435
|
+
width = len(ids[0]) - len(m.group(1))
|
|
1436
|
+
wanted = [m.group(1) + str(k).zfill(width) for k in range(int(r.group(1)), int(r.group(2)) + 1)]
|
|
1437
|
+
ids = [x for x in wanted if x in self.doc._jats_refs] or ids
|
|
1438
|
+
return self.doc.cite(ids, self.run(n))
|
|
1439
|
+
if kind in ("fig", "table", "sec", "disp-formula", "supplementary-material"):
|
|
1440
|
+
return '<a href="#j-%s">%s</a>' % (safe_id(rid.split()[0]) if rid else "", self.run(n))
|
|
1441
|
+
if kind == "fn":
|
|
1442
|
+
fn = self.doc._jats_fns.get(rid)
|
|
1443
|
+
if fn is not None:
|
|
1444
|
+
self.doc._jats_cited.add(rid)
|
|
1445
|
+
return self.doc.note(self.run(fn).strip())
|
|
1446
|
+
return self.run(n)
|
|
1447
|
+
if n.tag in ("inline-formula", "disp-formula", "alternatives"):
|
|
1448
|
+
return formula_html(n, self.doc) or self.run(n)
|
|
1449
|
+
if n.tag in ("label",) and n.parent is not None and n.parent.tag in ("fig", "table-wrap", "ref"):
|
|
1450
|
+
return ""
|
|
1451
|
+
if n.tag == "label" and n.parent is not None and n.parent.tag == "fn": # a table's notes need their marks
|
|
1452
|
+
in_table = n.parent.parent is not None and n.parent.parent.tag == "table-wrap-foot"
|
|
1453
|
+
return self.run(n).strip() + " " if in_table else ""
|
|
1454
|
+
if n.tag == "label" and n.parent is not None and n.parent.tag == "list-item":
|
|
1455
|
+
return self.run(n).strip() + " "
|
|
1456
|
+
if n.tag == "title" and n.parent is not None and n.parent.tag == "caption":
|
|
1457
|
+
return self.run(n).strip() + " " # the shared whitelist drops <title>, but here it is the caption's first line
|
|
1458
|
+
if n.tag == "ext-link":
|
|
1459
|
+
href = safe_href(n.attrs.get("href", ""))
|
|
1460
|
+
inner = self.run(n)
|
|
1461
|
+
return '<a href="%s" rel="noopener noreferrer">%s</a>' % (attr(href), inner) if href else inner
|
|
1462
|
+
return None
|
|
1463
|
+
|
|
1464
|
+
|
|
1465
|
+
def structured_ref(ix, cit):
|
|
1466
|
+
"""An element-citation has fields but no punctuation between them."""
|
|
1467
|
+
names = [ix.one(n).strip() for n in cit.findall(lambda x: x.tag in ("name", "string-name", "collab"))]
|
|
1468
|
+
fields = [ix.run(c).strip() for c in cit.children() if c.tag not in ("person-group", "name", "string-name", "collab")]
|
|
1469
|
+
return ". ".join(x for x in [", ".join(names)] + fields if x) + "."
|
|
1470
|
+
|
|
1471
|
+
|
|
1472
|
+
def jats_doc(meta, d):
|
|
1473
|
+
path = d / "jats.xml"
|
|
1474
|
+
if not path.exists():
|
|
1475
|
+
raise ReaderError("no JATS XML; run fetch --route jats first")
|
|
1476
|
+
root = et_to_node(parse_xml(path))
|
|
1477
|
+
doc = Doc(dict(meta))
|
|
1478
|
+
doc._jats_fns = {n.attrs.get("id"): n for n in root.findall(lambda x: x.tag == "fn") if n.attrs.get("id")}
|
|
1479
|
+
doc._jats_refs = {n.attrs.get("id") for n in root.findall(lambda x: x.tag == "ref")}
|
|
1480
|
+
doc._jats_cited = set()
|
|
1481
|
+
ix = JatsInline(doc)
|
|
1482
|
+
imgdir = d / "img"
|
|
1483
|
+
doc._imgdir = imgdir
|
|
1484
|
+
doc.pdf = source_pdf(meta)
|
|
1485
|
+
doc.meta["kicker"] = meta.get("venue") or "Article"
|
|
1486
|
+
doc.source = "Text from Europe PMC's full-text XML for this article; figures from Europe PMC's file bundle."
|
|
1487
|
+
art_meta = root.find(lambda x: x.tag == "article-meta")
|
|
1488
|
+
if art_meta is not None:
|
|
1489
|
+
t = art_meta.find(lambda x: x.tag == "article-title")
|
|
1490
|
+
if t is not None:
|
|
1491
|
+
for x in t.findall(lambda x: x.tag == "xref" and x.attrs.get("ref-type") == "fn"):
|
|
1492
|
+
x.kids, x.attrs["ref-type"] = [], "none" # a note on the title has no place in the heading or the browser tab
|
|
1493
|
+
doc.meta["title_html"] = ix.run(t).strip()
|
|
1494
|
+
all_abstracts = art_meta.findall(lambda x: x.tag == "abstract")
|
|
1495
|
+
full = [a for a in all_abstracts if a.attrs.get("abstract-type") not in ("toc", "teaser", "precis", "web-summary", "graphical")]
|
|
1496
|
+
for ab in full or all_abstracts:
|
|
1497
|
+
title = ab.find(lambda x: x.tag == "title" and x.parent is ab)
|
|
1498
|
+
head = title.text().strip() if title is not None else "Abstract"
|
|
1499
|
+
paras = []
|
|
1500
|
+
for p in ab.findall(lambda x: x.tag == "p"):
|
|
1501
|
+
sec = p.parent if p.parent is not None and p.parent.tag == "sec" else None
|
|
1502
|
+
label = sec.find(lambda x: x.tag == "title" and x.parent is sec) if sec is not None and sec.find(lambda x: x.tag == "p") is p else None
|
|
1503
|
+
paras.append(("<strong>%s</strong> " % ix.run(label).strip().rstrip(":.") if label is not None else "") + ix.run(p).strip())
|
|
1504
|
+
doc.abstracts.append((head, "".join("<p>%s</p>" % p for p in paras if p)))
|
|
1505
|
+
|
|
1506
|
+
def walk(n, depth, appendix=False):
|
|
1507
|
+
for k in n.children():
|
|
1508
|
+
t = k.tag
|
|
1509
|
+
if t == "sec" and k.find(lambda x: x.tag == "ref-list") is not None and k.find(lambda x: x.tag == "p") is None:
|
|
1510
|
+
continue # a section holding only the reference list, which has its own place on the page
|
|
1511
|
+
if t == "sec":
|
|
1512
|
+
title = k.find(lambda x: x.tag == "title" and x.parent is k)
|
|
1513
|
+
if title is not None:
|
|
1514
|
+
number = k.find(lambda x: x.tag == "label" and x.parent is k)
|
|
1515
|
+
doc.add(t="h", level=min(2 + depth, 4), html=ix.run(title).strip(), id="j-" + safe_id(k.attrs.get("id", "")) if k.attrs.get("id") else None,
|
|
1516
|
+
no=number.text().strip().rstrip(".") if number is not None else "", appendix=appendix)
|
|
1517
|
+
walk(k, depth + 1)
|
|
1518
|
+
elif t == "p":
|
|
1519
|
+
# a <p> can hold a figure or table; pull those out after the text
|
|
1520
|
+
# equations and lists stay where they fall in the sentence; figures and tables follow the paragraph
|
|
1521
|
+
floats, cur = [c for c in k.children() if c.tag in ("fig", "table-wrap")], Node("p", parent=k.parent)
|
|
1522
|
+
|
|
1523
|
+
def emit():
|
|
1524
|
+
h = ix.run(cur).strip()
|
|
1525
|
+
if h:
|
|
1526
|
+
doc.add(t="p", html=h)
|
|
1527
|
+
cur.kids = []
|
|
1528
|
+
for kid in k.kids:
|
|
1529
|
+
if isinstance(kid, Node) and kid.tag in ("disp-formula", "list"):
|
|
1530
|
+
emit()
|
|
1531
|
+
walk_one(kid)
|
|
1532
|
+
elif not (isinstance(kid, Node) and kid.tag in ("fig", "table-wrap")):
|
|
1533
|
+
cur.kids.append(kid)
|
|
1534
|
+
emit()
|
|
1535
|
+
for f in floats:
|
|
1536
|
+
walk_one(f)
|
|
1537
|
+
elif t in ("fig", "table-wrap", "disp-formula", "list"):
|
|
1538
|
+
walk_one(k)
|
|
1539
|
+
elif t in ("supplementary-material",):
|
|
1540
|
+
cap = k.find(lambda x: x.tag == "caption")
|
|
1541
|
+
if cap is not None:
|
|
1542
|
+
doc.add(t="p", html="<em>Supplementary material (not held locally):</em> " + ix.run(cap).strip())
|
|
1543
|
+
elif t == "def-list":
|
|
1544
|
+
items = []
|
|
1545
|
+
for di in k.findall(lambda x: x.tag == "def-item"):
|
|
1546
|
+
term, meaning = di.find(lambda x: x.tag == "term"), di.find(lambda x: x.tag == "def")
|
|
1547
|
+
items.append(" ".join(x for x in ["<strong>%s</strong>" % ix.run(term).strip() if term is not None else "",
|
|
1548
|
+
ix.run(meaning).strip() if meaning is not None else ""] if x))
|
|
1549
|
+
doc.add(t="list", ordered=False, plain=True, items=items)
|
|
1550
|
+
elif t in ("title", "label", "ref-list"):
|
|
1551
|
+
continue
|
|
1552
|
+
else:
|
|
1553
|
+
walk(k, depth, appendix)
|
|
1554
|
+
|
|
1555
|
+
def walk_one(k):
|
|
1556
|
+
t = k.tag
|
|
1557
|
+
if t == "fig" or t == "table-wrap":
|
|
1558
|
+
label = k.find(lambda x: x.tag == "label" and x.parent is k)
|
|
1559
|
+
cap = k.find(lambda x: x.tag == "caption")
|
|
1560
|
+
imgs = []
|
|
1561
|
+
for g in k.findall(lambda x: x.tag == "graphic"):
|
|
1562
|
+
href = g.attrs.get("href", "")
|
|
1563
|
+
base = os.path.basename(href)
|
|
1564
|
+
f = next((imgdir / (base + e) for e in ("", ".jpg", ".jpeg", ".png", ".gif") if base and (imgdir / (base + e)).is_file()), None)
|
|
1565
|
+
if f is not None:
|
|
1566
|
+
u = data_uri(f.read_bytes())
|
|
1567
|
+
if u:
|
|
1568
|
+
imgs.append((u, "%s, shown as an image" % (label.text().strip().rstrip(".:") if label is not None else ("Table" if t == "table-wrap" else "Figure"))))
|
|
1569
|
+
if len(imgs) > 1: # publishers often add a small preview of the same figure
|
|
1570
|
+
biggest = max(len(u) for u, _ in imgs)
|
|
1571
|
+
imgs = [x for x in imgs if len(x[0]) >= biggest * 0.2]
|
|
1572
|
+
table = ""
|
|
1573
|
+
tb = k.find(lambda x: x.tag == "table")
|
|
1574
|
+
if tb is not None:
|
|
1575
|
+
table = ix.one(tb)
|
|
1576
|
+
if table:
|
|
1577
|
+
imgs = [] # the same table supplied both as data and as a picture
|
|
1578
|
+
caption = ix.run(cap).strip() if cap is not None else ""
|
|
1579
|
+
foot = k.find(lambda x: x.tag == "table-wrap-foot")
|
|
1580
|
+
foot_text = " ".join(x for x in (ix.run(c).strip() for c in foot.children()) if x) if foot is not None else ""
|
|
1581
|
+
if foot_text:
|
|
1582
|
+
caption += ' <small class="tnote">%s</small>' % foot_text
|
|
1583
|
+
doc.add(t="fig", kind="table" if t == "table-wrap" else "figure",
|
|
1584
|
+
id="j-" + safe_id(k.attrs.get("id", "")) if k.attrs.get("id") else None,
|
|
1585
|
+
label=label.text().strip() if label is not None else "",
|
|
1586
|
+
caption=caption, imgs=imgs, table=table)
|
|
1587
|
+
elif t == "disp-formula":
|
|
1588
|
+
lab = k.find(lambda x: x.tag == "label" and x.parent is k)
|
|
1589
|
+
html_ = formula_html(k, doc)
|
|
1590
|
+
target = '<span id="j-%s"></span>' % safe_id(k.attrs["id"]) if k.attrs.get("id") else ""
|
|
1591
|
+
doc.add(t="eq", html=target + (html_ + (" " + esc(lab.text().strip()) if lab is not None else "") if html_ else ix.run(k)))
|
|
1592
|
+
elif t == "list":
|
|
1593
|
+
items = [ix.run(li).strip() for li in k.children() if li.tag == "list-item"]
|
|
1594
|
+
labelled = any(c.tag == "label" for li in k.children() if li.tag == "list-item" for c in li.children())
|
|
1595
|
+
doc.add(t="list", ordered=not labelled and k.attrs.get("list-type") in ("order", "arabic", "alpha-lower", "roman-lower"),
|
|
1596
|
+
plain=labelled, items=items)
|
|
1597
|
+
|
|
1598
|
+
body = root.find(lambda x: x.tag == "body")
|
|
1599
|
+
if body is not None:
|
|
1600
|
+
if body.children() and body.children()[0].tag == "p":
|
|
1601
|
+
doc.add(t="h", level=2, html="Introduction", id="j-intro")
|
|
1602
|
+
walk(body, 0)
|
|
1603
|
+
back = root.find(lambda x: x.tag == "back")
|
|
1604
|
+
if back is not None:
|
|
1605
|
+
ack = back.find(lambda x: x.tag == "ack")
|
|
1606
|
+
if ack is not None:
|
|
1607
|
+
doc.add(t="h", level=2, html="Acknowledgments", appendix=True)
|
|
1608
|
+
for p in ack.findall(lambda x: x.tag == "p"):
|
|
1609
|
+
doc.add(t="p", html=ix.run(p).strip())
|
|
1610
|
+
for k in back.children(): # appendices and sections such as data availability
|
|
1611
|
+
for sec in ([k] if k.tag in ("sec", "notes") else k.findall(lambda x: x.tag == "app") if k.tag == "app-group" else [k] if k.tag == "app" else []):
|
|
1612
|
+
title = sec.find(lambda x: x.tag == "title" and x.parent is sec)
|
|
1613
|
+
doc.add(t="h", level=2, html=ix.run(title).strip() if title is not None else ("Notes" if sec.tag == "notes" else "Appendix"),
|
|
1614
|
+
appendix=not any(b.get("appendix") for b in doc.blocks))
|
|
1615
|
+
walk(sec, 1)
|
|
1616
|
+
for ref in root.findall(lambda x: x.tag == "ref"): # usually in <back>, but some articles keep them in the body
|
|
1617
|
+
cit = ref.find(lambda x: x.tag in ("mixed-citation", "element-citation", "citation"))
|
|
1618
|
+
if cit is not None:
|
|
1619
|
+
doc.refs.append((ref.attrs.get("id", ""), structured_ref(ix, cit) if cit.tag == "element-citation" else ix.run(cit).strip()))
|
|
1620
|
+
fns = [f for f in root.findall(lambda x: x.tag == "fn" and x.parent is not None and x.parent.tag == "fn-group")
|
|
1621
|
+
if f.attrs.get("id") not in doc._jats_cited] # notes already shown where they are cited
|
|
1622
|
+
notes = root.findall(lambda x: x.tag == "author-notes")
|
|
1623
|
+
extra = [ix.run(f).strip() for f in fns] + [ix.run(fn).strip() for an in notes for fn in an.findall(lambda x: x.tag == "fn")]
|
|
1624
|
+
extra = [e for e in extra if e]
|
|
1625
|
+
if extra:
|
|
1626
|
+
first_app = not any(b.get("appendix") for b in doc.blocks)
|
|
1627
|
+
doc.add(t="h", level=2, html="Article notes", appendix=first_app)
|
|
1628
|
+
for e in extra:
|
|
1629
|
+
doc.add(t="p", html=e)
|
|
1630
|
+
return doc
|
|
1631
|
+
|
|
1632
|
+
|
|
1633
|
+
# --------------------------------------------------------------------------
|
|
1634
|
+
# Route: PDF via Docling
|
|
1635
|
+
# --------------------------------------------------------------------------
|
|
1636
|
+
|
|
1637
|
+
def run_docling(pdf, d, pages=None):
|
|
1638
|
+
"""Convert a private copy of the PDF, so Docling never touches the original."""
|
|
1639
|
+
(d / "in").mkdir(parents=True, exist_ok=True)
|
|
1640
|
+
(d / "docling").mkdir(exist_ok=True)
|
|
1641
|
+
src = d / "in" / "paper.pdf"
|
|
1642
|
+
if Path(pdf).resolve() != src.resolve():
|
|
1643
|
+
shutil.copyfile(str(pdf), str(src))
|
|
1644
|
+
exe = shutil.which("docling") or str(HOME / ".local" / "bin" / "docling")
|
|
1645
|
+
if not Path(exe).exists():
|
|
1646
|
+
raise ReaderError("PDFs need Docling: uv tool install docling --python 3.12")
|
|
1647
|
+
cmd = [exe, str(src), "--to", "json", "--image-export-mode", "placeholder", "--output", str(d / "docling")]
|
|
1648
|
+
if pages:
|
|
1649
|
+
cmd += ["--page-range", pages] # one argument, "253-314"
|
|
1650
|
+
print("running Docling on %s ..." % Path(pdf).name)
|
|
1651
|
+
r = subprocess.run(cmd, cwd=str(d), stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
|
|
1652
|
+
if r.returncode != 0:
|
|
1653
|
+
raise ReaderError("Docling failed:\n" + r.stdout.decode("utf-8", "replace")[-1500:])
|
|
1654
|
+
print("Docling JSON in %s" % (d / "docling"))
|
|
1655
|
+
|
|
1656
|
+
|
|
1657
|
+
def cmd_pdf_convert(a):
|
|
1658
|
+
meta = ledger_meta(a.ledger, a.pid)
|
|
1659
|
+
pdf = cache_file("pdf", meta["pid"], ".pdf")
|
|
1660
|
+
if not pdf:
|
|
1661
|
+
raise ReaderError("no cached PDF for %s" % a.pid)
|
|
1662
|
+
run_docling(pdf, SRC / slug(meta), a.pages)
|
|
1663
|
+
|
|
1664
|
+
|
|
1665
|
+
def crop_png(pdf, page, bbox, page_h, dpi=170):
|
|
1666
|
+
"""Render one region of one PDF page with pdftoppm. bbox is
|
|
1667
|
+
(l, t, r, b) in PDF points with the origin at the bottom left."""
|
|
1668
|
+
l, t, r, b = bbox
|
|
1669
|
+
scale = dpi / 72.0
|
|
1670
|
+
pad = 4
|
|
1671
|
+
x, y = max(0, int((l - pad) * scale)), max(0, int((page_h - t - pad) * scale))
|
|
1672
|
+
w, h = int((r - l + 2 * pad) * scale), int((t - b + 2 * pad) * scale)
|
|
1673
|
+
if w < 10 or h < 10:
|
|
1674
|
+
return None
|
|
1675
|
+
WORK.mkdir(parents=True, exist_ok=True)
|
|
1676
|
+
with tempfile.TemporaryDirectory(dir=str(WORK)) as tmp: # its own folder, so simultaneous builds can't mix up crops
|
|
1677
|
+
root = Path(tmp) / "crop"
|
|
1678
|
+
png = Path(tmp) / "crop.png"
|
|
1679
|
+
# pdftoppm 26 writes nothing when the output root is "-", so go through a file
|
|
1680
|
+
try:
|
|
1681
|
+
r_ = subprocess.run(["pdftoppm", "-f", str(page), "-l", str(page), "-r", str(dpi), "-x", str(x), "-y", str(y),
|
|
1682
|
+
"-W", str(w), "-H", str(h), "-png", "-singlefile", "--", str(pdf), str(root)],
|
|
1683
|
+
stdout=subprocess.PIPE, stderr=subprocess.PIPE, timeout=60)
|
|
1684
|
+
except subprocess.TimeoutExpired:
|
|
1685
|
+
return None # one very slow page shouldn't stop the build
|
|
1686
|
+
return png.read_bytes() if r_.returncode == 0 and png.exists() else None
|
|
1687
|
+
|
|
1688
|
+
|
|
1689
|
+
WATERMARK = re.compile(r"^(Downloaded from |Downloaded by |This article is protected by copyright|Copyright ©)", re.I)
|
|
1690
|
+
DICT = None
|
|
1691
|
+
|
|
1692
|
+
|
|
1693
|
+
def word_list():
|
|
1694
|
+
global DICT
|
|
1695
|
+
if DICT is None:
|
|
1696
|
+
p = Path("/usr/share/dict/words")
|
|
1697
|
+
DICT = set(w.strip().lower() for w in p.read_text(errors="ignore").splitlines()) if p.exists() else set()
|
|
1698
|
+
return DICT
|
|
1699
|
+
|
|
1700
|
+
|
|
1701
|
+
def join_drop_cap(text):
|
|
1702
|
+
"""'H uman' is a drop cap read as its own word; 'T cells' is not."""
|
|
1703
|
+
m = re.match(r"^([B-HJ-Z]) ([a-z]+)", text)
|
|
1704
|
+
if m and m.group(2) not in ("is", "are", "was", "were", "be", "in", "of", "and", "or", "to", "at", "on", "as", "has", "can", "may", "cells"):
|
|
1705
|
+
w = (m.group(1) + m.group(2)).lower()
|
|
1706
|
+
stems = [w, w[:-1] if w.endswith("s") else w, w[:-2] if w.endswith("ed") else w, w[:-1] if w.endswith("ed") else w,
|
|
1707
|
+
w[:-3] if w.endswith("ing") else w, w[:-3] + "e" if w.endswith("ing") else w]
|
|
1708
|
+
if any(x in word_list() for x in stems):
|
|
1709
|
+
return m.group(1) + text[2:]
|
|
1710
|
+
return text
|
|
1711
|
+
|
|
1712
|
+
|
|
1713
|
+
def dehyphen(a, b):
|
|
1714
|
+
"""Join 'perspec-' + 'tives' when the result is a word."""
|
|
1715
|
+
word_list()
|
|
1716
|
+
m1, m2 = re.search(r"(\w+)-$", a), re.match(r"([a-z]+)", b)
|
|
1717
|
+
if m1 and m2:
|
|
1718
|
+
joined = (m1.group(1) + m2.group(1)).lower()
|
|
1719
|
+
if joined in DICT or (joined.rstrip("s") in DICT) or (joined[:-2] in DICT and joined.endswith("ed")):
|
|
1720
|
+
return a[:-1] + b
|
|
1721
|
+
return a + b
|
|
1722
|
+
|
|
1723
|
+
|
|
1724
|
+
def docling_table_html(item):
|
|
1725
|
+
"""Docling's reading of a table as an HTML table, or "". It goes under the
|
|
1726
|
+
table's image, marked unchecked, so screen readers and phones get the data."""
|
|
1727
|
+
data = item.get("data") or {}
|
|
1728
|
+
cells = [c for c in data.get("table_cells") or [] if isinstance(c, dict)]
|
|
1729
|
+
try: # from the cells themselves too, so a missing or low count can't empty or cut the table
|
|
1730
|
+
nrows = max([data.get("num_rows") or 0] + [int(c.get("end_row_offset_idx") or 0) for c in cells])
|
|
1731
|
+
ncols = max([data.get("num_cols") or 0] + [int(c.get("end_col_offset_idx") or 0) for c in cells])
|
|
1732
|
+
except (TypeError, ValueError):
|
|
1733
|
+
return ""
|
|
1734
|
+
if nrows < 2 or not any((c.get("text") or "").strip() for c in cells):
|
|
1735
|
+
return ""
|
|
1736
|
+
if ncols < 1:
|
|
1737
|
+
return ""
|
|
1738
|
+
starts, covered = {}, set() # Docling leaves empty cells out, so each cell is placed by its offsets
|
|
1739
|
+
for c in cells:
|
|
1740
|
+
try:
|
|
1741
|
+
r0, r1 = int(c["start_row_offset_idx"]), int(c["end_row_offset_idx"])
|
|
1742
|
+
c0, c1 = int(c["start_col_offset_idx"]), int(c["end_col_offset_idx"])
|
|
1743
|
+
except (KeyError, TypeError, ValueError):
|
|
1744
|
+
return "" # not the shape we know: offer nothing rather than a garbled table
|
|
1745
|
+
r1, c1 = min(max(r1, r0 + 1), nrows), min(max(c1, c0 + 1), ncols)
|
|
1746
|
+
span = set((r, k) for r in range(r0, r1) for k in range(c0, c1))
|
|
1747
|
+
if not (0 <= r0 < nrows and 0 <= c0 < ncols) or span & covered:
|
|
1748
|
+
continue # overlaps a cell already placed: keep the first, so every row keeps its width
|
|
1749
|
+
starts[(r0, c0)] = (r1 - r0, c1 - c0, c)
|
|
1750
|
+
covered |= span
|
|
1751
|
+
out = []
|
|
1752
|
+
for r in range(nrows):
|
|
1753
|
+
tds, k = [], 0
|
|
1754
|
+
while k < ncols:
|
|
1755
|
+
if (r, k) in starts:
|
|
1756
|
+
rs, cs, c = starts[(r, k)]
|
|
1757
|
+
head = c.get("column_header") or c.get("row_header")
|
|
1758
|
+
tag = "th" if head else "td"
|
|
1759
|
+
scope = (' scope="col"' if c.get("column_header") else ' scope="row"') if head else "" # col and row already cover every column or row spanned
|
|
1760
|
+
span = (' rowspan="%d"' % rs if rs > 1 else "") + (' colspan="%d"' % cs if cs > 1 else "")
|
|
1761
|
+
tds.append("<%s%s%s>%s</%s>" % (tag, scope, span, esc((c.get("text") or "").strip()), tag))
|
|
1762
|
+
k += cs
|
|
1763
|
+
else:
|
|
1764
|
+
if (r, k) not in covered:
|
|
1765
|
+
tds.append("<td></td>") # an empty cell, or one Docling didn't read
|
|
1766
|
+
k += 1
|
|
1767
|
+
out.append("<tr>%s</tr>" % "".join(tds))
|
|
1768
|
+
return '<table class="data">%s</table>' % "".join(out)
|
|
1769
|
+
|
|
1770
|
+
|
|
1771
|
+
def docling_doc(meta, d, pages=None):
|
|
1772
|
+
js = sorted((d / "docling").glob("*.json"))
|
|
1773
|
+
if not js:
|
|
1774
|
+
raise ReaderError("no Docling output; run pdf-convert first")
|
|
1775
|
+
j = json.loads(js[0].read_text(encoding="utf-8"))
|
|
1776
|
+
doc = Doc(dict(meta))
|
|
1777
|
+
doc.pdf = source_pdf(meta)
|
|
1778
|
+
doc.meta["kicker"] = meta.get("venue") or "Article"
|
|
1779
|
+
doc.source = "Text converted from the PDF by Docling; figures and tables are images cropped from the PDF pages."
|
|
1780
|
+
page_h = {int(k): v["size"]["height"] for k, v in j.get("pages", {}).items()}
|
|
1781
|
+
offset = 0
|
|
1782
|
+
if pages and page_h and min(page_h) == 1:
|
|
1783
|
+
offset = int(pages.split("-")[0]) - 1 # only if Docling renumbered the range from 1
|
|
1784
|
+
|
|
1785
|
+
def resolve(ref):
|
|
1786
|
+
kind, idx = ref["$ref"].split("/")[1], int(ref["$ref"].split("/")[2])
|
|
1787
|
+
return kind, j[kind][idx]
|
|
1788
|
+
|
|
1789
|
+
# one figure can arrive as several pictures, with its labels read as text: gather them under the caption
|
|
1790
|
+
float_box, absorbed, figure_areas = {}, set(), {}
|
|
1791
|
+
pictures = [x for x in j.get("pictures", []) if x.get("prov")]
|
|
1792
|
+
|
|
1793
|
+
def area(it):
|
|
1794
|
+
pg = it["prov"][0]["page_no"]
|
|
1795
|
+
return docling_box(it["prov"][0]["bbox"], page_h.get(pg, 792))
|
|
1796
|
+
for pic in pictures:
|
|
1797
|
+
if not pic.get("captions"):
|
|
1798
|
+
continue
|
|
1799
|
+
pg = pic["prov"][0]["page_no"]
|
|
1800
|
+
l, t, r, b = area(pic)
|
|
1801
|
+
cap = resolve(pic["captions"][0])[1]
|
|
1802
|
+
cl, cr = (area(cap)[0], area(cap)[2]) if cap.get("prov") else (l, r)
|
|
1803
|
+
for other in pictures:
|
|
1804
|
+
if other is pic or other.get("captions") or other["prov"][0]["page_no"] != pg:
|
|
1805
|
+
continue
|
|
1806
|
+
ol, ot, orr, ob = area(other)
|
|
1807
|
+
if ob <= t + 30 and ot >= b - 30 and cl <= (ol + orr) / 2 <= cr: # under this caption, not in the next column
|
|
1808
|
+
l, t, r, b = min(l, ol), max(t, ot), max(r, orr), min(b, ob)
|
|
1809
|
+
absorbed.add(other.get("self_ref"))
|
|
1810
|
+
if cr - cl > 1.25 * (r - l): # the caption spans the whole figure; the picture found is only part of it
|
|
1811
|
+
l, r = min(l, cl), max(r, cr)
|
|
1812
|
+
float_box[pic.get("self_ref")] = (l, t, r, b)
|
|
1813
|
+
figure_areas.setdefault(pg, []).append((l, t + 12, r, b))
|
|
1814
|
+
head_counts = Counter((t.get("text") or "").strip() for t in j.get("texts", []) if t.get("label") == "section_header")
|
|
1815
|
+
|
|
1816
|
+
def figure_text(item, text):
|
|
1817
|
+
"""A label drawn inside a figure, which Docling reads as text or a heading."""
|
|
1818
|
+
if not item.get("prov") or len(text) >= 80:
|
|
1819
|
+
return False
|
|
1820
|
+
pg = item["prov"][0]["page_no"]
|
|
1821
|
+
il, it, ir, ib = area(item)
|
|
1822
|
+
cx, cy = (il + ir) / 2, (it + ib) / 2
|
|
1823
|
+
for l, t, r, b in figure_areas.get(pg, []):
|
|
1824
|
+
if l <= cx <= r and (b <= cy <= t or (head_counts[text] >= 2 and re.search(r"[^\W\d_]\d\b", text) and 0 <= ib - t <= 55)):
|
|
1825
|
+
return True
|
|
1826
|
+
return False
|
|
1827
|
+
|
|
1828
|
+
# reading order: the body tree, flattening groups
|
|
1829
|
+
order = []
|
|
1830
|
+
|
|
1831
|
+
def flatten(node):
|
|
1832
|
+
for ch in node.get("children", []):
|
|
1833
|
+
kind, item = resolve(ch)
|
|
1834
|
+
if kind == "groups":
|
|
1835
|
+
order.append(("group-start", item))
|
|
1836
|
+
flatten(item)
|
|
1837
|
+
order.append(("group-end", item))
|
|
1838
|
+
else:
|
|
1839
|
+
order.append((kind, item))
|
|
1840
|
+
if kind in ("pictures", "tables"):
|
|
1841
|
+
continue # captions are children; handled with the float
|
|
1842
|
+
flatten(item)
|
|
1843
|
+
flatten(j["body"])
|
|
1844
|
+
|
|
1845
|
+
# a repository's cover sheet ("Please cite the original version ... Take down policy") is not the paper
|
|
1846
|
+
cover = set()
|
|
1847
|
+
by_page = {}
|
|
1848
|
+
for t in j.get("texts", []):
|
|
1849
|
+
pg = (t.get("prov") or [{}])[0].get("page_no")
|
|
1850
|
+
by_page.setdefault(pg, []).append(t.get("text") or "")
|
|
1851
|
+
for pg, txts in by_page.items():
|
|
1852
|
+
blob = " ".join(txts).lower()
|
|
1853
|
+
if pg and pg <= 2 and sum(k in blob for k in ("please cite the original version", "take down policy",
|
|
1854
|
+
"general rights", "this is an electronic reprint",
|
|
1855
|
+
"self-archived", "to cite this article", "submit your article to this journal",
|
|
1856
|
+
"view related articles", "citing articles")) >= 2:
|
|
1857
|
+
cover.add(pg)
|
|
1858
|
+
names = set(re.sub(r"\W", "", a.lower()) for a in meta.get("authors") or [])
|
|
1859
|
+
venue = re.sub(r"\W", "", (meta.get("venue") or "").lower())
|
|
1860
|
+
|
|
1861
|
+
printed = printed_page_numbers(j)
|
|
1862
|
+
in_refs, refs_raw = False, []
|
|
1863
|
+
last_page = None
|
|
1864
|
+
pending = None # paragraph text waiting for its continuation
|
|
1865
|
+
list_items = None
|
|
1866
|
+
list_depth = 0 # a list inside a list joins the outer one
|
|
1867
|
+
list_numbered = False
|
|
1868
|
+
caption_ids = set()
|
|
1869
|
+
for kind, item in order:
|
|
1870
|
+
if kind in ("pictures", "tables"):
|
|
1871
|
+
for c in item.get("captions", []):
|
|
1872
|
+
caption_ids.add(c["$ref"])
|
|
1873
|
+
|
|
1874
|
+
total_words = sum(words(t.get("text") or "") for t in j.get("texts", []) if t.get("label") not in ("page_header", "page_footer"))
|
|
1875
|
+
deferred = []
|
|
1876
|
+
last_fn = [None] # the latest footnote, which may run on to the next page
|
|
1877
|
+
front_dropped = []
|
|
1878
|
+
seen_head = [False]
|
|
1879
|
+
summary = [None] # None, then "open" under a page-one summary, then "closed"
|
|
1880
|
+
pending_label = [None] # the run-in label of the held summary paragraph
|
|
1881
|
+
chapter_pre = [None]
|
|
1882
|
+
|
|
1883
|
+
def flush():
|
|
1884
|
+
nonlocal pending
|
|
1885
|
+
if pending is not None:
|
|
1886
|
+
doc.add(t="p", html=("<b>%s:</b> " % esc(pending_label[0]) if pending_label[0] else "") + esc(pending))
|
|
1887
|
+
pending = None
|
|
1888
|
+
pending_label[0] = None
|
|
1889
|
+
while deferred:
|
|
1890
|
+
fn = deferred.pop(0)
|
|
1891
|
+
fn["block"] = {"t": "p", "html": "<small>%s</small>" % esc(fn["text"])}
|
|
1892
|
+
doc.blocks.append(fn["block"])
|
|
1893
|
+
|
|
1894
|
+
def page_marker(p):
|
|
1895
|
+
nonlocal last_page
|
|
1896
|
+
if p and p != last_page:
|
|
1897
|
+
last_page = p
|
|
1898
|
+
if printed.get(p) == "":
|
|
1899
|
+
return
|
|
1900
|
+
label = printed.get(p) or str(p + offset)
|
|
1901
|
+
doc.add(t="page", label=label, pdf=p + offset)
|
|
1902
|
+
|
|
1903
|
+
for kind, item in order:
|
|
1904
|
+
if kind == "group-start":
|
|
1905
|
+
if item.get("label") in ("list", "ordered_list"):
|
|
1906
|
+
if list_depth == 0:
|
|
1907
|
+
flush()
|
|
1908
|
+
list_items = []
|
|
1909
|
+
list_numbered = False
|
|
1910
|
+
list_depth += 1
|
|
1911
|
+
continue
|
|
1912
|
+
if kind == "group-end":
|
|
1913
|
+
if list_depth and item.get("label") in ("list", "ordered_list"):
|
|
1914
|
+
list_depth -= 1
|
|
1915
|
+
if list_depth == 0:
|
|
1916
|
+
if list_items:
|
|
1917
|
+
doc.add(t="list", ordered=item.get("label") == "ordered_list" or list_numbered, items=list_items)
|
|
1918
|
+
list_items = None
|
|
1919
|
+
continue
|
|
1920
|
+
if item.get("self_ref") in caption_ids:
|
|
1921
|
+
continue
|
|
1922
|
+
label = item.get("label")
|
|
1923
|
+
prov = item.get("prov") or [{}]
|
|
1924
|
+
p = prov[0].get("page_no")
|
|
1925
|
+
if label in ("page_header", "page_footer") or p in cover:
|
|
1926
|
+
continue
|
|
1927
|
+
text = STAMP_IN_TEXT.sub("", item.get("text") or "").strip() # Docling can fuse a page's publisher line into a sentence
|
|
1928
|
+
if label in ("text", "paragraph", "list_item", "footnote", "caption", "section_header", "title", "formula", "code", "checkbox_selected", "checkbox_unselected", "reference"):
|
|
1929
|
+
if WATERMARK.match(text):
|
|
1930
|
+
continue
|
|
1931
|
+
if kind in ("pictures", "tables") and item.get("self_ref") in absorbed:
|
|
1932
|
+
continue
|
|
1933
|
+
if kind in ("pictures", "tables"):
|
|
1934
|
+
pnum = p
|
|
1935
|
+
caps = [resolve(c)[1].get("text", "") for c in item.get("captions", [])]
|
|
1936
|
+
cap_text = " ".join(caps).strip()
|
|
1937
|
+
lab = ""
|
|
1938
|
+
mm = re.match(r"^((?:Fig(?:ure)?|Table|TABLE|FIGURE)\.?\s*[\dA-Z]+[.:]?)\s*(.*)$", cap_text, re.S)
|
|
1939
|
+
if mm:
|
|
1940
|
+
lab, cap_text = mm.group(1), mm.group(2)
|
|
1941
|
+
imgs = []
|
|
1942
|
+
table_text = docling_table_html(item) if kind == "tables" else ""
|
|
1943
|
+
if pnum and doc.pdf:
|
|
1944
|
+
bb = prov[0]["bbox"]
|
|
1945
|
+
box = float_box.get(item.get("self_ref")) or docling_box(bb, page_h.get(pnum, 792))
|
|
1946
|
+
png = crop_png(doc.pdf, pnum + offset, box, page_h.get(pnum, 792))
|
|
1947
|
+
u = data_uri(png) if png else None
|
|
1948
|
+
if u:
|
|
1949
|
+
alt = "%s, shown as an image" % (lab.rstrip(".: ") or ("Table" if kind == "tables" else "Figure"))
|
|
1950
|
+
imgs.append((u, alt + (". The table as text follows it" if table_text else "")))
|
|
1951
|
+
if not imgs and not cap_text:
|
|
1952
|
+
continue
|
|
1953
|
+
bb = prov[0].get("bbox") or {}
|
|
1954
|
+
if not cap_text and (abs(bb.get("r", 0) - bb.get("l", 0)) < 120 or abs(bb.get("t", 0) - bb.get("b", 0)) < 80):
|
|
1955
|
+
continue # logos and icons
|
|
1956
|
+
held = None
|
|
1957
|
+
if not cap_text and pending and len(pending) < 400 and CAPTION_LIKE.match(pending):
|
|
1958
|
+
cm = CAPTION_LIKE.match(pending) # the caption above the float, left as a paragraph
|
|
1959
|
+
lab, cap_text, pending = cm.group(1), cm.group(2).strip(), None
|
|
1960
|
+
pending_label[0] = None # the held text was a caption, not a labelled summary part
|
|
1961
|
+
caption = esc(cap_text)
|
|
1962
|
+
gm = TABLE_NOTE_GLUED.search(pending) if (pending and kind == "tables") else None
|
|
1963
|
+
if gm: # Docling sometimes glues a table note onto the sentence the table cuts
|
|
1964
|
+
pending = pending[:gm.start()]
|
|
1965
|
+
caption += ' <small class="tnote">%s</small>' % esc(gm.group(1).strip())
|
|
1966
|
+
if pending is not None and not re.search(r"[.!?:;”\"')\]]$", pending):
|
|
1967
|
+
held, pending = pending, None # a sentence cut by the float: finish it after the float
|
|
1968
|
+
flush()
|
|
1969
|
+
page_marker(pnum)
|
|
1970
|
+
doc.add(t="fig", kind="table" if kind == "tables" else "figure", label=lab, caption=caption,
|
|
1971
|
+
imgs=imgs, table="", table_text=table_text if imgs else "", page=(pnum + offset) if pnum else None)
|
|
1972
|
+
pending = held
|
|
1973
|
+
continue
|
|
1974
|
+
if label == "formula" and not text: # Docling found an equation but couldn't read it: show it as printed
|
|
1975
|
+
png = crop_png(doc.pdf, p + offset, docling_box(prov[0]["bbox"], page_h.get(p, 792)), page_h.get(p, 792)) if doc.pdf and p else None
|
|
1976
|
+
u = data_uri(png) if png else None
|
|
1977
|
+
if u:
|
|
1978
|
+
held = None
|
|
1979
|
+
if pending is not None and not re.search(r"[.!?:;”\"')\]]$", pending):
|
|
1980
|
+
held, pending = pending, None # the sentence carries on after the equation
|
|
1981
|
+
flush()
|
|
1982
|
+
page_marker(p)
|
|
1983
|
+
doc.add(t="eq", html=img_tag(u, "Equation"))
|
|
1984
|
+
pending = held
|
|
1985
|
+
continue
|
|
1986
|
+
real_heading = label == "section_header" and p and p > 2 and re.fullmatch(r"perspectives?|reviews?|reports?|artificial intelligence", text, re.I)
|
|
1987
|
+
if not text or figure_text(item, text) or (furniture(text, doc.meta.get("title")) and not real_heading):
|
|
1988
|
+
continue
|
|
1989
|
+
last = doc.blocks[-1] if doc.blocks else None
|
|
1990
|
+
cm = CAPTION_LIKE.match(text)
|
|
1991
|
+
if cm and len(text) < 400 and last is not None and last["t"] == "fig" and not last.get("label") and not last.get("caption"):
|
|
1992
|
+
last["label"], last["caption"] = cm.group(1), esc(cm.group(2).strip())
|
|
1993
|
+
continue
|
|
1994
|
+
if last is not None and last["t"] == "fig" and len(text) < 600 and (
|
|
1995
|
+
re.match(r"(Notes?[:.]|Sources?:)\s", text) or (last.get("kind") == "table" and TABLE_NOTE.match(text))):
|
|
1996
|
+
last["caption"] += ' <small class="tnote">%s</small>' % esc(text) # "NS: Not significant ...", "Note: ..."
|
|
1997
|
+
continue
|
|
1998
|
+
if (pending is None and last is not None and last["t"] == "fig" and last.get("caption")
|
|
1999
|
+
and not re.search(r"[.!?)]$", strip_tags(last["caption"]).strip()) and re.match(r"^[a-z(]", text)):
|
|
2000
|
+
last["caption"] = esc(dehyphen(strip_tags(last["caption"]).strip(), text)) if last["caption"].endswith("-") \
|
|
2001
|
+
else last["caption"] + " " + esc(text) # a caption continued in the next column
|
|
2002
|
+
continue
|
|
2003
|
+
if label in ("section_header", "title") and re.sub(r"\W", "", text.lower()) in names | {venue}:
|
|
2004
|
+
continue # an author's name or the journal's name set as a heading
|
|
2005
|
+
if label in ("section_header", "title"):
|
|
2006
|
+
flush()
|
|
2007
|
+
seen_head[0] = True
|
|
2008
|
+
if REFS_HEAD.match(text):
|
|
2009
|
+
in_refs = True
|
|
2010
|
+
continue
|
|
2011
|
+
if in_refs and refs_raw:
|
|
2012
|
+
in_refs = False
|
|
2013
|
+
page_marker(p)
|
|
2014
|
+
if re.fullmatch(r"(Chapter|CHAPTER|Part|PART) [\dIVX]+", text):
|
|
2015
|
+
chapter_pre[0] = text # "Chapter 9" then "Restoring Trust": one heading
|
|
2016
|
+
continue
|
|
2017
|
+
if chapter_pre[0]:
|
|
2018
|
+
text, chapter_pre[0] = chapter_pre[0] + ": " + text, None
|
|
2019
|
+
lvl = 2 if (item.get("level") or 1) <= 1 else 3
|
|
2020
|
+
numbered = re.match(r"^\d{1,2}((?:\.\d{1,2})*)\.?\s", text)
|
|
2021
|
+
if numbered: # Docling often flattens levels; "3.2" says where it belongs
|
|
2022
|
+
lvl = 3 if numbered.group(1) else 2
|
|
2023
|
+
if text.isupper() and len(text) > 3:
|
|
2024
|
+
text = re.sub(r"[^\W\d_]", lambda c: c.group(0).upper(), text.lower(), count=1)
|
|
2025
|
+
# acknowledgments near the front of a book or thesis don't end the main text
|
|
2026
|
+
end_matter = bool(END_MATTER.match(text)) and not any(b.get("appendix") for b in doc.blocks) and (
|
|
2027
|
+
sum(words(strip_tags(block_text(b))) for b in doc.blocks) >= FRONT_MATTER_SHARE * total_words)
|
|
2028
|
+
doc.add(t="h", level=lvl, html=esc(text), appendix=end_matter)
|
|
2029
|
+
continue
|
|
2030
|
+
surnames = [a.split()[-1] for a in doc.meta.get("authors") or [] if a.split()]
|
|
2031
|
+
if not in_refs and p and p <= 2 and len(text) < 800 and len(surnames) >= 3 and sum(1 for s in surnames if s in text) >= 3:
|
|
2032
|
+
continue # the author line, with affiliation marks; the header shows the authors
|
|
2033
|
+
if not in_refs and not seen_head[0] and p and p <= 2 and len(text) < 1200 and len(re.findall(r"(?:^|\.\s+)\d{1,2}\s+[A-Z]", text)) >= 2 \
|
|
2034
|
+
and re.match(r"1\s", text) and re.search(r"\b(Universit|Institut|College|School|Department|Laborator|Centre|Center)", text):
|
|
2035
|
+
front_dropped.append(text) # a numbered affiliation list ("1 Google DeepMind, London, UK. 2 …"), set mid-column
|
|
2036
|
+
continue
|
|
2037
|
+
if not seen_head[0] and len(text.split()) < 25 and not re.match(r"keywords\b", text, re.I) and (
|
|
2038
|
+
not re.search(r"[.?!]$", text) or "@" in text or re.search(r"10\.\d{4,}/", text)):
|
|
2039
|
+
front_dropped.append(text) # front matter before the first heading: affiliations, article type, DOI lines
|
|
2040
|
+
continue
|
|
2041
|
+
if in_refs:
|
|
2042
|
+
mk = (item.get("marker") or "").strip()
|
|
2043
|
+
refs_raw.append((mk + " " + text) if re.fullmatch(r"\[?\d{1,3}[.\]]?", mk) else text)
|
|
2044
|
+
continue
|
|
2045
|
+
if label == "list_item" and list_items is not None:
|
|
2046
|
+
flush()
|
|
2047
|
+
list_numbered = list_numbered or bool(item.get("enumerated"))
|
|
2048
|
+
list_items.append(esc(re.sub(r"^[•·\-–]\s*", "", text)))
|
|
2049
|
+
continue
|
|
2050
|
+
if label == "footnote" and last is not None and last["t"] == "fig" and last.get("kind") == "table" and (
|
|
2051
|
+
TABLE_NOTE.match(text) or re.search(r"\bp\s?[<=≤]\s?0?\.\d", text)):
|
|
2052
|
+
last["caption"] += ' <small class="tnote">%s</small>' % esc(text)
|
|
2053
|
+
continue
|
|
2054
|
+
if label == "footnote":
|
|
2055
|
+
fn = last_fn[0]
|
|
2056
|
+
if (fn and p and fn["page"] and p > fn["page"] and re.match(r"^[a-z(]", text)
|
|
2057
|
+
and not re.search(r"[.!?:;”\"')\]]$", fn["text"])):
|
|
2058
|
+
# the rest of the previous page's last footnote, carried over the page break
|
|
2059
|
+
fn["text"] = dehyphen(fn["text"], text) if fn["text"].endswith("-") else fn["text"] + " " + text
|
|
2060
|
+
fn["page"] = p
|
|
2061
|
+
if fn.get("block"):
|
|
2062
|
+
fn["block"]["html"] = "<small>%s</small>" % esc(fn["text"])
|
|
2063
|
+
continue
|
|
2064
|
+
last_fn[0] = {"text": text, "page": p}
|
|
2065
|
+
deferred.append(last_fn[0]) # after the paragraph it interrupts, not inside it
|
|
2066
|
+
continue
|
|
2067
|
+
lab = SUMMARY_LABEL.match(text) if not seen_head[0] and label in ("text", "paragraph") else None
|
|
2068
|
+
if lab and re.fullmatch(r"KEY ?WORDS", lab.group(1).strip()):
|
|
2069
|
+
lab = None
|
|
2070
|
+
if lab: # a page-one summary in run-in labels: Science's "INTRODUCTION: …", "RATIONALE: …", with no heading over it
|
|
2071
|
+
flush()
|
|
2072
|
+
if not summary[0]:
|
|
2073
|
+
page_marker(p)
|
|
2074
|
+
doc.add(t="h", level=2, html="Abstract" if lab.group(1).strip() == "ABSTRACT" else "Summary")
|
|
2075
|
+
summary[0] = "open"
|
|
2076
|
+
pending = text[lab.end():] # held, so a part split at a column can still be joined
|
|
2077
|
+
pending_label[0] = None if lab.group(1).strip() == "ABSTRACT" else lab.group(1).strip().capitalize()
|
|
2078
|
+
continue
|
|
2079
|
+
if summary[0] == "open" and pending is not None and pending_label[0] and label in ("text", "paragraph") \
|
|
2080
|
+
and not re.search(r"[.!?:;”\"')\]]$", pending) and re.match(r"^[a-z(]", text):
|
|
2081
|
+
pending = dehyphen(pending, text) if pending.endswith("-") else pending + " " + text
|
|
2082
|
+
if p != last_page:
|
|
2083
|
+
page_marker(p)
|
|
2084
|
+
continue
|
|
2085
|
+
if summary[0] == "open" and not seen_head[0] and label in ("text", "paragraph") and len(text.split()) >= 40:
|
|
2086
|
+
flush() # the article proper starts here, untitled; without a heading it would sit inside the summary
|
|
2087
|
+
page_marker(p)
|
|
2088
|
+
doc.add(t="h", level=2, html="Introduction")
|
|
2089
|
+
summary[0] = "closed"
|
|
2090
|
+
if label == "formula":
|
|
2091
|
+
flush()
|
|
2092
|
+
doc.add(t="eq", html=esc(text))
|
|
2093
|
+
continue
|
|
2094
|
+
if label == "code":
|
|
2095
|
+
flush()
|
|
2096
|
+
doc.add(t="pre", text=text)
|
|
2097
|
+
continue
|
|
2098
|
+
# body text: join a paragraph broken by a column or page
|
|
2099
|
+
if pending is not None and (pending.endswith("-") or not re.search(r"[.!?:;”\"')\]]$", pending)) and re.match(r"^[a-z(]", text):
|
|
2100
|
+
pending = dehyphen(pending, text) if pending.endswith("-") else pending + " " + text
|
|
2101
|
+
if p != last_page:
|
|
2102
|
+
page_marker(p) # the marker lands before the joined paragraph; close enough
|
|
2103
|
+
continue
|
|
2104
|
+
flush()
|
|
2105
|
+
page_marker(p)
|
|
2106
|
+
pending = join_drop_cap(text)
|
|
2107
|
+
flush()
|
|
2108
|
+
link_endnotes(doc)
|
|
2109
|
+
link_references(doc, refs_raw)
|
|
2110
|
+
if front_dropped:
|
|
2111
|
+
print(" dropped %d short lines before the first heading: %s" % (
|
|
2112
|
+
len(front_dropped), " | ".join(t[:40] for t in front_dropped)[:400]))
|
|
2113
|
+
return doc
|
|
2114
|
+
|
|
2115
|
+
|
|
2116
|
+
SUMMARY_LABEL = re.compile(r"^((?:[A-Z]{3,} ?){1,3}):\s+(?=\S)") # "INTRODUCTION: ", "RATIONALE: "
|
|
2117
|
+
FRONT_MATTER_SHARE = 0.25 # end matter can't begin before this share of the document's words
|
|
2118
|
+
TABLE_NOTE = re.compile(r"^(Notes?[:.]\s|Sources?:\s|[A-Z][A-Za-z]{0,5}: [A-Z]?[a-z]|[*∗†‡]+\s?p\s?[<=≤])")
|
|
2119
|
+
CAPTION_LIKE = re.compile(r"^((?:Fig(?:ure)?|Table|TABLE|FIGURE)\.?\s*[A-Z]?\d+(?:\.\d+)?[.:])\s*(.+)$", re.S)
|
|
2120
|
+
TABLE_NOTE_GLUED = re.compile(r"(?<=[a-z,]) ((?:[A-Z]{1,5}: [a-z][a-z ,()-]{2,80}\.\s?)+)$")
|
|
2121
|
+
|
|
2122
|
+
FURNITURE = re.compile(
|
|
2123
|
+
r"^(research article( summary)?|research articles?|reports?|reviews?|perspectives?|artificial intelligence|"
|
|
2124
|
+
r"read the full article at|view the article online|permissions|use of this article is subject to.*|"
|
|
2125
|
+
r"cite this article as.*|the list of author affiliations is available in the full article online.*|downloaded from .*|this article is protected by copyright.*|"
|
|
2126
|
+
r"routledge|taylor & francis group|check for updates|open n? ?access)$", re.I)
|
|
2127
|
+
|
|
2128
|
+
|
|
2129
|
+
STAMP_IN_TEXT = re.compile(r"(?:[^©]{0,40}©\s*the author\(s\)\s+\d{4}\s*)?(?:https?://doi\.org/\S+\s+)?article reuse guidelines:.{0,160}?journals\.sagepub\.com/home/\w+\s*", re.I | re.S)
|
|
2130
|
+
PUBLISHER_STAMP = re.compile(r"provided proper attribution is provided|article reuse guidelines|journals-permissions|©\s*the author\(s\)\s+\d{4}", re.I)
|
|
2131
|
+
|
|
2132
|
+
|
|
2133
|
+
def furniture(text, title):
|
|
2134
|
+
"""Publisher page furniture: kicker labels, bare URLs, stray glyphs, and the
|
|
2135
|
+
title repeated on later pages."""
|
|
2136
|
+
t = text.strip()
|
|
2137
|
+
if len(t) < 400 and PUBLISHER_STAMP.search(t):
|
|
2138
|
+
return True
|
|
2139
|
+
if len(t) <= 2 or FURNITURE.match(t) or re.fullmatch(r"(https?://|www\.|doi\.org/|science\.org/)\S+", t):
|
|
2140
|
+
return True
|
|
2141
|
+
if title:
|
|
2142
|
+
norm = lambda s: re.sub(r"[^a-z0-9]", "", s.lower())
|
|
2143
|
+
if norm(t) and norm(t) == norm(title):
|
|
2144
|
+
return True
|
|
2145
|
+
return False
|
|
2146
|
+
|
|
2147
|
+
|
|
2148
|
+
END_MATTER = re.compile(r"^\s*(acknowledg|supplementary|funding|author contributions|competing interests|"
|
|
2149
|
+
r"declaration|data (and materials )?availability|disclosure|conflicts? of interest|notes on contributors|editor'?s summary)", re.I)
|
|
2150
|
+
|
|
2151
|
+
REFS_HEAD = re.compile(r"^\s*(references(\s+and\s+notes)?|bibliography|works cited|literature cited)\s*$", re.I)
|
|
2152
|
+
|
|
2153
|
+
|
|
2154
|
+
NOTES_HEAD = re.compile(r"^\s*(end)?notes\s*$", re.I)
|
|
2155
|
+
|
|
2156
|
+
|
|
2157
|
+
def link_endnotes(doc):
|
|
2158
|
+
"""Numbered endnotes whose markers lost their superscript. A marker is only
|
|
2159
|
+
recognised straight after punctuation ("in Korsholm. 2 The"), in order. One
|
|
2160
|
+
that can't be told from an ordinary number stays as plain text."""
|
|
2161
|
+
start = next((i for i, b in enumerate(doc.blocks) if b["t"] == "h" and NOTES_HEAD.match(strip_tags(b["html"]))), None)
|
|
2162
|
+
if start is None:
|
|
2163
|
+
return
|
|
2164
|
+
notes, count = {}, 0
|
|
2165
|
+
for b in doc.blocks[start + 1:]:
|
|
2166
|
+
if b["t"] == "h":
|
|
2167
|
+
break
|
|
2168
|
+
for h in (b["items"] if b["t"] == "list" else [b.get("html", "")] if b["t"] == "p" else []):
|
|
2169
|
+
count += 1
|
|
2170
|
+
m = re.match(r"^(?:<small>)?\s*(\d{1,3})[.)]?\s+(.*?)(?:</small>)?$", h, re.S)
|
|
2171
|
+
if m:
|
|
2172
|
+
notes[int(m.group(1))] = m.group(2)
|
|
2173
|
+
elif b["t"] == "list": # Docling keeps a list's numbers apart from its text
|
|
2174
|
+
notes[count] = h
|
|
2175
|
+
at, pos = 0, 0
|
|
2176
|
+
for n in sorted(notes):
|
|
2177
|
+
pat = re.compile(r"(?<=[^\W\d_][.,;:?!”’\")]) (%d)(?=\s|$)" % n)
|
|
2178
|
+
for i in range(at, start):
|
|
2179
|
+
b = doc.blocks[i]
|
|
2180
|
+
m = pat.search(b["html"], pos if i == at else 0) if b["t"] == "p" else None
|
|
2181
|
+
if m:
|
|
2182
|
+
note = doc.note(notes[n], n)
|
|
2183
|
+
b["html"] = b["html"][:m.start()] + note + b["html"][m.end():]
|
|
2184
|
+
at, pos = i, m.start() + len(note)
|
|
2185
|
+
break
|
|
2186
|
+
|
|
2187
|
+
|
|
2188
|
+
def link_references(doc, raw):
|
|
2189
|
+
"""Turn the PDF's reference list into doc.refs and link the citations in
|
|
2190
|
+
the text: numbered "(19)" / "(5, 8-10)" style, or author-year style."""
|
|
2191
|
+
if not raw:
|
|
2192
|
+
return
|
|
2193
|
+
numbered = sum(1 for r in raw if re.match(r"^\s*\[?\d{1,3}[.\])]?\s", r)) > len(raw) / 2
|
|
2194
|
+
if numbered:
|
|
2195
|
+
cur = None
|
|
2196
|
+
for r in raw:
|
|
2197
|
+
m = re.match(r"^\s*\[?(\d{1,3})[.\])]?\s+(.*)$", r, re.S)
|
|
2198
|
+
if m:
|
|
2199
|
+
cur = [m.group(1), m.group(2)]
|
|
2200
|
+
doc.refs.append(cur)
|
|
2201
|
+
elif cur is not None:
|
|
2202
|
+
cur[1] = dehyphen(cur[1], r) if cur[1].endswith("-") else cur[1] + " " + r
|
|
2203
|
+
doc.refs = [(n, esc(t)) for n, t in doc.refs]
|
|
2204
|
+
known = set(n for n, _ in doc.refs)
|
|
2205
|
+
|
|
2206
|
+
def numfix(m):
|
|
2207
|
+
before, after = m.string[max(0, m.start() - 14):m.start()], m.string[m.end():m.end() + 3]
|
|
2208
|
+
if re.search(r"(?i)\b(eqs?\.?|equations?|figs?\.?|figures?|tables?|sections?|steps?|models?|hypothes[ie]s)\s*$", before):
|
|
2209
|
+
return m.group(0) # "Eq. (3)" is not a citation
|
|
2210
|
+
if m.group(1) == "(" and re.fullmatch(r"\d", m.group(2)) and re.match(r"\s[a-z]", after) and re.search(r"(^|[:;,]|\b(?:and|or))\s*$", before):
|
|
2211
|
+
return m.group(0) # "two aims: (1) trust and (2) recall"
|
|
2212
|
+
out, linked = [], False
|
|
2213
|
+
for x in re.split(r"(\s*,\s*)", m.group(2)):
|
|
2214
|
+
r = re.fullmatch(r"(\d{1,3})\s*[–-]\s*(\d{1,3})", x)
|
|
2215
|
+
ids = [x] if x in known else []
|
|
2216
|
+
if r and int(r.group(1)) < int(r.group(2)) <= int(r.group(1)) + 50:
|
|
2217
|
+
ids = [str(k) for k in range(int(r.group(1)), int(r.group(2)) + 1) if str(k) in known]
|
|
2218
|
+
linked = linked or bool(ids)
|
|
2219
|
+
out.append(doc.cite(ids, x) if ids else x)
|
|
2220
|
+
return m.group(1) + "".join(out) + m.group(3) if linked else m.group(0)
|
|
2221
|
+
pat = re.compile(r"([(\[])(\d{1,3}(?:\s*[,–-]\s*\d{1,3})*)([)\]])")
|
|
2222
|
+
else:
|
|
2223
|
+
# author-year: a reference starts with a capitalised surname and contains a year
|
|
2224
|
+
refs, cur = [], None
|
|
2225
|
+
for r in raw:
|
|
2226
|
+
# "Abbott, A. (1988)", "Ackerman B and Fishkin JS (2002)", "van der Meer TWG (2017)", "Yle News (2019)"
|
|
2227
|
+
starts = re.match(r"^(?:(?:[Vv]an|[Vv]on|[Dd]e|[Dd]er|[Dd]en|[Dd]u|[Ll]a|[Ll]e)\s+)*[A-ZÀ-Þ][\w'’-]*(?:,| [A-ZÀ-Þ]{1,4}\b| \(| and | [A-ZÀ-Þ][a-zà-ÿ]+ \()", r) \
|
|
2228
|
+
and re.search(r"\b(1[89]\d\d|20\d\d)[a-z]?\b", r[:200])
|
|
2229
|
+
if starts or cur is None:
|
|
2230
|
+
cur = [r]
|
|
2231
|
+
refs.append(cur)
|
|
2232
|
+
else:
|
|
2233
|
+
cur.append(r)
|
|
2234
|
+
keys = {}
|
|
2235
|
+
for i, parts in enumerate(refs):
|
|
2236
|
+
t = " ".join(parts)
|
|
2237
|
+
rid = "r%d" % (i + 1)
|
|
2238
|
+
doc.refs.append((rid, esc(t)))
|
|
2239
|
+
sur = re.match(r"^(?:(?:[Vv]an|[Vv]on|[Dd]e|[Dd]er|[Dd]en|[Dd]u|[Ll]a|[Ll]e)\s+)*([A-ZÀ-Þ][\w'’-]+)", t)
|
|
2240
|
+
yr = re.search(r"(1[89]\d\d|20\d\d)([a-z]?)", t)
|
|
2241
|
+
if sur and yr:
|
|
2242
|
+
keys.setdefault((sur.group(1).lower(), yr.group(1) + yr.group(2)), rid)
|
|
2243
|
+
|
|
2244
|
+
def numfix(m):
|
|
2245
|
+
rid = keys.get((m.group(1).lower(), m.group(2) or m.group(3)))
|
|
2246
|
+
return doc.cite([rid], m.group(0)) if rid else m.group(0)
|
|
2247
|
+
pat = re.compile(r"\b(?:(?:[Vv]an|[Vv]on|[Dd]e|[Dd]er|[Dd]en|[Dd]u|[Ll]a|[Ll]e)\s+)*([A-ZÀ-Ý][\w'’-]+)(?:\s+et al\.?|\s+(?:and|&)\s+[A-ZÀ-Ý][\w'’-]+)?,?\s+(?:\(((?:1[89]|20)\d\d[a-z]?)\)|\(?((?:1[89]|20)\d\d[a-z]?)\b)")
|
|
2248
|
+
for b in doc.blocks:
|
|
2249
|
+
if b["t"] == "p":
|
|
2250
|
+
b["html"] = pat.sub(numfix, b["html"])
|
|
2251
|
+
elif b["t"] == "list":
|
|
2252
|
+
b["items"] = [pat.sub(numfix, x) for x in b["items"]]
|
|
2253
|
+
|
|
2254
|
+
|
|
2255
|
+
def printed_page_numbers(j):
|
|
2256
|
+
"""Printed page numbers from headers and footers, where they are bare numbers."""
|
|
2257
|
+
out = {}
|
|
2258
|
+
for t in j.get("texts", []):
|
|
2259
|
+
if t.get("label") in ("page_header", "page_footer"):
|
|
2260
|
+
s = (t.get("text") or "").strip()
|
|
2261
|
+
m = re.fullmatch(r"(?:Page\s+)?(\d{1,4})(?:\s+of\s+\d+)?", s)
|
|
2262
|
+
p = (t.get("prov") or [{}])[0].get("page_no")
|
|
2263
|
+
if m and p and p not in out:
|
|
2264
|
+
out[p] = m.group(1)
|
|
2265
|
+
# most pages share one gap between PDF page and printed page; use it to fill misses and overrule misreadings
|
|
2266
|
+
gaps = Counter(p - int(n) for p, n in out.items())
|
|
2267
|
+
if gaps:
|
|
2268
|
+
gap, seen = gaps.most_common(1)[0]
|
|
2269
|
+
if seen >= 2 and seen * 2 >= len(out):
|
|
2270
|
+
pages = [int(k) for k in j.get("pages", {})] or list(out)
|
|
2271
|
+
out = {p: str(p - gap) if p - gap >= 1 else "" for p in pages} # "": a cover page with no printed number
|
|
2272
|
+
return out
|
|
2273
|
+
|
|
2274
|
+
|
|
2275
|
+
# --------------------------------------------------------------------------
|
|
2276
|
+
# Coverage check
|
|
2277
|
+
# --------------------------------------------------------------------------
|
|
2278
|
+
|
|
2279
|
+
def tokens(text, pages_of=None):
|
|
2280
|
+
"""Lower-case words. With a list `pages_of`, also appends each word's page
|
|
2281
|
+
index, counted by the form feeds pdftotext puts between pages."""
|
|
2282
|
+
text = unicodedata.normalize("NFKC", text).lower()
|
|
2283
|
+
if pages_of is None:
|
|
2284
|
+
return re.findall(r"[a-z0-9]+", re.sub(r"(\w)-\s*\n\s*(\w)", r"\1\2", text))
|
|
2285
|
+
# a word hyphenated across a page break keeps the break as \x00 inside it, so the count stays right
|
|
2286
|
+
text = re.sub(r"(\w)-\s*\n\s*(\w)", lambda m: m.group(1) + "\x00" * m.group(0).count("\f") + m.group(2), text)
|
|
2287
|
+
out, page = [], 0
|
|
2288
|
+
for m in re.finditer(r"\f|[a-z0-9\x00]+", text):
|
|
2289
|
+
w = m.group()
|
|
2290
|
+
if w == "\f":
|
|
2291
|
+
page += 1
|
|
2292
|
+
continue
|
|
2293
|
+
breaks = w.count("\x00")
|
|
2294
|
+
w = w.replace("\x00", "")
|
|
2295
|
+
if w:
|
|
2296
|
+
out.append(w)
|
|
2297
|
+
pages_of.append(page)
|
|
2298
|
+
page += breaks
|
|
2299
|
+
return out
|
|
2300
|
+
|
|
2301
|
+
|
|
2302
|
+
def coverage(page_html, meta, pages=None, report=True):
|
|
2303
|
+
"""How much of the source text reached the page. Returns the counts and the
|
|
2304
|
+
missing runs, each with the PDF pages its words came from."""
|
|
2305
|
+
txt = source_text(meta)
|
|
2306
|
+
if not txt:
|
|
2307
|
+
if report:
|
|
2308
|
+
print(" coverage: no source text to compare against")
|
|
2309
|
+
return None
|
|
2310
|
+
raw = txt.read_text(encoding="utf-8", errors="replace")
|
|
2311
|
+
first = 1
|
|
2312
|
+
if pages: # compare against the converted pages only (pdftotext separates pages with form feeds)
|
|
2313
|
+
lo, hi = (int(x) for x in pages.split("-"))
|
|
2314
|
+
raw = "\f".join(raw.split("\f")[lo - 1:hi])
|
|
2315
|
+
first = lo
|
|
2316
|
+
pg = []
|
|
2317
|
+
src = tokens(raw, pg)
|
|
2318
|
+
body = re.sub(r"<(script|style)[^>]*>.*?</\1>", " ", page_html, flags=re.S)
|
|
2319
|
+
body = re.sub(r"<details class=\"(?:orig-page|table-text)\">.*?</details>", " ", body, flags=re.S) # aids, not the reading text
|
|
2320
|
+
body = re.sub(r'<img[^>]*>', " ", body)
|
|
2321
|
+
got = tokens(strip_tags(body))
|
|
2322
|
+
cs, cg = Counter(src), Counter(got)
|
|
2323
|
+
hit = sum(min(n, cg[w]) for w, n in cs.items())
|
|
2324
|
+
cov = {"hit": hit, "n": len(src), "label": meta.get("text_label", "cached text"), "runs": [], "first": first}
|
|
2325
|
+
grams = set(tuple(got[i:i + 5]) for i in range(len(got) - 4))
|
|
2326
|
+
miss = [tuple(src[i:i + 5]) not in grams for i in range(len(src) - 4)]
|
|
2327
|
+
runs, i = [], 0
|
|
2328
|
+
while i < len(miss):
|
|
2329
|
+
if miss[i]:
|
|
2330
|
+
k = i
|
|
2331
|
+
while k < len(miss) and miss[k]:
|
|
2332
|
+
k += 1
|
|
2333
|
+
if k - i >= 25:
|
|
2334
|
+
words_ = src[i:k + 4]
|
|
2335
|
+
cov["runs"].append({"n": k - i, "words": words_,
|
|
2336
|
+
"pages": Counter(first + p for p in pg[i:k + 4])})
|
|
2337
|
+
i = k
|
|
2338
|
+
else:
|
|
2339
|
+
i += 1
|
|
2340
|
+
cov["runs"].sort(key=lambda r: -r["n"])
|
|
2341
|
+
if report:
|
|
2342
|
+
print_coverage(cov)
|
|
2343
|
+
return cov
|
|
2344
|
+
|
|
2345
|
+
|
|
2346
|
+
def print_coverage(cov):
|
|
2347
|
+
print(" coverage: %.1f%% of the %d words in the %s reached the page" % (
|
|
2348
|
+
100.0 * cov["hit"] / max(1, cov["n"]), cov["n"], cov["label"]))
|
|
2349
|
+
for r in cov["runs"][:8]:
|
|
2350
|
+
print(" missing run of %d words: %s ..." % (r["n"], " ".join(r["words"][:18])))
|
|
2351
|
+
|
|
2352
|
+
|
|
2353
|
+
FRONT_MATTER_WORDS = {"copyright", "https", "http", "doi", "licence", "license", "licensed", "corresponding", "permissions"}
|
|
2354
|
+
|
|
2355
|
+
|
|
2356
|
+
def weak_pages(cov, limit=10):
|
|
2357
|
+
"""PDF pages where a long run of ordinary prose went missing: the pages worth
|
|
2358
|
+
showing as printed. Runs of numbers and labels (charts, tables) don't count,
|
|
2359
|
+
and nor do the licence and contact lines the conversion drops on purpose."""
|
|
2360
|
+
known = word_list()
|
|
2361
|
+
missing = Counter()
|
|
2362
|
+
for r in cov["runs"]:
|
|
2363
|
+
ws = r["words"]
|
|
2364
|
+
prose = sum(1 for w in ws if w.isalpha() and len(w) > 2 and (not known or w in known))
|
|
2365
|
+
if prose < 0.4 * len(ws): # sentences score about 0.5 or more; tables and charts 0.25 or less
|
|
2366
|
+
continue
|
|
2367
|
+
hits = sum(1 for w in ws if w in FRONT_MATTER_WORDS)
|
|
2368
|
+
if hits and (min(r["pages"]) <= cov["first"] + 1 or 30 * hits >= len(ws)):
|
|
2369
|
+
continue # licence and contact blocks: on the opening pages, or dense with these words. A page footer adds one or two.
|
|
2370
|
+
for p, n in r["pages"].items():
|
|
2371
|
+
if n >= 10:
|
|
2372
|
+
missing[p] += n
|
|
2373
|
+
return sorted(p for p, _ in missing.most_common(limit))
|
|
2374
|
+
|
|
2375
|
+
|
|
2376
|
+
def page_image(pdf, page):
|
|
2377
|
+
"""The whole printed page as a JPEG, for showing where text may be missing."""
|
|
2378
|
+
WORK.mkdir(parents=True, exist_ok=True)
|
|
2379
|
+
with tempfile.TemporaryDirectory(dir=str(WORK)) as tmp:
|
|
2380
|
+
root = Path(tmp) / "page"
|
|
2381
|
+
try:
|
|
2382
|
+
r = subprocess.run(["pdftoppm", "-f", str(page), "-l", str(page), "-scale-to", "1400", "-jpeg",
|
|
2383
|
+
"-jpegopt", "quality=75", "-singlefile", "--", str(pdf), str(root)],
|
|
2384
|
+
stdout=subprocess.PIPE, stderr=subprocess.PIPE, timeout=60)
|
|
2385
|
+
except subprocess.TimeoutExpired:
|
|
2386
|
+
return None
|
|
2387
|
+
out = Path(tmp) / "page.jpg"
|
|
2388
|
+
return out.read_bytes() if r.returncode == 0 and out.exists() else None
|
|
2389
|
+
|
|
2390
|
+
|
|
2391
|
+
def finish_page(doc, meta, pages=None):
|
|
2392
|
+
"""Render the page, check its coverage, and render again with the printed
|
|
2393
|
+
page offered wherever prose went missing. Returns (html, words, coverage)."""
|
|
2394
|
+
page, total = render(doc)
|
|
2395
|
+
cov = coverage(page, meta, pages, report=False)
|
|
2396
|
+
if cov and doc.pdf and shutil.which("pdftoppm") and any(b["t"] == "page" for b in doc.blocks):
|
|
2397
|
+
for p in weak_pages(cov):
|
|
2398
|
+
u = data_uri(page_image(doc.pdf, p) or b"")
|
|
2399
|
+
if u:
|
|
2400
|
+
doc.page_images[p] = u
|
|
2401
|
+
if doc.page_images:
|
|
2402
|
+
page, total = render(doc)
|
|
2403
|
+
return page, total, cov
|
|
2404
|
+
|
|
2405
|
+
|
|
2406
|
+
def report_coverage(cov, doc):
|
|
2407
|
+
if cov is None:
|
|
2408
|
+
print(" coverage: no source text to compare against")
|
|
2409
|
+
return
|
|
2410
|
+
print_coverage(cov)
|
|
2411
|
+
if doc.page_images:
|
|
2412
|
+
print(" original page offered for PDF pages %s, where text may be missing" % ", ".join(str(p) for p in sorted(doc.page_images)))
|
|
2413
|
+
|
|
2414
|
+
|
|
2415
|
+
# --------------------------------------------------------------------------
|
|
2416
|
+
# Commands
|
|
2417
|
+
# --------------------------------------------------------------------------
|
|
2418
|
+
|
|
2419
|
+
@functools.lru_cache(maxsize=1) # the same fonts for every page in a build
|
|
2420
|
+
def fonts_css():
|
|
2421
|
+
faces = []
|
|
2422
|
+
for style, f in (("normal", "literata-opsz-normal.woff2"), ("italic", "literata-opsz-italic.woff2")):
|
|
2423
|
+
p = FONTS / f
|
|
2424
|
+
if p.exists():
|
|
2425
|
+
faces.append("@font-face{font-family:'Literata';font-style:%s;font-weight:200 900;font-display:swap;"
|
|
2426
|
+
"src:url(data:font/woff2;base64,%s) format('woff2');}" % (style, base64.b64encode(p.read_bytes()).decode()))
|
|
2427
|
+
return "/* Literata, SIL Open Font License, via @fontsource-variable/literata@5.3.0 */\n" + "\n".join(faces)
|
|
2428
|
+
|
|
2429
|
+
|
|
2430
|
+
def write_assets(out=None):
|
|
2431
|
+
a = (out or OUT) / "assets"
|
|
2432
|
+
a.mkdir(parents=True, exist_ok=True)
|
|
2433
|
+
shutil.copyfile(str(HERE / "reader.css"), str(a / "reader.css"))
|
|
2434
|
+
shutil.copyfile(str(HERE / "reader.js"), str(a / "reader.js"))
|
|
2435
|
+
(a / "fonts.css").write_text(fonts_css())
|
|
2436
|
+
|
|
2437
|
+
|
|
2438
|
+
def inline_assets(page):
|
|
2439
|
+
"""The page with its styles, script and fonts inside it, so it works as one file.
|
|
2440
|
+
The script stays allowed only by its hash, so the page still runs no other script."""
|
|
2441
|
+
# Plain ASCII throughout: a file sent by email can be re-encoded on the way
|
|
2442
|
+
# (Gmail turned one into Latin-1 on 19 November 2026), which garbles every
|
|
2443
|
+
# accent and breaks the script's hash. Escapes survive that.
|
|
2444
|
+
def js_ascii(s): # reader.js keeps its non-ASCII inside strings, where \u escapes mean the same
|
|
2445
|
+
return re.sub(r"[^\x00-\x7f]", lambda m: json.dumps(m.group())[1:-1], s) # \uXXXX, surrogate pairs included
|
|
2446
|
+
def css_ascii(s):
|
|
2447
|
+
return re.sub(r"[^\x00-\x7f]", lambda m: "\\%x " % ord(m.group()), s)
|
|
2448
|
+
page = page.encode("ascii", "xmlcharrefreplace").decode("ascii")
|
|
2449
|
+
js = js_ascii((HERE / "reader.js").read_text(encoding="utf-8"))
|
|
2450
|
+
css = css_ascii((HERE / "reader.css").read_text(encoding="utf-8"))
|
|
2451
|
+
fonts = css_ascii(fonts_css())
|
|
2452
|
+
for text in (js, css, fonts):
|
|
2453
|
+
if re.search(r"</(script|style)", text, re.I): # would end the inline block early
|
|
2454
|
+
raise ReaderError("an asset contains a closing script or style tag and can't be inlined")
|
|
2455
|
+
digest = base64.b64encode(hashlib.sha256(js.encode("utf-8")).digest()).decode()
|
|
2456
|
+
swaps = [
|
|
2457
|
+
("script-src 'self' file:", "script-src 'sha256-%s'" % digest),
|
|
2458
|
+
('<link rel="stylesheet" href="assets/fonts.css">', "<style>%s</style>" % fonts),
|
|
2459
|
+
('<link rel="stylesheet" href="assets/reader.css">', "<style>%s</style>" % css),
|
|
2460
|
+
('<script src="assets/reader.js"></script>', "<script>%s</script>" % js),
|
|
2461
|
+
]
|
|
2462
|
+
for old, new in swaps:
|
|
2463
|
+
if page.count(old) != 1:
|
|
2464
|
+
raise ReaderError("page template changed; can't inline %s" % old)
|
|
2465
|
+
page = page.replace(old, new)
|
|
2466
|
+
return page
|
|
2467
|
+
|
|
2468
|
+
|
|
2469
|
+
def single_file_copy(doc):
|
|
2470
|
+
"""A copy of the page that needs nothing beside it, to send to a phone."""
|
|
2471
|
+
doc.meta["single_file"] = True
|
|
2472
|
+
try:
|
|
2473
|
+
page, _ = render(doc)
|
|
2474
|
+
finally:
|
|
2475
|
+
doc.meta.pop("single_file", None)
|
|
2476
|
+
return inline_assets(page)
|
|
2477
|
+
|
|
2478
|
+
|
|
2479
|
+
def build_one(pid, ledger, route, pages=None, part=None, epub=False, single=False, place=None, members=None):
|
|
2480
|
+
"""Returns ({"epub": path, "html": path, "page": html}, meta): the copies written this run, None where not written."""
|
|
2481
|
+
meta = ledger_meta(ledger, pid)
|
|
2482
|
+
d = SRC / slug(meta)
|
|
2483
|
+
if route == "arxiv":
|
|
2484
|
+
doc = arxiv_doc(meta, d)
|
|
2485
|
+
elif route == "jats":
|
|
2486
|
+
doc = jats_doc(meta, d)
|
|
2487
|
+
elif route == "pdf":
|
|
2488
|
+
doc = docling_doc(meta, d, pages)
|
|
2489
|
+
else:
|
|
2490
|
+
raise ReaderError("unknown route %s" % route)
|
|
2491
|
+
doc.meta["part"] = part
|
|
2492
|
+
doc.meta["place"] = place
|
|
2493
|
+
doc.meta["bundle_members"] = members
|
|
2494
|
+
page, total, cov = finish_page(doc, meta, pages)
|
|
2495
|
+
write_assets()
|
|
2496
|
+
name = slug(meta) + ".html"
|
|
2497
|
+
(OUT / name).write_text(page, encoding="utf-8")
|
|
2498
|
+
print("wrote %s (%d words, %d references, %d notes)" % (OUT / name, total, len(doc.refs), doc.nnotes))
|
|
2499
|
+
report_coverage(cov, doc)
|
|
2500
|
+
epub_path = write_epub_copy(page, meta, OUT / (slug(meta) + ".epub")) if epub else None
|
|
2501
|
+
single_path = write_single_copy(doc, OUT / "single-file" / name) if single else None
|
|
2502
|
+
return {"epub": epub_path, "html": single_path, "page": page}, doc.meta
|
|
2503
|
+
|
|
2504
|
+
|
|
2505
|
+
def load_list():
|
|
2506
|
+
if not LIST_FILE.exists():
|
|
2507
|
+
raise ReaderError("no %s" % LIST_FILE)
|
|
2508
|
+
return json.loads(LIST_FILE.read_text(encoding="utf-8"))
|
|
2509
|
+
|
|
2510
|
+
|
|
2511
|
+
def list_members(L):
|
|
2512
|
+
"""The papers of one list that have a reading copy, for links between them."""
|
|
2513
|
+
out = []
|
|
2514
|
+
for k, it in enumerate(L["items"], 1):
|
|
2515
|
+
if not it.get("route"):
|
|
2516
|
+
continue
|
|
2517
|
+
try:
|
|
2518
|
+
meta = ledger_meta(it["ledger"], it["pid"])
|
|
2519
|
+
except ReaderError:
|
|
2520
|
+
continue
|
|
2521
|
+
out.append({"pid": it["pid"], "title": meta.get("title"), "doi": meta.get("doi"), "arxiv": meta.get("arxiv"),
|
|
2522
|
+
"href": slug(meta) + ".html", "n": it.get("n") or k, "total": len(L["items"])})
|
|
2523
|
+
return out
|
|
2524
|
+
|
|
2525
|
+
|
|
2526
|
+
def list_place(L, it, k):
|
|
2527
|
+
return (it.get("n") or k, len(L["items"]))
|
|
2528
|
+
|
|
2529
|
+
|
|
2530
|
+
EPUB_MODES = ("papers", "bundle", "both")
|
|
2531
|
+
|
|
2532
|
+
|
|
2533
|
+
def epub_choice(v):
|
|
2534
|
+
if v not in EPUB_MODES:
|
|
2535
|
+
raise argparse.ArgumentTypeError("choose from papers, bundle, both")
|
|
2536
|
+
return v
|
|
2537
|
+
|
|
2538
|
+
|
|
2539
|
+
def epub_mode(given):
|
|
2540
|
+
"""Which EPUBs to write: None for none, or "papers", "bundle" or "both".
|
|
2541
|
+
A bare --epub (given as True: argparse would run a string const through
|
|
2542
|
+
the type check) means the settings file's `epub = "..."`, or else "papers"."""
|
|
2543
|
+
if given is not True:
|
|
2544
|
+
return given
|
|
2545
|
+
conf = read_settings(CONFIG) if CONFIG.is_file() else {}
|
|
2546
|
+
v = conf.get("epub") or "papers"
|
|
2547
|
+
if v not in EPUB_MODES:
|
|
2548
|
+
raise ReaderError('%s: epub must be "papers", "bundle" or "both", not "%s"' % (CONFIG, v))
|
|
2549
|
+
return v
|
|
2550
|
+
|
|
2551
|
+
|
|
2552
|
+
def cmd_build(a):
|
|
2553
|
+
lst = load_list() if LIST_FILE.exists() else {"lists": []}
|
|
2554
|
+
part = place = members = None
|
|
2555
|
+
for L in lst["lists"]:
|
|
2556
|
+
for k, it in enumerate(L["items"], 1):
|
|
2557
|
+
if it["pid"] == a.pid and place is None: # the first list it's on
|
|
2558
|
+
a.pages = a.pages or it.get("pages")
|
|
2559
|
+
part = it.get("part")
|
|
2560
|
+
place, members = list_place(L, it, k), list_members(L)
|
|
2561
|
+
mail_to = email_settings(a.email)
|
|
2562
|
+
copies, meta = build_one(a.pid, a.ledger, a.route, a.pages, part,
|
|
2563
|
+
a.epub or a.email == "epub", a.single_file or a.email == "html", place, members)
|
|
2564
|
+
index_error = None
|
|
2565
|
+
if LIST_FILE.exists():
|
|
2566
|
+
try:
|
|
2567
|
+
cmd_index(a)
|
|
2568
|
+
except ReaderError as e: # the paper is built, so still send it, then report this
|
|
2569
|
+
index_error = e
|
|
2570
|
+
if mail_to: # after the index, so a failed send leaves nothing else undone
|
|
2571
|
+
email_copy(mail_to, copies[a.email], meta)
|
|
2572
|
+
if index_error:
|
|
2573
|
+
raise index_error
|
|
2574
|
+
|
|
2575
|
+
|
|
2576
|
+
def cmd_all(a):
|
|
2577
|
+
mode = epub_mode(a.epub)
|
|
2578
|
+
lst = load_list()
|
|
2579
|
+
for L in lst["lists"]:
|
|
2580
|
+
members, built, missing = list_members(L), [], []
|
|
2581
|
+
for k, it in enumerate(L["items"], 1):
|
|
2582
|
+
if it.get("route"):
|
|
2583
|
+
try:
|
|
2584
|
+
copies, meta = build_one(it["pid"], it["ledger"], it["route"], it.get("pages"), it.get("part"),
|
|
2585
|
+
mode in ("papers", "both"), a.single_file, list_place(L, it, k), members)
|
|
2586
|
+
built.append((copies["page"], meta, slug(meta) + ".html"))
|
|
2587
|
+
except ReaderError as e:
|
|
2588
|
+
print("skipped %s: %s" % (it["pid"], e))
|
|
2589
|
+
try:
|
|
2590
|
+
missing.append(ledger_meta(it["ledger"], it["pid"]).get("title") or it["pid"])
|
|
2591
|
+
except ReaderError:
|
|
2592
|
+
missing.append(it["pid"])
|
|
2593
|
+
if mode in ("bundle", "both") and built:
|
|
2594
|
+
write_bundle_epub(built, L.get("name") or "Reading list", L.get("note") or "",
|
|
2595
|
+
OUT / (folder_name(L.get("name") or "reading-list") + ".epub"), missing)
|
|
2596
|
+
cmd_index(a)
|
|
2597
|
+
|
|
2598
|
+
|
|
2599
|
+
def paper_row(out, n, pid, meta, name, part=None, why_not=None):
|
|
2600
|
+
"""One paper on an index page. `name` is its page's file name in `out`."""
|
|
2601
|
+
exists = bool(name) and (out / name).exists()
|
|
2602
|
+
authors = meta.get("authors") or []
|
|
2603
|
+
who = authors[0].split()[-1] + (" et al." if len(authors) > 2 else (" and " + authors[1].split()[-1] if len(authors) == 2 else "")) if authors else ""
|
|
2604
|
+
desc = " · ".join(esc(x) for x in [who, str(meta.get("year") or ""), meta.get("venue") or "", (part or "").split(":")[0]] if x)
|
|
2605
|
+
quotes = ""
|
|
2606
|
+
if exists:
|
|
2607
|
+
mm = re.search(r"about (\d+) min of reading", (out / name).read_text(encoding="utf-8"))
|
|
2608
|
+
if mm:
|
|
2609
|
+
desc += " · about %s min" % mm.group(1)
|
|
2610
|
+
title = '<a href="%s">%s</a>' % (attr(name), esc(meta["title"]))
|
|
2611
|
+
t = meta.get("title") or ""
|
|
2612
|
+
quotes = ' data-title="%s" data-cite="%s" data-reference="%s"' % (attr(t), attr(short_citation(meta)), attr(full_reference(meta, t)))
|
|
2613
|
+
else:
|
|
2614
|
+
title = esc(meta["title"])
|
|
2615
|
+
desc += (" · " if desc else "") + esc(why_not or "no reading copy yet")
|
|
2616
|
+
return '<li class="%s" data-pid="%s"%s><span class="n">%s</span><span class="t">%s</span><span class="s"></span><span class="d">%s</span></li>' % (
|
|
2617
|
+
"" if exists else "nofile", attr(pid), quotes, esc(str(n)), title, desc)
|
|
2618
|
+
|
|
2619
|
+
|
|
2620
|
+
def write_index(out, heading, sections, kicker, bar, note=""):
|
|
2621
|
+
"""sections: (name, note, rows). A section without a name has no heading of its own."""
|
|
2622
|
+
write_assets(out)
|
|
2623
|
+
lists = "\n".join('<section class="list">%s%s<ol>%s</ol></section>' % (
|
|
2624
|
+
"<h2>%s</h2>" % esc(name) if name else "", '<p class="listnote">%s</p>' % esc(lnote) if lnote else "", "".join(rows))
|
|
2625
|
+
for name, lnote, rows in sections)
|
|
2626
|
+
page = INDEX.format(lists=lists, built=datetime.date.today().isoformat(), panel=PANEL, heading=esc(heading),
|
|
2627
|
+
bar=esc(bar), kicker=esc(kicker), note=esc(note + " ") if note else "")
|
|
2628
|
+
(out / "index.html").write_text(page, encoding="utf-8")
|
|
2629
|
+
print("wrote %s" % (out / "index.html"))
|
|
2630
|
+
|
|
2631
|
+
|
|
2632
|
+
def cmd_index(a):
|
|
2633
|
+
lst = load_list()
|
|
2634
|
+
sections = []
|
|
2635
|
+
for L in lst["lists"]:
|
|
2636
|
+
rows = []
|
|
2637
|
+
for it in L["items"]:
|
|
2638
|
+
try:
|
|
2639
|
+
meta = ledger_meta(it["ledger"], it["pid"])
|
|
2640
|
+
except ReaderError:
|
|
2641
|
+
meta = {"title": it["pid"], "authors": [], "year": "", "venue": "", "pid": it["pid"]}
|
|
2642
|
+
name = slug(meta) + ".html" if meta.get("authors") is not None else ""
|
|
2643
|
+
rows.append(paper_row(OUT, it.get("n", ""), it["pid"], meta, name, it.get("part"), it.get("why_not")))
|
|
2644
|
+
sections.append((L["name"], L.get("note", ""), rows))
|
|
2645
|
+
write_index(OUT, "Reading list", sections, INDEX_TEXT["index_kicker"], INDEX_TEXT["index_bar"], INDEX_TEXT["index_note"])
|
|
2646
|
+
|
|
2647
|
+
|
|
2648
|
+
INDEX = """<!doctype html>
|
|
2649
|
+
<html lang="en">
|
|
2650
|
+
<head>
|
|
2651
|
+
<meta charset="utf-8">
|
|
2652
|
+
<meta name="viewport" content="width=device-width, initial-scale=1">
|
|
2653
|
+
<meta name="generator" content="lit-reader index">
|
|
2654
|
+
<meta http-equiv="Content-Security-Policy" content="default-src 'none'; img-src data:; style-src 'self' 'unsafe-inline' file:; font-src data:; script-src 'self' file:">
|
|
2655
|
+
<title>{heading}</title>
|
|
2656
|
+
<link rel="stylesheet" href="assets/fonts.css">
|
|
2657
|
+
<link rel="stylesheet" href="assets/reader.css">
|
|
2658
|
+
</head>
|
|
2659
|
+
<body class="index">
|
|
2660
|
+
<div class="bar"><span class="where">{bar}</span><button id="aa" type="button" aria-label="Aa, reading settings" aria-expanded="false" aria-controls="panel">Aa</button></div>
|
|
2661
|
+
{panel}
|
|
2662
|
+
<div class="layout"><main>
|
|
2663
|
+
<header class="paper"><div class="kicker">{kicker}</div><h1>{heading}</h1>
|
|
2664
|
+
<div class="meta">{note}Built {built}.</div></header>
|
|
2665
|
+
{lists}
|
|
2666
|
+
<section class="bundle-quotes" id="bundle-quotes" aria-labelledby="bundle-quotes-h" hidden></section>
|
|
2667
|
+
</main></div>
|
|
2668
|
+
<script src="assets/reader.js"></script>
|
|
2669
|
+
</body>
|
|
2670
|
+
</html>
|
|
2671
|
+
"""
|
|
2672
|
+
|
|
2673
|
+
# Pages also have to work where no script runs: an iPhone's preview of an email
|
|
2674
|
+
# attachment runs none. This small script marks the page as scripted before it
|
|
2675
|
+
# is drawn, so the styles can offer plain links instead of dead buttons when it
|
|
2676
|
+
# doesn't run. It takes the mark back if reader.js never ran (a page opened
|
|
2677
|
+
# without its assets folder), since the buttons would then be dead too.
|
|
2678
|
+
JS_FLAG = ('document.documentElement.classList.add("js");'
|
|
2679
|
+
'document.addEventListener("DOMContentLoaded",function(){'
|
|
2680
|
+
'if(!window.litReaderRan)document.documentElement.classList.remove("js")})')
|
|
2681
|
+
|
|
2682
|
+
|
|
2683
|
+
def with_js_flag(tpl):
|
|
2684
|
+
digest = base64.b64encode(hashlib.sha256(JS_FLAG.encode("ascii")).digest()).decode()
|
|
2685
|
+
for old, new in (("script-src 'self' file:", "script-src 'self' file: 'sha256-%s'" % digest),
|
|
2686
|
+
("<title>", "<script>%s</script>\n<title>" % JS_FLAG.replace("{", "{{").replace("}", "}}"))): # the templates go through str.format
|
|
2687
|
+
if tpl.count(old) != 1: # not an assert: python -O would drop it, and pages would lose their script
|
|
2688
|
+
raise RuntimeError("page template changed; can't add the script flag at %r" % old)
|
|
2689
|
+
tpl = tpl.replace(old, new)
|
|
2690
|
+
return tpl
|
|
2691
|
+
|
|
2692
|
+
|
|
2693
|
+
PAGE = with_js_flag(PAGE)
|
|
2694
|
+
INDEX = with_js_flag(INDEX)
|
|
2695
|
+
|
|
2696
|
+
|
|
2697
|
+
# --------------------------------------------------------------------------
|
|
2698
|
+
# Standalone mode: one paper, no ledger
|
|
2699
|
+
# --------------------------------------------------------------------------
|
|
2700
|
+
|
|
2701
|
+
ARXIV_ID = r"(?:\d{4}\.\d{4,5}|[a-z-]+(?:\.[A-Z]{2})?/\d{7})(?:v\d+)?"
|
|
2702
|
+
DOI_RE = r"10\.\d{4,9}/[^\s\"<>]+"
|
|
2703
|
+
ATOM = "{http://www.w3.org/2005/Atom}"
|
|
2704
|
+
ARXIV_NS = "{http://arxiv.org/schemas/atom}"
|
|
2705
|
+
EPMC = "https://www.ebi.ac.uk/europepmc/webservices/rest/search?format=json&resultType=core&pageSize=5&query="
|
|
2706
|
+
|
|
2707
|
+
|
|
2708
|
+
def parse_input(text):
|
|
2709
|
+
"""(kind, value) for a PDF path, arXiv ID or URL, DOI or DOI URL, PMCID or PMC URL."""
|
|
2710
|
+
t = text.strip()
|
|
2711
|
+
p = Path(t).expanduser()
|
|
2712
|
+
if p.is_file():
|
|
2713
|
+
return "pdf", p.resolve()
|
|
2714
|
+
if t.lower().endswith(".pdf") and not re.match(r"https?://", t, re.I):
|
|
2715
|
+
raise ReaderError("no such file: %s" % t)
|
|
2716
|
+
m = re.match(r"^(?:https?://)?(?:www\.|export\.)?arxiv\.org/(?:abs|pdf|html)/(%s)(?:\.pdf)?/?(?:[?#].*)?$" % ARXIV_ID, t, re.I)
|
|
2717
|
+
if m:
|
|
2718
|
+
return "arxiv", m.group(1)
|
|
2719
|
+
m = re.match(r"^arxiv:\s*(%s)$" % ARXIV_ID, t, re.I)
|
|
2720
|
+
if m or re.match(r"^%s$" % ARXIV_ID, t):
|
|
2721
|
+
return "arxiv", m.group(1) if m else t
|
|
2722
|
+
m = re.search(r"\b(PMC\d+)\b", t, re.I)
|
|
2723
|
+
if m and (t.upper() == m.group(1).upper() or re.match(r"^https?://([\w-]+\.)*(nih\.gov|europepmc\.org)/", t, re.I)):
|
|
2724
|
+
return "pmcid", m.group(1).upper()
|
|
2725
|
+
m = re.match(r"^(?:https?://(?:dx\.)?doi\.org/|doi:\s*)?(%s)$" % DOI_RE, t, re.I)
|
|
2726
|
+
if m:
|
|
2727
|
+
doi = urllib.parse.unquote(m.group(1))
|
|
2728
|
+
a = re.match(r"^10\.48550/arxiv\.(%s)$" % ARXIV_ID, doi, re.I)
|
|
2729
|
+
return ("arxiv", a.group(1)) if a else ("doi", doi)
|
|
2730
|
+
raise ReaderError("not a PDF, arXiv ID, DOI or PMCID: %s" % text)
|
|
2731
|
+
|
|
2732
|
+
|
|
2733
|
+
def read_json(body, service):
|
|
2734
|
+
try:
|
|
2735
|
+
return json.loads(body.decode("utf-8"))
|
|
2736
|
+
except ValueError:
|
|
2737
|
+
raise ReaderError("%s sent an unreadable reply" % service)
|
|
2738
|
+
|
|
2739
|
+
|
|
2740
|
+
def clean_title(t):
|
|
2741
|
+
return re.sub(r"\s+", " ", html.unescape(re.sub(r"<[^>]+>", "", t or ""))).strip().rstrip(".")
|
|
2742
|
+
|
|
2743
|
+
|
|
2744
|
+
def arxiv_meta(aid):
|
|
2745
|
+
body, _ = http_get("https://export.arxiv.org/api/query?id_list=%s" % urllib.parse.quote(aid, safe="/.v"), {"export.arxiv.org"})
|
|
2746
|
+
try:
|
|
2747
|
+
e = ET.fromstring(body).find(ATOM + "entry")
|
|
2748
|
+
except ET.ParseError:
|
|
2749
|
+
raise ReaderError("arXiv sent an unreadable reply for %s" % aid)
|
|
2750
|
+
if e is None or e.find(ATOM + "published") is None:
|
|
2751
|
+
raise ReaderError("arXiv has no paper %s" % aid)
|
|
2752
|
+
return {"pid": "arxiv:" + aid, "title": clean_title(e.findtext(ATOM + "title")),
|
|
2753
|
+
"authors": [a.findtext(ATOM + "name").strip() for a in e.findall(ATOM + "author")],
|
|
2754
|
+
"year": int(e.findtext(ATOM + "published")[:4]), "venue": (e.findtext(ARXIV_NS + "journal_ref") or "").strip() or None,
|
|
2755
|
+
"doi": (e.findtext(ARXIV_NS + "doi") or "").strip() or None, "arxiv": aid, "pmcid": None}
|
|
2756
|
+
|
|
2757
|
+
|
|
2758
|
+
def crossref_meta(doi):
|
|
2759
|
+
body, _ = http_get("https://api.crossref.org/works/%s" % urllib.parse.quote(doi, safe=""), {"api.crossref.org"})
|
|
2760
|
+
m = read_json(body, "Crossref")["message"]
|
|
2761
|
+
authors = [" ".join(x for x in (a.get("given"), a.get("family")) if x) or a.get("name", "") for a in m.get("author", [])]
|
|
2762
|
+
parts = ((m.get("issued") or {}).get("date-parts") or [[None]])[0]
|
|
2763
|
+
return {"pid": "doi:" + doi, "title": clean_title((m.get("title") or [""])[0]), "authors": [a for a in authors if a],
|
|
2764
|
+
"year": parts[0] if parts else None, "venue": clean_title((m.get("container-title") or [""])[0]) or None,
|
|
2765
|
+
"doi": doi, "arxiv": None, "pmcid": None}
|
|
2766
|
+
|
|
2767
|
+
|
|
2768
|
+
def epmc_record(query):
|
|
2769
|
+
body, _ = http_get(EPMC + urllib.parse.quote(query, safe=""), {"www.ebi.ac.uk"})
|
|
2770
|
+
res = read_json(body, "Europe PMC").get("resultList", {}).get("result", [])
|
|
2771
|
+
return res[0] if res else None
|
|
2772
|
+
|
|
2773
|
+
|
|
2774
|
+
def epmc_meta(r):
|
|
2775
|
+
authors = []
|
|
2776
|
+
for a in (r.get("authorList") or {}).get("author", []):
|
|
2777
|
+
name = " ".join(x for x in (a.get("firstName"), a.get("lastName")) if x) or a.get("collectiveName") or a.get("fullName")
|
|
2778
|
+
if name:
|
|
2779
|
+
authors.append(name)
|
|
2780
|
+
year = r.get("pubYear")
|
|
2781
|
+
return {"pid": "pmc:" + (r.get("pmcid") or r.get("id", "")), "title": clean_title(r.get("title")), "authors": authors,
|
|
2782
|
+
"year": int(year) if year and year.isdigit() else None,
|
|
2783
|
+
"venue": ((r.get("journalInfo") or {}).get("journal") or {}).get("title"),
|
|
2784
|
+
"doi": r.get("doi"), "arxiv": None, "pmcid": r.get("pmcid")}
|
|
2785
|
+
|
|
2786
|
+
|
|
2787
|
+
def need_tool(name, why):
|
|
2788
|
+
if not shutil.which(name):
|
|
2789
|
+
raise ReaderError("%s needs poppler's %s (macOS: brew install poppler)" % (why, name))
|
|
2790
|
+
|
|
2791
|
+
|
|
2792
|
+
def pdf_to_text(pdf, out):
|
|
2793
|
+
"""pdftotext's reading of the PDF, or None without poppler."""
|
|
2794
|
+
if not shutil.which("pdftotext"):
|
|
2795
|
+
return None
|
|
2796
|
+
try:
|
|
2797
|
+
r = subprocess.run(["pdftotext", "-enc", "UTF-8", str(pdf), str(out)], stdout=subprocess.PIPE, stderr=subprocess.PIPE, timeout=180)
|
|
2798
|
+
except subprocess.TimeoutExpired:
|
|
2799
|
+
return None
|
|
2800
|
+
return out if r.returncode == 0 and out.exists() else None
|
|
2801
|
+
|
|
2802
|
+
|
|
2803
|
+
def on_first_page(title, first):
|
|
2804
|
+
norm = lambda x: re.sub(r"[^a-z0-9]", "", x.lower())
|
|
2805
|
+
return bool(norm(title)) and norm(title)[:60] in norm(first)
|
|
2806
|
+
|
|
2807
|
+
|
|
2808
|
+
def guess_title(pdf, first):
|
|
2809
|
+
try:
|
|
2810
|
+
r = subprocess.run(["pdfinfo", str(pdf)], stdout=subprocess.PIPE, stderr=subprocess.PIPE, timeout=60) if shutil.which("pdfinfo") else None
|
|
2811
|
+
except subprocess.TimeoutExpired:
|
|
2812
|
+
r = None
|
|
2813
|
+
m = re.search(r"^Title:\s*(.+)$", r.stdout.decode("utf-8", "replace"), re.M) if r and r.returncode == 0 else None
|
|
2814
|
+
t = m.group(1).strip() if m else ""
|
|
2815
|
+
if len(t.split()) >= 3 and not re.search(r"\.(pdf|docx?|tex|indd)$|microsoft word|untitled", t, re.I):
|
|
2816
|
+
return t
|
|
2817
|
+
for line in first.splitlines():
|
|
2818
|
+
line = line.strip()
|
|
2819
|
+
if len(line.split()) >= 4 and not re.search(r"https?:|doi|©|copyright|journal|vol\.|arxiv", line, re.I):
|
|
2820
|
+
return line
|
|
2821
|
+
return "Untitled paper"
|
|
2822
|
+
|
|
2823
|
+
|
|
2824
|
+
def pdf_meta(pdf, text, key):
|
|
2825
|
+
"""Metadata for a local PDF. An identifier printed on its first page is
|
|
2826
|
+
trusted only if the record it leads to has this PDF's title."""
|
|
2827
|
+
first = text.read_text(encoding="utf-8", errors="replace").split("\f")[0] if text else ""
|
|
2828
|
+
m = re.search(r"arXiv:\s*(%s)" % ARXIV_ID, first)
|
|
2829
|
+
if m:
|
|
2830
|
+
try:
|
|
2831
|
+
meta = arxiv_meta(m.group(1))
|
|
2832
|
+
if on_first_page(meta["title"], first):
|
|
2833
|
+
return meta
|
|
2834
|
+
except ReaderError:
|
|
2835
|
+
pass
|
|
2836
|
+
for raw in re.findall(DOI_RE, first):
|
|
2837
|
+
doi = raw.rstrip(".,;)]")
|
|
2838
|
+
try:
|
|
2839
|
+
meta = crossref_meta(doi)
|
|
2840
|
+
except (ReaderError, KeyError, ValueError):
|
|
2841
|
+
continue
|
|
2842
|
+
if on_first_page(meta["title"], first):
|
|
2843
|
+
return meta
|
|
2844
|
+
year = re.search(r"\b(19[5-9]\d|20[0-4]\d)\b", first)
|
|
2845
|
+
print(" no record found for this PDF: title guessed from its first page, authors left blank")
|
|
2846
|
+
return {"pid": "pdf:" + key, "title": guess_title(pdf, first), "authors": [], "year": int(year.group(1)) if year else None,
|
|
2847
|
+
"venue": None, "doi": None, "arxiv": None, "pmcid": None}
|
|
2848
|
+
|
|
2849
|
+
|
|
2850
|
+
def jats_text(xml, out):
|
|
2851
|
+
"""The article's own words, for the coverage check of a page built from XML."""
|
|
2852
|
+
skip = {"tex-math", "annotation", "annotation-xml", "graphic", "object-id", "label"}
|
|
2853
|
+
parts = []
|
|
2854
|
+
|
|
2855
|
+
def walk(e):
|
|
2856
|
+
if local(e.tag) in skip:
|
|
2857
|
+
return
|
|
2858
|
+
parts.append(e.text or "")
|
|
2859
|
+
for c in e:
|
|
2860
|
+
walk(c)
|
|
2861
|
+
parts.append(c.tail or "")
|
|
2862
|
+
|
|
2863
|
+
def outermost(e): # a reference list inside the body must not be counted twice
|
|
2864
|
+
if local(e.tag) in ("article-title", "abstract", "body", "ref-list", "ack", "app-group", "app") or (
|
|
2865
|
+
local(e.tag) in ("sec", "notes") and in_back[0]):
|
|
2866
|
+
walk(e)
|
|
2867
|
+
parts.append("\n")
|
|
2868
|
+
else:
|
|
2869
|
+
in_back[0] = in_back[0] or local(e.tag) == "back"
|
|
2870
|
+
for c in e:
|
|
2871
|
+
outermost(c)
|
|
2872
|
+
in_back = [False]
|
|
2873
|
+
outermost(parse_xml(xml))
|
|
2874
|
+
out.write_text(" ".join(parts), encoding="utf-8")
|
|
2875
|
+
return out
|
|
2876
|
+
|
|
2877
|
+
|
|
2878
|
+
def identify(kind, value):
|
|
2879
|
+
"""(metadata, work folder, local PDF or None) for one input."""
|
|
2880
|
+
pdf = None
|
|
2881
|
+
if kind == "pdf":
|
|
2882
|
+
data = value.read_bytes()
|
|
2883
|
+
if b"%PDF-" not in data[:1024]:
|
|
2884
|
+
raise ReaderError("%s is not a PDF" % value)
|
|
2885
|
+
key = hashlib.sha256(data).hexdigest()[:16]
|
|
2886
|
+
d = WORK / "standalone" / ("pdf-" + key)
|
|
2887
|
+
(d / "in").mkdir(parents=True, exist_ok=True)
|
|
2888
|
+
pdf = d / "in" / "paper.pdf"
|
|
2889
|
+
pdf.write_bytes(data)
|
|
2890
|
+
meta = pdf_meta(pdf, pdf_to_text(pdf, d / "paper.txt"), key)
|
|
2891
|
+
elif kind == "arxiv":
|
|
2892
|
+
meta = arxiv_meta(value)
|
|
2893
|
+
elif kind == "pmcid":
|
|
2894
|
+
r = epmc_record("PMCID:%s" % value)
|
|
2895
|
+
if not r:
|
|
2896
|
+
raise ReaderError("Europe PMC has no record of %s" % value)
|
|
2897
|
+
meta = epmc_meta(r)
|
|
2898
|
+
meta["pmcid"] = value
|
|
2899
|
+
else:
|
|
2900
|
+
try:
|
|
2901
|
+
meta = crossref_meta(value)
|
|
2902
|
+
except (ReaderError, KeyError, ValueError):
|
|
2903
|
+
r = epmc_record('DOI:"%s"' % value)
|
|
2904
|
+
if not r:
|
|
2905
|
+
raise ReaderError("neither Crossref nor Europe PMC has a record of %s" % value)
|
|
2906
|
+
meta = epmc_meta(r)
|
|
2907
|
+
meta["doi"] = value
|
|
2908
|
+
if kind != "pdf":
|
|
2909
|
+
d = WORK / "standalone" / safe_id(meta["pid"])
|
|
2910
|
+
d.mkdir(parents=True, exist_ok=True)
|
|
2911
|
+
meta["standalone"] = True
|
|
2912
|
+
return meta, d, pdf
|
|
2913
|
+
|
|
2914
|
+
|
|
2915
|
+
def open_pmcid(meta):
|
|
2916
|
+
"""The PMCID to fetch, if Europe PMC holds this paper's full text openly."""
|
|
2917
|
+
if meta.get("pmcid"):
|
|
2918
|
+
return meta["pmcid"]
|
|
2919
|
+
if not meta.get("doi"):
|
|
2920
|
+
return None
|
|
2921
|
+
r = epmc_record('DOI:"%s"' % meta["doi"]) # an outage is reported, not mistaken for "no open text"
|
|
2922
|
+
if r and r.get("pmcid") and r.get("isOpenAccess") == "Y" and r.get("inEPMC") == "Y":
|
|
2923
|
+
return r["pmcid"]
|
|
2924
|
+
return None
|
|
2925
|
+
|
|
2926
|
+
|
|
2927
|
+
def arxiv_pdf(aid, d):
|
|
2928
|
+
(d / "in").mkdir(parents=True, exist_ok=True)
|
|
2929
|
+
data, _ = http_get("https://arxiv.org/pdf/%s" % urllib.parse.quote(aid, safe="/.v"), {"arxiv.org"})
|
|
2930
|
+
if b"%PDF-" not in data[:1024]:
|
|
2931
|
+
raise ReaderError("arXiv did not return a PDF for %s" % aid)
|
|
2932
|
+
out = d / "in" / "paper.pdf"
|
|
2933
|
+
out.write_bytes(data)
|
|
2934
|
+
return out
|
|
2935
|
+
|
|
2936
|
+
|
|
2937
|
+
def take_route(want, meta, d, pdf, pages):
|
|
2938
|
+
"""A local PDF is used as given, because it may be a particular version of
|
|
2939
|
+
the paper. Otherwise, or with --route online, the best source comes first:
|
|
2940
|
+
arXiv HTML, then Europe PMC XML, then ar5iv's conversion, then the PDF."""
|
|
2941
|
+
tried = []
|
|
2942
|
+
arxiv_has_none = False # ar5iv is tried only when arXiv answered that it has no HTML, never after an outage
|
|
2943
|
+
if want == "auto" and pdf is not None:
|
|
2944
|
+
order = ["pdf"]
|
|
2945
|
+
if meta.get("arxiv") or meta.get("doi"):
|
|
2946
|
+
print(" using your PDF. To try the online version of this paper instead, add --route online")
|
|
2947
|
+
elif want in ("auto", "online"):
|
|
2948
|
+
order = ["arxiv", "jats", "ar5iv", "pdf"] # the publisher's XML is a better copy than ar5iv's older conversion
|
|
2949
|
+
else:
|
|
2950
|
+
order = [want]
|
|
2951
|
+
for route in order:
|
|
2952
|
+
try:
|
|
2953
|
+
if route == "ar5iv" and not arxiv_has_none:
|
|
2954
|
+
continue
|
|
2955
|
+
if route in ("arxiv", "ar5iv") and meta.get("arxiv"):
|
|
2956
|
+
fetch_arxiv(meta["arxiv"], d, ("arxiv", "ar5iv") if want == "arxiv" else (route,))
|
|
2957
|
+
try:
|
|
2958
|
+
copy = pdf or arxiv_pdf(meta["arxiv"], d)
|
|
2959
|
+
meta["text_path"], meta["text_label"] = pdf_to_text(copy, d / "paper.txt"), "PDF's text"
|
|
2960
|
+
meta["pdf_path"], meta["pdf_href"] = copy, "https://arxiv.org/pdf/%s" % meta["arxiv"]
|
|
2961
|
+
except ReaderError as e:
|
|
2962
|
+
print(" coverage check unavailable: %s" % e)
|
|
2963
|
+
return "arxiv", arxiv_doc(meta, d)
|
|
2964
|
+
if route == "jats":
|
|
2965
|
+
pmcid = open_pmcid(meta)
|
|
2966
|
+
if pmcid:
|
|
2967
|
+
fetch_jats(pmcid, d)
|
|
2968
|
+
meta["text_path"], meta["text_label"] = jats_text(d / "jats.xml", d / "jats.txt"), "article's XML"
|
|
2969
|
+
return "jats", jats_doc(meta, d)
|
|
2970
|
+
if route == "pdf":
|
|
2971
|
+
if pdf is None and meta.get("arxiv"):
|
|
2972
|
+
pdf = arxiv_pdf(meta["arxiv"], d)
|
|
2973
|
+
if pdf is None:
|
|
2974
|
+
continue
|
|
2975
|
+
need_tool("pdftoppm", "The PDF route")
|
|
2976
|
+
need_tool("pdftotext", "The PDF route")
|
|
2977
|
+
meta["pdf_path"], meta["pdf_href"] = pdf, "paper.pdf"
|
|
2978
|
+
if meta.get("arxiv"):
|
|
2979
|
+
meta["url"] = "https://arxiv.org/abs/%s" % meta["arxiv"]
|
|
2980
|
+
meta["text_path"], meta["text_label"] = pdf_to_text(pdf, d / "paper.txt"), "PDF's text"
|
|
2981
|
+
if not sorted((d / "docling").glob("*.json")):
|
|
2982
|
+
run_docling(pdf, d, pages)
|
|
2983
|
+
return "pdf", docling_doc(meta, d, pages)
|
|
2984
|
+
except ReaderError as e:
|
|
2985
|
+
arxiv_has_none = arxiv_has_none or (route == "arxiv" and getattr(e, "no_html", False))
|
|
2986
|
+
tried.append("%s: %s" % (route, e))
|
|
2987
|
+
print(" %s route unavailable: %s" % (route, e))
|
|
2988
|
+
if want not in ("auto", "online") and not tried:
|
|
2989
|
+
raise ReaderError("the %s route doesn't apply to this input" % want)
|
|
2990
|
+
if tried:
|
|
2991
|
+
raise ReaderError("couldn't get the full text:\n %s\nTry again later, or download the PDF and run: lit-reader paper.pdf" % "\n ".join(tried))
|
|
2992
|
+
raise ReaderError("no open full text found. Download the PDF and run: lit-reader paper.pdf")
|
|
2993
|
+
|
|
2994
|
+
|
|
2995
|
+
def cmd_standalone(a):
|
|
2996
|
+
if a.pages and not re.fullmatch(r"\d+-\d+", a.pages):
|
|
2997
|
+
raise ReaderError("--pages takes a range such as 12-30")
|
|
2998
|
+
mail_to = email_settings(a.email)
|
|
2999
|
+
a.epub = a.epub or a.email == "epub" # the copy to send has to be written first
|
|
3000
|
+
a.single_file = a.single_file or a.email == "html"
|
|
3001
|
+
kind, value = parse_input(a.input)
|
|
3002
|
+
meta, d, pdf = identify(kind, value)
|
|
3003
|
+
if a.pages: # a page range is a different conversion of the same PDF
|
|
3004
|
+
d = d / ("pages-" + a.pages)
|
|
3005
|
+
d.mkdir(parents=True, exist_ok=True)
|
|
3006
|
+
name = re.sub(r"-+", "-", slug(meta)).strip("-") or "paper"
|
|
3007
|
+
out = Path(a.output).expanduser() if a.output else Path.cwd() / name
|
|
3008
|
+
target = out / "index.html"
|
|
3009
|
+
ours = target.is_file() and 'content="lit-reader"' in target.read_text(encoding="utf-8", errors="replace")[:600]
|
|
3010
|
+
if out.exists() and (not out.is_dir() or (any(out.iterdir()) and not ours)):
|
|
3011
|
+
raise ReaderError("%s already holds other files; choose an empty folder with -o" % out)
|
|
3012
|
+
route, doc = take_route(a.route, meta, d, pdf, a.pages)
|
|
3013
|
+
page, total, cov = finish_page(doc, doc.meta, a.pages)
|
|
3014
|
+
out.mkdir(parents=True, exist_ok=True)
|
|
3015
|
+
write_assets(out)
|
|
3016
|
+
if route == "pdf":
|
|
3017
|
+
shutil.copyfile(str(meta["pdf_path"]), str(out / "paper.pdf"))
|
|
3018
|
+
target.write_text(page, encoding="utf-8")
|
|
3019
|
+
print("wrote %s (route %s, %d words, %d references, %d notes)" % (target, route, total, len(doc.refs), doc.nnotes))
|
|
3020
|
+
report_coverage(cov, doc)
|
|
3021
|
+
copies = {"epub": write_epub_copy(page, doc.meta, out / (name + ".epub")) if a.epub else None,
|
|
3022
|
+
"html": write_single_copy(doc, out / (name + ".html")) if a.single_file else None}
|
|
3023
|
+
if mail_to:
|
|
3024
|
+
email_copy(mail_to, copies[a.email], doc.meta)
|
|
3025
|
+
|
|
3026
|
+
|
|
3027
|
+
# --------------------------------------------------------------------------
|
|
3028
|
+
# Bundles: several papers from a plain text file, with an index page
|
|
3029
|
+
# --------------------------------------------------------------------------
|
|
3030
|
+
|
|
3031
|
+
BUNDLE_OPTS = re.compile(r"(?:\s+(?:pages|route)=\S+)+\s*$")
|
|
3032
|
+
ROUTES = ("auto", "online", "arxiv", "jats", "pdf")
|
|
3033
|
+
|
|
3034
|
+
|
|
3035
|
+
def read_bundle(path):
|
|
3036
|
+
"""A bundle file: `#` lines before the first paper give the title (the
|
|
3037
|
+
first) and a note (the rest); then one paper per line, as standalone mode
|
|
3038
|
+
takes it, optionally followed by pages=A-B and route=R. Relative PDF paths
|
|
3039
|
+
are read from the file's own folder."""
|
|
3040
|
+
try:
|
|
3041
|
+
lines = path.read_text(encoding="utf-8").splitlines()
|
|
3042
|
+
except (OSError, UnicodeDecodeError) as e:
|
|
3043
|
+
raise ReaderError("can't read the bundle file %s: %s" % (path, e))
|
|
3044
|
+
title, note, items = None, [], []
|
|
3045
|
+
for no, raw in enumerate(lines, 1):
|
|
3046
|
+
line = raw.strip()
|
|
3047
|
+
if not line:
|
|
3048
|
+
continue
|
|
3049
|
+
if line.startswith("#"):
|
|
3050
|
+
text = line.lstrip("#").strip()
|
|
3051
|
+
if not items and text: # comments among the papers are just comments
|
|
3052
|
+
if title is None:
|
|
3053
|
+
title = text
|
|
3054
|
+
else:
|
|
3055
|
+
note.append(text)
|
|
3056
|
+
continue
|
|
3057
|
+
opts = {}
|
|
3058
|
+
m = BUNDLE_OPTS.search(line)
|
|
3059
|
+
if m:
|
|
3060
|
+
for tok in m.group(0).split():
|
|
3061
|
+
k, v = tok.split("=", 1)
|
|
3062
|
+
opts[k] = v
|
|
3063
|
+
line = line[:m.start()].strip()
|
|
3064
|
+
if opts.get("pages") and not re.fullmatch(r"\d+-\d+", opts["pages"]):
|
|
3065
|
+
raise ReaderError("%s line %d: pages= takes a range such as 12-30" % (path, no))
|
|
3066
|
+
if opts.get("route") and opts["route"] not in ROUTES:
|
|
3067
|
+
raise ReaderError("%s line %d: route= must be one of %s" % (path, no, ", ".join(ROUTES)))
|
|
3068
|
+
p = Path(line).expanduser()
|
|
3069
|
+
if (not re.match(r"[a-z][a-z0-9+.-]*://", line, re.I) and not p.is_absolute()
|
|
3070
|
+
and ((path.parent / p).is_file() or line.lower().endswith(".pdf"))):
|
|
3071
|
+
line = str(path.parent / p)
|
|
3072
|
+
items.append({"input": line, "line": no, "pages": opts.get("pages"), "route": opts.get("route")})
|
|
3073
|
+
if not items:
|
|
3074
|
+
raise ReaderError("%s lists no papers" % path)
|
|
3075
|
+
return {"title": title or path.stem.replace("-", " ").replace("_", " "), "note": " ".join(note), "items": items}
|
|
3076
|
+
|
|
3077
|
+
|
|
3078
|
+
def folder_name(title):
|
|
3079
|
+
words_ = re.sub(r"[^a-z0-9]+", " ", unicodedata.normalize("NFKD", title).encode("ascii", "ignore").decode().lower()).split()
|
|
3080
|
+
return "-".join(words_[:6]) or "bundle"
|
|
3081
|
+
|
|
3082
|
+
|
|
3083
|
+
def write_bundle_epub(built, title, note, path, missing=()):
|
|
3084
|
+
"""One EPUB holding every paper in `built`, a list of (page html, meta, page file name).
|
|
3085
|
+
`missing` names the papers that weren't converted, so the book can say so."""
|
|
3086
|
+
from .epub import write_bundle
|
|
3087
|
+
try:
|
|
3088
|
+
write_bundle(built, title, note, path, missing)
|
|
3089
|
+
except Exception as e: # the reading copies are already written
|
|
3090
|
+
print(" bundle EPUB not written (%s): %s" % (type(e).__name__, e))
|
|
3091
|
+
return None
|
|
3092
|
+
print("wrote %s (%d papers in one EPUB)" % (path, len(built)))
|
|
3093
|
+
return path
|
|
3094
|
+
|
|
3095
|
+
|
|
3096
|
+
WHAT_IT_IS = {"doi": "the DOI", "arxiv": "the arXiv ID", "pmcid": "the PMCID"}
|
|
3097
|
+
|
|
3098
|
+
|
|
3099
|
+
def bundle_failure(e, p, bundle_file):
|
|
3100
|
+
"""(reason, what to do) for a paper of a bundle that wasn't converted. The
|
|
3101
|
+
advice is about the bundle file: a reading copy made on its own wouldn't
|
|
3102
|
+
join the bundle."""
|
|
3103
|
+
msg = str(e)
|
|
3104
|
+
where = "line %d of %s" % (p["line"], bundle_file.name)
|
|
3105
|
+
if p.get("kind") == "pdf": # the PDF is already there: what went wrong is on this machine or in the file
|
|
3106
|
+
said = next((l.strip()[5:].rstrip(":") for l in msg.splitlines() if l.strip().startswith("pdf: ")), None)
|
|
3107
|
+
if said and "needs poppler" in said:
|
|
3108
|
+
return said, "Install it, then run the command again."
|
|
3109
|
+
if said:
|
|
3110
|
+
return said, "Check that the PDF on %s opens, then run the command again." % where
|
|
3111
|
+
lines = [l.strip() for l in msg.splitlines() if l.strip()] or [msg]
|
|
3112
|
+
first = lines[1] if lines[0].endswith(":") and len(lines) > 1 else lines[0] # the first route's reason, not the header
|
|
3113
|
+
return first, "Check %s." % where
|
|
3114
|
+
instead = WHAT_IT_IS.get(p.get("kind"))
|
|
3115
|
+
swap = ("Download the PDF, put its path on %s in place of %s, and run the command again." % (where, instead) if instead
|
|
3116
|
+
else "Download the PDF, put its path on %s, and run the command again." % where)
|
|
3117
|
+
if msg.startswith("no open full text"):
|
|
3118
|
+
return "no open copy online", swap
|
|
3119
|
+
if msg.startswith("couldn't get the full text"):
|
|
3120
|
+
return "the online sources didn't give its full text", "Try again later. If it still fails: " + swap[0].lower() + swap[1:]
|
|
3121
|
+
return msg.splitlines()[0], "Check %s." % where
|
|
3122
|
+
|
|
3123
|
+
|
|
3124
|
+
def cmd_bundle(a):
|
|
3125
|
+
bundle_file = Path(a.file).expanduser().resolve()
|
|
3126
|
+
spec = read_bundle(bundle_file)
|
|
3127
|
+
mode = epub_mode(a.epub)
|
|
3128
|
+
out = Path(a.output).expanduser() if a.output else Path.cwd() / folder_name(spec["title"])
|
|
3129
|
+
index = out / "index.html"
|
|
3130
|
+
ours = index.is_file() and 'content="lit-reader index"' in index.read_text(encoding="utf-8", errors="replace")[:600]
|
|
3131
|
+
if out.exists() and (not out.is_dir() or (any(out.iterdir()) and not ours)):
|
|
3132
|
+
raise ReaderError("%s already holds other files; choose an empty folder with -o" % out)
|
|
3133
|
+
total = len(spec["items"])
|
|
3134
|
+
|
|
3135
|
+
# First every paper's record, so each page can link to the others
|
|
3136
|
+
papers, used = [], set()
|
|
3137
|
+
for n, it in enumerate(spec["items"], 1):
|
|
3138
|
+
print("[%d/%d] %s" % (n, total, it["input"]))
|
|
3139
|
+
p = dict(it, n=n)
|
|
3140
|
+
try:
|
|
3141
|
+
kind, value = parse_input(it["input"])
|
|
3142
|
+
p["kind"] = kind
|
|
3143
|
+
meta, d, pdf = identify(kind, value)
|
|
3144
|
+
except ReaderError as e:
|
|
3145
|
+
p["error"], p["fix"] = bundle_failure(e, p, bundle_file)
|
|
3146
|
+
print(" not converted: %s. %s" % (p["error"], p["fix"]))
|
|
3147
|
+
papers.append(p)
|
|
3148
|
+
continue
|
|
3149
|
+
if it["pages"]: # a page range is a different conversion of the same PDF
|
|
3150
|
+
d = d / ("pages-" + it["pages"])
|
|
3151
|
+
d.mkdir(parents=True, exist_ok=True)
|
|
3152
|
+
name = re.sub(r"-+", "-", slug(meta)).strip("-") or "paper"
|
|
3153
|
+
base, k = name, 2
|
|
3154
|
+
while name in used: # the same paper twice, say for two page ranges
|
|
3155
|
+
name, k = "%s-%d" % (base, k), k + 1
|
|
3156
|
+
used.add(name)
|
|
3157
|
+
# its own reading state: the same PDF's other ranges are other copies
|
|
3158
|
+
pid = "%s-pages-%s" % (meta["pid"], it["pages"]) if it["pages"] else meta["pid"]
|
|
3159
|
+
p.update(meta=meta, d=d, pdf=pdf, name=name, pid=pid)
|
|
3160
|
+
papers.append(p)
|
|
3161
|
+
|
|
3162
|
+
# Then every conversion, so the pages link only to papers that were converted
|
|
3163
|
+
for p in papers:
|
|
3164
|
+
if not p.get("meta"):
|
|
3165
|
+
continue
|
|
3166
|
+
print("[%d/%d] converting %s" % (p["n"], total, p["meta"].get("title") or p["input"]))
|
|
3167
|
+
meta = p["meta"]
|
|
3168
|
+
meta.update(bundle=True, place=(p["n"], total))
|
|
3169
|
+
try:
|
|
3170
|
+
p["route"], p["doc"] = take_route(p["route"] or a.route, meta, p["d"], p["pdf"], p["pages"])
|
|
3171
|
+
except ReaderError as e:
|
|
3172
|
+
p["error"], p["fix"] = bundle_failure(e, p, bundle_file)
|
|
3173
|
+
print(" not converted: %s. %s" % (p["error"], p["fix"]))
|
|
3174
|
+
continue
|
|
3175
|
+
p["doc"].meta["pid"] = p["pid"]
|
|
3176
|
+
done = [p for p in papers if p.get("doc")]
|
|
3177
|
+
members = [{"pid": p["doc"].meta["pid"], "title": p["meta"].get("title"), "doi": p["meta"].get("doi"),
|
|
3178
|
+
"arxiv": p["meta"].get("arxiv"), "href": p["name"] + ".html", "n": p["n"], "total": total}
|
|
3179
|
+
for p in done]
|
|
3180
|
+
|
|
3181
|
+
out.mkdir(parents=True, exist_ok=True)
|
|
3182
|
+
write_assets(out)
|
|
3183
|
+
built = []
|
|
3184
|
+
for p in done:
|
|
3185
|
+
doc, meta, route = p["doc"], p["meta"], p["route"]
|
|
3186
|
+
doc.meta["bundle_members"] = members
|
|
3187
|
+
if route == "pdf": # the page links to its own copy of the PDF, beside it
|
|
3188
|
+
doc.meta["pdf_href"] = p["name"] + ".pdf"
|
|
3189
|
+
shutil.copyfile(str(meta["pdf_path"]), str(out / (p["name"] + ".pdf")))
|
|
3190
|
+
page, words_, cov = finish_page(doc, doc.meta, p["pages"])
|
|
3191
|
+
(out / (p["name"] + ".html")).write_text(page, encoding="utf-8")
|
|
3192
|
+
print("wrote %s (route %s, %d words, %d references, %d notes)" % (out / (p["name"] + ".html"), route, words_, len(doc.refs), doc.nnotes))
|
|
3193
|
+
report_coverage(cov, doc)
|
|
3194
|
+
if mode in ("papers", "both"):
|
|
3195
|
+
write_epub_copy(page, doc.meta, out / (p["name"] + ".epub"))
|
|
3196
|
+
if a.single_file:
|
|
3197
|
+
write_single_copy(doc, out / "single-file" / (p["name"] + ".html"))
|
|
3198
|
+
built.append((page, doc.meta, p["name"] + ".html"))
|
|
3199
|
+
if mode in ("bundle", "both") and built:
|
|
3200
|
+
missing = [(p.get("meta") or {}).get("title") or p["input"] for p in papers if p.get("error")]
|
|
3201
|
+
write_bundle_epub(built, spec["title"], spec["note"], out / (folder_name(spec["title"]) + ".epub"), missing)
|
|
3202
|
+
|
|
3203
|
+
rows = []
|
|
3204
|
+
for p in papers:
|
|
3205
|
+
meta = (p["doc"].meta if p.get("doc") else p.get("meta")) or {"title": p["input"], "authors": [], "year": "", "venue": ""}
|
|
3206
|
+
why = ("not converted: %s. %s" % (p["error"], p["fix"])) if p.get("error") else None
|
|
3207
|
+
rows.append(paper_row(out, p["n"], p.get("pid") or "line-%d" % p["line"], meta,
|
|
3208
|
+
p["name"] + ".html" if p.get("name") and not p.get("error") else "", why_not=why))
|
|
3209
|
+
write_index(out, spec["title"], [("", spec["note"], rows)], "Bundle", "%d papers" % total)
|
|
3210
|
+
failed = [p for p in papers if p.get("error")]
|
|
3211
|
+
if failed:
|
|
3212
|
+
print("\n%d of %d papers weren't converted:" % (len(failed), total))
|
|
3213
|
+
for p in failed:
|
|
3214
|
+
print(" \u201c%s\u201d: %s. %s" % ((p.get("meta") or {}).get("title") or p["input"], p["error"], p["fix"]))
|
|
3215
|
+
|
|
3216
|
+
|
|
3217
|
+
def bundle_parser():
|
|
3218
|
+
ap = argparse.ArgumentParser(prog="lit-reader bundle",
|
|
3219
|
+
description="Build reading copies of several papers, with an index page, from a text file: "
|
|
3220
|
+
"one paper per line (a PDF path, arXiv ID or URL, DOI, or PMCID). "
|
|
3221
|
+
"Lines starting with # before the first paper give the bundle's title and a note.")
|
|
3222
|
+
ap.add_argument("file", help="the bundle file")
|
|
3223
|
+
ap.add_argument("-o", "--output", help="output folder (default: ./<title>/)")
|
|
3224
|
+
ap.add_argument("--route", choices=list(ROUTES), default="auto", help="source to use for every paper; a line's own route= wins")
|
|
3225
|
+
ap.add_argument("--epub", nargs="?", const=True, type=epub_choice, metavar="papers|bundle|both", help=EPUB_HELP)
|
|
3226
|
+
ap.add_argument("--single-file", action="store_true", help=SINGLE_HELP + ", in single-file/")
|
|
3227
|
+
return ap
|
|
3228
|
+
|
|
3229
|
+
|
|
3230
|
+
LEDGER_COMMANDS = ("fetch", "pdf-convert", "build", "index", "all")
|
|
3231
|
+
|
|
3232
|
+
|
|
3233
|
+
def write_epub_copy(page, meta, path):
|
|
3234
|
+
from .epub import write_epub # only needed when asked for
|
|
3235
|
+
try:
|
|
3236
|
+
write_epub(page, meta, path)
|
|
3237
|
+
except Exception as e: # the HTML copy is already written; one failed EPUB shouldn't stop a batch
|
|
3238
|
+
print(" EPUB not written (%s): %s" % (type(e).__name__, e))
|
|
3239
|
+
return None
|
|
3240
|
+
print("wrote %s" % path)
|
|
3241
|
+
return path
|
|
3242
|
+
|
|
3243
|
+
|
|
3244
|
+
def write_single_copy(doc, path):
|
|
3245
|
+
try:
|
|
3246
|
+
page = single_file_copy(doc)
|
|
3247
|
+
path.parent.mkdir(parents=True, exist_ok=True)
|
|
3248
|
+
path.write_text(page, encoding="utf-8")
|
|
3249
|
+
except Exception as e: # as with the EPUB: the main copy is already written, and one failure shouldn't stop a batch
|
|
3250
|
+
print(" single-file copy not written (%s): %s" % (type(e).__name__, e))
|
|
3251
|
+
return None
|
|
3252
|
+
print("wrote %s (one file, %.1f MB, for a phone)" % (path, len(page.encode("utf-8")) / 1e6))
|
|
3253
|
+
return path
|
|
3254
|
+
|
|
3255
|
+
|
|
3256
|
+
SINGLE_HELP = "also write a copy with everything inside one HTML file, to send to a phone"
|
|
3257
|
+
EPUB_HELP = ("also write EPUBs for an e-reader: papers (the default) writes one per paper, bundle one holding every paper, "
|
|
3258
|
+
"both writes both. A settings-file line epub = \"...\" sets the default")
|
|
3259
|
+
EMAIL_HELP = ("email a copy to yourself: the EPUB (the default, for Apple Books or a Kindle) or the single HTML file. "
|
|
3260
|
+
"Set it up first with: lit-reader setup-email")
|
|
3261
|
+
|
|
3262
|
+
|
|
3263
|
+
def email_settings(kind):
|
|
3264
|
+
"""Checked before the build starts, so a long conversion never ends in a missing password."""
|
|
3265
|
+
if not kind:
|
|
3266
|
+
return None
|
|
3267
|
+
from . import mail
|
|
3268
|
+
try:
|
|
3269
|
+
return mail.settings(read_settings(CONFIG) if CONFIG.is_file() else {}, CONFIG)
|
|
3270
|
+
except mail.MailError as e:
|
|
3271
|
+
raise ReaderError(str(e))
|
|
3272
|
+
|
|
3273
|
+
|
|
3274
|
+
def email_copy(s, path, meta):
|
|
3275
|
+
from . import mail
|
|
3276
|
+
if path is None: # never fall back to a copy left by an earlier run
|
|
3277
|
+
raise ReaderError("the copy to email wasn't written this time, so nothing was emailed")
|
|
3278
|
+
title = re.sub(r"\s+", " ", strip_tags(meta.get("title_html") or esc(meta.get("title", "")))).strip() or path.stem
|
|
3279
|
+
try:
|
|
3280
|
+
mail.send(s, mail.message(s, path, title, full_reference(meta, title), __version__))
|
|
3281
|
+
except mail.MailError as e:
|
|
3282
|
+
raise ReaderError("the copies are written, but %s wasn't emailed: %s" % (path.name, e))
|
|
3283
|
+
print("emailed %s to %s" % (path.name, s["to"]))
|
|
3284
|
+
|
|
3285
|
+
|
|
3286
|
+
def standalone_parser():
|
|
3287
|
+
ap = argparse.ArgumentParser(prog="lit-reader", description="Turn one academic paper into an offline HTML reading copy.",
|
|
3288
|
+
epilog="Ledger commands: %s (each has its own --help). "
|
|
3289
|
+
"To set up --email: lit-reader setup-email" % ", ".join(LEDGER_COMMANDS))
|
|
3290
|
+
ap.add_argument("--version", action="version", version="lit-reader " + __version__)
|
|
3291
|
+
ap.add_argument("input", help="a PDF path, arXiv ID or URL, DOI, or PMCID")
|
|
3292
|
+
ap.add_argument("-o", "--output", help="output folder (default: ./<slug>/)")
|
|
3293
|
+
ap.add_argument("--route", choices=["auto", "online", "arxiv", "jats", "pdf"], default="auto",
|
|
3294
|
+
help="source to use. auto (default) keeps a local PDF as given and otherwise picks the best source; "
|
|
3295
|
+
"online prefers arXiv or Europe PMC even when you supply a PDF")
|
|
3296
|
+
ap.add_argument("--pages", help="PDF route only: convert these pages, for example 12-30")
|
|
3297
|
+
ap.add_argument("--epub", action="store_true", help="also write an EPUB copy for an e-reader")
|
|
3298
|
+
ap.add_argument("--single-file", action="store_true", help=SINGLE_HELP)
|
|
3299
|
+
ap.add_argument("--email", nargs="?", const="epub", choices=["epub", "html"], help=EMAIL_HELP)
|
|
3300
|
+
return ap
|
|
3301
|
+
|
|
3302
|
+
|
|
3303
|
+
def email_first(argv):
|
|
3304
|
+
"""argv with a bare --email followed by something other than its own value
|
|
3305
|
+
written as --email=epub, so `lit-reader --email paper.pdf` doesn't take the
|
|
3306
|
+
paper for the copy type. A bare --epub of `bundle` or `all` moves to the end,
|
|
3307
|
+
where it keeps meaning "the default kind"."""
|
|
3308
|
+
out = list(argv)
|
|
3309
|
+
for i, v in enumerate(out[:-1]):
|
|
3310
|
+
if v == "--email" and out[i + 1] not in ("epub", "html"):
|
|
3311
|
+
out[i] = "--email=epub"
|
|
3312
|
+
if out[:1] in (["bundle"], ["all"]) and "--epub" in out[:-1] and out[out.index("--epub") + 1] not in EPUB_MODES:
|
|
3313
|
+
out.append(out.pop(out.index("--epub"))) # last, so it can't take the bundle file for its value
|
|
3314
|
+
return out
|
|
3315
|
+
|
|
3316
|
+
|
|
3317
|
+
def main(argv=None):
|
|
3318
|
+
argv = email_first(sys.argv[1:] if argv is None else argv)
|
|
3319
|
+
if argv[:1] == ["setup-email"]:
|
|
3320
|
+
argparse.ArgumentParser(prog="lit-reader setup-email",
|
|
3321
|
+
description="Set up --email: asks for your address and mail server, writes them to "
|
|
3322
|
+
"the settings file, stores the password in Keychain and sends a test.").parse_args(argv[1:])
|
|
3323
|
+
from . import mail
|
|
3324
|
+
try:
|
|
3325
|
+
return mail.setup(CONFIG, read_settings, __version__)
|
|
3326
|
+
except ReaderError as e: # an existing settings file that doesn't read
|
|
3327
|
+
print("error: %s" % e, file=sys.stderr)
|
|
3328
|
+
return 1
|
|
3329
|
+
if argv[:1] == ["bundle"]:
|
|
3330
|
+
try:
|
|
3331
|
+
cmd_bundle(bundle_parser().parse_args(argv[1:]))
|
|
3332
|
+
except ReaderError as e:
|
|
3333
|
+
print("error: %s" % e, file=sys.stderr)
|
|
3334
|
+
return 1
|
|
3335
|
+
return 0
|
|
3336
|
+
if not argv or argv[0] not in LEDGER_COMMANDS:
|
|
3337
|
+
try:
|
|
3338
|
+
cmd_standalone(standalone_parser().parse_args(argv))
|
|
3339
|
+
except ReaderError as e:
|
|
3340
|
+
print("error: %s" % e, file=sys.stderr)
|
|
3341
|
+
return 1
|
|
3342
|
+
return 0
|
|
3343
|
+
ap = argparse.ArgumentParser(prog="lit-reader", description=__doc__.split("\n\n")[0])
|
|
3344
|
+
sub = ap.add_subparsers(dest="cmd", required=True)
|
|
3345
|
+
f = sub.add_parser("fetch")
|
|
3346
|
+
f.add_argument("--pid", required=True)
|
|
3347
|
+
f.add_argument("--ledger", required=True)
|
|
3348
|
+
f.add_argument("--route", choices=["arxiv", "jats"], required=True)
|
|
3349
|
+
f.add_argument("--pmcid")
|
|
3350
|
+
c = sub.add_parser("pdf-convert")
|
|
3351
|
+
c.add_argument("--pid", required=True)
|
|
3352
|
+
c.add_argument("--ledger", required=True)
|
|
3353
|
+
c.add_argument("--pages")
|
|
3354
|
+
b = sub.add_parser("build")
|
|
3355
|
+
b.add_argument("--pid", required=True)
|
|
3356
|
+
b.add_argument("--ledger", required=True)
|
|
3357
|
+
b.add_argument("--route", choices=["arxiv", "jats", "pdf"], required=True)
|
|
3358
|
+
b.add_argument("--pages")
|
|
3359
|
+
b.add_argument("--epub", action="store_true")
|
|
3360
|
+
b.add_argument("--single-file", action="store_true", help=SINGLE_HELP + ", in single-file/")
|
|
3361
|
+
b.add_argument("--email", nargs="?", const="epub", choices=["epub", "html"], help=EMAIL_HELP)
|
|
3362
|
+
sub.add_parser("index")
|
|
3363
|
+
al = sub.add_parser("all")
|
|
3364
|
+
al.add_argument("--epub", nargs="?", const=True, type=epub_choice, metavar="papers|bundle|both", help=EPUB_HELP)
|
|
3365
|
+
al.add_argument("--single-file", action="store_true", help=SINGLE_HELP + ", in single-file/")
|
|
3366
|
+
a = ap.parse_args(argv)
|
|
3367
|
+
try:
|
|
3368
|
+
apply_settings()
|
|
3369
|
+
{"fetch": cmd_fetch, "pdf-convert": cmd_pdf_convert, "build": cmd_build, "index": cmd_index, "all": cmd_all}[a.cmd](a)
|
|
3370
|
+
except ReaderError as e:
|
|
3371
|
+
print("error: %s" % e, file=sys.stderr)
|
|
3372
|
+
return 1
|
|
3373
|
+
return 0
|
|
3374
|
+
|
|
3375
|
+
|
|
3376
|
+
if __name__ == "__main__":
|
|
3377
|
+
sys.exit(main())
|