unitoolkit 0.7.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
unitoolkit/docs.py ADDED
@@ -0,0 +1,394 @@
1
+ """Document text: download a course file by id, extract text once, cache the text.
2
+
3
+ Why this exists: a Canvas file is a name until someone reads it. `display_name` is all the
4
+ API offers, and the words a student searches for are inside the PDF. So each file is fetched
5
+ once, parsed, and the *text* is kept while the bytes are discarded, keyed by
6
+ `file id + updated_at` so a reindex costs nothing and an updated file is re-read.
7
+
8
+ Extraction, in order of preference:
9
+ pdf pdftotext (poppler; no dependency of ours)
10
+ docx zipfile + word/document.xml
11
+ pptx zipfile + ppt/slides/slideN.xml
12
+ xlsx zipfile + xl/sharedStrings.xml
13
+ html tag strip, entities decoded
14
+ image OCR (opt in), tesseract with the Swedish model
15
+
16
+ OCR is deliberately not part of every index run: it is slow (1.5-3 s per page at 300 dpi)
17
+ and most files do not need it. It runs when a document has no usable text layer, is capped,
18
+ and its result is cached like any other text.
19
+
20
+ python -m unitoolkit docs course=2416 # text layer only, fast
21
+ python -m unitoolkit docs course=2416 ocr=true # plus OCR for image-only files
22
+ """
23
+ import hashlib
24
+ import html
25
+ import io
26
+ import json
27
+ import pathlib
28
+ import re
29
+ import shutil
30
+ import subprocess
31
+ import tempfile
32
+ import zipfile
33
+
34
+ from . import canvas
35
+
36
+ CACHE = pathlib.Path.home() / ".cache/unitoolkit/docs"
37
+
38
+ TEXT_EXTS = {"txt", "md", "csv", "json", "html", "htm", "xml", "py", "js", "css"}
39
+ IMAGE_EXTS = {"png", "jpg", "jpeg", "gif", "bmp", "tif", "tiff"}
40
+ OFFICE_EXTS = {"docx", "pptx", "xlsx"}
41
+
42
+ # a document whose text is this thin is treated as image-only
43
+ MIN_USEFUL_CHARS = 100
44
+ MIN_CHARS_PER_PAGE = 40
45
+
46
+
47
+ def _have(binary):
48
+ return shutil.which(binary) is not None
49
+
50
+
51
+ def _run(cmd, **kw):
52
+ return subprocess.run(cmd, capture_output=True, text=True, **kw)
53
+
54
+
55
+ # ---------------------------------------------------------------- extractors
56
+ def docx_text(data):
57
+ with zipfile.ZipFile(io.BytesIO(data)) as z:
58
+ names = z.namelist()
59
+ parts = [n for n in names if n == "word/document.xml" or
60
+ re.match(r"word/(header|footer)\d*\.xml$", n)]
61
+ out = []
62
+ for n in parts:
63
+ xml = z.read(n).decode("utf-8", "ignore")
64
+ xml = re.sub(r"</w:p>|</w:tr>", "\n", xml)
65
+ xml = re.sub(r"</w:tc>", " ", xml)
66
+ out.append(re.sub(r"[ \t]+", " ", re.sub(r"<[^>]+>", "", xml)))
67
+ return "\n".join(out)
68
+
69
+
70
+ def pptx_text(data):
71
+ with zipfile.ZipFile(io.BytesIO(data)) as z:
72
+ slides = []
73
+ for n in z.namelist():
74
+ m = re.match(r"ppt/slides/slide(\d+)\.xml$", n)
75
+ if m:
76
+ slides.append((int(m.group(1)), n))
77
+ out = []
78
+ for num, n in sorted(slides):
79
+ xml = z.read(n).decode("utf-8", "ignore")
80
+ xml = re.sub(r"</a:p>", "\n", xml)
81
+ txt = re.sub(r"[ \t]+", " ", re.sub(r"<[^>]+>", "", xml)).strip()
82
+ if txt:
83
+ out.append(f"Bild {num}: {txt}")
84
+ return "\n".join(out)
85
+
86
+
87
+ def xlsx_text(data):
88
+ with zipfile.ZipFile(io.BytesIO(data)) as z:
89
+ if "xl/sharedStrings.xml" not in z.namelist():
90
+ return ""
91
+ xml = z.read("xl/sharedStrings.xml").decode("utf-8", "ignore")
92
+ cells = re.findall(r"<t[^>]*>(.*?)</t>", xml, re.S)
93
+ return "\n".join(html.unescape(c) for c in cells)
94
+
95
+
96
+ def html_text(data):
97
+ s = data.decode("utf-8", "ignore")
98
+ s = re.sub(r"<script[\s\S]*?</script>|<style[\s\S]*?</style>", " ", s, flags=re.I)
99
+ s = re.sub(r"</(p|div|li|tr|h[1-6])>|<br\s*/?>", "\n", s, flags=re.I)
100
+ s = re.sub(r"<[^>]+>", " ", s)
101
+ s = html.unescape(s).replace("\u00a0", " ")
102
+ s = re.sub(r"[ \t]+", " ", s)
103
+ return re.sub(r" *\n *", "\n", re.sub(r"\n{2,}", "\n", s)).strip()
104
+
105
+
106
+ def pdf_text(data):
107
+ """Text layer via pdftotext. Page breaks survive as \\f, which gives us page numbers."""
108
+ with tempfile.TemporaryDirectory() as td:
109
+ src = pathlib.Path(td) / "in.pdf"
110
+ src.write_bytes(data)
111
+ r = _run(["pdftotext", "-enc", "UTF-8", "-q", str(src), "-"])
112
+ return r.stdout or ""
113
+
114
+
115
+ def pdf_pages(data):
116
+ """Real page count from the PDF itself, so a capped OCR run can say it was capped."""
117
+ if not _have("pdfinfo"):
118
+ return None
119
+ with tempfile.TemporaryDirectory() as td:
120
+ src = pathlib.Path(td) / "in.pdf"
121
+ src.write_bytes(data)
122
+ r = _run(["pdfinfo", str(src)])
123
+ m = re.search(r"^Pages:\s+(\d+)", r.stdout or "", re.M)
124
+ return int(m.group(1)) if m else None
125
+
126
+
127
+ def ocr_pdf(data, max_pages=60, dpi=300, lang="swe"):
128
+ """Render pages to PNG and OCR them. Returns (text, pages_ocred)."""
129
+ if not (_have("pdftoppm") and _have("tesseract")):
130
+ return "", 0
131
+ out = []
132
+ with tempfile.TemporaryDirectory() as td:
133
+ src = pathlib.Path(td) / "in.pdf"
134
+ src.write_bytes(data)
135
+ pre = str(pathlib.Path(td) / "pg")
136
+ _run(["pdftoppm", "-r", str(dpi), "-gray", "-png", "-f", "1", "-l", str(max_pages),
137
+ str(src), pre])
138
+ pages = sorted(pathlib.Path(td).glob("pg*.png"))
139
+ for i, p in enumerate(pages, 1):
140
+ r = _run(["tesseract", str(p), "-", "-l", lang, "--psm", "3"])
141
+ out.append(r.stdout.strip())
142
+ return "\f".join(out), len(out)
143
+
144
+
145
+ def ocr_image(data, lang="swe"):
146
+ if not _have("tesseract"):
147
+ return ""
148
+ with tempfile.TemporaryDirectory() as td:
149
+ src = pathlib.Path(td) / "in.png"
150
+ src.write_bytes(data)
151
+ return _run(["tesseract", str(src), "-", "-l", lang, "--psm", "3"]).stdout.strip()
152
+
153
+
154
+ def extract(name, data, allow_ocr=False, ocr_max_pages=60):
155
+ """Extract text from bytes. Returns dict: text, kind, ocr_pages, needs_ocr."""
156
+ ext = (name.rsplit(".", 1)[-1] or "").lower()
157
+ info = {"kind": ext, "ocr_pages": 0, "needs_ocr": False, "text": ""}
158
+
159
+ if ext == "pdf":
160
+ text = pdf_text(data)
161
+ pages = max(1, text.count("\f") + 1)
162
+ thin = len(text.strip()) < MIN_USEFUL_CHARS or len(text.strip()) / pages < MIN_CHARS_PER_PAGE
163
+ info["text"] = text
164
+ if thin:
165
+ info["needs_ocr"] = True
166
+ info["pages_total"] = pdf_pages(data)
167
+ if allow_ocr:
168
+ ocr, n = ocr_pdf(data, max_pages=ocr_max_pages)
169
+ info["ocr_pages"] = n
170
+ # a capped run must be visible, or a half-read book looks complete
171
+ info["ocr_truncated"] = bool(info["pages_total"] and n < info["pages_total"])
172
+ if len(ocr.strip()) > len(text.strip()):
173
+ info["text"] = ocr
174
+ return info
175
+ if ext == "docx":
176
+ info["text"] = docx_text(data)
177
+ elif ext == "pptx":
178
+ info["text"] = pptx_text(data)
179
+ elif ext == "xlsx":
180
+ info["text"] = xlsx_text(data)
181
+ elif ext in ("html", "htm", "aspx"):
182
+ info["text"] = html_text(data)
183
+ elif ext in TEXT_EXTS:
184
+ info["text"] = data.decode("utf-8", "ignore")
185
+ elif ext in IMAGE_EXTS:
186
+ info["needs_ocr"] = True
187
+ if allow_ocr:
188
+ info["text"] = ocr_image(data)
189
+ info["ocr_pages"] = 1 if info["text"] else 0
190
+ else:
191
+ info["kind"] = "binary"
192
+ return info
193
+
194
+
195
+ # ---------------------------------------------------------------- cache
196
+ def _cache_dir(course_id):
197
+ d = CACHE / str(course_id)
198
+ d.mkdir(parents=True, exist_ok=True)
199
+ return d
200
+
201
+
202
+ def cache_key(file_meta):
203
+ return hashlib.sha1(
204
+ f"{file_meta.get('id')}:{file_meta.get('updated_at') or file_meta.get('created_at')}"
205
+ .encode()).hexdigest()[:16]
206
+
207
+
208
+ def read_cached(course_id, file_meta):
209
+ p = _cache_dir(course_id) / f"{cache_key(file_meta)}.json"
210
+ if p.exists():
211
+ try:
212
+ return json.loads(p.read_text())
213
+ except Exception: # noqa: BLE001 - a broken cache entry is just a miss
214
+ return None
215
+ return None
216
+
217
+
218
+ def write_cached(course_id, file_meta, payload):
219
+ p = _cache_dir(course_id) / f"{cache_key(file_meta)}.json"
220
+ p.write_text(json.dumps(payload, ensure_ascii=False))
221
+ return p
222
+
223
+
224
+ # ---------------------------------------------------------------- build
225
+ def build(course_id, ocr=False, force=False, max_files=None, ocr_max_pages=60,
226
+ max_bytes=40 * 1024 * 1024, only_contains=None, progress=None):
227
+ """Extract text for every file in a course. Cached by file id + updated_at."""
228
+ files = canvas.try_get(f"courses/{course_id}/files", per_page=100) or []
229
+ files = [f for f in files if isinstance(f, dict)]
230
+ files.sort(key=lambda f: (f.get("display_name") or "").lower())
231
+ if only_contains:
232
+ needle = only_contains.lower()
233
+ files = [f for f in files
234
+ if needle in (f.get("display_name") or f.get("filename") or "").lower()]
235
+ if max_files:
236
+ files = files[:max_files]
237
+
238
+ stats = {"course": course_id, "files": len(files), "cached": 0, "extracted": 0,
239
+ "failed": 0, "skipped_big": 0, "needs_ocr": 0, "ocr_pages": 0,
240
+ "with_text": 0, "no_text": 0, "chars": 0, "by_kind": {}}
241
+ for i, f in enumerate(files, 1):
242
+ name = f.get("display_name") or f.get("filename") or str(f.get("id"))
243
+ if progress:
244
+ progress(f"{i}/{len(files)} {name[:44]}")
245
+ if (f.get("size") or 0) > max_bytes:
246
+ stats["skipped_big"] += 1
247
+ continue
248
+ hit = None if force else read_cached(course_id, f)
249
+ if hit and ocr and hit.get("needs_ocr") and len((hit.get("text") or "").strip()) < MIN_USEFUL_CHARS:
250
+ hit = None # cached as image-only, and OCR is now requested: read it again
251
+ stats["re_ocred"] = stats.get("re_ocred", 0) + 1
252
+ if hit:
253
+ stats["cached"] += 1
254
+ payload = hit
255
+ else:
256
+ url = f.get("url") or f"courses/{course_id}/files/{f.get('id')}/download"
257
+ try:
258
+ data = canvas.download_bytes(url)
259
+ except Exception: # noqa: BLE001 - a failed file must not stop the run
260
+ stats["failed"] += 1
261
+ continue
262
+ info = extract(name, data, allow_ocr=ocr, ocr_max_pages=ocr_max_pages)
263
+ stats["extracted"] += 1
264
+ stats["ocr_pages"] += info["ocr_pages"]
265
+ payload = {"id": f.get("id"), "name": name, "ext": info["kind"],
266
+ "updated_at": f.get("updated_at") or f.get("created_at") or "",
267
+ "href": f"/courses/{course_id}/files/{f.get('id')}",
268
+ "url": url, "bytes": len(data), "pages": info["text"].count("\f") + 1,
269
+ "pages_total": info.get("pages_total"),
270
+ "ocr_pages": info["ocr_pages"], "needs_ocr": info["needs_ocr"],
271
+ "ocr_truncated": bool(info.get("ocr_truncated")),
272
+ "text": info["text"]}
273
+ if payload["ocr_truncated"]:
274
+ stats["ocr_truncated"] = stats.get("ocr_truncated", 0) + 1
275
+ write_cached(course_id, f, payload)
276
+ kind = payload.get("ext") or "?"
277
+ stats["by_kind"][kind] = stats["by_kind"].get(kind, 0) + 1
278
+ n = len((payload.get("text") or "").strip())
279
+ stats["chars"] += n
280
+ if n >= MIN_USEFUL_CHARS:
281
+ stats["with_text"] += 1
282
+ else:
283
+ stats["no_text"] += 1
284
+ if payload.get("needs_ocr"):
285
+ stats["needs_ocr"] += 1
286
+ return stats
287
+
288
+
289
+ HREF_RE = re.compile(r'href="([^"]+)"', re.I)
290
+
291
+
292
+ def page_links(course_id, mirror_root=None):
293
+ """Every link inside the course's pages, deduped, classified.
294
+
295
+ Pages line up with the mirror; without a mirror nothing is read, which keeps this from
296
+ turning into a page crawl of its own.
297
+ """
298
+ root = pathlib.Path(mirror_root or (pathlib.Path.home() / "courses/canvas" / str(course_id)))
299
+ pages = root / "pages"
300
+ if not pages.exists():
301
+ return []
302
+ seen, out = set(), []
303
+ for p in sorted(pages.glob("*.html")):
304
+ for url in HREF_RE.findall(p.read_text(errors="ignore")):
305
+ url = html.unescape(url).strip()
306
+ if not url or url.startswith(("#", "mailto:", "javascript:")) or url in seen:
307
+ continue
308
+ seen.add(url)
309
+ kind = "other"
310
+ if re.search(r"/files/\d+", url):
311
+ kind = "canvas_file"
312
+ elif re.search(r"/pages/", url):
313
+ kind = "canvas_page"
314
+ elif re.search(r"sharepoint\.com/:[a-z]:/", url, re.I):
315
+ kind = "share"
316
+ elif re.search(r"(kaltura|youtube|vimeo)", url, re.I):
317
+ kind = "video"
318
+ out.append({"url": url, "kind": kind, "from": p.name})
319
+ return out
320
+
321
+
322
+ def resolve_links(course_id, limit=None, allow_ocr=False, progress=None):
323
+ """Fetch linked documents that are not in the file tree (SharePoint share links) and
324
+ cache their text alongside the course files. Bytes are read once and dropped."""
325
+ from . import sharepoint
326
+
327
+ links = [l for l in page_links(course_id) if l["kind"] == "share"]
328
+ if limit:
329
+ links = links[:limit]
330
+ stats = {"course": course_id, "share_links": len(links), "stored": 0, "cached": 0,
331
+ "failed": 0, "with_text": 0, "no_text": 0, "chars": 0, "by_kind": {}}
332
+ for i, link in enumerate(links, 1):
333
+ url = link["url"]
334
+ if progress:
335
+ progress(f"{i}/{len(links)} {url[-28:]}")
336
+ meta = {"id": f"share:{url}", "updated_at": "fixed"} # a share link has no version
337
+ hit = read_cached(course_id, meta)
338
+ if hit:
339
+ stats["cached"] += 1
340
+ payload = hit
341
+ else:
342
+ body, kind, name = sharepoint.fetch_share(url)
343
+ if not body:
344
+ stats["failed"] += 1
345
+ continue
346
+ name = name or f"share-{len(body)}.{kind}"
347
+ info = extract(name, body, allow_ocr=allow_ocr)
348
+ stats["stored"] += 1
349
+ payload = {"id": meta["id"], "name": name, "ext": info["kind"],
350
+ "updated_at": "", "href": url, "url": url,
351
+ "bytes": len(body), "pages": info["text"].count("\f") + 1,
352
+ "pages_total": info.get("pages_total"),
353
+ "ocr_pages": info["ocr_pages"], "needs_ocr": info["needs_ocr"],
354
+ "ocr_truncated": bool(info.get("ocr_truncated")),
355
+ "text": info["text"]}
356
+ write_cached(course_id, meta, payload)
357
+ stats["by_kind"][payload.get("ext") or "?"] = stats["by_kind"].get(payload.get("ext") or "?", 0) + 1
358
+ n = len((payload.get("text") or "").strip())
359
+ stats["chars"] += n
360
+ stats["with_text" if n >= MIN_USEFUL_CHARS else "no_text"] += 1
361
+ return stats
362
+
363
+
364
+ def load(course_id):
365
+ """Cached documents for a course, shaped like search docs."""
366
+ out = []
367
+ for p in sorted(_cache_dir(course_id).glob("*.json")):
368
+ try:
369
+ d = json.loads(p.read_text())
370
+ except Exception: # noqa: BLE001
371
+ continue
372
+ text = d.get("text") or ""
373
+ if not text.strip():
374
+ continue
375
+ out.append({"type": "fildok", "title": d.get("name") or "(fil)",
376
+ "body": text, "href": d.get("href") or "#",
377
+ "ext": d.get("ext") or "", "date": (d.get("updated_at") or "")[:10],
378
+ "pages": d.get("pages"), "ocr_pages": d.get("ocr_pages") or 0,
379
+ "bytes": d.get("bytes")})
380
+ return out
381
+
382
+
383
+ def page_of(text, at):
384
+ """1-based page number for a character offset, using pdftotext's form feeds."""
385
+ if not text:
386
+ return None
387
+ return text.count("\f", 0, max(0, at)) + 1
388
+
389
+
390
+ def clear(course_id=None):
391
+ target = CACHE / str(course_id) if course_id else CACHE
392
+ if target.exists():
393
+ shutil.rmtree(target)
394
+ return {"cleared": str(target)}
unitoolkit/fulltext.py ADDED
@@ -0,0 +1,260 @@
1
+ """Full text of an article the school subscribes to, by DOI.
2
+
3
+ The DOI is resolved through the school's EZproxy (`shh.idm.oclc.org`), so the publisher
4
+ sees Sophiahemmet's subscription. The PDF is captured at the network layer (CDP Fetch
5
+ interception) rather than by a page-side fetch(): Wiley, Elsevier and others answer a
6
+ script fetch with an HTML viewer or a bot check, but hand the real PDF to a top-level
7
+ navigation.
8
+
9
+ Tested 2026-11-17: Wiley (two paywalled articles, Unpaywall is_oa=false) and Elsevier
10
+ (ScienceDirect, via pdf.sciencedirectassets.com) return the PDF. De Gruyter is not
11
+ rewritten by the school's EZproxy, so there is no subscription route; the function says so
12
+ instead of returning a paywall page.
13
+
14
+ Licensed content: the school's licences allow a student to read and download for their own
15
+ study. They do not allow systematic or bulk downloading, and publishers react to it by
16
+ blocking the institution's whole EZproxy. Hence the hourly cap below, which is a rule, not
17
+ a tuning knob.
18
+ """
19
+
20
+ import asyncio
21
+ import base64
22
+ import json
23
+ import os
24
+ import re
25
+ import subprocess
26
+ import time
27
+ import urllib.error
28
+ import urllib.parse
29
+ import urllib.request
30
+ from pathlib import Path
31
+
32
+ from .databases import DatabaseError, _Tab, EZ
33
+
34
+ OUT = Path(os.environ.get('UNITOOLKIT_FULLTEXT_DIR',
35
+ str(Path.home() / '.local/share/unitoolkit/fulltext')))
36
+ LEDGER = OUT / '.downloads.json'
37
+ MAX_PER_HOUR = 20 # licence hygiene: never let a loop turn into a bulk download
38
+
39
+ # Publisher hosts that put the PDF behind a URL we can build from the DOI.
40
+ PDF_PATTERNS = [
41
+ (re.compile(r'onlinelibrary-wiley-com|wiley-com\.shh'), '/doi/pdfdirect/{doi}?download=true'),
42
+ (re.compile(r'journals-sagepub-com'), '/doi/pdf/{doi}?download=true'),
43
+ (re.compile(r'tandfonline-com'), '/doi/pdf/{doi}?download=true'),
44
+ (re.compile(r'cochranelibrary-com'), '/cdsr/doi/{doi}/pdf/full'),
45
+ ]
46
+
47
+ PAGE_JS = r"""(() => {
48
+ const t = document.body ? document.body.innerText : '';
49
+ const meta = (document.querySelector('meta[name=citation_pdf_url]') || {}).content || '';
50
+ const links = [...new Set([...document.querySelectorAll('a')]
51
+ .map(a => a.href).filter(h => h && /pdf/i.test(h) && !/supplement|suppl|figure/i.test(h)))];
52
+ const body = document.querySelector('.article__body, section.article-section__full, #body, '
53
+ + 'article .Body, div.article-body, #article-body, main article, .cdsr-article, '
54
+ + 'section.abstract ~ section, #content article') || (t.length > 20000 ? document.body : null);
55
+ return JSON.stringify({
56
+ url: location.href, host: location.host, title: document.title,
57
+ proxied: /\.shh\.idm\.oclc\.org$/.test(location.host),
58
+ signin: /login\.microsoftonline\.com/.test(location.href),
59
+ meta_pdf: meta, links: links.slice(0, 10),
60
+ access: (t.match(/(Open Access|Free Access|Full Access|Access provided by[^\n]{0,60}|institutional access|Get access|Purchase PDF|Buy article)/i) || [''])[0],
61
+ body_text: body ? body.innerText : '',
62
+ });
63
+ })()"""
64
+
65
+
66
+ def _slug(doi):
67
+ return re.sub(r'[^\w.-]+', '_', doi.strip())
68
+
69
+
70
+ def _ledger():
71
+ try:
72
+ return json.loads(LEDGER.read_text())
73
+ except Exception:
74
+ return []
75
+
76
+
77
+ def _note_download(doi):
78
+ now = time.time()
79
+ rows = [r for r in _ledger() if now - r['t'] < 3600]
80
+ rows.append({'t': now, 'doi': doi})
81
+ OUT.mkdir(parents=True, exist_ok=True)
82
+ LEDGER.write_text(json.dumps(rows))
83
+
84
+
85
+ def _under_cap():
86
+ now = time.time()
87
+ return sum(1 for r in _ledger() if now - r['t'] < 3600) < MAX_PER_HOUR
88
+
89
+
90
+ def unpaywall(doi):
91
+ """Open-access status, so a free copy is preferred and the paid route is not used for it."""
92
+ try:
93
+ j = json.load(urllib.request.urlopen(
94
+ 'https://api.unpaywall.org/v2/%s?email=unitoolkit@users.noreply.github.com'
95
+ % urllib.parse.quote(doi), timeout=15))
96
+ except Exception:
97
+ return {'is_oa': None}
98
+ loc = j.get('best_oa_location') or {}
99
+ return {'is_oa': j.get('is_oa'), 'oa_pdf': loc.get('url_for_pdf') or '',
100
+ 'publisher': j.get('publisher') or '', 'title': j.get('title') or ''}
101
+
102
+
103
+ async def _capture_pdf(page, url, timeout=45):
104
+ """Navigate and return the first PDF response body seen on the wire."""
105
+ await page.call('Fetch.enable', {'patterns': [{'urlPattern': '*', 'requestStage': 'Response'}]},
106
+ page.sid)
107
+ page.n += 1
108
+ await page.ws.send(json.dumps({'id': page.n, 'method': 'Page.navigate',
109
+ 'params': {'url': url}, 'sessionId': page.sid}))
110
+ loop = asyncio.get_event_loop()
111
+ end = loop.time() + timeout
112
+ try:
113
+ while loop.time() < end:
114
+ try:
115
+ msg = json.loads(await asyncio.wait_for(page.ws.recv(), timeout=end - loop.time()))
116
+ except asyncio.TimeoutError:
117
+ break
118
+ if msg.get('method') != 'Fetch.requestPaused':
119
+ continue
120
+ ev = msg['params']
121
+ rid = ev['requestId']
122
+ ctype = next((h['value'] for h in ev.get('responseHeaders') or []
123
+ if h['name'].lower() == 'content-type'), '')
124
+ status = ev.get('responseStatusCode') or 0
125
+ if 'pdf' in ctype.lower() and status < 300:
126
+ page.n += 1
127
+ k = page.n
128
+ await page.ws.send(json.dumps({'id': k, 'method': 'Fetch.getResponseBody',
129
+ 'params': {'requestId': rid}, 'sessionId': page.sid}))
130
+ while True:
131
+ r = json.loads(await page.ws.recv())
132
+ if r.get('id') == k:
133
+ break
134
+ res = r.get('result') or {}
135
+ body = res.get('body', '')
136
+ raw = base64.b64decode(body) if res.get('base64Encoded') else body.encode('latin1')
137
+ page.n += 1
138
+ await page.ws.send(json.dumps({'id': page.n, 'method': 'Fetch.continueRequest',
139
+ 'params': {'requestId': rid}, 'sessionId': page.sid}))
140
+ return raw, ev['request']['url']
141
+ page.n += 1
142
+ await page.ws.send(json.dumps({'id': page.n, 'method': 'Fetch.continueRequest',
143
+ 'params': {'requestId': rid}, 'sessionId': page.sid}))
144
+ finally:
145
+ try:
146
+ await page.call('Fetch.disable', {}, page.sid)
147
+ except Exception:
148
+ pass
149
+ return None, None
150
+
151
+
152
+ def resolve(doi):
153
+ """Publisher landing URL for a DOI, resolved outside the proxy.
154
+
155
+ Some DOI redirects break when they run through EZproxy (Cochrane lands on a Chrome
156
+ error page), while the publisher URL itself proxies fine. Resolving first and proxying
157
+ the result works for both kinds.
158
+ """
159
+ if doi.lower().startswith('10.1002/14651858.'): # Cochrane reviews: Wiley's resolver
160
+ return 'https://www.cochranelibrary.com/cdsr/doi/%s/full' % doi # stops at doi.wiley.com
161
+ req = urllib.request.Request('https://doi.org/' + doi,
162
+ headers={'User-Agent': 'Mozilla/5.0 unitoolkit'})
163
+ try:
164
+ with urllib.request.urlopen(req, timeout=20) as r:
165
+ return r.geturl()
166
+ except urllib.error.HTTPError as e: # many publishers refuse HEAD but still redirected
167
+ return e.geturl() or ''
168
+ except Exception:
169
+ return ''
170
+
171
+
172
+ def _pdf_text(path):
173
+ try:
174
+ return subprocess.run(['pdftotext', '-layout', str(path), '-'], capture_output=True,
175
+ text=True, timeout=60).stdout
176
+ except Exception:
177
+ return ''
178
+
179
+
180
+ def fetch(doi, text=True):
181
+ """Download the full text of one article through the school's subscription.
182
+
183
+ Returns the saved PDF path (or the HTML full text when the publisher only offers that),
184
+ page count, and optionally the extracted text. Raises with a reason when the school has
185
+ no route to it.
186
+ """
187
+ doi = re.sub(r'^(https?://(dx\.)?doi\.org/|doi:\s*)', '', (doi or '').strip(), flags=re.I)
188
+ if not doi.startswith('10.'):
189
+ raise DatabaseError('not a DOI: %r' % doi)
190
+ OUT.mkdir(parents=True, exist_ok=True)
191
+ pdf_path = OUT / (_slug(doi) + '.pdf')
192
+ if pdf_path.exists() and pdf_path.stat().st_size > 1000:
193
+ return _result(doi, pdf_path, 'cache', text)
194
+ if not _under_cap():
195
+ raise DatabaseError('hourly cap of %d full-text downloads reached; the licence forbids '
196
+ 'bulk downloading, wait or narrow the list' % MAX_PER_HOUR)
197
+
198
+ oa = unpaywall(doi)
199
+ landing = resolve(doi)
200
+ with _Tab(EZ + 'https://doi.org/' + doi) as page:
201
+ page._run(asyncio.sleep(12))
202
+ info = json.loads(page.eval(PAGE_JS) or '{}')
203
+ if (not info.get('proxied')) and landing and not info.get('signin'):
204
+ page.goto(EZ + landing, settle=12)
205
+ info = json.loads(page.eval(PAGE_JS) or '{}')
206
+ if info.get('signin'):
207
+ raise DatabaseError('school sign-in expired; run db_status / sign in once in the '
208
+ 'school browser')
209
+ candidates = []
210
+ for rx, tmpl in PDF_PATTERNS:
211
+ if rx.search(info.get('host', '')):
212
+ candidates.append('https://%s%s' % (info['host'], tmpl.format(doi=doi)))
213
+ if info.get('meta_pdf'):
214
+ candidates.append(info['meta_pdf'])
215
+ candidates += [l for l in info.get('links', []) if doi.split('/')[-1].lower() in l.lower()
216
+ or 'pdfft' in l or '/pdf' in l]
217
+ seen = set()
218
+ for url in candidates:
219
+ if url in seen:
220
+ continue
221
+ seen.add(url)
222
+ raw, src = page._run(_capture_pdf(page, url))
223
+ if raw and raw[:5] == b'%PDF-':
224
+ pdf_path.write_bytes(raw)
225
+ _note_download(doi)
226
+ return _result(doi, pdf_path, (src or url).split('/')[2], text, oa=oa,
227
+ access=info.get('access'))
228
+ # no PDF: keep the HTML full text if the page has it
229
+ body = info.get('body_text') or ''
230
+ if len(body) > 4000 and info.get('proxied'):
231
+ html_path = OUT / (_slug(doi) + '.txt')
232
+ html_path.write_text(body)
233
+ _note_download(doi)
234
+ return {'doi': doi, 'format': 'html-text', 'path': str(html_path),
235
+ 'chars': len(body), 'via': info.get('host'), 'open_access': oa.get('is_oa'),
236
+ 'text': body if text else ''}
237
+ if oa.get('oa_pdf'):
238
+ reason = 'no subscribed copy found, but an open-access PDF exists: %s' % oa['oa_pdf']
239
+ elif not info.get('proxied'):
240
+ host = (landing or '').split('/')[2] if landing.count('/') >= 2 else info.get('host')
241
+ reason = ('%s is not rewritten by the school EZproxy, so Sophiahemmet has no '
242
+ 'subscription route to it; order it via the library (fjärrlån)' % host)
243
+ else:
244
+ reason = 'the publisher page was reached through EZproxy but offered no PDF (%s)' % (
245
+ info.get('access') or 'no access marker')
246
+ raise DatabaseError(reason)
247
+
248
+
249
+ def _result(doi, path, via, text, oa=None, access=''):
250
+ pages = 0
251
+ try:
252
+ out = subprocess.run(['pdfinfo', str(path)], capture_output=True, text=True, timeout=20).stdout
253
+ m = re.search(r'Pages:\s+(\d+)', out)
254
+ pages = int(m.group(1)) if m else 0
255
+ except Exception:
256
+ pass
257
+ body = _pdf_text(path) if text else ''
258
+ return {'doi': doi, 'format': 'pdf', 'path': str(path), 'bytes': path.stat().st_size,
259
+ 'pages': pages, 'via': via, 'access': access,
260
+ 'open_access': (oa or {}).get('is_oa'), 'chars': len(body), 'text': body}