unitoolkit 0.7.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- ladok_vakt/__init__.py +7 -0
- ladok_vakt/__main__.py +51 -0
- ladok_vakt/core.py +553 -0
- ladok_vakt/login.py +232 -0
- tools.json +623 -0
- unitoolkit/__init__.py +1 -0
- unitoolkit/__main__.py +63 -0
- unitoolkit/agent.py +270 -0
- unitoolkit/browser.py +147 -0
- unitoolkit/canvas.py +189 -0
- unitoolkit/chainproxy.py +113 -0
- unitoolkit/citation.py +165 -0
- unitoolkit/databases.py +497 -0
- unitoolkit/docs.py +394 -0
- unitoolkit/fulltext.py +260 -0
- unitoolkit/koha.py +729 -0
- unitoolkit/mcp.py +85 -0
- unitoolkit/programme.py +147 -0
- unitoolkit/search.py +507 -0
- unitoolkit/serve.py +153 -0
- unitoolkit/sharepoint.py +86 -0
- unitoolkit/surface.py +86 -0
- unitoolkit/timeedit.py +209 -0
- unitoolkit/tools.py +555 -0
- unitoolkit-0.7.0.dist-info/METADATA +177 -0
- unitoolkit-0.7.0.dist-info/RECORD +30 -0
- unitoolkit-0.7.0.dist-info/WHEEL +5 -0
- unitoolkit-0.7.0.dist-info/entry_points.txt +4 -0
- unitoolkit-0.7.0.dist-info/licenses/LICENSE +21 -0
- unitoolkit-0.7.0.dist-info/top_level.txt +2 -0
unitoolkit/docs.py
ADDED
|
@@ -0,0 +1,394 @@
|
|
|
1
|
+
"""Document text: download a course file by id, extract text once, cache the text.
|
|
2
|
+
|
|
3
|
+
Why this exists: a Canvas file is a name until someone reads it. `display_name` is all the
|
|
4
|
+
API offers, and the words a student searches for are inside the PDF. So each file is fetched
|
|
5
|
+
once, parsed, and the *text* is kept while the bytes are discarded, keyed by
|
|
6
|
+
`file id + updated_at` so a reindex costs nothing and an updated file is re-read.
|
|
7
|
+
|
|
8
|
+
Extraction, in order of preference:
|
|
9
|
+
pdf pdftotext (poppler; no dependency of ours)
|
|
10
|
+
docx zipfile + word/document.xml
|
|
11
|
+
pptx zipfile + ppt/slides/slideN.xml
|
|
12
|
+
xlsx zipfile + xl/sharedStrings.xml
|
|
13
|
+
html tag strip, entities decoded
|
|
14
|
+
image OCR (opt in), tesseract with the Swedish model
|
|
15
|
+
|
|
16
|
+
OCR is deliberately not part of every index run: it is slow (1.5-3 s per page at 300 dpi)
|
|
17
|
+
and most files do not need it. It runs when a document has no usable text layer, is capped,
|
|
18
|
+
and its result is cached like any other text.
|
|
19
|
+
|
|
20
|
+
python -m unitoolkit docs course=2416 # text layer only, fast
|
|
21
|
+
python -m unitoolkit docs course=2416 ocr=true # plus OCR for image-only files
|
|
22
|
+
"""
|
|
23
|
+
import hashlib
|
|
24
|
+
import html
|
|
25
|
+
import io
|
|
26
|
+
import json
|
|
27
|
+
import pathlib
|
|
28
|
+
import re
|
|
29
|
+
import shutil
|
|
30
|
+
import subprocess
|
|
31
|
+
import tempfile
|
|
32
|
+
import zipfile
|
|
33
|
+
|
|
34
|
+
from . import canvas
|
|
35
|
+
|
|
36
|
+
CACHE = pathlib.Path.home() / ".cache/unitoolkit/docs"
|
|
37
|
+
|
|
38
|
+
TEXT_EXTS = {"txt", "md", "csv", "json", "html", "htm", "xml", "py", "js", "css"}
|
|
39
|
+
IMAGE_EXTS = {"png", "jpg", "jpeg", "gif", "bmp", "tif", "tiff"}
|
|
40
|
+
OFFICE_EXTS = {"docx", "pptx", "xlsx"}
|
|
41
|
+
|
|
42
|
+
# a document whose text is this thin is treated as image-only
|
|
43
|
+
MIN_USEFUL_CHARS = 100
|
|
44
|
+
MIN_CHARS_PER_PAGE = 40
|
|
45
|
+
|
|
46
|
+
|
|
47
|
+
def _have(binary):
|
|
48
|
+
return shutil.which(binary) is not None
|
|
49
|
+
|
|
50
|
+
|
|
51
|
+
def _run(cmd, **kw):
|
|
52
|
+
return subprocess.run(cmd, capture_output=True, text=True, **kw)
|
|
53
|
+
|
|
54
|
+
|
|
55
|
+
# ---------------------------------------------------------------- extractors
|
|
56
|
+
def docx_text(data):
|
|
57
|
+
with zipfile.ZipFile(io.BytesIO(data)) as z:
|
|
58
|
+
names = z.namelist()
|
|
59
|
+
parts = [n for n in names if n == "word/document.xml" or
|
|
60
|
+
re.match(r"word/(header|footer)\d*\.xml$", n)]
|
|
61
|
+
out = []
|
|
62
|
+
for n in parts:
|
|
63
|
+
xml = z.read(n).decode("utf-8", "ignore")
|
|
64
|
+
xml = re.sub(r"</w:p>|</w:tr>", "\n", xml)
|
|
65
|
+
xml = re.sub(r"</w:tc>", " ", xml)
|
|
66
|
+
out.append(re.sub(r"[ \t]+", " ", re.sub(r"<[^>]+>", "", xml)))
|
|
67
|
+
return "\n".join(out)
|
|
68
|
+
|
|
69
|
+
|
|
70
|
+
def pptx_text(data):
|
|
71
|
+
with zipfile.ZipFile(io.BytesIO(data)) as z:
|
|
72
|
+
slides = []
|
|
73
|
+
for n in z.namelist():
|
|
74
|
+
m = re.match(r"ppt/slides/slide(\d+)\.xml$", n)
|
|
75
|
+
if m:
|
|
76
|
+
slides.append((int(m.group(1)), n))
|
|
77
|
+
out = []
|
|
78
|
+
for num, n in sorted(slides):
|
|
79
|
+
xml = z.read(n).decode("utf-8", "ignore")
|
|
80
|
+
xml = re.sub(r"</a:p>", "\n", xml)
|
|
81
|
+
txt = re.sub(r"[ \t]+", " ", re.sub(r"<[^>]+>", "", xml)).strip()
|
|
82
|
+
if txt:
|
|
83
|
+
out.append(f"Bild {num}: {txt}")
|
|
84
|
+
return "\n".join(out)
|
|
85
|
+
|
|
86
|
+
|
|
87
|
+
def xlsx_text(data):
|
|
88
|
+
with zipfile.ZipFile(io.BytesIO(data)) as z:
|
|
89
|
+
if "xl/sharedStrings.xml" not in z.namelist():
|
|
90
|
+
return ""
|
|
91
|
+
xml = z.read("xl/sharedStrings.xml").decode("utf-8", "ignore")
|
|
92
|
+
cells = re.findall(r"<t[^>]*>(.*?)</t>", xml, re.S)
|
|
93
|
+
return "\n".join(html.unescape(c) for c in cells)
|
|
94
|
+
|
|
95
|
+
|
|
96
|
+
def html_text(data):
|
|
97
|
+
s = data.decode("utf-8", "ignore")
|
|
98
|
+
s = re.sub(r"<script[\s\S]*?</script>|<style[\s\S]*?</style>", " ", s, flags=re.I)
|
|
99
|
+
s = re.sub(r"</(p|div|li|tr|h[1-6])>|<br\s*/?>", "\n", s, flags=re.I)
|
|
100
|
+
s = re.sub(r"<[^>]+>", " ", s)
|
|
101
|
+
s = html.unescape(s).replace("\u00a0", " ")
|
|
102
|
+
s = re.sub(r"[ \t]+", " ", s)
|
|
103
|
+
return re.sub(r" *\n *", "\n", re.sub(r"\n{2,}", "\n", s)).strip()
|
|
104
|
+
|
|
105
|
+
|
|
106
|
+
def pdf_text(data):
|
|
107
|
+
"""Text layer via pdftotext. Page breaks survive as \\f, which gives us page numbers."""
|
|
108
|
+
with tempfile.TemporaryDirectory() as td:
|
|
109
|
+
src = pathlib.Path(td) / "in.pdf"
|
|
110
|
+
src.write_bytes(data)
|
|
111
|
+
r = _run(["pdftotext", "-enc", "UTF-8", "-q", str(src), "-"])
|
|
112
|
+
return r.stdout or ""
|
|
113
|
+
|
|
114
|
+
|
|
115
|
+
def pdf_pages(data):
|
|
116
|
+
"""Real page count from the PDF itself, so a capped OCR run can say it was capped."""
|
|
117
|
+
if not _have("pdfinfo"):
|
|
118
|
+
return None
|
|
119
|
+
with tempfile.TemporaryDirectory() as td:
|
|
120
|
+
src = pathlib.Path(td) / "in.pdf"
|
|
121
|
+
src.write_bytes(data)
|
|
122
|
+
r = _run(["pdfinfo", str(src)])
|
|
123
|
+
m = re.search(r"^Pages:\s+(\d+)", r.stdout or "", re.M)
|
|
124
|
+
return int(m.group(1)) if m else None
|
|
125
|
+
|
|
126
|
+
|
|
127
|
+
def ocr_pdf(data, max_pages=60, dpi=300, lang="swe"):
|
|
128
|
+
"""Render pages to PNG and OCR them. Returns (text, pages_ocred)."""
|
|
129
|
+
if not (_have("pdftoppm") and _have("tesseract")):
|
|
130
|
+
return "", 0
|
|
131
|
+
out = []
|
|
132
|
+
with tempfile.TemporaryDirectory() as td:
|
|
133
|
+
src = pathlib.Path(td) / "in.pdf"
|
|
134
|
+
src.write_bytes(data)
|
|
135
|
+
pre = str(pathlib.Path(td) / "pg")
|
|
136
|
+
_run(["pdftoppm", "-r", str(dpi), "-gray", "-png", "-f", "1", "-l", str(max_pages),
|
|
137
|
+
str(src), pre])
|
|
138
|
+
pages = sorted(pathlib.Path(td).glob("pg*.png"))
|
|
139
|
+
for i, p in enumerate(pages, 1):
|
|
140
|
+
r = _run(["tesseract", str(p), "-", "-l", lang, "--psm", "3"])
|
|
141
|
+
out.append(r.stdout.strip())
|
|
142
|
+
return "\f".join(out), len(out)
|
|
143
|
+
|
|
144
|
+
|
|
145
|
+
def ocr_image(data, lang="swe"):
|
|
146
|
+
if not _have("tesseract"):
|
|
147
|
+
return ""
|
|
148
|
+
with tempfile.TemporaryDirectory() as td:
|
|
149
|
+
src = pathlib.Path(td) / "in.png"
|
|
150
|
+
src.write_bytes(data)
|
|
151
|
+
return _run(["tesseract", str(src), "-", "-l", lang, "--psm", "3"]).stdout.strip()
|
|
152
|
+
|
|
153
|
+
|
|
154
|
+
def extract(name, data, allow_ocr=False, ocr_max_pages=60):
|
|
155
|
+
"""Extract text from bytes. Returns dict: text, kind, ocr_pages, needs_ocr."""
|
|
156
|
+
ext = (name.rsplit(".", 1)[-1] or "").lower()
|
|
157
|
+
info = {"kind": ext, "ocr_pages": 0, "needs_ocr": False, "text": ""}
|
|
158
|
+
|
|
159
|
+
if ext == "pdf":
|
|
160
|
+
text = pdf_text(data)
|
|
161
|
+
pages = max(1, text.count("\f") + 1)
|
|
162
|
+
thin = len(text.strip()) < MIN_USEFUL_CHARS or len(text.strip()) / pages < MIN_CHARS_PER_PAGE
|
|
163
|
+
info["text"] = text
|
|
164
|
+
if thin:
|
|
165
|
+
info["needs_ocr"] = True
|
|
166
|
+
info["pages_total"] = pdf_pages(data)
|
|
167
|
+
if allow_ocr:
|
|
168
|
+
ocr, n = ocr_pdf(data, max_pages=ocr_max_pages)
|
|
169
|
+
info["ocr_pages"] = n
|
|
170
|
+
# a capped run must be visible, or a half-read book looks complete
|
|
171
|
+
info["ocr_truncated"] = bool(info["pages_total"] and n < info["pages_total"])
|
|
172
|
+
if len(ocr.strip()) > len(text.strip()):
|
|
173
|
+
info["text"] = ocr
|
|
174
|
+
return info
|
|
175
|
+
if ext == "docx":
|
|
176
|
+
info["text"] = docx_text(data)
|
|
177
|
+
elif ext == "pptx":
|
|
178
|
+
info["text"] = pptx_text(data)
|
|
179
|
+
elif ext == "xlsx":
|
|
180
|
+
info["text"] = xlsx_text(data)
|
|
181
|
+
elif ext in ("html", "htm", "aspx"):
|
|
182
|
+
info["text"] = html_text(data)
|
|
183
|
+
elif ext in TEXT_EXTS:
|
|
184
|
+
info["text"] = data.decode("utf-8", "ignore")
|
|
185
|
+
elif ext in IMAGE_EXTS:
|
|
186
|
+
info["needs_ocr"] = True
|
|
187
|
+
if allow_ocr:
|
|
188
|
+
info["text"] = ocr_image(data)
|
|
189
|
+
info["ocr_pages"] = 1 if info["text"] else 0
|
|
190
|
+
else:
|
|
191
|
+
info["kind"] = "binary"
|
|
192
|
+
return info
|
|
193
|
+
|
|
194
|
+
|
|
195
|
+
# ---------------------------------------------------------------- cache
|
|
196
|
+
def _cache_dir(course_id):
|
|
197
|
+
d = CACHE / str(course_id)
|
|
198
|
+
d.mkdir(parents=True, exist_ok=True)
|
|
199
|
+
return d
|
|
200
|
+
|
|
201
|
+
|
|
202
|
+
def cache_key(file_meta):
|
|
203
|
+
return hashlib.sha1(
|
|
204
|
+
f"{file_meta.get('id')}:{file_meta.get('updated_at') or file_meta.get('created_at')}"
|
|
205
|
+
.encode()).hexdigest()[:16]
|
|
206
|
+
|
|
207
|
+
|
|
208
|
+
def read_cached(course_id, file_meta):
|
|
209
|
+
p = _cache_dir(course_id) / f"{cache_key(file_meta)}.json"
|
|
210
|
+
if p.exists():
|
|
211
|
+
try:
|
|
212
|
+
return json.loads(p.read_text())
|
|
213
|
+
except Exception: # noqa: BLE001 - a broken cache entry is just a miss
|
|
214
|
+
return None
|
|
215
|
+
return None
|
|
216
|
+
|
|
217
|
+
|
|
218
|
+
def write_cached(course_id, file_meta, payload):
|
|
219
|
+
p = _cache_dir(course_id) / f"{cache_key(file_meta)}.json"
|
|
220
|
+
p.write_text(json.dumps(payload, ensure_ascii=False))
|
|
221
|
+
return p
|
|
222
|
+
|
|
223
|
+
|
|
224
|
+
# ---------------------------------------------------------------- build
|
|
225
|
+
def build(course_id, ocr=False, force=False, max_files=None, ocr_max_pages=60,
|
|
226
|
+
max_bytes=40 * 1024 * 1024, only_contains=None, progress=None):
|
|
227
|
+
"""Extract text for every file in a course. Cached by file id + updated_at."""
|
|
228
|
+
files = canvas.try_get(f"courses/{course_id}/files", per_page=100) or []
|
|
229
|
+
files = [f for f in files if isinstance(f, dict)]
|
|
230
|
+
files.sort(key=lambda f: (f.get("display_name") or "").lower())
|
|
231
|
+
if only_contains:
|
|
232
|
+
needle = only_contains.lower()
|
|
233
|
+
files = [f for f in files
|
|
234
|
+
if needle in (f.get("display_name") or f.get("filename") or "").lower()]
|
|
235
|
+
if max_files:
|
|
236
|
+
files = files[:max_files]
|
|
237
|
+
|
|
238
|
+
stats = {"course": course_id, "files": len(files), "cached": 0, "extracted": 0,
|
|
239
|
+
"failed": 0, "skipped_big": 0, "needs_ocr": 0, "ocr_pages": 0,
|
|
240
|
+
"with_text": 0, "no_text": 0, "chars": 0, "by_kind": {}}
|
|
241
|
+
for i, f in enumerate(files, 1):
|
|
242
|
+
name = f.get("display_name") or f.get("filename") or str(f.get("id"))
|
|
243
|
+
if progress:
|
|
244
|
+
progress(f"{i}/{len(files)} {name[:44]}")
|
|
245
|
+
if (f.get("size") or 0) > max_bytes:
|
|
246
|
+
stats["skipped_big"] += 1
|
|
247
|
+
continue
|
|
248
|
+
hit = None if force else read_cached(course_id, f)
|
|
249
|
+
if hit and ocr and hit.get("needs_ocr") and len((hit.get("text") or "").strip()) < MIN_USEFUL_CHARS:
|
|
250
|
+
hit = None # cached as image-only, and OCR is now requested: read it again
|
|
251
|
+
stats["re_ocred"] = stats.get("re_ocred", 0) + 1
|
|
252
|
+
if hit:
|
|
253
|
+
stats["cached"] += 1
|
|
254
|
+
payload = hit
|
|
255
|
+
else:
|
|
256
|
+
url = f.get("url") or f"courses/{course_id}/files/{f.get('id')}/download"
|
|
257
|
+
try:
|
|
258
|
+
data = canvas.download_bytes(url)
|
|
259
|
+
except Exception: # noqa: BLE001 - a failed file must not stop the run
|
|
260
|
+
stats["failed"] += 1
|
|
261
|
+
continue
|
|
262
|
+
info = extract(name, data, allow_ocr=ocr, ocr_max_pages=ocr_max_pages)
|
|
263
|
+
stats["extracted"] += 1
|
|
264
|
+
stats["ocr_pages"] += info["ocr_pages"]
|
|
265
|
+
payload = {"id": f.get("id"), "name": name, "ext": info["kind"],
|
|
266
|
+
"updated_at": f.get("updated_at") or f.get("created_at") or "",
|
|
267
|
+
"href": f"/courses/{course_id}/files/{f.get('id')}",
|
|
268
|
+
"url": url, "bytes": len(data), "pages": info["text"].count("\f") + 1,
|
|
269
|
+
"pages_total": info.get("pages_total"),
|
|
270
|
+
"ocr_pages": info["ocr_pages"], "needs_ocr": info["needs_ocr"],
|
|
271
|
+
"ocr_truncated": bool(info.get("ocr_truncated")),
|
|
272
|
+
"text": info["text"]}
|
|
273
|
+
if payload["ocr_truncated"]:
|
|
274
|
+
stats["ocr_truncated"] = stats.get("ocr_truncated", 0) + 1
|
|
275
|
+
write_cached(course_id, f, payload)
|
|
276
|
+
kind = payload.get("ext") or "?"
|
|
277
|
+
stats["by_kind"][kind] = stats["by_kind"].get(kind, 0) + 1
|
|
278
|
+
n = len((payload.get("text") or "").strip())
|
|
279
|
+
stats["chars"] += n
|
|
280
|
+
if n >= MIN_USEFUL_CHARS:
|
|
281
|
+
stats["with_text"] += 1
|
|
282
|
+
else:
|
|
283
|
+
stats["no_text"] += 1
|
|
284
|
+
if payload.get("needs_ocr"):
|
|
285
|
+
stats["needs_ocr"] += 1
|
|
286
|
+
return stats
|
|
287
|
+
|
|
288
|
+
|
|
289
|
+
HREF_RE = re.compile(r'href="([^"]+)"', re.I)
|
|
290
|
+
|
|
291
|
+
|
|
292
|
+
def page_links(course_id, mirror_root=None):
|
|
293
|
+
"""Every link inside the course's pages, deduped, classified.
|
|
294
|
+
|
|
295
|
+
Pages line up with the mirror; without a mirror nothing is read, which keeps this from
|
|
296
|
+
turning into a page crawl of its own.
|
|
297
|
+
"""
|
|
298
|
+
root = pathlib.Path(mirror_root or (pathlib.Path.home() / "courses/canvas" / str(course_id)))
|
|
299
|
+
pages = root / "pages"
|
|
300
|
+
if not pages.exists():
|
|
301
|
+
return []
|
|
302
|
+
seen, out = set(), []
|
|
303
|
+
for p in sorted(pages.glob("*.html")):
|
|
304
|
+
for url in HREF_RE.findall(p.read_text(errors="ignore")):
|
|
305
|
+
url = html.unescape(url).strip()
|
|
306
|
+
if not url or url.startswith(("#", "mailto:", "javascript:")) or url in seen:
|
|
307
|
+
continue
|
|
308
|
+
seen.add(url)
|
|
309
|
+
kind = "other"
|
|
310
|
+
if re.search(r"/files/\d+", url):
|
|
311
|
+
kind = "canvas_file"
|
|
312
|
+
elif re.search(r"/pages/", url):
|
|
313
|
+
kind = "canvas_page"
|
|
314
|
+
elif re.search(r"sharepoint\.com/:[a-z]:/", url, re.I):
|
|
315
|
+
kind = "share"
|
|
316
|
+
elif re.search(r"(kaltura|youtube|vimeo)", url, re.I):
|
|
317
|
+
kind = "video"
|
|
318
|
+
out.append({"url": url, "kind": kind, "from": p.name})
|
|
319
|
+
return out
|
|
320
|
+
|
|
321
|
+
|
|
322
|
+
def resolve_links(course_id, limit=None, allow_ocr=False, progress=None):
|
|
323
|
+
"""Fetch linked documents that are not in the file tree (SharePoint share links) and
|
|
324
|
+
cache their text alongside the course files. Bytes are read once and dropped."""
|
|
325
|
+
from . import sharepoint
|
|
326
|
+
|
|
327
|
+
links = [l for l in page_links(course_id) if l["kind"] == "share"]
|
|
328
|
+
if limit:
|
|
329
|
+
links = links[:limit]
|
|
330
|
+
stats = {"course": course_id, "share_links": len(links), "stored": 0, "cached": 0,
|
|
331
|
+
"failed": 0, "with_text": 0, "no_text": 0, "chars": 0, "by_kind": {}}
|
|
332
|
+
for i, link in enumerate(links, 1):
|
|
333
|
+
url = link["url"]
|
|
334
|
+
if progress:
|
|
335
|
+
progress(f"{i}/{len(links)} {url[-28:]}")
|
|
336
|
+
meta = {"id": f"share:{url}", "updated_at": "fixed"} # a share link has no version
|
|
337
|
+
hit = read_cached(course_id, meta)
|
|
338
|
+
if hit:
|
|
339
|
+
stats["cached"] += 1
|
|
340
|
+
payload = hit
|
|
341
|
+
else:
|
|
342
|
+
body, kind, name = sharepoint.fetch_share(url)
|
|
343
|
+
if not body:
|
|
344
|
+
stats["failed"] += 1
|
|
345
|
+
continue
|
|
346
|
+
name = name or f"share-{len(body)}.{kind}"
|
|
347
|
+
info = extract(name, body, allow_ocr=allow_ocr)
|
|
348
|
+
stats["stored"] += 1
|
|
349
|
+
payload = {"id": meta["id"], "name": name, "ext": info["kind"],
|
|
350
|
+
"updated_at": "", "href": url, "url": url,
|
|
351
|
+
"bytes": len(body), "pages": info["text"].count("\f") + 1,
|
|
352
|
+
"pages_total": info.get("pages_total"),
|
|
353
|
+
"ocr_pages": info["ocr_pages"], "needs_ocr": info["needs_ocr"],
|
|
354
|
+
"ocr_truncated": bool(info.get("ocr_truncated")),
|
|
355
|
+
"text": info["text"]}
|
|
356
|
+
write_cached(course_id, meta, payload)
|
|
357
|
+
stats["by_kind"][payload.get("ext") or "?"] = stats["by_kind"].get(payload.get("ext") or "?", 0) + 1
|
|
358
|
+
n = len((payload.get("text") or "").strip())
|
|
359
|
+
stats["chars"] += n
|
|
360
|
+
stats["with_text" if n >= MIN_USEFUL_CHARS else "no_text"] += 1
|
|
361
|
+
return stats
|
|
362
|
+
|
|
363
|
+
|
|
364
|
+
def load(course_id):
|
|
365
|
+
"""Cached documents for a course, shaped like search docs."""
|
|
366
|
+
out = []
|
|
367
|
+
for p in sorted(_cache_dir(course_id).glob("*.json")):
|
|
368
|
+
try:
|
|
369
|
+
d = json.loads(p.read_text())
|
|
370
|
+
except Exception: # noqa: BLE001
|
|
371
|
+
continue
|
|
372
|
+
text = d.get("text") or ""
|
|
373
|
+
if not text.strip():
|
|
374
|
+
continue
|
|
375
|
+
out.append({"type": "fildok", "title": d.get("name") or "(fil)",
|
|
376
|
+
"body": text, "href": d.get("href") or "#",
|
|
377
|
+
"ext": d.get("ext") or "", "date": (d.get("updated_at") or "")[:10],
|
|
378
|
+
"pages": d.get("pages"), "ocr_pages": d.get("ocr_pages") or 0,
|
|
379
|
+
"bytes": d.get("bytes")})
|
|
380
|
+
return out
|
|
381
|
+
|
|
382
|
+
|
|
383
|
+
def page_of(text, at):
|
|
384
|
+
"""1-based page number for a character offset, using pdftotext's form feeds."""
|
|
385
|
+
if not text:
|
|
386
|
+
return None
|
|
387
|
+
return text.count("\f", 0, max(0, at)) + 1
|
|
388
|
+
|
|
389
|
+
|
|
390
|
+
def clear(course_id=None):
|
|
391
|
+
target = CACHE / str(course_id) if course_id else CACHE
|
|
392
|
+
if target.exists():
|
|
393
|
+
shutil.rmtree(target)
|
|
394
|
+
return {"cleared": str(target)}
|
unitoolkit/fulltext.py
ADDED
|
@@ -0,0 +1,260 @@
|
|
|
1
|
+
"""Full text of an article the school subscribes to, by DOI.
|
|
2
|
+
|
|
3
|
+
The DOI is resolved through the school's EZproxy (`shh.idm.oclc.org`), so the publisher
|
|
4
|
+
sees Sophiahemmet's subscription. The PDF is captured at the network layer (CDP Fetch
|
|
5
|
+
interception) rather than by a page-side fetch(): Wiley, Elsevier and others answer a
|
|
6
|
+
script fetch with an HTML viewer or a bot check, but hand the real PDF to a top-level
|
|
7
|
+
navigation.
|
|
8
|
+
|
|
9
|
+
Tested 2026-11-17: Wiley (two paywalled articles, Unpaywall is_oa=false) and Elsevier
|
|
10
|
+
(ScienceDirect, via pdf.sciencedirectassets.com) return the PDF. De Gruyter is not
|
|
11
|
+
rewritten by the school's EZproxy, so there is no subscription route; the function says so
|
|
12
|
+
instead of returning a paywall page.
|
|
13
|
+
|
|
14
|
+
Licensed content: the school's licences allow a student to read and download for their own
|
|
15
|
+
study. They do not allow systematic or bulk downloading, and publishers react to it by
|
|
16
|
+
blocking the institution's whole EZproxy. Hence the hourly cap below, which is a rule, not
|
|
17
|
+
a tuning knob.
|
|
18
|
+
"""
|
|
19
|
+
|
|
20
|
+
import asyncio
|
|
21
|
+
import base64
|
|
22
|
+
import json
|
|
23
|
+
import os
|
|
24
|
+
import re
|
|
25
|
+
import subprocess
|
|
26
|
+
import time
|
|
27
|
+
import urllib.error
|
|
28
|
+
import urllib.parse
|
|
29
|
+
import urllib.request
|
|
30
|
+
from pathlib import Path
|
|
31
|
+
|
|
32
|
+
from .databases import DatabaseError, _Tab, EZ
|
|
33
|
+
|
|
34
|
+
OUT = Path(os.environ.get('UNITOOLKIT_FULLTEXT_DIR',
|
|
35
|
+
str(Path.home() / '.local/share/unitoolkit/fulltext')))
|
|
36
|
+
LEDGER = OUT / '.downloads.json'
|
|
37
|
+
MAX_PER_HOUR = 20 # licence hygiene: never let a loop turn into a bulk download
|
|
38
|
+
|
|
39
|
+
# Publisher hosts that put the PDF behind a URL we can build from the DOI.
|
|
40
|
+
PDF_PATTERNS = [
|
|
41
|
+
(re.compile(r'onlinelibrary-wiley-com|wiley-com\.shh'), '/doi/pdfdirect/{doi}?download=true'),
|
|
42
|
+
(re.compile(r'journals-sagepub-com'), '/doi/pdf/{doi}?download=true'),
|
|
43
|
+
(re.compile(r'tandfonline-com'), '/doi/pdf/{doi}?download=true'),
|
|
44
|
+
(re.compile(r'cochranelibrary-com'), '/cdsr/doi/{doi}/pdf/full'),
|
|
45
|
+
]
|
|
46
|
+
|
|
47
|
+
PAGE_JS = r"""(() => {
|
|
48
|
+
const t = document.body ? document.body.innerText : '';
|
|
49
|
+
const meta = (document.querySelector('meta[name=citation_pdf_url]') || {}).content || '';
|
|
50
|
+
const links = [...new Set([...document.querySelectorAll('a')]
|
|
51
|
+
.map(a => a.href).filter(h => h && /pdf/i.test(h) && !/supplement|suppl|figure/i.test(h)))];
|
|
52
|
+
const body = document.querySelector('.article__body, section.article-section__full, #body, '
|
|
53
|
+
+ 'article .Body, div.article-body, #article-body, main article, .cdsr-article, '
|
|
54
|
+
+ 'section.abstract ~ section, #content article') || (t.length > 20000 ? document.body : null);
|
|
55
|
+
return JSON.stringify({
|
|
56
|
+
url: location.href, host: location.host, title: document.title,
|
|
57
|
+
proxied: /\.shh\.idm\.oclc\.org$/.test(location.host),
|
|
58
|
+
signin: /login\.microsoftonline\.com/.test(location.href),
|
|
59
|
+
meta_pdf: meta, links: links.slice(0, 10),
|
|
60
|
+
access: (t.match(/(Open Access|Free Access|Full Access|Access provided by[^\n]{0,60}|institutional access|Get access|Purchase PDF|Buy article)/i) || [''])[0],
|
|
61
|
+
body_text: body ? body.innerText : '',
|
|
62
|
+
});
|
|
63
|
+
})()"""
|
|
64
|
+
|
|
65
|
+
|
|
66
|
+
def _slug(doi):
|
|
67
|
+
return re.sub(r'[^\w.-]+', '_', doi.strip())
|
|
68
|
+
|
|
69
|
+
|
|
70
|
+
def _ledger():
|
|
71
|
+
try:
|
|
72
|
+
return json.loads(LEDGER.read_text())
|
|
73
|
+
except Exception:
|
|
74
|
+
return []
|
|
75
|
+
|
|
76
|
+
|
|
77
|
+
def _note_download(doi):
|
|
78
|
+
now = time.time()
|
|
79
|
+
rows = [r for r in _ledger() if now - r['t'] < 3600]
|
|
80
|
+
rows.append({'t': now, 'doi': doi})
|
|
81
|
+
OUT.mkdir(parents=True, exist_ok=True)
|
|
82
|
+
LEDGER.write_text(json.dumps(rows))
|
|
83
|
+
|
|
84
|
+
|
|
85
|
+
def _under_cap():
|
|
86
|
+
now = time.time()
|
|
87
|
+
return sum(1 for r in _ledger() if now - r['t'] < 3600) < MAX_PER_HOUR
|
|
88
|
+
|
|
89
|
+
|
|
90
|
+
def unpaywall(doi):
|
|
91
|
+
"""Open-access status, so a free copy is preferred and the paid route is not used for it."""
|
|
92
|
+
try:
|
|
93
|
+
j = json.load(urllib.request.urlopen(
|
|
94
|
+
'https://api.unpaywall.org/v2/%s?email=unitoolkit@users.noreply.github.com'
|
|
95
|
+
% urllib.parse.quote(doi), timeout=15))
|
|
96
|
+
except Exception:
|
|
97
|
+
return {'is_oa': None}
|
|
98
|
+
loc = j.get('best_oa_location') or {}
|
|
99
|
+
return {'is_oa': j.get('is_oa'), 'oa_pdf': loc.get('url_for_pdf') or '',
|
|
100
|
+
'publisher': j.get('publisher') or '', 'title': j.get('title') or ''}
|
|
101
|
+
|
|
102
|
+
|
|
103
|
+
async def _capture_pdf(page, url, timeout=45):
|
|
104
|
+
"""Navigate and return the first PDF response body seen on the wire."""
|
|
105
|
+
await page.call('Fetch.enable', {'patterns': [{'urlPattern': '*', 'requestStage': 'Response'}]},
|
|
106
|
+
page.sid)
|
|
107
|
+
page.n += 1
|
|
108
|
+
await page.ws.send(json.dumps({'id': page.n, 'method': 'Page.navigate',
|
|
109
|
+
'params': {'url': url}, 'sessionId': page.sid}))
|
|
110
|
+
loop = asyncio.get_event_loop()
|
|
111
|
+
end = loop.time() + timeout
|
|
112
|
+
try:
|
|
113
|
+
while loop.time() < end:
|
|
114
|
+
try:
|
|
115
|
+
msg = json.loads(await asyncio.wait_for(page.ws.recv(), timeout=end - loop.time()))
|
|
116
|
+
except asyncio.TimeoutError:
|
|
117
|
+
break
|
|
118
|
+
if msg.get('method') != 'Fetch.requestPaused':
|
|
119
|
+
continue
|
|
120
|
+
ev = msg['params']
|
|
121
|
+
rid = ev['requestId']
|
|
122
|
+
ctype = next((h['value'] for h in ev.get('responseHeaders') or []
|
|
123
|
+
if h['name'].lower() == 'content-type'), '')
|
|
124
|
+
status = ev.get('responseStatusCode') or 0
|
|
125
|
+
if 'pdf' in ctype.lower() and status < 300:
|
|
126
|
+
page.n += 1
|
|
127
|
+
k = page.n
|
|
128
|
+
await page.ws.send(json.dumps({'id': k, 'method': 'Fetch.getResponseBody',
|
|
129
|
+
'params': {'requestId': rid}, 'sessionId': page.sid}))
|
|
130
|
+
while True:
|
|
131
|
+
r = json.loads(await page.ws.recv())
|
|
132
|
+
if r.get('id') == k:
|
|
133
|
+
break
|
|
134
|
+
res = r.get('result') or {}
|
|
135
|
+
body = res.get('body', '')
|
|
136
|
+
raw = base64.b64decode(body) if res.get('base64Encoded') else body.encode('latin1')
|
|
137
|
+
page.n += 1
|
|
138
|
+
await page.ws.send(json.dumps({'id': page.n, 'method': 'Fetch.continueRequest',
|
|
139
|
+
'params': {'requestId': rid}, 'sessionId': page.sid}))
|
|
140
|
+
return raw, ev['request']['url']
|
|
141
|
+
page.n += 1
|
|
142
|
+
await page.ws.send(json.dumps({'id': page.n, 'method': 'Fetch.continueRequest',
|
|
143
|
+
'params': {'requestId': rid}, 'sessionId': page.sid}))
|
|
144
|
+
finally:
|
|
145
|
+
try:
|
|
146
|
+
await page.call('Fetch.disable', {}, page.sid)
|
|
147
|
+
except Exception:
|
|
148
|
+
pass
|
|
149
|
+
return None, None
|
|
150
|
+
|
|
151
|
+
|
|
152
|
+
def resolve(doi):
|
|
153
|
+
"""Publisher landing URL for a DOI, resolved outside the proxy.
|
|
154
|
+
|
|
155
|
+
Some DOI redirects break when they run through EZproxy (Cochrane lands on a Chrome
|
|
156
|
+
error page), while the publisher URL itself proxies fine. Resolving first and proxying
|
|
157
|
+
the result works for both kinds.
|
|
158
|
+
"""
|
|
159
|
+
if doi.lower().startswith('10.1002/14651858.'): # Cochrane reviews: Wiley's resolver
|
|
160
|
+
return 'https://www.cochranelibrary.com/cdsr/doi/%s/full' % doi # stops at doi.wiley.com
|
|
161
|
+
req = urllib.request.Request('https://doi.org/' + doi,
|
|
162
|
+
headers={'User-Agent': 'Mozilla/5.0 unitoolkit'})
|
|
163
|
+
try:
|
|
164
|
+
with urllib.request.urlopen(req, timeout=20) as r:
|
|
165
|
+
return r.geturl()
|
|
166
|
+
except urllib.error.HTTPError as e: # many publishers refuse HEAD but still redirected
|
|
167
|
+
return e.geturl() or ''
|
|
168
|
+
except Exception:
|
|
169
|
+
return ''
|
|
170
|
+
|
|
171
|
+
|
|
172
|
+
def _pdf_text(path):
|
|
173
|
+
try:
|
|
174
|
+
return subprocess.run(['pdftotext', '-layout', str(path), '-'], capture_output=True,
|
|
175
|
+
text=True, timeout=60).stdout
|
|
176
|
+
except Exception:
|
|
177
|
+
return ''
|
|
178
|
+
|
|
179
|
+
|
|
180
|
+
def fetch(doi, text=True):
|
|
181
|
+
"""Download the full text of one article through the school's subscription.
|
|
182
|
+
|
|
183
|
+
Returns the saved PDF path (or the HTML full text when the publisher only offers that),
|
|
184
|
+
page count, and optionally the extracted text. Raises with a reason when the school has
|
|
185
|
+
no route to it.
|
|
186
|
+
"""
|
|
187
|
+
doi = re.sub(r'^(https?://(dx\.)?doi\.org/|doi:\s*)', '', (doi or '').strip(), flags=re.I)
|
|
188
|
+
if not doi.startswith('10.'):
|
|
189
|
+
raise DatabaseError('not a DOI: %r' % doi)
|
|
190
|
+
OUT.mkdir(parents=True, exist_ok=True)
|
|
191
|
+
pdf_path = OUT / (_slug(doi) + '.pdf')
|
|
192
|
+
if pdf_path.exists() and pdf_path.stat().st_size > 1000:
|
|
193
|
+
return _result(doi, pdf_path, 'cache', text)
|
|
194
|
+
if not _under_cap():
|
|
195
|
+
raise DatabaseError('hourly cap of %d full-text downloads reached; the licence forbids '
|
|
196
|
+
'bulk downloading, wait or narrow the list' % MAX_PER_HOUR)
|
|
197
|
+
|
|
198
|
+
oa = unpaywall(doi)
|
|
199
|
+
landing = resolve(doi)
|
|
200
|
+
with _Tab(EZ + 'https://doi.org/' + doi) as page:
|
|
201
|
+
page._run(asyncio.sleep(12))
|
|
202
|
+
info = json.loads(page.eval(PAGE_JS) or '{}')
|
|
203
|
+
if (not info.get('proxied')) and landing and not info.get('signin'):
|
|
204
|
+
page.goto(EZ + landing, settle=12)
|
|
205
|
+
info = json.loads(page.eval(PAGE_JS) or '{}')
|
|
206
|
+
if info.get('signin'):
|
|
207
|
+
raise DatabaseError('school sign-in expired; run db_status / sign in once in the '
|
|
208
|
+
'school browser')
|
|
209
|
+
candidates = []
|
|
210
|
+
for rx, tmpl in PDF_PATTERNS:
|
|
211
|
+
if rx.search(info.get('host', '')):
|
|
212
|
+
candidates.append('https://%s%s' % (info['host'], tmpl.format(doi=doi)))
|
|
213
|
+
if info.get('meta_pdf'):
|
|
214
|
+
candidates.append(info['meta_pdf'])
|
|
215
|
+
candidates += [l for l in info.get('links', []) if doi.split('/')[-1].lower() in l.lower()
|
|
216
|
+
or 'pdfft' in l or '/pdf' in l]
|
|
217
|
+
seen = set()
|
|
218
|
+
for url in candidates:
|
|
219
|
+
if url in seen:
|
|
220
|
+
continue
|
|
221
|
+
seen.add(url)
|
|
222
|
+
raw, src = page._run(_capture_pdf(page, url))
|
|
223
|
+
if raw and raw[:5] == b'%PDF-':
|
|
224
|
+
pdf_path.write_bytes(raw)
|
|
225
|
+
_note_download(doi)
|
|
226
|
+
return _result(doi, pdf_path, (src or url).split('/')[2], text, oa=oa,
|
|
227
|
+
access=info.get('access'))
|
|
228
|
+
# no PDF: keep the HTML full text if the page has it
|
|
229
|
+
body = info.get('body_text') or ''
|
|
230
|
+
if len(body) > 4000 and info.get('proxied'):
|
|
231
|
+
html_path = OUT / (_slug(doi) + '.txt')
|
|
232
|
+
html_path.write_text(body)
|
|
233
|
+
_note_download(doi)
|
|
234
|
+
return {'doi': doi, 'format': 'html-text', 'path': str(html_path),
|
|
235
|
+
'chars': len(body), 'via': info.get('host'), 'open_access': oa.get('is_oa'),
|
|
236
|
+
'text': body if text else ''}
|
|
237
|
+
if oa.get('oa_pdf'):
|
|
238
|
+
reason = 'no subscribed copy found, but an open-access PDF exists: %s' % oa['oa_pdf']
|
|
239
|
+
elif not info.get('proxied'):
|
|
240
|
+
host = (landing or '').split('/')[2] if landing.count('/') >= 2 else info.get('host')
|
|
241
|
+
reason = ('%s is not rewritten by the school EZproxy, so Sophiahemmet has no '
|
|
242
|
+
'subscription route to it; order it via the library (fjärrlån)' % host)
|
|
243
|
+
else:
|
|
244
|
+
reason = 'the publisher page was reached through EZproxy but offered no PDF (%s)' % (
|
|
245
|
+
info.get('access') or 'no access marker')
|
|
246
|
+
raise DatabaseError(reason)
|
|
247
|
+
|
|
248
|
+
|
|
249
|
+
def _result(doi, path, via, text, oa=None, access=''):
|
|
250
|
+
pages = 0
|
|
251
|
+
try:
|
|
252
|
+
out = subprocess.run(['pdfinfo', str(path)], capture_output=True, text=True, timeout=20).stdout
|
|
253
|
+
m = re.search(r'Pages:\s+(\d+)', out)
|
|
254
|
+
pages = int(m.group(1)) if m else 0
|
|
255
|
+
except Exception:
|
|
256
|
+
pass
|
|
257
|
+
body = _pdf_text(path) if text else ''
|
|
258
|
+
return {'doi': doi, 'format': 'pdf', 'path': str(path), 'bytes': path.stat().st_size,
|
|
259
|
+
'pages': pages, 'via': via, 'access': access,
|
|
260
|
+
'open_access': (oa or {}).get('is_oa'), 'chars': len(body), 'text': body}
|