unitoolkit 0.7.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
unitoolkit/search.py ADDED
@@ -0,0 +1,507 @@
1
+ """Full-text search over a Canvas course: from the live API or from a mirror on disk.
2
+
3
+ Same semantics as the Canvas Text Search extension (canvas-text-search repo), so an agent
4
+ and a student searching the same course get comparable answers: implicit AND, "phrases",
5
+ -exclusion, +required, OR, type:, filetype:, intitle:, before:, after:, regex, fuzzy,
6
+ title-weighted ranking, snippets, and deep links that scroll to the match.
7
+
8
+ from unitoolkit import search
9
+ idx = search.build_index(2416) # live Canvas
10
+ idx = search.index_from_mirror("~/courses/canvas/2416")
11
+ search.run(idx, 'plagiatkontroll type:sida')
12
+ """
13
+ import datetime
14
+ import html
15
+ import json
16
+ import os
17
+ import pathlib
18
+ import re
19
+ import urllib.parse
20
+
21
+ from . import canvas as canvas_api
22
+
23
+ TYPES = {
24
+ "sida": "sida", "sidor": "sida", "page": "sida", "pages": "sida",
25
+ "uppgift": "uppgift", "uppgifter": "uppgift", "assignment": "uppgift",
26
+ "fil": "fil", "filer": "fil", "file": "fil",
27
+ "modul": "modul", "moduler": "modul", "module": "modul",
28
+ "diskussion": "diskussion", "discussion": "diskussion",
29
+ "anvisning": "anvisning", "announcement": "anvisning",
30
+ }
31
+
32
+
33
+ def fold(s):
34
+ return re.sub(r"\s+", " ", (s or "").lower().replace("å", "a").replace("ä", "a")
35
+ .replace("ö", "o").replace("é", "e")).strip()
36
+
37
+
38
+ def strip_html(h):
39
+ """HTML -> text with block elements turned into "\\n".
40
+
41
+ The newlines matter: a text-fragment anchor can never cross a block boundary, so the
42
+ index has to remember where the page's paragraphs were. Without them "</p><p>" glues
43
+ two sentences into "(OURGINAL)Observera" and every anchor spanning them fails.
44
+ """
45
+ s = re.sub(r"</(p|div|li|ul|ol|h[1-6]|tr|table|blockquote|section|article)>", "\n",
46
+ h or "", flags=re.I)
47
+ s = re.sub(r"<br\s*/?>", "\n", s, flags=re.I)
48
+ s = re.sub(r"<li[^>]*>", "\n", s, flags=re.I)
49
+ s = re.sub(r"<[^>]+>", " ", s)
50
+ s = html.unescape(s).replace("\u00a0", " ")
51
+ s = re.sub(r"[ \t]+", " ", s)
52
+ s = re.sub(r" *\n *", "\n", s)
53
+ return re.sub(r"\n{2,}", "\n", s).strip()
54
+
55
+
56
+ _FOLD1 = str.maketrans("åäöéèÅÄÖÉ", "aaoeeAAOE")
57
+
58
+
59
+ def norm_len(s):
60
+ """Lowercase + diacritic fold that preserves length exactly, so offsets survive.
61
+
62
+ Offsets measured in a whitespace-collapsed string drift, and a folded string of a
63
+ different length points at the wrong page of a document.
64
+ """
65
+ return (s or "").lower().translate(_FOLD1)
66
+
67
+
68
+ def norm_keep(s):
69
+ """Lowercase + Swedish folding, but keeps "\n" so block boundaries survive."""
70
+ s = (s or "").lower().replace("å", "a").replace("ä", "a").replace("ö", "o").replace("é", "e")
71
+ return re.sub(r"[ \t]+", " ", s)
72
+
73
+
74
+ def edit_distance(a, b, max_d=2):
75
+ """Damerau-Levenshtein, bounded: returns max_d+1 when over the bound."""
76
+ if abs(len(a) - len(b)) > max_d:
77
+ return max_d + 1
78
+ prev2 = None
79
+ prev = list(range(len(b) + 1))
80
+ for i in range(1, len(a) + 1):
81
+ cur = [i]
82
+ rowmin = i
83
+ for j in range(1, len(b) + 1):
84
+ cost = 0 if a[i - 1] == b[j - 1] else 1
85
+ v = min(prev[j] + 1, cur[j - 1] + 1, prev[j - 1] + cost)
86
+ if prev2 and i > 1 and j > 1 and a[i - 1] == b[j - 2] and a[i - 2] == b[j - 1]:
87
+ v = min(v, prev2[j - 2] + 1)
88
+ cur.append(v)
89
+ rowmin = min(rowmin, v)
90
+ if rowmin > max_d:
91
+ return max_d + 1
92
+ prev2, prev = prev, cur
93
+ return prev[len(b)]
94
+
95
+
96
+ def _words(text):
97
+ return {w for w in re.split(r"[^a-z0-9]+", text) if len(w) > 2}
98
+
99
+
100
+ def _doc(type_, title, body, href, ext="", date=""):
101
+ f = fold(f"{title} {body}")
102
+ return {"type": type_, "title": title or "(utan titel)", "body": body or "",
103
+ "href": href or "#", "ext": ext, "date": (date or "")[:10],
104
+ "f": f, "w": sorted(_words(f))}
105
+
106
+
107
+ # ------------------------------------------------------------------ builders
108
+ def _adder(docs):
109
+ """Return an add() that keeps the richest version of a document.
110
+
111
+ A module item creates an empty stub for its assignment or page; the full document arrives
112
+ later with the same key. First-write-wins would drop rubric text and task descriptions.
113
+ """
114
+ at = {}
115
+
116
+ def add(type_, title, body, href, ext="", date=""):
117
+ d = _doc(type_, title, body, href, ext=ext, date=date)
118
+ key = (d["type"], d["title"], d["href"])
119
+ if key not in at:
120
+ at[key] = len(docs)
121
+ docs.append(d)
122
+ return
123
+ pos = at[key]
124
+ if len(d["body"]) + len(d["date"]) > len(docs[pos]["body"]) + len(docs[pos]["date"]):
125
+ docs[pos] = d
126
+
127
+ return add
128
+
129
+
130
+ def build_index(course_id, base=None):
131
+ """Index a course from the live Canvas API. Read-only, token from $CANVAS_TOKEN."""
132
+ docs = []
133
+ add = _adder(docs)
134
+
135
+ modules = canvas_api.get(f"courses/{course_id}/modules", **{"include[]": ["items"]})
136
+ page_slugs = []
137
+ for m in modules:
138
+ for it in m.get("items", []):
139
+ add("modul", f"{m['name']}: {it.get('title')}", m["name"], it.get("html_url") or it.get("url"))
140
+ if it.get("type") == "Page":
141
+ slug = it.get("page_url") or (it.get("url") or "").rsplit("/", 1)[-1]
142
+ if slug:
143
+ page_slugs.append(slug)
144
+ elif it.get("type") == "Assignment" and it.get("content_id"):
145
+ add("uppgift", it.get("title"), "", f"/courses/{course_id}/assignments/{it['content_id']}")
146
+ elif it.get("type") == "Discussion" and it.get("content_id"):
147
+ add("diskussion", it.get("title"), "", f"/courses/{course_id}/discussion_topics/{it['content_id']}")
148
+
149
+ for slug in dict.fromkeys(page_slugs):
150
+ try:
151
+ p = canvas_api.get(f"courses/{course_id}/pages/{urllib.parse.quote(slug)}")
152
+ except Exception:
153
+ continue
154
+ add("sida", p.get("title") or slug, strip_html(p.get("body")), f"/courses/{course_id}/pages/{slug}",
155
+ date=p.get("updated_at") or "")
156
+
157
+ # include[]=rubric: without it Canvas returns rubric: [] and the grading criteria,
158
+ # which students search for by name, never enter the index
159
+ for a in canvas_api.try_get(f"courses/{course_id}/assignments", **{"include[]": ["rubric"]}) or []:
160
+ if not isinstance(a, dict):
161
+ continue
162
+ rubric = " ".join(f"{r.get('description','')} {r.get('long_description','')}"
163
+ for r in (a.get("rubric") or []))
164
+ add("uppgift", a.get("name"), f"{strip_html(a.get('description'))} {rubric}",
165
+ f"/courses/{course_id}/assignments/{a.get('id')}", date=a.get("updated_at") or "")
166
+
167
+ for d in canvas_api.try_get(f"courses/{course_id}/discussion_topics", only_announcements="true") or []:
168
+ if isinstance(d, dict):
169
+ add("anvisning", d.get("title"), strip_html(d.get("message")),
170
+ f"/courses/{course_id}/discussion_topics/{d.get('id')}", date=d.get("posted_at") or "")
171
+ for d in canvas_api.try_get(f"courses/{course_id}/discussion_topics") or []:
172
+ if isinstance(d, dict) and not d.get("pinned"):
173
+ add("diskussion", d.get("title"), strip_html(d.get("message")),
174
+ f"/courses/{course_id}/discussion_topics/{d.get('id')}", date=d.get("posted_at") or "")
175
+
176
+ for f_ in canvas_api.try_get(f"courses/{course_id}/files") or []:
177
+ if isinstance(f_, dict):
178
+ name = f_.get("display_name") or f_.get("filename") or ""
179
+ add("fil", name, name, f_.get("url") or "", ext=(name.rsplit(".", 1)[-1] or "").lower(),
180
+ date=f_.get("updated_at") or f_.get("created_at") or "")
181
+ return {"course_id": course_id, "source": "api", **({"base": base} if base else {}), "docs": docs}
182
+
183
+
184
+ def index_from_mirror(path):
185
+ """Index an existing `python -m unitoolkit mirror` output directory."""
186
+ root = pathlib.Path(path).expanduser()
187
+ docs = []
188
+ add = _adder(docs)
189
+
190
+ mods_file = root / "modules.json"
191
+ course_id = root.name
192
+ if mods_file.exists():
193
+ for m in json.loads(mods_file.read_text()):
194
+ for it in m.get("items", []):
195
+ add("modul", f"{m['name']}: {it.get('title')}", m["name"], it.get("html_url") or it.get("url"))
196
+ if it.get("type") == "Assignment" and it.get("content_id"):
197
+ add("uppgift", it.get("title"), "", f"/courses/{course_id}/assignments/{it['content_id']}")
198
+ if it.get("type") == "Discussion" and it.get("content_id"):
199
+ add("diskussion", it.get("title"), "", f"/courses/{course_id}/discussion_topics/{it['content_id']}")
200
+
201
+ for p in sorted((root / "pages").glob("*.html")) if (root / "pages").exists() else []:
202
+ raw = p.read_text(errors="ignore")
203
+ slug = re.sub(r"^\d+_\d+_", "", p.stem)
204
+ title = next((m.group(1) for m in [re.search(r"title: ([^|]+)", raw[:400])] if m), slug)
205
+ add("sida", title.strip(), strip_html(raw), f"/courses/{course_id}/pages/{slug}")
206
+
207
+ for a in (json.loads((root / "assignments.json").read_text()) if (root / "assignments.json").exists() else []):
208
+ if not isinstance(a, dict):
209
+ continue
210
+ rubric = " ".join(f"{r.get('description','')} {r.get('long_description','')}"
211
+ for r in (a.get("rubric") or []))
212
+ add("uppgift", a.get("name"), f"{strip_html(a.get('description'))} {rubric}",
213
+ f"/courses/{course_id}/assignments/{a.get('id')}", date=a.get("updated_at") or "")
214
+
215
+ for name, kind in (("announcements.json", "anvisning"), ("discussions.json", "diskussion")):
216
+ f = root / name
217
+ for d in (json.loads(f.read_text()) if f.exists() else []):
218
+ if isinstance(d, dict):
219
+ add(kind, d.get("title"), strip_html(d.get("message")),
220
+ f"/courses/{course_id}/discussion_topics/{d.get('id')}", date=d.get("posted_at") or "")
221
+
222
+ fl = root / "files.json"
223
+ for f_ in (json.loads(fl.read_text()) if fl.exists() else []):
224
+ if isinstance(f_, dict):
225
+ name = f_.get("display_name") or f_.get("filename") or ""
226
+ add("fil", name, name, f_.get("url") or "", ext=(name.rsplit(".", 1)[-1] or "").lower(),
227
+ date=f_.get("updated_at") or "")
228
+ return {"course_id": course_id, "source": "mirror", "docs": docs}
229
+
230
+
231
+ # ------------------------------------------------------------------ query
232
+ def parse_query(raw, case_sensitive=False, regex=False):
233
+ q = {"terms": [], "not": [], "types": [], "exts": [], "title": [],
234
+ "before": None, "after": None, "regex": None, "error": None}
235
+ if regex:
236
+ flags = 0 if case_sensitive else re.IGNORECASE
237
+ try:
238
+ q["regex"] = re.compile(raw, flags)
239
+ except re.error as e:
240
+ q["error"] = f"Ogiltigt reguljärt uttryck: {e}"
241
+ return q
242
+ for phrase, word in re.findall(r'"([^"]*)"|(\S+)', raw):
243
+ is_phrase = phrase != "" or (phrase == "" and word == "")
244
+ s = phrase if phrase else word
245
+ if word:
246
+ kv = re.match(r"^(type|filetype|intitle|before|after):(.+)$", word, re.I)
247
+ if kv:
248
+ k, v = kv.group(1).lower(), kv.group(2)
249
+ if k == "intitle":
250
+ q["title"].append(v)
251
+ continue
252
+ if k == "filetype":
253
+ q["exts"].append(v.lstrip(".").lower())
254
+ continue
255
+ if k in ("before", "after"):
256
+ q[k] = _norm_date(v)
257
+ continue
258
+ mapped = TYPES.get(v.lower())
259
+ if mapped:
260
+ q["types"].append(mapped)
261
+ else:
262
+ q["error"] = f"Okänd typ {v!r}. Använd: {', '.join(sorted(set(TYPES.values())))}"
263
+ continue
264
+ if word.startswith("-") and len(word) > 1:
265
+ q["not"].append(word[1:])
266
+ continue
267
+ if word.startswith("+") and len(word) > 1:
268
+ q["terms"].append(word[1:])
269
+ continue
270
+ if word.lower() in ("and", "or"):
271
+ continue
272
+ if s:
273
+ q["terms"].append(s)
274
+ return q
275
+
276
+
277
+ def _norm_date(v):
278
+ m = re.match(r"^(\d{4})[-/.](\d{1,2})(?:[-/.](\d{1,2}))?$", str(v))
279
+ if m:
280
+ return f"{m.group(1)}-{m.group(2).zfill(2)}-{m.group(3).zfill(2) if m.group(3) else '01'}"
281
+ if re.match(r"^\d{4}$", str(v)):
282
+ return f"{v}-01-01"
283
+ return None
284
+
285
+
286
+ def _count_plain(hay, needle):
287
+ if not needle:
288
+ return 0
289
+ return hay.count(needle)
290
+
291
+
292
+ def _count_word(hay, term):
293
+ return len(re.findall(rf"(?<![a-z0-9]){re.escape(term)}(?![a-z0-9])", hay))
294
+
295
+
296
+ def _fuzzy_hits(doc, term):
297
+ max_d = 1 if len(term) <= 5 else 2
298
+ hits = []
299
+ for w in doc["w"]:
300
+ if w == term or w.startswith(term):
301
+ hits.append(w)
302
+ elif abs(len(w) - len(term)) <= max_d and edit_distance(term, w, max_d) <= max_d:
303
+ hits.append(w)
304
+ return hits
305
+
306
+
307
+ MIN_FRAGMENT = 12
308
+
309
+
310
+ def fragment_matches(anchor, title, body):
311
+ """Would a browser text directive for this phrase actually find it?
312
+
313
+ The phrase must occur inside a single block, so the comparison is line by line.
314
+ """
315
+ a = norm_keep(anchor or "").strip()
316
+ if not a or len(a) < 8:
317
+ return False
318
+ lines = norm_keep(f"{title or ''}\n{body or ''}").split("\n")
319
+ if any(a in ln for ln in lines):
320
+ return True
321
+ words = a.split(" ")
322
+ return len(words) > 2 and any(" ".join(words[:3]) in ln for ln in lines)
323
+
324
+
325
+ def anchor_for(title, at, body):
326
+ """Longest verified phrase at the match: whole words, max 8, inside one block.
327
+
328
+ The offset is moved back to the start of its word, then forward to the next word,
329
+ because a text directive that starts inside a word never matches.
330
+ """
331
+ b = body or ""
332
+ is_w = lambda ch: bool(ch) and (ch.isalnum() or ch in "åäöé")
333
+ i = max(0, min(at or 0, len(b) - 1)) if b else 0
334
+ while i > 0 and is_w(b[i - 1]) and is_w(b[i]):
335
+ i -= 1
336
+ while i < len(b) and not is_w(b[i]):
337
+ i += 1
338
+
339
+ # work inside the block that contains the match
340
+ line_start = b.rfind("\n", 0, max(0, i - 1)) + 1
341
+ line_end = b.find("\n", i)
342
+ if line_end < 0:
343
+ line_end = len(b)
344
+ line = re.sub(r"[ \t\u00a0]+", " ", b[line_start:line_end])
345
+
346
+ words = [(m.group(0), m.start()) for m in re.finditer(r"[0-9a-zA-ZåäöÅÄÖé]+", line)]
347
+ if not words:
348
+ return ""
349
+ k = next((n for n, (w, pos) in enumerate(words) if pos + len(w) > i - line_start), 0)
350
+
351
+ for start in range(max(0, k - 3), k + 1):
352
+ for n in range(min(8, len(words) - start), 0, -1):
353
+ end = words[start + n - 1][1] + len(words[start + n - 1][0])
354
+ cand = line[words[start][1]:end].strip()
355
+ if len(cand) >= MIN_FRAGMENT and fragment_matches(cand, title, b):
356
+ return cand
357
+ return words[k][0] if fragment_matches(words[k][0], title, b) else ""
358
+
359
+
360
+ def anchor_target(doc, raw, regex=False, fuzzy=False, case_sensitive=False):
361
+ body = doc.get("body") or ""
362
+ seed = doc.get("anchor") or ""
363
+ if not regex and not fuzzy:
364
+ terms = sorted(parse_query(raw, case_sensitive)["terms"], key=len, reverse=True)
365
+ if terms:
366
+ seed = terms[0]
367
+ at = doc.get("at") or 0
368
+ if seed:
369
+ # offsets must be measured in the original body: fold() collapses whitespace and shifts them
370
+ pos = body.lower().find(str(seed).lower())
371
+ if pos >= 0:
372
+ at = pos
373
+ return anchor_for(doc.get("title"), at, body)
374
+
375
+
376
+ def _match(doc, q, case_sensitive=False, fuzzy=False, whole_word=False):
377
+ if q["types"] and doc["type"] not in q["types"]:
378
+ return None
379
+ if q["exts"] and doc["ext"] not in q["exts"]:
380
+ return None
381
+ if q["before"] or q["after"]:
382
+ d = doc["date"][:10]
383
+ if not d:
384
+ return None
385
+ if q["before"] and not d < q["before"]:
386
+ return None
387
+ if q["after"] and not d > q["after"]:
388
+ return None
389
+
390
+ if q["regex"]:
391
+ hay = f"{doc['title']}\n{doc['body']}"
392
+ m = list(q["regex"].finditer(hay))
393
+ if not m:
394
+ return None
395
+ return {"score": len(m) + 5, "at": m[0].start(), "anchor": m[0].group(0)}
396
+
397
+ prep = (lambda s: (s or "")) if case_sensitive else fold
398
+ hay_src, title_src = f"{doc['title']}\n{doc['body']}", doc["title"]
399
+ hay, title = prep(hay_src), prep(title_src)
400
+ if any(prep(n) in hay for n in q["not"]):
401
+ return None
402
+ if any(prep(t) not in title for t in q["title"]):
403
+ return None
404
+
405
+ score, at, best = 0, -1, ""
406
+ for term in q["terms"]:
407
+ t = prep(term)
408
+ if fuzzy:
409
+ fw = _fuzzy_hits(doc, t)
410
+ c, anchor = len(fw), (fw[0] if fw else t)
411
+ elif whole_word or len(t) <= 3:
412
+ c, anchor = _count_word(hay, t), t
413
+ else:
414
+ c, anchor = _count_plain(hay, t), t
415
+ if not c:
416
+ return None
417
+ if at < 0:
418
+ # offsets must be measured in the original text: fold() collapses whitespace,
419
+ # so a folded offset drifts and page numbers and snippets land in the wrong place
420
+ # A phrase may match only after folding (a line break inside it), so locate its
421
+ # leading words with whitespace tolerance. Whole phrase first, then fewer words,
422
+ # so "sida 100 text" is not located at the first "sida" on page 1.
423
+
424
+ hay_norm = hay_src if case_sensitive else norm_len(hay_src)
425
+ target = anchor if case_sensitive else norm_len(anchor)
426
+ words = [w for w in re.split(r"\s+", target) if w][:3]
427
+ at = 0
428
+ for n in range(len(words), 0, -1):
429
+ m = re.search(r"\s+".join(re.escape(w) for w in words[:n]), hay_norm)
430
+ if m:
431
+ at = m.start()
432
+ break
433
+ best = best if len(best) > len(anchor) else anchor
434
+ score += (12 if anchor in title else 0) + min(c, 6)
435
+ if doc["type"] in ("anvisning", "sida"):
436
+ score += 3
437
+ return {"score": score, "at": at, "anchor": best}
438
+
439
+
440
+ def run(index, raw, case_sensitive=False, regex=False, fuzzy=False, whole_word=False, limit=40):
441
+ """Search an index. Returns {'hits': [...], 'total': n} or {'error': '...'}."""
442
+ alts = [raw] if regex else (re.split(r"\s+OR\s+", raw, flags=re.I) if re.search(r"\s+OR\s+", raw, flags=re.I) else [raw])
443
+ found, constrained = {}, False
444
+ for alt in alts:
445
+ q = parse_query(alt, case_sensitive, regex)
446
+ if q["error"]:
447
+ return {"error": q["error"], "hits": [], "total": 0}
448
+ if not (q["terms"] or q["regex"] or q["title"] or q["types"] or q["exts"] or q["before"] or q["after"]):
449
+ continue
450
+ constrained = True
451
+ for doc in index["docs"]:
452
+ r = _match(doc, q, case_sensitive, fuzzy, whole_word)
453
+ if not r:
454
+ continue
455
+ key = (doc["href"], doc["title"])
456
+ if key not in found or r["score"] > found[key]["score"]:
457
+ found[key] = {**doc, "score": r["score"], "at": r["at"]}
458
+ cur = found[key]
459
+ if r.get("anchor") and len(r["anchor"]) > len(cur.get("anchor") or ""):
460
+ cur["anchor"] = r["anchor"]
461
+ if not constrained:
462
+ return {"hits": [], "total": 0, "empty": True}
463
+ hits = sorted(found.values(), key=lambda d: -d["score"])[:limit]
464
+ for h in hits:
465
+ h["snippet"] = snippet(h, h["at"])
466
+ h["anchor"] = anchor_target(h, raw, regex=regex, fuzzy=fuzzy, case_sensitive=case_sensitive)
467
+ # an unverifiable anchor yields the bare link rather than a dead fragment
468
+ h["url"] = deep_link(h["href"], h["anchor"])
469
+ return {"hits": hits, "total": len(found)}
470
+
471
+
472
+ def suggest(index, raw):
473
+ q = parse_query(raw)
474
+ if not q["terms"]:
475
+ return None
476
+ term = fold(q["terms"][0])
477
+ if len(term) < 4:
478
+ return None
479
+ best = None
480
+ seen = set()
481
+ for doc in index["docs"]:
482
+ for w in doc["w"]:
483
+ if w in seen:
484
+ continue
485
+ seen.add(w)
486
+ if len(w) < 4:
487
+ continue
488
+ d = edit_distance(term, w, 2)
489
+ if d == 0 or d > 2:
490
+ continue
491
+ if best is None or d < best[0] or (d == best[0] and len(w) < len(best[1])):
492
+ best = (d, w)
493
+ return best[1] if best else None
494
+
495
+
496
+ def snippet(doc, at, width=230):
497
+ body = doc["body"] if len(doc["body"] or "") > 20 else doc["title"]
498
+ start = max(0, (at or 0) - width // 3)
499
+ return ("…" if start else "") + re.sub(r"\s+", " ", body[start:start + width])
500
+
501
+
502
+ def deep_link(href, phrase):
503
+ if not href or not phrase:
504
+ return href
505
+ clean = re.sub(r"[\r\n]+", " ", phrase).strip()[:120]
506
+ enc = urllib.parse.quote(clean, safe="").replace("-", "%2D").replace(",", "%2C")
507
+ return f"{href}{'&' if '#' in href else '#:~:'}text={enc}"
unitoolkit/serve.py ADDED
@@ -0,0 +1,153 @@
1
+ """Local HTTP endpoint for the deep index, for the browser plugin and for other tools.
2
+
3
+ The plugin is the fast path (in-browser index of pages, assignments, rubrics, announcements,
4
+ module items). This server is the deep path: document text, OCR results, mirrors, the whole
5
+ programme. Running it lets the plugin merge both layers without bundling a PDF parser into a
6
+ Chrome extension, and without the plugin ever needing a Canvas token.
7
+
8
+ Binds loopback only. Reach it from another machine with `tailscale serve`:
9
+
10
+ python -m unitoolkit serve # 127.0.0.1:8790
11
+ tailscale serve --bg 8790 # https://<host>.<tailnet>.ts.net/
12
+
13
+ Endpoints (all GET, JSON):
14
+ /health service state, cached document count
15
+ /courses the student's courses
16
+ /search?course=&q=&limit= course search; add docs=1 to include document text
17
+ /docs?course=&q= document text only, with page numbers
18
+ /index what is cached per course
19
+ """
20
+ import json
21
+ import pathlib
22
+ import urllib.parse
23
+ from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
24
+
25
+ from . import docs as docs_mod
26
+ from .tools import TOOLS, call
27
+
28
+ PORT = 8791 # 8790 is the agentrecourse site preview on this machine
29
+ ALLOWED_PREFIXES = ("https://shh.instructure.com",
30
+ "https://cloud.timeedit.net",
31
+ "http://127.0.0.1", "http://localhost")
32
+
33
+
34
+ def _cached_index():
35
+ root = docs_mod.CACHE
36
+ out = {}
37
+ if not root.exists():
38
+ return out
39
+ for d in sorted(root.iterdir()):
40
+ if d.is_dir():
41
+ files = list(d.glob("*.json"))
42
+ chars = 0
43
+ for f in files:
44
+ try:
45
+ chars += len((json.loads(f.read_text()).get("text") or ""))
46
+ except Exception: # noqa: BLE001
47
+ pass
48
+ out[d.name] = {"documents": len(files), "chars": chars}
49
+ return out
50
+
51
+
52
+ class Handler(BaseHTTPRequestHandler):
53
+ server_version = "unitoolkit"
54
+ protocol_version = "HTTP/1.1"
55
+
56
+ def log_message(self, format, *args): # noqa: A002 - matches the base signature
57
+ pass
58
+
59
+ def _cors(self):
60
+ origin = self.headers.get("Origin") or ""
61
+ if origin.startswith(ALLOWED_PREFIXES):
62
+ self.send_header("Access-Control-Allow-Origin", origin)
63
+ self.send_header("Vary", "Origin")
64
+ self.send_header("Access-Control-Allow-Methods", "GET, OPTIONS")
65
+ self.send_header("Access-Control-Allow-Headers", "content-type")
66
+ self.send_header("Access-Control-Max-Age", "600")
67
+ # Chrome refuses a loopback fetch from an https page (Private Network Access) unless
68
+ # the server answers the preflight with this header.
69
+ self.send_header("Access-Control-Allow-Private-Network", "true")
70
+
71
+ def _json(self, obj, code=200):
72
+ body = json.dumps(obj, ensure_ascii=False, default=str).encode("utf-8")
73
+ self.send_response(code)
74
+ self.send_header("Content-Type", "application/json; charset=utf-8")
75
+ self._cors()
76
+ self.send_header("Content-Length", str(len(body)))
77
+ self.end_headers()
78
+ self.wfile.write(body)
79
+
80
+ def do_OPTIONS(self): # noqa: N802 - http.server API
81
+ self.send_response(204)
82
+ self._cors()
83
+ self.send_header("Content-Length", "0")
84
+ self.end_headers()
85
+
86
+ def do_GET(self): # noqa: N802 - http.server API
87
+ u = urllib.parse.urlparse(self.path)
88
+ q = {k: v for k, v in urllib.parse.parse_qsl(u.query)}
89
+ try:
90
+ if u.path in ("/", "/health"):
91
+ return self._json({"ok": True, "service": "unitoolkit", "version": _version(),
92
+ "tools": len(TOOLS), "cached": _cached_index()})
93
+ if u.path == "/courses":
94
+ return self._json(call("courses", {}))
95
+ if u.path == "/index":
96
+ return self._json(_cached_index())
97
+ if u.path == "/search":
98
+ course = q.get("course")
99
+ query = q.get("q") or q.get("query") or ""
100
+ if not course:
101
+ return self._json({"error": "course is required"}, 400)
102
+ args = {"course": str(course), "query": query,
103
+ "limit": int(q.get("limit") or 25),
104
+ "include_docs": (q.get("docs") or "") in ("1", "true", "yes"),
105
+ "fuzzy": (q.get("fuzzy") or "") in ("1", "true"),
106
+ "regex": (q.get("regex") or "") in ("1", "true")}
107
+ if q.get("source"):
108
+ args["source"] = q["source"]
109
+ return self._json(call("search", args))
110
+ if u.path == "/docs":
111
+ course = q.get("course")
112
+ if not course:
113
+ return self._json({"error": "course is required"}, 400)
114
+ args = {"course": str(course), "query": q.get("q") or "",
115
+ "include_docs": True, "limit": int(q.get("limit") or 25)}
116
+ if q.get("source"):
117
+ args["source"] = q["source"]
118
+ return self._json(call("search", args))
119
+ return self._json({"error": "unknown path", "paths": ["/health", "/courses",
120
+ "/search", "/docs", "/index"]}, 404)
121
+ except Exception as e: # noqa: BLE001 - a server must answer, not crash
122
+ return self._json({"error": f"{type(e).__name__}: {e}"}, 500)
123
+
124
+
125
+ def _version():
126
+ try:
127
+ from . import __version__
128
+ return __version__
129
+ except Exception: # noqa: BLE001
130
+ return "?"
131
+
132
+
133
+ def serve(host="127.0.0.1", port=PORT):
134
+ httpd = None
135
+ last = None
136
+ for candidate in range(port, port + 6):
137
+ try:
138
+ httpd = ThreadingHTTPServer((host, candidate), Handler)
139
+ port = candidate
140
+ break
141
+ except OSError as e: # port taken by something else: try the next one
142
+ last = e
143
+ if httpd is None:
144
+ raise SystemExit(f"no free port in {port}-{port + 5}: {last}")
145
+ print(f"unitoolkit serving on http://{host}:{port}", flush=True)
146
+ print(" tailscale serve --bg " + str(port) + " # then https://<machine>.<tailnet>.ts.net/",
147
+ flush=True)
148
+ try:
149
+ httpd.serve_forever()
150
+ except KeyboardInterrupt:
151
+ print("\nstopped", flush=True)
152
+ finally:
153
+ httpd.server_close()