unitoolkit 0.7.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- ladok_vakt/__init__.py +7 -0
- ladok_vakt/__main__.py +51 -0
- ladok_vakt/core.py +553 -0
- ladok_vakt/login.py +232 -0
- tools.json +623 -0
- unitoolkit/__init__.py +1 -0
- unitoolkit/__main__.py +63 -0
- unitoolkit/agent.py +270 -0
- unitoolkit/browser.py +147 -0
- unitoolkit/canvas.py +189 -0
- unitoolkit/chainproxy.py +113 -0
- unitoolkit/citation.py +165 -0
- unitoolkit/databases.py +497 -0
- unitoolkit/docs.py +394 -0
- unitoolkit/fulltext.py +260 -0
- unitoolkit/koha.py +729 -0
- unitoolkit/mcp.py +85 -0
- unitoolkit/programme.py +147 -0
- unitoolkit/search.py +507 -0
- unitoolkit/serve.py +153 -0
- unitoolkit/sharepoint.py +86 -0
- unitoolkit/surface.py +86 -0
- unitoolkit/timeedit.py +209 -0
- unitoolkit/tools.py +555 -0
- unitoolkit-0.7.0.dist-info/METADATA +177 -0
- unitoolkit-0.7.0.dist-info/RECORD +30 -0
- unitoolkit-0.7.0.dist-info/WHEEL +5 -0
- unitoolkit-0.7.0.dist-info/entry_points.txt +4 -0
- unitoolkit-0.7.0.dist-info/licenses/LICENSE +21 -0
- unitoolkit-0.7.0.dist-info/top_level.txt +2 -0
unitoolkit/search.py
ADDED
|
@@ -0,0 +1,507 @@
|
|
|
1
|
+
"""Full-text search over a Canvas course: from the live API or from a mirror on disk.
|
|
2
|
+
|
|
3
|
+
Same semantics as the Canvas Text Search extension (canvas-text-search repo), so an agent
|
|
4
|
+
and a student searching the same course get comparable answers: implicit AND, "phrases",
|
|
5
|
+
-exclusion, +required, OR, type:, filetype:, intitle:, before:, after:, regex, fuzzy,
|
|
6
|
+
title-weighted ranking, snippets, and deep links that scroll to the match.
|
|
7
|
+
|
|
8
|
+
from unitoolkit import search
|
|
9
|
+
idx = search.build_index(2416) # live Canvas
|
|
10
|
+
idx = search.index_from_mirror("~/courses/canvas/2416")
|
|
11
|
+
search.run(idx, 'plagiatkontroll type:sida')
|
|
12
|
+
"""
|
|
13
|
+
import datetime
|
|
14
|
+
import html
|
|
15
|
+
import json
|
|
16
|
+
import os
|
|
17
|
+
import pathlib
|
|
18
|
+
import re
|
|
19
|
+
import urllib.parse
|
|
20
|
+
|
|
21
|
+
from . import canvas as canvas_api
|
|
22
|
+
|
|
23
|
+
TYPES = {
|
|
24
|
+
"sida": "sida", "sidor": "sida", "page": "sida", "pages": "sida",
|
|
25
|
+
"uppgift": "uppgift", "uppgifter": "uppgift", "assignment": "uppgift",
|
|
26
|
+
"fil": "fil", "filer": "fil", "file": "fil",
|
|
27
|
+
"modul": "modul", "moduler": "modul", "module": "modul",
|
|
28
|
+
"diskussion": "diskussion", "discussion": "diskussion",
|
|
29
|
+
"anvisning": "anvisning", "announcement": "anvisning",
|
|
30
|
+
}
|
|
31
|
+
|
|
32
|
+
|
|
33
|
+
def fold(s):
|
|
34
|
+
return re.sub(r"\s+", " ", (s or "").lower().replace("å", "a").replace("ä", "a")
|
|
35
|
+
.replace("ö", "o").replace("é", "e")).strip()
|
|
36
|
+
|
|
37
|
+
|
|
38
|
+
def strip_html(h):
|
|
39
|
+
"""HTML -> text with block elements turned into "\\n".
|
|
40
|
+
|
|
41
|
+
The newlines matter: a text-fragment anchor can never cross a block boundary, so the
|
|
42
|
+
index has to remember where the page's paragraphs were. Without them "</p><p>" glues
|
|
43
|
+
two sentences into "(OURGINAL)Observera" and every anchor spanning them fails.
|
|
44
|
+
"""
|
|
45
|
+
s = re.sub(r"</(p|div|li|ul|ol|h[1-6]|tr|table|blockquote|section|article)>", "\n",
|
|
46
|
+
h or "", flags=re.I)
|
|
47
|
+
s = re.sub(r"<br\s*/?>", "\n", s, flags=re.I)
|
|
48
|
+
s = re.sub(r"<li[^>]*>", "\n", s, flags=re.I)
|
|
49
|
+
s = re.sub(r"<[^>]+>", " ", s)
|
|
50
|
+
s = html.unescape(s).replace("\u00a0", " ")
|
|
51
|
+
s = re.sub(r"[ \t]+", " ", s)
|
|
52
|
+
s = re.sub(r" *\n *", "\n", s)
|
|
53
|
+
return re.sub(r"\n{2,}", "\n", s).strip()
|
|
54
|
+
|
|
55
|
+
|
|
56
|
+
_FOLD1 = str.maketrans("åäöéèÅÄÖÉ", "aaoeeAAOE")
|
|
57
|
+
|
|
58
|
+
|
|
59
|
+
def norm_len(s):
|
|
60
|
+
"""Lowercase + diacritic fold that preserves length exactly, so offsets survive.
|
|
61
|
+
|
|
62
|
+
Offsets measured in a whitespace-collapsed string drift, and a folded string of a
|
|
63
|
+
different length points at the wrong page of a document.
|
|
64
|
+
"""
|
|
65
|
+
return (s or "").lower().translate(_FOLD1)
|
|
66
|
+
|
|
67
|
+
|
|
68
|
+
def norm_keep(s):
|
|
69
|
+
"""Lowercase + Swedish folding, but keeps "\n" so block boundaries survive."""
|
|
70
|
+
s = (s or "").lower().replace("å", "a").replace("ä", "a").replace("ö", "o").replace("é", "e")
|
|
71
|
+
return re.sub(r"[ \t]+", " ", s)
|
|
72
|
+
|
|
73
|
+
|
|
74
|
+
def edit_distance(a, b, max_d=2):
|
|
75
|
+
"""Damerau-Levenshtein, bounded: returns max_d+1 when over the bound."""
|
|
76
|
+
if abs(len(a) - len(b)) > max_d:
|
|
77
|
+
return max_d + 1
|
|
78
|
+
prev2 = None
|
|
79
|
+
prev = list(range(len(b) + 1))
|
|
80
|
+
for i in range(1, len(a) + 1):
|
|
81
|
+
cur = [i]
|
|
82
|
+
rowmin = i
|
|
83
|
+
for j in range(1, len(b) + 1):
|
|
84
|
+
cost = 0 if a[i - 1] == b[j - 1] else 1
|
|
85
|
+
v = min(prev[j] + 1, cur[j - 1] + 1, prev[j - 1] + cost)
|
|
86
|
+
if prev2 and i > 1 and j > 1 and a[i - 1] == b[j - 2] and a[i - 2] == b[j - 1]:
|
|
87
|
+
v = min(v, prev2[j - 2] + 1)
|
|
88
|
+
cur.append(v)
|
|
89
|
+
rowmin = min(rowmin, v)
|
|
90
|
+
if rowmin > max_d:
|
|
91
|
+
return max_d + 1
|
|
92
|
+
prev2, prev = prev, cur
|
|
93
|
+
return prev[len(b)]
|
|
94
|
+
|
|
95
|
+
|
|
96
|
+
def _words(text):
|
|
97
|
+
return {w for w in re.split(r"[^a-z0-9]+", text) if len(w) > 2}
|
|
98
|
+
|
|
99
|
+
|
|
100
|
+
def _doc(type_, title, body, href, ext="", date=""):
|
|
101
|
+
f = fold(f"{title} {body}")
|
|
102
|
+
return {"type": type_, "title": title or "(utan titel)", "body": body or "",
|
|
103
|
+
"href": href or "#", "ext": ext, "date": (date or "")[:10],
|
|
104
|
+
"f": f, "w": sorted(_words(f))}
|
|
105
|
+
|
|
106
|
+
|
|
107
|
+
# ------------------------------------------------------------------ builders
|
|
108
|
+
def _adder(docs):
|
|
109
|
+
"""Return an add() that keeps the richest version of a document.
|
|
110
|
+
|
|
111
|
+
A module item creates an empty stub for its assignment or page; the full document arrives
|
|
112
|
+
later with the same key. First-write-wins would drop rubric text and task descriptions.
|
|
113
|
+
"""
|
|
114
|
+
at = {}
|
|
115
|
+
|
|
116
|
+
def add(type_, title, body, href, ext="", date=""):
|
|
117
|
+
d = _doc(type_, title, body, href, ext=ext, date=date)
|
|
118
|
+
key = (d["type"], d["title"], d["href"])
|
|
119
|
+
if key not in at:
|
|
120
|
+
at[key] = len(docs)
|
|
121
|
+
docs.append(d)
|
|
122
|
+
return
|
|
123
|
+
pos = at[key]
|
|
124
|
+
if len(d["body"]) + len(d["date"]) > len(docs[pos]["body"]) + len(docs[pos]["date"]):
|
|
125
|
+
docs[pos] = d
|
|
126
|
+
|
|
127
|
+
return add
|
|
128
|
+
|
|
129
|
+
|
|
130
|
+
def build_index(course_id, base=None):
|
|
131
|
+
"""Index a course from the live Canvas API. Read-only, token from $CANVAS_TOKEN."""
|
|
132
|
+
docs = []
|
|
133
|
+
add = _adder(docs)
|
|
134
|
+
|
|
135
|
+
modules = canvas_api.get(f"courses/{course_id}/modules", **{"include[]": ["items"]})
|
|
136
|
+
page_slugs = []
|
|
137
|
+
for m in modules:
|
|
138
|
+
for it in m.get("items", []):
|
|
139
|
+
add("modul", f"{m['name']}: {it.get('title')}", m["name"], it.get("html_url") or it.get("url"))
|
|
140
|
+
if it.get("type") == "Page":
|
|
141
|
+
slug = it.get("page_url") or (it.get("url") or "").rsplit("/", 1)[-1]
|
|
142
|
+
if slug:
|
|
143
|
+
page_slugs.append(slug)
|
|
144
|
+
elif it.get("type") == "Assignment" and it.get("content_id"):
|
|
145
|
+
add("uppgift", it.get("title"), "", f"/courses/{course_id}/assignments/{it['content_id']}")
|
|
146
|
+
elif it.get("type") == "Discussion" and it.get("content_id"):
|
|
147
|
+
add("diskussion", it.get("title"), "", f"/courses/{course_id}/discussion_topics/{it['content_id']}")
|
|
148
|
+
|
|
149
|
+
for slug in dict.fromkeys(page_slugs):
|
|
150
|
+
try:
|
|
151
|
+
p = canvas_api.get(f"courses/{course_id}/pages/{urllib.parse.quote(slug)}")
|
|
152
|
+
except Exception:
|
|
153
|
+
continue
|
|
154
|
+
add("sida", p.get("title") or slug, strip_html(p.get("body")), f"/courses/{course_id}/pages/{slug}",
|
|
155
|
+
date=p.get("updated_at") or "")
|
|
156
|
+
|
|
157
|
+
# include[]=rubric: without it Canvas returns rubric: [] and the grading criteria,
|
|
158
|
+
# which students search for by name, never enter the index
|
|
159
|
+
for a in canvas_api.try_get(f"courses/{course_id}/assignments", **{"include[]": ["rubric"]}) or []:
|
|
160
|
+
if not isinstance(a, dict):
|
|
161
|
+
continue
|
|
162
|
+
rubric = " ".join(f"{r.get('description','')} {r.get('long_description','')}"
|
|
163
|
+
for r in (a.get("rubric") or []))
|
|
164
|
+
add("uppgift", a.get("name"), f"{strip_html(a.get('description'))} {rubric}",
|
|
165
|
+
f"/courses/{course_id}/assignments/{a.get('id')}", date=a.get("updated_at") or "")
|
|
166
|
+
|
|
167
|
+
for d in canvas_api.try_get(f"courses/{course_id}/discussion_topics", only_announcements="true") or []:
|
|
168
|
+
if isinstance(d, dict):
|
|
169
|
+
add("anvisning", d.get("title"), strip_html(d.get("message")),
|
|
170
|
+
f"/courses/{course_id}/discussion_topics/{d.get('id')}", date=d.get("posted_at") or "")
|
|
171
|
+
for d in canvas_api.try_get(f"courses/{course_id}/discussion_topics") or []:
|
|
172
|
+
if isinstance(d, dict) and not d.get("pinned"):
|
|
173
|
+
add("diskussion", d.get("title"), strip_html(d.get("message")),
|
|
174
|
+
f"/courses/{course_id}/discussion_topics/{d.get('id')}", date=d.get("posted_at") or "")
|
|
175
|
+
|
|
176
|
+
for f_ in canvas_api.try_get(f"courses/{course_id}/files") or []:
|
|
177
|
+
if isinstance(f_, dict):
|
|
178
|
+
name = f_.get("display_name") or f_.get("filename") or ""
|
|
179
|
+
add("fil", name, name, f_.get("url") or "", ext=(name.rsplit(".", 1)[-1] or "").lower(),
|
|
180
|
+
date=f_.get("updated_at") or f_.get("created_at") or "")
|
|
181
|
+
return {"course_id": course_id, "source": "api", **({"base": base} if base else {}), "docs": docs}
|
|
182
|
+
|
|
183
|
+
|
|
184
|
+
def index_from_mirror(path):
|
|
185
|
+
"""Index an existing `python -m unitoolkit mirror` output directory."""
|
|
186
|
+
root = pathlib.Path(path).expanduser()
|
|
187
|
+
docs = []
|
|
188
|
+
add = _adder(docs)
|
|
189
|
+
|
|
190
|
+
mods_file = root / "modules.json"
|
|
191
|
+
course_id = root.name
|
|
192
|
+
if mods_file.exists():
|
|
193
|
+
for m in json.loads(mods_file.read_text()):
|
|
194
|
+
for it in m.get("items", []):
|
|
195
|
+
add("modul", f"{m['name']}: {it.get('title')}", m["name"], it.get("html_url") or it.get("url"))
|
|
196
|
+
if it.get("type") == "Assignment" and it.get("content_id"):
|
|
197
|
+
add("uppgift", it.get("title"), "", f"/courses/{course_id}/assignments/{it['content_id']}")
|
|
198
|
+
if it.get("type") == "Discussion" and it.get("content_id"):
|
|
199
|
+
add("diskussion", it.get("title"), "", f"/courses/{course_id}/discussion_topics/{it['content_id']}")
|
|
200
|
+
|
|
201
|
+
for p in sorted((root / "pages").glob("*.html")) if (root / "pages").exists() else []:
|
|
202
|
+
raw = p.read_text(errors="ignore")
|
|
203
|
+
slug = re.sub(r"^\d+_\d+_", "", p.stem)
|
|
204
|
+
title = next((m.group(1) for m in [re.search(r"title: ([^|]+)", raw[:400])] if m), slug)
|
|
205
|
+
add("sida", title.strip(), strip_html(raw), f"/courses/{course_id}/pages/{slug}")
|
|
206
|
+
|
|
207
|
+
for a in (json.loads((root / "assignments.json").read_text()) if (root / "assignments.json").exists() else []):
|
|
208
|
+
if not isinstance(a, dict):
|
|
209
|
+
continue
|
|
210
|
+
rubric = " ".join(f"{r.get('description','')} {r.get('long_description','')}"
|
|
211
|
+
for r in (a.get("rubric") or []))
|
|
212
|
+
add("uppgift", a.get("name"), f"{strip_html(a.get('description'))} {rubric}",
|
|
213
|
+
f"/courses/{course_id}/assignments/{a.get('id')}", date=a.get("updated_at") or "")
|
|
214
|
+
|
|
215
|
+
for name, kind in (("announcements.json", "anvisning"), ("discussions.json", "diskussion")):
|
|
216
|
+
f = root / name
|
|
217
|
+
for d in (json.loads(f.read_text()) if f.exists() else []):
|
|
218
|
+
if isinstance(d, dict):
|
|
219
|
+
add(kind, d.get("title"), strip_html(d.get("message")),
|
|
220
|
+
f"/courses/{course_id}/discussion_topics/{d.get('id')}", date=d.get("posted_at") or "")
|
|
221
|
+
|
|
222
|
+
fl = root / "files.json"
|
|
223
|
+
for f_ in (json.loads(fl.read_text()) if fl.exists() else []):
|
|
224
|
+
if isinstance(f_, dict):
|
|
225
|
+
name = f_.get("display_name") or f_.get("filename") or ""
|
|
226
|
+
add("fil", name, name, f_.get("url") or "", ext=(name.rsplit(".", 1)[-1] or "").lower(),
|
|
227
|
+
date=f_.get("updated_at") or "")
|
|
228
|
+
return {"course_id": course_id, "source": "mirror", "docs": docs}
|
|
229
|
+
|
|
230
|
+
|
|
231
|
+
# ------------------------------------------------------------------ query
|
|
232
|
+
def parse_query(raw, case_sensitive=False, regex=False):
|
|
233
|
+
q = {"terms": [], "not": [], "types": [], "exts": [], "title": [],
|
|
234
|
+
"before": None, "after": None, "regex": None, "error": None}
|
|
235
|
+
if regex:
|
|
236
|
+
flags = 0 if case_sensitive else re.IGNORECASE
|
|
237
|
+
try:
|
|
238
|
+
q["regex"] = re.compile(raw, flags)
|
|
239
|
+
except re.error as e:
|
|
240
|
+
q["error"] = f"Ogiltigt reguljärt uttryck: {e}"
|
|
241
|
+
return q
|
|
242
|
+
for phrase, word in re.findall(r'"([^"]*)"|(\S+)', raw):
|
|
243
|
+
is_phrase = phrase != "" or (phrase == "" and word == "")
|
|
244
|
+
s = phrase if phrase else word
|
|
245
|
+
if word:
|
|
246
|
+
kv = re.match(r"^(type|filetype|intitle|before|after):(.+)$", word, re.I)
|
|
247
|
+
if kv:
|
|
248
|
+
k, v = kv.group(1).lower(), kv.group(2)
|
|
249
|
+
if k == "intitle":
|
|
250
|
+
q["title"].append(v)
|
|
251
|
+
continue
|
|
252
|
+
if k == "filetype":
|
|
253
|
+
q["exts"].append(v.lstrip(".").lower())
|
|
254
|
+
continue
|
|
255
|
+
if k in ("before", "after"):
|
|
256
|
+
q[k] = _norm_date(v)
|
|
257
|
+
continue
|
|
258
|
+
mapped = TYPES.get(v.lower())
|
|
259
|
+
if mapped:
|
|
260
|
+
q["types"].append(mapped)
|
|
261
|
+
else:
|
|
262
|
+
q["error"] = f"Okänd typ {v!r}. Använd: {', '.join(sorted(set(TYPES.values())))}"
|
|
263
|
+
continue
|
|
264
|
+
if word.startswith("-") and len(word) > 1:
|
|
265
|
+
q["not"].append(word[1:])
|
|
266
|
+
continue
|
|
267
|
+
if word.startswith("+") and len(word) > 1:
|
|
268
|
+
q["terms"].append(word[1:])
|
|
269
|
+
continue
|
|
270
|
+
if word.lower() in ("and", "or"):
|
|
271
|
+
continue
|
|
272
|
+
if s:
|
|
273
|
+
q["terms"].append(s)
|
|
274
|
+
return q
|
|
275
|
+
|
|
276
|
+
|
|
277
|
+
def _norm_date(v):
|
|
278
|
+
m = re.match(r"^(\d{4})[-/.](\d{1,2})(?:[-/.](\d{1,2}))?$", str(v))
|
|
279
|
+
if m:
|
|
280
|
+
return f"{m.group(1)}-{m.group(2).zfill(2)}-{m.group(3).zfill(2) if m.group(3) else '01'}"
|
|
281
|
+
if re.match(r"^\d{4}$", str(v)):
|
|
282
|
+
return f"{v}-01-01"
|
|
283
|
+
return None
|
|
284
|
+
|
|
285
|
+
|
|
286
|
+
def _count_plain(hay, needle):
|
|
287
|
+
if not needle:
|
|
288
|
+
return 0
|
|
289
|
+
return hay.count(needle)
|
|
290
|
+
|
|
291
|
+
|
|
292
|
+
def _count_word(hay, term):
|
|
293
|
+
return len(re.findall(rf"(?<![a-z0-9]){re.escape(term)}(?![a-z0-9])", hay))
|
|
294
|
+
|
|
295
|
+
|
|
296
|
+
def _fuzzy_hits(doc, term):
|
|
297
|
+
max_d = 1 if len(term) <= 5 else 2
|
|
298
|
+
hits = []
|
|
299
|
+
for w in doc["w"]:
|
|
300
|
+
if w == term or w.startswith(term):
|
|
301
|
+
hits.append(w)
|
|
302
|
+
elif abs(len(w) - len(term)) <= max_d and edit_distance(term, w, max_d) <= max_d:
|
|
303
|
+
hits.append(w)
|
|
304
|
+
return hits
|
|
305
|
+
|
|
306
|
+
|
|
307
|
+
MIN_FRAGMENT = 12
|
|
308
|
+
|
|
309
|
+
|
|
310
|
+
def fragment_matches(anchor, title, body):
|
|
311
|
+
"""Would a browser text directive for this phrase actually find it?
|
|
312
|
+
|
|
313
|
+
The phrase must occur inside a single block, so the comparison is line by line.
|
|
314
|
+
"""
|
|
315
|
+
a = norm_keep(anchor or "").strip()
|
|
316
|
+
if not a or len(a) < 8:
|
|
317
|
+
return False
|
|
318
|
+
lines = norm_keep(f"{title or ''}\n{body or ''}").split("\n")
|
|
319
|
+
if any(a in ln for ln in lines):
|
|
320
|
+
return True
|
|
321
|
+
words = a.split(" ")
|
|
322
|
+
return len(words) > 2 and any(" ".join(words[:3]) in ln for ln in lines)
|
|
323
|
+
|
|
324
|
+
|
|
325
|
+
def anchor_for(title, at, body):
|
|
326
|
+
"""Longest verified phrase at the match: whole words, max 8, inside one block.
|
|
327
|
+
|
|
328
|
+
The offset is moved back to the start of its word, then forward to the next word,
|
|
329
|
+
because a text directive that starts inside a word never matches.
|
|
330
|
+
"""
|
|
331
|
+
b = body or ""
|
|
332
|
+
is_w = lambda ch: bool(ch) and (ch.isalnum() or ch in "åäöé")
|
|
333
|
+
i = max(0, min(at or 0, len(b) - 1)) if b else 0
|
|
334
|
+
while i > 0 and is_w(b[i - 1]) and is_w(b[i]):
|
|
335
|
+
i -= 1
|
|
336
|
+
while i < len(b) and not is_w(b[i]):
|
|
337
|
+
i += 1
|
|
338
|
+
|
|
339
|
+
# work inside the block that contains the match
|
|
340
|
+
line_start = b.rfind("\n", 0, max(0, i - 1)) + 1
|
|
341
|
+
line_end = b.find("\n", i)
|
|
342
|
+
if line_end < 0:
|
|
343
|
+
line_end = len(b)
|
|
344
|
+
line = re.sub(r"[ \t\u00a0]+", " ", b[line_start:line_end])
|
|
345
|
+
|
|
346
|
+
words = [(m.group(0), m.start()) for m in re.finditer(r"[0-9a-zA-ZåäöÅÄÖé]+", line)]
|
|
347
|
+
if not words:
|
|
348
|
+
return ""
|
|
349
|
+
k = next((n for n, (w, pos) in enumerate(words) if pos + len(w) > i - line_start), 0)
|
|
350
|
+
|
|
351
|
+
for start in range(max(0, k - 3), k + 1):
|
|
352
|
+
for n in range(min(8, len(words) - start), 0, -1):
|
|
353
|
+
end = words[start + n - 1][1] + len(words[start + n - 1][0])
|
|
354
|
+
cand = line[words[start][1]:end].strip()
|
|
355
|
+
if len(cand) >= MIN_FRAGMENT and fragment_matches(cand, title, b):
|
|
356
|
+
return cand
|
|
357
|
+
return words[k][0] if fragment_matches(words[k][0], title, b) else ""
|
|
358
|
+
|
|
359
|
+
|
|
360
|
+
def anchor_target(doc, raw, regex=False, fuzzy=False, case_sensitive=False):
|
|
361
|
+
body = doc.get("body") or ""
|
|
362
|
+
seed = doc.get("anchor") or ""
|
|
363
|
+
if not regex and not fuzzy:
|
|
364
|
+
terms = sorted(parse_query(raw, case_sensitive)["terms"], key=len, reverse=True)
|
|
365
|
+
if terms:
|
|
366
|
+
seed = terms[0]
|
|
367
|
+
at = doc.get("at") or 0
|
|
368
|
+
if seed:
|
|
369
|
+
# offsets must be measured in the original body: fold() collapses whitespace and shifts them
|
|
370
|
+
pos = body.lower().find(str(seed).lower())
|
|
371
|
+
if pos >= 0:
|
|
372
|
+
at = pos
|
|
373
|
+
return anchor_for(doc.get("title"), at, body)
|
|
374
|
+
|
|
375
|
+
|
|
376
|
+
def _match(doc, q, case_sensitive=False, fuzzy=False, whole_word=False):
|
|
377
|
+
if q["types"] and doc["type"] not in q["types"]:
|
|
378
|
+
return None
|
|
379
|
+
if q["exts"] and doc["ext"] not in q["exts"]:
|
|
380
|
+
return None
|
|
381
|
+
if q["before"] or q["after"]:
|
|
382
|
+
d = doc["date"][:10]
|
|
383
|
+
if not d:
|
|
384
|
+
return None
|
|
385
|
+
if q["before"] and not d < q["before"]:
|
|
386
|
+
return None
|
|
387
|
+
if q["after"] and not d > q["after"]:
|
|
388
|
+
return None
|
|
389
|
+
|
|
390
|
+
if q["regex"]:
|
|
391
|
+
hay = f"{doc['title']}\n{doc['body']}"
|
|
392
|
+
m = list(q["regex"].finditer(hay))
|
|
393
|
+
if not m:
|
|
394
|
+
return None
|
|
395
|
+
return {"score": len(m) + 5, "at": m[0].start(), "anchor": m[0].group(0)}
|
|
396
|
+
|
|
397
|
+
prep = (lambda s: (s or "")) if case_sensitive else fold
|
|
398
|
+
hay_src, title_src = f"{doc['title']}\n{doc['body']}", doc["title"]
|
|
399
|
+
hay, title = prep(hay_src), prep(title_src)
|
|
400
|
+
if any(prep(n) in hay for n in q["not"]):
|
|
401
|
+
return None
|
|
402
|
+
if any(prep(t) not in title for t in q["title"]):
|
|
403
|
+
return None
|
|
404
|
+
|
|
405
|
+
score, at, best = 0, -1, ""
|
|
406
|
+
for term in q["terms"]:
|
|
407
|
+
t = prep(term)
|
|
408
|
+
if fuzzy:
|
|
409
|
+
fw = _fuzzy_hits(doc, t)
|
|
410
|
+
c, anchor = len(fw), (fw[0] if fw else t)
|
|
411
|
+
elif whole_word or len(t) <= 3:
|
|
412
|
+
c, anchor = _count_word(hay, t), t
|
|
413
|
+
else:
|
|
414
|
+
c, anchor = _count_plain(hay, t), t
|
|
415
|
+
if not c:
|
|
416
|
+
return None
|
|
417
|
+
if at < 0:
|
|
418
|
+
# offsets must be measured in the original text: fold() collapses whitespace,
|
|
419
|
+
# so a folded offset drifts and page numbers and snippets land in the wrong place
|
|
420
|
+
# A phrase may match only after folding (a line break inside it), so locate its
|
|
421
|
+
# leading words with whitespace tolerance. Whole phrase first, then fewer words,
|
|
422
|
+
# so "sida 100 text" is not located at the first "sida" on page 1.
|
|
423
|
+
|
|
424
|
+
hay_norm = hay_src if case_sensitive else norm_len(hay_src)
|
|
425
|
+
target = anchor if case_sensitive else norm_len(anchor)
|
|
426
|
+
words = [w for w in re.split(r"\s+", target) if w][:3]
|
|
427
|
+
at = 0
|
|
428
|
+
for n in range(len(words), 0, -1):
|
|
429
|
+
m = re.search(r"\s+".join(re.escape(w) for w in words[:n]), hay_norm)
|
|
430
|
+
if m:
|
|
431
|
+
at = m.start()
|
|
432
|
+
break
|
|
433
|
+
best = best if len(best) > len(anchor) else anchor
|
|
434
|
+
score += (12 if anchor in title else 0) + min(c, 6)
|
|
435
|
+
if doc["type"] in ("anvisning", "sida"):
|
|
436
|
+
score += 3
|
|
437
|
+
return {"score": score, "at": at, "anchor": best}
|
|
438
|
+
|
|
439
|
+
|
|
440
|
+
def run(index, raw, case_sensitive=False, regex=False, fuzzy=False, whole_word=False, limit=40):
|
|
441
|
+
"""Search an index. Returns {'hits': [...], 'total': n} or {'error': '...'}."""
|
|
442
|
+
alts = [raw] if regex else (re.split(r"\s+OR\s+", raw, flags=re.I) if re.search(r"\s+OR\s+", raw, flags=re.I) else [raw])
|
|
443
|
+
found, constrained = {}, False
|
|
444
|
+
for alt in alts:
|
|
445
|
+
q = parse_query(alt, case_sensitive, regex)
|
|
446
|
+
if q["error"]:
|
|
447
|
+
return {"error": q["error"], "hits": [], "total": 0}
|
|
448
|
+
if not (q["terms"] or q["regex"] or q["title"] or q["types"] or q["exts"] or q["before"] or q["after"]):
|
|
449
|
+
continue
|
|
450
|
+
constrained = True
|
|
451
|
+
for doc in index["docs"]:
|
|
452
|
+
r = _match(doc, q, case_sensitive, fuzzy, whole_word)
|
|
453
|
+
if not r:
|
|
454
|
+
continue
|
|
455
|
+
key = (doc["href"], doc["title"])
|
|
456
|
+
if key not in found or r["score"] > found[key]["score"]:
|
|
457
|
+
found[key] = {**doc, "score": r["score"], "at": r["at"]}
|
|
458
|
+
cur = found[key]
|
|
459
|
+
if r.get("anchor") and len(r["anchor"]) > len(cur.get("anchor") or ""):
|
|
460
|
+
cur["anchor"] = r["anchor"]
|
|
461
|
+
if not constrained:
|
|
462
|
+
return {"hits": [], "total": 0, "empty": True}
|
|
463
|
+
hits = sorted(found.values(), key=lambda d: -d["score"])[:limit]
|
|
464
|
+
for h in hits:
|
|
465
|
+
h["snippet"] = snippet(h, h["at"])
|
|
466
|
+
h["anchor"] = anchor_target(h, raw, regex=regex, fuzzy=fuzzy, case_sensitive=case_sensitive)
|
|
467
|
+
# an unverifiable anchor yields the bare link rather than a dead fragment
|
|
468
|
+
h["url"] = deep_link(h["href"], h["anchor"])
|
|
469
|
+
return {"hits": hits, "total": len(found)}
|
|
470
|
+
|
|
471
|
+
|
|
472
|
+
def suggest(index, raw):
|
|
473
|
+
q = parse_query(raw)
|
|
474
|
+
if not q["terms"]:
|
|
475
|
+
return None
|
|
476
|
+
term = fold(q["terms"][0])
|
|
477
|
+
if len(term) < 4:
|
|
478
|
+
return None
|
|
479
|
+
best = None
|
|
480
|
+
seen = set()
|
|
481
|
+
for doc in index["docs"]:
|
|
482
|
+
for w in doc["w"]:
|
|
483
|
+
if w in seen:
|
|
484
|
+
continue
|
|
485
|
+
seen.add(w)
|
|
486
|
+
if len(w) < 4:
|
|
487
|
+
continue
|
|
488
|
+
d = edit_distance(term, w, 2)
|
|
489
|
+
if d == 0 or d > 2:
|
|
490
|
+
continue
|
|
491
|
+
if best is None or d < best[0] or (d == best[0] and len(w) < len(best[1])):
|
|
492
|
+
best = (d, w)
|
|
493
|
+
return best[1] if best else None
|
|
494
|
+
|
|
495
|
+
|
|
496
|
+
def snippet(doc, at, width=230):
|
|
497
|
+
body = doc["body"] if len(doc["body"] or "") > 20 else doc["title"]
|
|
498
|
+
start = max(0, (at or 0) - width // 3)
|
|
499
|
+
return ("…" if start else "") + re.sub(r"\s+", " ", body[start:start + width])
|
|
500
|
+
|
|
501
|
+
|
|
502
|
+
def deep_link(href, phrase):
|
|
503
|
+
if not href or not phrase:
|
|
504
|
+
return href
|
|
505
|
+
clean = re.sub(r"[\r\n]+", " ", phrase).strip()[:120]
|
|
506
|
+
enc = urllib.parse.quote(clean, safe="").replace("-", "%2D").replace(",", "%2C")
|
|
507
|
+
return f"{href}{'&' if '#' in href else '#:~:'}text={enc}"
|
unitoolkit/serve.py
ADDED
|
@@ -0,0 +1,153 @@
|
|
|
1
|
+
"""Local HTTP endpoint for the deep index, for the browser plugin and for other tools.
|
|
2
|
+
|
|
3
|
+
The plugin is the fast path (in-browser index of pages, assignments, rubrics, announcements,
|
|
4
|
+
module items). This server is the deep path: document text, OCR results, mirrors, the whole
|
|
5
|
+
programme. Running it lets the plugin merge both layers without bundling a PDF parser into a
|
|
6
|
+
Chrome extension, and without the plugin ever needing a Canvas token.
|
|
7
|
+
|
|
8
|
+
Binds loopback only. Reach it from another machine with `tailscale serve`:
|
|
9
|
+
|
|
10
|
+
python -m unitoolkit serve # 127.0.0.1:8790
|
|
11
|
+
tailscale serve --bg 8790 # https://<host>.<tailnet>.ts.net/
|
|
12
|
+
|
|
13
|
+
Endpoints (all GET, JSON):
|
|
14
|
+
/health service state, cached document count
|
|
15
|
+
/courses the student's courses
|
|
16
|
+
/search?course=&q=&limit= course search; add docs=1 to include document text
|
|
17
|
+
/docs?course=&q= document text only, with page numbers
|
|
18
|
+
/index what is cached per course
|
|
19
|
+
"""
|
|
20
|
+
import json
|
|
21
|
+
import pathlib
|
|
22
|
+
import urllib.parse
|
|
23
|
+
from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
|
|
24
|
+
|
|
25
|
+
from . import docs as docs_mod
|
|
26
|
+
from .tools import TOOLS, call
|
|
27
|
+
|
|
28
|
+
PORT = 8791 # 8790 is the agentrecourse site preview on this machine
|
|
29
|
+
ALLOWED_PREFIXES = ("https://shh.instructure.com",
|
|
30
|
+
"https://cloud.timeedit.net",
|
|
31
|
+
"http://127.0.0.1", "http://localhost")
|
|
32
|
+
|
|
33
|
+
|
|
34
|
+
def _cached_index():
|
|
35
|
+
root = docs_mod.CACHE
|
|
36
|
+
out = {}
|
|
37
|
+
if not root.exists():
|
|
38
|
+
return out
|
|
39
|
+
for d in sorted(root.iterdir()):
|
|
40
|
+
if d.is_dir():
|
|
41
|
+
files = list(d.glob("*.json"))
|
|
42
|
+
chars = 0
|
|
43
|
+
for f in files:
|
|
44
|
+
try:
|
|
45
|
+
chars += len((json.loads(f.read_text()).get("text") or ""))
|
|
46
|
+
except Exception: # noqa: BLE001
|
|
47
|
+
pass
|
|
48
|
+
out[d.name] = {"documents": len(files), "chars": chars}
|
|
49
|
+
return out
|
|
50
|
+
|
|
51
|
+
|
|
52
|
+
class Handler(BaseHTTPRequestHandler):
|
|
53
|
+
server_version = "unitoolkit"
|
|
54
|
+
protocol_version = "HTTP/1.1"
|
|
55
|
+
|
|
56
|
+
def log_message(self, format, *args): # noqa: A002 - matches the base signature
|
|
57
|
+
pass
|
|
58
|
+
|
|
59
|
+
def _cors(self):
|
|
60
|
+
origin = self.headers.get("Origin") or ""
|
|
61
|
+
if origin.startswith(ALLOWED_PREFIXES):
|
|
62
|
+
self.send_header("Access-Control-Allow-Origin", origin)
|
|
63
|
+
self.send_header("Vary", "Origin")
|
|
64
|
+
self.send_header("Access-Control-Allow-Methods", "GET, OPTIONS")
|
|
65
|
+
self.send_header("Access-Control-Allow-Headers", "content-type")
|
|
66
|
+
self.send_header("Access-Control-Max-Age", "600")
|
|
67
|
+
# Chrome refuses a loopback fetch from an https page (Private Network Access) unless
|
|
68
|
+
# the server answers the preflight with this header.
|
|
69
|
+
self.send_header("Access-Control-Allow-Private-Network", "true")
|
|
70
|
+
|
|
71
|
+
def _json(self, obj, code=200):
|
|
72
|
+
body = json.dumps(obj, ensure_ascii=False, default=str).encode("utf-8")
|
|
73
|
+
self.send_response(code)
|
|
74
|
+
self.send_header("Content-Type", "application/json; charset=utf-8")
|
|
75
|
+
self._cors()
|
|
76
|
+
self.send_header("Content-Length", str(len(body)))
|
|
77
|
+
self.end_headers()
|
|
78
|
+
self.wfile.write(body)
|
|
79
|
+
|
|
80
|
+
def do_OPTIONS(self): # noqa: N802 - http.server API
|
|
81
|
+
self.send_response(204)
|
|
82
|
+
self._cors()
|
|
83
|
+
self.send_header("Content-Length", "0")
|
|
84
|
+
self.end_headers()
|
|
85
|
+
|
|
86
|
+
def do_GET(self): # noqa: N802 - http.server API
|
|
87
|
+
u = urllib.parse.urlparse(self.path)
|
|
88
|
+
q = {k: v for k, v in urllib.parse.parse_qsl(u.query)}
|
|
89
|
+
try:
|
|
90
|
+
if u.path in ("/", "/health"):
|
|
91
|
+
return self._json({"ok": True, "service": "unitoolkit", "version": _version(),
|
|
92
|
+
"tools": len(TOOLS), "cached": _cached_index()})
|
|
93
|
+
if u.path == "/courses":
|
|
94
|
+
return self._json(call("courses", {}))
|
|
95
|
+
if u.path == "/index":
|
|
96
|
+
return self._json(_cached_index())
|
|
97
|
+
if u.path == "/search":
|
|
98
|
+
course = q.get("course")
|
|
99
|
+
query = q.get("q") or q.get("query") or ""
|
|
100
|
+
if not course:
|
|
101
|
+
return self._json({"error": "course is required"}, 400)
|
|
102
|
+
args = {"course": str(course), "query": query,
|
|
103
|
+
"limit": int(q.get("limit") or 25),
|
|
104
|
+
"include_docs": (q.get("docs") or "") in ("1", "true", "yes"),
|
|
105
|
+
"fuzzy": (q.get("fuzzy") or "") in ("1", "true"),
|
|
106
|
+
"regex": (q.get("regex") or "") in ("1", "true")}
|
|
107
|
+
if q.get("source"):
|
|
108
|
+
args["source"] = q["source"]
|
|
109
|
+
return self._json(call("search", args))
|
|
110
|
+
if u.path == "/docs":
|
|
111
|
+
course = q.get("course")
|
|
112
|
+
if not course:
|
|
113
|
+
return self._json({"error": "course is required"}, 400)
|
|
114
|
+
args = {"course": str(course), "query": q.get("q") or "",
|
|
115
|
+
"include_docs": True, "limit": int(q.get("limit") or 25)}
|
|
116
|
+
if q.get("source"):
|
|
117
|
+
args["source"] = q["source"]
|
|
118
|
+
return self._json(call("search", args))
|
|
119
|
+
return self._json({"error": "unknown path", "paths": ["/health", "/courses",
|
|
120
|
+
"/search", "/docs", "/index"]}, 404)
|
|
121
|
+
except Exception as e: # noqa: BLE001 - a server must answer, not crash
|
|
122
|
+
return self._json({"error": f"{type(e).__name__}: {e}"}, 500)
|
|
123
|
+
|
|
124
|
+
|
|
125
|
+
def _version():
|
|
126
|
+
try:
|
|
127
|
+
from . import __version__
|
|
128
|
+
return __version__
|
|
129
|
+
except Exception: # noqa: BLE001
|
|
130
|
+
return "?"
|
|
131
|
+
|
|
132
|
+
|
|
133
|
+
def serve(host="127.0.0.1", port=PORT):
|
|
134
|
+
httpd = None
|
|
135
|
+
last = None
|
|
136
|
+
for candidate in range(port, port + 6):
|
|
137
|
+
try:
|
|
138
|
+
httpd = ThreadingHTTPServer((host, candidate), Handler)
|
|
139
|
+
port = candidate
|
|
140
|
+
break
|
|
141
|
+
except OSError as e: # port taken by something else: try the next one
|
|
142
|
+
last = e
|
|
143
|
+
if httpd is None:
|
|
144
|
+
raise SystemExit(f"no free port in {port}-{port + 5}: {last}")
|
|
145
|
+
print(f"unitoolkit serving on http://{host}:{port}", flush=True)
|
|
146
|
+
print(" tailscale serve --bg " + str(port) + " # then https://<machine>.<tailnet>.ts.net/",
|
|
147
|
+
flush=True)
|
|
148
|
+
try:
|
|
149
|
+
httpd.serve_forever()
|
|
150
|
+
except KeyboardInterrupt:
|
|
151
|
+
print("\nstopped", flush=True)
|
|
152
|
+
finally:
|
|
153
|
+
httpd.server_close()
|