unitoolkit 0.7.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- ladok_vakt/__init__.py +7 -0
- ladok_vakt/__main__.py +51 -0
- ladok_vakt/core.py +553 -0
- ladok_vakt/login.py +232 -0
- tools.json +623 -0
- unitoolkit/__init__.py +1 -0
- unitoolkit/__main__.py +63 -0
- unitoolkit/agent.py +270 -0
- unitoolkit/browser.py +147 -0
- unitoolkit/canvas.py +189 -0
- unitoolkit/chainproxy.py +113 -0
- unitoolkit/citation.py +165 -0
- unitoolkit/databases.py +497 -0
- unitoolkit/docs.py +394 -0
- unitoolkit/fulltext.py +260 -0
- unitoolkit/koha.py +729 -0
- unitoolkit/mcp.py +85 -0
- unitoolkit/programme.py +147 -0
- unitoolkit/search.py +507 -0
- unitoolkit/serve.py +153 -0
- unitoolkit/sharepoint.py +86 -0
- unitoolkit/surface.py +86 -0
- unitoolkit/timeedit.py +209 -0
- unitoolkit/tools.py +555 -0
- unitoolkit-0.7.0.dist-info/METADATA +177 -0
- unitoolkit-0.7.0.dist-info/RECORD +30 -0
- unitoolkit-0.7.0.dist-info/WHEEL +5 -0
- unitoolkit-0.7.0.dist-info/entry_points.txt +4 -0
- unitoolkit-0.7.0.dist-info/licenses/LICENSE +21 -0
- unitoolkit-0.7.0.dist-info/top_level.txt +2 -0
unitoolkit/agent.py
ADDED
|
@@ -0,0 +1,270 @@
|
|
|
1
|
+
"""The agent-facing surface: a few tools shaped like the questions a student asks.
|
|
2
|
+
|
|
3
|
+
The full registry has 30-odd tools, and every tool description sits in the agent's context
|
|
4
|
+
on every turn. Most requests are one of a handful of questions, so the MCP server lists only
|
|
5
|
+
CORE by default and the rest stay one environment variable away (UNITOOLKIT_TOOLS=all).
|
|
6
|
+
Every tool stays callable by name either way, and all of them stay importable as a library.
|
|
7
|
+
|
|
8
|
+
Design rules for this layer:
|
|
9
|
+
- One question, one call. `today` answers "what do I need to do" across Canvas, Ladok, the
|
|
10
|
+
library and the timetable, so the agent does not have to know which system holds what.
|
|
11
|
+
- Partial answers beat no answer. A dead Ladok session must not hide the Canvas deadlines,
|
|
12
|
+
so each source reports its own error next to the others' results.
|
|
13
|
+
- `doctor` names the fix, not just the fault, because the fix is always a human action.
|
|
14
|
+
"""
|
|
15
|
+
|
|
16
|
+
import datetime as dt
|
|
17
|
+
import os
|
|
18
|
+
import time
|
|
19
|
+
|
|
20
|
+
CORE = ('today', 'doctor', 'find_sources', 'db_fulltext', 'search', 'assignment', 'library',
|
|
21
|
+
'schedule', 'free_rooms', 'announcements', 'discussions')
|
|
22
|
+
|
|
23
|
+
|
|
24
|
+
def listed(all_tools):
|
|
25
|
+
"""Names the MCP server should list."""
|
|
26
|
+
mode = os.environ.get('UNITOOLKIT_TOOLS', 'core').strip().lower()
|
|
27
|
+
if mode == 'all':
|
|
28
|
+
return list(all_tools)
|
|
29
|
+
if mode and mode != 'core': # explicit comma list
|
|
30
|
+
return [n for n in (x.strip() for x in mode.split(',')) if n in all_tools]
|
|
31
|
+
return [n for n in CORE if n in all_tools]
|
|
32
|
+
|
|
33
|
+
|
|
34
|
+
def _days(date_str, today=None):
|
|
35
|
+
if not date_str:
|
|
36
|
+
return None
|
|
37
|
+
try:
|
|
38
|
+
d = dt.date.fromisoformat(str(date_str)[:10])
|
|
39
|
+
except ValueError:
|
|
40
|
+
return None
|
|
41
|
+
return (d - (today or dt.date.today())).days
|
|
42
|
+
|
|
43
|
+
|
|
44
|
+
def _guard(fn):
|
|
45
|
+
t0 = time.time()
|
|
46
|
+
try:
|
|
47
|
+
return {'ok': True, 'data': fn(), 'seconds': round(time.time() - t0, 1)}
|
|
48
|
+
except BaseException as e: # SystemExit from vendored code too
|
|
49
|
+
return {'ok': False, 'error': '%s: %s' % (type(e).__name__, str(e)[:240]),
|
|
50
|
+
'seconds': round(time.time() - t0, 1)}
|
|
51
|
+
|
|
52
|
+
|
|
53
|
+
# ------------------------------------------------------------------------- today
|
|
54
|
+
|
|
55
|
+
def _canvas_due(days):
|
|
56
|
+
from .tools import call
|
|
57
|
+
out = []
|
|
58
|
+
for item in call('upcoming'):
|
|
59
|
+
left = _days(item.get('due_utc'))
|
|
60
|
+
if item.get('submitted') or left is None or left > days:
|
|
61
|
+
continue
|
|
62
|
+
out.append({'due_utc': item['due_utc'], 'days_left': left, 'course': item.get('course'),
|
|
63
|
+
'title': item.get('title'), 'type': item.get('type')})
|
|
64
|
+
return sorted(out, key=lambda r: r['due_utc'])
|
|
65
|
+
|
|
66
|
+
|
|
67
|
+
def _canvas_announcements(days):
|
|
68
|
+
from .tools import call
|
|
69
|
+
out = []
|
|
70
|
+
for a in call('announcements', {'days': days}):
|
|
71
|
+
out.append({'posted_utc': a.get('posted_utc'), 'days_ago': abs(_days(a.get('posted_utc')) or 0),
|
|
72
|
+
'course_id': a.get('course_id'), 'title': a.get('title')})
|
|
73
|
+
return out
|
|
74
|
+
|
|
75
|
+
|
|
76
|
+
def _canvas_discussions(days):
|
|
77
|
+
from .tools import call
|
|
78
|
+
out = []
|
|
79
|
+
for d in call('discussions', {'days': days, 'unread_only': True}):
|
|
80
|
+
out.append({'last_reply_utc': d.get('last_reply_utc'), 'course_id': d.get('course_id'),
|
|
81
|
+
'title': d.get('title'), 'entries': d.get('entries')})
|
|
82
|
+
return out
|
|
83
|
+
|
|
84
|
+
|
|
85
|
+
def _ladok_windows(days):
|
|
86
|
+
from .tools import call
|
|
87
|
+
res = call('ladok_windows', {})
|
|
88
|
+
if 'error' in res:
|
|
89
|
+
raise RuntimeError(res['error'] + ' (' + res.get('hint', '') + ')')
|
|
90
|
+
out = []
|
|
91
|
+
for r in res.get('windows') or []:
|
|
92
|
+
opens, closes = _days(r.get('reg_opens')), _days(r.get('reg_closes'))
|
|
93
|
+
open_now = opens is not None and opens <= 0 and (closes is None or closes >= 0)
|
|
94
|
+
soon = opens is not None and 0 < opens <= days
|
|
95
|
+
if open_now or soon:
|
|
96
|
+
out.append({'course': r.get('course'), 'code': r.get('code'), 'kind': r.get('kind'),
|
|
97
|
+
'opens': r.get('reg_opens'), 'closes': r.get('reg_closes'),
|
|
98
|
+
'open_now': open_now, 'days_to_close': closes, 'days_to_open': opens})
|
|
99
|
+
return out
|
|
100
|
+
|
|
101
|
+
|
|
102
|
+
def _loans_due(days):
|
|
103
|
+
from . import koha
|
|
104
|
+
acc = koha.loans()
|
|
105
|
+
return [{'title': c['title'], 'due': c['due'], 'days_left': c['days_left']}
|
|
106
|
+
for c in acc.get('checkouts') or []
|
|
107
|
+
if c.get('days_left') is not None and c['days_left'] <= days]
|
|
108
|
+
|
|
109
|
+
|
|
110
|
+
def _schedule(days):
|
|
111
|
+
group = os.environ.get('UNITOOLKIT_GROUP', '').strip()
|
|
112
|
+
if group: # the whole cohort, any period, no login
|
|
113
|
+
from . import timeedit
|
|
114
|
+
end = (dt.date.today() + dt.timedelta(days=days)).isoformat()
|
|
115
|
+
return timeedit.schedule(group, 'group', dt.date.today().isoformat(), end)['bookings']
|
|
116
|
+
url = os.environ.get('UNITOOLKIT_TIMEEDIT_URL', '').strip()
|
|
117
|
+
if not url:
|
|
118
|
+
raise RuntimeError('no timetable configured: set UNITOOLKIT_GROUP to your student group '
|
|
119
|
+
'(e.g. H26SK), or UNITOOLKIT_TIMEEDIT_URL to a saved TimeEdit link')
|
|
120
|
+
from .tools import call
|
|
121
|
+
today = dt.date.today()
|
|
122
|
+
return [b for b in call('timeedit', {'url': url})
|
|
123
|
+
if (_days(b.get('date'), today) or 0) >= 0 and (_days(b.get('date'), today) or 0) <= days]
|
|
124
|
+
|
|
125
|
+
|
|
126
|
+
def today(days=14, library=False):
|
|
127
|
+
"""What needs doing in the next `days` days, from every system at once.
|
|
128
|
+
|
|
129
|
+
library=False by default: the Koha login goes through a residential proxy, takes up to
|
|
130
|
+
two minutes when it fails, and repeated failed logins risk locking the student's library
|
|
131
|
+
account. Ask for it explicitly.
|
|
132
|
+
"""
|
|
133
|
+
days = int(days or 14)
|
|
134
|
+
out = {'date': dt.date.today().isoformat(), 'window_days': days,
|
|
135
|
+
'canvas': _guard(lambda: _canvas_due(days)),
|
|
136
|
+
'announcements': _guard(lambda: _canvas_announcements(days)),
|
|
137
|
+
'discussions': _guard(lambda: _canvas_discussions(days)),
|
|
138
|
+
'ladok': _guard(lambda: _ladok_windows(days)),
|
|
139
|
+
'schedule': _guard(lambda: _schedule(min(days, 7)))}
|
|
140
|
+
if library:
|
|
141
|
+
out['library'] = _guard(lambda: _loans_due(days))
|
|
142
|
+
out['problems'] = [k for k, v in out.items() if isinstance(v, dict) and v.get('ok') is False]
|
|
143
|
+
if out['problems']:
|
|
144
|
+
out['hint'] = 'run doctor for the fix; the other sections are still valid'
|
|
145
|
+
return out
|
|
146
|
+
|
|
147
|
+
|
|
148
|
+
# ------------------------------------------------------------------------ doctor
|
|
149
|
+
|
|
150
|
+
def _check_canvas():
|
|
151
|
+
from . import canvas
|
|
152
|
+
tok = canvas.token()
|
|
153
|
+
if not tok:
|
|
154
|
+
return 'fail', 'no Canvas token', 'Canvas > Account > Settings > New access token, save it to ~/.secrets/CANVAS_TOKEN'
|
|
155
|
+
me = canvas.get('users/self/profile')
|
|
156
|
+
expiry = None
|
|
157
|
+
try:
|
|
158
|
+
for t in canvas.get('users/self/user_generated_tokens') or []:
|
|
159
|
+
prefix = (t.get('visible_token') or '').rstrip('.')
|
|
160
|
+
if prefix and tok.startswith(prefix):
|
|
161
|
+
expiry = t.get('expires_at')
|
|
162
|
+
except Exception:
|
|
163
|
+
pass
|
|
164
|
+
left = _days(expiry) if expiry else None
|
|
165
|
+
detail = 'signed in as %s' % me.get('name', '?') + (
|
|
166
|
+
', token expires %s (%s days)' % (expiry[:10], left) if expiry else ', token expiry unknown')
|
|
167
|
+
if left is not None and left <= 14:
|
|
168
|
+
return 'warn', detail, 'make a new token before it expires: Canvas > Account > Settings'
|
|
169
|
+
return 'ok', detail, ''
|
|
170
|
+
|
|
171
|
+
|
|
172
|
+
def _check_browser():
|
|
173
|
+
from . import browser
|
|
174
|
+
try:
|
|
175
|
+
n = len([t for t in browser.targets() if t.get('type') == 'page'])
|
|
176
|
+
except Exception:
|
|
177
|
+
return 'fail', 'agent browser not running on CDP %s' % os.environ.get('UNITOOLKIT_CDP', ':9233'), \
|
|
178
|
+
'start the agent Chrome with --remote-debugging-port=9233 (see skill/SKILL.md, Setup)'
|
|
179
|
+
return ('warn' if n > 15 else 'ok'), '%d open tabs' % n, ('close stray tabs' if n > 15 else '')
|
|
180
|
+
|
|
181
|
+
|
|
182
|
+
def _check_ladok():
|
|
183
|
+
from .tools import call
|
|
184
|
+
res = call('ladok_windows', {})
|
|
185
|
+
if 'error' in res:
|
|
186
|
+
return 'fail', res['error'][:160], 'sign in to Ladok once in the agent browser'
|
|
187
|
+
rows = [r for r in res.get('windows') or [] if isinstance(r, dict)] # rows, not text lines
|
|
188
|
+
return 'ok', '%d registration windows readable' % len(rows), ''
|
|
189
|
+
|
|
190
|
+
|
|
191
|
+
def _check_databases():
|
|
192
|
+
from . import databases
|
|
193
|
+
st = databases.status('cinahl')
|
|
194
|
+
if not st.get('authenticated'):
|
|
195
|
+
return 'fail', 'EZproxy sends us to the Microsoft sign-in', \
|
|
196
|
+
'sign in once at https://shh.idm.oclc.org in the agent browser (Authenticator)'
|
|
197
|
+
return 'ok', 'EZproxy admits us (%s)' % (st.get('institution') or 'institution not shown'), ''
|
|
198
|
+
|
|
199
|
+
|
|
200
|
+
def _check_library():
|
|
201
|
+
from . import koha
|
|
202
|
+
st = koha.status()
|
|
203
|
+
if st.get('proxy_enabled', True) and not st.get('chain_up'):
|
|
204
|
+
return 'warn', 'library proxy not running on %s (starts on demand)' % st.get('chain_proxy'), \
|
|
205
|
+
'only needed where the library drops your IP; otherwise set KOHA_PROXY=0'
|
|
206
|
+
if not st.get('cdp_up'):
|
|
207
|
+
return 'warn', 'library browser is down (starts on demand)', ''
|
|
208
|
+
try:
|
|
209
|
+
user, pin = koha.credentials()
|
|
210
|
+
except Exception:
|
|
211
|
+
user = pin = None
|
|
212
|
+
if not (user and pin):
|
|
213
|
+
return 'warn', 'no library credentials on file', 'put card number and pinkod in ~/.secrets/KOHA_CREDENTIALS (two lines)'
|
|
214
|
+
return 'ok', 'proxy, browser and credentials in place (login not attempted)', ''
|
|
215
|
+
|
|
216
|
+
|
|
217
|
+
def _check_timeedit():
|
|
218
|
+
group = os.environ.get('UNITOOLKIT_GROUP', '').strip()
|
|
219
|
+
if group:
|
|
220
|
+
from . import timeedit
|
|
221
|
+
hits = timeedit.find(group, 'group')
|
|
222
|
+
if not hits:
|
|
223
|
+
return 'fail', 'TimeEdit has no group %r' % group, 'set UNITOOLKIT_GROUP to the group on your timetable (e.g. H26SK)'
|
|
224
|
+
return 'ok', 'group %s found (%d subgroups)' % (group, len(hits)), ''
|
|
225
|
+
url = os.environ.get('UNITOOLKIT_TIMEEDIT_URL', '').strip()
|
|
226
|
+
if not url:
|
|
227
|
+
return 'warn', 'no timetable configured', 'set UNITOOLKIT_GROUP to your student group (e.g. H26SK)'
|
|
228
|
+
from .tools import call
|
|
229
|
+
n = len(call('timeedit', {'url': url}))
|
|
230
|
+
return ('ok' if n else 'warn'), '%d bookings in the saved link' % n, (
|
|
231
|
+
'' if n else 'the link covers an old period: make a new one in TimeEdit')
|
|
232
|
+
|
|
233
|
+
|
|
234
|
+
CHECKS = [('canvas', _check_canvas), ('browser', _check_browser), ('ladok', _check_ladok),
|
|
235
|
+
('databases', _check_databases), ('library', _check_library),
|
|
236
|
+
('timeedit', _check_timeedit)]
|
|
237
|
+
|
|
238
|
+
|
|
239
|
+
def doctor(only=None):
|
|
240
|
+
"""Every session and credential the toolkit depends on, each with the human fix."""
|
|
241
|
+
wanted = {x.strip() for x in (only or '').split(',') if x.strip()}
|
|
242
|
+
rows = []
|
|
243
|
+
for name, fn in CHECKS:
|
|
244
|
+
if wanted and name not in wanted:
|
|
245
|
+
continue
|
|
246
|
+
t0 = time.time()
|
|
247
|
+
try:
|
|
248
|
+
status, detail, fix = fn()
|
|
249
|
+
except BaseException as e:
|
|
250
|
+
status, detail, fix = 'fail', '%s: %s' % (type(e).__name__, str(e)[:200]), \
|
|
251
|
+
'see the error; most often the agent browser is down or signed out'
|
|
252
|
+
rows.append({'check': name, 'status': status, 'detail': detail, 'fix': fix,
|
|
253
|
+
'seconds': round(time.time() - t0, 1)})
|
|
254
|
+
worst = 'fail' if any(r['status'] == 'fail' for r in rows) else (
|
|
255
|
+
'warn' if any(r['status'] == 'warn' for r in rows) else 'ok')
|
|
256
|
+
return {'overall': worst, 'checks': rows}
|
|
257
|
+
|
|
258
|
+
|
|
259
|
+
# ----------------------------------------------------------------------- library
|
|
260
|
+
|
|
261
|
+
def library(query=None, limit=10):
|
|
262
|
+
"""Catalogue search when given a query, otherwise the student's own loans."""
|
|
263
|
+
from . import koha
|
|
264
|
+
if query:
|
|
265
|
+
return {'mode': 'search', 'query': query, 'results': koha.search(query, limit=limit or 10)}
|
|
266
|
+
acc = koha.loans()
|
|
267
|
+
return {'mode': 'loans', 'loans': [
|
|
268
|
+
{'title': c['title'], 'author': c.get('author', ''), 'due': c['due'],
|
|
269
|
+
'days_left': c['days_left']} for c in acc.get('checkouts') or []],
|
|
270
|
+
'holds': acc.get('holds') or [], 'fines': acc.get('fines') or []}
|
unitoolkit/browser.py
ADDED
|
@@ -0,0 +1,147 @@
|
|
|
1
|
+
"""Logged-in browser access over Chrome DevTools Protocol.
|
|
2
|
+
|
|
3
|
+
For SSO-walled systems with no student API (SharePoint, Ladok, Inspera, library), run a
|
|
4
|
+
dedicated Chrome profile with remote debugging, log in once by hand, and let the agent
|
|
5
|
+
read pages through it. Never point this at the user's default profile (Chrome refuses,
|
|
6
|
+
and it should not be the agent's).
|
|
7
|
+
|
|
8
|
+
flatpak run com.google.Chrome --user-data-dir=~/.var/app/com.google.Chrome/config/unitoolkit \
|
|
9
|
+
--remote-debugging-port=9233
|
|
10
|
+
"""
|
|
11
|
+
import asyncio
|
|
12
|
+
import json
|
|
13
|
+
import os
|
|
14
|
+
import urllib.request
|
|
15
|
+
|
|
16
|
+
PORT = int(os.environ.get("SHHKIT_CDP_PORT", "9233"))
|
|
17
|
+
|
|
18
|
+
|
|
19
|
+
def targets():
|
|
20
|
+
with urllib.request.urlopen(f"http://127.0.0.1:{PORT}/json", timeout=5) as r:
|
|
21
|
+
return [t for t in json.load(r) if t.get("type") == "page"]
|
|
22
|
+
|
|
23
|
+
|
|
24
|
+
def new_tab(url="about:blank"):
|
|
25
|
+
req = urllib.request.Request(f"http://127.0.0.1:{PORT}/json/new?{url}", method="PUT")
|
|
26
|
+
with urllib.request.urlopen(req, timeout=10) as r:
|
|
27
|
+
return json.load(r)
|
|
28
|
+
|
|
29
|
+
|
|
30
|
+
def close_tab(tid):
|
|
31
|
+
urllib.request.urlopen(f"http://127.0.0.1:{PORT}/json/close/{tid}", timeout=5).read()
|
|
32
|
+
|
|
33
|
+
|
|
34
|
+
async def _session(ws_url, steps):
|
|
35
|
+
import websockets # only dependency outside stdlib
|
|
36
|
+
|
|
37
|
+
async with websockets.connect(ws_url, max_size=64 * 2**20) as ws:
|
|
38
|
+
n = 0
|
|
39
|
+
|
|
40
|
+
async def call(method, **params):
|
|
41
|
+
nonlocal n
|
|
42
|
+
n += 1
|
|
43
|
+
mid = n
|
|
44
|
+
await ws.send(json.dumps({"id": mid, "method": method, "params": params}))
|
|
45
|
+
while True:
|
|
46
|
+
msg = json.loads(await ws.recv())
|
|
47
|
+
if msg.get("id") == mid:
|
|
48
|
+
if "error" in msg:
|
|
49
|
+
raise RuntimeError(msg["error"])
|
|
50
|
+
return msg.get("result", {})
|
|
51
|
+
|
|
52
|
+
return await steps(call)
|
|
53
|
+
|
|
54
|
+
|
|
55
|
+
def _eval(call, expr):
|
|
56
|
+
async def run():
|
|
57
|
+
r = await call("Runtime.evaluate", expression=expr, awaitPromise=True, returnByValue=True)
|
|
58
|
+
return r.get("result", {}).get("value")
|
|
59
|
+
return run()
|
|
60
|
+
|
|
61
|
+
|
|
62
|
+
def page(url, wait=4.0, text=True):
|
|
63
|
+
"""Open url in a fresh tab of the logged-in profile, wait, return {url,title,text|html}."""
|
|
64
|
+
t = new_tab(url)
|
|
65
|
+
|
|
66
|
+
async def steps(call):
|
|
67
|
+
await call("Page.enable")
|
|
68
|
+
await call("Page.navigate", url=url)
|
|
69
|
+
for _ in range(int(wait * 4)):
|
|
70
|
+
await asyncio.sleep(0.25)
|
|
71
|
+
if await _eval(call, "document.readyState") == "complete" and (await _eval(call, "location.href")) != "about:blank":
|
|
72
|
+
break
|
|
73
|
+
await asyncio.sleep(1.0) # SPAs (SharePoint, Ladok) render after load
|
|
74
|
+
body = "document.body.innerText" if text else "document.documentElement.outerHTML"
|
|
75
|
+
return {"url": await _eval(call, "location.href"),
|
|
76
|
+
"title": await _eval(call, "document.title"),
|
|
77
|
+
("text" if text else "html"): await _eval(call, body)}
|
|
78
|
+
|
|
79
|
+
try:
|
|
80
|
+
return asyncio.run(_session(t["webSocketDebuggerUrl"], steps))
|
|
81
|
+
finally:
|
|
82
|
+
close_tab(t["id"])
|
|
83
|
+
|
|
84
|
+
|
|
85
|
+
def fetch(url, origin, as_json=True):
|
|
86
|
+
"""fetch() from inside a tab on `origin`, so the site's own cookies/SSO apply.
|
|
87
|
+
Use for internal JSON APIs (e.g. Ladok student, SharePoint REST)."""
|
|
88
|
+
t = new_tab(origin)
|
|
89
|
+
|
|
90
|
+
async def steps(call):
|
|
91
|
+
await call("Page.enable")
|
|
92
|
+
await asyncio.sleep(3)
|
|
93
|
+
js = (f"fetch({json.dumps(url)},{{credentials:'include',headers:{{Accept:'application/json'}}}})"
|
|
94
|
+
f".then(async r=>({{status:r.status,body:await r.text()}}))")
|
|
95
|
+
return await _eval(call, js)
|
|
96
|
+
|
|
97
|
+
try:
|
|
98
|
+
r = asyncio.run(_session(t["webSocketDebuggerUrl"], steps))
|
|
99
|
+
finally:
|
|
100
|
+
close_tab(t["id"])
|
|
101
|
+
if as_json and r and r.get("status") == 200:
|
|
102
|
+
try:
|
|
103
|
+
r["body"] = json.loads(r["body"])
|
|
104
|
+
except ValueError:
|
|
105
|
+
pass
|
|
106
|
+
return r
|
|
107
|
+
|
|
108
|
+
|
|
109
|
+
LOGIN_MARKERS = ("login.microsoftonline.com", "/login", "wayf", "discovery", "seamlessaccess")
|
|
110
|
+
|
|
111
|
+
|
|
112
|
+
def fetch_binary(url, origin):
|
|
113
|
+
"""Fetch bytes with the site's cookies. SharePoint serves binary only to a fetch inside
|
|
114
|
+
its own origin, so this goes through arrayBuffer -> base64 -> Python."""
|
|
115
|
+
t = new_tab(origin)
|
|
116
|
+
|
|
117
|
+
async def steps(call):
|
|
118
|
+
await call("Page.enable")
|
|
119
|
+
await asyncio.sleep(3)
|
|
120
|
+
js = (f"fetch({json.dumps(url)},{{credentials:'include'}}).then(async r=>{{"
|
|
121
|
+
"const b=new Uint8Array(await r.arrayBuffer());let s='';"
|
|
122
|
+
"for(let i=0;i<b.length;i+=8192)s+=String.fromCharCode.apply(null,b.subarray(i,i+8192));"
|
|
123
|
+
"return {status:r.status,type:r.headers.get('content-type'),"
|
|
124
|
+
"cd:r.headers.get('content-disposition'),b64:btoa(s)};})")
|
|
125
|
+
return await _eval(call, js)
|
|
126
|
+
|
|
127
|
+
try:
|
|
128
|
+
r = asyncio.run(_session(t["webSocketDebuggerUrl"], steps))
|
|
129
|
+
finally:
|
|
130
|
+
close_tab(t["id"])
|
|
131
|
+
if r and r.get("b64"):
|
|
132
|
+
import base64
|
|
133
|
+
r["bytes"] = base64.b64decode(r["b64"])
|
|
134
|
+
del r["b64"]
|
|
135
|
+
return r
|
|
136
|
+
|
|
137
|
+
|
|
138
|
+
def logged_in(url):
|
|
139
|
+
"""True when url renders without bouncing to an SSO login page."""
|
|
140
|
+
p = page(url, wait=6)
|
|
141
|
+
u = (p.get("url") or "").lower()
|
|
142
|
+
head = (p.get("text") or "")[:1500].lower()
|
|
143
|
+
# Some SPAs (Ladok) show the login screen without changing the URL.
|
|
144
|
+
walled = any(m in u for m in LOGIN_MARKERS) or any(
|
|
145
|
+
s in head for s in ("log in through your university", "logga in via ditt lärosäte",
|
|
146
|
+
"sign in to your account", "logga in"))
|
|
147
|
+
return not walled, p.get("url")
|
unitoolkit/canvas.py
ADDED
|
@@ -0,0 +1,189 @@
|
|
|
1
|
+
"""Canvas LMS REST client (stdlib only). Token from $CANVAS_TOKEN or ~/.secrets/CANVAS_TOKEN."""
|
|
2
|
+
import json
|
|
3
|
+
import os
|
|
4
|
+
import pathlib
|
|
5
|
+
import re
|
|
6
|
+
import time
|
|
7
|
+
import urllib.error
|
|
8
|
+
import urllib.parse
|
|
9
|
+
import urllib.request
|
|
10
|
+
|
|
11
|
+
BASE = os.environ.get("CANVAS_BASE", "https://shh.instructure.com")
|
|
12
|
+
|
|
13
|
+
|
|
14
|
+
def token():
|
|
15
|
+
t = os.environ.get("CANVAS_TOKEN")
|
|
16
|
+
if t:
|
|
17
|
+
return t.strip()
|
|
18
|
+
p = pathlib.Path(os.environ.get("CANVAS_TOKEN_FILE", "~/.secrets/CANVAS_TOKEN")).expanduser()
|
|
19
|
+
return p.read_text().strip()
|
|
20
|
+
|
|
21
|
+
|
|
22
|
+
def _req(url, raw=False):
|
|
23
|
+
if not url.startswith("http"):
|
|
24
|
+
url = f"{BASE}/api/v1/{url.lstrip('/')}"
|
|
25
|
+
r = urllib.request.Request(url, headers={"Authorization": f"Bearer {token()}"})
|
|
26
|
+
for attempt in range(4):
|
|
27
|
+
try:
|
|
28
|
+
with urllib.request.urlopen(r, timeout=60) as resp:
|
|
29
|
+
body = resp.read()
|
|
30
|
+
link = resp.headers.get("Link", "")
|
|
31
|
+
return (body if raw else json.loads(body or b"null")), link
|
|
32
|
+
except urllib.error.HTTPError as e:
|
|
33
|
+
if e.code in (429, 403) and "Rate Limit" in e.read().decode(errors="ignore") and attempt < 3:
|
|
34
|
+
time.sleep(2 ** attempt)
|
|
35
|
+
continue
|
|
36
|
+
raise
|
|
37
|
+
|
|
38
|
+
|
|
39
|
+
def get(path, **params):
|
|
40
|
+
"""GET one resource or follow pagination and return the full list."""
|
|
41
|
+
q = urllib.parse.urlencode({"per_page": 100, **params}, doseq=True)
|
|
42
|
+
base = path if path.startswith("http") else f"{BASE}/api/v1/{path.lstrip('/')}"
|
|
43
|
+
url = base + ("&" if "?" in path else "?") + q
|
|
44
|
+
data, link = _req(url)
|
|
45
|
+
if not isinstance(data, list):
|
|
46
|
+
return data
|
|
47
|
+
out = list(data)
|
|
48
|
+
while True:
|
|
49
|
+
m = re.search(r'<([^>]+)>;\s*rel="next"', link or "")
|
|
50
|
+
if not m:
|
|
51
|
+
return out
|
|
52
|
+
data, link = _req(m.group(1))
|
|
53
|
+
out.extend(data)
|
|
54
|
+
|
|
55
|
+
|
|
56
|
+
def try_get(path, **params):
|
|
57
|
+
try:
|
|
58
|
+
return get(path, **params)
|
|
59
|
+
except urllib.error.HTTPError as e:
|
|
60
|
+
return {"_error": e.code, "_detail": e.read().decode(errors="ignore")[:300]}
|
|
61
|
+
|
|
62
|
+
|
|
63
|
+
def download(url, dest):
|
|
64
|
+
body, _ = _req(url, raw=True)
|
|
65
|
+
dest.parent.mkdir(parents=True, exist_ok=True)
|
|
66
|
+
dest.write_bytes(body)
|
|
67
|
+
return len(body)
|
|
68
|
+
|
|
69
|
+
|
|
70
|
+
def download_bytes(url):
|
|
71
|
+
"""Fetch a file into memory. Used by the document extractor, which keeps text, not bytes."""
|
|
72
|
+
body, _ = _req(url, raw=True)
|
|
73
|
+
return body
|
|
74
|
+
|
|
75
|
+
|
|
76
|
+
def safe(name):
|
|
77
|
+
return re.sub(r'[\\/:*?"<>|\n\r\t]+', "_", name).strip()[:150] or "unnamed"
|
|
78
|
+
|
|
79
|
+
|
|
80
|
+
def courses():
|
|
81
|
+
return get("courses", **{"include[]": ["term", "syllabus_body"], "state[]": ["available", "completed"]})
|
|
82
|
+
|
|
83
|
+
|
|
84
|
+
def mirror(course_id, out, files=True):
|
|
85
|
+
"""Dump a course to `out`: course.json, modules.json, pages/*.html, assignments.json,
|
|
86
|
+
announcements.json, files/. Pages are reached via module items, since SHH disables
|
|
87
|
+
the page index for students. Returns a summary dict."""
|
|
88
|
+
out = pathlib.Path(out).expanduser()
|
|
89
|
+
out.mkdir(parents=True, exist_ok=True)
|
|
90
|
+
w = lambda n, d: (out / n).write_text(json.dumps(d, ensure_ascii=False, indent=1))
|
|
91
|
+
course = get(f"courses/{course_id}", **{"include[]": ["syllabus_body", "term"]})
|
|
92
|
+
w("course.json", course)
|
|
93
|
+
modules = get(f"courses/{course_id}/modules", **{"include[]": ["items", "content_details"]})
|
|
94
|
+
w("modules.json", modules)
|
|
95
|
+
pages_dir = out / "pages"
|
|
96
|
+
pages_dir.mkdir(exist_ok=True)
|
|
97
|
+
n_pages = 0
|
|
98
|
+
for m in modules:
|
|
99
|
+
for it in m.get("items", []):
|
|
100
|
+
if it.get("type") == "Page" and it.get("url"):
|
|
101
|
+
pg = try_get(it["url"])
|
|
102
|
+
if isinstance(pg, dict) and "body" in pg:
|
|
103
|
+
name = f"{m['position']:02d}_{it['position']:02d}_{safe(pg.get('url', 'page'))}.html"
|
|
104
|
+
(pages_dir / name).write_text(
|
|
105
|
+
f"<!-- module: {m['name']} | title: {pg.get('title')} | updated: {pg.get('updated_at')} -->\n"
|
|
106
|
+
+ (pg.get("body") or ""))
|
|
107
|
+
n_pages += 1
|
|
108
|
+
assignments = try_get(f"courses/{course_id}/assignments", **{"include[]": ["rubric_assessment", "submission"]})
|
|
109
|
+
w("assignments.json", assignments)
|
|
110
|
+
ann = try_get(f"courses/{course_id}/discussion_topics", only_announcements="true")
|
|
111
|
+
w("announcements.json", ann)
|
|
112
|
+
disc = try_get(f"courses/{course_id}/discussion_topics")
|
|
113
|
+
w("discussions.json", disc)
|
|
114
|
+
flist = try_get(f"courses/{course_id}/files")
|
|
115
|
+
w("files.json", flist)
|
|
116
|
+
n_files, n_bytes = 0, 0
|
|
117
|
+
if files and isinstance(flist, list):
|
|
118
|
+
for f in flist:
|
|
119
|
+
dest = out / "files" / safe(f.get("display_name") or f.get("filename"))
|
|
120
|
+
if dest.exists() and dest.stat().st_size == f.get("size"):
|
|
121
|
+
continue
|
|
122
|
+
try:
|
|
123
|
+
n_bytes += download(f["url"], dest)
|
|
124
|
+
n_files += 1
|
|
125
|
+
except Exception as e: # locked/unpublished files
|
|
126
|
+
(out / "files_errors.log").open("a").write(f"{f.get('id')} {f.get('display_name')}: {e}\n")
|
|
127
|
+
return {"course": course.get("name"), "modules": len(modules), "pages": n_pages,
|
|
128
|
+
"assignments": len(assignments) if isinstance(assignments, list) else assignments,
|
|
129
|
+
"announcements": len(ann) if isinstance(ann, list) else ann,
|
|
130
|
+
"files_downloaded": n_files, "bytes": n_bytes, "out": str(out)}
|
|
131
|
+
|
|
132
|
+
|
|
133
|
+
def discussions(course=None, days=None, unread_only=False):
|
|
134
|
+
"""Discussion topics (not announcements), across courses.
|
|
135
|
+
|
|
136
|
+
`read_state` is Canvas's per-student state, so this answers "is anything waiting for me in
|
|
137
|
+
a group discussion" - the digital group discussion in PRY is that kind of obligation.
|
|
138
|
+
"""
|
|
139
|
+
out = []
|
|
140
|
+
scope = [course] if course else [c['id'] for c in courses()]
|
|
141
|
+
for cid in scope:
|
|
142
|
+
for t in try_get('/courses/%s/discussion_topics' % cid, per_page=50) or []:
|
|
143
|
+
if t.get('is_announcement'):
|
|
144
|
+
continue
|
|
145
|
+
if unread_only and t.get('read_state') == 'read':
|
|
146
|
+
continue
|
|
147
|
+
out.append({'id': t.get('id'), 'course_id': cid, 'title': t.get('title'),
|
|
148
|
+
'posted_utc': t.get('posted_at'), 'last_reply_utc': t.get('last_reply_at'),
|
|
149
|
+
'entries': t.get('discussion_subentry_count'),
|
|
150
|
+
'read_state': t.get('read_state'), 'author': t.get('user_name'),
|
|
151
|
+
'url': t.get('html_url'),
|
|
152
|
+
'assignment_id': t.get('assignment_id'),
|
|
153
|
+
'requires_entry': t.get('require_initial_post')})
|
|
154
|
+
if days:
|
|
155
|
+
cut = time.time() - days * 86400
|
|
156
|
+
out = [d for d in out if not d['last_reply_utc'] or
|
|
157
|
+
time.mktime(time.strptime(d['last_reply_utc'][:19], '%Y-%m-%dT%H:%M:%S')) >= cut]
|
|
158
|
+
return out
|
|
159
|
+
|
|
160
|
+
|
|
161
|
+
def announcements(course=None, days=30):
|
|
162
|
+
"""Course announcements, newest first.
|
|
163
|
+
|
|
164
|
+
Not `discussion_topics`: without `only_announcements` Canvas leaves the announcements out
|
|
165
|
+
and the list looks empty, which is how a teacher's post went unnoticed on 2026-11-18.
|
|
166
|
+
A course's announcements may live on its parent (a section or the account), so both are read.
|
|
167
|
+
"""
|
|
168
|
+
out = []
|
|
169
|
+
scope = [course] if course else [c['id'] for c in courses()]
|
|
170
|
+
for cid in scope:
|
|
171
|
+
rows = try_get('/courses/%s/discussion_topics' % cid, only_announcements=True,
|
|
172
|
+
per_page=50) or []
|
|
173
|
+
for a in rows:
|
|
174
|
+
out.append({'id': a.get('id'), 'course_id': cid, 'title': a.get('title'),
|
|
175
|
+
'posted_utc': a.get('posted_at') or a.get('delayed_post_at'),
|
|
176
|
+
'author': (a.get('author') or {}).get('display_name'),
|
|
177
|
+
'url': a.get('html_url'),
|
|
178
|
+
'body': (a.get('message') or '')[:2000]})
|
|
179
|
+
out.sort(key=lambda r: r.get('posted_utc') or '', reverse=True)
|
|
180
|
+
if days:
|
|
181
|
+
cut = time.time() - days * 86400
|
|
182
|
+
out = [a for a in out if not a['posted_utc'] or
|
|
183
|
+
time.mktime(time.strptime(a['posted_utc'][:19], '%Y-%m-%dT%H:%M:%S')) >= cut]
|
|
184
|
+
return out
|
|
185
|
+
|
|
186
|
+
|
|
187
|
+
def upcoming():
|
|
188
|
+
"""Planner items (assignments, quizzes, events) from today, across all courses."""
|
|
189
|
+
return get("planner/items", start_date=time.strftime("%Y-%m-%d"))
|