monash-study-kit 1.2.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- monash_study_kit/__init__.py +4 -0
- monash_study_kit/__main__.py +5 -0
- monash_study_kit/browser_login.py +213 -0
- monash_study_kit/cdp.py +188 -0
- monash_study_kit/claude_setup.py +152 -0
- monash_study_kit/cli.py +843 -0
- monash_study_kit/cli_ed.py +238 -0
- monash_study_kit/clients.py +230 -0
- monash_study_kit/content_index.py +324 -0
- monash_study_kit/doctor.py +228 -0
- monash_study_kit/doctor_deep.py +275 -0
- monash_study_kit/edlib.py +551 -0
- monash_study_kit/edquery.py +431 -0
- monash_study_kit/edsync.py +183 -0
- monash_study_kit/features.py +591 -0
- monash_study_kit/help_menu.py +121 -0
- monash_study_kit/htmldom.py +162 -0
- monash_study_kit/jobs.py +233 -0
- monash_study_kit/lesson_reader.py +94 -0
- monash_study_kit/lessons.py +231 -0
- monash_study_kit/mcp_server.py +713 -0
- monash_study_kit/moodlelib.py +485 -0
- monash_study_kit/output.py +73 -0
- monash_study_kit/paths.py +126 -0
- monash_study_kit/recordings.py +312 -0
- monash_study_kit/syncer.py +393 -0
- monash_study_kit/textextract.py +89 -0
- monash_study_kit/todo.py +178 -0
- monash_study_kit/transcribe.py +128 -0
- monash_study_kit/update_check.py +113 -0
- monash_study_kit/webnotes.py +312 -0
- monash_study_kit-1.2.0.dist-info/METADATA +464 -0
- monash_study_kit-1.2.0.dist-info/RECORD +36 -0
- monash_study_kit-1.2.0.dist-info/WHEEL +4 -0
- monash_study_kit-1.2.0.dist-info/entry_points.txt +2 -0
- monash_study_kit-1.2.0.dist-info/licenses/LICENSE +21 -0
|
@@ -0,0 +1,485 @@
|
|
|
1
|
+
"""Monash Moodle 的最小客户端:会话 cookie、AJAX 接口、页面解析、文件下载。
|
|
2
|
+
|
|
3
|
+
登录靠浏览器里拿到的 cookie(Okta SAML 没法在这里走,见 browser_login.py)。Monash 关掉了
|
|
4
|
+
移动端 Web Service(tool_mobile_get_public_config 里 enablemobilewebservice=0),
|
|
5
|
+
所以 moodle-cli 那种"用移动端令牌无限续期"的办法在这里用不了,只能靠定时
|
|
6
|
+
core_session_touch 让会话一直不空闲。
|
|
7
|
+
"""
|
|
8
|
+
from __future__ import annotations
|
|
9
|
+
|
|
10
|
+
import json
|
|
11
|
+
import os
|
|
12
|
+
import re
|
|
13
|
+
import shlex
|
|
14
|
+
import sqlite3
|
|
15
|
+
import time
|
|
16
|
+
import urllib.error
|
|
17
|
+
import urllib.parse
|
|
18
|
+
import urllib.request
|
|
19
|
+
from dataclasses import dataclass
|
|
20
|
+
from datetime import datetime, timezone
|
|
21
|
+
from html.parser import HTMLParser
|
|
22
|
+
from pathlib import Path
|
|
23
|
+
|
|
24
|
+
from .paths import DATA_DIR, FILES_DIR, SECRETS_DIR, write_secret # noqa: F401 FILES_DIR 给 syncer 等模块用
|
|
25
|
+
|
|
26
|
+
BASE_URL = os.environ.get("MOODLE_BASE_URL", "https://learning.monash.edu").rstrip("/")
|
|
27
|
+
HOST = urllib.parse.urlsplit(BASE_URL).hostname or ""
|
|
28
|
+
BASE_SCHEME = urllib.parse.urlsplit(BASE_URL).scheme
|
|
29
|
+
COOKIE_FILE = SECRETS_DIR / "moodle-cookie"
|
|
30
|
+
UA = "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/140.0 Safari/537.36"
|
|
31
|
+
# 请求之间歇一下:这是用个人账号访问学校系统,别像爬虫一样猛打
|
|
32
|
+
REQUEST_GAP = float(os.environ.get("MONASH_KIT_GAP", "0.4"))
|
|
33
|
+
# 会话失效时提示用户跑的命令。服务器上的部署登录方式不一样,用环境变量换掉
|
|
34
|
+
LOGIN_HINT = os.environ.get("MONASH_KIT_LOGIN_HINT", "monash login")
|
|
35
|
+
|
|
36
|
+
|
|
37
|
+
class MoodleAuthError(RuntimeError):
|
|
38
|
+
"""会话失效或 cookie 不对。调用方应退出码 2,提示运行 `monash login`。"""
|
|
39
|
+
|
|
40
|
+
|
|
41
|
+
class MoodleError(RuntimeError):
|
|
42
|
+
"""Moodle 返回了意料之外的东西:HTTP 错误、跳转太多、AJAX 接口报错。命令行退出码 3。"""
|
|
43
|
+
|
|
44
|
+
|
|
45
|
+
def now_iso() -> str:
|
|
46
|
+
return datetime.now(timezone.utc).isoformat(timespec="seconds")
|
|
47
|
+
|
|
48
|
+
|
|
49
|
+
# ---------------------------------------------------------------- cookie
|
|
50
|
+
|
|
51
|
+
def parse_cookie_input(text: str) -> dict[str, str]:
|
|
52
|
+
"""从用户粘贴的东西里抠出 cookie。
|
|
53
|
+
|
|
54
|
+
接受三种:DevTools 里 "Copy as cURL" 的整段命令、一行 `a=b; c=d` 的
|
|
55
|
+
cookie 头、或者只有 MoodleSession 的值。
|
|
56
|
+
"""
|
|
57
|
+
text = text.strip()
|
|
58
|
+
if not text:
|
|
59
|
+
return {}
|
|
60
|
+
header = None
|
|
61
|
+
if text.startswith("curl "):
|
|
62
|
+
try:
|
|
63
|
+
argv = shlex.split(text.replace("\\\n", " "))
|
|
64
|
+
except ValueError:
|
|
65
|
+
argv = text.split()
|
|
66
|
+
for i, arg in enumerate(argv):
|
|
67
|
+
nxt = argv[i + 1] if i + 1 < len(argv) else ""
|
|
68
|
+
if arg in ("-b", "--cookie"):
|
|
69
|
+
header = nxt
|
|
70
|
+
elif arg in ("-H", "--header") and nxt.lower().startswith("cookie:"):
|
|
71
|
+
header = nxt.split(":", 1)[1]
|
|
72
|
+
if header is None:
|
|
73
|
+
return {}
|
|
74
|
+
elif "=" in text:
|
|
75
|
+
header = text
|
|
76
|
+
else:
|
|
77
|
+
return {"MoodleSession": text}
|
|
78
|
+
jar = {}
|
|
79
|
+
for part in header.split(";"):
|
|
80
|
+
if "=" in part:
|
|
81
|
+
k, v = part.split("=", 1)
|
|
82
|
+
if k.strip():
|
|
83
|
+
jar[k.strip()] = v.strip()
|
|
84
|
+
return jar
|
|
85
|
+
|
|
86
|
+
|
|
87
|
+
def load_cookies() -> dict[str, str]:
|
|
88
|
+
if not COOKIE_FILE.exists():
|
|
89
|
+
raise MoodleAuthError(f"还没登录 Moodle,先运行 `{LOGIN_HINT}`")
|
|
90
|
+
try:
|
|
91
|
+
jar = json.loads(COOKIE_FILE.read_text(encoding="utf-8"))
|
|
92
|
+
except json.JSONDecodeError:
|
|
93
|
+
jar = parse_cookie_input(COOKIE_FILE.read_text(encoding="utf-8"))
|
|
94
|
+
if not any(k.startswith("MoodleSession") for k in jar):
|
|
95
|
+
raise MoodleAuthError(f"{COOKIE_FILE} 里没有 MoodleSession")
|
|
96
|
+
return jar
|
|
97
|
+
|
|
98
|
+
|
|
99
|
+
def save_cookies(jar: dict[str, str]) -> None:
|
|
100
|
+
write_secret(COOKIE_FILE, json.dumps(jar))
|
|
101
|
+
|
|
102
|
+
|
|
103
|
+
# ---------------------------------------------------------------- HTTP
|
|
104
|
+
|
|
105
|
+
class _NoRedirect(urllib.request.HTTPRedirectHandler):
|
|
106
|
+
def redirect_request(self, *args, **kwargs):
|
|
107
|
+
return None
|
|
108
|
+
|
|
109
|
+
|
|
110
|
+
_opener = urllib.request.build_opener(_NoRedirect)
|
|
111
|
+
|
|
112
|
+
|
|
113
|
+
@dataclass
|
|
114
|
+
class Response:
|
|
115
|
+
status: int
|
|
116
|
+
url: str
|
|
117
|
+
headers: object
|
|
118
|
+
body: bytes = b""
|
|
119
|
+
location: str | None = None
|
|
120
|
+
|
|
121
|
+
def text(self) -> str:
|
|
122
|
+
return self.body.decode("utf-8", "replace")
|
|
123
|
+
|
|
124
|
+
|
|
125
|
+
class MoodleClient:
|
|
126
|
+
def __init__(self, jar: dict[str, str] | None = None, persist: bool = True):
|
|
127
|
+
self.jar = jar if jar is not None else load_cookies()
|
|
128
|
+
self.persist = persist
|
|
129
|
+
self._sesskey: str | None = None
|
|
130
|
+
self.userid: int | None = None
|
|
131
|
+
self._last = 0.0
|
|
132
|
+
|
|
133
|
+
# 只有发往 Moodle 本站的请求才带 cookie。下载会被 302 到 CloudFront,
|
|
134
|
+
# 自己一步步跟跳转,保证 MoodleSession 不会被带到别的域名去。
|
|
135
|
+
def _request(self, url: str, *, data: bytes | None = None, headers: dict | None = None,
|
|
136
|
+
stream_to: Path | None = None) -> Response:
|
|
137
|
+
gap = REQUEST_GAP - (time.monotonic() - self._last)
|
|
138
|
+
if gap > 0:
|
|
139
|
+
time.sleep(gap)
|
|
140
|
+
self._last = time.monotonic()
|
|
141
|
+
req = urllib.request.Request(url, data=data, method="POST" if data is not None else "GET")
|
|
142
|
+
req.add_header("User-Agent", UA)
|
|
143
|
+
for k, v in (headers or {}).items():
|
|
144
|
+
req.add_header(k, v)
|
|
145
|
+
parts = urllib.parse.urlsplit(url)
|
|
146
|
+
# 只发给本站,而且协议要和 BASE_URL 一样:http:// 的链接不带 cookie,免得明文发出去
|
|
147
|
+
if parts.hostname == HOST and parts.scheme == BASE_SCHEME:
|
|
148
|
+
req.add_unredirected_header("Cookie", "; ".join(f"{k}={v}" for k, v in self.jar.items()))
|
|
149
|
+
try:
|
|
150
|
+
resp = _opener.open(req, timeout=60)
|
|
151
|
+
except urllib.error.HTTPError as e:
|
|
152
|
+
resp = e
|
|
153
|
+
status = resp.status if hasattr(resp, "status") else resp.code
|
|
154
|
+
if parts.hostname == HOST and parts.scheme == BASE_SCHEME:
|
|
155
|
+
self._absorb_cookies(resp.headers) # http:// 的回复可能被人改过,里面的 Set-Cookie 不收
|
|
156
|
+
location = resp.headers.get("Location")
|
|
157
|
+
out = Response(status, url, resp.headers,
|
|
158
|
+
location=urllib.parse.urljoin(url, location) if location else None)
|
|
159
|
+
if stream_to is not None and status == 200:
|
|
160
|
+
tmp = stream_to.with_name(stream_to.name + ".part")
|
|
161
|
+
tmp.parent.mkdir(parents=True, exist_ok=True)
|
|
162
|
+
try:
|
|
163
|
+
with open(tmp, "wb") as f:
|
|
164
|
+
while chunk := resp.read(1 << 16):
|
|
165
|
+
f.write(chunk)
|
|
166
|
+
except BaseException:
|
|
167
|
+
tmp.unlink(missing_ok=True) # 下到一半断了,别在课件目录里留半个文件
|
|
168
|
+
resp.close()
|
|
169
|
+
raise
|
|
170
|
+
os.replace(tmp, stream_to)
|
|
171
|
+
else:
|
|
172
|
+
out.body = resp.read()
|
|
173
|
+
resp.close()
|
|
174
|
+
return out
|
|
175
|
+
|
|
176
|
+
def _absorb_cookies(self, headers) -> None:
|
|
177
|
+
changed = False
|
|
178
|
+
for raw in headers.get_all("Set-Cookie") or []:
|
|
179
|
+
name, _, rest = raw.partition("=")
|
|
180
|
+
value = rest.split(";", 1)[0]
|
|
181
|
+
name = name.strip()
|
|
182
|
+
if not name or value in ("deleted", ""):
|
|
183
|
+
continue
|
|
184
|
+
if self.jar.get(name) != value:
|
|
185
|
+
self.jar[name] = value
|
|
186
|
+
changed = True
|
|
187
|
+
if changed and self.persist:
|
|
188
|
+
save_cookies(self.jar)
|
|
189
|
+
|
|
190
|
+
def get(self, url: str, max_hops: int = 8) -> Response:
|
|
191
|
+
"""GET 本站页面,只在本站内跟跳转;被踢去 Okta 就当会话失效。"""
|
|
192
|
+
url = self.abs(url)
|
|
193
|
+
for _ in range(max_hops):
|
|
194
|
+
r = self._request(url)
|
|
195
|
+
if not _is_redirect(r):
|
|
196
|
+
return r
|
|
197
|
+
_check_login_url(r.location)
|
|
198
|
+
url = r.location
|
|
199
|
+
raise MoodleError(f"跳转太多:{url}")
|
|
200
|
+
|
|
201
|
+
def post(self, url: str, fields: list[tuple[str, str]] | None = None, *, body: bytes | None = None,
|
|
202
|
+
content_type: str | None = None, max_hops: int = 8) -> Response:
|
|
203
|
+
"""POST 一个表单(或 multipart),然后像浏览器一样跟着 303 用 GET 走到最终页面。
|
|
204
|
+
|
|
205
|
+
返回的 Response.url 是最终落地的地址,Moodle 的表单靠"落在哪一页"
|
|
206
|
+
判断成功与否(比如生成日历链接后停在导出页)。
|
|
207
|
+
"""
|
|
208
|
+
url = self.abs(url)
|
|
209
|
+
if body is None:
|
|
210
|
+
body = urllib.parse.urlencode(fields or []).encode()
|
|
211
|
+
content_type = "application/x-www-form-urlencoded"
|
|
212
|
+
r = self._request(url, data=body, headers={"Content-Type": content_type or "application/octet-stream"})
|
|
213
|
+
for _ in range(max_hops):
|
|
214
|
+
if not _is_redirect(r):
|
|
215
|
+
return r
|
|
216
|
+
_check_login_url(r.location)
|
|
217
|
+
r = self._request(r.location)
|
|
218
|
+
raise MoodleError(f"跳转太多:{url}")
|
|
219
|
+
|
|
220
|
+
def html(self, url: str) -> str:
|
|
221
|
+
r = self.get(url)
|
|
222
|
+
if r.status != 200:
|
|
223
|
+
raise MoodleError(f"GET {url} -> {r.status}")
|
|
224
|
+
return r.text()
|
|
225
|
+
|
|
226
|
+
@staticmethod
|
|
227
|
+
def abs(url: str) -> str:
|
|
228
|
+
return url if url.startswith("http") else BASE_URL + url
|
|
229
|
+
|
|
230
|
+
|
|
231
|
+
# ---------------------------------------------------------- AJAX
|
|
232
|
+
|
|
233
|
+
def sesskey(self) -> str:
|
|
234
|
+
if self._sesskey is None:
|
|
235
|
+
page = self.html("/my/courses.php")
|
|
236
|
+
m = re.search(r'"sesskey":"([^"]+)"', page)
|
|
237
|
+
if not m:
|
|
238
|
+
raise MoodleAuthError("页面里没有 sesskey,大概是没登录上")
|
|
239
|
+
self._sesskey = m.group(1)
|
|
240
|
+
return self._sesskey
|
|
241
|
+
|
|
242
|
+
def call(self, method: str, args: dict | None = None):
|
|
243
|
+
url = f"{BASE_URL}/lib/ajax/service.php?sesskey={self.sesskey()}&info={method}"
|
|
244
|
+
body = json.dumps([{"index": 0, "methodname": method, "args": args or {}}]).encode()
|
|
245
|
+
r = self._request(url, data=body, headers={"Content-Type": "application/json"})
|
|
246
|
+
if _is_redirect(r):
|
|
247
|
+
_check_login_url(r.location)
|
|
248
|
+
try:
|
|
249
|
+
payload = json.loads(r.body)
|
|
250
|
+
except json.JSONDecodeError:
|
|
251
|
+
raise MoodleAuthError(f"{method} 返回的不是 JSON({r.status}),会话可能已失效")
|
|
252
|
+
first = payload[0] if isinstance(payload, list) and payload else payload
|
|
253
|
+
if isinstance(first, dict) and first.get("error"):
|
|
254
|
+
exc = first.get("exception") or {}
|
|
255
|
+
code = exc.get("errorcode", "")
|
|
256
|
+
if code in ("servicerequireslogin", "invalidsesskey", "requireloginerror"):
|
|
257
|
+
raise MoodleAuthError(f"{method}: {code}")
|
|
258
|
+
raise MoodleError(f"{method}: {exc.get('message') or first}")
|
|
259
|
+
if isinstance(first, dict) and "exception" in first:
|
|
260
|
+
raise MoodleError(f"{method}: {first.get('message')}")
|
|
261
|
+
return first.get("data") if isinstance(first, dict) else first
|
|
262
|
+
|
|
263
|
+
def time_remaining(self) -> int | None:
|
|
264
|
+
data = self.call("core_session_time_remaining")
|
|
265
|
+
if not isinstance(data, dict):
|
|
266
|
+
return None
|
|
267
|
+
self.userid = data.get("userid") or self.userid
|
|
268
|
+
return data.get("timeremaining")
|
|
269
|
+
|
|
270
|
+
def touch(self) -> int | None:
|
|
271
|
+
self.call("core_session_touch")
|
|
272
|
+
return self.time_remaining()
|
|
273
|
+
|
|
274
|
+
def courses(self) -> list[dict]:
|
|
275
|
+
data = self.call("core_course_get_enrolled_courses_by_timeline_classification",
|
|
276
|
+
{"classification": "all", "limit": 0, "offset": 0, "sort": "fullname"})
|
|
277
|
+
return data["courses"]
|
|
278
|
+
|
|
279
|
+
def course_state(self, course_id: int) -> dict:
|
|
280
|
+
return json.loads(self.call("core_courseformat_get_state", {"courseid": course_id}))
|
|
281
|
+
|
|
282
|
+
# ---------------------------------------------------------- 下载
|
|
283
|
+
|
|
284
|
+
def resolve(self, url: str, max_hops: int = 8) -> tuple[str, str | None, Response]:
|
|
285
|
+
"""只在本站内跟跳转,不下载正文。
|
|
286
|
+
|
|
287
|
+
返回 (最后一个本站 URL, 第一个站外地址或 None, 最后的响应)。本站 pluginfile
|
|
288
|
+
地址里带着文件的 revision 和文件名,拿它当变更指纹;站外地址(CloudFront、
|
|
289
|
+
Google Slides…)只记下来,不带 cookie 去碰。
|
|
290
|
+
"""
|
|
291
|
+
url = self.abs(url)
|
|
292
|
+
for _ in range(max_hops):
|
|
293
|
+
r = self._request(url)
|
|
294
|
+
if not _is_redirect(r):
|
|
295
|
+
return url, None, r
|
|
296
|
+
_check_login_url(r.location)
|
|
297
|
+
if urllib.parse.urlsplit(r.location).hostname != HOST:
|
|
298
|
+
return url, r.location, r
|
|
299
|
+
url = r.location
|
|
300
|
+
raise MoodleError(f"跳转太多:{url}")
|
|
301
|
+
|
|
302
|
+
def download(self, url: str, dest: Path, max_hops: int = 8) -> Response:
|
|
303
|
+
"""下载到 dest。站外那一跳(CloudFront 签名链接)不带 cookie。"""
|
|
304
|
+
url = self.abs(url)
|
|
305
|
+
for _ in range(max_hops):
|
|
306
|
+
r = self._request(url, stream_to=dest)
|
|
307
|
+
if not _is_redirect(r):
|
|
308
|
+
if r.status != 200:
|
|
309
|
+
raise MoodleError(f"下载失败 {url} -> {r.status}")
|
|
310
|
+
return r
|
|
311
|
+
_check_login_url(r.location)
|
|
312
|
+
url = r.location
|
|
313
|
+
raise MoodleError(f"跳转太多:{url}")
|
|
314
|
+
|
|
315
|
+
|
|
316
|
+
def multipart(fields: list[tuple[str, str]], files: list[tuple[str, str, bytes]]) -> tuple[bytes, str]:
|
|
317
|
+
"""标准库没有 multipart 编码器,自己拼一个。files: (字段名, 文件名, 内容)。
|
|
318
|
+
|
|
319
|
+
本包只读,用不到;自己在服务器上部署、要交作业的话用它上传文件。"""
|
|
320
|
+
import secrets
|
|
321
|
+
boundary = "----mdl" + secrets.token_hex(12)
|
|
322
|
+
out = bytearray()
|
|
323
|
+
for name, value in fields:
|
|
324
|
+
out += (f'--{boundary}\r\nContent-Disposition: form-data; name="{name}"\r\n\r\n').encode()
|
|
325
|
+
out += str(value).encode() + b"\r\n"
|
|
326
|
+
for name, filename, data in files:
|
|
327
|
+
safe = filename.replace('"', "%22")
|
|
328
|
+
out += (f'--{boundary}\r\nContent-Disposition: form-data; name="{name}"; filename="{safe}"\r\n'
|
|
329
|
+
f"Content-Type: application/octet-stream\r\n\r\n").encode()
|
|
330
|
+
out += data + b"\r\n"
|
|
331
|
+
out += f"--{boundary}--\r\n".encode()
|
|
332
|
+
return bytes(out), f"multipart/form-data; boundary={boundary}"
|
|
333
|
+
|
|
334
|
+
|
|
335
|
+
def _is_redirect(r: Response) -> bool:
|
|
336
|
+
return bool(r.location) and r.status in (301, 302, 303, 307, 308)
|
|
337
|
+
|
|
338
|
+
|
|
339
|
+
def _check_login_url(url: str) -> None:
|
|
340
|
+
"""在跟进跳转之前检查:要是往 Okta / 登录页跳,就是会话没了。
|
|
341
|
+
|
|
342
|
+
必须在请求之前查,不然下载时会把 Okta 登录页当成课件写进文件里。
|
|
343
|
+
"""
|
|
344
|
+
parts = urllib.parse.urlsplit(url)
|
|
345
|
+
host = parts.hostname or ""
|
|
346
|
+
if host.endswith("okta.com") or (host == HOST and parts.path.startswith("/login/")) \
|
|
347
|
+
or (host == HOST and "/auth/saml" in parts.path):
|
|
348
|
+
raise MoodleAuthError(f"Moodle 登录已过期(被跳到登录页),运行 `{LOGIN_HINT}` 重新登录")
|
|
349
|
+
|
|
350
|
+
|
|
351
|
+
def filename_from(resp: Response, fallback: str) -> str:
|
|
352
|
+
cd = resp.headers.get("Content-Disposition") or ""
|
|
353
|
+
m = re.search(r"filename\*=UTF-8''([^;]+)", cd) or re.search(r'filename="?([^";]+)"?', cd)
|
|
354
|
+
if m:
|
|
355
|
+
return urllib.parse.unquote(m.group(1))
|
|
356
|
+
path = urllib.parse.urlsplit(resp.url).path
|
|
357
|
+
name = urllib.parse.unquote(path.rsplit("/", 1)[-1])
|
|
358
|
+
return name or fallback
|
|
359
|
+
|
|
360
|
+
|
|
361
|
+
# ---------------------------------------------------------------- HTML
|
|
362
|
+
|
|
363
|
+
class LinkCollector(HTMLParser):
|
|
364
|
+
"""按活动(li[data-id] / [data-for=cmitem])归类收集链接。
|
|
365
|
+
|
|
366
|
+
课程页里 label、CMS 块的内容只出现在 HTML 里,接口不给,所以得自己扫。
|
|
367
|
+
"""
|
|
368
|
+
|
|
369
|
+
def __init__(self):
|
|
370
|
+
super().__init__(convert_charrefs=True)
|
|
371
|
+
self.stack: list[tuple[str, str | None]] = [] # (tag, activity id or None)
|
|
372
|
+
self.by_activity: dict[str, list[tuple[str, str]]] = {}
|
|
373
|
+
self.all: list[tuple[str, str]] = []
|
|
374
|
+
self._a: tuple[str, list[str]] | None = None
|
|
375
|
+
|
|
376
|
+
def _current(self) -> str | None:
|
|
377
|
+
for _, act in reversed(self.stack):
|
|
378
|
+
if act:
|
|
379
|
+
return act
|
|
380
|
+
return None
|
|
381
|
+
|
|
382
|
+
VOID = {"br", "img", "input", "meta", "link", "hr", "source", "area", "base", "col", "wbr"}
|
|
383
|
+
|
|
384
|
+
def handle_starttag(self, tag, attrs):
|
|
385
|
+
a = dict(attrs)
|
|
386
|
+
act = None
|
|
387
|
+
cls = a.get("class") or ""
|
|
388
|
+
if tag == "li" and "activity" in cls.split() and a.get("data-id"):
|
|
389
|
+
act = a["data-id"]
|
|
390
|
+
if tag not in self.VOID:
|
|
391
|
+
self.stack.append((tag, act))
|
|
392
|
+
if tag == "a" and a.get("href"):
|
|
393
|
+
self._a = (a["href"], [])
|
|
394
|
+
|
|
395
|
+
def handle_endtag(self, tag):
|
|
396
|
+
if tag == "a" and self._a:
|
|
397
|
+
href, text = self._a
|
|
398
|
+
item = (href, " ".join("".join(text).split()))
|
|
399
|
+
self.all.append(item)
|
|
400
|
+
act = self._current()
|
|
401
|
+
if act:
|
|
402
|
+
self.by_activity.setdefault(act, []).append(item)
|
|
403
|
+
self._a = None
|
|
404
|
+
for i in range(len(self.stack) - 1, -1, -1):
|
|
405
|
+
if self.stack[i][0] == tag:
|
|
406
|
+
del self.stack[i:]
|
|
407
|
+
break
|
|
408
|
+
|
|
409
|
+
def handle_data(self, data):
|
|
410
|
+
if self._a:
|
|
411
|
+
self._a[1].append(data)
|
|
412
|
+
|
|
413
|
+
|
|
414
|
+
def collect_links(html_text: str) -> LinkCollector:
|
|
415
|
+
p = LinkCollector()
|
|
416
|
+
p.feed(html_text)
|
|
417
|
+
p.close()
|
|
418
|
+
return p
|
|
419
|
+
|
|
420
|
+
|
|
421
|
+
DOC_EXT = re.compile(r"\.(pdf|pptx?|docx?|xlsx?|zip|tar|gz|tgz|7z|rar|ipynb|py|hs|ts|js|java|c|cpp|h|"
|
|
422
|
+
r"txt|md|csv|json|mzn|dzn|r|rmd|sql|mp4|mov|key|pages|numbers)$", re.I)
|
|
423
|
+
|
|
424
|
+
|
|
425
|
+
def is_attachment(url: str) -> bool:
|
|
426
|
+
"""本站 pluginfile 里真正的课件(排除主题图片、背景图之类)。"""
|
|
427
|
+
parts = urllib.parse.urlsplit(url)
|
|
428
|
+
if parts.hostname != HOST or "/pluginfile.php/" not in parts.path:
|
|
429
|
+
return False
|
|
430
|
+
if re.search(r"/(theme_|msttools_imglib|user/icon|core_admin|block_)", parts.path):
|
|
431
|
+
return False
|
|
432
|
+
return bool(DOC_EXT.search(urllib.parse.unquote(parts.path)))
|
|
433
|
+
|
|
434
|
+
|
|
435
|
+
def db_connect(path: Path | None = None) -> sqlite3.Connection:
|
|
436
|
+
# MCP 的后台同步和工具调用会同时开库:WAL 让读写互不阻塞,timeout 兜住偶尔的写锁
|
|
437
|
+
path = path or DATA_DIR / "moodle.db"
|
|
438
|
+
path.parent.mkdir(parents=True, exist_ok=True)
|
|
439
|
+
con = sqlite3.connect(path, timeout=30, check_same_thread=False)
|
|
440
|
+
con.row_factory = sqlite3.Row
|
|
441
|
+
con.execute("PRAGMA journal_mode=WAL")
|
|
442
|
+
con.executescript("""
|
|
443
|
+
CREATE TABLE IF NOT EXISTS files (
|
|
444
|
+
course_id INTEGER NOT NULL,
|
|
445
|
+
source TEXT NOT NULL, -- 本站最后一个 URL(含 revision),当指纹
|
|
446
|
+
cmid INTEGER,
|
|
447
|
+
path TEXT NOT NULL, -- 相对 FILES_DIR
|
|
448
|
+
size INTEGER,
|
|
449
|
+
section TEXT,
|
|
450
|
+
title TEXT,
|
|
451
|
+
synced_at TEXT NOT NULL,
|
|
452
|
+
removed_at TEXT, -- Moodle 上已经没有了:文件先留着,列表和搜索里不再出现
|
|
453
|
+
PRIMARY KEY (course_id, source)
|
|
454
|
+
);
|
|
455
|
+
CREATE TABLE IF NOT EXISTS links (
|
|
456
|
+
course_id INTEGER NOT NULL, folder TEXT NOT NULL, section TEXT,
|
|
457
|
+
title TEXT, url TEXT NOT NULL, synced_at TEXT NOT NULL,
|
|
458
|
+
PRIMARY KEY (course_id, folder, url)
|
|
459
|
+
);
|
|
460
|
+
CREATE TABLE IF NOT EXISTS courses (
|
|
461
|
+
id INTEGER PRIMARY KEY, code TEXT, folder TEXT, fullname TEXT,
|
|
462
|
+
tracked INTEGER DEFAULT 0, synced_at TEXT
|
|
463
|
+
);
|
|
464
|
+
CREATE TABLE IF NOT EXISTS state (k TEXT PRIMARY KEY, v TEXT);
|
|
465
|
+
""")
|
|
466
|
+
# 1.0.3 以前的库没有 removed_at,补上
|
|
467
|
+
if "removed_at" not in {r["name"] for r in con.execute("PRAGMA table_info(files)")}:
|
|
468
|
+
con.execute("ALTER TABLE files ADD COLUMN removed_at TEXT")
|
|
469
|
+
return con
|
|
470
|
+
|
|
471
|
+
|
|
472
|
+
def removed_paths(con) -> set[str]:
|
|
473
|
+
"""Moodle 上已经下架的文件(相对 FILES_DIR)。"""
|
|
474
|
+
return {r["path"] for r in con.execute("SELECT path FROM files WHERE removed_at IS NOT NULL")}
|
|
475
|
+
|
|
476
|
+
|
|
477
|
+
def set_state(con, k: str, v) -> None:
|
|
478
|
+
con.execute("INSERT INTO state VALUES (?, ?) ON CONFLICT(k) DO UPDATE SET v=excluded.v",
|
|
479
|
+
(k, json.dumps(v)))
|
|
480
|
+
con.commit()
|
|
481
|
+
|
|
482
|
+
|
|
483
|
+
def get_state(con, k: str, default=None):
|
|
484
|
+
row = con.execute("SELECT v FROM state WHERE k=?", (k,)).fetchone()
|
|
485
|
+
return json.loads(row["v"]) if row else default
|
|
@@ -0,0 +1,73 @@
|
|
|
1
|
+
"""终端输出:交互终端里打表格,管道或 --json 时打 JSON。"""
|
|
2
|
+
from __future__ import annotations
|
|
3
|
+
|
|
4
|
+
import json
|
|
5
|
+
import shutil
|
|
6
|
+
import sys
|
|
7
|
+
import unicodedata
|
|
8
|
+
from datetime import datetime, timezone
|
|
9
|
+
|
|
10
|
+
|
|
11
|
+
def want_json(args) -> bool:
|
|
12
|
+
if getattr(args, "json", False):
|
|
13
|
+
return True
|
|
14
|
+
if getattr(args, "text", False):
|
|
15
|
+
return False
|
|
16
|
+
return not sys.stdout.isatty()
|
|
17
|
+
|
|
18
|
+
|
|
19
|
+
def emit_json(data) -> None:
|
|
20
|
+
print(json.dumps(data, ensure_ascii=False, indent=2))
|
|
21
|
+
|
|
22
|
+
|
|
23
|
+
def width(s: str) -> int:
|
|
24
|
+
return sum(2 if unicodedata.east_asian_width(ch) in "WF" else 1 for ch in s)
|
|
25
|
+
|
|
26
|
+
|
|
27
|
+
def cut(s: str, n: int) -> str:
|
|
28
|
+
s = " ".join(str(s or "").split())
|
|
29
|
+
if width(s) <= n:
|
|
30
|
+
return s + " " * (n - width(s))
|
|
31
|
+
out, w = "", 0
|
|
32
|
+
for ch in s:
|
|
33
|
+
cw = width(ch)
|
|
34
|
+
if w + cw > n - 1:
|
|
35
|
+
break
|
|
36
|
+
out += ch
|
|
37
|
+
w += cw
|
|
38
|
+
return out + "…" + " " * (n - w - 1)
|
|
39
|
+
|
|
40
|
+
|
|
41
|
+
def table(rows: list[dict], cols: list[tuple[str, str, int]]) -> None:
|
|
42
|
+
"""cols: (key, 表头, 最大宽度)。最后一列吃掉剩余宽度。"""
|
|
43
|
+
if not rows:
|
|
44
|
+
print("(没有内容)")
|
|
45
|
+
return
|
|
46
|
+
term = shutil.get_terminal_size((120, 20)).columns
|
|
47
|
+
widths = []
|
|
48
|
+
for key, head, maxw in cols:
|
|
49
|
+
w = max([width(head)] + [width(" ".join(str(r.get(key) or "").split())) for r in rows])
|
|
50
|
+
widths.append(min(w, maxw))
|
|
51
|
+
used = sum(widths[:-1]) + 2 * (len(cols) - 1)
|
|
52
|
+
widths[-1] = max(10, min(widths[-1], term - used))
|
|
53
|
+
print(" ".join(cut(h, w) for (_, h, _), w in zip(cols, widths)).rstrip())
|
|
54
|
+
print(" ".join("─" * w for w in widths))
|
|
55
|
+
for r in rows:
|
|
56
|
+
print(" ".join(cut(r.get(k, ""), w) for (k, _, _), w in zip(cols, widths)).rstrip())
|
|
57
|
+
|
|
58
|
+
|
|
59
|
+
def local_time(iso: str | None, fmt: str = "%m-%d %a %H:%M") -> str:
|
|
60
|
+
if not iso:
|
|
61
|
+
return "—"
|
|
62
|
+
return datetime.fromisoformat(iso).astimezone().strftime(fmt) # 按这台电脑的时区
|
|
63
|
+
|
|
64
|
+
|
|
65
|
+
def relative(iso: str | None) -> str:
|
|
66
|
+
if not iso:
|
|
67
|
+
return ""
|
|
68
|
+
delta = datetime.fromisoformat(iso) - datetime.now(timezone.utc)
|
|
69
|
+
secs = delta.total_seconds()
|
|
70
|
+
if secs < 0:
|
|
71
|
+
return "已过期"
|
|
72
|
+
days = int(secs // 86400)
|
|
73
|
+
return f"{days} 天后" if days else f"{int(secs // 3600)} 小时后"
|
|
@@ -0,0 +1,126 @@
|
|
|
1
|
+
"""所有东西放在哪:一个目录装下数据、课件、登录凭据和专用浏览器配置。
|
|
2
|
+
|
|
3
|
+
macOS ~/Library/Application Support/monash-study-kit
|
|
4
|
+
Windows %LOCALAPPDATA%\\monash-study-kit
|
|
5
|
+
Linux ~/.local/share/monash-study-kit
|
|
6
|
+
|
|
7
|
+
设 MONASH_KIT_HOME 可以换位置(测试也靠它)。卸载时删掉这个目录就干净了。
|
|
8
|
+
用户可改的设置在 settings.json(`monash config`)。
|
|
9
|
+
|
|
10
|
+
服务器上的部署把数据、凭据、Ed 库放在各自的目录,用这几个环境变量指过去:
|
|
11
|
+
MONASH_KIT_DATA moodle.db 所在目录
|
|
12
|
+
MONASH_KIT_SECRETS cookie、日历链接、Ed 令牌所在目录
|
|
13
|
+
MONASH_KIT_FILES 课件目录
|
|
14
|
+
MONASH_KIT_ED_DATA Ed 的数据目录(里面是 ed.db 和 files/,ed-digest 就是这个布局)
|
|
15
|
+
MONASH_KIT_ED_TOKEN Ed 令牌文件
|
|
16
|
+
settings.json 里的每一项也能用 MONASH_KIT_<大写名字> 覆盖,比如 MONASH_KIT_DOWNLOAD_VIDEOS=true。
|
|
17
|
+
"""
|
|
18
|
+
from __future__ import annotations
|
|
19
|
+
|
|
20
|
+
import json
|
|
21
|
+
import os
|
|
22
|
+
import sys
|
|
23
|
+
from pathlib import Path
|
|
24
|
+
|
|
25
|
+
|
|
26
|
+
def _default_home() -> Path:
|
|
27
|
+
if sys.platform == "darwin":
|
|
28
|
+
return Path.home() / "Library" / "Application Support" / "monash-study-kit"
|
|
29
|
+
if os.name == "nt":
|
|
30
|
+
base = os.environ.get("LOCALAPPDATA") or str(Path.home() / "AppData" / "Local")
|
|
31
|
+
return Path(base) / "monash-study-kit"
|
|
32
|
+
base = os.environ.get("XDG_DATA_HOME") or str(Path.home() / ".local" / "share")
|
|
33
|
+
return Path(base) / "monash-study-kit"
|
|
34
|
+
|
|
35
|
+
|
|
36
|
+
HOME = Path(os.environ.get("MONASH_KIT_HOME") or _default_home())
|
|
37
|
+
SETTINGS_FILE = HOME / "settings.json"
|
|
38
|
+
|
|
39
|
+
DEFAULTS = {
|
|
40
|
+
"download_videos": False, # 录像动辄几百 MB;没装 [media] 转不了字幕,下下来 Claude 也看不了
|
|
41
|
+
"auto_sync_hours": 1, # MCP 在跑的时候,多久在后台同步一次
|
|
42
|
+
"keepalive_minutes": 20, # MCP 在跑的时候,多久给 Moodle 会话续一次期(空闲 4 小时就过期)
|
|
43
|
+
"tz_offset": 8, # Moodle 账号的时区(页面上的时间按它显示):马来西亚 8,澳洲 10(夏令时 11)
|
|
44
|
+
"files_dir": "", # 课件放哪;空 = 数据目录下的 files。Windows 路径太长时改成 C:\\Monash 之类
|
|
45
|
+
"browser": "", # 登录窗口用哪个浏览器程序;空 = 自动找 Chrome / Edge / Brave
|
|
46
|
+
"web_notes": True, # 抓 Moodle / Ed 链接到的课程笔记网页(老师的 github.io 讲义站之类)进全文搜索
|
|
47
|
+
"update_check": True, # 每天查一次 GitHub 上有没有新版本,有就提示
|
|
48
|
+
"name_limit": 80, # 文件名最长几个字符:Windows 整条路径不能超过 260,课程/周次/文件夹/文件四层很容易超
|
|
49
|
+
"pdf_extractor": "pypdf", # PDF 抽文字用什么:pypdf(随包装好)或 pdftotext(要另装 poppler,排版更准)
|
|
50
|
+
}
|
|
51
|
+
|
|
52
|
+
|
|
53
|
+
def _env_value(key: str, default):
|
|
54
|
+
"""MONASH_KIT_<KEY> 覆盖一项设置,按默认值的类型转换。"""
|
|
55
|
+
raw = os.environ.get("MONASH_KIT_" + key.upper())
|
|
56
|
+
if raw is None:
|
|
57
|
+
return None
|
|
58
|
+
if isinstance(default, bool):
|
|
59
|
+
return raw.strip().lower() in ("1", "true", "yes", "on")
|
|
60
|
+
if isinstance(default, (int, float)):
|
|
61
|
+
try:
|
|
62
|
+
return type(default)(float(raw))
|
|
63
|
+
except ValueError:
|
|
64
|
+
return None
|
|
65
|
+
return raw
|
|
66
|
+
|
|
67
|
+
|
|
68
|
+
def load_settings() -> dict:
|
|
69
|
+
try:
|
|
70
|
+
out = {**DEFAULTS, **json.loads(SETTINGS_FILE.read_text(encoding="utf-8"))}
|
|
71
|
+
except (OSError, ValueError):
|
|
72
|
+
out = dict(DEFAULTS)
|
|
73
|
+
for key, default in DEFAULTS.items():
|
|
74
|
+
v = _env_value(key, default)
|
|
75
|
+
if v is not None:
|
|
76
|
+
out[key] = v
|
|
77
|
+
return out
|
|
78
|
+
|
|
79
|
+
|
|
80
|
+
# 设置在导入时读一次:Claude Desktop 启动的 MCP 读不到终端里设的环境变量,所以常用的都进 settings.json;
|
|
81
|
+
# 环境变量仍然优先(测试和高级用法)。
|
|
82
|
+
_s = load_settings()
|
|
83
|
+
SECRETS_DIR = Path(os.environ.get("MONASH_KIT_SECRETS") or HOME / "secrets") # Moodle cookie、Ed 令牌、日历链接(600 权限)
|
|
84
|
+
DATA_DIR = Path(os.environ.get("MONASH_KIT_DATA") or HOME / "data") # moodle.db、ed.db
|
|
85
|
+
FILES_DIR = Path(os.environ.get("MONASH_KIT_FILES") or _s["files_dir"] or HOME / "files").expanduser()
|
|
86
|
+
_ED_DATA = os.environ.get("MONASH_KIT_ED_DATA")
|
|
87
|
+
ED_DB = Path(_ED_DATA) / "ed.db" if _ED_DATA else DATA_DIR / "ed.db"
|
|
88
|
+
ED_FILES_DIR = Path(_ED_DATA) / "files" if _ED_DATA else DATA_DIR / "ed-files" # Ed Lessons 的课件 PDF
|
|
89
|
+
ED_TOKEN_FILE = Path(os.environ.get("MONASH_KIT_ED_TOKEN") or SECRETS_DIR / "ed-token")
|
|
90
|
+
BROWSER_PROFILE = HOME / "browser-profile" # 专用登录窗口的配置目录,和日常浏览器分开
|
|
91
|
+
LOG_FILE = HOME / "mcp.log"
|
|
92
|
+
TZ_OFFSET = float(os.environ.get("MONASH_KIT_TZ_OFFSET") or _s["tz_offset"])
|
|
93
|
+
BROWSER = os.environ.get("MONASH_KIT_BROWSER") or _s["browser"]
|
|
94
|
+
|
|
95
|
+
|
|
96
|
+
def ensure_private_dir(path: Path) -> None:
|
|
97
|
+
path.mkdir(parents=True, exist_ok=True)
|
|
98
|
+
try:
|
|
99
|
+
os.chmod(path, 0o700)
|
|
100
|
+
except OSError:
|
|
101
|
+
pass
|
|
102
|
+
|
|
103
|
+
|
|
104
|
+
def write_secret(path: Path, text: str) -> None:
|
|
105
|
+
"""原子写入、只有自己能读(Windows 上 chmod 不起作用,但目录本来就在用户自己的目录下)。"""
|
|
106
|
+
ensure_private_dir(path.parent)
|
|
107
|
+
# 临时文件名带进程号:Claude Desktop 和 Claude Code 各开一个 MCP,后台同步也会续 cookie,
|
|
108
|
+
# 共用一个 .tmp 会互相覆盖,后 replace 的那个直接报错
|
|
109
|
+
tmp = path.with_name(f"{path.name}.{os.getpid()}.tmp")
|
|
110
|
+
fd = os.open(tmp, os.O_WRONLY | os.O_CREAT | os.O_TRUNC, 0o600)
|
|
111
|
+
with os.fdopen(fd, "w", encoding="utf-8") as f:
|
|
112
|
+
f.write(text)
|
|
113
|
+
os.replace(tmp, path)
|
|
114
|
+
|
|
115
|
+
|
|
116
|
+
def save_settings(values: dict) -> None:
|
|
117
|
+
HOME.mkdir(parents=True, exist_ok=True)
|
|
118
|
+
cur = {}
|
|
119
|
+
try:
|
|
120
|
+
cur = json.loads(SETTINGS_FILE.read_text(encoding="utf-8"))
|
|
121
|
+
except (OSError, ValueError):
|
|
122
|
+
pass
|
|
123
|
+
cur.update(values)
|
|
124
|
+
tmp = SETTINGS_FILE.with_suffix(".tmp")
|
|
125
|
+
tmp.write_text(json.dumps(cur, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
|
|
126
|
+
os.replace(tmp, SETTINGS_FILE)
|