monash-study-kit 1.2.0__py3-none-any.whl

monash_study_kit/transcribe.py

@@ -0,0 +1,128 @@
+"""把同步下来的录像（Moodle 上的 mp4、Ed 里的 Zoom 录像）转成带时间戳的字幕稿。
+
+    monash media [--limit N] [--model small.en] [--dry-run]
+
+每个视频旁边生成 <原名>.transcript.md，并登记进 files 表，list_files、read_file、
+全文搜索都能看到，Claude 读字幕稿就知道老师讲了什么。
+
+用 faster-whisper（CTranslate2，CPU int8），要装 [media]。普通笔记本上 small.en 大约几倍实时，
+一节 2 小时的课要跑二三十分钟，所以只在手动运行 monash media 时做，不放进后台同步。
+  * 内容完全一样的视频只识别一次（Moodle 上同一段录像常出现在两周），另一份写一句指过去
+  * 给模型一段提示：课程名 + 这一周的标题，专有名词（Functor、beta reduction…）会准很多
+  * 没有讲话的录屏（演示动画）会写明"没有识别到讲话"，不会硬编内容
+"""
+from __future__ import annotations
+
+import hashlib
+import os
+import re
+import time
+from pathlib import Path
+
+from .moodlelib import FILES_DIR, get_state, now_iso, set_state
+from .recordings import fmt_ts, register, write_transcript
+
+MEDIA = (".mp4", ".mov", ".m4a", ".mp3", ".webm", ".mkv", ".wav")
+MODEL = os.environ.get("MONASH_KIT_WHISPER_MODEL", "small.en")
+THREADS = int(os.environ.get("MONASH_KIT_WHISPER_THREADS", str(max(1, (os.cpu_count() or 4) // 2))))
+
+
+def transcript_path(media_rel: str) -> str:
+    return re.sub(r"\.[^./]+$", "", media_rel) + ".transcript.md"
+
+
+def file_hash(p: Path) -> str:
+    """大小 + 开头 16 MB 的 SHA-1：够分辨"是不是同一个文件"，不用读完几百 MB。"""
+    h = hashlib.sha1(str(p.stat().st_size).encode())
+    with open(p, "rb") as f:
+        h.update(f.read(16 << 20))
+    return h.hexdigest()
+
+
+def cached_hash(con, rel: str) -> str:
+    """file_hash 记在库里，文件大小和修改时间没变就直接用。
+
+    每次开跑都要给所有转写过的视频算一遍哈希来认重复，视频一多就是每次读几个 GB。
+    """
+    st = (FILES_DIR / rel).stat()
+    key = f"hash:{rel}"
+    hit = get_state(con, key)
+    if hit and hit[:2] == [st.st_size, st.st_mtime]:
+        return hit[2]
+    h = file_hash(FILES_DIR / rel)
+    set_state(con, key, [st.st_size, st.st_mtime, h])
+    return h
+
+
+def pending(con) -> list[dict]:
+    rows = con.execute("SELECT f.course_id, f.path, f.title, f.section, c.fullname FROM files f "
+                       "LEFT JOIN courses c ON c.id=f.course_id WHERE f.removed_at IS NULL ORDER BY f.path").fetchall()
+    have = {r["path"] for r in rows}
+    out = []
+    for r in rows:
+        if r["path"].lower().endswith(MEDIA) and transcript_path(r["path"]) not in have \
+                and (FILES_DIR / r["path"]).exists():
+            out.append(dict(r))
+    return out
+
+
+def prompt_for(item: dict) -> str:
+    """课程名 + 周目录名，比如 "FIT2102 Programming paradigms. Week 08 - Functors and Applicatives in Haskell."。"""
+    parts = item["path"].split("/")
+    week = next((p for p in parts if p.startswith("Week ")), "")
+    return f"{item.get('fullname') or parts[0]}. {week}.".replace("..", ".")
+
+
+def run(con, *, model_name: str = MODEL, limit: int = 0, match: str | None = None, dry_run: bool = False,
+        log=print) -> int:
+    """转写还没有字幕稿的视频，返回处理了几个。"""
+    todo = pending(con)
+    if match:
+        todo = [t for t in todo if match.lower() in t["path"].lower()]
+    if limit:
+        todo = todo[:limit]
+    log(f"{len(todo)} 个视频待转写（模型 {model_name}）")
+    if dry_run or not todo:
+        for t in todo:
+            log(f"  · {t['path']}")
+        return 0
+
+    try:
+        from faster_whisper import WhisperModel
+    except ImportError:
+        raise SystemExit("没装转写组件。重新安装时带上 [media]：见 README 的“录播字幕”一节") from None
+    model = WhisperModel(model_name, device="cpu", compute_type="int8", cpu_threads=THREADS)
+    # 以前几次已经转写过的视频也算进去，跨次运行一样能认出重复
+    seen: dict[str, str] = {}
+    have = {r["path"] for r in con.execute("SELECT path FROM files")}
+    for r in con.execute("SELECT path FROM files").fetchall():
+        p = r["path"]
+        if p.lower().endswith(MEDIA) and transcript_path(p) in have and (FILES_DIR / p).exists():
+            seen.setdefault(cached_hash(con, p), transcript_path(p))
+    for item in todo:
+        media = FILES_DIR / item["path"]
+        out_rel = transcript_path(item["path"])
+        h = cached_hash(con, item["path"])
+        name = Path(item["path"]).name
+        title = item["title"] or Path(name).stem
+        if h in seen:
+            (FILES_DIR / out_rel).write_text(
+                f"# {title} 字幕稿\n\n这段视频和 `{seen[h]}` 完全相同，字幕稿见那一份。\n", encoding="utf-8")
+            register(con, item["course_id"], f"transcript:{item['path']}", out_rel, item["section"], f"字幕稿：{title}")
+            log(f"  = {name}（和 {seen[h]} 相同）")
+            continue
+        t0 = time.time()
+        log(f"  ♪ {item['path']}")
+        segments, info = model.transcribe(str(media), language="en", beam_size=5, vad_filter=True,
+                                          initial_prompt=prompt_for(item), condition_on_previous_text=False)
+        lines = [(s.start, s.text.strip()) for s in segments if s.text.strip()]
+        write_transcript(FILES_DIR / out_rel, title,
+                         [f"视频：{name}", f"时长 {fmt_ts(info.duration)}",
+                          f"识别：faster-whisper {model_name}", f"生成于 {now_iso()[:16].replace('T', ' ')} UTC"],
+                         lines)
+        register(con, item["course_id"], f"transcript:{item['path']}", out_rel, item["section"], f"字幕稿：{title}")
+        seen[h] = out_rel
+        took = time.time() - t0
+        log(f"    → {out_rel}（{fmt_ts(info.duration)} 的视频用了 {fmt_ts(took)}，"
+            f"{info.duration / max(took, 1):.1f}× 实时）")
+    return len(todo)

monash_study_kit/update_check.py

@@ -0,0 +1,113 @@
+"""新版本提示：每天最多查一次 PyPI 上的最新版本号，结果缓存在本地。
+
+朋友不会主动运行 monash update，修了 bug 也拿不到，所以在他们会看到的地方提一句：
+命令行（stderr 一行）、Claude 里的 study_todo / monash_status、monash doctor。
+
+  * 命令行只读缓存，绝不等网络；缓存过期就在后台线程里刷新，下一次运行才看得到。
+  * 查的是 PyPI 的公开 JSON 接口，不带任何身份信息。PyPI 上还没有这个包或者连不上时，
+    退回去看 GitHub 上 main 分支的 __init__.py（1.1.0 以前就是这么查的）。
+  * 关掉：monash config update_check false，或者设 MONASH_KIT_NO_UPDATE_CHECK=1（CI 里自动关）。
+"""
+from __future__ import annotations
+
+import json
+import os
+import re
+import threading
+import time
+import urllib.request
+
+from . import __version__
+from .paths import HOME, load_settings
+
+PYPI_URL = "https://pypi.org/pypi/monash-study-kit/json"
+VERSION_URL = "https://raw.githubusercontent.com/Waldo0926/monash-study-kit/main/src/monash_study_kit/__init__.py"
+CACHE_FILE = HOME / "update-check.json"
+EVERY = 24 * 3600
+VERSION_RE = re.compile(r'__version__\s*=\s*"([0-9][0-9A-Za-z.\-+]*)"')
+
+
+def parse(v: str) -> tuple:
+    """"0.10.2" → (0, 10, 2)；认不出来的部分当 0，免得一个怪版本号让比较崩掉。"""
+    return tuple(int(p) if p.isdigit() else 0 for p in re.split(r"[.\-+]", v or "0"))
+
+
+def is_newer(latest: str | None, current: str = __version__) -> bool:
+    return bool(latest) and parse(latest) > parse(current)
+
+
+def enabled() -> bool:
+    if os.environ.get("MONASH_KIT_NO_UPDATE_CHECK") or os.environ.get("CI"):
+        return False
+    return bool(load_settings().get("update_check", True))
+
+
+def read_cache() -> dict:
+    try:
+        return json.loads(CACHE_FILE.read_text(encoding="utf-8"))
+    except (OSError, ValueError):
+        return {}
+
+
+def _get(url: str, timeout: float) -> bytes:
+    req = urllib.request.Request(url, headers={"User-Agent": f"monash-study-kit/{__version__}"})
+    with urllib.request.urlopen(req, timeout=timeout) as r:
+        return r.read(2_000_000)
+
+
+def fetch_latest(timeout: float = 5) -> str | None:
+    try:
+        version = json.loads(_get(PYPI_URL, timeout))["info"]["version"]
+        if VERSION_RE.fullmatch(f'__version__ = "{version}"'):
+            return version
+    except (OSError, ValueError, KeyError, TypeError):
+        pass
+    m = VERSION_RE.search(_get(VERSION_URL, timeout)[:4096].decode("utf-8", "replace"))
+    return m.group(1) if m else None
+
+
+def refresh(force: bool = False) -> dict:
+    """需要的话查一次，写缓存。网络出错就记下时间、下次再说，绝不抛出去。"""
+    cache = read_cache()
+    if not force and time.time() - cache.get("checked_at", 0) < EVERY:
+        return cache
+    try:
+        latest = fetch_latest()
+    except Exception:  # noqa: BLE001  没网、GitHub 抽风：安静地跳过
+        latest = cache.get("latest")
+    cache = {"checked_at": time.time(), "latest": latest}
+    try:
+        HOME.mkdir(parents=True, exist_ok=True)
+        tmp = CACHE_FILE.with_suffix(".tmp")
+        tmp.write_text(json.dumps(cache), encoding="utf-8")
+        os.replace(tmp, CACHE_FILE)
+    except OSError:
+        pass
+    return cache
+
+
+def refresh_in_background() -> None:
+    if enabled() and time.time() - read_cache().get("checked_at", 0) >= EVERY:
+        threading.Thread(target=refresh, name="update-check", daemon=True).start()
+
+
+def available() -> str | None:
+    """缓存里有比当前新的版本就返回它（只读缓存，不联网）。"""
+    if not enabled():
+        return None
+    latest = read_cache().get("latest")
+    return latest if is_newer(latest) else None
+
+
+def notice() -> str | None:
+    latest = available()
+    if not latest:
+        return None
+    return f"monash 有新版本 {latest}（现在是 {__version__}）：运行 monash update 更新，然后重启 Claude Desktop"
+
+
+def clear() -> None:
+    try:
+        CACHE_FILE.unlink()
+    except OSError:
+        pass

monash_study_kit/webnotes.py

@@ -0,0 +1,312 @@
+"""课程笔记网页：Moodle 和 Ed Lessons 里链接到的讲义网站，抓下来转成文字，进全文索引。
+
+很多课的讲义不在 Moodle / Ed 里，而在老师的公开网站上（FIT2102 → tgdwyer.github.io，
+FIT2109 → yqtian-se.github.io）。只抓被课程直接链接到的那几页，不顺着页面里的链接往下爬。
+
+哪些站算“讲义站”：
+  * github.io / gitlab.io / readthedocs.io / netlify.app / pages.dev 这类静态站；
+  * 或者同一门课里被链接了很多次的站（CLASS_SITE_MIN 次以上）；
+  * 学校官网、Moodle / Ed 自己、视频、会议、网盘、Google 文档之类一律不抓。
+
+存在 FILES_DIR/<课程文件夹>/Course notes (web)/<标题>.md，开头写来源网址。放在课件目录里，
+list_files / read_file / search_content 就都能直接用。每页最多每 REFRESH_DAYS 天重抓一次，
+带 ETag / Last-Modified 条件请求；请求之间歇 GAP 秒；robots.txt 不让抓的页面跳过。
+"""
+from __future__ import annotations
+
+import hashlib
+import re
+import sqlite3
+import time
+import urllib.error
+import urllib.parse
+import urllib.request
+import urllib.robotparser
+from collections import Counter
+from datetime import datetime, timedelta, timezone
+
+from . import htmldom
+from .edlib import DB_PATH as ED_DB
+from .moodlelib import FILES_DIR, now_iso
+from .syncer import safe_name
+
+UA = "monash-study-kit (personal study helper; fetches course notes linked from Moodle/Ed)"
+ROBOTS_AGENT = "monash-study-kit"     # robots.txt 里按这个名字匹配 User-agent
+ROBOTS_TTL = 24 * 3600
+STATIC_HOSTS = ("github.io", "gitlab.io", "readthedocs.io", "netlify.app", "pages.dev", "vercel.app")
+BLOCKED_HOSTS = ("monash.edu", "monash.edu.my", "edstem.org", "zoom.us", "youtube.com", "youtu.be",
+                 "google.com", "googleusercontent.com", "sharepoint.com", "microsoft.com", "office.com",
+                 "okta.com", "instructure.com", "panopto.com", "vimeo.com", "github.com")
+SKIP_EXT = re.compile(r"\.(zip|tar|gz|tgz|7z|rar|mp4|mov|mp3|png|jpe?g|gif|svg|ipynb|pptx?|docx?|xlsx?)$", re.I)
+CLASS_SITE_MIN = 5        # 同一门课链到同一个站这么多次，就当它是讲义站
+MAX_PAGES_PER_COURSE = 200
+MAX_BYTES = 5 * 1024 * 1024
+REFRESH_DAYS = 7
+GAP = 1.0
+FOLDER = "Course notes (web)"
+
+SCHEMA = """
+CREATE TABLE IF NOT EXISTS web_pages (
+    url TEXT PRIMARY KEY, course TEXT, title TEXT, path TEXT, etag TEXT, last_modified TEXT,
+    status TEXT, fetched_at TEXT
+);
+"""
+
+
+# ---------------------------------------------------------------- 挑页面
+
+def host_of(url: str) -> str:
+    return (urllib.parse.urlsplit(url).hostname or "").lower()
+
+
+def _host_matches(host: str, suffixes: tuple) -> bool:
+    return any(host == s or host.endswith("." + s) for s in suffixes)
+
+
+def normalize(url: str) -> str | None:
+    p = urllib.parse.urlsplit(url.strip())
+    if p.scheme not in ("http", "https") or not p.hostname:
+        return None
+    return urllib.parse.urlunsplit((p.scheme, p.netloc.lower(), p.path or "/", p.query, ""))
+
+
+def choose(links: list[tuple[str, str, str]]) -> list[tuple[str, str, str]]:
+    """[(课号, url, 标题)] → 要抓的那些。按课去重、过滤、每门课限量。"""
+    per_course: dict[str, dict[str, str]] = {}
+    for code, url, title in links:
+        u = normalize(url)
+        if code and u:
+            per_course.setdefault(code, {}).setdefault(u, title or "")
+    out = []
+    for code, urls in per_course.items():
+        counts = Counter(host_of(u) for u in urls)
+        picked = []
+        for u, title in urls.items():
+            host = host_of(u)
+            if _host_matches(host, BLOCKED_HOSTS) or SKIP_EXT.search(urllib.parse.urlsplit(u).path):
+                continue
+            if _host_matches(host, STATIC_HOSTS) or counts[host] >= CLASS_SITE_MIN:
+                picked.append((code, u, title))
+        out += picked[:MAX_PAGES_PER_COURSE]
+    return out
+
+
+def collect_links(moodle_con: sqlite3.Connection) -> list[tuple[str, str, str]]:
+    """跟踪的课里所有外部链接：Moodle 的 links 表 + Ed Lessons 的网页 slide。"""
+    rows = [(r["code"], r["url"], r["title"]) for r in moodle_con.execute(
+        "SELECT c.code, l.url, l.title FROM links l JOIN courses c ON c.id = l.course_id WHERE c.tracked = 1")]
+    if ED_DB.exists():
+        try:
+            con = sqlite3.connect(f"file:{ED_DB.as_posix()}?mode=ro", uri=True)
+            for code, url, title in con.execute(
+                    "SELECT c.code, f.url, f.title FROM lesson_files f JOIN courses c ON c.id = f.course_id "
+                    "WHERE c.tracked = 1 AND f.type = 'webpage' AND f.url IS NOT NULL"):
+                m = re.search(r"[A-Z]{3}\d{4}", code or "")
+                rows.append((m.group(0) if m else None, url, title))
+            con.close()
+        except sqlite3.Error:
+            pass
+    return rows
+
+
+# ---------------------------------------------------------------- HTML → 文字
+
+DROP = {"script", "style", "nav", "header", "footer", "aside", "noscript", "svg", "form", "button",
+        "iframe", "template", "select"}
+BLOCK = {"p", "div", "section", "article", "main", "ul", "ol", "table", "tr", "blockquote", "figure",
+         "details", "summary", "dl", "dt", "dd", "br", "hr"}
+
+
+def html_to_text(html_text: str) -> tuple[str, str]:
+    """返回 (标题, Markdown 风格的正文)。只保留阅读需要的：标题层级、段落、列表、代码块。"""
+    doc = htmldom.parse(html_text)
+    title_node = doc.find("title")
+    title = title_node.text() if title_node else ""
+    root = doc.find("main") or doc.find("article") or doc.find(role="main") or doc.find("body") or doc
+    h1 = root.find("h1")
+    if h1 and h1.text():
+        title = h1.text()
+    out: list[str] = []
+
+    def walk(n):
+        for ch in n.children:
+            if isinstance(ch, str):
+                out.append(re.sub(r"\s+", " ", ch))
+                continue
+            tag = ch.tag
+            if tag in DROP or ch.get("hidden") is not None or ch.get("role") == "tooltip" \
+                    or ch.classes & {"sr-only", "glossary-popup", "tooltip", "tooltip-text"}:
+                continue       # 术语悬浮提示之类，混进正文会把句子打断
+            if tag == "pre":
+                code = "".join(t for t in _texts(ch)).rstrip()
+                out.append(f"\n\n```\n{code}\n```\n\n")
+            elif tag in ("h1", "h2", "h3", "h4", "h5", "h6"):
+                out.append(f"\n\n{'#' * int(tag[1])} {ch.text()}\n\n")
+            elif tag == "li":
+                out.append("\n- ")
+                walk(ch)
+            elif tag == "code":
+                out.append(f"`{ch.text()}`")
+            elif tag in ("td", "th"):
+                walk(ch)
+                out.append(" | ")
+            elif tag == "img":
+                alt = ch.get("alt")
+                if alt:
+                    out.append(f"[图：{alt}]")
+            else:
+                if tag in BLOCK:
+                    out.append("\n\n")
+                walk(ch)
+                if tag in BLOCK:
+                    out.append("\n\n")
+
+    walk(root)
+    # 只整理代码块外面的空白：代码的缩进要原样保留（Haskell、Python 靠缩进）
+    parts = re.split(r"(\n\n```\n.*?\n```\n\n)", "".join(out), flags=re.S)
+    for i in range(0, len(parts), 2):
+        t = re.sub(r"[ \t]+\n", "\n", parts[i])
+        t = re.sub(r"\n[ \t]+", "\n", t)
+        parts[i] = re.sub(r"\n{3,}", "\n\n", t)
+    text = re.sub(r"\n{3,}", "\n\n", "".join(parts)).strip()
+    return " ".join(title.split()), text
+
+
+def _texts(n):
+    for ch in n.children:
+        if isinstance(ch, str):
+            yield ch
+        else:
+            if ch.tag == "br":
+                yield "\n"
+            yield from _texts(ch)
+
+
+# ---------------------------------------------------------------- 抓取
+
+_robots: dict[str, tuple[float, urllib.robotparser.RobotFileParser | None]] = {}
+
+
+def load_robots(site: str) -> urllib.robotparser.RobotFileParser | None:
+    """取 site 的 robots.txt。规则跟 urllib.robotparser.read() 一样：401/403 当全站禁止，
+    别的 4xx 当没有限制；连不上返回 None（说不清，交给后面的正式请求去报错）。"""
+    rp = urllib.robotparser.RobotFileParser(site + "/robots.txt")
+    req = urllib.request.Request(site + "/robots.txt", headers={"User-Agent": UA})
+    try:
+        with urllib.request.urlopen(req, timeout=15) as r:
+            rp.parse(r.read(512 * 1024).decode("utf-8", "replace").splitlines())
+    except urllib.error.HTTPError as e:
+        if e.code in (401, 403):
+            rp.disallow_all = True
+        else:
+            rp.allow_all = True
+    except (urllib.error.URLError, TimeoutError, OSError):
+        return None
+    return rp
+
+
+def robots_allowed(url: str, loader=load_robots) -> bool:
+    """robots.txt 允不允许抓这一页。每个站一天最多取一次 robots.txt（MCP 会一直开着）。"""
+    p = urllib.parse.urlsplit(url)
+    site = f"{p.scheme}://{p.netloc}"
+    at, rp = _robots.get(site, (0.0, None))
+    if site not in _robots or time.time() - at > ROBOTS_TTL:
+        rp = loader(site)
+        _robots[site] = (time.time(), rp)
+    return rp is None or rp.can_fetch(ROBOTS_AGENT, url)
+
+
+def fetch(url: str, etag: str | None = None, last_modified: str | None = None) -> dict:
+    """{"status": "ok"/"unchanged"/"skipped"/"error", ...}。不带任何登录信息。"""
+    if not robots_allowed(url):
+        return {"status": "skipped", "reason": "robots.txt 不允许"}
+    req = urllib.request.Request(url, headers={"User-Agent": UA, "Accept": "text/html,application/xhtml+xml"})
+    if etag:
+        req.add_header("If-None-Match", etag)
+    if last_modified:
+        req.add_header("If-Modified-Since", last_modified)
+    try:
+        with urllib.request.urlopen(req, timeout=30) as r:
+            ctype = (r.headers.get("Content-Type") or "").split(";")[0].strip().lower()
+            if ctype not in ("text/html", "application/xhtml+xml"):
+                return {"status": "skipped", "reason": ctype or "未知类型"}
+            body = r.read(MAX_BYTES + 1)
+            if len(body) > MAX_BYTES:
+                return {"status": "skipped", "reason": "太大"}
+            charset = r.headers.get_content_charset() or "utf-8"
+            return {"status": "ok", "html": body.decode(charset, "replace"), "final_url": r.geturl(),
+                    "etag": r.headers.get("ETag"), "last_modified": r.headers.get("Last-Modified")}
+    except urllib.error.HTTPError as e:
+        if e.code == 304:
+            return {"status": "unchanged"}
+        return {"status": "error", "reason": f"HTTP {e.code}"}
+    except (urllib.error.URLError, TimeoutError, OSError) as e:
+        return {"status": "error", "reason": str(getattr(e, "reason", e))}
+
+
+def _course_folder(moodle_con, code: str) -> tuple[int | None, str]:
+    row = moodle_con.execute("SELECT id, folder FROM courses WHERE code=? ORDER BY tracked DESC, id DESC LIMIT 1",
+                             (code,)).fetchone()
+    return (row["id"], row["folder"]) if row else (None, f"{code} (Ed)")
+
+
+def _dest(folder: str, title: str, url: str, taken: set[str]) -> str:
+    base = safe_name(title or urllib.parse.urlsplit(url).path.strip("/").replace("/", " ") or host_of(url), 70)
+    rel = f"{folder}/{FOLDER}/{base}.md"
+    if rel in taken:
+        rel = f"{folder}/{FOLDER}/{base} ({hashlib.sha1(url.encode()).hexdigest()[:6]}).md"
+    return rel
+
+
+def sync(moodle_con: sqlite3.Connection, log=print, now: datetime | None = None, fetcher=fetch) -> dict:
+    """抓该抓的讲义页。返回 {"fetched", "unchanged", "skipped", "errors"}。"""
+    moodle_con.executescript(SCHEMA)
+    now = now or datetime.now(timezone.utc)
+    stale = (now - timedelta(days=REFRESH_DAYS)).isoformat(timespec="seconds")
+    retry = (now - timedelta(days=1)).isoformat(timespec="seconds")     # 抓失败的第二天再试
+    known = {r["url"]: dict(r) for r in moodle_con.execute("SELECT * FROM web_pages")}
+    taken = {r["path"] for r in known.values() if r["path"]}
+    stats = {"fetched": 0, "unchanged": 0, "skipped": 0, "errors": 0}
+    last = 0.0
+    for code, url, link_title in choose(collect_links(moodle_con)):
+        prev = known.get(url)
+        if prev and (prev["fetched_at"] or "") > (retry if (prev["status"] or "").startswith("error") else stale):
+            continue
+        wait = GAP - (time.monotonic() - last)
+        if wait > 0:
+            time.sleep(wait)
+        last = time.monotonic()
+        res = fetcher(url, (prev or {}).get("etag"), (prev or {}).get("last_modified"))
+        course_id, folder = _course_folder(moodle_con, code)
+        path = (prev or {}).get("path")
+        if res["status"] == "ok":
+            title, text = html_to_text(res["html"])
+            if len(text) < 200:
+                res = {"status": "skipped", "reason": "没有正文"}
+            else:
+                path = path or _dest(folder, title or link_title, url, taken)
+                taken.add(path)
+                dest = FILES_DIR / path
+                dest.parent.mkdir(parents=True, exist_ok=True)
+                dest.write_text(f"# {title or link_title}\n\n来源：{url}\n\n{text}\n", encoding="utf-8")
+                if course_id is not None:
+                    moodle_con.execute(
+                        "INSERT INTO files (course_id, source, cmid, path, size, section, title, synced_at)"
+                        " VALUES (?,?,NULL,?,?,?,?,?) ON CONFLICT(course_id, source) DO UPDATE SET"
+                        " path=excluded.path, size=excluded.size, synced_at=excluded.synced_at",
+                        (course_id, f"web:{url}", path, dest.stat().st_size, FOLDER, title or link_title, now_iso()))
+                log(f"  ↓ 讲义网页 {path}")
+                link_title = title or link_title
+        key = {"ok": "fetched", "unchanged": "unchanged", "skipped": "skipped"}.get(res["status"], "errors")
+        stats[key] += 1
+        if key == "errors":
+            log(f"  ! {url}: {res.get('reason')}")
+        moodle_con.execute(
+            "INSERT OR REPLACE INTO web_pages VALUES (?,?,?,?,?,?,?,?)",
+            (url, code, link_title, path if res["status"] in ("ok", "unchanged") else (prev or {}).get("path"),
+             res.get("etag") or (prev or {}).get("etag"),
+             res.get("last_modified") or (prev or {}).get("last_modified"),
+             res["status"] if res["status"] != "error" else f"error: {res.get('reason')}",
+             now.isoformat(timespec="seconds")))
+        moodle_con.commit()
+    return stats