canvas-pull 2.4.0__py3-none-any.whl

canvas_pull/collector.py

@@ -0,0 +1,308 @@
+"""Find courses and list every downloadable file in them.
+
+Files are gathered from several places in a course, in priority order. When the same
+file shows up in more than one place it keeps the path from the first source, so a file
+in the Files tab never moves just because an assignment also links to it.
+"""
+
+from __future__ import annotations
+
+import re
+import unicodedata
+from collections.abc import Callable, Iterable, Iterator, Sequence
+from concurrent.futures import ThreadPoolExecutor, as_completed
+from dataclasses import dataclass, field
+from datetime import UTC, datetime
+from pathlib import PurePosixPath
+
+from .api import CanvasClient, CanvasError, ForbiddenError
+from .models import CanvasFile, Course, Folder
+
+SOURCES: tuple[str, ...] = (
+    "files",
+    "modules",
+    "syllabus",
+    "assignments",
+    "pages",
+    "announcements",
+    "discussions",
+)
+
+# Only the Files tab by default; the others are opt-in through the `sources` setting.
+DEFAULT_SOURCES: tuple[str, ...] = ("files",)
+
+_UNSAFE = re.compile(r'[\\/:*?"<>|\x00-\x1f]')
+_WINDOWS_RESERVED = {"CON", "PRN", "AUX", "NUL"} | {
+    f"{p}{i}" for p in ("COM", "LPT") for i in range(1, 10)
+}
+# Matches /files/123 and /courses/45/files/123 in links, images and API endpoints.
+_FILE_LINK = re.compile(r"/(?:courses/(\d+)/)?files/(\d+)")
+
+
+def sanitize(name: str, max_len: int = 150) -> str:
+    """Make a Canvas name safe as a single path component on macOS/Linux/Windows."""
+    name = unicodedata.normalize("NFC", name)
+    name = _UNSAFE.sub("_", name).strip().rstrip(". ")
+    if name.split(".", 1)[0].upper() in _WINDOWS_RESERVED:
+        name = "_" + name
+    if len(name) > max_len:
+        stem, dot, ext = name.rpartition(".")
+        if dot and 0 < len(ext) <= 10:
+            name = stem[: max_len - len(ext) - 1].rstrip() + "." + ext
+        else:
+            name = name[:max_len].rstrip()
+    return name or "_"
+
+
+@dataclass(frozen=True)
+class RemoteFile:
+    file: CanvasFile
+    course: Course
+    path: str  # POSIX path relative to the save directory
+    source: str = "files"
+
+    @property
+    def locked(self) -> bool:
+        return self.file.locked_for_user or not self.file.url
+
+
+@dataclass
+class CourseScan:
+    course: Course
+    files: list[RemoteFile] = field(default_factory=list)
+    files_tab_hidden: bool = False
+    error: str | None = None
+
+    def count(self, source: str) -> int:
+        return sum(1 for f in self.files if f.source == source)
+
+
+# -- courses ------------------------------------------------------------------
+
+
+def list_courses(
+    client: CanvasClient,
+    *,
+    include_past: bool = False,
+    include_completed: bool = False,
+    now: datetime | None = None,
+) -> list[Course]:
+    """Active courses (minus ones whose term ended), optionally plus completed ones."""
+    now = now or datetime.now(UTC)
+    courses: dict[int, Course] = {}
+    states = ["active"] + (["completed"] if include_completed else [])
+    for state in states:
+        params = {"enrollment_state": state, "include[]": ["term"]}
+        for raw in client.paginate("courses", params):
+            course = Course.model_validate(raw)
+            if course.access_restricted_by_date or course.id in courses:
+                continue
+            if state == "completed":
+                course = course.model_copy(update={"completed": True})
+            elif not include_past and not include_completed:
+                end = (course.term.end_at if course.term else None) or course.end_at
+                if end and end < now:
+                    continue
+            courses[course.id] = course
+    return sorted(courses.values(), key=lambda c: c.label.lower())
+
+
+def course_dirnames(courses: Iterable[Course]) -> dict[int, str]:
+    """One folder name per course; disambiguates courses that share a name."""
+    courses = list(courses)
+    base = {c.id: sanitize(c.label) for c in courses}
+    counts: dict[str, int] = {}
+    for name in base.values():
+        counts[name.lower()] = counts.get(name.lower(), 0) + 1
+    return {
+        cid: (f"{name} ({cid})" if counts[name.lower()] > 1 else name) for cid, name in base.items()
+    }
+
+
+# -- sources ------------------------------------------------------------------
+# Each source yields (file, folder parts) or (file id, owning course id, folder parts)
+# when only a link is known; links are resolved later, after deduplication.
+
+Found = tuple[CanvasFile, list[str]]
+Link = tuple[int, int, list[str]]
+
+
+def file_links(html: str | None, course_id: int) -> list[tuple[int, int]]:
+    """(file id, course id) pairs linked from a piece of Canvas HTML, in order, unique."""
+    out: dict[int, int] = {}
+    for cid, fid in _FILE_LINK.findall(html or ""):
+        out.setdefault(int(fid), int(cid) if cid else course_id)
+    return list(out.items())
+
+
+def _files_tab(client: CanvasClient, course: Course) -> Iterator[Found]:
+    # The folder list and the file list are independent: fetch them at the same time.
+    with ThreadPoolExecutor(max_workers=1) as pool:
+        folders_future = pool.submit(lambda: list(client.paginate(f"courses/{course.id}/folders")))
+        files_raw = list(client.paginate(f"courses/{course.id}/files"))
+        folders = {
+            f.id: [sanitize(p) for p in PurePosixPath(f.full_name).parts[1:]]
+            for f in (Folder.model_validate(r) for r in folders_future.result())
+        }
+    for raw in files_raw:
+        f = CanvasFile.model_validate(raw)
+        yield f, folders.get(f.folder_id or -1, [])
+
+
+def _modules(client: CanvasClient, course: Course) -> Iterator[Link]:
+    # Ask for items inline; Canvas leaves them out for very large modules, and only those
+    # need a request of their own.
+    params = {"include[]": ["items"]}
+    for module in client.paginate(f"courses/{course.id}/modules", params):
+        folder = ["Modules", sanitize(module.get("name") or f"Module {module.get('id')}")]
+        items = module.get("items")
+        if items is None:
+            items = client.paginate(f"courses/{course.id}/modules/{module['id']}/items")
+        for item in items:
+            if item.get("type") == "File" and item.get("content_id"):
+                yield int(item["content_id"]), course.id, folder
+
+
+def _syllabus(client: CanvasClient, course: Course) -> Iterator[Link]:
+    raw = client.get(f"courses/{course.id}", {"include[]": ["syllabus_body"]})
+    for fid, cid in file_links(raw.get("syllabus_body"), course.id):
+        yield fid, cid, ["Syllabus"]
+
+
+def _html_items(
+    client: CanvasClient, course: Course, path: str, params: dict, root: str, title: str, body: str
+) -> Iterator[Found | Link]:
+    for item in client.paginate(path, params):
+        folder = [root, sanitize(str(item.get(title) or item.get("id")))]
+        for att in item.get("attachments") or []:
+            if isinstance(att, dict) and att.get("id"):
+                yield CanvasFile.model_validate(att), folder
+        for fid, cid in file_links(item.get(body), course.id):
+            yield fid, cid, folder
+
+
+def _assignments(client: CanvasClient, course: Course):
+    path = f"courses/{course.id}/assignments"
+    return _html_items(client, course, path, {}, "Assignments", "name", "description")
+
+
+def _pages(client: CanvasClient, course: Course):
+    params = {"include[]": ["body"], "published": "true"}
+    return _html_items(
+        client, course, f"courses/{course.id}/pages", params, "Pages", "title", "body"
+    )
+
+
+def _announcements(client: CanvasClient, course: Course):
+    path = f"courses/{course.id}/discussion_topics"
+    params = {"only_announcements": "true"}
+    return _html_items(client, course, path, params, "Announcements", "title", "message")
+
+
+def _discussions(client: CanvasClient, course: Course):
+    path = f"courses/{course.id}/discussion_topics"
+    return _html_items(client, course, path, {}, "Discussions", "title", "message")
+
+
+_SCANNERS = {
+    "files": _files_tab,
+    "modules": _modules,
+    "syllabus": _syllabus,
+    "assignments": _assignments,
+    "pages": _pages,
+    "announcements": _announcements,
+    "discussions": _discussions,
+}
+
+
+def _resolve(client: CanvasClient, fid: int, cid: int) -> CanvasFile | None:
+    for path in (f"courses/{cid}/files/{fid}", f"files/{fid}"):
+        try:
+            return CanvasFile.model_validate(client.get(path))
+        except ForbiddenError:
+            continue
+    return None
+
+
+def scan_courses(
+    client: CanvasClient,
+    courses: Sequence[Course],
+    dirnames: dict[int, str],
+    sources: Sequence[str] = DEFAULT_SOURCES,
+    *,
+    workers: int = 6,
+    on_done: Callable[[CourseScan], None] | None = None,
+) -> list[CourseScan]:
+    """Scan several courses at once. Results come back in the order of `courses`."""
+    results: dict[int, CourseScan] = {}
+    with ThreadPoolExecutor(max_workers=max(1, workers)) as pool:
+        futures = {pool.submit(scan_course, client, c, dirnames[c.id], sources): c for c in courses}
+        for future in as_completed(futures):
+            scan = future.result()  # scan_course turns Canvas errors into scan.error
+            results[scan.course.id] = scan
+            if on_done:
+                on_done(scan)
+    return [results[c.id] for c in courses]
+
+
+def _assign_paths(
+    found: list[tuple[CanvasFile, list[str], str]], course: Course, root: str
+) -> list[RemoteFile]:
+    """Unique, case-insensitively distinct paths. Older files keep the plain name."""
+    taken: set[str] = set()
+    out: list[RemoteFile] = []
+    for f, parts, source in sorted(found, key=lambda e: e[0].id):
+        name = sanitize(f.display_name or f.filename or str(f.id))
+        path = str(PurePosixPath(root, *parts, name))
+        if path.lower() in taken:
+            stem, dot, ext = name.rpartition(".")
+            name = f"{stem} ({f.id}).{ext}" if dot and stem else f"{name} ({f.id})"
+            path = str(PurePosixPath(root, *parts, name))
+        taken.add(path.lower())
+        out.append(RemoteFile(file=f, course=course, path=path, source=source))
+    return out
+
+
+def scan_course(
+    client: CanvasClient, course: Course, dirname: str, sources: Sequence[str] = DEFAULT_SOURCES
+) -> CourseScan:
+    """List a course's files from every enabled source.
+
+    When the Files tab is enabled but hidden or empty, Modules is scanned as a fallback
+    even if it is not enabled, so such courses still get their files.
+    A hidden tab (401/403/404) just skips that source. Any other error marks the whole
+    scan as failed, so no file of this course is reported as deleted.
+    """
+    scan = CourseScan(course=course)
+    files: dict[int, tuple[CanvasFile, list[str], str]] = {}
+    links: dict[int, tuple[int, list[str], str]] = {}
+    try:
+        for source in SOURCES:
+            fallback = (
+                source == "modules" and "files" in sources and (scan.files_tab_hidden or not files)
+            )
+            if source not in sources and not fallback:
+                continue
+            try:
+                for entry in _SCANNERS[source](client, course):
+                    if isinstance(entry[0], CanvasFile):
+                        f, parts = entry  # type: ignore[misc]
+                        if f.id not in files and f.id not in links:
+                            files[f.id] = (f, parts, source)
+                    else:
+                        fid, cid, parts = entry  # type: ignore[misc]
+                        if fid not in files and fid not in links:
+                            links[fid] = (cid, parts, source)
+            except ForbiddenError:
+                if source == "files":
+                    scan.files_tab_hidden = True
+        if links:
+            with ThreadPoolExecutor(max_workers=8) as pool:
+                resolved = pool.map(lambda kv: _resolve(client, kv[0], kv[1][0]), links.items())
+                for (fid, (_cid, parts, source)), f in zip(links.items(), resolved, strict=True):
+                    if f is not None:
+                        files[fid] = (f, parts, source)
+        scan.files = _assign_paths(list(files.values()), course, dirname)
+    except CanvasError as exc:
+        scan.error = str(exc)
+    return scan

canvas_pull/config.py

@@ -0,0 +1,211 @@
+"""Load and save settings.
+
+Sources, lowest to highest priority:
+  1. ~/.config/canvas-pull/config.toml (written by `canvas-pull init`)
+  2. a `.env` file in the current directory
+  3. environment variables
+
+The token is read from those sources first, then from the system keychain.
+Legacy keys from v1 (WEBSITE, ACCESS_TOKEN, SAVE_DIR) are still accepted.
+"""
+
+from __future__ import annotations
+
+import os
+import tomllib
+from collections.abc import Mapping
+from dataclasses import dataclass, field
+from pathlib import Path
+
+from platformdirs import user_config_path
+
+from .collector import DEFAULT_SOURCES, SOURCES
+
+APP_NAME = "canvas-pull"
+KEYRING_SERVICE = "canvas-pull"
+DEFAULT_SAVE_DIR = "~/Canvas"
+
+# setting name -> accepted environment / .env keys (first match wins)
+ENV_KEYS: dict[str, tuple[str, ...]] = {
+    "website": ("CANVAS_URL", "WEBSITE"),
+    "token": ("CANVAS_TOKEN", "ACCESS_TOKEN"),
+    "save_dir": ("CANVAS_SAVE_DIR", "SAVE_DIR"),
+}
+
+
+class ConfigError(Exception):
+    pass
+
+
+@dataclass
+class Config:
+    website: str
+    save_dir: Path
+    token: str = ""
+    concurrency: int = 4
+    scan_concurrency: int = 8
+    keep_history: bool = True
+    git: bool = False
+    exclude: list[str] = field(default_factory=list)
+    include_completed: bool = False
+    sources: list[str] = field(default_factory=lambda: list(DEFAULT_SOURCES))
+
+
+def default_config_file() -> Path:
+    return user_config_path(APP_NAME) / "config.toml"
+
+
+def normalize_website(value: str) -> str:
+    """'oc.sjtu.edu.cn/' -> 'https://oc.sjtu.edu.cn'"""
+    url = value.strip().rstrip("/")
+    if not url:
+        raise ConfigError("Canvas URL is empty.")
+    if "://" not in url:
+        url = "https://" + url
+    if url.endswith("/api/v1"):
+        url = url[: -len("/api/v1")]
+    return url
+
+
+def parse_dotenv(path: Path) -> dict[str, str]:
+    values: dict[str, str] = {}
+    if not path.is_file():
+        return values
+    for raw in path.read_text(encoding="utf-8").splitlines():
+        line = raw.strip()
+        if not line or line.startswith("#") or "=" not in line:
+            continue
+        key, _, value = line.partition("=")
+        key = key.strip().removeprefix("export ").strip()
+        value = value.strip()
+        if len(value) >= 2 and value[0] == value[-1] and value[0] in "'\"":
+            value = value[1:-1]
+        values[key] = value
+    return values
+
+
+def _from_env(source: Mapping[str, str]) -> dict[str, str]:
+    out: dict[str, str] = {}
+    for name, keys in ENV_KEYS.items():
+        for key in keys:
+            if source.get(key):
+                out[name] = source[key]
+                break
+    return out
+
+
+def _keyring_get(website: str) -> str:
+    try:
+        import keyring
+
+        return keyring.get_password(KEYRING_SERVICE, website) or ""
+    except Exception:  # no backend, locked keychain, etc.
+        return ""
+
+
+def load_config(
+    *,
+    require_token: bool = True,
+    config_file: Path | None = None,
+    cwd: Path | None = None,
+    environ: Mapping[str, str] | None = None,
+    use_keyring: bool = True,
+) -> Config:
+    config_file = config_file or default_config_file()
+    cwd = cwd or Path.cwd()
+    environ = os.environ if environ is None else environ
+
+    data: dict = {}
+    if config_file.is_file():
+        try:
+            data = tomllib.loads(config_file.read_text(encoding="utf-8"))
+        except tomllib.TOMLDecodeError as exc:
+            raise ConfigError(f"Cannot parse {config_file}: {exc}") from exc
+    data.update(_from_env(parse_dotenv(cwd / ".env")))
+    data.update(_from_env(environ))
+
+    if not data.get("website"):
+        raise ConfigError("No Canvas URL configured. Run `canvas-pull init` first.")
+    website = normalize_website(str(data["website"]))
+
+    save_dir = Path(str(data.get("save_dir") or DEFAULT_SAVE_DIR)).expanduser()
+    if not save_dir.is_absolute():
+        save_dir = (cwd / save_dir).resolve()
+
+    token = str(data.get("token") or "")
+    if not token and use_keyring:
+        token = _keyring_get(website)
+    if require_token and not token:
+        raise ConfigError("No access token found. Run `canvas-pull init`, or set CANVAS_TOKEN.")
+
+    try:
+        return Config(
+            website=website,
+            save_dir=save_dir,
+            token=token,
+            concurrency=max(1, int(data.get("concurrency", 4))),
+            scan_concurrency=max(1, min(16, int(data.get("scan_concurrency", 8)))),
+            keep_history=bool(data.get("keep_history", True)),
+            git=bool(data.get("git", False)),
+            exclude=[str(x) for x in data.get("exclude", [])],
+            include_completed=bool(data.get("include_completed", False)),
+            sources=_sources(data.get("sources", list(DEFAULT_SOURCES))),
+        )
+    except (TypeError, ValueError) as exc:
+        raise ConfigError(f"Invalid value in {config_file}: {exc}") from exc
+
+
+def _sources(value: object) -> list[str]:
+    if not isinstance(value, list):
+        raise ConfigError(f"`sources` must be a list, e.g. {list(SOURCES)}")
+    unknown = [s for s in value if s not in SOURCES]
+    if unknown:
+        raise ConfigError(f"Unknown source(s) {unknown}. Choose from {list(SOURCES)}.")
+    return [str(s) for s in value]
+
+
+def _toml_value(value: object) -> str:
+    if isinstance(value, bool):
+        return "true" if value else "false"
+    if isinstance(value, int):
+        return str(value)
+    if isinstance(value, list):
+        return "[" + ", ".join(_toml_value(v) for v in value) + "]"
+    escaped = str(value).replace("\\", "\\\\").replace('"', '\\"')
+    return f'"{escaped}"'
+
+
+def save_config(cfg: Config, *, config_file: Path | None = None) -> str:
+    """Write settings; store the token in the keychain when possible.
+
+    Returns where the token ended up: "keychain" or the config file path.
+    """
+    config_file = config_file or default_config_file()
+    config_file.parent.mkdir(parents=True, exist_ok=True)
+
+    values: dict[str, object] = {
+        "website": cfg.website,
+        "save_dir": str(cfg.save_dir),
+        "concurrency": cfg.concurrency,
+        "scan_concurrency": cfg.scan_concurrency,
+        "keep_history": cfg.keep_history,
+        "git": cfg.git,
+        "exclude": cfg.exclude,
+        "include_completed": cfg.include_completed,
+        "sources": cfg.sources,
+    }
+    stored_in = "keychain"
+    try:
+        import keyring
+
+        keyring.set_password(KEYRING_SERVICE, cfg.website, cfg.token)
+    except Exception:
+        values["token"] = cfg.token
+        stored_in = str(config_file)
+
+    lines = ["# canvas-pull settings. Edit freely; run `canvas-pull config` to check."]
+    lines += [f"{k} = {_toml_value(v)}" for k, v in values.items()]
+    config_file.write_text("\n".join(lines) + "\n", encoding="utf-8")
+    if "token" in values:
+        config_file.chmod(0o600)
+    return stored_in

canvas_pull/gitops.py

@@ -0,0 +1,52 @@
+"""Optional: commit each sync in the save directory so `git log` shows history."""
+
+from __future__ import annotations
+
+import shutil
+import subprocess
+from collections import Counter
+from pathlib import Path
+
+from .manifest import STATE_DIR
+from .sync import Change
+
+GITIGNORE = f"{STATE_DIR}/tmp/\n{STATE_DIR}/history/\n.DS_Store\n"
+
+
+class GitError(Exception):
+    pass
+
+
+def _git(save_dir: Path, *args: str) -> subprocess.CompletedProcess[str]:
+    proc = subprocess.run(
+        ["git", "-C", str(save_dir), *args], capture_output=True, text=True, check=False
+    )
+    if proc.returncode != 0:
+        raise GitError(proc.stderr.strip() or f"git {' '.join(args)} failed")
+    return proc
+
+
+def commit_sync(save_dir: Path, changes: list[Change]) -> str | None:
+    """Commit everything in save_dir. Returns the commit summary, or None if nothing changed."""
+    if shutil.which("git") is None:
+        raise GitError("git is not installed")
+    if not (save_dir / ".git").exists():
+        _git(save_dir, "init", "-q")
+    ignore = save_dir / ".gitignore"
+    if not ignore.exists():
+        ignore.write_text(GITIGNORE, encoding="utf-8")
+
+    _git(save_dir, "add", "-A")
+    if not _git(save_dir, "status", "--porcelain").stdout.strip():
+        return None
+
+    counts = Counter(str(c.kind) for c in changes)
+    summary = "sync: " + (", ".join(f"{n} {k}" for k, n in sorted(counts.items())) or "update")
+    body = "\n".join(
+        f"{c.kind}: {c.old_path} -> {c.path}"
+        if c.old_path and c.old_path != c.path
+        else f"{c.kind}: {c.path}"
+        for c in changes
+    )
+    _git(save_dir, "commit", "-q", "-m", summary, "-m", body or "metadata update")
+    return summary