moodle-scraper 0.1.0__tar.gz

moodle_scraper-0.1.0/.env.example

@@ -0,0 +1,30 @@
+# Copy to `.env` and fill in. `.env` is git-ignored — never commit a real cookie.
+#
+# The MoodleSession cookie is a secret: anyone holding it can act as you until the
+# session times out (8 hours on learn.haw-kiel.de). Treat it like a password.
+
+# Base URL of the Moodle instance.
+MOODLE_BASE_URL=https://learn.haw-kiel.de
+
+# Option A: paste the cookie value (or a full "Cookie:" header) directly.
+# MOODLE_COOKIE=MoodleSession=replace-me
+
+# Option B: point at an exported Netscape cookies.txt file.
+# MOODLE_COOKIE_FILE=./cookies.txt
+
+# Where mirrored course files and manifest.json files are written.
+MOODLE_OUT=data
+
+# Politeness: seconds between requests, and parallel downloads (1 = sequential).
+MOODLE_DELAY=0.5
+MOODLE_CONCURRENCY=1
+
+# Per-request timeout in seconds. Raise this if the server answers slowly
+# (learn.haw-kiel.de dashboard pages can take well over 30 seconds).
+MOODLE_TIMEOUT=30
+
+# Optional byte cap per file; unset/0 means unlimited.
+# MOODLE_MAX_FILE_SIZE=0
+
+# Set to 1 only behind an intercepting campus proxy that breaks TLS verification.
+MOODLE_INSECURE=0

moodle_scraper-0.1.0/.github/workflows/ci.yml

@@ -0,0 +1,36 @@
+name: CI
+
+on:
+  push:
+    branches: [main]
+  pull_request:
+
+jobs:
+  test:
+    name: Test (Python ${{ matrix.python-version }})
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        python-version: ["3.11", "3.12", "3.13"]
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Set up Python ${{ matrix.python-version }}
+        uses: actions/setup-python@v5
+        with:
+          python-version: ${{ matrix.python-version }}
+
+      - name: Install package with dev dependencies
+        run: pip install -e ".[dev]"
+
+      - name: Lint (ruff)
+        run: |
+          ruff check .
+          ruff format --check .
+
+      - name: Type check (mypy)
+        run: mypy src
+
+      - name: Test (pytest)
+        run: pytest -q

moodle_scraper-0.1.0/.github/workflows/release.yml

@@ -0,0 +1,47 @@
+name: Release
+
+on:
+  push:
+    tags: ["v*"]
+
+jobs:
+  build:
+    name: Build distribution
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: "3.13"
+
+      - name: Build sdist and wheel
+        run: |
+          pip install build twine
+          python -m build
+          twine check dist/*
+
+      - name: Upload distribution artifact
+        uses: actions/upload-artifact@v4
+        with:
+          name: dist
+          path: dist/
+
+  publish:
+    name: Publish to PyPI
+    needs: build
+    runs-on: ubuntu-latest
+    environment: pypi
+    permissions:
+      # Required for PyPI trusted publishing (OIDC) — no API token stored.
+      id-token: write
+    steps:
+      - name: Download distribution artifact
+        uses: actions/download-artifact@v4
+        with:
+          name: dist
+          path: dist/
+
+      - name: Publish
+        uses: pypa/gh-action-pypi-publish@release/v1

moodle_scraper-0.1.0/.gitignore

@@ -0,0 +1,30 @@
+.venv/
+venv/
+__pycache__/
+*.py[cod]
+*.egg-info/
+.eggs/
+
+.pytest_cache/
+.mypy_cache/
+.ruff_cache/
+.coverage
+htmlcov/
+
+dist/
+build/
+
+# Secrets and scraped content must never be committed
+.env
+*.session
+*.part
+data/
+
+# Kilo-managed worktrees
+.kilo/
+
+# Editors / OS
+.vscode/
+.idea/
+Thumbs.db
+desktop.ini

moodle_scraper-0.1.0/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2026 Simon Schwärzler
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

moodle_scraper-0.1.0/PKG-INFO

@@ -0,0 +1,335 @@
+Metadata-Version: 2.5
+Name: moodle-scraper
+Version: 0.1.0
+Summary: Mirror the course files of your learn.haw-kiel.de Moodle courses into a local, resumable folder tree.
+Project-URL: Homepage, https://github.com/RTXC01/moodle_scrape
+Project-URL: Repository, https://github.com/RTXC01/moodle_scrape
+Project-URL: Issues, https://github.com/RTXC01/moodle_scrape/issues
+Author-email: RTXC1 <audiundlambofan@gmail.com>
+License-Expression: MIT
+License-File: LICENSE
+Keywords: backup,course-files,e-learning,haw-kiel,moodle,scraper
+Classifier: Development Status :: 4 - Beta
+Classifier: Environment :: Console
+Classifier: Intended Audience :: Education
+Classifier: Operating System :: OS Independent
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.11
+Classifier: Programming Language :: Python :: 3.12
+Classifier: Programming Language :: Python :: 3.13
+Classifier: Topic :: Education
+Classifier: Topic :: Utilities
+Requires-Python: >=3.11
+Requires-Dist: beautifulsoup4>=4.12
+Requires-Dist: httpx>=0.27
+Requires-Dist: lxml>=5.2
+Requires-Dist: python-dotenv>=1.0
+Requires-Dist: rich>=13.7
+Requires-Dist: typer>=0.12
+Provides-Extra: dev
+Requires-Dist: build>=1.2; extra == 'dev'
+Requires-Dist: mypy>=1.11; extra == 'dev'
+Requires-Dist: pytest-cov>=5.0; extra == 'dev'
+Requires-Dist: pytest>=8.2; extra == 'dev'
+Requires-Dist: ruff>=0.6; extra == 'dev'
+Requires-Dist: twine>=5.1; extra == 'dev'
+Description-Content-Type: text/markdown
+
+# moodle-scraper
+
+[![CI](https://github.com/RTXC01/moodle_scrape/actions/workflows/ci.yml/badge.svg)](https://github.com/RTXC01/moodle_scrape/actions/workflows/ci.yml)
+[![PyPI](https://img.shields.io/pypi/v/moodle-scraper)](https://pypi.org/project/moodle-scraper/)
+[![Python](https://img.shields.io/pypi/pyversions/moodle-scraper)](https://pypi.org/project/moodle-scraper/)
+[![License: MIT](https://img.shields.io/badge/license-MIT-green.svg)](LICENSE)
+
+Mirror the **course files** of your enrolled Moodle courses on
+[`learn.haw-kiel.de`](https://learn.haw-kiel.de) into a local folder tree, with a per-course
+`manifest.json` that makes repeat runs incremental and resumable.
+
+It is a personal, read-only backup tool: it downloads the files you already have access to, and
+never touches grades, submissions, forums, quizzes, or messages.
+
+Built for HAW Kiel's Moodle instance, but any Moodle base URL works (`MOODLE_BASE_URL`).
+
+## What it does
+
+- Discovers your courses (dashboard where available, or directly by course id/URL).
+- Walks each course's sections and activities and resolves the file links behind
+  `mod_resource`, `mod_folder`, `mod_url`, `mod_page`, `mod_book` and `mod_imscp`.
+- Mirrors those files under `data/<shortname>-<id>/<NN>-<section>/<activity>/...`.
+- Records size / `Last-Modified` / ETag / optional SHA-1 in `manifest.json`.
+- On later runs it skips unchanged files, resumes interrupted `.part` downloads, and can prune
+  files that were removed upstream.
+
+## What it does *not* do
+
+Forums, grades, assignment submissions, quizzes, calendar, messages, streaming video/HLS capture,
+HTML→Markdown conversion, scheduling, or any upload/POST — all out of scope. It also does not
+automate SSO login.
+
+## Legal and ethical use
+
+- Only use this for content you are **already enrolled in**, as a **personal backup**. Do not
+  redistribute any downloaded material.
+- The tool only issues `GET`/`HEAD` requests. It never posts, submits, or grades anything.
+- Defaults are deliberately polite: sequential requests, a `0.5s` delay, one download at a time,
+  exponential backoff on `429`/`5xx`, and an identifying `User-Agent`. Please keep the delay unless
+  you have a reason not to.
+- Your `MoodleSession` cookie is a secret. It is read from `.env`/environment/a cookie file, never
+  logged, and never written into a manifest. `.env` and `data/` are git-ignored.
+
+## Requirements
+
+- Python **3.11+**
+- A browser where you are logged in to your Moodle instance.
+
+## Install
+
+From PyPI:
+
+```powershell
+pip install moodle-scraper
+```
+
+From source:
+
+```powershell
+git clone https://github.com/RTXC01/moodle_scrape.git moodle_scrape
+cd moodle_scrape
+python -m venv .venv
+.\.venv\Scripts\Activate.ps1
+pip install -e ".[dev]"
+```
+
+Then copy the example config and fill in your cookie:
+
+```powershell
+Copy-Item .env.example .env
+```
+
+## Cookie export
+
+The tool reuses the browser session you already have; it never asks for your password and does not
+drive SSO. Export the `MoodleSession` cookie and put it in `.env` as `MOODLE_COOKIE` (or save a
+Netscape `cookies.txt` and point `MOODLE_COOKIE_FILE` at it).
+
+### Chrome / Edge
+
+1. Log in at <https://learn.haw-kiel.de>.
+2. Open DevTools (`F12`) → **Application** → **Storage** → **Cookies** →
+   `https://learn.haw-kiel.de`.
+3. Copy the **Value** of the `MoodleSession` cookie.
+4. Put it in `.env`:
+
+   ```dotenv
+   MOODLE_COOKIE=MoodleSession=<paste-value>
+   ```
+
+   A bare value also works (`MOODLE_COOKIE=<paste-value>`), as does a full header
+   (`MOODLE_COOKIE=Cookie: MoodleSession=...`).
+
+### Firefox
+
+1. Log in at <https://learn.haw-kiel.de>.
+2. DevTools (`F12`) → **Storage** → **Cookies** → `https://learn.haw-kiel.de` → copy the
+   `MoodleSession` **Value**.
+3. Paste it into `.env` as above. Alternatively, use a "cookies.txt" export extension, save the
+   Netscape file, and set `MOODLE_COOKIE_FILE=./cookies.txt`.
+
+> The Moodle session times out after ~8 hours (`sessiontimeout: 28800`). When it expires, commands
+> exit with code `3` and tell you to re-export the cookie.
+
+## Commands
+
+```text
+moodle-scrape doctor
+moodle-scrape courses
+moodle-scrape files <course>
+moodle-scrape sync [--course ID]... [options]
+```
+
+### `doctor`
+
+Validates the cookie, prints the logged-in account and how many courses are visible.
+Exit code `0` when the session is good, `3` when it is expired or invalid.
+
+```powershell
+moodle-scrape doctor
+```
+
+### `courses`
+
+Prints a table of enrolled courses (id, shortname, fullname, discovered file count).
+
+```powershell
+moodle-scrape courses
+```
+
+### `files`
+
+Dry discovery for one course: lists the files that *would* be downloaded, without downloading.
+
+```powershell
+moodle-scrape files 12345
+moodle-scrape files https://learn.haw-kiel.de/course/view.php?id=12345
+```
+
+### `sync`
+
+The main command.
+
+```text
+--course ID|URL     Repeatable. Defaults to all enrolled courses.
+--out DIR           Output directory (default: data).
+--delay SECONDS     Delay between requests (default: 0.5).
+--timeout SECONDS   Per-request timeout (default: 30; raise it for slow servers).
+--concurrency N     Parallel downloads (default: 1 = sequential).
+--dry-run           Resolve and report everything, download nothing.
+--prune             Delete local files that no longer exist upstream.
+--checksum          Compute SHA-1 while downloading (slower).
+--max-file-size N   Skip files larger than N bytes (default: unlimited).
+--insecure          Disable TLS verification (only behind a broken campus proxy).
+--verbose           Verbose logging; the cookie is always redacted.
+```
+
+Examples:
+
+```powershell
+# Everything, politely
+moodle-scrape sync
+
+# One course, preview only
+moodle-scrape sync --course 12345 --dry-run
+
+# Two courses, checksums, prune removed files
+moodle-scrape sync --course 12345 --course 67890 --checksum --prune
+```
+
+> **Slow dashboard?** Some Moodle instances answer very slowly under load. The tool retries
+> transient login bounces and transport errors automatically; for this site a timeout of
+> `MOODLE_TIMEOUT=180` in `.env` is a good idea. Single-course runs fetch the course page directly
+> and skip the dashboard entirely.
+
+## Output layout
+
+```text
+data/
+  Analysis-12345/                 # <shortname>-<id>
+    00-General/
+      Slides.pdf
+      Scripts/
+        chapter1.pdf              # nested folder structure preserved
+    01-Week 1/
+      Exercises/
+        Notes.txt
+    _external/                    # recorded links, not downloaded
+    manifest.json
+  Physics-67890/
+    ...
+    manifest.json
+```
+
+Filenames and directories are sanitised for Windows (`<>:"/\|?*`, control characters, trailing
+dots/spaces, reserved device names such as `CON`/`PRN`, over-long components, and path-length
+limits). If two different URLs would map to the same path, a short deterministic hash
+(`__<8 hex chars>`) is inserted before the extension.
+
+## Manifest format
+
+One `manifest.json` per course, written atomically after the course finishes (and rewritten after
+partial failures so a rerun resumes instead of restarting):
+
+```json
+{
+  "schema_version": 1,
+  "course": { "id": 12345, "shortname": "Analysis", "fullname": "Analysis I" },
+  "generated_at": "2026-09-15T19:30:00+00:00",
+  "files": [
+    {
+      "rel_path": "Analysis-12345/00-General/Slides.pdf",
+      "url": "https://learn.haw-kiel.de/pluginfile.php/1/mod_resource/content/0/Slides.pdf",
+      "section": "General",
+      "activity": "",
+      "filename": "Slides.pdf",
+      "size": 1048576,
+      "last_modified": "Wed, 03 Sep 2026 10:00:00 GMT",
+      "etag": "\"abc123\"",
+      "sha1": null,
+      "downloaded": true,
+      "external": false,
+      "status": "downloaded",
+      "fetched_at": "2026-09-15T19:31:00+00:00"
+    }
+  ]
+}
+```
+
+`status` is one of `pending`, `downloaded`, `skipped`, `missing`, `failed`, `external`,
+`unsupported`. `external: true` entries are links pointing outside the Moodle host; they are
+recorded, never fetched.
+
+## Exit codes
+
+| Code | Meaning |
+| --- | --- |
+| `0` | Success |
+| `1` | Unexpected error |
+| `2` | Bad usage or configuration (no cookie, bad values, missing cookie file) |
+| `3` | Session expired or invalid |
+
+## Incremental behaviour
+
+1. If the manifest entry for a URL matches the local file's size (and, when available,
+   `Last-Modified`/ETag via a conditional request), the file is **skipped**.
+2. Otherwise the file is downloaded to `<name>.part`, flushed, then atomically renamed. If the run
+   is interrupted, the `.part` file stays and the next run resumes it with a `Range` request when
+   the server supports it.
+3. `404` is recorded as `missing` and the run continues. `401`/`403` triggers a session check and
+   aborts the run.
+
+## Development
+
+```powershell
+pip install -e ".[dev]"
+ruff check .
+ruff format --check .
+mypy src
+pytest -q
+```
+
+Tests are fully offline: HTML fixtures under `tests/fixtures/` plus `httpx.MockTransport` cover
+discovery, path sanitisation, manifest diffing/pruning, cookie parsing, session-expiry detection
+and retry/backoff behaviour.
+
+### Releasing (maintainers)
+
+1. Bump `__version__` in `src/moodle_scraper/__init__.py`.
+2. Commit and tag: `git tag v0.1.0 && git push origin v0.1.0` — the
+   [release workflow](.github/workflows/release.yml) builds the sdist/wheel, runs `twine check`,
+   and publishes to PyPI via [trusted publishing](https://docs.pypi.org/trusted-publishers/)
+   (configure the PyPI project once: owner `RTXC01`, repository `moodle_scrape`, workflow
+   `release.yml`, environment `pypi`).
+3. Manual alternative: `python -m build && twine upload dist/*`.
+
+## Troubleshooting
+
+- **Exit code 3 / "session expired"** — log in again in your browser and re-export the cookie.
+- **A course page returns no files** — Moodle markup varies by version/theme. Run
+  `moodle-scrape files <id> --verbose` and check whether the activity selectors in
+  `discovery.py` match. All selectors live in that one module.
+- **A single activity is skipped** — some activities bounce to login even with a valid session
+  (insufficient rights); they are skipped individually instead of aborting the run. Run with
+  `--verbose` to see which.
+- **Files look larger than expected / slow** — use `--max-file-size` to skip big video files, and
+  keep `--concurrency 1`.
+- **TLS errors behind a campus proxy** — `--insecure` disables verification for that run only.
+
+## Roadmap
+
+- `browser_cookie3` browser-store extraction as an optional extra.
+- Optionally record `mod_page`/`mod_book` text content in the manifest (today only their
+  attachments are collected).
+
+## License
+
+[MIT](LICENSE)