qguider 1.0.0__tar.gz

qguider-1.0.0/.claude/settings.local.json

@@ -0,0 +1,7 @@
+{
+  "permissions": {
+    "allow": [
+      "Bash(uv run *)"
+    ]
+  }
+}

qguider-1.0.0/.env.example

@@ -0,0 +1 @@
+SESSION="..."

qguider-1.0.0/.gitattributes

@@ -0,0 +1,2 @@
+# Auto detect text files and perform LF normalization
+* text=auto

qguider-1.0.0/.gitignore

@@ -0,0 +1,7 @@
+.env
+__pycache__/
+*.html
+!fixtures/*.html
+.venv/
+pytest_cache/
+qguider_data/

qguider-1.0.0/PKG-INFO

@@ -0,0 +1,214 @@
+Metadata-Version: 2.4
+Name: qguider
+Version: 1.0.0
+Summary: A Python package to download and scrape QGuides.
+Requires-Python: >=3.10
+Requires-Dist: beautifulsoup4
+Requires-Dist: pydantic
+Requires-Dist: python-dotenv
+Requires-Dist: requests
+Requires-Dist: rich>=15.0.0
+Requires-Dist: tdqm>=0.0.1
+Provides-Extra: dev
+Requires-Dist: pytest; extra == 'dev'
+Description-Content-Type: text/markdown
+
+# QGuider
+
+QGuider is a Python library for downloading, parsing, and querying Harvard QGuides — the university's course evaluation reports. It scrapes QGuide HTML pages, normalizes the data into typed Pydantic models, and provides a fluent API for filtering and exporting results.
+
+## Features
+
+- Download QGuides for multiple semesters with checkpointing and resume support
+- Parse HTML reports into structured, typed models
+- Filter by semester, subject, department, or instructor
+- Aggregate multi-instructor courses into a single record
+- Export to JSON or pandas DataFrame
+- Import previously exported JSON back into model objects
+
+## Installation
+
+```bash
+pip install qguider
+```
+
+Or install from source:
+
+```bash
+git clone https://github.com/ivanharvard/qguider
+cd qguider
+pip install .
+```
+
+## Setup
+
+QGuider requires your Harvard Key credentials to access the QGuide portal. 
+
+1. Login to [Harvard QGuide Portal](https://qreports.fas.harvard.edu/browse/index).
+2. Either (press F12 on your keyboard) or (right click anywhere on the page, and click `Inspect`).
+3. Press the arrow pointing to the right, and then click on `Application`.
+![Inspect Element](images/inspect.png)
+4. Under Storage, find Cookies, and under Cookies, find the option that looks like `https://qreports.fas.har...`
+![Cookies](images/cookies.png)
+5. Find the row that's labeled `SESSION`. In that row, double click the cell under the column `Value`. Copy it to your clipboard.
+6. Create a `.env` file in your working directory if it does not already exist:
+```
+SESSION="..."
+```
+7. Paste the value into your `.env`.
+8. When initializing the `QGuider`, pass in the path to your `.env` file.
+
+This `SESSION` key is temporary! You will need to replace it every 30-40 minutes or so if you wish to download any sources. You'll know it's time to replace it when you get 0 QGuide listings when attempting to download QGuides.
+
+## Quick Start
+
+```python
+import qguider
+
+qgdr = qguider.QGuider(creds=".env")
+
+results = (
+    qgdr.query()
+    .semesters("Fall 2024", "Spring 2025")
+    .download(checkpoint=True, checkpoint_interval=15)
+    .parse()
+    .agg(by="id") # all courses with the same id will be merged
+)
+
+qguider.exporter.to_json(results, "qguider_data/output.json")
+```
+
+## API Reference
+
+### `QGuider`
+
+The top-level entry point.
+
+```python
+qgdr = qguider.QGuider(creds=".env", outpath="qguider_data")
+query = qgdr.query()
+```
+
+- `creds` — path to a `.env` file containing credentials
+- `outpath` — directory where downloaded HTML files are stored (default: `qguider_data`)
+
+### `Query` (fluent builder)
+
+Chain filters before downloading:
+
+| Method | Description |
+|---|---|
+| `.semesters("Fall 2024", ...)` | Filter by one or more semesters |
+| `.subjects("CS", "MATH", ...)` | Filter by subject code |
+| `.departments("Computer Science", ...)` | Filter by department name |
+| `.instructor_last_name("Smith")` | Filter by instructor last name |
+| `.search("algorithms")` | Free-text search |
+| `.progress(rich_progress)` | Attach a Rich progress bar |
+| `.outpath("path/")` | Override output directory |
+
+After setting filters, call:
+
+```python
+# Download HTML files to disk
+.download(checkpoint=True, checkpoint_interval=15, report_failed=True)
+
+# Parse previously downloaded files
+.parse(skip_failed=True)
+
+# Download and parse in one step
+.run(checkpoint=True, skip_failed=True)
+```
+
+### `QGuideSet`
+
+`download().parse()` returns a `QGuideSet`, a list-like container of `QGuide` objects.
+
+```python
+len(results)          # number of QGuides
+results[0]            # access by index
+for guide in results: # iterate
+    print(guide.course.title)
+
+# Merge entries that share the same QGuide ID (e.g., multi-instructor courses)
+merged = results.agg(by="id")
+```
+
+### Data Models
+
+Each `QGuide` contains:
+
+| Field | Type | Description |
+|---|---|---|
+| `id` | `str` | Unique QGuide identifier |
+| `course` | `Course` | Course metadata |
+| `response_rate` | `ResponseRate` | Survey response counts and ratio |
+| `course_feedback` | `CourseFeedback` | Likert ratings for overall course, materials, assignments, etc. |
+| `instructor_feedback` | `list[InstructorFeedback]` | Per-instructor Likert ratings |
+| `hours_per_week` | `HoursPerWeek` | Reported weekly workload distribution |
+| `recommendation_strength` | `RecommendationStrength` | How strongly students recommend the course |
+| `reasons_for_enrollment` | `ReasonsForEnrollment` | Distribution of enrollment motivations |
+| `comments` | `list[Comment]` | Free-text student comments |
+
+`Course` fields: `title`, `subject`, `department`, `number`, `section`, `instructors`, `semester`, `aliases`.
+
+### Exporting
+
+```python
+# Write to JSON file
+qguider.exporter.to_json(results, "output.json")
+
+# Return JSON string without writing
+json_str = qguider.exporter.to_json(results)
+
+# Convert to pandas DataFrame (requires pandas)
+df = qguider.exporter.to_dataframe(results)
+```
+
+### Importing
+
+```python
+results = qguider.importer.from_json("output.json")
+```
+
+## CLI Example
+
+A reference CLI is provided in [`examples/cli.py`](examples/cli.py):
+
+```bash
+# Download and parse all semesters, write JSON
+python -m examples.cli --download
+
+# Download with Rich progress bar
+python -m examples.cli --download --progress
+
+# Parse previously downloaded HTML files
+python -m examples.cli --parse
+
+# Import from a previously exported JSON
+python -m examples.cli --import
+
+# Skip aggregation of multi-instructor courses
+python -m examples.cli --download --no-agg
+
+# Set logging verbosity
+python -m examples.cli --download --log-level DEBUG
+
+# Clear all downloaded data
+python -m examples.cli --clear-all
+```
+
+## Supported Semesters
+
+QGuider currently supports FAS (Faculty of Arts and Sciences) evaluations for:
+
+- Fall 2023
+- Spring 2024
+- Fall 2024
+- Spring 2025
+- Fall 2025
+- Spring 2026
+
+## Notes
+
+- Downloaded HTML files are cached under `qguider_data/` by semester, department, and subject — re-running a download with `checkpoint=True` skips already-downloaded files.
+- `agg(by="id")` merges records that share a QGuide ID, deduplicating instructor feedback and comments across entries for the same course offering.

qguider-1.0.0/README.md

@@ -0,0 +1,199 @@
+# QGuider
+
+QGuider is a Python library for downloading, parsing, and querying Harvard QGuides — the university's course evaluation reports. It scrapes QGuide HTML pages, normalizes the data into typed Pydantic models, and provides a fluent API for filtering and exporting results.
+
+## Features
+
+- Download QGuides for multiple semesters with checkpointing and resume support
+- Parse HTML reports into structured, typed models
+- Filter by semester, subject, department, or instructor
+- Aggregate multi-instructor courses into a single record
+- Export to JSON or pandas DataFrame
+- Import previously exported JSON back into model objects
+
+## Installation
+
+```bash
+pip install qguider
+```
+
+Or install from source:
+
+```bash
+git clone https://github.com/ivanharvard/qguider
+cd qguider
+pip install .
+```
+
+## Setup
+
+QGuider requires your Harvard Key credentials to access the QGuide portal. 
+
+1. Login to [Harvard QGuide Portal](https://qreports.fas.harvard.edu/browse/index).
+2. Either (press F12 on your keyboard) or (right click anywhere on the page, and click `Inspect`).
+3. Press the arrow pointing to the right, and then click on `Application`.
+![Inspect Element](images/inspect.png)
+4. Under Storage, find Cookies, and under Cookies, find the option that looks like `https://qreports.fas.har...`
+![Cookies](images/cookies.png)
+5. Find the row that's labeled `SESSION`. In that row, double click the cell under the column `Value`. Copy it to your clipboard.
+6. Create a `.env` file in your working directory if it does not already exist:
+```
+SESSION="..."
+```
+7. Paste the value into your `.env`.
+8. When initializing the `QGuider`, pass in the path to your `.env` file.
+
+This `SESSION` key is temporary! You will need to replace it every 30-40 minutes or so if you wish to download any sources. You'll know it's time to replace it when you get 0 QGuide listings when attempting to download QGuides.
+
+## Quick Start
+
+```python
+import qguider
+
+qgdr = qguider.QGuider(creds=".env")
+
+results = (
+    qgdr.query()
+    .semesters("Fall 2024", "Spring 2025")
+    .download(checkpoint=True, checkpoint_interval=15)
+    .parse()
+    .agg(by="id") # all courses with the same id will be merged
+)
+
+qguider.exporter.to_json(results, "qguider_data/output.json")
+```
+
+## API Reference
+
+### `QGuider`
+
+The top-level entry point.
+
+```python
+qgdr = qguider.QGuider(creds=".env", outpath="qguider_data")
+query = qgdr.query()
+```
+
+- `creds` — path to a `.env` file containing credentials
+- `outpath` — directory where downloaded HTML files are stored (default: `qguider_data`)
+
+### `Query` (fluent builder)
+
+Chain filters before downloading:
+
+| Method | Description |
+|---|---|
+| `.semesters("Fall 2024", ...)` | Filter by one or more semesters |
+| `.subjects("CS", "MATH", ...)` | Filter by subject code |
+| `.departments("Computer Science", ...)` | Filter by department name |
+| `.instructor_last_name("Smith")` | Filter by instructor last name |
+| `.search("algorithms")` | Free-text search |
+| `.progress(rich_progress)` | Attach a Rich progress bar |
+| `.outpath("path/")` | Override output directory |
+
+After setting filters, call:
+
+```python
+# Download HTML files to disk
+.download(checkpoint=True, checkpoint_interval=15, report_failed=True)
+
+# Parse previously downloaded files
+.parse(skip_failed=True)
+
+# Download and parse in one step
+.run(checkpoint=True, skip_failed=True)
+```
+
+### `QGuideSet`
+
+`download().parse()` returns a `QGuideSet`, a list-like container of `QGuide` objects.
+
+```python
+len(results)          # number of QGuides
+results[0]            # access by index
+for guide in results: # iterate
+    print(guide.course.title)
+
+# Merge entries that share the same QGuide ID (e.g., multi-instructor courses)
+merged = results.agg(by="id")
+```
+
+### Data Models
+
+Each `QGuide` contains:
+
+| Field | Type | Description |
+|---|---|---|
+| `id` | `str` | Unique QGuide identifier |
+| `course` | `Course` | Course metadata |
+| `response_rate` | `ResponseRate` | Survey response counts and ratio |
+| `course_feedback` | `CourseFeedback` | Likert ratings for overall course, materials, assignments, etc. |
+| `instructor_feedback` | `list[InstructorFeedback]` | Per-instructor Likert ratings |
+| `hours_per_week` | `HoursPerWeek` | Reported weekly workload distribution |
+| `recommendation_strength` | `RecommendationStrength` | How strongly students recommend the course |
+| `reasons_for_enrollment` | `ReasonsForEnrollment` | Distribution of enrollment motivations |
+| `comments` | `list[Comment]` | Free-text student comments |
+
+`Course` fields: `title`, `subject`, `department`, `number`, `section`, `instructors`, `semester`, `aliases`.
+
+### Exporting
+
+```python
+# Write to JSON file
+qguider.exporter.to_json(results, "output.json")
+
+# Return JSON string without writing
+json_str = qguider.exporter.to_json(results)
+
+# Convert to pandas DataFrame (requires pandas)
+df = qguider.exporter.to_dataframe(results)
+```
+
+### Importing
+
+```python
+results = qguider.importer.from_json("output.json")
+```
+
+## CLI Example
+
+A reference CLI is provided in [`examples/cli.py`](examples/cli.py):
+
+```bash
+# Download and parse all semesters, write JSON
+python -m examples.cli --download
+
+# Download with Rich progress bar
+python -m examples.cli --download --progress
+
+# Parse previously downloaded HTML files
+python -m examples.cli --parse
+
+# Import from a previously exported JSON
+python -m examples.cli --import
+
+# Skip aggregation of multi-instructor courses
+python -m examples.cli --download --no-agg
+
+# Set logging verbosity
+python -m examples.cli --download --log-level DEBUG
+
+# Clear all downloaded data
+python -m examples.cli --clear-all
+```
+
+## Supported Semesters
+
+QGuider currently supports FAS (Faculty of Arts and Sciences) evaluations for:
+
+- Fall 2023
+- Spring 2024
+- Fall 2024
+- Spring 2025
+- Fall 2025
+- Spring 2026
+
+## Notes
+
+- Downloaded HTML files are cached under `qguider_data/` by semester, department, and subject — re-running a download with `checkpoint=True` skips already-downloaded files.
+- `agg(by="id")` merges records that share a QGuide ID, deduplicating instructor feedback and comments across entries for the same course offering.

qguider-1.0.0/examples/_ui.py

@@ -0,0 +1,52 @@
+# Used by cli.py. Not an example.
+
+from collections import deque
+import logging
+
+from rich.console import Console
+from rich.layout import Layout
+from rich.live import Live
+from rich.panel import Panel
+from rich.progress import (
+    Progress,
+    BarColumn,
+    TextColumn,
+    TimeRemainingColumn,
+)
+
+
+def make_rich_ui(log_level: str = "INFO"):
+    console = Console()
+    log_lines = deque(maxlen=20)
+
+    progress = Progress(
+        TextColumn("[bold]{task.description}"),
+        BarColumn(),
+        TextColumn("{task.completed}/{task.total}"),
+        TimeRemainingColumn(),
+    )
+
+    layout = Layout()
+    layout.split_column(
+        Layout(Panel(progress), name="progress", size=5),
+        Layout(Panel("", title="Logs"), name="logs"),
+    )
+
+    class PanelLogHandler(logging.Handler):
+        def emit(self, record):
+            log_lines.append(self.format(record))
+            layout["logs"].update(
+                Panel("\n".join(log_lines), title="Logs")
+            )
+
+    handler = PanelLogHandler()
+    handler.setFormatter(
+        logging.Formatter("%(levelname)s:%(name)s:%(message)s")
+    )
+
+    root = logging.getLogger()
+    root.handlers.clear()
+    root.addHandler(handler)
+    root.setLevel(getattr(logging, log_level.upper(), logging.INFO))
+
+    return console, layout, progress, Live

qguider-1.0.0/examples/cli.py

@@ -0,0 +1,131 @@
+import argparse
+import qguider
+import logging
+from pathlib import Path
+from examples._ui import make_rich_ui
+import shutil
+
+
+class ColorFormatter(logging.Formatter):
+    COLORS = {
+        logging.DEBUG: "\033[37m",
+        logging.INFO: "\033[36m",
+        logging.WARNING: "\033[33m",
+        logging.ERROR: "\033[31m",
+        logging.CRITICAL: "\033[1;31m",
+    }
+
+    RESET = "\033[0m"
+
+    def format(self, record):
+        color = self.COLORS.get(record.levelno, "")
+        return (
+            f"{color}[{record.levelname}]{self.RESET} "
+            f"{record.getMessage()}"
+        )
+    
+def configure_logging(level: str = "INFO"):
+    root = logging.getLogger()
+    root.handlers.clear()
+
+    handler = logging.StreamHandler()
+    handler.setFormatter(ColorFormatter())
+
+    root.addHandler(handler)
+    root.setLevel(getattr(logging, level.upper(), logging.INFO))
+
+def download_all(progress = None, agg = False):
+    qgdr = qguider.QGuider(creds=".env")
+
+    results = (
+        qgdr
+        .query()
+        .semesters(
+            "Fall 2023",
+            "Spring 2024",
+            "Fall 2024",
+            "Spring 2025",
+            "Fall 2025",
+            "Spring 2026",
+        )
+        .progress(progress)
+        .download(
+            checkpoint=True,
+            checkpoint_interval=15,
+            report_failed=True
+        )
+        .parse(
+            skip_failed=True,
+        )
+    )
+
+    if agg:
+        results = results.agg(by="id")
+
+    qguider.exporter.to_json(results, "qguider_data/all_qguides.json")
+
+def parse_all(progress=None):
+    import qguider.parser
+
+    qguide_files = list(Path("qguider_data").rglob("*.html"))
+    task_id = None
+
+    if progress:
+        task_id = progress.add_task("Parsing QGuides", total=len(qguide_files))
+
+    for file in qguide_files:
+        try:
+            qguide = qguider.parser.QGuideParser(file).parse()
+            print(qguide.course, qguide.instructor)
+        finally:
+            if progress and task_id is not None:
+                progress.advance(task_id)
+
+def import_all():
+    results = qguider.importer.from_json("qguider_data/all_qguides.json")
+    print(f"Imported {len(results)} QGuides.")
+
+def clear_all():
+    shutil.rmtree("qguider_data", ignore_errors=True)
+    print("Cleared all downloaded data.")
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser(description="Download and parse QGuides")
+    parser.add_argument("--import", dest="do_import", action="store_true")
+    parser.add_argument("--parse", action="store_true")
+    parser.add_argument("--download", action="store_true")
+    parser.add_argument("--progress", action="store_true")
+    parser.add_argument("--no-agg", action="store_true")
+    parser.add_argument("--clear-all", action="store_true")
+    parser.add_argument("--log-level", default="INFO")
+
+    args = parser.parse_args()
+
+    if args.clear_all:
+        clear_all()
+
+    if args.progress:
+        console, layout, progress, Live = make_rich_ui(args.log_level)
+
+        with Live(layout, console=console, refresh_per_second=10):
+            if args.download:
+                download_all(progress=progress, agg=not args.no_agg)
+
+            if args.parse:
+                parse_all(progress=progress)
+
+            if args.do_import:
+                import_all()
+    else:
+        configure_logging(args.log_level)
+
+        if args.download:
+            download_all(progress=None, agg=not args.no_agg)
+
+        if args.parse:
+            parse_all(progress=None)
+
+        if args.do_import:
+            import_all()
+
+    

qguider-1.0.0/pyproject.toml

@@ -0,0 +1,26 @@
+[build-system]
+requires = ["hatchling"]
+build-backend = "hatchling.build"
+
+[project]
+name = "qguider"
+version = "1.0.0"
+description = "A Python package to download and scrape QGuides."
+readme = "README.md"
+requires-python = ">=3.10"
+dependencies = [
+    "requests",
+    "beautifulsoup4",
+    "python-dotenv",
+    "pydantic",
+    "tdqm>=0.0.1",
+    "rich>=15.0.0",
+]
+
+[project.optional-dependencies]
+dev = [
+    "pytest",
+]
+
+[tool.pytest.ini_options]
+testpaths = ["tests"]

qguider-1.0.0/qguider/__init__.py

@@ -0,0 +1,6 @@
+from .parser import QGuideParser
+from .api import QGuider
+from .agg import QGuideSet
+from . import exporter, importer
+
+__all__ = ["QGuideParser", "QGuider", "QGuideSet", "exporter", "importer"]

qguider-1.0.0/qguider/_logging.py

@@ -0,0 +1,7 @@
+import logging
+
+logger = logging.getLogger("qguider")
+
+# Prevent "No handler found" warnings while allowing the
+# application using qguider to configure logging however it wants.
+logger.addHandler(logging.NullHandler())