pl-e2e-test 0.1.0__py3-none-any.whl

prairielearn_e2e/backend.py

@@ -0,0 +1,168 @@
+from __future__ import annotations
+
+import io
+import os
+import sys
+from contextlib import redirect_stderr, redirect_stdout
+from pathlib import Path
+from typing import Any, cast
+
+from prairielearn.internal import question_phases
+from prairielearn.internal.question_phases import ElementInfo, RenderContext
+
+from .contract import Phase, snapshot, validate_mutations, validate_serializable
+from .errors import HarnessError
+from .metadata import load_json
+from .repository import (
+    PrairieLearnRepository,
+    installed_prairielearn_source,
+)
+
+
+def _load_elements(root: Path, element_type: str) -> dict[str, ElementInfo]:
+    result: dict[str, ElementInfo] = {}
+    if not root.is_dir():
+        return result
+    for directory in sorted(root.iterdir()):
+        info_path = directory / "info.json"
+        if not directory.is_dir() or not info_path.is_file():
+            continue
+        raw = load_json(info_path)
+        if not isinstance(raw, dict) or not isinstance(raw.get("controller"), str):
+            raise HarnessError(f"{info_path}: element info must declare a controller")
+        info = cast(
+            ElementInfo,
+            {
+                "name": directory.name,
+                "controller": raw["controller"],
+                "type": element_type,
+            },
+        )
+        result[directory.name] = info
+        if element_type == "core":
+            result[directory.name.replace("-", "_")] = info
+            for alias in raw.get("additionalNames", []):
+                if not isinstance(alias, str):
+                    raise HarnessError(
+                        f"{info_path}: additionalNames must contain strings"
+                    )
+                result[alias] = info
+                result[alias.replace("-", "_")] = info
+    return result
+
+
+def _load_extensions(course_root: Path) -> dict[str, dict[str, dict[Any, Any]]]:
+    result: dict[str, dict[str, dict[Any, Any]]] = {}
+    root = course_root / "elementExtensions"
+    if not root.is_dir():
+        return result
+    for info_path in sorted(root.glob("*/*/info.json")):
+        raw = load_json(info_path)
+        if not isinstance(raw, dict) or not isinstance(raw.get("controller"), str):
+            raise HarnessError(f"{info_path}: extension info must declare a controller")
+        element_name = info_path.parent.parent.name
+        extension_name = info_path.parent.name
+        result.setdefault(element_name, {})[extension_name] = {
+            "name": extension_name,
+            "directory": str(info_path.parent),
+            **raw,
+        }
+    return result
+
+
+class PrairieLearnBackend:
+    """Real PrairieLearn Python element phases backed by a pinned clone archive."""
+
+    def __init__(
+        self,
+        *,
+        course_root: Path,
+        core_elements_path: Path,
+        commit: str,
+    ) -> None:
+        self.course_root = course_root.resolve()
+        self.core_elements_path = core_elements_path.resolve()
+        self.commit = commit
+        core = _load_elements(self.core_elements_path, "core")
+        course = _load_elements(self.course_root / "elements", "course")
+        self.elements = {**core, **course}
+        self.element_extensions = _load_extensions(self.course_root)
+        question_phases.CORE_ELEMENTS_PATH = self.core_elements_path
+
+    @classmethod
+    def from_clone(
+        cls,
+        clone_path: str | Path,
+        course_root: str | Path,
+        materialize_root: str | Path,
+    ) -> PrairieLearnBackend:
+        root = Path(course_root).resolve()
+        destination = Path(materialize_root).resolve()
+        source = installed_prairielearn_source(root)
+        elements = PrairieLearnRepository(source).materialize_clone(
+            clone_path, destination
+        )
+        return cls(course_root=root, core_elements_path=elements, commit=source.commit)
+
+    @classmethod
+    def from_environment(
+        cls,
+        course_root: str | Path,
+        materialize_root: str | Path,
+        prairielearn_path: str | Path | None = None,
+    ) -> PrairieLearnBackend:
+        """Build a backend from an override repository or the managed cache."""
+
+        root = Path(course_root).resolve()
+        destination = Path(materialize_root).resolve()
+        source = installed_prairielearn_source(root)
+        elements = PrairieLearnRepository(source).materialize(
+            destination, prairielearn_path
+        )
+        return cls(course_root=root, core_elements_path=elements, commit=source.commit)
+
+    def context(self, html: str) -> RenderContext:
+        return {
+            "html": html,
+            "elements": self.elements,
+            "element_extensions": self.element_extensions,
+            "course_path": str(self.course_root),
+        }
+
+    def process(
+        self, phase: Phase, html: str, data: dict[str, Any]
+    ) -> tuple[str | None, set[str]]:
+        before = snapshot(data, phase)
+        old_cwd = Path.cwd()
+        old_path = sys.path.copy()
+        stdout = io.StringIO()
+        stderr = io.StringIO()
+        try:
+            with redirect_stdout(stdout), redirect_stderr(stderr):
+                result, processed = question_phases.process(
+                    phase, data, self.context(html)
+                )
+        finally:
+            os.chdir(old_cwd)
+            sys.path[:] = old_path
+        output = stdout.getvalue() + stderr.getvalue()
+        if output:
+            raise HarnessError(
+                f"{phase}: PrairieLearn element emitted unexpected console output: {output!r}"
+            )
+        validate_mutations(before, data, phase)
+        validate_serializable(data, phase)
+        return result, processed
+
+    def unresolved_elements(self, html: str) -> set[str]:
+        from lxml import html as lxml_html
+
+        fragments = lxml_html.fragments_fromstring(html)
+        registered = set(self.elements)
+        return {
+            node.tag
+            for fragment in fragments
+            if not isinstance(fragment, str)
+            for node in fragment.iter()
+            if isinstance(node.tag, str) and node.tag in registered
+        }

prairielearn_e2e/config.json

@@ -0,0 +1,7 @@
+{
+  "element_solution_answer_selectors": ["pl-sketch"],
+  "non_gradable_answer_selectors": [
+    "pl-sketch[read-only=\"true\"]",
+    "pl-sketch[read-only=\"1\"]"
+  ]
+}

prairielearn_e2e/config.py

@@ -0,0 +1,30 @@
+"""Load declarative configuration for the question harness."""
+
+from __future__ import annotations
+
+import json
+from pathlib import Path
+from typing import Any
+
+_CONFIG_PATH = Path(__file__).with_name("config.json")
+
+
+def _load_config() -> dict[str, Any]:
+    with _CONFIG_PATH.open(encoding="utf-8") as config_file:
+        config = json.load(config_file)
+    if not isinstance(config, dict):
+        raise TypeError("e2e-tests config must be a JSON object")
+    return config
+
+
+_CONFIG = _load_config()
+ELEMENT_SOLUTION_ANSWER_SELECTORS: tuple[str, ...] = tuple(
+    selector
+    for selector in _CONFIG.get("element_solution_answer_selectors", [])
+    if isinstance(selector, str) and selector.strip()
+)
+NON_GRADABLE_ANSWER_SELECTORS: tuple[str, ...] = tuple(
+    selector
+    for selector in _CONFIG.get("non_gradable_answer_selectors", [])
+    if isinstance(selector, str) and selector.strip()
+)

prairielearn_e2e/contract.py

@@ -0,0 +1,199 @@
+from __future__ import annotations
+
+import copy
+import json
+import math
+from dataclasses import dataclass
+from typing import Any, Literal
+
+from prairielearn.internal.zygote_utils import assert_all_integers_within_limits
+
+from .errors import HarnessError
+
+type Phase = Literal["generate", "prepare", "render", "parse", "grade", "test", "file"]
+
+
+@dataclass(frozen=True, slots=True)
+class Field:
+    kind: Literal["dict", "integer", "number", "string", "boolean"]
+
+
+BASE_FIELDS = {
+    "params": Field("dict"),
+    "correct_answers": Field("dict"),
+    "variant_seed": Field("integer"),
+    "options": Field("dict"),
+    "preferences": Field("dict"),
+}
+
+PHASE_FIELDS: dict[Phase, dict[str, Field]] = {
+    "generate": BASE_FIELDS,
+    "prepare": {**BASE_FIELDS, "answers_names": Field("dict")},
+    "render": {
+        **BASE_FIELDS,
+        "submitted_answers": Field("dict"),
+        "format_errors": Field("dict"),
+        "raw_submitted_answers": Field("dict"),
+        "partial_scores": Field("dict"),
+        "score": Field("number"),
+        "feedback": Field("dict"),
+        "editable": Field("boolean"),
+        "manual_grading": Field("boolean"),
+        "ai_grading": Field("boolean"),
+        "panel": Field("string"),
+        "correct_answer_shown": Field("boolean"),
+        "num_valid_submissions": Field("integer"),
+    },
+    "parse": {
+        **BASE_FIELDS,
+        "submitted_answers": Field("dict"),
+        "format_errors": Field("dict"),
+        "raw_submitted_answers": Field("dict"),
+        "feedback": Field("dict"),
+        "gradable": Field("boolean"),
+    },
+    "grade": {
+        **BASE_FIELDS,
+        "submitted_answers": Field("dict"),
+        "format_errors": Field("dict"),
+        "raw_submitted_answers": Field("dict"),
+        "partial_scores": Field("dict"),
+        "score": Field("number"),
+        "feedback": Field("dict"),
+        "gradable": Field("boolean"),
+    },
+    "test": {
+        **BASE_FIELDS,
+        "format_errors": Field("dict"),
+        "raw_submitted_answers": Field("dict"),
+        "partial_scores": Field("dict"),
+        "score": Field("number"),
+        "feedback": Field("dict"),
+        "gradable": Field("boolean"),
+        "test_type": Field("string"),
+    },
+    "file": {
+        **BASE_FIELDS,
+        "submitted_answers": Field("dict"),
+        "format_errors": Field("dict"),
+        "raw_submitted_answers": Field("dict"),
+        "partial_scores": Field("dict"),
+        "score": Field("number"),
+        "feedback": Field("dict"),
+        "num_valid_submissions": Field("integer"),
+        "filename": Field("string"),
+    },
+}
+
+EDITABLE_FIELDS: dict[Phase, frozenset[str]] = {
+    "generate": frozenset({"params", "correct_answers"}),
+    "prepare": frozenset({"params", "correct_answers", "answers_names"}),
+    "render": frozenset(),
+    "parse": frozenset(
+        {"params", "correct_answers", "submitted_answers", "format_errors", "feedback"}
+    ),
+    "grade": frozenset(
+        {
+            "params",
+            "correct_answers",
+            "submitted_answers",
+            "format_errors",
+            "partial_scores",
+            "score",
+            "feedback",
+        }
+    ),
+    "test": frozenset(
+        {
+            "raw_submitted_answers",
+            "format_errors",
+            "partial_scores",
+            "score",
+            "feedback",
+        }
+    ),
+    "file": frozenset(),
+}
+
+
+def _valid_type(value: Any, kind: str) -> bool:
+    if kind == "dict":
+        return isinstance(value, dict)
+    if kind == "integer":
+        return isinstance(value, int) and not isinstance(value, bool)
+    if kind == "number":
+        return (
+            isinstance(value, (int, float))
+            and not isinstance(value, bool)
+            and math.isfinite(value)
+        )
+    if kind == "string":
+        return isinstance(value, str)
+    if kind == "boolean":
+        return isinstance(value, bool)
+    return False
+
+
+def validate_shape(data: dict[str, Any], phase: Phase) -> None:
+    fields = PHASE_FIELDS[phase]
+    actual = set(data)
+    expected = set(fields)
+    if missing := expected - actual:
+        raise HarnessError(
+            f"{phase}: data is missing fields: {', '.join(sorted(missing))}"
+        )
+    if extra := actual - expected:
+        raise HarnessError(
+            f"{phase}: data has extra fields: {', '.join(sorted(extra))}"
+        )
+    for name, field in fields.items():
+        if not _valid_type(data[name], field.kind):
+            raise HarnessError(
+                f"{phase}: data[{name!r}] must have type {field.kind}, "
+                f"got {type(data[name]).__name__}"
+            )
+
+
+def validate_mutations(
+    before: dict[str, Any], after: dict[str, Any], phase: Phase
+) -> None:
+    validate_shape(after, phase)
+    editable = EDITABLE_FIELDS[phase]
+    for name in PHASE_FIELDS[phase]:
+        if name not in editable and before[name] != after[name]:
+            raise HarnessError(f"{phase}: data[{name!r}] was illegally modified")
+
+
+def validate_serializable(data: dict[str, Any], phase: Phase) -> None:
+    def check_json(value: Any, path: str) -> None:
+        if value is None or isinstance(value, (str, bool, int)):
+            return
+        if isinstance(value, float):
+            if not math.isfinite(value):
+                raise TypeError(f"{path} contains NaN or infinity")
+            return
+        if isinstance(value, list):
+            for index, item in enumerate(value):
+                check_json(item, f"{path}[{index}]")
+            return
+        if isinstance(value, dict):
+            for key, item in value.items():
+                if not isinstance(key, str):
+                    raise TypeError(f"{path} has a non-string object key")
+                check_json(item, f"{path}[{key!r}]")
+            return
+        raise TypeError(f"{path} contains {type(value).__name__}, which is not JSON")
+
+    try:
+        check_json(data, "data")
+        assert_all_integers_within_limits(data)
+        json.dumps(data, allow_nan=False)
+    except (TypeError, ValueError) as exc:
+        raise HarnessError(
+            f"{phase}: data is not PrairieLearn JSON serializable: {exc}"
+        ) from exc
+
+
+def snapshot(data: dict[str, Any], phase: Phase) -> dict[str, Any]:
+    validate_shape(data, phase)
+    return copy.deepcopy(data)

prairielearn_e2e/discovery.py

@@ -0,0 +1,212 @@
+from __future__ import annotations
+
+import random
+from collections.abc import Sequence
+from dataclasses import dataclass
+from pathlib import Path
+from typing import Any, Literal
+
+from .errors import HarnessError
+from .metadata import load_json
+
+type LifecycleMethod = Literal["generate", "prepare", "render", "grade"]
+
+E2E_CONFIG_FILENAME = ".question-e2e.json"
+LIFECYCLE_METHODS: frozenset[str] = frozenset(
+    {
+        "generate",
+        "prepare",
+        "render",
+        "grade",
+    }
+)
+
+
+@dataclass(frozen=True, slots=True)
+class E2EConfig:
+    """Question-local end-to-end test configuration."""
+
+    skip_methods: dict[str, str]
+    regression_seeds: tuple[int, ...]
+
+
+def load_e2e_config(config_path: Path) -> E2EConfig:
+    """Load and validate a question-local E2E configuration file."""
+
+    if not config_path.is_file():
+        return E2EConfig(skip_methods={}, regression_seeds=())
+
+    config = load_json(config_path)
+    if not isinstance(config, dict):
+        raise HarnessError(f"{config_path}: top-level JSON value must be an object")
+
+    if unknown_keys := set(config) - {"regression_seeds", "skip_methods"}:
+        raise HarnessError(
+            f"{config_path}: unknown keys: {', '.join(sorted(unknown_keys))}"
+        )
+
+    skip_methods = config.get("skip_methods", {})
+    if not isinstance(skip_methods, dict):
+        raise HarnessError(f"{config_path}: skip_methods must be an object")
+
+    if unknown_methods := set(skip_methods) - LIFECYCLE_METHODS:
+        raise HarnessError(
+            f"{config_path}: unknown skip methods: {', '.join(sorted(unknown_methods))}"
+        )
+
+    validated_skip_methods: dict[str, str] = {}
+    for method, reason in skip_methods.items():
+        if not isinstance(reason, str) or not reason.strip():
+            raise HarnessError(
+                f"{config_path}: skip reason for {method!r} must be a non-empty string"
+            )
+        validated_skip_methods[method] = reason.strip()
+
+    regression_seeds = config.get("regression_seeds", [])
+    if not isinstance(regression_seeds, list):
+        raise HarnessError(f"{config_path}: regression_seeds must be an array")
+
+    validated_regression_seeds: list[int] = []
+    seen_regression_seeds: set[int] = set()
+    for index, seed in enumerate(regression_seeds):
+        if isinstance(seed, bool) or not isinstance(seed, int):
+            raise HarnessError(
+                f"{config_path}: regression_seeds[{index}] must be an integer"
+            )
+        if not 0 <= seed < 2**32:
+            raise HarnessError(
+                f"{config_path}: regression_seeds[{index}] must be between "
+                f"0 and {2**32 - 1}"
+            )
+        if seed in seen_regression_seeds:
+            raise HarnessError(
+                f"{config_path}: regression_seeds contains duplicate seed {seed}"
+            )
+        validated_regression_seeds.append(seed)
+        seen_regression_seeds.add(seed)
+
+    return E2EConfig(
+        skip_methods=validated_skip_methods,
+        regression_seeds=tuple(validated_regression_seeds),
+    )
+
+
+def load_e2e_skip_methods(config_path: Path) -> dict[str, str]:
+    """Load method-specific E2E skips from a question-local config file."""
+
+    return load_e2e_config(config_path).skip_methods
+
+
+@dataclass(frozen=True, slots=True)
+class QuestionCase:
+    """A question directory discovered from an ``info.json`` file."""
+
+    directory: Path
+    course_root: Path
+
+    @property
+    def info_path(self) -> Path:
+        return self.directory / "info.json"
+
+    @property
+    def qid(self) -> str:
+        questions_root = self.course_root / "questions"
+        try:
+            return self.directory.relative_to(questions_root).as_posix()
+        except ValueError:
+            return self.directory.relative_to(self.course_root).as_posix()
+
+    @property
+    def e2e_config_path(self) -> Path:
+        return self.directory / E2E_CONFIG_FILENAME
+
+    def e2e_config(self) -> E2EConfig:
+        return load_e2e_config(self.e2e_config_path)
+
+    def e2e_skip_reason(self, method: LifecycleMethod) -> str | None:
+        return self.e2e_config().skip_methods.get(method)
+
+    def unvalidated_metadata(self) -> dict[str, Any]:
+        value = load_json(self.info_path)
+        return value if isinstance(value, dict) else {}
+
+    def is_manual_grading(self) -> bool:
+        """Whether this question contains instructor-only grading content."""
+
+        metadata = self.unvalidated_metadata()
+        declared = metadata.get("gradingMethod") == "Manual"
+        question_path = self.directory / "question.html"
+        if not question_path.is_file():
+            return declared
+        has_manual_content = (
+            "<pl-manual-grading-only"
+            in question_path.read_text(encoding="utf-8").lower()
+        )
+        if has_manual_content and not declared:
+            raise HarnessError(
+                f"{self.info_path}: question uses pl-manual-grading-only "
+                'but info.json does not declare gradingMethod: "Manual"'
+            )
+        return declared or has_manual_content
+
+
+def variant_seeds(
+    metadata: dict[str, Any],
+    seed_count: int,
+    fuzz_seed: int | None = None,
+    *,
+    regression_seeds: Sequence[int] = (),
+) -> tuple[int, ...]:
+    """Return saved regressions followed by generated variant seeds."""
+    if seed_count < 1:
+        raise ValueError("seed count must be at least 1")
+    if seed_count > 2**32:
+        raise ValueError("fuzz seed count cannot exceed 2**32")
+    if metadata.get("singleVariant") is True:
+        if any(seed != 0 for seed in regression_seeds):
+            raise ValueError("singleVariant questions may only use regression seed 0")
+        generated_seeds = (0,)
+    elif fuzz_seed is None:
+        generated_seeds = tuple(range(seed_count))
+    else:
+        generated_seeds = tuple(
+            random.Random(fuzz_seed).sample(range(2**32), seed_count)
+        )
+    return tuple(dict.fromkeys((*regression_seeds, *generated_seeds)))
+
+
+def _info_paths(input_path: Path) -> list[Path]:
+    if input_path.is_file():
+        if input_path.name != "info.json":
+            raise ValueError(f"{input_path}: expected an info.json file")
+        return [input_path]
+    if not input_path.is_dir():
+        raise ValueError(f"{input_path}: path does not exist")
+
+    paths = list(input_path.rglob("info.json"))
+    direct = input_path / "info.json"
+    if direct.is_file() and direct not in paths:
+        paths.append(direct)
+    if not paths:
+        raise ValueError(f"{input_path}: no info.json files found")
+    return paths
+
+
+def discover_questions(
+    paths: Sequence[str | Path],
+    course_root: str | Path,
+) -> list[QuestionCase]:
+    """Discover question directories beneath explicit files or directories."""
+
+    root = Path(course_root).resolve()
+    info_paths: set[Path] = set()
+    for raw_path in paths:
+        path = Path(raw_path)
+        if not path.is_absolute():
+            path = root / path
+        info_paths.update(candidate.resolve() for candidate in _info_paths(path))
+
+    return [
+        QuestionCase(info_path.parent, root)
+        for info_path in sorted(info_paths, key=lambda item: item.as_posix())
+    ]

prairielearn_e2e/errors.py

@@ -0,0 +1,9 @@
+"""Errors raised by the PrairieLearn end-to-end harness."""
+
+
+class HarnessError(AssertionError):
+    """A selected question violated the PrairieLearn lifecycle contract."""
+
+
+class UnsupportedQuestion(HarnessError):
+    """A selected question uses a format that this harness does not execute."""