pl-e2e-test 0.1.0__py3-none-any.whl

prairielearn_e2e/pytest_plugin.py

@@ -0,0 +1,335 @@
+"""Pytest integration for installed PrairieLearn question lifecycle tests."""
+
+from __future__ import annotations
+
+import secrets
+import shlex
+import sys
+import warnings
+from collections.abc import Hashable
+from pathlib import Path
+from typing import Any, cast
+
+import pytest
+
+from .backend import PrairieLearnBackend
+from .discovery import QuestionCase, discover_questions, variant_seeds
+from .errors import HarnessError
+from .reporting import ShardReportRecorder, serialize_pytest_report
+from .sharding import Shard, parse_shard, selected_group_keys
+
+_REPORTER_KEY: pytest.StashKey[ShardReportRecorder] = pytest.StashKey()
+_EMPTY_SHARD_KEY: pytest.StashKey[bool] = pytest.StashKey()
+_active_recorder: ShardReportRecorder | None = None
+
+
+def pytest_addoption(parser: pytest.Parser) -> None:
+    group = parser.getgroup("prairielearn-e2e")
+    group.addoption(
+        "--course-root",
+        metavar="PATH",
+        help="PrairieLearn course root (default: current working directory)",
+    )
+    group.addoption(
+        "--question-path",
+        action="append",
+        default=[],
+        metavar="PATH",
+        help="info.json, question directory, or parent directory (repeatable)",
+    )
+    group.addoption(
+        "--prairielearn-path",
+        metavar="PATH",
+        help=(
+            "PrairieLearn Git repository containing the installed pinned revision "
+            "(default: managed user cache)"
+        ),
+    )
+    group.addoption(
+        "--seed-count",
+        type=int,
+        default=3,
+        metavar="N",
+        help="number of variant seeds to test (default: 3)",
+    )
+    group.addoption(
+        "--fuzz-seed",
+        type=int,
+        metavar="N",
+        help="master seed used to generate reproducible pseudo-random variant seeds",
+    )
+    group.addoption(
+        "--fuzz-seeds",
+        action="store_true",
+        help="generate a master seed when --fuzz-seed is not provided",
+    )
+    group.addoption(
+        "--seed-replay-command",
+        default="pl-e2e-test",
+        metavar="COMMAND",
+        help="command shown in fuzz-seed replay instructions",
+    )
+    group.addoption(
+        "--shard",
+        type=parse_shard,
+        metavar="INDEX/COUNT",
+        help="run one one-based shard of the selected question variants",
+    )
+    group.addoption(
+        "--shard-report",
+        metavar="PATH",
+        help="write a mergeable JSON result blob for this shard",
+    )
+
+
+def _course_root(config: pytest.Config) -> Path:
+    configured = cast(str | None, config.getoption("course_root"))
+    return Path(configured or Path.cwd()).resolve()
+
+
+def pytest_configure(config: pytest.Config) -> None:
+    global _active_recorder
+
+    for marker, description in (
+        ("generate", "question generate-phase lifecycle checks"),
+        ("prepare", "question prepare-phase lifecycle checks"),
+        ("render", "question render-phase lifecycle checks"),
+        ("grade", "question grading lifecycle checks"),
+    ):
+        config.addinivalue_line("markers", f"{marker}: {description}")
+
+    fuzz_seeds = cast(bool, config.getoption("fuzz_seeds"))
+    fuzz_seed = cast(int | None, config.getoption("fuzz_seed"))
+    shard = cast(Shard | None, config.getoption("shard"))
+    shard_report = cast(str | None, config.getoption("shard_report"))
+    if shard_report is not None and shard is None:
+        raise pytest.UsageError("--shard-report requires --shard")
+    if shard is not None and fuzz_seeds and fuzz_seed is None:
+        raise pytest.UsageError(
+            "sharded fuzz runs require an explicit shared --fuzz-seed"
+        )
+    if fuzz_seeds and fuzz_seed is None:
+        config.option.fuzz_seed = secrets.randbits(64)
+        fuzz_seed = cast(int, config.option.fuzz_seed)
+    if shard_report is not None:
+        report_path = Path(shard_report)
+        try:
+            report_path.unlink(missing_ok=True)
+        except OSError as exc:
+            raise pytest.UsageError(
+                f"unable to replace shard report {report_path}: {exc}"
+            ) from exc
+        recorder = ShardReportRecorder(
+            path=report_path, shard=cast(Shard, shard), fuzz_seed=fuzz_seed
+        )
+        config.stash[_REPORTER_KEY] = recorder
+        _active_recorder = recorder
+
+
+def pytest_report_header(config: pytest.Config) -> str | None:
+    paths = cast(list[str], config.getoption("question_path"))
+    fuzz_seeds = cast(bool, config.getoption("fuzz_seeds"))
+    fuzz_seed = cast(int | None, config.getoption("fuzz_seed"))
+    replay_command = cast(str, config.getoption("seed_replay_command"))
+    prairielearn_path = cast(str | None, config.getoption("prairielearn_path"))
+    shard = cast(Shard | None, config.getoption("shard"))
+    if not paths and shard is None:
+        return None
+
+    lines: list[str] = []
+    if shard is not None:
+        lines.append(f"E2E shard: {shard}")
+    if paths:
+        seed_count = cast(int, config.getoption("seed_count"))
+        try:
+            seeds = variant_seeds({}, seed_count, fuzz_seed)
+        except ValueError:
+            return "\n".join(lines) or None
+        seed_list = f"E2E variant seeds: {', '.join(str(seed) for seed in seeds)}"
+        if fuzz_seed is None:
+            lines.append(seed_list)
+        elif fuzz_seeds:
+            replay = f"{replay_command} --fuzz-seed {fuzz_seed}"
+            if prairielearn_path is not None:
+                replay += f" --prairielearn-path {shlex.quote(prairielearn_path)}"
+            if shard is not None:
+                replay += f" --shard {shard}"
+            lines.extend(
+                (
+                    f"E2E fuzz seed: {fuzz_seed}",
+                    seed_list,
+                    f"Replay with: {replay}",
+                )
+            )
+    return "\n".join(lines) or None
+
+
+def _cases(config: pytest.Config) -> list[tuple[QuestionCase | None, int]]:
+    paths = cast(list[str], config.getoption("question_path"))
+    if not paths:
+        return [(None, 0)]
+    seed_count = cast(int, config.getoption("seed_count"))
+    fuzz_seed = cast(int | None, config.getoption("fuzz_seed"))
+    if seed_count < 1:
+        raise pytest.UsageError("--seed-count must be at least 1")
+    try:
+        questions = discover_questions(paths, _course_root(config))
+    except (HarnessError, ValueError) as exc:
+        raise pytest.UsageError(str(exc)) from exc
+
+    result: list[tuple[QuestionCase | None, int]] = []
+    for question in questions:
+        try:
+            metadata: dict[str, Any] = question.unvalidated_metadata()
+        except HarnessError:
+            metadata = {}
+        try:
+            e2e_config = question.e2e_config()
+            seeds = variant_seeds(
+                metadata,
+                seed_count,
+                fuzz_seed,
+                regression_seeds=e2e_config.regression_seeds,
+            )
+        except HarnessError as exc:
+            raise pytest.UsageError(str(exc)) from exc
+        except ValueError as exc:
+            raise pytest.UsageError(f"{question.qid}: {exc}") from exc
+        result.extend((question, seed) for seed in seeds)
+    return result
+
+
+def pytest_generate_tests(metafunc: pytest.Metafunc) -> None:
+    if "question_variant" not in metafunc.fixturenames:
+        return
+    cases = _cases(metafunc.config)
+    ids = [
+        "no-question-paths" if question is None else f"{question.qid}[seed={seed}]"
+        for question, seed in cases
+    ]
+    metafunc.parametrize("question_variant", cases, ids=ids)
+
+
+def _shard_group(item: pytest.Item) -> Hashable:
+    callspec = getattr(item, "callspec", None)
+    if callspec is None:
+        return ("item", item.nodeid)
+    question_variant = callspec.params.get("question_variant")
+    if not (
+        isinstance(question_variant, tuple)
+        and len(question_variant) == 2
+        and isinstance(question_variant[1], int)
+    ):
+        return ("item", item.nodeid)
+    question, seed = question_variant
+    qid = None if question is None else question.qid
+    return ("question-variant", qid, seed)
+
+
+@pytest.hookimpl(trylast=True)
+def pytest_collection_modifyitems(
+    config: pytest.Config, items: list[pytest.Item]
+) -> None:
+    shard = cast(Shard | None, config.getoption("shard"))
+    if shard is None:
+        return
+
+    full_items = list(items)
+    keys = [_shard_group(item) for item in full_items]
+    selected_keys = selected_group_keys(keys, shard)
+    selected = [
+        item for item, key in zip(full_items, keys, strict=True) if key in selected_keys
+    ]
+    deselected = [item for item in full_items if item not in selected]
+    items[:] = selected
+    if deselected:
+        config.hook.pytest_deselected(items=deselected)
+    config.stash[_EMPTY_SHARD_KEY] = bool(full_items) and not selected
+
+    recorder = config.stash.get(_REPORTER_KEY, None)
+    if recorder is not None:
+        recorder.set_collection(
+            [item.nodeid for item in full_items], [item.nodeid for item in selected]
+        )
+
+
+def pytest_runtest_logreport(report: pytest.TestReport) -> None:
+    if _active_recorder is not None:
+        _active_recorder.reports.append(serialize_pytest_report(report))
+
+
+def pytest_collectreport(report: pytest.CollectReport) -> None:
+    if _active_recorder is not None and report.failed:
+        _active_recorder.collection_errors.append(serialize_pytest_report(report))
+
+
+def pytest_warning_recorded(
+    warning_message: warnings.WarningMessage,
+    when: str,
+    nodeid: str,
+    location: tuple[str, int, str] | None,
+) -> None:
+    if _active_recorder is not None:
+        message = warning_message.message
+        _active_recorder.warnings.append(
+            {
+                "message": str(message),
+                "category": type(message).__name__,
+                "when": when,
+                "nodeid": nodeid,
+                "location": list(location) if location is not None else None,
+            }
+        )
+
+
+def pytest_internalerror(excrepr: object, excinfo: object) -> None:
+    del excinfo
+    if _active_recorder is not None:
+        _active_recorder.internal_errors.append(str(excrepr))
+
+
+def pytest_keyboard_interrupt(excinfo: object) -> None:
+    if _active_recorder is not None:
+        _active_recorder.interrupted.append(str(excinfo))
+
+
+@pytest.hookimpl(trylast=True)
+def pytest_sessionfinish(session: pytest.Session, exitstatus: int) -> None:
+    if (
+        session.config.stash.get(_EMPTY_SHARD_KEY, False)
+        and exitstatus == pytest.ExitCode.NO_TESTS_COLLECTED
+    ):
+        session.exitstatus = pytest.ExitCode.OK
+    recorder = session.config.stash.get(_REPORTER_KEY, None)
+    if recorder is None:
+        return
+    if not recorder.fingerprint:
+        recorder.set_collection((), ())
+    try:
+        recorder.write(int(session.exitstatus))
+    except OSError as exc:
+        session.exitstatus = pytest.ExitCode.INTERNAL_ERROR
+        print(
+            f"ERROR: unable to write shard report {recorder.path}: {exc}",
+            file=sys.stderr,
+        )
+
+
+def pytest_unconfigure(config: pytest.Config) -> None:
+    global _active_recorder
+
+    recorder = config.stash.get(_REPORTER_KEY, None)
+    if recorder is _active_recorder:
+        _active_recorder = None
+
+
+@pytest.fixture(scope="session")
+def prairielearn_backend(
+    request: pytest.FixtureRequest, tmp_path_factory: pytest.TempPathFactory
+) -> PrairieLearnBackend:
+    clone = cast(str | None, request.config.getoption("prairielearn_path"))
+    return PrairieLearnBackend.from_environment(
+        _course_root(request.config),
+        tmp_path_factory.mktemp("prairielearn-core"),
+        clone,
+    )

prairielearn_e2e/question_tests.py

@@ -0,0 +1,68 @@
+"""Installed pytest cases for the PrairieLearn question lifecycle."""
+
+from __future__ import annotations
+
+from collections.abc import Callable
+
+import pytest
+
+from .backend import PrairieLearnBackend
+from .discovery import LifecycleMethod, QuestionCase
+from .errors import UnsupportedQuestion
+from .harness import QuestionHarness
+
+type LifecycleCheck = Callable[[QuestionHarness, int], object]
+
+
+def _run_check(
+    question_variant: tuple[QuestionCase | None, int],
+    request: pytest.FixtureRequest,
+    method: LifecycleMethod,
+    check: LifecycleCheck,
+) -> None:
+    question, seed = question_variant
+    if question is None:
+        pytest.skip("pass one or more --question-path values to select content")
+    assert question is not None
+    if reason := question.e2e_skip_reason(method):
+        pytest.skip(reason)
+    backend: PrairieLearnBackend = request.getfixturevalue("prairielearn_backend")
+    try:
+        check(QuestionHarness(question, backend), seed)
+    except UnsupportedQuestion as exc:
+        pytest.skip(str(exc))
+
+
+@pytest.mark.generate
+def test_question_generate(
+    question_variant: tuple[QuestionCase | None, int],
+    request: pytest.FixtureRequest,
+) -> None:
+    _run_check(question_variant, request, "generate", QuestionHarness.run_generate)
+
+
+@pytest.mark.prepare
+def test_question_prepare(
+    question_variant: tuple[QuestionCase | None, int],
+    request: pytest.FixtureRequest,
+) -> None:
+    _run_check(question_variant, request, "prepare", QuestionHarness.run_prepare)
+
+
+@pytest.mark.render
+def test_question_render(
+    question_variant: tuple[QuestionCase | None, int],
+    request: pytest.FixtureRequest,
+) -> None:
+    _run_check(question_variant, request, "render", QuestionHarness.run_render)
+
+
+@pytest.mark.grade
+def test_question_grade(
+    question_variant: tuple[QuestionCase | None, int],
+    request: pytest.FixtureRequest,
+) -> None:
+    question, _ = question_variant
+    if question is not None and question.is_manual_grading():
+        pytest.skip("question is configured for manual grading")
+    _run_check(question_variant, request, "grade", QuestionHarness.run_grade)

prairielearn_e2e/regression.py

@@ -0,0 +1,128 @@
+"""Record a concrete variant seed in a question's regression corpus."""
+
+from __future__ import annotations
+
+import argparse
+import json
+import os
+import subprocess
+import tempfile
+from collections.abc import Sequence
+from pathlib import Path
+from typing import Any
+
+
+def _git_root(start: Path) -> Path:
+    result = subprocess.run(
+        ["git", "-C", os.fspath(start), "rev-parse", "--show-toplevel"],
+        check=True,
+        capture_output=True,
+        text=True,
+    )
+    return Path(result.stdout.strip()).resolve()
+
+
+def _variant_seed(value: str) -> int:
+    if (
+        not value.isascii()
+        or not value.isdecimal()
+        or (len(value) > 1 and value.startswith("0"))
+    ):
+        raise argparse.ArgumentTypeError("must be an integer between 0 and 4294967295")
+    seed = int(value)
+    if seed >= 2**32:
+        raise argparse.ArgumentTypeError("must be an integer between 0 and 4294967295")
+    return seed
+
+
+def add_regression_seed(course_root: Path, question_path: Path, seed: int) -> bool:
+    """Add *seed* atomically, returning whether the config changed."""
+
+    root = course_root.resolve()
+    question = question_path if question_path.is_absolute() else root / question_path
+    question = question.resolve()
+    questions_root = (root / "questions").resolve()
+    try:
+        question.relative_to(questions_root)
+    except ValueError as exc:
+        raise ValueError(
+            f"QUESTION_PATH must be a directory beneath questions/: {question_path}"
+        ) from exc
+    if not question.is_dir():
+        raise ValueError(f"Question directory does not exist: {question_path}")
+    if not (question / "info.json").is_file():
+        raise ValueError(f"Question directory has no info.json: {question_path}")
+
+    config_path = question / ".question-e2e.json"
+    if config_path.is_file():
+        try:
+            config: Any = json.loads(config_path.read_text(encoding="utf-8"))
+        except (OSError, UnicodeError, json.JSONDecodeError) as exc:
+            raise ValueError(f"Invalid E2E config {config_path}: {exc}") from exc
+        if not isinstance(config, dict):
+            raise ValueError(f"E2E config must contain a JSON object: {config_path}")
+    else:
+        config = {}
+
+    raw_seeds = config.get("regression_seeds", [])
+    if not isinstance(raw_seeds, list) or any(
+        isinstance(item, bool) or not isinstance(item, int) for item in raw_seeds
+    ):
+        raise ValueError(
+            f"regression_seeds must be an array of integers: {config_path}"
+        )
+    if seed in raw_seeds:
+        return False
+
+    config["regression_seeds"] = sorted({*raw_seeds, seed})
+    temporary_path: Path | None = None
+    try:
+        with tempfile.NamedTemporaryFile(
+            "w",
+            encoding="utf-8",
+            dir=question,
+            prefix=".question-e2e.json.tmp.",
+            delete=False,
+        ) as temporary:
+            json.dump(config, temporary, indent=2)
+            temporary.write("\n")
+            temporary_path = Path(temporary.name)
+        temporary_path.chmod(0o644)
+        temporary_path.replace(config_path)
+    finally:
+        if temporary_path is not None:
+            temporary_path.unlink(missing_ok=True)
+    return True
+
+
+def main(argv: Sequence[str] | None = None) -> int:
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument("question_path", type=Path)
+    parser.add_argument("variant_seed", type=_variant_seed)
+    parser.add_argument("--course-root", type=Path)
+    args = parser.parse_args(argv)
+
+    try:
+        course_root = (
+            args.course_root.resolve()
+            if args.course_root is not None
+            else _git_root(Path.cwd())
+        )
+        changed = add_regression_seed(
+            course_root, args.question_path, args.variant_seed
+        )
+    except (OSError, subprocess.CalledProcessError, ValueError) as exc:
+        parser.error(str(exc))
+
+    config_path = (
+        args.question_path / ".question-e2e.json"
+        if not args.question_path.is_absolute()
+        else args.question_path.resolve() / ".question-e2e.json"
+    )
+    if changed:
+        print(f"Added regression seed {args.variant_seed} to {config_path}")
+    else:
+        print(
+            f"Regression seed {args.variant_seed} is already recorded in {config_path}"
+        )
+    return 0