pl-e2e-test 0.1.0__py3-none-any.whl

prairielearn_e2e/fuzz.py

@@ -0,0 +1,293 @@
+"""Run lifecycle fuzz tests for questions changed between two Git refs."""
+
+from __future__ import annotations
+
+import argparse
+import os
+import secrets
+import shlex
+import subprocess
+from collections.abc import Sequence
+from pathlib import Path
+from tempfile import TemporaryDirectory
+
+from .discovery import variant_seeds
+from .reporting import write_empty_shard_report
+from .runner import run_tests
+from .sharding import Shard, parse_shard
+
+QUESTIONS_ROOT = Path("questions")
+DEFAULT_E2E_DIFF_IGNORE = Path(".question-e2e.diffignore")
+
+
+def git_repository_root(start: Path) -> Path:
+    """Return the Git repository containing *start*."""
+
+    result = subprocess.run(
+        ["git", "-C", os.fspath(start), "rev-parse", "--show-toplevel"],
+        check=True,
+        capture_output=True,
+        text=True,
+    )
+    return Path(result.stdout.strip()).resolve()
+
+
+def load_e2e_diff_ignore(config_path: Path, *, missing_ok: bool = False) -> list[str]:
+    """Load Git-style ignore rules used by E2E change detection."""
+
+    try:
+        return config_path.read_text(encoding="utf-8").splitlines()
+    except FileNotFoundError:
+        if missing_ok:
+            return []
+        raise ValueError(f"Unable to read {config_path}: file does not exist") from None
+    except OSError as exc:
+        raise ValueError(f"Unable to read {config_path}: {exc}") from exc
+
+
+def e2e_diff_ignored_files(
+    changed_files: Sequence[Path], ignore_rules: Sequence[str]
+) -> set[Path]:
+    """Return changed files matched using Git's native ignore semantics."""
+
+    if not changed_files or not ignore_rules:
+        return set()
+
+    with TemporaryDirectory(prefix="e2e-diff-ignore-") as temporary_directory:
+        temporary_root = Path(temporary_directory)
+        repository = temporary_root / "repository"
+        ignore_file = temporary_root / "exclude"
+        ignore_file.write_text("\n".join(ignore_rules), encoding="utf-8")
+
+        subprocess.run(
+            ["git", "init", "--quiet", os.fspath(repository)],
+            check=True,
+            stdout=subprocess.DEVNULL,
+            stderr=subprocess.PIPE,
+        )
+        result = subprocess.run(
+            [
+                "git",
+                "-C",
+                os.fspath(repository),
+                "-c",
+                f"core.excludesFile={ignore_file}",
+                "check-ignore",
+                "--no-index",
+                "--stdin",
+                "-z",
+            ],
+            check=False,
+            capture_output=True,
+            input=b"\0".join(os.fsencode(path) for path in changed_files) + b"\0",
+        )
+        if result.returncode not in (0, 1):
+            raise subprocess.CalledProcessError(
+                result.returncode,
+                result.args,
+                output=result.stdout,
+                stderr=result.stderr,
+            )
+
+    return {Path(os.fsdecode(path)) for path in result.stdout.split(b"\0") if path}
+
+
+def nonignored_files_outside_questions(
+    changed_files: Sequence[Path], ignore_rules: Sequence[str]
+) -> list[Path]:
+    """Return sorted changed files that cause every question to be tested."""
+
+    ignored_files = e2e_diff_ignored_files(changed_files, ignore_rules)
+    return sorted(
+        (
+            path
+            for path in changed_files
+            if QUESTIONS_ROOT not in path.parents and path not in ignored_files
+        ),
+        key=lambda path: path.as_posix(),
+    )
+
+
+def log_full_question_run(
+    changed_files: Sequence[Path], ignored_patterns: Sequence[str]
+) -> None:
+    """Log the changed files responsible for testing every question."""
+
+    outside_files = nonignored_files_outside_questions(changed_files, ignored_patterns)
+    print("Non-ignored changed files outside questions; testing every question:")
+    for path in outside_files:
+        print(f"  - {shlex.quote(path.as_posix())}")
+
+
+def question_paths_from_changed_files(
+    changed_files: Sequence[Path],
+    repository_root: Path,
+    ignore_rules: Sequence[str] = (),
+) -> list[Path]:
+    """Select changed questions, or all questions after a shared-file change."""
+
+    ignored_files = e2e_diff_ignored_files(changed_files, ignore_rules)
+    relevant_files = [path for path in changed_files if path not in ignored_files]
+    if any(QUESTIONS_ROOT not in path.parents for path in relevant_files):
+        return [QUESTIONS_ROOT]
+
+    question_paths: set[Path] = set()
+    for changed_file in relevant_files:
+        candidate = changed_file.parent
+        while candidate != QUESTIONS_ROOT and QUESTIONS_ROOT in candidate.parents:
+            if (repository_root / candidate / "info.json").is_file():
+                question_paths.add(candidate)
+                break
+            candidate = candidate.parent
+
+    return sorted(question_paths, key=lambda path: path.as_posix())
+
+
+def changed_files_between(base: str, head: str, repository_root: Path) -> list[Path]:
+    """Return repository-relative files changed between two Git refs."""
+
+    result = subprocess.run(
+        [
+            "git",
+            "diff",
+            "--name-only",
+            "-z",
+            "--no-renames",
+            "--diff-filter=ACDMRT",
+            base,
+            head,
+            "--",
+        ],
+        cwd=repository_root,
+        check=True,
+        stdout=subprocess.PIPE,
+    )
+    return [Path(os.fsdecode(path)) for path in result.stdout.split(b"\0") if path]
+
+
+def positive_integer(value: str) -> int:
+    parsed = int(value)
+    if parsed < 1:
+        raise argparse.ArgumentTypeError("must be at least 1")
+    if parsed > 2**32:
+        raise argparse.ArgumentTypeError("must not exceed 2**32")
+    return parsed
+
+
+def main(argv: Sequence[str] | None = None) -> int:
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument("--course-root", type=Path)
+    parser.add_argument("--base", default="main", help="base Git ref (default: main)")
+    parser.add_argument("--head", default="HEAD", help="head Git ref (default: HEAD)")
+    parser.add_argument("--seed-count", type=positive_integer, default=10)
+    parser.add_argument(
+        "--fuzz-seed",
+        type=int,
+        help="master seed for reproducible pseudo-random variant seeds",
+    )
+    parser.add_argument(
+        "--prairielearn-path",
+        type=Path,
+        help=(
+            "PrairieLearn Git repository containing the installed pinned revision "
+            "(default: managed user cache)"
+        ),
+    )
+    parser.add_argument(
+        "--e2ediffignore",
+        type=Path,
+        help="E2E change-detection ignore file (default: COURSE_ROOT/.question-e2e.diffignore)",
+    )
+    parser.add_argument(
+        "--shard",
+        type=parse_shard,
+        metavar="INDEX/COUNT",
+        help="run one one-based shard of the selected question variants",
+    )
+    parser.add_argument(
+        "--shard-report",
+        type=Path,
+        metavar="PATH",
+        help="write a mergeable JSON result blob for this shard",
+    )
+    args, pytest_args = parser.parse_known_args(argv)
+    shard: Shard | None = args.shard
+    if args.shard_report is not None and shard is None:
+        parser.error("--shard-report requires --shard")
+    if shard is not None and args.fuzz_seed is None:
+        parser.error("sharded fuzz runs require an explicit shared --fuzz-seed")
+
+    try:
+        repository_root = (
+            args.course_root.resolve()
+            if args.course_root is not None
+            else git_repository_root(Path.cwd())
+        )
+    except (OSError, subprocess.CalledProcessError) as exc:
+        parser.error(f"could not find the course Git repository: {exc}")
+
+    ignore_path = args.e2ediffignore
+    if ignore_path is None:
+        ignore_path = repository_root / DEFAULT_E2E_DIFF_IGNORE
+        missing_ok = True
+    else:
+        if not ignore_path.is_absolute():
+            ignore_path = repository_root / ignore_path
+        missing_ok = False
+    try:
+        ignored_patterns = load_e2e_diff_ignore(ignore_path, missing_ok=missing_ok)
+    except ValueError as exc:
+        parser.error(str(exc))
+
+    changed_files = changed_files_between(args.base, args.head, repository_root)
+    question_paths = question_paths_from_changed_files(
+        changed_files, repository_root, ignored_patterns
+    )
+    if not question_paths:
+        print("No non-ignored changed questions; skipping end-to-end fuzz tests.")
+        if args.shard_report is not None:
+            assert shard is not None
+            write_empty_shard_report(args.shard_report, shard, fuzz_seed=args.fuzz_seed)
+        return 0
+    if question_paths == [QUESTIONS_ROOT]:
+        log_full_question_run(changed_files, ignored_patterns)
+
+    fuzz_seed = args.fuzz_seed
+    if fuzz_seed is None:
+        fuzz_seed = secrets.randbits(64)
+    seeds = variant_seeds({}, args.seed_count, fuzz_seed)
+    print(f"E2E fuzz seed: {fuzz_seed}")
+    print(f"Variant seeds: {', '.join(str(seed) for seed in seeds)}")
+    replay_arguments = ["pl-e2e-test-diff", "--fuzz-seed", str(fuzz_seed)]
+    test_arguments: list[str] = []
+    if args.prairielearn_path is not None:
+        replay_arguments.extend(
+            ("--prairielearn-path", os.fspath(args.prairielearn_path))
+        )
+        test_arguments.extend(
+            ("--prairielearn-path", os.fspath(args.prairielearn_path))
+        )
+    if shard is not None:
+        replay_arguments.extend(("--shard", str(shard)))
+        test_arguments.extend(("--shard", str(shard)))
+    if args.shard_report is not None:
+        test_arguments.extend(("--shard-report", os.fspath(args.shard_report)))
+    print(f"Replay with: {shlex.join(replay_arguments)}", flush=True)
+
+    test_arguments.extend(
+        [
+            "--seed-count",
+            str(args.seed_count),
+            "--fuzz-seed",
+            str(fuzz_seed),
+        ]
+    )
+    for question_path in question_paths:
+        print(
+            f"Testing {shlex.quote(os.fspath(question_path))} "
+            f"with {args.seed_count} fuzz seeds.",
+            flush=True,
+        )
+        test_arguments.extend(("--question-path", os.fspath(question_path)))
+    test_arguments.extend(pytest_args)
+    return run_tests(test_arguments, course_root=repository_root)