pl-e2e-test 0.1.0__py3-none-any.whl

prairielearn_e2e/repository.py

@@ -0,0 +1,298 @@
+"""Resolve and materialize the PrairieLearn source used by the harness."""
+
+from __future__ import annotations
+
+import hashlib
+import io
+import json
+import tarfile
+import tempfile
+import tomllib
+from contextlib import AbstractContextManager
+from dataclasses import dataclass, field
+from importlib.metadata import distribution
+from pathlib import Path
+from typing import Protocol
+from urllib.parse import parse_qsl, urlencode, urlsplit, urlunsplit
+
+from filelock import FileLock, Timeout
+from platformdirs import user_cache_path
+
+from ._git import _GitBackend, _GitError, _GitFetchError, _SubprocessGitBackend
+from .errors import HarnessError
+
+_CACHE_LOCK_TIMEOUT_SECONDS = 120
+_UV_GIT_QUERY_PARAMETERS = frozenset({"branch", "rev", "subdirectory", "tag"})
+
+
+class _LockFactory(Protocol):
+    def __call__(
+        self, path: Path, timeout: float
+    ) -> AbstractContextManager[object]: ...
+
+
+@dataclass(frozen=True, slots=True)
+class PrairieLearnSource:
+    """The exact Git source installed in the current Python environment."""
+
+    url: str
+    commit: str
+
+
+def _valid_source(url: object, commit: object) -> PrairieLearnSource | None:
+    if not isinstance(url, str) or not url:
+        return None
+    if not isinstance(commit, str) or not commit:
+        return None
+    if any(character not in "0123456789abcdefABCDEF" for character in commit):
+        return None
+    return PrairieLearnSource(url=url, commit=commit)
+
+
+def _source_from_direct_url() -> PrairieLearnSource | None:
+    direct_url = distribution("prairielearn").read_text("direct_url.json")
+    if not direct_url:
+        return None
+    try:
+        parsed = json.loads(direct_url)
+    except (json.JSONDecodeError, TypeError):
+        return None
+    if not isinstance(parsed, dict):
+        return None
+    vcs_info = parsed.get("vcs_info")
+    if not isinstance(vcs_info, dict) or vcs_info.get("vcs") != "git":
+        return None
+    return _valid_source(parsed.get("url"), vcs_info.get("commit_id"))
+
+
+def _remote_url(git_source: str) -> str:
+    parsed = urlsplit(git_source)
+    query = urlencode(
+        [
+            (key, value)
+            for key, value in parse_qsl(parsed.query, keep_blank_values=True)
+            if key not in _UV_GIT_QUERY_PARAMETERS
+        ]
+    )
+    return urlunsplit((parsed.scheme, parsed.netloc, parsed.path, query, ""))
+
+
+def _source_from_lock(course_root: Path) -> PrairieLearnSource | None:
+    try:
+        lock = tomllib.loads((course_root / "uv.lock").read_text(encoding="utf-8"))
+    except (OSError, tomllib.TOMLDecodeError):
+        return None
+    for package in lock.get("package", []):
+        if not isinstance(package, dict) or package.get("name") != "prairielearn":
+            continue
+        source = package.get("source")
+        if not isinstance(source, dict):
+            return None
+        git_source = source.get("git")
+        if not isinstance(git_source, str):
+            return None
+        parsed = urlsplit(git_source)
+        return _valid_source(_remote_url(git_source), parsed.fragment)
+    return None
+
+
+def installed_prairielearn_source(course_root: Path) -> PrairieLearnSource:
+    """Return the exact PrairieLearn Git source used by this environment."""
+
+    source = _source_from_direct_url() or _source_from_lock(course_root)
+    if source is None:
+        raise HarnessError(
+            "Could not determine the installed PrairieLearn Git source and revision"
+        )
+    return source
+
+
+def _file_lock(path: Path, timeout: float) -> AbstractContextManager[object]:
+    return FileLock(path, timeout=timeout)
+
+
+@dataclass(slots=True)
+class PrairieLearnRepository:
+    """Materialize one installed PrairieLearn source from a cache or override."""
+
+    source: PrairieLearnSource
+    cache_root: Path | None = field(default=None, kw_only=True)
+    _git: _GitBackend = field(
+        default_factory=_SubprocessGitBackend,
+        repr=False,
+        compare=False,
+        kw_only=True,
+    )
+    _lock_factory: _LockFactory = field(
+        default=_file_lock,
+        repr=False,
+        compare=False,
+        kw_only=True,
+    )
+
+    @property
+    def cache_path(self) -> Path:
+        """Return the private bare-repository path for this source."""
+
+        source_key = hashlib.sha256(self.source.url.encode()).hexdigest()
+        root = self.cache_root or (
+            user_cache_path("pl-e2e-test", appauthor=False) / "prairielearn"
+        )
+        return root / f"{source_key}.git"
+
+    def materialize(
+        self, destination: Path, clone_path: str | Path | None = None
+    ) -> Path:
+        """Materialize elements from an override or the managed cache."""
+
+        if clone_path is not None:
+            return self.materialize_clone(clone_path, destination)
+        return self.materialize_cached(destination)
+
+    def materialize_cached(self, destination: Path) -> Path:
+        """Materialize core elements from the managed repository cache."""
+
+        repository = self.cache_path
+        lock_path = repository.with_suffix(f"{repository.suffix}.lock")
+        try:
+            repository.parent.mkdir(parents=True, exist_ok=True)
+            with self._lock_factory(lock_path, _CACHE_LOCK_TIMEOUT_SECONDS):
+                self._ensure_cached_commit(repository)
+                return self._archive_elements(
+                    repository, destination, managed_cache=True
+                )
+        except Timeout as exc:
+            raise HarnessError(
+                f"Timed out after {_CACHE_LOCK_TIMEOUT_SECONDS} seconds waiting for "
+                f"managed PrairieLearn cache {repository}"
+            ) from exc
+        except OSError as exc:
+            raise HarnessError(
+                f"Unable to use managed PrairieLearn cache {repository} for revision "
+                f"{self.source.commit} from {self.source.url}: {exc}. Supply "
+                "--prairielearn-path to use another Git repository"
+            ) from exc
+
+    def materialize_clone(self, clone_path: str | Path, destination: Path) -> Path:
+        """Materialize elements from an authoritative user-supplied repository."""
+
+        repository = Path(clone_path).resolve()
+        try:
+            is_repository = self._git.is_repository(repository)
+        except _GitError as exc:
+            raise HarnessError(f"Unable to inspect {repository}: {exc}") from exc
+        if not is_repository:
+            raise HarnessError(
+                f"{repository}: --prairielearn-path must name a Git repository"
+            )
+        try:
+            has_commit = self._git.has_commit(repository, self.source.commit)
+        except _GitError as exc:
+            raise HarnessError(f"Unable to inspect {repository}: {exc}") from exc
+        if not has_commit:
+            raise HarnessError(
+                f"{repository}: --prairielearn-path does not contain installed "
+                f"PrairieLearn revision {self.source.commit}; the supplied repository "
+                "is authoritative and will not be fetched"
+            )
+        return self._archive_elements(repository, destination)
+
+    def _validate_managed_cache(self, repository: Path) -> None:
+        try:
+            is_repository = self._git.is_repository(repository)
+            is_bare = is_repository and self._git.is_bare_repository(repository)
+            remote = self._git.origin_url(repository) if is_bare else None
+        except _GitError as exc:
+            raise HarnessError(
+                f"Unable to inspect managed PrairieLearn cache {repository}: {exc}"
+            ) from exc
+        if not is_repository:
+            raise HarnessError(
+                f"Managed PrairieLearn cache {repository} is not a Git repository; "
+                "remove it and retry"
+            )
+        if not is_bare:
+            raise HarnessError(
+                f"Managed PrairieLearn cache {repository} is not a bare Git "
+                "repository; remove it and retry"
+            )
+        assert remote is not None
+        if remote != self.source.url:
+            raise HarnessError(
+                f"Managed PrairieLearn cache {repository} has unexpected origin "
+                f"{remote!r}; remove it and retry"
+            )
+
+    def _fetch_commit(
+        self, repository: Path, *, cache_path: Path | None = None
+    ) -> None:
+        reported_path = cache_path or repository
+        try:
+            self._git.fetch_commit(repository, self.source.commit)
+        except (_GitFetchError, _GitError) as exc:
+            raise HarnessError(
+                f"Unable to fetch PrairieLearn revision {self.source.commit} from "
+                f"{self.source.url} into {reported_path}: {exc}. Supply "
+                "--prairielearn-path with a Git repository containing revision "
+                f"{self.source.commit} to run without this fetch"
+            ) from exc
+
+    def _initialize_cache(self, repository: Path) -> None:
+        with tempfile.TemporaryDirectory(
+            prefix=f".{repository.stem}-", dir=repository.parent
+        ) as temporary_directory:
+            temporary_repository = Path(temporary_directory) / "repository.git"
+            try:
+                self._git.initialize_bare(temporary_repository, self.source.url)
+            except _GitError as exc:
+                raise HarnessError(
+                    f"Unable to initialize managed PrairieLearn cache "
+                    f"{repository}: {exc}"
+                ) from exc
+            self._fetch_commit(temporary_repository, cache_path=repository)
+            temporary_repository.rename(repository)
+
+    def _ensure_cached_commit(self, repository: Path) -> None:
+        if not repository.exists():
+            self._initialize_cache(repository)
+            return
+        self._validate_managed_cache(repository)
+        try:
+            has_commit = self._git.has_commit(repository, self.source.commit)
+        except _GitError as exc:
+            raise HarnessError(
+                f"Unable to inspect managed PrairieLearn cache {repository}: {exc}"
+            ) from exc
+        if not has_commit:
+            self._fetch_commit(repository)
+
+    def _archive_elements(
+        self,
+        repository: Path,
+        destination: Path,
+        *,
+        managed_cache: bool = False,
+    ) -> Path:
+        destination.mkdir(parents=True, exist_ok=True)
+        try:
+            archive = self._git.archive_elements(repository, self.source.commit)
+        except _GitError as exc:
+            if not managed_cache:
+                raise HarnessError(
+                    f"Unable to archive PrairieLearn elements at revision "
+                    f"{self.source.commit} from {repository}: {exc}"
+                ) from exc
+            raise HarnessError(
+                f"Unable to archive PrairieLearn elements at revision "
+                f"{self.source.commit} from {repository}: {exc}. The managed cache "
+                f"uses {self.source.url}; supply --prairielearn-path with a Git "
+                f"repository containing revision {self.source.commit} to bypass it"
+            ) from exc
+        with tarfile.open(fileobj=io.BytesIO(archive), mode="r:") as tar:
+            tar.extractall(destination, filter="data")
+        elements = destination / "elements"
+        if not elements.is_dir():
+            raise HarnessError(
+                f"PrairieLearn revision {self.source.commit} contains no core elements"
+            )
+        return elements

prairielearn_e2e/runner.py

@@ -0,0 +1,28 @@
+"""Run the installed PrairieLearn question lifecycle test suite."""
+
+from __future__ import annotations
+
+import os
+import sys
+from collections.abc import Sequence
+from pathlib import Path
+
+import pytest
+
+from . import pytest_plugin
+
+
+def run_tests(
+    arguments: Sequence[str], *, course_root: str | Path | None = None
+) -> int:
+    """Run the installed lifecycle cases with ordinary pytest arguments."""
+
+    root = Path(course_root or Path.cwd()).resolve()
+    test_module = Path(__file__).with_name("question_tests.py").resolve()
+    pytest_arguments = [os.fspath(test_module), "--course-root", os.fspath(root)]
+    pytest_arguments.extend(arguments)
+    return pytest.main(pytest_arguments, plugins=[pytest_plugin])
+
+
+def main(argv: Sequence[str] | None = None) -> int:
+    return run_tests(sys.argv[1:] if argv is None else argv)

prairielearn_e2e/server.py

@@ -0,0 +1,128 @@
+from __future__ import annotations
+
+import inspect
+import io
+import os
+import sys
+from contextlib import redirect_stderr, redirect_stdout
+from pathlib import Path
+from typing import Any
+
+from prairielearn.internal.zygote_utils import get_module_function
+
+from .contract import Phase, snapshot, validate_mutations, validate_serializable
+from .errors import HarnessError
+
+HOOKS: tuple[Phase, ...] = (
+    "generate",
+    "prepare",
+    "render",
+    "parse",
+    "grade",
+    "test",
+    "file",
+)
+
+
+class ServerModule:
+    def __init__(self, path: Path, course_root: Path) -> None:
+        self.path = path
+        self.course_root = course_root
+        self.namespace: dict[str, Any] = {}
+        if path.is_file():
+            self.namespace = self._execute()
+        self._validate_signatures()
+
+    def _environment(self) -> tuple[Path, list[str]]:
+        old_cwd = Path.cwd()
+        old_path = sys.path.copy()
+        os.chdir(self.path.parent)
+        sys.path.insert(0, str(self.course_root / "serverFilesCourse"))
+        sys.path.insert(0, str(self.path.parent))
+        return old_cwd, old_path
+
+    @staticmethod
+    def _restore(old_cwd: Path, old_path: list[str]) -> None:
+        os.chdir(old_cwd)
+        sys.path[:] = old_path
+
+    def _execute(self) -> dict[str, Any]:
+        namespace: dict[str, Any] = {"__file__": str(self.path)}
+        stdout = io.StringIO()
+        stderr = io.StringIO()
+        old_cwd, old_path = self._environment()
+        try:
+            source = self.path.read_text(encoding="utf-8")
+            code = compile(source, self.path, "exec")
+            with redirect_stdout(stdout), redirect_stderr(stderr):
+                exec(code, namespace)  # noqa: S102 -- server.py is the code under test.
+        finally:
+            self._restore(old_cwd, old_path)
+        output = stdout.getvalue() + stderr.getvalue()
+        if output:
+            raise HarnessError(
+                f"{self.path}: import emitted unexpected console output: {output!r}"
+            )
+        return namespace
+
+    def _validate_signatures(self) -> None:
+        for phase in HOOKS:
+            declared = self.namespace.get(phase)
+            if declared is None:
+                continue
+            if not callable(declared):
+                raise HarnessError(f"{self.path}: {phase} must be callable")
+            function = get_module_function(self.namespace, phase)
+            if function is None:
+                continue
+            expected = 2 if phase == "render" else 1
+            try:
+                inspect.signature(function).bind(*([object()] * expected))
+            except TypeError as exc:
+                arguments = "data, html" if phase == "render" else "data"
+                raise HarnessError(
+                    f"{self.path}: {phase} must accept ({arguments}): {exc}"
+                ) from exc
+
+    def has_hook(self, phase: Phase) -> bool:
+        return get_module_function(self.namespace, phase) is not None
+
+    def invoke(
+        self,
+        phase: Phase,
+        data: dict[str, Any],
+        html: str | None = None,
+    ) -> Any:
+        function = get_module_function(self.namespace, phase)
+        if function is None:
+            return html if phase == "render" else None
+        before = snapshot(data, phase)
+        stdout = io.StringIO()
+        stderr = io.StringIO()
+        old_cwd, old_path = self._environment()
+        try:
+            args = (data, html) if phase == "render" else (data,)
+            with redirect_stdout(stdout), redirect_stderr(stderr):
+                result = function(*args)
+        finally:
+            self._restore(old_cwd, old_path)
+        output = stdout.getvalue() + stderr.getvalue()
+        if output:
+            raise HarnessError(
+                f"{self.path}: {phase} emitted unexpected console output: {output!r}"
+            )
+        if phase == "render":
+            if not isinstance(result, str):
+                raise HarnessError(f"{self.path}: render must return HTML text")
+        elif phase == "file":
+            if result is not None and not isinstance(
+                result, (str, bytes, bytearray, memoryview, io.IOBase)
+            ):
+                raise HarnessError(f"{self.path}: file returned an unsupported value")
+        elif result is not None and result is not data:
+            raise HarnessError(
+                f"{self.path}: {phase} must mutate data in place and return None"
+            )
+        validate_mutations(before, data, phase)
+        validate_serializable(data, phase)
+        return result

prairielearn_e2e/sharding.py

@@ -0,0 +1,46 @@
+"""Stable test-suite sharding helpers."""
+
+from __future__ import annotations
+
+import argparse
+from collections.abc import Hashable, Sequence
+from dataclasses import dataclass
+
+
+@dataclass(frozen=True, slots=True)
+class Shard:
+    """A one-based shard index and the total number of shards."""
+
+    index: int
+    count: int
+
+    def __str__(self) -> str:
+        return f"{self.index}/{self.count}"
+
+
+def parse_shard(value: str) -> Shard:
+    """Parse an ``INDEX/COUNT`` shard specification."""
+
+    parts = value.split("/")
+    if len(parts) != 2:
+        raise argparse.ArgumentTypeError("must use INDEX/COUNT format")
+    try:
+        index, count = (int(part) for part in parts)
+    except ValueError:
+        raise argparse.ArgumentTypeError("index and count must be integers") from None
+    if count < 1:
+        raise argparse.ArgumentTypeError("count must be at least 1")
+    if index < 1 or index > count:
+        raise argparse.ArgumentTypeError(f"index must be between 1 and {count}")
+    return Shard(index=index, count=count)
+
+
+def selected_group_keys(keys: Sequence[Hashable], shard: Shard) -> set[Hashable]:
+    """Select stable first-seen groups for *shard* using round-robin assignment."""
+
+    unique_keys = list(dict.fromkeys(keys))
+    return {
+        key
+        for position, key in enumerate(unique_keys)
+        if position % shard.count == shard.index - 1
+    }