pl-e2e-test 0.1.4__tar.gz → 0.1.5__tar.gz

{pl_e2e_test-0.1.4/src/pl_e2e_test.egg-info → pl_e2e_test-0.1.5}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: pl-e2e-test
-Version: 0.1.4
+Version: 0.1.5
 Summary: Fuzz PrairieLearn question lifecycles against the real Python element runtime
 License-Expression: MIT
 Project-URL: Homepage, https://github.com/PrairieLearn/pl-e2e-test
@@ -276,14 +276,22 @@ Use `--config path/to/settings.toml` to select another TOML file with the same
 
 ### Question-local configuration
 
-Store saved concrete seeds and narrow phase skips in `.question-e2e.json` beside the
-question's `info.json`:
+Store saved concrete seeds, narrow phase skips, and expected upstream errors in
+`.question-e2e.json` beside the question's `info.json`:
 
 ```json
 {
   "regression_seeds": [1843927501],
   "skip_methods": {
     "grade": "Temporarily blocked by an upstream issue"
+  },
+  "expected_errors": {
+    "render": [
+      {
+        "match": "^render: known upstream failure$",
+        "reason": "https://github.com/PrairieLearn/PrairieLearn/issues/12345"
+      }
+    ]
   }
 }
 ```
@@ -292,6 +300,21 @@ Saved regression seeds run before newly generated seeds. Skips require a nonempt
 reason and may name only `generate`, `prepare`, `render`, or `grade`. Manual-grading
 questions automatically skip only the grade check.
 
+Prefer `expected_errors` to a skip when the lifecycle can run far enough to expose a
+known upstream defect. Each rule requires a human-readable reason or issue URL and
+exactly one matcher: either `type`, containing a fully qualified exception type, or
+`match`, containing a Python regular expression searched against the exception message.
+A matching exception is reported as `XFAIL`; a different exception fails normally,
+while seeds that do not raise an exception pass normally. A method may not be both
+skipped and expected to fail.
+
+Rules under one method are alternatives. At least one rule must match across the selected
+seeds for that question and method, or an otherwise-successful run fails an audit asking
+you to remove or update the stale expectation. Parallel runs perform this audit after
+merging all shards. Because one upstream error can prevent later code from running,
+obsolete individual alternatives cannot be detected while another alternative still
+matches.
+
 Record the concrete seed shown in a failing pytest case with:
 
 ```sh

{pl_e2e_test-0.1.4 → pl_e2e_test-0.1.5}/README.md

@@ -252,14 +252,22 @@ Use `--config path/to/settings.toml` to select another TOML file with the same
 
 ### Question-local configuration
 
-Store saved concrete seeds and narrow phase skips in `.question-e2e.json` beside the
-question's `info.json`:
+Store saved concrete seeds, narrow phase skips, and expected upstream errors in
+`.question-e2e.json` beside the question's `info.json`:
 
 ```json
 {
   "regression_seeds": [1843927501],
   "skip_methods": {
     "grade": "Temporarily blocked by an upstream issue"
+  },
+  "expected_errors": {
+    "render": [
+      {
+        "match": "^render: known upstream failure$",
+        "reason": "https://github.com/PrairieLearn/PrairieLearn/issues/12345"
+      }
+    ]
   }
 }
 ```
@@ -268,6 +276,21 @@ Saved regression seeds run before newly generated seeds. Skips require a nonempt
 reason and may name only `generate`, `prepare`, `render`, or `grade`. Manual-grading
 questions automatically skip only the grade check.
 
+Prefer `expected_errors` to a skip when the lifecycle can run far enough to expose a
+known upstream defect. Each rule requires a human-readable reason or issue URL and
+exactly one matcher: either `type`, containing a fully qualified exception type, or
+`match`, containing a Python regular expression searched against the exception message.
+A matching exception is reported as `XFAIL`; a different exception fails normally,
+while seeds that do not raise an exception pass normally. A method may not be both
+skipped and expected to fail.
+
+Rules under one method are alternatives. At least one rule must match across the selected
+seeds for that question and method, or an otherwise-successful run fails an audit asking
+you to remove or update the stale expectation. Parallel runs perform this audit after
+merging all shards. Because one upstream error can prevent later code from running,
+obsolete individual alternatives cannot be detected while another alternative still
+matches.
+
 Record the concrete seed shown in a failing pytest case with:
 
 ```sh

{pl_e2e_test-0.1.4 → pl_e2e_test-0.1.5}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "pl-e2e-test"
-version = "0.1.4"
+version = "0.1.5"
 description = "Fuzz PrairieLearn question lifecycles against the real Python element runtime"
 readme = "README.md"
 requires-python = ">=3.13"
@@ -39,6 +39,12 @@ where = ["src"]
 
 [tool.pytest.ini_options]
 testpaths = ["tests"]
+markers = [
+  "generate: question generate-phase lifecycle checks",
+  "prepare: question prepare-phase lifecycle checks",
+  "render: question render-phase lifecycle checks",
+  "grade: question grading lifecycle checks",
+]
 
 [tool.pyright]
 include = ["src/prairielearn_e2e", "tests"]

{pl_e2e_test-0.1.4 → pl_e2e_test-0.1.5/src/pl_e2e_test.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: pl-e2e-test
-Version: 0.1.4
+Version: 0.1.5
 Summary: Fuzz PrairieLearn question lifecycles against the real Python element runtime
 License-Expression: MIT
 Project-URL: Homepage, https://github.com/PrairieLearn/pl-e2e-test
@@ -276,14 +276,22 @@ Use `--config path/to/settings.toml` to select another TOML file with the same
 
 ### Question-local configuration
 
-Store saved concrete seeds and narrow phase skips in `.question-e2e.json` beside the
-question's `info.json`:
+Store saved concrete seeds, narrow phase skips, and expected upstream errors in
+`.question-e2e.json` beside the question's `info.json`:
 
 ```json
 {
   "regression_seeds": [1843927501],
   "skip_methods": {
     "grade": "Temporarily blocked by an upstream issue"
+  },
+  "expected_errors": {
+    "render": [
+      {
+        "match": "^render: known upstream failure$",
+        "reason": "https://github.com/PrairieLearn/PrairieLearn/issues/12345"
+      }
+    ]
   }
 }
 ```
@@ -292,6 +300,21 @@ Saved regression seeds run before newly generated seeds. Skips require a nonempt
 reason and may name only `generate`, `prepare`, `render`, or `grade`. Manual-grading
 questions automatically skip only the grade check.
 
+Prefer `expected_errors` to a skip when the lifecycle can run far enough to expose a
+known upstream defect. Each rule requires a human-readable reason or issue URL and
+exactly one matcher: either `type`, containing a fully qualified exception type, or
+`match`, containing a Python regular expression searched against the exception message.
+A matching exception is reported as `XFAIL`; a different exception fails normally,
+while seeds that do not raise an exception pass normally. A method may not be both
+skipped and expected to fail.
+
+Rules under one method are alternatives. At least one rule must match across the selected
+seeds for that question and method, or an otherwise-successful run fails an audit asking
+you to remove or update the stale expectation. Parallel runs perform this audit after
+merging all shards. Because one upstream error can prevent later code from running,
+obsolete individual alternatives cannot be detected while another alternative still
+matches.
+
 Record the concrete seed shown in a failing pytest case with:
 
 ```sh

{pl_e2e_test-0.1.4 → pl_e2e_test-0.1.5}/src/pl_e2e_test.egg-info/SOURCES.txt

@@ -17,6 +17,7 @@ src/prairielearn_e2e/config.py
 src/prairielearn_e2e/contract.py
 src/prairielearn_e2e/discovery.py
 src/prairielearn_e2e/errors.py
+src/prairielearn_e2e/expected_errors.py
 src/prairielearn_e2e/fuzz.py
 src/prairielearn_e2e/harness.py
 src/prairielearn_e2e/html_checks.py
@@ -45,6 +46,7 @@ tests/test_metadata.py
 tests/test_parallel.py
 tests/test_publish_version.py
 tests/test_pytest_plugin.py
+tests/test_question_tests.py
 tests/test_regression.py
 tests/test_reporting.py
 tests/test_repository.py

{pl_e2e_test-0.1.4 → pl_e2e_test-0.1.5}/src/prairielearn_e2e/discovery.py

@@ -1,8 +1,9 @@
 from __future__ import annotations
 
 import random
+import re
 from collections.abc import Sequence
-from dataclasses import dataclass
+from dataclasses import dataclass, field
 from pathlib import Path
 from typing import Any, Literal
 
@@ -22,25 +23,49 @@ LIFECYCLE_METHODS: frozenset[str] = frozenset(
 )
 
 
+@dataclass(frozen=True, slots=True)
+class ExpectedError:
+    """One acceptable exception signature for a lifecycle method."""
+
+    reason: str
+    exception_type: str | None = None
+    match: str | None = None
+
+    def matches(self, exception: Exception) -> bool:
+        if self.exception_type is not None:
+            exception_type = type(exception)
+            qualified_type = (
+                f"{exception_type.__module__}.{exception_type.__qualname__}"
+            )
+            return self.exception_type == qualified_type
+        assert self.match is not None
+        return re.search(self.match, str(exception)) is not None
+
+
 @dataclass(frozen=True, slots=True)
 class E2EConfig:
     """Question-local end-to-end test configuration."""
 
     skip_methods: dict[str, str]
     regression_seeds: tuple[int, ...]
+    expected_errors: dict[str, tuple[ExpectedError, ...]] = field(default_factory=dict)
 
 
 def load_e2e_config(config_path: Path) -> E2EConfig:
     """Load and validate a question-local E2E configuration file."""
 
     if not config_path.is_file():
-        return E2EConfig(skip_methods={}, regression_seeds=())
+        return E2EConfig(skip_methods={}, regression_seeds=(), expected_errors={})
 
     config = load_json(config_path)
     if not isinstance(config, dict):
         raise HarnessError(f"{config_path}: top-level JSON value must be an object")
 
-    if unknown_keys := set(config) - {"regression_seeds", "skip_methods"}:
+    if unknown_keys := set(config) - {
+        "expected_errors",
+        "regression_seeds",
+        "skip_methods",
+    }:
         raise HarnessError(
             f"{config_path}: unknown keys: {', '.join(sorted(unknown_keys))}"
         )
@@ -62,6 +87,80 @@ def load_e2e_config(config_path: Path) -> E2EConfig:
             )
         validated_skip_methods[method] = reason.strip()
 
+    expected_errors = config.get("expected_errors", {})
+    if not isinstance(expected_errors, dict):
+        raise HarnessError(f"{config_path}: expected_errors must be an object")
+    if unknown_methods := set(expected_errors) - LIFECYCLE_METHODS:
+        raise HarnessError(
+            f"{config_path}: unknown expected-error methods: "
+            f"{', '.join(sorted(unknown_methods))}"
+        )
+    if conflicts := set(expected_errors) & set(validated_skip_methods):
+        raise HarnessError(
+            f"{config_path}: methods cannot be both skipped and expected to fail: "
+            f"{', '.join(sorted(conflicts))}"
+        )
+
+    validated_expected_errors: dict[str, tuple[ExpectedError, ...]] = {}
+    allowed_rule_keys = {"type", "match", "reason"}
+    for method, rules in expected_errors.items():
+        context = f"{config_path}: expected_errors[{method!r}]"
+        if not isinstance(rules, list) or not rules:
+            raise HarnessError(f"{context} must be a non-empty array")
+        validated_rules: list[ExpectedError] = []
+        for index, rule in enumerate(rules):
+            rule_context = f"{context}[{index}]"
+            if not isinstance(rule, dict):
+                raise HarnessError(f"{rule_context} must be an object")
+            if unknown_rule_keys := set(rule) - allowed_rule_keys:
+                raise HarnessError(
+                    f"{rule_context} has unknown keys: "
+                    f"{', '.join(sorted(unknown_rule_keys))}"
+                )
+            if "reason" not in rule:
+                raise HarnessError(f"{rule_context} is missing key: reason")
+            has_type = "type" in rule
+            has_match = "match" in rule
+            if has_type == has_match:
+                raise HarnessError(
+                    f"{rule_context} must contain exactly one of type or match"
+                )
+            exception_type = rule.get("type")
+            match = rule.get("match")
+            reason = rule["reason"]
+            if has_type:
+                if not isinstance(exception_type, str):
+                    raise HarnessError(
+                        f"{rule_context}.type must be a fully qualified exception type"
+                    )
+                type_parts = exception_type.strip().split(".")
+                if len(type_parts) < 2 or not all(
+                    part.isidentifier() for part in type_parts
+                ):
+                    raise HarnessError(
+                        f"{rule_context}.type must be a fully qualified exception type"
+                    )
+                exception_type = exception_type.strip()
+            else:
+                if not isinstance(match, str) or not match:
+                    raise HarnessError(
+                        f"{rule_context}.match must be a non-empty string"
+                    )
+                try:
+                    re.compile(match)
+                except re.error as exc:
+                    raise HarnessError(
+                        f"{rule_context}.match is not a valid regular expression: {exc}"
+                    ) from exc
+            if not isinstance(reason, str) or not reason.strip():
+                raise HarnessError(f"{rule_context}.reason must be a non-empty string")
+            validated_rules.append(
+                ExpectedError(
+                    reason=reason.strip(), exception_type=exception_type, match=match
+                )
+            )
+        validated_expected_errors[method] = tuple(validated_rules)
+
     regression_seeds = config.get("regression_seeds", [])
     if not isinstance(regression_seeds, list):
         raise HarnessError(f"{config_path}: regression_seeds must be an array")
@@ -88,6 +187,7 @@ def load_e2e_config(config_path: Path) -> E2EConfig:
     return E2EConfig(
         skip_methods=validated_skip_methods,
         regression_seeds=tuple(validated_regression_seeds),
+        expected_errors=validated_expected_errors,
     )
 
 
@@ -126,6 +226,9 @@ class QuestionCase:
     def e2e_skip_reason(self, method: LifecycleMethod) -> str | None:
         return self.e2e_config().skip_methods.get(method)
 
+    def e2e_expected_errors(self, method: LifecycleMethod) -> tuple[ExpectedError, ...]:
+        return self.e2e_config().expected_errors.get(method, ())
+
     def unvalidated_metadata(self) -> dict[str, Any]:
         value = load_json(self.info_path)
         return value if isinstance(value, dict) else {}

pl_e2e_test-0.1.5/src/prairielearn_e2e/expected_errors.py

@@ -0,0 +1,62 @@
+"""Session-level accounting for configured expected upstream errors."""
+
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+
+
+@dataclass(frozen=True, slots=True)
+class ExpectedErrorGroup:
+    """A question lifecycle method whose expected errors are alternatives."""
+
+    group_id: str
+    question: str
+    method: str
+    config_path: str
+
+    def payload(self) -> dict[str, str]:
+        return {
+            "id": self.group_id,
+            "question": self.question,
+            "method": self.method,
+            "config_path": self.config_path,
+        }
+
+
+@dataclass(slots=True)
+class ExpectedErrorAudit:
+    """Track expected-error groups reached and observed during one pytest run."""
+
+    required: dict[str, ExpectedErrorGroup] = field(default_factory=dict)
+    matched: set[str] = field(default_factory=set)
+
+    def require(
+        self, *, question: str, method: str, config_path: str
+    ) -> ExpectedErrorGroup:
+        group_id = f"{question}::{method}"
+        group = ExpectedErrorGroup(group_id, question, method, config_path)
+        existing = self.required.setdefault(group_id, group)
+        if existing != group:
+            raise ValueError(f"conflicting expected-error group {group_id!r}")
+        return existing
+
+    def record_match(self, group: ExpectedErrorGroup) -> None:
+        if self.required.get(group.group_id) != group:
+            raise ValueError(
+                f"expected-error group {group.group_id!r} was not registered"
+            )
+        self.matched.add(group.group_id)
+
+    def missing(self) -> tuple[ExpectedErrorGroup, ...]:
+        return tuple(
+            self.required[group_id]
+            for group_id in sorted(set(self.required) - self.matched)
+        )
+
+    def payload(self) -> dict[str, object]:
+        return {
+            "required": [
+                self.required[group_id].payload() for group_id in sorted(self.required)
+            ],
+            "matched": sorted(self.matched),
+        }

{pl_e2e_test-0.1.4 → pl_e2e_test-0.1.5}/src/prairielearn_e2e/pytest_plugin.py

@@ -5,8 +5,9 @@ from __future__ import annotations
 import secrets
 import shlex
 import sys
+import time
 import warnings
-from collections.abc import Hashable
+from collections.abc import Generator, Hashable
 from pathlib import Path
 from typing import Any, cast
 
@@ -16,12 +17,14 @@ from .backend import PrairieLearnBackend
 from .config import HarnessConfig, load_config
 from .discovery import QuestionCase, discover_questions, variant_seeds
 from .errors import HarnessError
+from .expected_errors import ExpectedErrorAudit, ExpectedErrorGroup
 from .reporting import ShardReportRecorder, serialize_pytest_report
 from .sharding import Shard, parse_shard, selected_group_keys
 
 _CONFIG_KEY: pytest.StashKey[HarnessConfig] = pytest.StashKey()
 _REPORTER_KEY: pytest.StashKey[ShardReportRecorder] = pytest.StashKey()
 _EMPTY_SHARD_KEY: pytest.StashKey[bool] = pytest.StashKey()
+_EXPECTED_ERROR_AUDIT_KEY: pytest.StashKey[ExpectedErrorAudit] = pytest.StashKey()
 _active_recorder: ShardReportRecorder | None = None
 
 
@@ -98,6 +101,9 @@ def _course_root(config: pytest.Config) -> Path:
 def pytest_configure(config: pytest.Config) -> None:
     global _active_recorder
 
+    expected_error_audit = ExpectedErrorAudit()
+    config.stash[_EXPECTED_ERROR_AUDIT_KEY] = expected_error_audit
+
     for marker, description in (
         ("generate", "question generate-phase lifecycle checks"),
         ("prepare", "question prepare-phase lifecycle checks"),
@@ -140,6 +146,7 @@ def pytest_configure(config: pytest.Config) -> None:
         recorder = ShardReportRecorder(
             path=report_path, shard=cast(Shard, shard), fuzz_seed=fuzz_seed
         )
+        recorder.expected_error_audit = expected_error_audit
         config.stash[_REPORTER_KEY] = recorder
         _active_recorder = recorder
 
@@ -279,6 +286,44 @@ def pytest_runtest_logreport(report: pytest.TestReport) -> None:
         _active_recorder.reports.append(serialize_pytest_report(report))
 
 
+def _missing_expected_error_report(group: ExpectedErrorGroup) -> pytest.TestReport:
+    now = time.time()
+    return pytest.TestReport(
+        nodeid=(
+            f"prairielearn_e2e::expected_error_audit[{group.question}:{group.method}]"
+        ),
+        location=(group.config_path, None, f"expected error audit: {group.method}"),
+        keywords={},
+        outcome="failed",
+        longrepr=(
+            f"Expected upstream error was not observed for {group.question} during "
+            f"{group.method}; all selected cases passed. Remove or update "
+            f"expected_errors in {group.config_path}."
+        ),
+        when="call",
+        duration=0.0,
+        start=now,
+        stop=now,
+    )
+
+
+@pytest.hookimpl(wrapper=True)
+def pytest_runtestloop(session: pytest.Session) -> Generator[None, object, object]:
+    result = yield
+    shard = cast(Shard | None, session.config.getoption("shard"))
+    if shard is not None or session.testsfailed:
+        return result
+    audit = session.config.stash[_EXPECTED_ERROR_AUDIT_KEY]
+    missing = audit.missing()
+    session.testscollected += len(missing)
+    for group in missing:
+        session.config.hook.pytest_runtest_logreport(
+            report=_missing_expected_error_report(group)
+        )
+        session.testsfailed += 1
+    return result
+
+
 def pytest_collectreport(report: pytest.CollectReport) -> None:
     if _active_recorder is not None and report.failed:
         _active_recorder.collection_errors.append(serialize_pytest_report(report))
@@ -359,3 +404,8 @@ def prairielearn_backend(
 @pytest.fixture(scope="session")
 def harness_config(request: pytest.FixtureRequest) -> HarnessConfig:
     return request.config.stash[_CONFIG_KEY]
+
+
+@pytest.fixture(scope="session")
+def expected_error_audit(request: pytest.FixtureRequest) -> ExpectedErrorAudit:
+    return request.config.stash[_EXPECTED_ERROR_AUDIT_KEY]

{pl_e2e_test-0.1.4 → pl_e2e_test-0.1.5}/src/prairielearn_e2e/question_tests.py

@@ -3,12 +3,14 @@
 from __future__ import annotations
 
 from collections.abc import Callable
+from typing import cast
 
 import pytest
 
 from .backend import PrairieLearnBackend
 from .discovery import LifecycleMethod, QuestionCase
 from .errors import UnsupportedQuestion
+from .expected_errors import ExpectedErrorAudit
 from .harness import QuestionHarness
 
 type LifecycleCheck = Callable[[QuestionHarness, int], object]
@@ -26,6 +28,18 @@ def _run_check(
     assert question is not None
     if reason := question.e2e_skip_reason(method):
         pytest.skip(reason)
+    expected_errors = question.e2e_expected_errors(method)
+    audit: ExpectedErrorAudit | None = None
+    group = None
+    if expected_errors:
+        audit = cast(
+            ExpectedErrorAudit, request.getfixturevalue("expected_error_audit")
+        )
+        group = audit.require(
+            question=question.qid,
+            method=method,
+            config_path=str(question.e2e_config_path),
+        )
     backend: PrairieLearnBackend = request.getfixturevalue("prairielearn_backend")
     try:
         check(
@@ -36,6 +50,19 @@ def _run_check(
         )
     except UnsupportedQuestion as exc:
         pytest.skip(str(exc))
+    except Exception as exc:
+        matching_rule = next(
+            (rule for rule in expected_errors if rule.matches(exc)), None
+        )
+        if matching_rule is None:
+            raise
+        assert audit is not None and group is not None
+        audit.record_match(group)
+        exception_type = type(exc)
+        qualified_type = f"{exception_type.__module__}.{exception_type.__qualname__}"
+        pytest.xfail(
+            f"expected upstream error: {matching_rule.reason} [{qualified_type}: {exc}]"
+        )
 
 
 @pytest.mark.generate