pl-e2e-test 0.1.3__tar.gz → 0.1.4__tar.gz

{pl_e2e_test-0.1.3/src/pl_e2e_test.egg-info → pl_e2e_test-0.1.4}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: pl-e2e-test
-Version: 0.1.3
+Version: 0.1.4
 Summary: Fuzz PrairieLearn question lifecycles against the real Python element runtime
 License-Expression: MIT
 Project-URL: Homepage, https://github.com/PrairieLearn/pl-e2e-test
@@ -255,6 +255,25 @@ element controllers are archived into the test run's temporary directory. Standa
 cache environment settings such as `XDG_CACHE_HOME` can relocate the managed cache;
 CI systems can persist that directory between runs if desired.
 
+### Harness configuration
+
+Selector settings are read automatically from the course's `pyproject.toml`:
+
+```toml
+[tool.pl-e2e-test]
+elementSolutionSelectors = ["pl-sketch"]
+nonGradableSelectors = ['pl-sketch[read-only="true"]', 'pl-sketch[read-only="1"]']
+```
+
+These are the defaults. `elementSolutionSelectors` identifies elements that provide
+solutions without a `correct_answers` entry. `nonGradableSelectors` excludes
+display-only answers from grading checks. Each supplied list replaces its default;
+omitted settings keep their defaults, and `[]` disables a list.
+
+Use `--config path/to/settings.toml` to select another TOML file with the same
+`[tool.pl-e2e-test]` table. This works with normal runs, `diff`, and arguments after
+`parallel run` or `parallel diff`. Explicit paths are relative to the working directory.
+
 ### Question-local configuration
 
 Store saved concrete seeds and narrow phase skips in `.question-e2e.json` beside the
@@ -331,6 +350,11 @@ from prairielearn_e2e import PrairieLearnBackend, QuestionCase, QuestionHarness
 increasingly complete portions of the lifecycle. `run_variant` remains an alias for
 the complete grade path.
 
+Pass `config=load_config(course_root=course_root)` to `QuestionHarness` to load the
+course's TOML settings, or `config=load_config("settings.toml")` for an explicit file.
+`load_config` and `HarnessConfig` are exported from `prairielearn_e2e`. Omitting
+`config` uses the built-in defaults.
+
 ## Development
 
 This project uses uv 0.9 or newer:

{pl_e2e_test-0.1.3 → pl_e2e_test-0.1.4}/README.md

@@ -231,6 +231,25 @@ element controllers are archived into the test run's temporary directory. Standa
 cache environment settings such as `XDG_CACHE_HOME` can relocate the managed cache;
 CI systems can persist that directory between runs if desired.
 
+### Harness configuration
+
+Selector settings are read automatically from the course's `pyproject.toml`:
+
+```toml
+[tool.pl-e2e-test]
+elementSolutionSelectors = ["pl-sketch"]
+nonGradableSelectors = ['pl-sketch[read-only="true"]', 'pl-sketch[read-only="1"]']
+```
+
+These are the defaults. `elementSolutionSelectors` identifies elements that provide
+solutions without a `correct_answers` entry. `nonGradableSelectors` excludes
+display-only answers from grading checks. Each supplied list replaces its default;
+omitted settings keep their defaults, and `[]` disables a list.
+
+Use `--config path/to/settings.toml` to select another TOML file with the same
+`[tool.pl-e2e-test]` table. This works with normal runs, `diff`, and arguments after
+`parallel run` or `parallel diff`. Explicit paths are relative to the working directory.
+
 ### Question-local configuration
 
 Store saved concrete seeds and narrow phase skips in `.question-e2e.json` beside the
@@ -307,6 +326,11 @@ from prairielearn_e2e import PrairieLearnBackend, QuestionCase, QuestionHarness
 increasingly complete portions of the lifecycle. `run_variant` remains an alias for
 the complete grade path.
 
+Pass `config=load_config(course_root=course_root)` to `QuestionHarness` to load the
+course's TOML settings, or `config=load_config("settings.toml")` for an explicit file.
+`load_config` and `HarnessConfig` are exported from `prairielearn_e2e`. Omitting
+`config` uses the built-in defaults.
+
 ## Development
 
 This project uses uv 0.9 or newer:

{pl_e2e_test-0.1.3 → pl_e2e_test-0.1.4}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "pl-e2e-test"
-version = "0.1.3"
+version = "0.1.4"
 description = "Fuzz PrairieLearn question lifecycles against the real Python element runtime"
 readme = "README.md"
 requires-python = ">=3.13"
@@ -37,9 +37,6 @@ pl-e2e-test = "prairielearn_e2e.cli:main"
 [tool.setuptools.packages.find]
 where = ["src"]
 
-[tool.setuptools.package-data]
-prairielearn_e2e = ["config.json"]
-
 [tool.pytest.ini_options]
 testpaths = ["tests"]
 

{pl_e2e_test-0.1.3 → pl_e2e_test-0.1.4/src/pl_e2e_test.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: pl-e2e-test
-Version: 0.1.3
+Version: 0.1.4
 Summary: Fuzz PrairieLearn question lifecycles against the real Python element runtime
 License-Expression: MIT
 Project-URL: Homepage, https://github.com/PrairieLearn/pl-e2e-test
@@ -255,6 +255,25 @@ element controllers are archived into the test run's temporary directory. Standa
 cache environment settings such as `XDG_CACHE_HOME` can relocate the managed cache;
 CI systems can persist that directory between runs if desired.
 
+### Harness configuration
+
+Selector settings are read automatically from the course's `pyproject.toml`:
+
+```toml
+[tool.pl-e2e-test]
+elementSolutionSelectors = ["pl-sketch"]
+nonGradableSelectors = ['pl-sketch[read-only="true"]', 'pl-sketch[read-only="1"]']
+```
+
+These are the defaults. `elementSolutionSelectors` identifies elements that provide
+solutions without a `correct_answers` entry. `nonGradableSelectors` excludes
+display-only answers from grading checks. Each supplied list replaces its default;
+omitted settings keep their defaults, and `[]` disables a list.
+
+Use `--config path/to/settings.toml` to select another TOML file with the same
+`[tool.pl-e2e-test]` table. This works with normal runs, `diff`, and arguments after
+`parallel run` or `parallel diff`. Explicit paths are relative to the working directory.
+
 ### Question-local configuration
 
 Store saved concrete seeds and narrow phase skips in `.question-e2e.json` beside the
@@ -331,6 +350,11 @@ from prairielearn_e2e import PrairieLearnBackend, QuestionCase, QuestionHarness
 increasingly complete portions of the lifecycle. `run_variant` remains an alias for
 the complete grade path.
 
+Pass `config=load_config(course_root=course_root)` to `QuestionHarness` to load the
+course's TOML settings, or `config=load_config("settings.toml")` for an explicit file.
+`load_config` and `HarnessConfig` are exported from `prairielearn_e2e`. Omitting
+`config` uses the built-in defaults.
+
 ## Development
 
 This project uses uv 0.9 or newer:

{pl_e2e_test-0.1.3 → pl_e2e_test-0.1.4}/src/pl_e2e_test.egg-info/SOURCES.txt

@@ -13,7 +13,6 @@ src/prairielearn_e2e/__main__.py
 src/prairielearn_e2e/_git.py
 src/prairielearn_e2e/backend.py
 src/prairielearn_e2e/cli.py
-src/prairielearn_e2e/config.json
 src/prairielearn_e2e/config.py
 src/prairielearn_e2e/contract.py
 src/prairielearn_e2e/discovery.py
@@ -35,6 +34,7 @@ tests/__init__.py
 tests/conftest.py
 tests/stub_git.py
 tests/test_cli.py
+tests/test_config.py
 tests/test_contract.py
 tests/test_discovery.py
 tests/test_fuzz.py

{pl_e2e_test-0.1.3 → pl_e2e_test-0.1.4}/src/prairielearn_e2e/__init__.py

@@ -3,6 +3,7 @@
 from importlib.metadata import version
 
 from .backend import PrairieLearnBackend
+from .config import HarnessConfig, load_config
 from .discovery import QuestionCase, discover_questions
 from .harness import LifecycleResult, QuestionHarness
 from .parallel import run_parallel
@@ -10,11 +11,13 @@ from .parallel import run_parallel
 __version__ = version("pl-e2e-test")
 
 __all__ = (
+    "HarnessConfig",
     "LifecycleResult",
     "PrairieLearnBackend",
     "QuestionCase",
     "QuestionHarness",
     "__version__",
     "discover_questions",
+    "load_config",
     "run_parallel",
 )

pl_e2e_test-0.1.4/src/prairielearn_e2e/config.py

@@ -0,0 +1,68 @@
+"""Load harness settings from the course's pyproject.toml."""
+
+from __future__ import annotations
+
+import tomllib
+from dataclasses import dataclass
+from pathlib import Path
+
+
+@dataclass(frozen=True, slots=True)
+class HarnessConfig:
+    element_solution_answer_selectors: tuple[str, ...] = ("pl-sketch",)
+    non_gradable_answer_selectors: tuple[str, ...] = (
+        'pl-sketch[read-only="true"]',
+        'pl-sketch[read-only="1"]',
+    )
+
+
+DEFAULT_CONFIG = HarnessConfig()
+
+
+def load_config(
+    path: str | Path | None = None, *, course_root: str | Path | None = None
+) -> HarnessConfig:
+    """Read [tool.pl-e2e-test], preserving defaults for omitted settings.
+
+    Without an explicit path, look for pyproject.toml in the course root.
+    A missing implicit file is allowed; a missing explicit file is an error.
+    """
+    config_path = (
+        Path(path)
+        if path is not None
+        else Path(course_root or Path.cwd()) / "pyproject.toml"
+    )
+    try:
+        with config_path.open("rb") as config_file:
+            document = tomllib.load(config_file)
+    except FileNotFoundError:
+        if path is not None:
+            raise
+        return DEFAULT_CONFIG
+    tool = document.get("tool", {})
+    if not isinstance(tool, dict):
+        raise TypeError("tool must be a table")
+    config = tool.get("pl-e2e-test", {})
+    if not isinstance(config, dict):
+        raise TypeError("tool.pl-e2e-test must be a table")
+
+    def selectors(key: str, default: tuple[str, ...]) -> tuple[str, ...]:
+        if key not in config:
+            return default
+        values = config[key]
+        if not isinstance(values, list) or any(
+            not isinstance(value, str) or not value.strip() for value in values
+        ):
+            raise TypeError(
+                f"tool.pl-e2e-test.{key} must be an array of non-empty strings"
+            )
+        return tuple(values)
+
+    return HarnessConfig(
+        element_solution_answer_selectors=selectors(
+            "elementSolutionSelectors", DEFAULT_CONFIG.element_solution_answer_selectors
+        ),
+        non_gradable_answer_selectors=selectors(
+            "nonGradableSelectors", DEFAULT_CONFIG.non_gradable_answer_selectors
+        ),
+    )

{pl_e2e_test-0.1.3 → pl_e2e_test-0.1.4}/src/prairielearn_e2e/fuzz.py

@@ -197,6 +197,9 @@ def positive_integer(value: str) -> int:
 def main(argv: Sequence[str] | None = None) -> int:
     parser = argparse.ArgumentParser(prog="pl-e2e-test diff", description=__doc__)
     parser.add_argument("--course-root", type=Path)
+    parser.add_argument(
+        "--config", type=Path, help="TOML config (default: COURSE_ROOT/pyproject.toml)"
+    )
     parser.add_argument("--base", default="main", help="base Git ref (default: main)")
     parser.add_argument("--head", default="HEAD", help="head Git ref (default: HEAD)")
     parser.add_argument("--seed-count", type=positive_integer, default=10)
@@ -276,6 +279,9 @@ def main(argv: Sequence[str] | None = None) -> int:
     print(f"Variant seeds: {', '.join(str(seed) for seed in seeds)}")
     replay_arguments = ["pl-e2e-test", "diff", "--fuzz-seed", str(fuzz_seed)]
     test_arguments: list[str] = []
+    if args.config is not None:
+        replay_arguments.extend(("--config", os.fspath(args.config)))
+        test_arguments.extend(("--config", os.fspath(args.config)))
     if args.prairielearn_path is not None:
         replay_arguments.extend(
             ("--prairielearn-path", os.fspath(args.prairielearn_path))

{pl_e2e_test-0.1.3 → pl_e2e_test-0.1.4}/src/prairielearn_e2e/harness.py

@@ -13,6 +13,7 @@ import lxml.html
 import numpy as np
 
 from .backend import PrairieLearnBackend
+from .config import DEFAULT_CONFIG, HarnessConfig
 from .contract import PHASE_FIELDS, Phase
 from .discovery import QuestionCase
 from .errors import HarnessError, UnsupportedQuestion
@@ -112,6 +113,7 @@ class QuestionHarness:
         backend: PrairieLearnBackend,
         *,
         course_root: str | Path | None = None,
+        config: HarnessConfig = DEFAULT_CONFIG,
     ) -> None:
         if isinstance(question, QuestionCase):
             self.question = question
@@ -124,6 +126,7 @@ class QuestionHarness:
                 directory = directory.parent
             self.question = QuestionCase(directory.resolve(), root)
         self.backend = backend
+        self.config = config
 
     def _options(self) -> dict[str, Any]:
         directory = self.question.directory
@@ -333,12 +336,12 @@ class QuestionHarness:
 
             show_correct = metadata.get("showCorrectAnswer", True)
             answer_names = validate_declared_answers(
-                prepared_html, base["correct_answers"], show_correct
+                prepared_html, base["correct_answers"], show_correct, self.config
             )
             tested_answer_names = tuple(
                 name
                 for name in answer_names
-                if name not in non_gradable_answer_names(prepared_html)
+                if name not in non_gradable_answer_names(prepared_html, self.config)
             )
             rendered_question = self._render(
                 source, base, server, panel="question", show_correct=show_correct

{pl_e2e_test-0.1.3 → pl_e2e_test-0.1.4}/src/prairielearn_e2e/html_checks.py

@@ -6,7 +6,7 @@ from typing import Any
 
 import lxml.html
 
-from .config import ELEMENT_SOLUTION_ANSWER_SELECTORS, NON_GRADABLE_ANSWER_SELECTORS
+from .config import DEFAULT_CONFIG, HarnessConfig
 from .errors import HarnessError
 
 _SIMPLE_CSS_SELECTOR = re.compile(
@@ -100,17 +100,23 @@ def _answer_names_matching_selectors(
     return frozenset(names)
 
 
-def element_solution_answer_names(html: str) -> frozenset[str]:
+def element_solution_answer_names(
+    html: str, config: HarnessConfig = DEFAULT_CONFIG
+) -> frozenset[str]:
     """Return answer names whose solution is owned by the element.
 
     These elements do not need an entry in ``correct_answers``, but they can
     still be graded and must remain part of the parse/grade lifecycle checks.
     """
 
-    return _answer_names_matching_selectors(html, ELEMENT_SOLUTION_ANSWER_SELECTORS)
+    return _answer_names_matching_selectors(
+        html, config.element_solution_answer_selectors
+    )
 
 
-def non_gradable_answer_names(html: str) -> frozenset[str]:
+def non_gradable_answer_names(
+    html: str, config: HarnessConfig = DEFAULT_CONFIG
+) -> frozenset[str]:
     """Return answer names intentionally excluded from server-side grading.
 
     Configured display-only elements still carry an ``answers-name`` for
@@ -118,15 +124,20 @@ def non_gradable_answer_names(html: str) -> frozenset[str]:
     partial score.
     """
 
-    return _answer_names_matching_selectors(html, NON_GRADABLE_ANSWER_SELECTORS)
+    return _answer_names_matching_selectors(html, config.non_gradable_answer_selectors)
 
 
 def validate_declared_answers(
-    html: str, correct_answers: dict[str, Any], show_correct_answer: bool
+    html: str,
+    correct_answers: dict[str, Any],
+    show_correct_answer: bool,
+    config: HarnessConfig = DEFAULT_CONFIG,
 ) -> tuple[str, ...]:
     names = declared_answer_names(html)
     if show_correct_answer:
-        ignored = non_gradable_answer_names(html) | element_solution_answer_names(html)
+        ignored = non_gradable_answer_names(
+            html, config
+        ) | element_solution_answer_names(html, config)
         require_correct_answers(
             tuple(name for name in names if name not in ignored), correct_answers
         )

{pl_e2e_test-0.1.3 → pl_e2e_test-0.1.4}/src/prairielearn_e2e/pytest_plugin.py

@@ -13,11 +13,13 @@ from typing import Any, cast
 import pytest
 
 from .backend import PrairieLearnBackend
+from .config import HarnessConfig, load_config
 from .discovery import QuestionCase, discover_questions, variant_seeds
 from .errors import HarnessError
 from .reporting import ShardReportRecorder, serialize_pytest_report
 from .sharding import Shard, parse_shard, selected_group_keys
 
+_CONFIG_KEY: pytest.StashKey[HarnessConfig] = pytest.StashKey()
 _REPORTER_KEY: pytest.StashKey[ShardReportRecorder] = pytest.StashKey()
 _EMPTY_SHARD_KEY: pytest.StashKey[bool] = pytest.StashKey()
 _active_recorder: ShardReportRecorder | None = None
@@ -25,6 +27,12 @@ _active_recorder: ShardReportRecorder | None = None
 
 def pytest_addoption(parser: pytest.Parser) -> None:
     group = parser.getgroup("prairielearn-e2e")
+    group.addoption(
+        "--config",
+        dest="e2e_config",
+        metavar="PATH",
+        help="TOML config (default: COURSE_ROOT/pyproject.toml)",
+    )
     group.addoption(
         "--course-root",
         metavar="PATH",
@@ -98,6 +106,16 @@ def pytest_configure(config: pytest.Config) -> None:
     ):
         config.addinivalue_line("markers", f"{marker}: {description}")
 
+    config_path = cast(str | None, config.getoption("e2e_config"))
+    try:
+        config.stash[_CONFIG_KEY] = load_config(
+            config_path, course_root=_course_root(config)
+        )
+    except (OSError, ValueError, TypeError) as exc:
+        raise pytest.UsageError(
+            f"unable to load harness config {config_path!r}: {exc}"
+        ) from exc
+
     fuzz_seeds = cast(bool, config.getoption("fuzz_seeds"))
     fuzz_seed = cast(int | None, config.getoption("fuzz_seed"))
     shard = cast(Shard | None, config.getoption("shard"))
@@ -152,6 +170,9 @@ def pytest_report_header(config: pytest.Config) -> str | None:
             replay = f"{replay_command} --fuzz-seed {fuzz_seed}"
             if prairielearn_path is not None:
                 replay += f" --prairielearn-path {shlex.quote(prairielearn_path)}"
+            config_path = cast(str | None, config.getoption("e2e_config"))
+            if config_path is not None:
+                replay += f" --config {shlex.quote(config_path)}"
             if shard is not None:
                 replay += f" --shard {shard}"
             lines.extend(
@@ -333,3 +354,8 @@ def prairielearn_backend(
         tmp_path_factory.mktemp("prairielearn-core"),
         clone,
     )
+
+
+@pytest.fixture(scope="session")
+def harness_config(request: pytest.FixtureRequest) -> HarnessConfig:
+    return request.config.stash[_CONFIG_KEY]

{pl_e2e_test-0.1.3 → pl_e2e_test-0.1.4}/src/prairielearn_e2e/question_tests.py

@@ -28,7 +28,12 @@ def _run_check(
         pytest.skip(reason)
     backend: PrairieLearnBackend = request.getfixturevalue("prairielearn_backend")
     try:
-        check(QuestionHarness(question, backend), seed)
+        check(
+            QuestionHarness(
+                question, backend, config=request.getfixturevalue("harness_config")
+            ),
+            seed,
+        )
     except UnsupportedQuestion as exc:
         pytest.skip(str(exc))
 

{pl_e2e_test-0.1.3 → pl_e2e_test-0.1.4}/src/prairielearn_e2e/reporting.py

@@ -503,7 +503,7 @@ def _print_terminal(results: Sequence[ItemResult], warnings: Sequence[str]) -> N
             print(f"{result.outcome.upper():8} {result.nodeid}")
     duration = sum(result.duration for result in results)
     print(
-        f"==================== {_summary(results)} in {duration:.2f}s "
+        f"==================== {_summary(results)}; combined test time: {duration:.2f}s "
         "===================="
     )
 

pl_e2e_test-0.1.4/tests/test_config.py

@@ -0,0 +1,88 @@
+from pathlib import Path
+from typing import cast
+
+import pytest
+
+from prairielearn_e2e import load_config, pytest_plugin
+from prairielearn_e2e.config import DEFAULT_CONFIG
+from prairielearn_e2e.errors import HarnessError
+from prairielearn_e2e.html_checks import (
+    non_gradable_answer_names,
+    validate_declared_answers,
+)
+
+from .test_pytest_plugin import FakeConfig
+
+
+def test_missing_course_config_uses_defaults(tmp_path: Path) -> None:
+    assert load_config(course_root=tmp_path) == DEFAULT_CONFIG
+    with pytest.raises(FileNotFoundError):
+        load_config(tmp_path / "missing.toml")
+
+
+def test_custom_selectors_replace_lists_without_changing_defaults(
+    tmp_path: Path,
+) -> None:
+    path = tmp_path / "pyproject.toml"
+    path.write_text(
+        "[tool.pl-e2e-test]\n"
+        'elementSolutionSelectors = ["custom-input"]\n'
+        'nonGradableSelectors = ["custom-display"]\n'
+    )
+    config = load_config(course_root=tmp_path)
+    html = '<custom-input answers-name="x"></custom-input><custom-display answers-name="y"></custom-display>'
+    assert validate_declared_answers(html, {}, True, config) == ("x", "y")
+    assert non_gradable_answer_names(html, config) == {"y"}
+    with pytest.raises(HarnessError, match="non-null correct answers"):
+        validate_declared_answers(html, {}, True)
+    with pytest.raises(HarnessError, match="non-null correct answers"):
+        validate_declared_answers('<pl-sketch answers-name="x"/>', {}, True, config)
+
+
+def test_omitted_settings_keep_defaults_and_empty_lists_disable(tmp_path: Path) -> None:
+    path = tmp_path / "settings.toml"
+    path.write_text("[tool.pl-e2e-test]\nnonGradableSelectors = []\n")
+    config = load_config(path)
+    assert config.element_solution_answer_selectors == ("pl-sketch",)
+    assert config.non_gradable_answer_selectors == ()
+    path.write_text('[project]\nname = "course"\n')
+    assert load_config(path) == DEFAULT_CONFIG
+
+
+@pytest.mark.parametrize(
+    "contents",
+    [
+        "invalid toml",
+        'tool = "bad"',
+        '[tool]\npl-e2e-test = "bad"',
+        '[tool.pl-e2e-test]\nelementSolutionSelectors = "pl-sketch"',
+        "[tool.pl-e2e-test]\nnonGradableSelectors = [1]",
+        '[tool.pl-e2e-test]\nnonGradableSelectors = [" "]',
+    ],
+)
+def test_invalid_config_is_a_usage_error(tmp_path: Path, contents: str) -> None:
+    path = tmp_path / "settings.toml"
+    path.write_text(contents)
+    config = FakeConfig(None)
+    config.options["e2e_config"] = str(path)
+    with pytest.raises(pytest.UsageError, match="unable to load harness config"):
+        pytest_plugin.pytest_configure(cast(pytest.Config, config))
+
+
+def test_plugin_loads_course_config_and_explicit_override(tmp_path: Path) -> None:
+    (tmp_path / "pyproject.toml").write_text(
+        "[tool.pl-e2e-test]\nnonGradableSelectors = []\n"
+    )
+    config = FakeConfig(None)
+    config.options["course_root"] = str(tmp_path)
+    pytest_plugin.pytest_configure(cast(pytest.Config, config))
+    assert config.stash[pytest_plugin._CONFIG_KEY] == load_config(course_root=tmp_path)
+
+    explicit = tmp_path / "custom settings.toml"
+    explicit.write_text("[tool.pl-e2e-test]\nelementSolutionSelectors = []\n")
+    config.options["e2e_config"] = str(explicit)
+    pytest_plugin.pytest_configure(cast(pytest.Config, config))
+    assert config.stash[pytest_plugin._CONFIG_KEY] == load_config(explicit)
+    header = pytest_plugin.pytest_report_header(cast(pytest.Config, config))
+    assert header is not None
+    assert f"--config '{explicit}'" in header

{pl_e2e_test-0.1.3 → pl_e2e_test-0.1.4}/tests/test_fuzz.py

@@ -175,6 +175,8 @@ def test_main_forwards_selected_questions_and_seed(
         [
             "--course-root",
             str(tmp_path),
+            "--config",
+            "custom settings.toml",
             "--prairielearn-path",
             "/tmp/prairielearn",
             "--seed-count",
@@ -189,6 +191,8 @@ def test_main_forwards_selected_questions_and_seed(
     assert calls == [
         (
             [
+                "--config",
+                "custom settings.toml",
                 "--prairielearn-path",
                 "/tmp/prairielearn",
                 "--seed-count",
@@ -202,7 +206,9 @@ def test_main_forwards_selected_questions_and_seed(
             tmp_path,
         )
     ]
-    assert "E2E fuzz seed: 12345" in capsys.readouterr().out
+    output = capsys.readouterr().out
+    assert "E2E fuzz seed: 12345" in output
+    assert "--config 'custom settings.toml'" in output
 
 
 def test_main_reports_invalid_pyproject(

{pl_e2e_test-0.1.3 → pl_e2e_test-0.1.4}/tests/test_harness.py

@@ -69,3 +69,32 @@ def test_grade_check_reaches_grading_hooks(tmp_path: Path) -> None:
 
     with pytest.raises(RuntimeError, match="grading reached"):
         harness.run_grade(0)
+
+
+def test_custom_config_applies_to_render_and_grade(tmp_path: Path) -> None:
+    from prairielearn_e2e import HarnessConfig
+    from prairielearn_e2e.errors import HarnessError
+
+    harness, backend = _harness(tmp_path)
+    (harness.question.directory / "question.html").write_text(
+        '<custom-display answers-name="display"></custom-display>'
+        '<custom-input answers-name="answer"></custom-input>'
+    )
+    (harness.question.directory / "server.py").write_text(
+        "def test(data):\n"
+        '    data["raw_submitted_answers"]["answer"] = "yes"\n'
+        '    data["partial_scores"]["answer"] = {"score": 1}\n'
+    )
+    with pytest.raises(HarnessError, match="non-null correct answers"):
+        harness.run_render(0)
+
+    configured = QuestionHarness(
+        harness.question,
+        cast(PrairieLearnBackend, backend),
+        config=HarnessConfig(
+            element_solution_answer_selectors=("custom-input",),
+            non_gradable_answer_selectors=("custom-display",),
+        ),
+    )
+    configured.run_render(0)
+    assert configured.run_grade(0).score == 1

{pl_e2e_test-0.1.3 → pl_e2e_test-0.1.4}/tests/test_pytest_plugin.py

@@ -11,6 +11,8 @@ from prairielearn_e2e.sharding import Shard
 class FakeConfig:
     def __init__(self, prairielearn_path: str | None) -> None:
         self.options: dict[str, Any] = {
+            "e2e_config": None,
+            "course_root": None,
             "question_path": ["questions/example"],
             "fuzz_seeds": True,
             "fuzz_seed": 12345,

{pl_e2e_test-0.1.3 → pl_e2e_test-0.1.4}/tests/test_reporting.py

@@ -95,7 +95,7 @@ def test_merge_reports_writes_terminal_and_junit(
 
     assert merge_reports([report_dir], junit) == 0
 
-    assert "2 passed, 1 skipped" in capsys.readouterr().out
+    assert "2 passed, 1 skipped; combined test time: 2.25s" in capsys.readouterr().out
     suite = ET.parse(junit).getroot().find("testsuite")
     assert suite is not None
     assert suite.attrib == {

pl_e2e_test-0.1.3/src/prairielearn_e2e/config.json

@@ -1,7 +0,0 @@
-{
-  "element_solution_answer_selectors": ["pl-sketch"],
-  "non_gradable_answer_selectors": [
-    "pl-sketch[read-only=\"true\"]",
-    "pl-sketch[read-only=\"1\"]"
-  ]
-}

pl_e2e_test-0.1.3/src/prairielearn_e2e/config.py

@@ -1,30 +0,0 @@
-"""Load declarative configuration for the question harness."""
-
-from __future__ import annotations
-
-import json
-from pathlib import Path
-from typing import Any
-
-_CONFIG_PATH = Path(__file__).with_name("config.json")
-
-
-def _load_config() -> dict[str, Any]:
-    with _CONFIG_PATH.open(encoding="utf-8") as config_file:
-        config = json.load(config_file)
-    if not isinstance(config, dict):
-        raise TypeError("e2e-tests config must be a JSON object")
-    return config
-
-
-_CONFIG = _load_config()
-ELEMENT_SOLUTION_ANSWER_SELECTORS: tuple[str, ...] = tuple(
-    selector
-    for selector in _CONFIG.get("element_solution_answer_selectors", [])
-    if isinstance(selector, str) and selector.strip()
-)
-NON_GRADABLE_ANSWER_SELECTORS: tuple[str, ...] = tuple(
-    selector
-    for selector in _CONFIG.get("non_gradable_answer_selectors", [])
-    if isinstance(selector, str) and selector.strip()
-)