pl-e2e-test 0.1.7__tar.gz → 0.1.8__tar.gz

{pl_e2e_test-0.1.7/src/pl_e2e_test.egg-info → pl_e2e_test-0.1.8}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: pl-e2e-test
-Version: 0.1.7
+Version: 0.1.8
 Summary: Fuzz PrairieLearn question lifecycles against the real Python element runtime
 License-Expression: MIT
 Project-URL: Homepage, https://github.com/PrairieLearn/pl-e2e-test
@@ -303,7 +303,7 @@ does not provide one itself.
 For every gradable `answers-name`, the correct test case must provide:
 
 - a non-null value in `data["raw_submitted_answers"]` under the answer name, or one or
-  more compound input keys prefixed with `answers-name-`; and
+  more compound input keys prefixed with `answers-name-` or `answers-name.`; and
 - `data["partial_scores"][answers_name]["score"] = 1`.
 
 Raw values must survive PrairieLearn's JSON boundary and have the same browser transport
@@ -332,11 +332,12 @@ def test(data):
 ```
 
 Compound elements keep the declared answer name as their grading identity while using
-namespaced child fields for browser inputs. For example, an element with
-`answers-name="match"` may test and parse `match-dropdown-0` and `match-dropdown-1`,
-then store its score in `partial_scores["match"]`; it does not need to invent a
-`submitted_answers["match"]` value. After parsing, every raw field associated with a
-declared gradable answer must either remain non-null under its child key or produce a
+namespaced child fields for browser inputs. Hyphen and dot namespace boundaries are
+supported. For example, an element with `answers-name="match"` may test and parse
+`match-dropdown-0` and `match-dropdown-1`, while `answers-name="average"` may use
+`average.main`. The element stores its score under the declared answer name and does not
+need to invent a direct raw value for it. After parsing, every raw field associated with
+a declared gradable answer must either remain non-null under its child key or produce a
 non-null value under its declared parent answer. This also supports elements that
 normalize `answer-input` into `answer`. Unassociated transport fields from file-producing
 elements may instead be consumed into `submitted_answers["_files"]`; they are not answer

{pl_e2e_test-0.1.7 → pl_e2e_test-0.1.8}/README.md

@@ -279,7 +279,7 @@ does not provide one itself.
 For every gradable `answers-name`, the correct test case must provide:
 
 - a non-null value in `data["raw_submitted_answers"]` under the answer name, or one or
-  more compound input keys prefixed with `answers-name-`; and
+  more compound input keys prefixed with `answers-name-` or `answers-name.`; and
 - `data["partial_scores"][answers_name]["score"] = 1`.
 
 Raw values must survive PrairieLearn's JSON boundary and have the same browser transport
@@ -308,11 +308,12 @@ def test(data):
 ```
 
 Compound elements keep the declared answer name as their grading identity while using
-namespaced child fields for browser inputs. For example, an element with
-`answers-name="match"` may test and parse `match-dropdown-0` and `match-dropdown-1`,
-then store its score in `partial_scores["match"]`; it does not need to invent a
-`submitted_answers["match"]` value. After parsing, every raw field associated with a
-declared gradable answer must either remain non-null under its child key or produce a
+namespaced child fields for browser inputs. Hyphen and dot namespace boundaries are
+supported. For example, an element with `answers-name="match"` may test and parse
+`match-dropdown-0` and `match-dropdown-1`, while `answers-name="average"` may use
+`average.main`. The element stores its score under the declared answer name and does not
+need to invent a direct raw value for it. After parsing, every raw field associated with
+a declared gradable answer must either remain non-null under its child key or produce a
 non-null value under its declared parent answer. This also supports elements that
 normalize `answer-input` into `answer`. Unassociated transport fields from file-producing
 elements may instead be consumed into `submitted_answers["_files"]`; they are not answer

{pl_e2e_test-0.1.7 → pl_e2e_test-0.1.8}/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "pl-e2e-test"
-version = "0.1.7"
+version = "0.1.8"
 description = "Fuzz PrairieLearn question lifecycles against the real Python element runtime"
 readme = "README.md"
 requires-python = ">=3.13"

{pl_e2e_test-0.1.7 → pl_e2e_test-0.1.8/src/pl_e2e_test.egg-info}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: pl-e2e-test
-Version: 0.1.7
+Version: 0.1.8
 Summary: Fuzz PrairieLearn question lifecycles against the real Python element runtime
 License-Expression: MIT
 Project-URL: Homepage, https://github.com/PrairieLearn/pl-e2e-test
@@ -303,7 +303,7 @@ does not provide one itself.
 For every gradable `answers-name`, the correct test case must provide:
 
 - a non-null value in `data["raw_submitted_answers"]` under the answer name, or one or
-  more compound input keys prefixed with `answers-name-`; and
+  more compound input keys prefixed with `answers-name-` or `answers-name.`; and
 - `data["partial_scores"][answers_name]["score"] = 1`.
 
 Raw values must survive PrairieLearn's JSON boundary and have the same browser transport
@@ -332,11 +332,12 @@ def test(data):
 ```
 
 Compound elements keep the declared answer name as their grading identity while using
-namespaced child fields for browser inputs. For example, an element with
-`answers-name="match"` may test and parse `match-dropdown-0` and `match-dropdown-1`,
-then store its score in `partial_scores["match"]`; it does not need to invent a
-`submitted_answers["match"]` value. After parsing, every raw field associated with a
-declared gradable answer must either remain non-null under its child key or produce a
+namespaced child fields for browser inputs. Hyphen and dot namespace boundaries are
+supported. For example, an element with `answers-name="match"` may test and parse
+`match-dropdown-0` and `match-dropdown-1`, while `answers-name="average"` may use
+`average.main`. The element stores its score under the declared answer name and does not
+need to invent a direct raw value for it. After parsing, every raw field associated with
+a declared gradable answer must either remain non-null under its child key or produce a
 non-null value under its declared parent answer. This also supports elements that
 normalize `answer-input` into `answer`. Unassociated transport fields from file-producing
 elements may instead be consumed into `submitted_answers["_files"]`; they are not answer

{pl_e2e_test-0.1.7 → pl_e2e_test-0.1.8}/src/prairielearn_e2e/discovery.py

@@ -234,23 +234,9 @@ class QuestionCase:
         return value if isinstance(value, dict) else {}
 
     def is_manual_grading(self) -> bool:
-        """Whether this question contains instructor-only grading content."""
-
-        metadata = self.unvalidated_metadata()
-        declared = metadata.get("gradingMethod") == "Manual"
-        question_path = self.directory / "question.html"
-        if not question_path.is_file():
-            return declared
-        has_manual_content = (
-            "<pl-manual-grading-only"
-            in question_path.read_text(encoding="utf-8").lower()
-        )
-        if has_manual_content and not declared:
-            raise HarnessError(
-                f"{self.info_path}: question uses pl-manual-grading-only "
-                'but info.json does not declare gradingMethod: "Manual"'
-            )
-        return declared or has_manual_content
+        """Whether question metadata configures manual-only grading."""
+
+        return self.unvalidated_metadata().get("gradingMethod") == "Manual"
 
 
 def variant_seeds(

{pl_e2e_test-0.1.7 → pl_e2e_test-0.1.8}/src/prairielearn_e2e/harness.py

@@ -75,7 +75,7 @@ def _associated_answer_name(raw_name: str, answer_names: tuple[str, ...]) -> str
         (
             answer_name
             for answer_name in answer_names
-            if raw_name.startswith(f"{answer_name}-")
+            if raw_name.startswith((f"{answer_name}-", f"{answer_name}."))
         ),
         key=len,
         default=None,

{pl_e2e_test-0.1.7 → pl_e2e_test-0.1.8}/tests/test_discovery.py

@@ -1,5 +1,6 @@
 from __future__ import annotations
 
+import json
 from pathlib import Path
 
 import pytest
@@ -269,28 +270,37 @@ def test_question_local_e2e_config_is_strict(
         load_e2e_config(config_path)
 
 
-def test_manual_grading_question_is_detected(tmp_path: Path) -> None:
+@pytest.mark.parametrize(
+    ("grading_method", "has_manual_content", "expected"),
+    [
+        (None, False, False),
+        (None, True, False),
+        ("Internal", True, False),
+        ("External", True, False),
+        ("Manual", False, True),
+        ("Manual", True, True),
+    ],
+)
+def test_manual_only_classification_uses_metadata(
+    tmp_path: Path,
+    grading_method: str | None,
+    has_manual_content: bool,
+    expected: bool,
+) -> None:
     question = tmp_path / "question"
     question.mkdir()
-    (question / "info.json").write_text(
-        '{"type": "v3", "gradingMethod": "Manual"}', encoding="utf-8"
-    )
-    (question / "question.html").write_text(
-        "<pl-manual-grading-only>guidance</pl-manual-grading-only>",
-        encoding="utf-8",
+    metadata = {"type": "v3"}
+    if grading_method is not None:
+        metadata["gradingMethod"] = grading_method
+    (question / "info.json").write_text(json.dumps(metadata), encoding="utf-8")
+    manual_content = (
+        "<pl-manual-grading-only>guidance</pl-manual-grading-only>"
+        if has_manual_content
+        else ""
     )
-
-    assert QuestionCase(question, tmp_path).is_manual_grading()
-
-
-def test_manual_grading_marker_requires_metadata(tmp_path: Path) -> None:
-    question = tmp_path / "question"
-    question.mkdir()
-    (question / "info.json").write_text('{"type": "v3"}', encoding="utf-8")
     (question / "question.html").write_text(
-        "<pl-manual-grading-only>guidance</pl-manual-grading-only>",
+        f"<p>Question</p>{manual_content}",
         encoding="utf-8",
     )
 
-    with pytest.raises(HarnessError, match="does not declare gradingMethod"):
-        QuestionCase(question, tmp_path).is_manual_grading()
+    assert QuestionCase(question, tmp_path).is_manual_grading() is expected

{pl_e2e_test-0.1.7 → pl_e2e_test-0.1.8}/tests/test_harness.py

@@ -8,6 +8,7 @@ import pytest
 from prairielearn_e2e import HarnessConfig, PrairieLearnBackend, QuestionHarness
 from prairielearn_e2e.contract import Phase
 from prairielearn_e2e.errors import HarnessError
+from prairielearn_e2e.harness import _associated_answer_name
 
 
 class RecordingBackend:
@@ -96,6 +97,20 @@ class ParentNormalizedAnswerBackend(RecordingBackend):
         return result
 
 
+class DotNormalizedAnswerBackend(RecordingBackend):
+    def process(
+        self, phase: Phase, html: str, data: dict[str, Any]
+    ) -> tuple[str | None, set[str]]:
+        result = super().process(phase, html, data)
+        if phase == "test":
+            data["raw_submitted_answers"]["average.main"] = '{"solution":[]}'
+            data["partial_scores"]["average"] = {"score": 1}
+        elif phase == "parse":
+            data["submitted_answers"].pop("average.main")
+            data["submitted_answers"]["average"] = "def average(values): ..."
+        return result
+
+
 def _harness(tmp_path: Path) -> tuple[QuestionHarness, RecordingBackend]:
     schemas = tmp_path / ".prairielearn" / "schemas"
     schemas.mkdir(parents=True)
@@ -382,3 +397,38 @@ def test_namespaced_raw_value_can_parse_into_declared_parent(tmp_path: Path) ->
 
     assert result.raw_submitted_answers == {"ordering-input": '[{"id":"a"}]'}
     assert result.submitted_answers == {"ordering": [{"id": "a"}]}
+
+
+def test_dot_namespaced_raw_value_can_parse_into_declared_parent(
+    tmp_path: Path,
+) -> None:
+    harness = _answer_harness(
+        tmp_path,
+        DotNormalizedAnswerBackend(),
+        html='<pl-faded-parsons answers-name="average"></pl-faded-parsons>',
+        selector="pl-faded-parsons",
+    )
+
+    result = harness.run_grade(0)
+
+    assert result.raw_submitted_answers == {"average.main": '{"solution":[]}'}
+    assert "average" not in result.raw_submitted_answers
+    assert result.submitted_answers == {"average": "def average(values): ..."}
+
+
+@pytest.mark.parametrize(
+    ("raw_name", "answer_names", "expected"),
+    [
+        ("average", ("average",), "average"),
+        ("match-dropdown-0", ("match",), "match"),
+        ("average.main", ("average",), "average"),
+        ("average.main", ("average", "average.main"), "average.main"),
+        ("average.main-value", ("average", "average.main"), "average.main"),
+        ("average_main", ("average",), None),
+        ("averageish", ("average",), None),
+    ],
+)
+def test_raw_answer_association_uses_supported_namespace_boundaries(
+    raw_name: str, answer_names: tuple[str, ...], expected: str | None
+) -> None:
+    assert _associated_answer_name(raw_name, answer_names) == expected

{pl_e2e_test-0.1.7 → pl_e2e_test-0.1.8}/tests/test_question_tests.py

@@ -7,17 +7,21 @@ from typing import Any, cast
 import pytest
 
 from prairielearn_e2e.config import DEFAULT_CONFIG
+from prairielearn_e2e.contract import Phase
 from prairielearn_e2e.discovery import QuestionCase
 from prairielearn_e2e.errors import HarnessError
 from prairielearn_e2e.expected_errors import ExpectedErrorAudit
 from prairielearn_e2e.question_tests import _run_check
+from prairielearn_e2e.question_tests import test_question_grade as run_question_grade
 
 
 class FakeRequest:
-    def __init__(self, audit: ExpectedErrorAudit) -> None:
+    def __init__(
+        self, audit: ExpectedErrorAudit, backend: object | None = None
+    ) -> None:
         self.fixtures: dict[str, Any] = {
             "expected_error_audit": audit,
-            "prairielearn_backend": object(),
+            "prairielearn_backend": backend if backend is not None else object(),
             "harness_config": DEFAULT_CONFIG,
         }
 
@@ -25,6 +29,23 @@ class FakeRequest:
         return self.fixtures[name]
 
 
+class LifecycleBackend:
+    def __init__(self) -> None:
+        self.phases: list[Phase] = []
+        self.grade_score: object | None = None
+
+    def process(
+        self, phase: Phase, html: str, data: dict[str, Any]
+    ) -> tuple[str | None, set[str]]:
+        self.phases.append(phase)
+        if phase == "grade":
+            self.grade_score = data["score"]
+        return (html if phase == "render" else None), set()
+
+    def unresolved_elements(self, _html: str) -> set[str]:
+        return set()
+
+
 def _question(tmp_path: Path, rules: list[dict[str, str]]) -> QuestionCase:
     directory = tmp_path / "questions" / "example"
     directory.mkdir(parents=True)
@@ -35,6 +56,44 @@ def _question(tmp_path: Path, rules: list[dict[str, str]]) -> QuestionCase:
     return QuestionCase(directory, tmp_path)
 
 
+def _lifecycle_question(tmp_path: Path, grading_method: str) -> QuestionCase:
+    schemas = tmp_path / ".prairielearn" / "schemas"
+    schemas.mkdir(parents=True)
+    (schemas / "infoQuestion.json").write_text("{}", encoding="utf-8")
+
+    directory = tmp_path / "questions" / "hybrid"
+    directory.mkdir(parents=True)
+    (directory / "info.json").write_text(
+        json.dumps(
+            {
+                "type": "v3",
+                "gradingMethod": grading_method,
+                "singleVariant": True,
+            }
+        ),
+        encoding="utf-8",
+    )
+    (directory / "question.html").write_text(
+        '<pl-number-input answers-name="answer"></pl-number-input>'
+        "<pl-question-panel>"
+        "<pl-manual-grading-only>"
+        "<p>Rubric for the manually graded explanation.</p>"
+        "</pl-manual-grading-only>"
+        "</pl-question-panel>",
+        encoding="utf-8",
+    )
+    (directory / "server.py").write_text(
+        "def generate(data):\n"
+        "    data['correct_answers']['answer'] = 1\n"
+        "\n"
+        "def test(data):\n"
+        "    data['raw_submitted_answers']['answer'] = '1'\n"
+        "    data['partial_scores']['answer'] = {'score': 1}\n",
+        encoding="utf-8",
+    )
+    return QuestionCase(directory, tmp_path)
+
+
 def _rule(
     *,
     exception_type: str | None = None,
@@ -128,3 +187,38 @@ def test_nonmatching_error_is_not_hidden(
             "grade",
             fail,
         )
+
+
+def test_hybrid_question_runs_automatic_grade_lifecycle(tmp_path: Path) -> None:
+    question = _lifecycle_question(tmp_path, "Internal")
+    backend = LifecycleBackend()
+
+    run_question_grade(
+        (question, 0),
+        cast(
+            pytest.FixtureRequest,
+            FakeRequest(ExpectedErrorAudit(), backend),
+        ),
+    )
+
+    assert "grade" in backend.phases
+    assert backend.phases[-1] == "file"
+    assert backend.grade_score == 1
+
+
+def test_manual_only_question_skips_grade_lifecycle(tmp_path: Path) -> None:
+    question = _lifecycle_question(tmp_path, "Manual")
+    backend = LifecycleBackend()
+
+    with pytest.raises(
+        pytest.skip.Exception, match="question is configured for manual grading"
+    ):
+        run_question_grade(
+            (question, 0),
+            cast(
+                pytest.FixtureRequest,
+                FakeRequest(ExpectedErrorAudit(), backend),
+            ),
+        )
+
+    assert backend.phases == []