pl-e2e-test 0.1.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (50) hide show
  1. pl_e2e_test-0.1.0/LICENSE +21 -0
  2. pl_e2e_test-0.1.0/MANIFEST.in +2 -0
  3. pl_e2e_test-0.1.0/PKG-INFO +322 -0
  4. pl_e2e_test-0.1.0/README.md +299 -0
  5. pl_e2e_test-0.1.0/pyproject.toml +59 -0
  6. pl_e2e_test-0.1.0/setup.cfg +4 -0
  7. pl_e2e_test-0.1.0/src/pl_e2e_test.egg-info/PKG-INFO +322 -0
  8. pl_e2e_test-0.1.0/src/pl_e2e_test.egg-info/SOURCES.txt +48 -0
  9. pl_e2e_test-0.1.0/src/pl_e2e_test.egg-info/dependency_links.txt +1 -0
  10. pl_e2e_test-0.1.0/src/pl_e2e_test.egg-info/entry_points.txt +5 -0
  11. pl_e2e_test-0.1.0/src/pl_e2e_test.egg-info/requires.txt +10 -0
  12. pl_e2e_test-0.1.0/src/pl_e2e_test.egg-info/top_level.txt +1 -0
  13. pl_e2e_test-0.1.0/src/prairielearn_e2e/__init__.py +18 -0
  14. pl_e2e_test-0.1.0/src/prairielearn_e2e/__main__.py +3 -0
  15. pl_e2e_test-0.1.0/src/prairielearn_e2e/_git.py +167 -0
  16. pl_e2e_test-0.1.0/src/prairielearn_e2e/backend.py +168 -0
  17. pl_e2e_test-0.1.0/src/prairielearn_e2e/config.json +7 -0
  18. pl_e2e_test-0.1.0/src/prairielearn_e2e/config.py +30 -0
  19. pl_e2e_test-0.1.0/src/prairielearn_e2e/contract.py +199 -0
  20. pl_e2e_test-0.1.0/src/prairielearn_e2e/discovery.py +212 -0
  21. pl_e2e_test-0.1.0/src/prairielearn_e2e/errors.py +9 -0
  22. pl_e2e_test-0.1.0/src/prairielearn_e2e/fuzz.py +293 -0
  23. pl_e2e_test-0.1.0/src/prairielearn_e2e/harness.py +457 -0
  24. pl_e2e_test-0.1.0/src/prairielearn_e2e/html_checks.py +133 -0
  25. pl_e2e_test-0.1.0/src/prairielearn_e2e/metadata.py +56 -0
  26. pl_e2e_test-0.1.0/src/prairielearn_e2e/pytest_plugin.py +335 -0
  27. pl_e2e_test-0.1.0/src/prairielearn_e2e/question_tests.py +68 -0
  28. pl_e2e_test-0.1.0/src/prairielearn_e2e/regression.py +128 -0
  29. pl_e2e_test-0.1.0/src/prairielearn_e2e/reporting.py +675 -0
  30. pl_e2e_test-0.1.0/src/prairielearn_e2e/repository.py +298 -0
  31. pl_e2e_test-0.1.0/src/prairielearn_e2e/runner.py +28 -0
  32. pl_e2e_test-0.1.0/src/prairielearn_e2e/server.py +128 -0
  33. pl_e2e_test-0.1.0/src/prairielearn_e2e/sharding.py +46 -0
  34. pl_e2e_test-0.1.0/tests/__init__.py +1 -0
  35. pl_e2e_test-0.1.0/tests/conftest.py +20 -0
  36. pl_e2e_test-0.1.0/tests/stub_git.py +128 -0
  37. pl_e2e_test-0.1.0/tests/test_contract.py +93 -0
  38. pl_e2e_test-0.1.0/tests/test_discovery.py +210 -0
  39. pl_e2e_test-0.1.0/tests/test_fuzz.py +306 -0
  40. pl_e2e_test-0.1.0/tests/test_git.py +65 -0
  41. pl_e2e_test-0.1.0/tests/test_harness.py +71 -0
  42. pl_e2e_test-0.1.0/tests/test_html_checks.py +56 -0
  43. pl_e2e_test-0.1.0/tests/test_metadata.py +40 -0
  44. pl_e2e_test-0.1.0/tests/test_publish_version.py +134 -0
  45. pl_e2e_test-0.1.0/tests/test_pytest_plugin.py +84 -0
  46. pl_e2e_test-0.1.0/tests/test_regression.py +49 -0
  47. pl_e2e_test-0.1.0/tests/test_reporting.py +286 -0
  48. pl_e2e_test-0.1.0/tests/test_repository.py +334 -0
  49. pl_e2e_test-0.1.0/tests/test_server.py +77 -0
  50. pl_e2e_test-0.1.0/tests/test_sharding.py +90 -0
@@ -0,0 +1,21 @@
1
+ MIT License
2
+
3
+ Copyright (c) 2026 PrairieLearn Course Template contributors
4
+
5
+ Permission is hereby granted, free of charge, to any person obtaining a copy
6
+ of this software and associated documentation files (the "Software"), to deal
7
+ in the Software without restriction, including without limitation the rights
8
+ to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
9
+ copies of the Software, and to permit persons to whom the Software is
10
+ furnished to do so, subject to the following conditions:
11
+
12
+ The above copyright notice and this permission notice shall be included in all
13
+ copies or substantial portions of the Software.
14
+
15
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16
+ IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17
+ FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18
+ AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19
+ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20
+ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
21
+ SOFTWARE.
@@ -0,0 +1,2 @@
1
+ graft tests
2
+ global-exclude *.py[cod]
@@ -0,0 +1,322 @@
1
+ Metadata-Version: 2.4
2
+ Name: pl-e2e-test
3
+ Version: 0.1.0
4
+ Summary: Fuzz PrairieLearn question lifecycles against the real Python element runtime
5
+ License-Expression: MIT
6
+ Project-URL: Homepage, https://github.com/PrairieLearn/pl-e2e-test
7
+ Project-URL: Issues, https://github.com/PrairieLearn/pl-e2e-test/issues
8
+ Project-URL: Repository, https://github.com/PrairieLearn/pl-e2e-test
9
+ Requires-Python: >=3.13
10
+ Description-Content-Type: text/markdown
11
+ License-File: LICENSE
12
+ Requires-Dist: chevron==0.14.0
13
+ Requires-Dist: filelock>=3.20
14
+ Requires-Dist: jsonschema>=4.26
15
+ Requires-Dist: lxml>=6.1
16
+ Requires-Dist: numpy>=2.4
17
+ Requires-Dist: platformdirs>=4.5
18
+ Requires-Dist: prairielearn
19
+ Requires-Dist: pyright>=1.1.414
20
+ Requires-Dist: pytest>=9.1
21
+ Requires-Dist: ruff>=0.16
22
+ Dynamic: license-file
23
+
24
+ # pl-e2e-test
25
+
26
+ `pl-e2e-test` fuzzes PrairieLearn v3 questions through their real Python
27
+ generate, prepare, render, parse, grade, test, and file phases. It was extracted
28
+ from the `pl-oer-lv102` course so that courses can install one uv-managed tool
29
+ instead of copying the harness and its support scripts.
30
+
31
+ The package installs the reusable `prairielearn_e2e` Python API and these commands:
32
+
33
+ - `pl-e2e-test` runs the installed lifecycle pytest suite.
34
+ - `pl-e2e-test-diff` fuzzes questions selected from a Git diff.
35
+ - `pl-e2e-add-regression-seed` records a failing concrete variant seed.
36
+ - `pl-e2e-merge-reports` merges parallel shard results into terminal and JUnit reports.
37
+
38
+ ## Requirements and installation
39
+
40
+ - Python 3.13 or newer
41
+ - Git 2.30 or newer
42
+
43
+ The harness depends on pytest, Pyright, Ruff, Chevron, JSON Schema, lxml, NumPy,
44
+ and PrairieLearn. PrairieLearn's Python package is currently consumed from its Git
45
+ repository, so the course project must tell uv where to resolve that dependency:
46
+
47
+ ```toml
48
+ [dependency-groups]
49
+ dev = ["pl-e2e-test"]
50
+
51
+ [tool.uv.sources.prairielearn]
52
+ git = "https://github.com/PrairieLearn/PrairieLearn.git"
53
+ subdirectory = "apps/prairielearn/python"
54
+ ```
55
+
56
+ Then lock and install the environment:
57
+
58
+ ```sh
59
+ uv lock
60
+ uv sync
61
+ ```
62
+
63
+ For local development of this package, use a path source in the consuming project:
64
+
65
+ ```toml
66
+ [tool.uv.sources.pl-e2e-test]
67
+ path = "../pl-e2e-test"
68
+ editable = true
69
+ ```
70
+
71
+ ## Running lifecycle tests
72
+
73
+ Pass a question, a directory of questions, or an `info.json` file. Repeat
74
+ `--question-path` to select multiple roots:
75
+
76
+ ```sh
77
+ uv run pl-e2e-test \
78
+ --question-path questions \
79
+ --seed-count 3
80
+ ```
81
+
82
+ The installed suite exposes separate `generate`, `prepare`, `render`, and `grade`
83
+ pytest markers. Any remaining arguments are passed to pytest:
84
+
85
+ ```sh
86
+ uv run pl-e2e-test \
87
+ --question-path questions/chapter/example \
88
+ --fuzz-seeds -m render -x -vv
89
+ ```
90
+
91
+ `--fuzz-seeds` creates and reports a random 64-bit master seed. Reproduce the same
92
+ variant set with `--fuzz-seed N`. Without either option, deterministic seeds start at
93
+ zero. Question metadata with `singleVariant: true` always uses seed zero.
94
+
95
+ ### CI sharding
96
+
97
+ Use `--shard INDEX/COUNT` to divide the selected question variants between parallel
98
+ jobs. The index is one-based. All selected lifecycle checks for one question and seed
99
+ stay together, and variants are assigned round-robin so shard sizes differ by at most
100
+ one. Pytest filters such as `-m` and `-k` are applied before sharding.
101
+
102
+ Add `--shard-report PATH` to write a mergeable result blob. Every shard must use the
103
+ same checkout and arguments. Sharded fuzz runs must also use one explicit shared
104
+ `--fuzz-seed`; automatic random fuzz seeds are rejected. After downloading all blobs,
105
+ merge them into a combined terminal summary and JUnit report:
106
+
107
+ ```sh
108
+ uv run pl-e2e-merge-reports e2e-reports \
109
+ --junitxml pl-e2e-results.xml
110
+ ```
111
+
112
+ The merger rejects missing or duplicate shards, mismatched test collections, mixed
113
+ fuzz seeds, corrupt reports, and tests left unexecuted by interruption or `-x`.
114
+ It exits unsuccessfully when the aggregate suite failed or was incomplete.
115
+
116
+ For example, a course can use this GitHub Actions matrix:
117
+
118
+ ```yaml
119
+ jobs:
120
+ e2e:
121
+ runs-on: ubuntu-latest
122
+ strategy:
123
+ fail-fast: false
124
+ matrix:
125
+ shard: [1, 2, 3, 4]
126
+ steps:
127
+ - uses: actions/checkout@v4
128
+ - uses: astral-sh/setup-uv@v9
129
+ with:
130
+ enable-cache: true
131
+ - run: uv python install && uv sync --locked
132
+ - name: Run shard
133
+ run: |
134
+ uv run pl-e2e-test \
135
+ --question-path questions \
136
+ --seed-count 10 \
137
+ --fuzz-seed "${{ github.run_id }}" \
138
+ --shard "${{ matrix.shard }}/4" \
139
+ --shard-report "e2e-reports/shard-${{ matrix.shard }}.pl-e2e-report.json"
140
+ - name: Upload shard report
141
+ if: ${{ !cancelled() }}
142
+ uses: actions/upload-artifact@v4
143
+ with:
144
+ name: e2e-report-${{ matrix.shard }}
145
+ path: e2e-reports/
146
+ if-no-files-found: error
147
+ retention-days: 1
148
+
149
+ e2e-report:
150
+ if: ${{ !cancelled() }}
151
+ needs: e2e
152
+ runs-on: ubuntu-latest
153
+ steps:
154
+ - uses: actions/checkout@v4
155
+ - uses: astral-sh/setup-uv@v9
156
+ - run: uv python install && uv sync --locked
157
+ - uses: actions/download-artifact@v4
158
+ with:
159
+ pattern: e2e-report-*
160
+ path: e2e-reports
161
+ merge-multiple: true
162
+ - name: Merge reports
163
+ run: |
164
+ uv run pl-e2e-merge-reports e2e-reports \
165
+ --junitxml pl-e2e-results.xml
166
+ - name: Upload merged JUnit report
167
+ if: ${{ always() }}
168
+ uses: actions/upload-artifact@v4
169
+ with:
170
+ name: pl-e2e-results
171
+ path: pl-e2e-results.xml
172
+ if-no-files-found: warn
173
+ ```
174
+
175
+ The course must provide `.prairielearn/schemas/infoQuestion.json`. The harness checks
176
+ question metadata against that schema and uses the exact PrairieLearn revision
177
+ installed in the uv environment. It stores a bare, partial PrairieLearn repository in
178
+ the platform-standard user cache directory. The first use of a new Git source or
179
+ revision requires network access; later runs reuse that exact revision offline. The
180
+ uv lockfile, rather than the cache, decides which revision is tested.
181
+
182
+ To use an existing PrairieLearn worktree or bare repository instead, pass
183
+ `--prairielearn-path /path/to/PrairieLearn`. The repository must already contain the
184
+ installed revision. An explicit path is authoritative: the harness does not fetch into
185
+ it or fall back to the managed cache. In either mode, only the pinned revision's Python
186
+ element controllers are archived into the test run's temporary directory. Standard
187
+ cache environment settings such as `XDG_CACHE_HOME` can relocate the managed cache;
188
+ CI systems can persist that directory between runs if desired.
189
+
190
+ ### Question-local configuration
191
+
192
+ Store saved concrete seeds and narrow phase skips in `.question-e2e.json` beside the
193
+ question's `info.json`:
194
+
195
+ ```json
196
+ {
197
+ "regression_seeds": [1843927501],
198
+ "skip_methods": {
199
+ "grade": "Temporarily blocked by an upstream issue"
200
+ }
201
+ }
202
+ ```
203
+
204
+ Saved regression seeds run before newly generated seeds. Skips require a nonempty
205
+ reason and may name only `generate`, `prepare`, `render`, or `grade`. Manual-grading
206
+ questions automatically skip only the grade check.
207
+
208
+ Record the concrete seed shown in a failing pytest case with:
209
+
210
+ ```sh
211
+ uv run pl-e2e-add-regression-seed \
212
+ questions/chapter/example 1843927501
213
+ ```
214
+
215
+ The command atomically creates or updates `.question-e2e.json`, preserves other
216
+ settings, and sorts and deduplicates the seed list.
217
+
218
+ ## Diff-based fuzzing
219
+
220
+ The diff runner tests changed question directories. A non-ignored changed file outside
221
+ `questions/` selects every question:
222
+
223
+ ```sh
224
+ uv run pl-e2e-test-diff \
225
+ --base origin/main \
226
+ --head HEAD \
227
+ --seed-count 10
228
+ ```
229
+
230
+ Optional repository-relative ignore rules live in `.question-e2e.diffignore` and use
231
+ Git's native ignore syntax. Missing default configuration means that no paths are
232
+ ignored. Pass `--e2ediffignore PATH` to require and use another file. Additional
233
+ arguments are forwarded to pytest.
234
+
235
+ In pull-request CI, fetch the base commit before running the command and pass the
236
+ appropriate base and head SHAs. Set `--fuzz-seed` to replay a reported run.
237
+ `pl-e2e-test-diff` accepts the same `--shard` and `--shard-report` options. If a diff
238
+ selects no questions, it writes a successful empty shard report so the merge job can
239
+ still verify that every matrix job completed.
240
+
241
+ ## Python API
242
+
243
+ The original API remains available:
244
+
245
+ ```python
246
+ from prairielearn_e2e import PrairieLearnBackend, QuestionCase, QuestionHarness
247
+ ```
248
+
249
+ `QuestionHarness.run_generate`, `run_prepare`, `run_render`, and `run_grade` execute
250
+ increasingly complete portions of the lifecycle. `run_variant` remains an alias for
251
+ the complete grade path.
252
+
253
+ ## Development
254
+
255
+ This project uses uv 0.9 or newer:
256
+
257
+ ```sh
258
+ uv sync
259
+ make test
260
+ make format
261
+ make smoke-dist
262
+ ```
263
+
264
+ `make test` runs pytest, Pyright, Ruff linting, and Ruff's formatting check.
265
+ `make smoke-dist` builds both distribution formats, installs each without dependencies
266
+ into an isolated target, and imports it using the locked development dependencies.
267
+
268
+ ## Publishing
269
+
270
+ There are three supported publishing paths. Choose one for a release; do not run the
271
+ local and GitHub paths for the same version because PyPI versions are immutable.
272
+
273
+ ### GitHub-hosted trusted publishing
274
+
275
+ Configure the PyPI project to trust the `publish.yml` workflow in the GitHub `pypi`
276
+ environment. From a clean branch, run:
277
+
278
+ ```sh
279
+ make publish-version VERSION=0.2.0
280
+ ```
281
+
282
+ This runs the full checks, updates `pyproject.toml` and `uv.lock`, builds the
283
+ distributions, creates a release commit and annotated `v0.2.0` tag, and atomically
284
+ pushes the branch and tag. The tag starts the workflow on `ubuntu-latest`; the workflow
285
+ rechecks the tag/version match, tests, builds, generates attestations, and publishes
286
+ with PyPI trusted-publishing OIDC credentials. Use `REMOTE=name` for a different Git
287
+ remote.
288
+
289
+ If an unpushed commit already contains the requested version, use the guarded bypass:
290
+
291
+ ```sh
292
+ make publish-version VERSION=0.2.0 ALLOW_PREBUMPED_VERSION=1
293
+ ```
294
+
295
+ ### Direct local publishing
296
+
297
+ To publish the version already present in `pyproject.toml` directly from this machine,
298
+ configure a uv-supported PyPI credential such as `UV_PUBLISH_TOKEN`, then run:
299
+
300
+ ```sh
301
+ make publish-local
302
+ ```
303
+
304
+ This runs all checks, builds and smoke-tests the current wheel and source archive, and
305
+ passes only those two version-matched files to `uv publish`. It does not create a Git
306
+ commit or tag.
307
+
308
+ ### Local self-hosted GitHub runner
309
+
310
+ The same trusted-publishing workflow can run on a repository-scoped Linux ARM64 runner
311
+ inside Docker. The container has no host mounts and does not receive the Docker socket.
312
+ Authenticate `gh` with repository administration permission, then run:
313
+
314
+ ```sh
315
+ make runner-start
316
+ make publish-self-hosted RELEASE_TAG=v0.2.0
317
+ ```
318
+
319
+ The tag must already exist. This dispatches `publish.yml` from `main` with the
320
+ `self-hosted` runner choice. `make runner-ensure` starts Docker Desktop on macOS when
321
+ needed and repairs the runner before dispatch. Other lifecycle commands are
322
+ `runner-status`, `runner-stop`, `runner-remove`, `runner-logs`, and `clean-runner`.
@@ -0,0 +1,299 @@
1
+ # pl-e2e-test
2
+
3
+ `pl-e2e-test` fuzzes PrairieLearn v3 questions through their real Python
4
+ generate, prepare, render, parse, grade, test, and file phases. It was extracted
5
+ from the `pl-oer-lv102` course so that courses can install one uv-managed tool
6
+ instead of copying the harness and its support scripts.
7
+
8
+ The package installs the reusable `prairielearn_e2e` Python API and these commands:
9
+
10
+ - `pl-e2e-test` runs the installed lifecycle pytest suite.
11
+ - `pl-e2e-test-diff` fuzzes questions selected from a Git diff.
12
+ - `pl-e2e-add-regression-seed` records a failing concrete variant seed.
13
+ - `pl-e2e-merge-reports` merges parallel shard results into terminal and JUnit reports.
14
+
15
+ ## Requirements and installation
16
+
17
+ - Python 3.13 or newer
18
+ - Git 2.30 or newer
19
+
20
+ The harness depends on pytest, Pyright, Ruff, Chevron, JSON Schema, lxml, NumPy,
21
+ and PrairieLearn. PrairieLearn's Python package is currently consumed from its Git
22
+ repository, so the course project must tell uv where to resolve that dependency:
23
+
24
+ ```toml
25
+ [dependency-groups]
26
+ dev = ["pl-e2e-test"]
27
+
28
+ [tool.uv.sources.prairielearn]
29
+ git = "https://github.com/PrairieLearn/PrairieLearn.git"
30
+ subdirectory = "apps/prairielearn/python"
31
+ ```
32
+
33
+ Then lock and install the environment:
34
+
35
+ ```sh
36
+ uv lock
37
+ uv sync
38
+ ```
39
+
40
+ For local development of this package, use a path source in the consuming project:
41
+
42
+ ```toml
43
+ [tool.uv.sources.pl-e2e-test]
44
+ path = "../pl-e2e-test"
45
+ editable = true
46
+ ```
47
+
48
+ ## Running lifecycle tests
49
+
50
+ Pass a question, a directory of questions, or an `info.json` file. Repeat
51
+ `--question-path` to select multiple roots:
52
+
53
+ ```sh
54
+ uv run pl-e2e-test \
55
+ --question-path questions \
56
+ --seed-count 3
57
+ ```
58
+
59
+ The installed suite exposes separate `generate`, `prepare`, `render`, and `grade`
60
+ pytest markers. Any remaining arguments are passed to pytest:
61
+
62
+ ```sh
63
+ uv run pl-e2e-test \
64
+ --question-path questions/chapter/example \
65
+ --fuzz-seeds -m render -x -vv
66
+ ```
67
+
68
+ `--fuzz-seeds` creates and reports a random 64-bit master seed. Reproduce the same
69
+ variant set with `--fuzz-seed N`. Without either option, deterministic seeds start at
70
+ zero. Question metadata with `singleVariant: true` always uses seed zero.
71
+
72
+ ### CI sharding
73
+
74
+ Use `--shard INDEX/COUNT` to divide the selected question variants between parallel
75
+ jobs. The index is one-based. All selected lifecycle checks for one question and seed
76
+ stay together, and variants are assigned round-robin so shard sizes differ by at most
77
+ one. Pytest filters such as `-m` and `-k` are applied before sharding.
78
+
79
+ Add `--shard-report PATH` to write a mergeable result blob. Every shard must use the
80
+ same checkout and arguments. Sharded fuzz runs must also use one explicit shared
81
+ `--fuzz-seed`; automatic random fuzz seeds are rejected. After downloading all blobs,
82
+ merge them into a combined terminal summary and JUnit report:
83
+
84
+ ```sh
85
+ uv run pl-e2e-merge-reports e2e-reports \
86
+ --junitxml pl-e2e-results.xml
87
+ ```
88
+
89
+ The merger rejects missing or duplicate shards, mismatched test collections, mixed
90
+ fuzz seeds, corrupt reports, and tests left unexecuted by interruption or `-x`.
91
+ It exits unsuccessfully when the aggregate suite failed or was incomplete.
92
+
93
+ For example, a course can use this GitHub Actions matrix:
94
+
95
+ ```yaml
96
+ jobs:
97
+ e2e:
98
+ runs-on: ubuntu-latest
99
+ strategy:
100
+ fail-fast: false
101
+ matrix:
102
+ shard: [1, 2, 3, 4]
103
+ steps:
104
+ - uses: actions/checkout@v4
105
+ - uses: astral-sh/setup-uv@v9
106
+ with:
107
+ enable-cache: true
108
+ - run: uv python install && uv sync --locked
109
+ - name: Run shard
110
+ run: |
111
+ uv run pl-e2e-test \
112
+ --question-path questions \
113
+ --seed-count 10 \
114
+ --fuzz-seed "${{ github.run_id }}" \
115
+ --shard "${{ matrix.shard }}/4" \
116
+ --shard-report "e2e-reports/shard-${{ matrix.shard }}.pl-e2e-report.json"
117
+ - name: Upload shard report
118
+ if: ${{ !cancelled() }}
119
+ uses: actions/upload-artifact@v4
120
+ with:
121
+ name: e2e-report-${{ matrix.shard }}
122
+ path: e2e-reports/
123
+ if-no-files-found: error
124
+ retention-days: 1
125
+
126
+ e2e-report:
127
+ if: ${{ !cancelled() }}
128
+ needs: e2e
129
+ runs-on: ubuntu-latest
130
+ steps:
131
+ - uses: actions/checkout@v4
132
+ - uses: astral-sh/setup-uv@v9
133
+ - run: uv python install && uv sync --locked
134
+ - uses: actions/download-artifact@v4
135
+ with:
136
+ pattern: e2e-report-*
137
+ path: e2e-reports
138
+ merge-multiple: true
139
+ - name: Merge reports
140
+ run: |
141
+ uv run pl-e2e-merge-reports e2e-reports \
142
+ --junitxml pl-e2e-results.xml
143
+ - name: Upload merged JUnit report
144
+ if: ${{ always() }}
145
+ uses: actions/upload-artifact@v4
146
+ with:
147
+ name: pl-e2e-results
148
+ path: pl-e2e-results.xml
149
+ if-no-files-found: warn
150
+ ```
151
+
152
+ The course must provide `.prairielearn/schemas/infoQuestion.json`. The harness checks
153
+ question metadata against that schema and uses the exact PrairieLearn revision
154
+ installed in the uv environment. It stores a bare, partial PrairieLearn repository in
155
+ the platform-standard user cache directory. The first use of a new Git source or
156
+ revision requires network access; later runs reuse that exact revision offline. The
157
+ uv lockfile, rather than the cache, decides which revision is tested.
158
+
159
+ To use an existing PrairieLearn worktree or bare repository instead, pass
160
+ `--prairielearn-path /path/to/PrairieLearn`. The repository must already contain the
161
+ installed revision. An explicit path is authoritative: the harness does not fetch into
162
+ it or fall back to the managed cache. In either mode, only the pinned revision's Python
163
+ element controllers are archived into the test run's temporary directory. Standard
164
+ cache environment settings such as `XDG_CACHE_HOME` can relocate the managed cache;
165
+ CI systems can persist that directory between runs if desired.
166
+
167
+ ### Question-local configuration
168
+
169
+ Store saved concrete seeds and narrow phase skips in `.question-e2e.json` beside the
170
+ question's `info.json`:
171
+
172
+ ```json
173
+ {
174
+ "regression_seeds": [1843927501],
175
+ "skip_methods": {
176
+ "grade": "Temporarily blocked by an upstream issue"
177
+ }
178
+ }
179
+ ```
180
+
181
+ Saved regression seeds run before newly generated seeds. Skips require a nonempty
182
+ reason and may name only `generate`, `prepare`, `render`, or `grade`. Manual-grading
183
+ questions automatically skip only the grade check.
184
+
185
+ Record the concrete seed shown in a failing pytest case with:
186
+
187
+ ```sh
188
+ uv run pl-e2e-add-regression-seed \
189
+ questions/chapter/example 1843927501
190
+ ```
191
+
192
+ The command atomically creates or updates `.question-e2e.json`, preserves other
193
+ settings, and sorts and deduplicates the seed list.
194
+
195
+ ## Diff-based fuzzing
196
+
197
+ The diff runner tests changed question directories. A non-ignored changed file outside
198
+ `questions/` selects every question:
199
+
200
+ ```sh
201
+ uv run pl-e2e-test-diff \
202
+ --base origin/main \
203
+ --head HEAD \
204
+ --seed-count 10
205
+ ```
206
+
207
+ Optional repository-relative ignore rules live in `.question-e2e.diffignore` and use
208
+ Git's native ignore syntax. Missing default configuration means that no paths are
209
+ ignored. Pass `--e2ediffignore PATH` to require and use another file. Additional
210
+ arguments are forwarded to pytest.
211
+
212
+ In pull-request CI, fetch the base commit before running the command and pass the
213
+ appropriate base and head SHAs. Set `--fuzz-seed` to replay a reported run.
214
+ `pl-e2e-test-diff` accepts the same `--shard` and `--shard-report` options. If a diff
215
+ selects no questions, it writes a successful empty shard report so the merge job can
216
+ still verify that every matrix job completed.
217
+
218
+ ## Python API
219
+
220
+ The original API remains available:
221
+
222
+ ```python
223
+ from prairielearn_e2e import PrairieLearnBackend, QuestionCase, QuestionHarness
224
+ ```
225
+
226
+ `QuestionHarness.run_generate`, `run_prepare`, `run_render`, and `run_grade` execute
227
+ increasingly complete portions of the lifecycle. `run_variant` remains an alias for
228
+ the complete grade path.
229
+
230
+ ## Development
231
+
232
+ This project uses uv 0.9 or newer:
233
+
234
+ ```sh
235
+ uv sync
236
+ make test
237
+ make format
238
+ make smoke-dist
239
+ ```
240
+
241
+ `make test` runs pytest, Pyright, Ruff linting, and Ruff's formatting check.
242
+ `make smoke-dist` builds both distribution formats, installs each without dependencies
243
+ into an isolated target, and imports it using the locked development dependencies.
244
+
245
+ ## Publishing
246
+
247
+ There are three supported publishing paths. Choose one for a release; do not run the
248
+ local and GitHub paths for the same version because PyPI versions are immutable.
249
+
250
+ ### GitHub-hosted trusted publishing
251
+
252
+ Configure the PyPI project to trust the `publish.yml` workflow in the GitHub `pypi`
253
+ environment. From a clean branch, run:
254
+
255
+ ```sh
256
+ make publish-version VERSION=0.2.0
257
+ ```
258
+
259
+ This runs the full checks, updates `pyproject.toml` and `uv.lock`, builds the
260
+ distributions, creates a release commit and annotated `v0.2.0` tag, and atomically
261
+ pushes the branch and tag. The tag starts the workflow on `ubuntu-latest`; the workflow
262
+ rechecks the tag/version match, tests, builds, generates attestations, and publishes
263
+ with PyPI trusted-publishing OIDC credentials. Use `REMOTE=name` for a different Git
264
+ remote.
265
+
266
+ If an unpushed commit already contains the requested version, use the guarded bypass:
267
+
268
+ ```sh
269
+ make publish-version VERSION=0.2.0 ALLOW_PREBUMPED_VERSION=1
270
+ ```
271
+
272
+ ### Direct local publishing
273
+
274
+ To publish the version already present in `pyproject.toml` directly from this machine,
275
+ configure a uv-supported PyPI credential such as `UV_PUBLISH_TOKEN`, then run:
276
+
277
+ ```sh
278
+ make publish-local
279
+ ```
280
+
281
+ This runs all checks, builds and smoke-tests the current wheel and source archive, and
282
+ passes only those two version-matched files to `uv publish`. It does not create a Git
283
+ commit or tag.
284
+
285
+ ### Local self-hosted GitHub runner
286
+
287
+ The same trusted-publishing workflow can run on a repository-scoped Linux ARM64 runner
288
+ inside Docker. The container has no host mounts and does not receive the Docker socket.
289
+ Authenticate `gh` with repository administration permission, then run:
290
+
291
+ ```sh
292
+ make runner-start
293
+ make publish-self-hosted RELEASE_TAG=v0.2.0
294
+ ```
295
+
296
+ The tag must already exist. This dispatches `publish.yml` from `main` with the
297
+ `self-hosted` runner choice. `make runner-ensure` starts Docker Desktop on macOS when
298
+ needed and repairs the runner before dispatch. Other lifecycle commands are
299
+ `runner-status`, `runner-stop`, `runner-remove`, `runner-logs`, and `clean-runner`.
@@ -0,0 +1,59 @@
1
+ [build-system]
2
+ requires = ["setuptools>=69"]
3
+ build-backend = "setuptools.build_meta"
4
+
5
+ [project]
6
+ name = "pl-e2e-test"
7
+ version = "0.1.0"
8
+ description = "Fuzz PrairieLearn question lifecycles against the real Python element runtime"
9
+ readme = "README.md"
10
+ requires-python = ">=3.13"
11
+ license = "MIT"
12
+ dependencies = [
13
+ "chevron==0.14.0",
14
+ "filelock>=3.20",
15
+ "jsonschema>=4.26",
16
+ "lxml>=6.1",
17
+ "numpy>=2.4",
18
+ "platformdirs>=4.5",
19
+ "prairielearn",
20
+ "pyright>=1.1.414",
21
+ "pytest>=9.1",
22
+ "ruff>=0.16",
23
+ ]
24
+
25
+ [dependency-groups]
26
+ dev = ["pyfakefs>=6.2"]
27
+
28
+ [project.urls]
29
+ Homepage = "https://github.com/PrairieLearn/pl-e2e-test"
30
+ Issues = "https://github.com/PrairieLearn/pl-e2e-test/issues"
31
+ Repository = "https://github.com/PrairieLearn/pl-e2e-test"
32
+
33
+ [project.scripts]
34
+ pl-e2e-test = "prairielearn_e2e.runner:main"
35
+ pl-e2e-test-diff = "prairielearn_e2e.fuzz:main"
36
+ pl-e2e-add-regression-seed = "prairielearn_e2e.regression:main"
37
+ pl-e2e-merge-reports = "prairielearn_e2e.reporting:main"
38
+
39
+ [tool.setuptools.packages.find]
40
+ where = ["src"]
41
+
42
+ [tool.setuptools.package-data]
43
+ prairielearn_e2e = ["config.json"]
44
+
45
+ [tool.pytest.ini_options]
46
+ testpaths = ["tests"]
47
+
48
+ [tool.pyright]
49
+ include = ["src/prairielearn_e2e", "tests"]
50
+
51
+ [tool.uv]
52
+ required-version = ">=0.9.0"
53
+
54
+ [tool.uv.sources.prairielearn]
55
+ git = "https://github.com/PrairieLearn/PrairieLearn.git"
56
+ subdirectory = "apps/prairielearn/python"
57
+
58
+ [tool.ruff]
59
+ target-version = "py313"