pl-e2e-test 0.1.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- pl_e2e_test-0.1.0/LICENSE +21 -0
- pl_e2e_test-0.1.0/MANIFEST.in +2 -0
- pl_e2e_test-0.1.0/PKG-INFO +322 -0
- pl_e2e_test-0.1.0/README.md +299 -0
- pl_e2e_test-0.1.0/pyproject.toml +59 -0
- pl_e2e_test-0.1.0/setup.cfg +4 -0
- pl_e2e_test-0.1.0/src/pl_e2e_test.egg-info/PKG-INFO +322 -0
- pl_e2e_test-0.1.0/src/pl_e2e_test.egg-info/SOURCES.txt +48 -0
- pl_e2e_test-0.1.0/src/pl_e2e_test.egg-info/dependency_links.txt +1 -0
- pl_e2e_test-0.1.0/src/pl_e2e_test.egg-info/entry_points.txt +5 -0
- pl_e2e_test-0.1.0/src/pl_e2e_test.egg-info/requires.txt +10 -0
- pl_e2e_test-0.1.0/src/pl_e2e_test.egg-info/top_level.txt +1 -0
- pl_e2e_test-0.1.0/src/prairielearn_e2e/__init__.py +18 -0
- pl_e2e_test-0.1.0/src/prairielearn_e2e/__main__.py +3 -0
- pl_e2e_test-0.1.0/src/prairielearn_e2e/_git.py +167 -0
- pl_e2e_test-0.1.0/src/prairielearn_e2e/backend.py +168 -0
- pl_e2e_test-0.1.0/src/prairielearn_e2e/config.json +7 -0
- pl_e2e_test-0.1.0/src/prairielearn_e2e/config.py +30 -0
- pl_e2e_test-0.1.0/src/prairielearn_e2e/contract.py +199 -0
- pl_e2e_test-0.1.0/src/prairielearn_e2e/discovery.py +212 -0
- pl_e2e_test-0.1.0/src/prairielearn_e2e/errors.py +9 -0
- pl_e2e_test-0.1.0/src/prairielearn_e2e/fuzz.py +293 -0
- pl_e2e_test-0.1.0/src/prairielearn_e2e/harness.py +457 -0
- pl_e2e_test-0.1.0/src/prairielearn_e2e/html_checks.py +133 -0
- pl_e2e_test-0.1.0/src/prairielearn_e2e/metadata.py +56 -0
- pl_e2e_test-0.1.0/src/prairielearn_e2e/pytest_plugin.py +335 -0
- pl_e2e_test-0.1.0/src/prairielearn_e2e/question_tests.py +68 -0
- pl_e2e_test-0.1.0/src/prairielearn_e2e/regression.py +128 -0
- pl_e2e_test-0.1.0/src/prairielearn_e2e/reporting.py +675 -0
- pl_e2e_test-0.1.0/src/prairielearn_e2e/repository.py +298 -0
- pl_e2e_test-0.1.0/src/prairielearn_e2e/runner.py +28 -0
- pl_e2e_test-0.1.0/src/prairielearn_e2e/server.py +128 -0
- pl_e2e_test-0.1.0/src/prairielearn_e2e/sharding.py +46 -0
- pl_e2e_test-0.1.0/tests/__init__.py +1 -0
- pl_e2e_test-0.1.0/tests/conftest.py +20 -0
- pl_e2e_test-0.1.0/tests/stub_git.py +128 -0
- pl_e2e_test-0.1.0/tests/test_contract.py +93 -0
- pl_e2e_test-0.1.0/tests/test_discovery.py +210 -0
- pl_e2e_test-0.1.0/tests/test_fuzz.py +306 -0
- pl_e2e_test-0.1.0/tests/test_git.py +65 -0
- pl_e2e_test-0.1.0/tests/test_harness.py +71 -0
- pl_e2e_test-0.1.0/tests/test_html_checks.py +56 -0
- pl_e2e_test-0.1.0/tests/test_metadata.py +40 -0
- pl_e2e_test-0.1.0/tests/test_publish_version.py +134 -0
- pl_e2e_test-0.1.0/tests/test_pytest_plugin.py +84 -0
- pl_e2e_test-0.1.0/tests/test_regression.py +49 -0
- pl_e2e_test-0.1.0/tests/test_reporting.py +286 -0
- pl_e2e_test-0.1.0/tests/test_repository.py +334 -0
- pl_e2e_test-0.1.0/tests/test_server.py +77 -0
- pl_e2e_test-0.1.0/tests/test_sharding.py +90 -0
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 PrairieLearn Course Template contributors
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
|
@@ -0,0 +1,322 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: pl-e2e-test
|
|
3
|
+
Version: 0.1.0
|
|
4
|
+
Summary: Fuzz PrairieLearn question lifecycles against the real Python element runtime
|
|
5
|
+
License-Expression: MIT
|
|
6
|
+
Project-URL: Homepage, https://github.com/PrairieLearn/pl-e2e-test
|
|
7
|
+
Project-URL: Issues, https://github.com/PrairieLearn/pl-e2e-test/issues
|
|
8
|
+
Project-URL: Repository, https://github.com/PrairieLearn/pl-e2e-test
|
|
9
|
+
Requires-Python: >=3.13
|
|
10
|
+
Description-Content-Type: text/markdown
|
|
11
|
+
License-File: LICENSE
|
|
12
|
+
Requires-Dist: chevron==0.14.0
|
|
13
|
+
Requires-Dist: filelock>=3.20
|
|
14
|
+
Requires-Dist: jsonschema>=4.26
|
|
15
|
+
Requires-Dist: lxml>=6.1
|
|
16
|
+
Requires-Dist: numpy>=2.4
|
|
17
|
+
Requires-Dist: platformdirs>=4.5
|
|
18
|
+
Requires-Dist: prairielearn
|
|
19
|
+
Requires-Dist: pyright>=1.1.414
|
|
20
|
+
Requires-Dist: pytest>=9.1
|
|
21
|
+
Requires-Dist: ruff>=0.16
|
|
22
|
+
Dynamic: license-file
|
|
23
|
+
|
|
24
|
+
# pl-e2e-test
|
|
25
|
+
|
|
26
|
+
`pl-e2e-test` fuzzes PrairieLearn v3 questions through their real Python
|
|
27
|
+
generate, prepare, render, parse, grade, test, and file phases. It was extracted
|
|
28
|
+
from the `pl-oer-lv102` course so that courses can install one uv-managed tool
|
|
29
|
+
instead of copying the harness and its support scripts.
|
|
30
|
+
|
|
31
|
+
The package installs the reusable `prairielearn_e2e` Python API and these commands:
|
|
32
|
+
|
|
33
|
+
- `pl-e2e-test` runs the installed lifecycle pytest suite.
|
|
34
|
+
- `pl-e2e-test-diff` fuzzes questions selected from a Git diff.
|
|
35
|
+
- `pl-e2e-add-regression-seed` records a failing concrete variant seed.
|
|
36
|
+
- `pl-e2e-merge-reports` merges parallel shard results into terminal and JUnit reports.
|
|
37
|
+
|
|
38
|
+
## Requirements and installation
|
|
39
|
+
|
|
40
|
+
- Python 3.13 or newer
|
|
41
|
+
- Git 2.30 or newer
|
|
42
|
+
|
|
43
|
+
The harness depends on pytest, Pyright, Ruff, Chevron, JSON Schema, lxml, NumPy,
|
|
44
|
+
and PrairieLearn. PrairieLearn's Python package is currently consumed from its Git
|
|
45
|
+
repository, so the course project must tell uv where to resolve that dependency:
|
|
46
|
+
|
|
47
|
+
```toml
|
|
48
|
+
[dependency-groups]
|
|
49
|
+
dev = ["pl-e2e-test"]
|
|
50
|
+
|
|
51
|
+
[tool.uv.sources.prairielearn]
|
|
52
|
+
git = "https://github.com/PrairieLearn/PrairieLearn.git"
|
|
53
|
+
subdirectory = "apps/prairielearn/python"
|
|
54
|
+
```
|
|
55
|
+
|
|
56
|
+
Then lock and install the environment:
|
|
57
|
+
|
|
58
|
+
```sh
|
|
59
|
+
uv lock
|
|
60
|
+
uv sync
|
|
61
|
+
```
|
|
62
|
+
|
|
63
|
+
For local development of this package, use a path source in the consuming project:
|
|
64
|
+
|
|
65
|
+
```toml
|
|
66
|
+
[tool.uv.sources.pl-e2e-test]
|
|
67
|
+
path = "../pl-e2e-test"
|
|
68
|
+
editable = true
|
|
69
|
+
```
|
|
70
|
+
|
|
71
|
+
## Running lifecycle tests
|
|
72
|
+
|
|
73
|
+
Pass a question, a directory of questions, or an `info.json` file. Repeat
|
|
74
|
+
`--question-path` to select multiple roots:
|
|
75
|
+
|
|
76
|
+
```sh
|
|
77
|
+
uv run pl-e2e-test \
|
|
78
|
+
--question-path questions \
|
|
79
|
+
--seed-count 3
|
|
80
|
+
```
|
|
81
|
+
|
|
82
|
+
The installed suite exposes separate `generate`, `prepare`, `render`, and `grade`
|
|
83
|
+
pytest markers. Any remaining arguments are passed to pytest:
|
|
84
|
+
|
|
85
|
+
```sh
|
|
86
|
+
uv run pl-e2e-test \
|
|
87
|
+
--question-path questions/chapter/example \
|
|
88
|
+
--fuzz-seeds -m render -x -vv
|
|
89
|
+
```
|
|
90
|
+
|
|
91
|
+
`--fuzz-seeds` creates and reports a random 64-bit master seed. Reproduce the same
|
|
92
|
+
variant set with `--fuzz-seed N`. Without either option, deterministic seeds start at
|
|
93
|
+
zero. Question metadata with `singleVariant: true` always uses seed zero.
|
|
94
|
+
|
|
95
|
+
### CI sharding
|
|
96
|
+
|
|
97
|
+
Use `--shard INDEX/COUNT` to divide the selected question variants between parallel
|
|
98
|
+
jobs. The index is one-based. All selected lifecycle checks for one question and seed
|
|
99
|
+
stay together, and variants are assigned round-robin so shard sizes differ by at most
|
|
100
|
+
one. Pytest filters such as `-m` and `-k` are applied before sharding.
|
|
101
|
+
|
|
102
|
+
Add `--shard-report PATH` to write a mergeable result blob. Every shard must use the
|
|
103
|
+
same checkout and arguments. Sharded fuzz runs must also use one explicit shared
|
|
104
|
+
`--fuzz-seed`; automatic random fuzz seeds are rejected. After downloading all blobs,
|
|
105
|
+
merge them into a combined terminal summary and JUnit report:
|
|
106
|
+
|
|
107
|
+
```sh
|
|
108
|
+
uv run pl-e2e-merge-reports e2e-reports \
|
|
109
|
+
--junitxml pl-e2e-results.xml
|
|
110
|
+
```
|
|
111
|
+
|
|
112
|
+
The merger rejects missing or duplicate shards, mismatched test collections, mixed
|
|
113
|
+
fuzz seeds, corrupt reports, and tests left unexecuted by interruption or `-x`.
|
|
114
|
+
It exits unsuccessfully when the aggregate suite failed or was incomplete.
|
|
115
|
+
|
|
116
|
+
For example, a course can use this GitHub Actions matrix:
|
|
117
|
+
|
|
118
|
+
```yaml
|
|
119
|
+
jobs:
|
|
120
|
+
e2e:
|
|
121
|
+
runs-on: ubuntu-latest
|
|
122
|
+
strategy:
|
|
123
|
+
fail-fast: false
|
|
124
|
+
matrix:
|
|
125
|
+
shard: [1, 2, 3, 4]
|
|
126
|
+
steps:
|
|
127
|
+
- uses: actions/checkout@v4
|
|
128
|
+
- uses: astral-sh/setup-uv@v9
|
|
129
|
+
with:
|
|
130
|
+
enable-cache: true
|
|
131
|
+
- run: uv python install && uv sync --locked
|
|
132
|
+
- name: Run shard
|
|
133
|
+
run: |
|
|
134
|
+
uv run pl-e2e-test \
|
|
135
|
+
--question-path questions \
|
|
136
|
+
--seed-count 10 \
|
|
137
|
+
--fuzz-seed "${{ github.run_id }}" \
|
|
138
|
+
--shard "${{ matrix.shard }}/4" \
|
|
139
|
+
--shard-report "e2e-reports/shard-${{ matrix.shard }}.pl-e2e-report.json"
|
|
140
|
+
- name: Upload shard report
|
|
141
|
+
if: ${{ !cancelled() }}
|
|
142
|
+
uses: actions/upload-artifact@v4
|
|
143
|
+
with:
|
|
144
|
+
name: e2e-report-${{ matrix.shard }}
|
|
145
|
+
path: e2e-reports/
|
|
146
|
+
if-no-files-found: error
|
|
147
|
+
retention-days: 1
|
|
148
|
+
|
|
149
|
+
e2e-report:
|
|
150
|
+
if: ${{ !cancelled() }}
|
|
151
|
+
needs: e2e
|
|
152
|
+
runs-on: ubuntu-latest
|
|
153
|
+
steps:
|
|
154
|
+
- uses: actions/checkout@v4
|
|
155
|
+
- uses: astral-sh/setup-uv@v9
|
|
156
|
+
- run: uv python install && uv sync --locked
|
|
157
|
+
- uses: actions/download-artifact@v4
|
|
158
|
+
with:
|
|
159
|
+
pattern: e2e-report-*
|
|
160
|
+
path: e2e-reports
|
|
161
|
+
merge-multiple: true
|
|
162
|
+
- name: Merge reports
|
|
163
|
+
run: |
|
|
164
|
+
uv run pl-e2e-merge-reports e2e-reports \
|
|
165
|
+
--junitxml pl-e2e-results.xml
|
|
166
|
+
- name: Upload merged JUnit report
|
|
167
|
+
if: ${{ always() }}
|
|
168
|
+
uses: actions/upload-artifact@v4
|
|
169
|
+
with:
|
|
170
|
+
name: pl-e2e-results
|
|
171
|
+
path: pl-e2e-results.xml
|
|
172
|
+
if-no-files-found: warn
|
|
173
|
+
```
|
|
174
|
+
|
|
175
|
+
The course must provide `.prairielearn/schemas/infoQuestion.json`. The harness checks
|
|
176
|
+
question metadata against that schema and uses the exact PrairieLearn revision
|
|
177
|
+
installed in the uv environment. It stores a bare, partial PrairieLearn repository in
|
|
178
|
+
the platform-standard user cache directory. The first use of a new Git source or
|
|
179
|
+
revision requires network access; later runs reuse that exact revision offline. The
|
|
180
|
+
uv lockfile, rather than the cache, decides which revision is tested.
|
|
181
|
+
|
|
182
|
+
To use an existing PrairieLearn worktree or bare repository instead, pass
|
|
183
|
+
`--prairielearn-path /path/to/PrairieLearn`. The repository must already contain the
|
|
184
|
+
installed revision. An explicit path is authoritative: the harness does not fetch into
|
|
185
|
+
it or fall back to the managed cache. In either mode, only the pinned revision's Python
|
|
186
|
+
element controllers are archived into the test run's temporary directory. Standard
|
|
187
|
+
cache environment settings such as `XDG_CACHE_HOME` can relocate the managed cache;
|
|
188
|
+
CI systems can persist that directory between runs if desired.
|
|
189
|
+
|
|
190
|
+
### Question-local configuration
|
|
191
|
+
|
|
192
|
+
Store saved concrete seeds and narrow phase skips in `.question-e2e.json` beside the
|
|
193
|
+
question's `info.json`:
|
|
194
|
+
|
|
195
|
+
```json
|
|
196
|
+
{
|
|
197
|
+
"regression_seeds": [1843927501],
|
|
198
|
+
"skip_methods": {
|
|
199
|
+
"grade": "Temporarily blocked by an upstream issue"
|
|
200
|
+
}
|
|
201
|
+
}
|
|
202
|
+
```
|
|
203
|
+
|
|
204
|
+
Saved regression seeds run before newly generated seeds. Skips require a nonempty
|
|
205
|
+
reason and may name only `generate`, `prepare`, `render`, or `grade`. Manual-grading
|
|
206
|
+
questions automatically skip only the grade check.
|
|
207
|
+
|
|
208
|
+
Record the concrete seed shown in a failing pytest case with:
|
|
209
|
+
|
|
210
|
+
```sh
|
|
211
|
+
uv run pl-e2e-add-regression-seed \
|
|
212
|
+
questions/chapter/example 1843927501
|
|
213
|
+
```
|
|
214
|
+
|
|
215
|
+
The command atomically creates or updates `.question-e2e.json`, preserves other
|
|
216
|
+
settings, and sorts and deduplicates the seed list.
|
|
217
|
+
|
|
218
|
+
## Diff-based fuzzing
|
|
219
|
+
|
|
220
|
+
The diff runner tests changed question directories. A non-ignored changed file outside
|
|
221
|
+
`questions/` selects every question:
|
|
222
|
+
|
|
223
|
+
```sh
|
|
224
|
+
uv run pl-e2e-test-diff \
|
|
225
|
+
--base origin/main \
|
|
226
|
+
--head HEAD \
|
|
227
|
+
--seed-count 10
|
|
228
|
+
```
|
|
229
|
+
|
|
230
|
+
Optional repository-relative ignore rules live in `.question-e2e.diffignore` and use
|
|
231
|
+
Git's native ignore syntax. Missing default configuration means that no paths are
|
|
232
|
+
ignored. Pass `--e2ediffignore PATH` to require and use another file. Additional
|
|
233
|
+
arguments are forwarded to pytest.
|
|
234
|
+
|
|
235
|
+
In pull-request CI, fetch the base commit before running the command and pass the
|
|
236
|
+
appropriate base and head SHAs. Set `--fuzz-seed` to replay a reported run.
|
|
237
|
+
`pl-e2e-test-diff` accepts the same `--shard` and `--shard-report` options. If a diff
|
|
238
|
+
selects no questions, it writes a successful empty shard report so the merge job can
|
|
239
|
+
still verify that every matrix job completed.
|
|
240
|
+
|
|
241
|
+
## Python API
|
|
242
|
+
|
|
243
|
+
The original API remains available:
|
|
244
|
+
|
|
245
|
+
```python
|
|
246
|
+
from prairielearn_e2e import PrairieLearnBackend, QuestionCase, QuestionHarness
|
|
247
|
+
```
|
|
248
|
+
|
|
249
|
+
`QuestionHarness.run_generate`, `run_prepare`, `run_render`, and `run_grade` execute
|
|
250
|
+
increasingly complete portions of the lifecycle. `run_variant` remains an alias for
|
|
251
|
+
the complete grade path.
|
|
252
|
+
|
|
253
|
+
## Development
|
|
254
|
+
|
|
255
|
+
This project uses uv 0.9 or newer:
|
|
256
|
+
|
|
257
|
+
```sh
|
|
258
|
+
uv sync
|
|
259
|
+
make test
|
|
260
|
+
make format
|
|
261
|
+
make smoke-dist
|
|
262
|
+
```
|
|
263
|
+
|
|
264
|
+
`make test` runs pytest, Pyright, Ruff linting, and Ruff's formatting check.
|
|
265
|
+
`make smoke-dist` builds both distribution formats, installs each without dependencies
|
|
266
|
+
into an isolated target, and imports it using the locked development dependencies.
|
|
267
|
+
|
|
268
|
+
## Publishing
|
|
269
|
+
|
|
270
|
+
There are three supported publishing paths. Choose one for a release; do not run the
|
|
271
|
+
local and GitHub paths for the same version because PyPI versions are immutable.
|
|
272
|
+
|
|
273
|
+
### GitHub-hosted trusted publishing
|
|
274
|
+
|
|
275
|
+
Configure the PyPI project to trust the `publish.yml` workflow in the GitHub `pypi`
|
|
276
|
+
environment. From a clean branch, run:
|
|
277
|
+
|
|
278
|
+
```sh
|
|
279
|
+
make publish-version VERSION=0.2.0
|
|
280
|
+
```
|
|
281
|
+
|
|
282
|
+
This runs the full checks, updates `pyproject.toml` and `uv.lock`, builds the
|
|
283
|
+
distributions, creates a release commit and annotated `v0.2.0` tag, and atomically
|
|
284
|
+
pushes the branch and tag. The tag starts the workflow on `ubuntu-latest`; the workflow
|
|
285
|
+
rechecks the tag/version match, tests, builds, generates attestations, and publishes
|
|
286
|
+
with PyPI trusted-publishing OIDC credentials. Use `REMOTE=name` for a different Git
|
|
287
|
+
remote.
|
|
288
|
+
|
|
289
|
+
If an unpushed commit already contains the requested version, use the guarded bypass:
|
|
290
|
+
|
|
291
|
+
```sh
|
|
292
|
+
make publish-version VERSION=0.2.0 ALLOW_PREBUMPED_VERSION=1
|
|
293
|
+
```
|
|
294
|
+
|
|
295
|
+
### Direct local publishing
|
|
296
|
+
|
|
297
|
+
To publish the version already present in `pyproject.toml` directly from this machine,
|
|
298
|
+
configure a uv-supported PyPI credential such as `UV_PUBLISH_TOKEN`, then run:
|
|
299
|
+
|
|
300
|
+
```sh
|
|
301
|
+
make publish-local
|
|
302
|
+
```
|
|
303
|
+
|
|
304
|
+
This runs all checks, builds and smoke-tests the current wheel and source archive, and
|
|
305
|
+
passes only those two version-matched files to `uv publish`. It does not create a Git
|
|
306
|
+
commit or tag.
|
|
307
|
+
|
|
308
|
+
### Local self-hosted GitHub runner
|
|
309
|
+
|
|
310
|
+
The same trusted-publishing workflow can run on a repository-scoped Linux ARM64 runner
|
|
311
|
+
inside Docker. The container has no host mounts and does not receive the Docker socket.
|
|
312
|
+
Authenticate `gh` with repository administration permission, then run:
|
|
313
|
+
|
|
314
|
+
```sh
|
|
315
|
+
make runner-start
|
|
316
|
+
make publish-self-hosted RELEASE_TAG=v0.2.0
|
|
317
|
+
```
|
|
318
|
+
|
|
319
|
+
The tag must already exist. This dispatches `publish.yml` from `main` with the
|
|
320
|
+
`self-hosted` runner choice. `make runner-ensure` starts Docker Desktop on macOS when
|
|
321
|
+
needed and repairs the runner before dispatch. Other lifecycle commands are
|
|
322
|
+
`runner-status`, `runner-stop`, `runner-remove`, `runner-logs`, and `clean-runner`.
|
|
@@ -0,0 +1,299 @@
|
|
|
1
|
+
# pl-e2e-test
|
|
2
|
+
|
|
3
|
+
`pl-e2e-test` fuzzes PrairieLearn v3 questions through their real Python
|
|
4
|
+
generate, prepare, render, parse, grade, test, and file phases. It was extracted
|
|
5
|
+
from the `pl-oer-lv102` course so that courses can install one uv-managed tool
|
|
6
|
+
instead of copying the harness and its support scripts.
|
|
7
|
+
|
|
8
|
+
The package installs the reusable `prairielearn_e2e` Python API and these commands:
|
|
9
|
+
|
|
10
|
+
- `pl-e2e-test` runs the installed lifecycle pytest suite.
|
|
11
|
+
- `pl-e2e-test-diff` fuzzes questions selected from a Git diff.
|
|
12
|
+
- `pl-e2e-add-regression-seed` records a failing concrete variant seed.
|
|
13
|
+
- `pl-e2e-merge-reports` merges parallel shard results into terminal and JUnit reports.
|
|
14
|
+
|
|
15
|
+
## Requirements and installation
|
|
16
|
+
|
|
17
|
+
- Python 3.13 or newer
|
|
18
|
+
- Git 2.30 or newer
|
|
19
|
+
|
|
20
|
+
The harness depends on pytest, Pyright, Ruff, Chevron, JSON Schema, lxml, NumPy,
|
|
21
|
+
and PrairieLearn. PrairieLearn's Python package is currently consumed from its Git
|
|
22
|
+
repository, so the course project must tell uv where to resolve that dependency:
|
|
23
|
+
|
|
24
|
+
```toml
|
|
25
|
+
[dependency-groups]
|
|
26
|
+
dev = ["pl-e2e-test"]
|
|
27
|
+
|
|
28
|
+
[tool.uv.sources.prairielearn]
|
|
29
|
+
git = "https://github.com/PrairieLearn/PrairieLearn.git"
|
|
30
|
+
subdirectory = "apps/prairielearn/python"
|
|
31
|
+
```
|
|
32
|
+
|
|
33
|
+
Then lock and install the environment:
|
|
34
|
+
|
|
35
|
+
```sh
|
|
36
|
+
uv lock
|
|
37
|
+
uv sync
|
|
38
|
+
```
|
|
39
|
+
|
|
40
|
+
For local development of this package, use a path source in the consuming project:
|
|
41
|
+
|
|
42
|
+
```toml
|
|
43
|
+
[tool.uv.sources.pl-e2e-test]
|
|
44
|
+
path = "../pl-e2e-test"
|
|
45
|
+
editable = true
|
|
46
|
+
```
|
|
47
|
+
|
|
48
|
+
## Running lifecycle tests
|
|
49
|
+
|
|
50
|
+
Pass a question, a directory of questions, or an `info.json` file. Repeat
|
|
51
|
+
`--question-path` to select multiple roots:
|
|
52
|
+
|
|
53
|
+
```sh
|
|
54
|
+
uv run pl-e2e-test \
|
|
55
|
+
--question-path questions \
|
|
56
|
+
--seed-count 3
|
|
57
|
+
```
|
|
58
|
+
|
|
59
|
+
The installed suite exposes separate `generate`, `prepare`, `render`, and `grade`
|
|
60
|
+
pytest markers. Any remaining arguments are passed to pytest:
|
|
61
|
+
|
|
62
|
+
```sh
|
|
63
|
+
uv run pl-e2e-test \
|
|
64
|
+
--question-path questions/chapter/example \
|
|
65
|
+
--fuzz-seeds -m render -x -vv
|
|
66
|
+
```
|
|
67
|
+
|
|
68
|
+
`--fuzz-seeds` creates and reports a random 64-bit master seed. Reproduce the same
|
|
69
|
+
variant set with `--fuzz-seed N`. Without either option, deterministic seeds start at
|
|
70
|
+
zero. Question metadata with `singleVariant: true` always uses seed zero.
|
|
71
|
+
|
|
72
|
+
### CI sharding
|
|
73
|
+
|
|
74
|
+
Use `--shard INDEX/COUNT` to divide the selected question variants between parallel
|
|
75
|
+
jobs. The index is one-based. All selected lifecycle checks for one question and seed
|
|
76
|
+
stay together, and variants are assigned round-robin so shard sizes differ by at most
|
|
77
|
+
one. Pytest filters such as `-m` and `-k` are applied before sharding.
|
|
78
|
+
|
|
79
|
+
Add `--shard-report PATH` to write a mergeable result blob. Every shard must use the
|
|
80
|
+
same checkout and arguments. Sharded fuzz runs must also use one explicit shared
|
|
81
|
+
`--fuzz-seed`; automatic random fuzz seeds are rejected. After downloading all blobs,
|
|
82
|
+
merge them into a combined terminal summary and JUnit report:
|
|
83
|
+
|
|
84
|
+
```sh
|
|
85
|
+
uv run pl-e2e-merge-reports e2e-reports \
|
|
86
|
+
--junitxml pl-e2e-results.xml
|
|
87
|
+
```
|
|
88
|
+
|
|
89
|
+
The merger rejects missing or duplicate shards, mismatched test collections, mixed
|
|
90
|
+
fuzz seeds, corrupt reports, and tests left unexecuted by interruption or `-x`.
|
|
91
|
+
It exits unsuccessfully when the aggregate suite failed or was incomplete.
|
|
92
|
+
|
|
93
|
+
For example, a course can use this GitHub Actions matrix:
|
|
94
|
+
|
|
95
|
+
```yaml
|
|
96
|
+
jobs:
|
|
97
|
+
e2e:
|
|
98
|
+
runs-on: ubuntu-latest
|
|
99
|
+
strategy:
|
|
100
|
+
fail-fast: false
|
|
101
|
+
matrix:
|
|
102
|
+
shard: [1, 2, 3, 4]
|
|
103
|
+
steps:
|
|
104
|
+
- uses: actions/checkout@v4
|
|
105
|
+
- uses: astral-sh/setup-uv@v9
|
|
106
|
+
with:
|
|
107
|
+
enable-cache: true
|
|
108
|
+
- run: uv python install && uv sync --locked
|
|
109
|
+
- name: Run shard
|
|
110
|
+
run: |
|
|
111
|
+
uv run pl-e2e-test \
|
|
112
|
+
--question-path questions \
|
|
113
|
+
--seed-count 10 \
|
|
114
|
+
--fuzz-seed "${{ github.run_id }}" \
|
|
115
|
+
--shard "${{ matrix.shard }}/4" \
|
|
116
|
+
--shard-report "e2e-reports/shard-${{ matrix.shard }}.pl-e2e-report.json"
|
|
117
|
+
- name: Upload shard report
|
|
118
|
+
if: ${{ !cancelled() }}
|
|
119
|
+
uses: actions/upload-artifact@v4
|
|
120
|
+
with:
|
|
121
|
+
name: e2e-report-${{ matrix.shard }}
|
|
122
|
+
path: e2e-reports/
|
|
123
|
+
if-no-files-found: error
|
|
124
|
+
retention-days: 1
|
|
125
|
+
|
|
126
|
+
e2e-report:
|
|
127
|
+
if: ${{ !cancelled() }}
|
|
128
|
+
needs: e2e
|
|
129
|
+
runs-on: ubuntu-latest
|
|
130
|
+
steps:
|
|
131
|
+
- uses: actions/checkout@v4
|
|
132
|
+
- uses: astral-sh/setup-uv@v9
|
|
133
|
+
- run: uv python install && uv sync --locked
|
|
134
|
+
- uses: actions/download-artifact@v4
|
|
135
|
+
with:
|
|
136
|
+
pattern: e2e-report-*
|
|
137
|
+
path: e2e-reports
|
|
138
|
+
merge-multiple: true
|
|
139
|
+
- name: Merge reports
|
|
140
|
+
run: |
|
|
141
|
+
uv run pl-e2e-merge-reports e2e-reports \
|
|
142
|
+
--junitxml pl-e2e-results.xml
|
|
143
|
+
- name: Upload merged JUnit report
|
|
144
|
+
if: ${{ always() }}
|
|
145
|
+
uses: actions/upload-artifact@v4
|
|
146
|
+
with:
|
|
147
|
+
name: pl-e2e-results
|
|
148
|
+
path: pl-e2e-results.xml
|
|
149
|
+
if-no-files-found: warn
|
|
150
|
+
```
|
|
151
|
+
|
|
152
|
+
The course must provide `.prairielearn/schemas/infoQuestion.json`. The harness checks
|
|
153
|
+
question metadata against that schema and uses the exact PrairieLearn revision
|
|
154
|
+
installed in the uv environment. It stores a bare, partial PrairieLearn repository in
|
|
155
|
+
the platform-standard user cache directory. The first use of a new Git source or
|
|
156
|
+
revision requires network access; later runs reuse that exact revision offline. The
|
|
157
|
+
uv lockfile, rather than the cache, decides which revision is tested.
|
|
158
|
+
|
|
159
|
+
To use an existing PrairieLearn worktree or bare repository instead, pass
|
|
160
|
+
`--prairielearn-path /path/to/PrairieLearn`. The repository must already contain the
|
|
161
|
+
installed revision. An explicit path is authoritative: the harness does not fetch into
|
|
162
|
+
it or fall back to the managed cache. In either mode, only the pinned revision's Python
|
|
163
|
+
element controllers are archived into the test run's temporary directory. Standard
|
|
164
|
+
cache environment settings such as `XDG_CACHE_HOME` can relocate the managed cache;
|
|
165
|
+
CI systems can persist that directory between runs if desired.
|
|
166
|
+
|
|
167
|
+
### Question-local configuration
|
|
168
|
+
|
|
169
|
+
Store saved concrete seeds and narrow phase skips in `.question-e2e.json` beside the
|
|
170
|
+
question's `info.json`:
|
|
171
|
+
|
|
172
|
+
```json
|
|
173
|
+
{
|
|
174
|
+
"regression_seeds": [1843927501],
|
|
175
|
+
"skip_methods": {
|
|
176
|
+
"grade": "Temporarily blocked by an upstream issue"
|
|
177
|
+
}
|
|
178
|
+
}
|
|
179
|
+
```
|
|
180
|
+
|
|
181
|
+
Saved regression seeds run before newly generated seeds. Skips require a nonempty
|
|
182
|
+
reason and may name only `generate`, `prepare`, `render`, or `grade`. Manual-grading
|
|
183
|
+
questions automatically skip only the grade check.
|
|
184
|
+
|
|
185
|
+
Record the concrete seed shown in a failing pytest case with:
|
|
186
|
+
|
|
187
|
+
```sh
|
|
188
|
+
uv run pl-e2e-add-regression-seed \
|
|
189
|
+
questions/chapter/example 1843927501
|
|
190
|
+
```
|
|
191
|
+
|
|
192
|
+
The command atomically creates or updates `.question-e2e.json`, preserves other
|
|
193
|
+
settings, and sorts and deduplicates the seed list.
|
|
194
|
+
|
|
195
|
+
## Diff-based fuzzing
|
|
196
|
+
|
|
197
|
+
The diff runner tests changed question directories. A non-ignored changed file outside
|
|
198
|
+
`questions/` selects every question:
|
|
199
|
+
|
|
200
|
+
```sh
|
|
201
|
+
uv run pl-e2e-test-diff \
|
|
202
|
+
--base origin/main \
|
|
203
|
+
--head HEAD \
|
|
204
|
+
--seed-count 10
|
|
205
|
+
```
|
|
206
|
+
|
|
207
|
+
Optional repository-relative ignore rules live in `.question-e2e.diffignore` and use
|
|
208
|
+
Git's native ignore syntax. Missing default configuration means that no paths are
|
|
209
|
+
ignored. Pass `--e2ediffignore PATH` to require and use another file. Additional
|
|
210
|
+
arguments are forwarded to pytest.
|
|
211
|
+
|
|
212
|
+
In pull-request CI, fetch the base commit before running the command and pass the
|
|
213
|
+
appropriate base and head SHAs. Set `--fuzz-seed` to replay a reported run.
|
|
214
|
+
`pl-e2e-test-diff` accepts the same `--shard` and `--shard-report` options. If a diff
|
|
215
|
+
selects no questions, it writes a successful empty shard report so the merge job can
|
|
216
|
+
still verify that every matrix job completed.
|
|
217
|
+
|
|
218
|
+
## Python API
|
|
219
|
+
|
|
220
|
+
The original API remains available:
|
|
221
|
+
|
|
222
|
+
```python
|
|
223
|
+
from prairielearn_e2e import PrairieLearnBackend, QuestionCase, QuestionHarness
|
|
224
|
+
```
|
|
225
|
+
|
|
226
|
+
`QuestionHarness.run_generate`, `run_prepare`, `run_render`, and `run_grade` execute
|
|
227
|
+
increasingly complete portions of the lifecycle. `run_variant` remains an alias for
|
|
228
|
+
the complete grade path.
|
|
229
|
+
|
|
230
|
+
## Development
|
|
231
|
+
|
|
232
|
+
This project uses uv 0.9 or newer:
|
|
233
|
+
|
|
234
|
+
```sh
|
|
235
|
+
uv sync
|
|
236
|
+
make test
|
|
237
|
+
make format
|
|
238
|
+
make smoke-dist
|
|
239
|
+
```
|
|
240
|
+
|
|
241
|
+
`make test` runs pytest, Pyright, Ruff linting, and Ruff's formatting check.
|
|
242
|
+
`make smoke-dist` builds both distribution formats, installs each without dependencies
|
|
243
|
+
into an isolated target, and imports it using the locked development dependencies.
|
|
244
|
+
|
|
245
|
+
## Publishing
|
|
246
|
+
|
|
247
|
+
There are three supported publishing paths. Choose one for a release; do not run the
|
|
248
|
+
local and GitHub paths for the same version because PyPI versions are immutable.
|
|
249
|
+
|
|
250
|
+
### GitHub-hosted trusted publishing
|
|
251
|
+
|
|
252
|
+
Configure the PyPI project to trust the `publish.yml` workflow in the GitHub `pypi`
|
|
253
|
+
environment. From a clean branch, run:
|
|
254
|
+
|
|
255
|
+
```sh
|
|
256
|
+
make publish-version VERSION=0.2.0
|
|
257
|
+
```
|
|
258
|
+
|
|
259
|
+
This runs the full checks, updates `pyproject.toml` and `uv.lock`, builds the
|
|
260
|
+
distributions, creates a release commit and annotated `v0.2.0` tag, and atomically
|
|
261
|
+
pushes the branch and tag. The tag starts the workflow on `ubuntu-latest`; the workflow
|
|
262
|
+
rechecks the tag/version match, tests, builds, generates attestations, and publishes
|
|
263
|
+
with PyPI trusted-publishing OIDC credentials. Use `REMOTE=name` for a different Git
|
|
264
|
+
remote.
|
|
265
|
+
|
|
266
|
+
If an unpushed commit already contains the requested version, use the guarded bypass:
|
|
267
|
+
|
|
268
|
+
```sh
|
|
269
|
+
make publish-version VERSION=0.2.0 ALLOW_PREBUMPED_VERSION=1
|
|
270
|
+
```
|
|
271
|
+
|
|
272
|
+
### Direct local publishing
|
|
273
|
+
|
|
274
|
+
To publish the version already present in `pyproject.toml` directly from this machine,
|
|
275
|
+
configure a uv-supported PyPI credential such as `UV_PUBLISH_TOKEN`, then run:
|
|
276
|
+
|
|
277
|
+
```sh
|
|
278
|
+
make publish-local
|
|
279
|
+
```
|
|
280
|
+
|
|
281
|
+
This runs all checks, builds and smoke-tests the current wheel and source archive, and
|
|
282
|
+
passes only those two version-matched files to `uv publish`. It does not create a Git
|
|
283
|
+
commit or tag.
|
|
284
|
+
|
|
285
|
+
### Local self-hosted GitHub runner
|
|
286
|
+
|
|
287
|
+
The same trusted-publishing workflow can run on a repository-scoped Linux ARM64 runner
|
|
288
|
+
inside Docker. The container has no host mounts and does not receive the Docker socket.
|
|
289
|
+
Authenticate `gh` with repository administration permission, then run:
|
|
290
|
+
|
|
291
|
+
```sh
|
|
292
|
+
make runner-start
|
|
293
|
+
make publish-self-hosted RELEASE_TAG=v0.2.0
|
|
294
|
+
```
|
|
295
|
+
|
|
296
|
+
The tag must already exist. This dispatches `publish.yml` from `main` with the
|
|
297
|
+
`self-hosted` runner choice. `make runner-ensure` starts Docker Desktop on macOS when
|
|
298
|
+
needed and repairs the runner before dispatch. Other lifecycle commands are
|
|
299
|
+
`runner-status`, `runner-stop`, `runner-remove`, `runner-logs`, and `clean-runner`.
|
|
@@ -0,0 +1,59 @@
|
|
|
1
|
+
[build-system]
|
|
2
|
+
requires = ["setuptools>=69"]
|
|
3
|
+
build-backend = "setuptools.build_meta"
|
|
4
|
+
|
|
5
|
+
[project]
|
|
6
|
+
name = "pl-e2e-test"
|
|
7
|
+
version = "0.1.0"
|
|
8
|
+
description = "Fuzz PrairieLearn question lifecycles against the real Python element runtime"
|
|
9
|
+
readme = "README.md"
|
|
10
|
+
requires-python = ">=3.13"
|
|
11
|
+
license = "MIT"
|
|
12
|
+
dependencies = [
|
|
13
|
+
"chevron==0.14.0",
|
|
14
|
+
"filelock>=3.20",
|
|
15
|
+
"jsonschema>=4.26",
|
|
16
|
+
"lxml>=6.1",
|
|
17
|
+
"numpy>=2.4",
|
|
18
|
+
"platformdirs>=4.5",
|
|
19
|
+
"prairielearn",
|
|
20
|
+
"pyright>=1.1.414",
|
|
21
|
+
"pytest>=9.1",
|
|
22
|
+
"ruff>=0.16",
|
|
23
|
+
]
|
|
24
|
+
|
|
25
|
+
[dependency-groups]
|
|
26
|
+
dev = ["pyfakefs>=6.2"]
|
|
27
|
+
|
|
28
|
+
[project.urls]
|
|
29
|
+
Homepage = "https://github.com/PrairieLearn/pl-e2e-test"
|
|
30
|
+
Issues = "https://github.com/PrairieLearn/pl-e2e-test/issues"
|
|
31
|
+
Repository = "https://github.com/PrairieLearn/pl-e2e-test"
|
|
32
|
+
|
|
33
|
+
[project.scripts]
|
|
34
|
+
pl-e2e-test = "prairielearn_e2e.runner:main"
|
|
35
|
+
pl-e2e-test-diff = "prairielearn_e2e.fuzz:main"
|
|
36
|
+
pl-e2e-add-regression-seed = "prairielearn_e2e.regression:main"
|
|
37
|
+
pl-e2e-merge-reports = "prairielearn_e2e.reporting:main"
|
|
38
|
+
|
|
39
|
+
[tool.setuptools.packages.find]
|
|
40
|
+
where = ["src"]
|
|
41
|
+
|
|
42
|
+
[tool.setuptools.package-data]
|
|
43
|
+
prairielearn_e2e = ["config.json"]
|
|
44
|
+
|
|
45
|
+
[tool.pytest.ini_options]
|
|
46
|
+
testpaths = ["tests"]
|
|
47
|
+
|
|
48
|
+
[tool.pyright]
|
|
49
|
+
include = ["src/prairielearn_e2e", "tests"]
|
|
50
|
+
|
|
51
|
+
[tool.uv]
|
|
52
|
+
required-version = ">=0.9.0"
|
|
53
|
+
|
|
54
|
+
[tool.uv.sources.prairielearn]
|
|
55
|
+
git = "https://github.com/PrairieLearn/PrairieLearn.git"
|
|
56
|
+
subdirectory = "apps/prairielearn/python"
|
|
57
|
+
|
|
58
|
+
[tool.ruff]
|
|
59
|
+
target-version = "py313"
|