pl-e2e-test 0.1.8__tar.gz → 0.2.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {pl_e2e_test-0.1.8/src/pl_e2e_test.egg-info → pl_e2e_test-0.2.0}/PKG-INFO +64 -44
- {pl_e2e_test-0.1.8 → pl_e2e_test-0.2.0}/README.md +63 -43
- {pl_e2e_test-0.1.8 → pl_e2e_test-0.2.0}/pyproject.toml +1 -1
- {pl_e2e_test-0.1.8 → pl_e2e_test-0.2.0/src/pl_e2e_test.egg-info}/PKG-INFO +64 -44
- {pl_e2e_test-0.1.8 → pl_e2e_test-0.2.0}/src/pl_e2e_test.egg-info/SOURCES.txt +4 -0
- pl_e2e_test-0.2.0/src/prairielearn_e2e/__init__.py +50 -0
- {pl_e2e_test-0.1.8 → pl_e2e_test-0.2.0}/src/prairielearn_e2e/_git.py +47 -3
- {pl_e2e_test-0.1.8 → pl_e2e_test-0.2.0}/src/prairielearn_e2e/cli.py +37 -6
- {pl_e2e_test-0.1.8 → pl_e2e_test-0.2.0}/src/prairielearn_e2e/config.py +10 -0
- {pl_e2e_test-0.1.8 → pl_e2e_test-0.2.0}/src/prairielearn_e2e/fuzz.py +236 -42
- pl_e2e_test-0.2.0/src/prairielearn_e2e/instances.py +161 -0
- {pl_e2e_test-0.1.8 → pl_e2e_test-0.2.0}/src/prairielearn_e2e/parallel.py +96 -9
- {pl_e2e_test-0.1.8 → pl_e2e_test-0.2.0}/src/prairielearn_e2e/pytest_plugin.py +90 -1
- {pl_e2e_test-0.1.8 → pl_e2e_test-0.2.0}/src/prairielearn_e2e/reporting.py +48 -3
- pl_e2e_test-0.2.0/src/prairielearn_e2e/repository.py +602 -0
- pl_e2e_test-0.2.0/src/prairielearn_e2e/runner.py +48 -0
- pl_e2e_test-0.2.0/src/prairielearn_e2e/runtime.py +387 -0
- {pl_e2e_test-0.1.8 → pl_e2e_test-0.2.0}/tests/stub_git.py +33 -7
- {pl_e2e_test-0.1.8 → pl_e2e_test-0.2.0}/tests/test_cli.py +14 -4
- {pl_e2e_test-0.1.8 → pl_e2e_test-0.2.0}/tests/test_config.py +4 -0
- {pl_e2e_test-0.1.8 → pl_e2e_test-0.2.0}/tests/test_fuzz.py +211 -1
- {pl_e2e_test-0.1.8 → pl_e2e_test-0.2.0}/tests/test_git.py +25 -1
- {pl_e2e_test-0.1.8 → pl_e2e_test-0.2.0}/tests/test_harness.py +34 -1
- pl_e2e_test-0.2.0/tests/test_instances.py +222 -0
- {pl_e2e_test-0.1.8 → pl_e2e_test-0.2.0}/tests/test_parallel.py +186 -4
- pl_e2e_test-0.2.0/tests/test_publish_version.py +479 -0
- {pl_e2e_test-0.1.8 → pl_e2e_test-0.2.0}/tests/test_pytest_plugin.py +125 -0
- {pl_e2e_test-0.1.8 → pl_e2e_test-0.2.0}/tests/test_reporting.py +35 -2
- {pl_e2e_test-0.1.8 → pl_e2e_test-0.2.0}/tests/test_repository.py +240 -1
- pl_e2e_test-0.2.0/tests/test_runtime.py +473 -0
- pl_e2e_test-0.1.8/src/prairielearn_e2e/__init__.py +0 -23
- pl_e2e_test-0.1.8/src/prairielearn_e2e/repository.py +0 -298
- pl_e2e_test-0.1.8/src/prairielearn_e2e/runner.py +0 -28
- pl_e2e_test-0.1.8/tests/test_publish_version.py +0 -169
- {pl_e2e_test-0.1.8 → pl_e2e_test-0.2.0}/LICENSE +0 -0
- {pl_e2e_test-0.1.8 → pl_e2e_test-0.2.0}/MANIFEST.in +0 -0
- {pl_e2e_test-0.1.8 → pl_e2e_test-0.2.0}/setup.cfg +0 -0
- {pl_e2e_test-0.1.8 → pl_e2e_test-0.2.0}/src/pl_e2e_test.egg-info/dependency_links.txt +0 -0
- {pl_e2e_test-0.1.8 → pl_e2e_test-0.2.0}/src/pl_e2e_test.egg-info/entry_points.txt +0 -0
- {pl_e2e_test-0.1.8 → pl_e2e_test-0.2.0}/src/pl_e2e_test.egg-info/requires.txt +0 -0
- {pl_e2e_test-0.1.8 → pl_e2e_test-0.2.0}/src/pl_e2e_test.egg-info/top_level.txt +0 -0
- {pl_e2e_test-0.1.8 → pl_e2e_test-0.2.0}/src/prairielearn_e2e/__main__.py +0 -0
- {pl_e2e_test-0.1.8 → pl_e2e_test-0.2.0}/src/prairielearn_e2e/backend.py +0 -0
- {pl_e2e_test-0.1.8 → pl_e2e_test-0.2.0}/src/prairielearn_e2e/contract.py +0 -0
- {pl_e2e_test-0.1.8 → pl_e2e_test-0.2.0}/src/prairielearn_e2e/discovery.py +0 -0
- {pl_e2e_test-0.1.8 → pl_e2e_test-0.2.0}/src/prairielearn_e2e/errors.py +0 -0
- {pl_e2e_test-0.1.8 → pl_e2e_test-0.2.0}/src/prairielearn_e2e/expected_errors.py +0 -0
- {pl_e2e_test-0.1.8 → pl_e2e_test-0.2.0}/src/prairielearn_e2e/harness.py +0 -0
- {pl_e2e_test-0.1.8 → pl_e2e_test-0.2.0}/src/prairielearn_e2e/html_checks.py +0 -0
- {pl_e2e_test-0.1.8 → pl_e2e_test-0.2.0}/src/prairielearn_e2e/metadata.py +0 -0
- {pl_e2e_test-0.1.8 → pl_e2e_test-0.2.0}/src/prairielearn_e2e/question_tests.py +0 -0
- {pl_e2e_test-0.1.8 → pl_e2e_test-0.2.0}/src/prairielearn_e2e/regression.py +0 -0
- {pl_e2e_test-0.1.8 → pl_e2e_test-0.2.0}/src/prairielearn_e2e/server.py +0 -0
- {pl_e2e_test-0.1.8 → pl_e2e_test-0.2.0}/src/prairielearn_e2e/sharding.py +0 -0
- {pl_e2e_test-0.1.8 → pl_e2e_test-0.2.0}/tests/__init__.py +0 -0
- {pl_e2e_test-0.1.8 → pl_e2e_test-0.2.0}/tests/conftest.py +0 -0
- {pl_e2e_test-0.1.8 → pl_e2e_test-0.2.0}/tests/test_contract.py +0 -0
- {pl_e2e_test-0.1.8 → pl_e2e_test-0.2.0}/tests/test_discovery.py +0 -0
- {pl_e2e_test-0.1.8 → pl_e2e_test-0.2.0}/tests/test_html_checks.py +0 -0
- {pl_e2e_test-0.1.8 → pl_e2e_test-0.2.0}/tests/test_metadata.py +0 -0
- {pl_e2e_test-0.1.8 → pl_e2e_test-0.2.0}/tests/test_question_tests.py +0 -0
- {pl_e2e_test-0.1.8 → pl_e2e_test-0.2.0}/tests/test_regression.py +0 -0
- {pl_e2e_test-0.1.8 → pl_e2e_test-0.2.0}/tests/test_server.py +0 -0
- {pl_e2e_test-0.1.8 → pl_e2e_test-0.2.0}/tests/test_sharding.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: pl-e2e-test
|
|
3
|
-
Version: 0.
|
|
3
|
+
Version: 0.2.0
|
|
4
4
|
Summary: Fuzz PrairieLearn question lifecycles against the real Python element runtime
|
|
5
5
|
License-Expression: MIT
|
|
6
6
|
Project-URL: Homepage, https://github.com/PrairieLearn/pl-e2e-test
|
|
@@ -38,6 +38,8 @@ The package installs the reusable `prairielearn_e2e` Python API and a single
|
|
|
38
38
|
- `pl-e2e-test parallel` runs local shards and merges their results.
|
|
39
39
|
- `pl-e2e-test add-seed` records a failing concrete variant seed.
|
|
40
40
|
- `pl-e2e-test reports` merges parallel shard results into terminal and JUnit reports.
|
|
41
|
+
- `pl-e2e-test instances` shows configured, installed, cached, and local PrairieLearn
|
|
42
|
+
revisions.
|
|
41
43
|
- `pl-e2e-test version` prints the installed version.
|
|
42
44
|
- `pl-e2e-test help [COMMAND]` prints top-level help, or one command's help.
|
|
43
45
|
|
|
@@ -45,6 +47,7 @@ The package installs the reusable `prairielearn_e2e` Python API and a single
|
|
|
45
47
|
|
|
46
48
|
- Python 3.13 or newer
|
|
47
49
|
- Git 2.30 or newer
|
|
50
|
+
- uv 0.9 or newer
|
|
48
51
|
|
|
49
52
|
The harness depends on pytest, Pyright, Ruff, Chevron, JSON Schema, lxml, NumPy,
|
|
50
53
|
and PrairieLearn. PrairieLearn's Python package is currently consumed from its Git
|
|
@@ -241,19 +244,34 @@ jobs:
|
|
|
241
244
|
```
|
|
242
245
|
|
|
243
246
|
The course must provide `.prairielearn/schemas/infoQuestion.json`. The harness checks
|
|
244
|
-
question metadata against that schema and
|
|
245
|
-
|
|
246
|
-
|
|
247
|
-
|
|
248
|
-
|
|
247
|
+
question metadata against that schema and resolves the configured PrairieLearn version
|
|
248
|
+
before each run. The default, `latest`, resolves the Git source's remote `HEAD`; a full
|
|
249
|
+
commit, branch, or unambiguous tag can be selected instead. Moving selectors require
|
|
250
|
+
network access and never silently fall back to stale cache state.
|
|
251
|
+
|
|
252
|
+
The selected revision's Python package and core elements are always used together. The
|
|
253
|
+
harness keeps a blob-filtered bare repository in the platform-standard user cache and
|
|
254
|
+
hydrates only `apps/prairielearn/python` and `apps/prairielearn/elements`. It launches
|
|
255
|
+
the selected Python package through a frozen uv overlay, without changing the course's
|
|
256
|
+
`pyproject.toml`, `uv.lock`, or virtual environment. A previously hydrated exact commit
|
|
257
|
+
can be reused offline when its uv dependencies are also cached.
|
|
249
258
|
|
|
250
259
|
To use an existing PrairieLearn worktree or bare repository instead, pass
|
|
251
260
|
`--prairielearn-path /path/to/PrairieLearn`. The repository must already contain the
|
|
252
|
-
|
|
253
|
-
|
|
254
|
-
|
|
255
|
-
|
|
256
|
-
|
|
261
|
+
selected revision. An explicit path is authoritative: the harness resolves selectors
|
|
262
|
+
against its local refs, does not fetch into it, and does not fall back to the managed
|
|
263
|
+
cache. For a local repository, `latest` means its current `HEAD`.
|
|
264
|
+
|
|
265
|
+
Inspect available revisions without contacting remotes:
|
|
266
|
+
|
|
267
|
+
```sh
|
|
268
|
+
uv run pl-e2e-test instances
|
|
269
|
+
```
|
|
270
|
+
|
|
271
|
+
Add `--refresh` to resolve the configured moving selector and verify remote access.
|
|
272
|
+
The table distinguishes fully hydrated runtime revisions from legacy element-only cache
|
|
273
|
+
entries. Standard cache environment settings such as `XDG_CACHE_HOME` can relocate the
|
|
274
|
+
managed cache; CI systems can persist that directory between runs if desired.
|
|
257
275
|
|
|
258
276
|
### Harness configuration
|
|
259
277
|
|
|
@@ -261,11 +279,17 @@ Selector settings are read automatically from the course's `pyproject.toml`:
|
|
|
261
279
|
|
|
262
280
|
```toml
|
|
263
281
|
[tool.pl-e2e-test]
|
|
282
|
+
prairielearnVersion = "latest"
|
|
264
283
|
elementSolutionSelectors = ["pl-sketch"]
|
|
265
284
|
nonGradableSelectors = ['pl-sketch[read-only="true"]', 'pl-sketch[read-only="1"]']
|
|
266
285
|
```
|
|
267
286
|
|
|
268
|
-
These are the defaults. `
|
|
287
|
+
These are the defaults. Omitting `prairielearnVersion` is equivalent to `"latest"`.
|
|
288
|
+
Use a full commit for reproducible or offline runs, or a branch/tag when a named moving
|
|
289
|
+
target is intentional. `--prairielearn-version SELECTOR` overrides the TOML value for
|
|
290
|
+
one invocation, and replay commands always print the resolved full commit.
|
|
291
|
+
|
|
292
|
+
`elementSolutionSelectors` identifies elements that provide
|
|
269
293
|
solutions without a `correct_answers` entry. It only relaxes that entry requirement;
|
|
270
294
|
it does not generate a test submission. `nonGradableSelectors` excludes display-only
|
|
271
295
|
answers from grading checks. Each supplied list replaces its default; omitted settings
|
|
@@ -455,12 +479,16 @@ course's TOML settings, or `config=load_config("settings.toml")` for an explicit
|
|
|
455
479
|
`load_config` and `HarnessConfig` are exported from `prairielearn_e2e`. Omitting
|
|
456
480
|
`config` uses the built-in defaults.
|
|
457
481
|
|
|
482
|
+
The command-line runner and high-level `run_parallel` API perform version resolution
|
|
483
|
+
and uv bootstrapping. Manually constructed `PrairieLearnBackend` and `QuestionHarness`
|
|
484
|
+
objects continue to use the PrairieLearn package already loaded by their interpreter.
|
|
485
|
+
|
|
458
486
|
## Development
|
|
459
487
|
|
|
460
488
|
This project uses uv 0.9 or newer:
|
|
461
489
|
|
|
462
490
|
```sh
|
|
463
|
-
|
|
491
|
+
make install
|
|
464
492
|
make test
|
|
465
493
|
make format
|
|
466
494
|
make smoke-dist
|
|
@@ -472,44 +500,36 @@ into an isolated target, and imports it using the locked development dependencie
|
|
|
472
500
|
|
|
473
501
|
## Publishing
|
|
474
502
|
|
|
475
|
-
There are three supported publishing paths. Choose one for a release; do not run the
|
|
476
|
-
local and GitHub paths for the same version because PyPI versions are immutable.
|
|
477
|
-
|
|
478
|
-
### GitHub-hosted trusted publishing
|
|
479
|
-
|
|
480
503
|
Configure the PyPI project to trust the `publish.yml` workflow in the GitHub `pypi`
|
|
481
|
-
environment.
|
|
504
|
+
environment. Releases must start from a clean `main` that is synchronized with
|
|
505
|
+
`origin/main`, with `gh` authenticated and the publishing workflow available. To bump
|
|
506
|
+
the current version, run one of:
|
|
482
507
|
|
|
483
508
|
```sh
|
|
484
|
-
make publish
|
|
509
|
+
make publish patch
|
|
510
|
+
make publish minor
|
|
511
|
+
make publish major
|
|
485
512
|
```
|
|
486
513
|
|
|
487
|
-
|
|
488
|
-
the full checks, updates `pyproject.toml` and `uv.lock`, builds and smoke-tests the
|
|
489
|
-
distributions, creates a release commit and annotated `v0.2.0` tag, and atomically
|
|
490
|
-
pushes the branch and tag. The tag starts the workflow on `ubuntu-latest`; the workflow
|
|
491
|
-
rechecks the tag/version match, tests, builds, generates attestations, and publishes
|
|
492
|
-
with PyPI trusted-publishing OIDC credentials. Use `REMOTE=name` for a different Git
|
|
493
|
-
remote.
|
|
494
|
-
|
|
495
|
-
If an unpushed commit already contains the requested version, use the guarded bypass:
|
|
514
|
+
If the synchronized commit already contains the intended, not-yet-tagged version, use:
|
|
496
515
|
|
|
497
516
|
```sh
|
|
498
|
-
make publish
|
|
517
|
+
make publish current
|
|
499
518
|
```
|
|
500
519
|
|
|
501
|
-
|
|
502
|
-
|
|
503
|
-
|
|
504
|
-
|
|
520
|
+
The release command runs the shared `make release-check` gate, updates
|
|
521
|
+
`pyproject.toml` and `uv.lock` when needed, creates a release commit and annotated tag,
|
|
522
|
+
atomically pushes `main` and the tag, and watches the trusted-publishing workflow to
|
|
523
|
+
completion. Preview the full process without changing files, refs, remotes, or workflows
|
|
524
|
+
with:
|
|
505
525
|
|
|
506
526
|
```sh
|
|
507
|
-
make publish
|
|
527
|
+
make publish patch DRY_RUN=1
|
|
508
528
|
```
|
|
509
529
|
|
|
510
|
-
|
|
511
|
-
|
|
512
|
-
|
|
530
|
+
If the release tag exists but trusted publishing needs to be dispatched again, run
|
|
531
|
+
`make publish-pypi`. This validates that the tag matches the current version and is
|
|
532
|
+
reachable from `origin/main`, then dispatches and watches a new workflow run.
|
|
513
533
|
|
|
514
534
|
### Local self-hosted GitHub runner
|
|
515
535
|
|
|
@@ -518,11 +538,11 @@ inside Docker. The container has no host mounts and does not receive the Docker
|
|
|
518
538
|
Authenticate `gh` with repository administration permission, then run:
|
|
519
539
|
|
|
520
540
|
```sh
|
|
521
|
-
make
|
|
522
|
-
make publish-self-hosted RELEASE_TAG=v0.2.0
|
|
541
|
+
make publish-self-hosted
|
|
523
542
|
```
|
|
524
543
|
|
|
525
|
-
The tag must already exist. This
|
|
526
|
-
`
|
|
527
|
-
|
|
528
|
-
|
|
544
|
+
The current version's tag must already exist. This ensures the local runner is online,
|
|
545
|
+
dispatches `publish.yml` from `main` with the `self-hosted` runner choice, and watches
|
|
546
|
+
the run. `make runner-ensure` starts Docker Desktop on macOS when needed and repairs the
|
|
547
|
+
runner before dispatch. Other lifecycle commands are `runner-status`, `runner-stop`,
|
|
548
|
+
`runner-remove`, `runner-logs`, and `clean-runner`.
|
|
@@ -14,6 +14,8 @@ The package installs the reusable `prairielearn_e2e` Python API and a single
|
|
|
14
14
|
- `pl-e2e-test parallel` runs local shards and merges their results.
|
|
15
15
|
- `pl-e2e-test add-seed` records a failing concrete variant seed.
|
|
16
16
|
- `pl-e2e-test reports` merges parallel shard results into terminal and JUnit reports.
|
|
17
|
+
- `pl-e2e-test instances` shows configured, installed, cached, and local PrairieLearn
|
|
18
|
+
revisions.
|
|
17
19
|
- `pl-e2e-test version` prints the installed version.
|
|
18
20
|
- `pl-e2e-test help [COMMAND]` prints top-level help, or one command's help.
|
|
19
21
|
|
|
@@ -21,6 +23,7 @@ The package installs the reusable `prairielearn_e2e` Python API and a single
|
|
|
21
23
|
|
|
22
24
|
- Python 3.13 or newer
|
|
23
25
|
- Git 2.30 or newer
|
|
26
|
+
- uv 0.9 or newer
|
|
24
27
|
|
|
25
28
|
The harness depends on pytest, Pyright, Ruff, Chevron, JSON Schema, lxml, NumPy,
|
|
26
29
|
and PrairieLearn. PrairieLearn's Python package is currently consumed from its Git
|
|
@@ -217,19 +220,34 @@ jobs:
|
|
|
217
220
|
```
|
|
218
221
|
|
|
219
222
|
The course must provide `.prairielearn/schemas/infoQuestion.json`. The harness checks
|
|
220
|
-
question metadata against that schema and
|
|
221
|
-
|
|
222
|
-
|
|
223
|
-
|
|
224
|
-
|
|
223
|
+
question metadata against that schema and resolves the configured PrairieLearn version
|
|
224
|
+
before each run. The default, `latest`, resolves the Git source's remote `HEAD`; a full
|
|
225
|
+
commit, branch, or unambiguous tag can be selected instead. Moving selectors require
|
|
226
|
+
network access and never silently fall back to stale cache state.
|
|
227
|
+
|
|
228
|
+
The selected revision's Python package and core elements are always used together. The
|
|
229
|
+
harness keeps a blob-filtered bare repository in the platform-standard user cache and
|
|
230
|
+
hydrates only `apps/prairielearn/python` and `apps/prairielearn/elements`. It launches
|
|
231
|
+
the selected Python package through a frozen uv overlay, without changing the course's
|
|
232
|
+
`pyproject.toml`, `uv.lock`, or virtual environment. A previously hydrated exact commit
|
|
233
|
+
can be reused offline when its uv dependencies are also cached.
|
|
225
234
|
|
|
226
235
|
To use an existing PrairieLearn worktree or bare repository instead, pass
|
|
227
236
|
`--prairielearn-path /path/to/PrairieLearn`. The repository must already contain the
|
|
228
|
-
|
|
229
|
-
|
|
230
|
-
|
|
231
|
-
|
|
232
|
-
|
|
237
|
+
selected revision. An explicit path is authoritative: the harness resolves selectors
|
|
238
|
+
against its local refs, does not fetch into it, and does not fall back to the managed
|
|
239
|
+
cache. For a local repository, `latest` means its current `HEAD`.
|
|
240
|
+
|
|
241
|
+
Inspect available revisions without contacting remotes:
|
|
242
|
+
|
|
243
|
+
```sh
|
|
244
|
+
uv run pl-e2e-test instances
|
|
245
|
+
```
|
|
246
|
+
|
|
247
|
+
Add `--refresh` to resolve the configured moving selector and verify remote access.
|
|
248
|
+
The table distinguishes fully hydrated runtime revisions from legacy element-only cache
|
|
249
|
+
entries. Standard cache environment settings such as `XDG_CACHE_HOME` can relocate the
|
|
250
|
+
managed cache; CI systems can persist that directory between runs if desired.
|
|
233
251
|
|
|
234
252
|
### Harness configuration
|
|
235
253
|
|
|
@@ -237,11 +255,17 @@ Selector settings are read automatically from the course's `pyproject.toml`:
|
|
|
237
255
|
|
|
238
256
|
```toml
|
|
239
257
|
[tool.pl-e2e-test]
|
|
258
|
+
prairielearnVersion = "latest"
|
|
240
259
|
elementSolutionSelectors = ["pl-sketch"]
|
|
241
260
|
nonGradableSelectors = ['pl-sketch[read-only="true"]', 'pl-sketch[read-only="1"]']
|
|
242
261
|
```
|
|
243
262
|
|
|
244
|
-
These are the defaults. `
|
|
263
|
+
These are the defaults. Omitting `prairielearnVersion` is equivalent to `"latest"`.
|
|
264
|
+
Use a full commit for reproducible or offline runs, or a branch/tag when a named moving
|
|
265
|
+
target is intentional. `--prairielearn-version SELECTOR` overrides the TOML value for
|
|
266
|
+
one invocation, and replay commands always print the resolved full commit.
|
|
267
|
+
|
|
268
|
+
`elementSolutionSelectors` identifies elements that provide
|
|
245
269
|
solutions without a `correct_answers` entry. It only relaxes that entry requirement;
|
|
246
270
|
it does not generate a test submission. `nonGradableSelectors` excludes display-only
|
|
247
271
|
answers from grading checks. Each supplied list replaces its default; omitted settings
|
|
@@ -431,12 +455,16 @@ course's TOML settings, or `config=load_config("settings.toml")` for an explicit
|
|
|
431
455
|
`load_config` and `HarnessConfig` are exported from `prairielearn_e2e`. Omitting
|
|
432
456
|
`config` uses the built-in defaults.
|
|
433
457
|
|
|
458
|
+
The command-line runner and high-level `run_parallel` API perform version resolution
|
|
459
|
+
and uv bootstrapping. Manually constructed `PrairieLearnBackend` and `QuestionHarness`
|
|
460
|
+
objects continue to use the PrairieLearn package already loaded by their interpreter.
|
|
461
|
+
|
|
434
462
|
## Development
|
|
435
463
|
|
|
436
464
|
This project uses uv 0.9 or newer:
|
|
437
465
|
|
|
438
466
|
```sh
|
|
439
|
-
|
|
467
|
+
make install
|
|
440
468
|
make test
|
|
441
469
|
make format
|
|
442
470
|
make smoke-dist
|
|
@@ -448,44 +476,36 @@ into an isolated target, and imports it using the locked development dependencie
|
|
|
448
476
|
|
|
449
477
|
## Publishing
|
|
450
478
|
|
|
451
|
-
There are three supported publishing paths. Choose one for a release; do not run the
|
|
452
|
-
local and GitHub paths for the same version because PyPI versions are immutable.
|
|
453
|
-
|
|
454
|
-
### GitHub-hosted trusted publishing
|
|
455
|
-
|
|
456
479
|
Configure the PyPI project to trust the `publish.yml` workflow in the GitHub `pypi`
|
|
457
|
-
environment.
|
|
480
|
+
environment. Releases must start from a clean `main` that is synchronized with
|
|
481
|
+
`origin/main`, with `gh` authenticated and the publishing workflow available. To bump
|
|
482
|
+
the current version, run one of:
|
|
458
483
|
|
|
459
484
|
```sh
|
|
460
|
-
make publish
|
|
485
|
+
make publish patch
|
|
486
|
+
make publish minor
|
|
487
|
+
make publish major
|
|
461
488
|
```
|
|
462
489
|
|
|
463
|
-
|
|
464
|
-
the full checks, updates `pyproject.toml` and `uv.lock`, builds and smoke-tests the
|
|
465
|
-
distributions, creates a release commit and annotated `v0.2.0` tag, and atomically
|
|
466
|
-
pushes the branch and tag. The tag starts the workflow on `ubuntu-latest`; the workflow
|
|
467
|
-
rechecks the tag/version match, tests, builds, generates attestations, and publishes
|
|
468
|
-
with PyPI trusted-publishing OIDC credentials. Use `REMOTE=name` for a different Git
|
|
469
|
-
remote.
|
|
470
|
-
|
|
471
|
-
If an unpushed commit already contains the requested version, use the guarded bypass:
|
|
490
|
+
If the synchronized commit already contains the intended, not-yet-tagged version, use:
|
|
472
491
|
|
|
473
492
|
```sh
|
|
474
|
-
make publish
|
|
493
|
+
make publish current
|
|
475
494
|
```
|
|
476
495
|
|
|
477
|
-
|
|
478
|
-
|
|
479
|
-
|
|
480
|
-
|
|
496
|
+
The release command runs the shared `make release-check` gate, updates
|
|
497
|
+
`pyproject.toml` and `uv.lock` when needed, creates a release commit and annotated tag,
|
|
498
|
+
atomically pushes `main` and the tag, and watches the trusted-publishing workflow to
|
|
499
|
+
completion. Preview the full process without changing files, refs, remotes, or workflows
|
|
500
|
+
with:
|
|
481
501
|
|
|
482
502
|
```sh
|
|
483
|
-
make publish
|
|
503
|
+
make publish patch DRY_RUN=1
|
|
484
504
|
```
|
|
485
505
|
|
|
486
|
-
|
|
487
|
-
|
|
488
|
-
|
|
506
|
+
If the release tag exists but trusted publishing needs to be dispatched again, run
|
|
507
|
+
`make publish-pypi`. This validates that the tag matches the current version and is
|
|
508
|
+
reachable from `origin/main`, then dispatches and watches a new workflow run.
|
|
489
509
|
|
|
490
510
|
### Local self-hosted GitHub runner
|
|
491
511
|
|
|
@@ -494,11 +514,11 @@ inside Docker. The container has no host mounts and does not receive the Docker
|
|
|
494
514
|
Authenticate `gh` with repository administration permission, then run:
|
|
495
515
|
|
|
496
516
|
```sh
|
|
497
|
-
make
|
|
498
|
-
make publish-self-hosted RELEASE_TAG=v0.2.0
|
|
517
|
+
make publish-self-hosted
|
|
499
518
|
```
|
|
500
519
|
|
|
501
|
-
The tag must already exist. This
|
|
502
|
-
`
|
|
503
|
-
|
|
504
|
-
|
|
520
|
+
The current version's tag must already exist. This ensures the local runner is online,
|
|
521
|
+
dispatches `publish.yml` from `main` with the `self-hosted` runner choice, and watches
|
|
522
|
+
the run. `make runner-ensure` starts Docker Desktop on macOS when needed and repairs the
|
|
523
|
+
runner before dispatch. Other lifecycle commands are `runner-status`, `runner-stop`,
|
|
524
|
+
`runner-remove`, `runner-logs`, and `clean-runner`.
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: pl-e2e-test
|
|
3
|
-
Version: 0.
|
|
3
|
+
Version: 0.2.0
|
|
4
4
|
Summary: Fuzz PrairieLearn question lifecycles against the real Python element runtime
|
|
5
5
|
License-Expression: MIT
|
|
6
6
|
Project-URL: Homepage, https://github.com/PrairieLearn/pl-e2e-test
|
|
@@ -38,6 +38,8 @@ The package installs the reusable `prairielearn_e2e` Python API and a single
|
|
|
38
38
|
- `pl-e2e-test parallel` runs local shards and merges their results.
|
|
39
39
|
- `pl-e2e-test add-seed` records a failing concrete variant seed.
|
|
40
40
|
- `pl-e2e-test reports` merges parallel shard results into terminal and JUnit reports.
|
|
41
|
+
- `pl-e2e-test instances` shows configured, installed, cached, and local PrairieLearn
|
|
42
|
+
revisions.
|
|
41
43
|
- `pl-e2e-test version` prints the installed version.
|
|
42
44
|
- `pl-e2e-test help [COMMAND]` prints top-level help, or one command's help.
|
|
43
45
|
|
|
@@ -45,6 +47,7 @@ The package installs the reusable `prairielearn_e2e` Python API and a single
|
|
|
45
47
|
|
|
46
48
|
- Python 3.13 or newer
|
|
47
49
|
- Git 2.30 or newer
|
|
50
|
+
- uv 0.9 or newer
|
|
48
51
|
|
|
49
52
|
The harness depends on pytest, Pyright, Ruff, Chevron, JSON Schema, lxml, NumPy,
|
|
50
53
|
and PrairieLearn. PrairieLearn's Python package is currently consumed from its Git
|
|
@@ -241,19 +244,34 @@ jobs:
|
|
|
241
244
|
```
|
|
242
245
|
|
|
243
246
|
The course must provide `.prairielearn/schemas/infoQuestion.json`. The harness checks
|
|
244
|
-
question metadata against that schema and
|
|
245
|
-
|
|
246
|
-
|
|
247
|
-
|
|
248
|
-
|
|
247
|
+
question metadata against that schema and resolves the configured PrairieLearn version
|
|
248
|
+
before each run. The default, `latest`, resolves the Git source's remote `HEAD`; a full
|
|
249
|
+
commit, branch, or unambiguous tag can be selected instead. Moving selectors require
|
|
250
|
+
network access and never silently fall back to stale cache state.
|
|
251
|
+
|
|
252
|
+
The selected revision's Python package and core elements are always used together. The
|
|
253
|
+
harness keeps a blob-filtered bare repository in the platform-standard user cache and
|
|
254
|
+
hydrates only `apps/prairielearn/python` and `apps/prairielearn/elements`. It launches
|
|
255
|
+
the selected Python package through a frozen uv overlay, without changing the course's
|
|
256
|
+
`pyproject.toml`, `uv.lock`, or virtual environment. A previously hydrated exact commit
|
|
257
|
+
can be reused offline when its uv dependencies are also cached.
|
|
249
258
|
|
|
250
259
|
To use an existing PrairieLearn worktree or bare repository instead, pass
|
|
251
260
|
`--prairielearn-path /path/to/PrairieLearn`. The repository must already contain the
|
|
252
|
-
|
|
253
|
-
|
|
254
|
-
|
|
255
|
-
|
|
256
|
-
|
|
261
|
+
selected revision. An explicit path is authoritative: the harness resolves selectors
|
|
262
|
+
against its local refs, does not fetch into it, and does not fall back to the managed
|
|
263
|
+
cache. For a local repository, `latest` means its current `HEAD`.
|
|
264
|
+
|
|
265
|
+
Inspect available revisions without contacting remotes:
|
|
266
|
+
|
|
267
|
+
```sh
|
|
268
|
+
uv run pl-e2e-test instances
|
|
269
|
+
```
|
|
270
|
+
|
|
271
|
+
Add `--refresh` to resolve the configured moving selector and verify remote access.
|
|
272
|
+
The table distinguishes fully hydrated runtime revisions from legacy element-only cache
|
|
273
|
+
entries. Standard cache environment settings such as `XDG_CACHE_HOME` can relocate the
|
|
274
|
+
managed cache; CI systems can persist that directory between runs if desired.
|
|
257
275
|
|
|
258
276
|
### Harness configuration
|
|
259
277
|
|
|
@@ -261,11 +279,17 @@ Selector settings are read automatically from the course's `pyproject.toml`:
|
|
|
261
279
|
|
|
262
280
|
```toml
|
|
263
281
|
[tool.pl-e2e-test]
|
|
282
|
+
prairielearnVersion = "latest"
|
|
264
283
|
elementSolutionSelectors = ["pl-sketch"]
|
|
265
284
|
nonGradableSelectors = ['pl-sketch[read-only="true"]', 'pl-sketch[read-only="1"]']
|
|
266
285
|
```
|
|
267
286
|
|
|
268
|
-
These are the defaults. `
|
|
287
|
+
These are the defaults. Omitting `prairielearnVersion` is equivalent to `"latest"`.
|
|
288
|
+
Use a full commit for reproducible or offline runs, or a branch/tag when a named moving
|
|
289
|
+
target is intentional. `--prairielearn-version SELECTOR` overrides the TOML value for
|
|
290
|
+
one invocation, and replay commands always print the resolved full commit.
|
|
291
|
+
|
|
292
|
+
`elementSolutionSelectors` identifies elements that provide
|
|
269
293
|
solutions without a `correct_answers` entry. It only relaxes that entry requirement;
|
|
270
294
|
it does not generate a test submission. `nonGradableSelectors` excludes display-only
|
|
271
295
|
answers from grading checks. Each supplied list replaces its default; omitted settings
|
|
@@ -455,12 +479,16 @@ course's TOML settings, or `config=load_config("settings.toml")` for an explicit
|
|
|
455
479
|
`load_config` and `HarnessConfig` are exported from `prairielearn_e2e`. Omitting
|
|
456
480
|
`config` uses the built-in defaults.
|
|
457
481
|
|
|
482
|
+
The command-line runner and high-level `run_parallel` API perform version resolution
|
|
483
|
+
and uv bootstrapping. Manually constructed `PrairieLearnBackend` and `QuestionHarness`
|
|
484
|
+
objects continue to use the PrairieLearn package already loaded by their interpreter.
|
|
485
|
+
|
|
458
486
|
## Development
|
|
459
487
|
|
|
460
488
|
This project uses uv 0.9 or newer:
|
|
461
489
|
|
|
462
490
|
```sh
|
|
463
|
-
|
|
491
|
+
make install
|
|
464
492
|
make test
|
|
465
493
|
make format
|
|
466
494
|
make smoke-dist
|
|
@@ -472,44 +500,36 @@ into an isolated target, and imports it using the locked development dependencie
|
|
|
472
500
|
|
|
473
501
|
## Publishing
|
|
474
502
|
|
|
475
|
-
There are three supported publishing paths. Choose one for a release; do not run the
|
|
476
|
-
local and GitHub paths for the same version because PyPI versions are immutable.
|
|
477
|
-
|
|
478
|
-
### GitHub-hosted trusted publishing
|
|
479
|
-
|
|
480
503
|
Configure the PyPI project to trust the `publish.yml` workflow in the GitHub `pypi`
|
|
481
|
-
environment.
|
|
504
|
+
environment. Releases must start from a clean `main` that is synchronized with
|
|
505
|
+
`origin/main`, with `gh` authenticated and the publishing workflow available. To bump
|
|
506
|
+
the current version, run one of:
|
|
482
507
|
|
|
483
508
|
```sh
|
|
484
|
-
make publish
|
|
509
|
+
make publish patch
|
|
510
|
+
make publish minor
|
|
511
|
+
make publish major
|
|
485
512
|
```
|
|
486
513
|
|
|
487
|
-
|
|
488
|
-
the full checks, updates `pyproject.toml` and `uv.lock`, builds and smoke-tests the
|
|
489
|
-
distributions, creates a release commit and annotated `v0.2.0` tag, and atomically
|
|
490
|
-
pushes the branch and tag. The tag starts the workflow on `ubuntu-latest`; the workflow
|
|
491
|
-
rechecks the tag/version match, tests, builds, generates attestations, and publishes
|
|
492
|
-
with PyPI trusted-publishing OIDC credentials. Use `REMOTE=name` for a different Git
|
|
493
|
-
remote.
|
|
494
|
-
|
|
495
|
-
If an unpushed commit already contains the requested version, use the guarded bypass:
|
|
514
|
+
If the synchronized commit already contains the intended, not-yet-tagged version, use:
|
|
496
515
|
|
|
497
516
|
```sh
|
|
498
|
-
make publish
|
|
517
|
+
make publish current
|
|
499
518
|
```
|
|
500
519
|
|
|
501
|
-
|
|
502
|
-
|
|
503
|
-
|
|
504
|
-
|
|
520
|
+
The release command runs the shared `make release-check` gate, updates
|
|
521
|
+
`pyproject.toml` and `uv.lock` when needed, creates a release commit and annotated tag,
|
|
522
|
+
atomically pushes `main` and the tag, and watches the trusted-publishing workflow to
|
|
523
|
+
completion. Preview the full process without changing files, refs, remotes, or workflows
|
|
524
|
+
with:
|
|
505
525
|
|
|
506
526
|
```sh
|
|
507
|
-
make publish
|
|
527
|
+
make publish patch DRY_RUN=1
|
|
508
528
|
```
|
|
509
529
|
|
|
510
|
-
|
|
511
|
-
|
|
512
|
-
|
|
530
|
+
If the release tag exists but trusted publishing needs to be dispatched again, run
|
|
531
|
+
`make publish-pypi`. This validates that the tag matches the current version and is
|
|
532
|
+
reachable from `origin/main`, then dispatches and watches a new workflow run.
|
|
513
533
|
|
|
514
534
|
### Local self-hosted GitHub runner
|
|
515
535
|
|
|
@@ -518,11 +538,11 @@ inside Docker. The container has no host mounts and does not receive the Docker
|
|
|
518
538
|
Authenticate `gh` with repository administration permission, then run:
|
|
519
539
|
|
|
520
540
|
```sh
|
|
521
|
-
make
|
|
522
|
-
make publish-self-hosted RELEASE_TAG=v0.2.0
|
|
541
|
+
make publish-self-hosted
|
|
523
542
|
```
|
|
524
543
|
|
|
525
|
-
The tag must already exist. This
|
|
526
|
-
`
|
|
527
|
-
|
|
528
|
-
|
|
544
|
+
The current version's tag must already exist. This ensures the local runner is online,
|
|
545
|
+
dispatches `publish.yml` from `main` with the `self-hosted` runner choice, and watches
|
|
546
|
+
the run. `make runner-ensure` starts Docker Desktop on macOS when needed and repairs the
|
|
547
|
+
runner before dispatch. Other lifecycle commands are `runner-status`, `runner-stop`,
|
|
548
|
+
`runner-remove`, `runner-logs`, and `clean-runner`.
|
|
@@ -21,6 +21,7 @@ src/prairielearn_e2e/expected_errors.py
|
|
|
21
21
|
src/prairielearn_e2e/fuzz.py
|
|
22
22
|
src/prairielearn_e2e/harness.py
|
|
23
23
|
src/prairielearn_e2e/html_checks.py
|
|
24
|
+
src/prairielearn_e2e/instances.py
|
|
24
25
|
src/prairielearn_e2e/metadata.py
|
|
25
26
|
src/prairielearn_e2e/parallel.py
|
|
26
27
|
src/prairielearn_e2e/pytest_plugin.py
|
|
@@ -29,6 +30,7 @@ src/prairielearn_e2e/regression.py
|
|
|
29
30
|
src/prairielearn_e2e/reporting.py
|
|
30
31
|
src/prairielearn_e2e/repository.py
|
|
31
32
|
src/prairielearn_e2e/runner.py
|
|
33
|
+
src/prairielearn_e2e/runtime.py
|
|
32
34
|
src/prairielearn_e2e/server.py
|
|
33
35
|
src/prairielearn_e2e/sharding.py
|
|
34
36
|
tests/__init__.py
|
|
@@ -42,6 +44,7 @@ tests/test_fuzz.py
|
|
|
42
44
|
tests/test_git.py
|
|
43
45
|
tests/test_harness.py
|
|
44
46
|
tests/test_html_checks.py
|
|
47
|
+
tests/test_instances.py
|
|
45
48
|
tests/test_metadata.py
|
|
46
49
|
tests/test_parallel.py
|
|
47
50
|
tests/test_publish_version.py
|
|
@@ -50,5 +53,6 @@ tests/test_question_tests.py
|
|
|
50
53
|
tests/test_regression.py
|
|
51
54
|
tests/test_reporting.py
|
|
52
55
|
tests/test_repository.py
|
|
56
|
+
tests/test_runtime.py
|
|
53
57
|
tests/test_server.py
|
|
54
58
|
tests/test_sharding.py
|
|
@@ -0,0 +1,50 @@
|
|
|
1
|
+
"""End-to-end contract tests for PrairieLearn question directories."""
|
|
2
|
+
|
|
3
|
+
from importlib import import_module
|
|
4
|
+
from importlib.metadata import version
|
|
5
|
+
from typing import TYPE_CHECKING, Any
|
|
6
|
+
|
|
7
|
+
if TYPE_CHECKING:
|
|
8
|
+
from .backend import PrairieLearnBackend
|
|
9
|
+
from .config import HarnessConfig, load_config
|
|
10
|
+
from .discovery import QuestionCase, discover_questions
|
|
11
|
+
from .harness import LifecycleResult, QuestionHarness
|
|
12
|
+
from .parallel import run_parallel
|
|
13
|
+
|
|
14
|
+
__version__ = version("pl-e2e-test")
|
|
15
|
+
|
|
16
|
+
__all__ = (
|
|
17
|
+
"HarnessConfig",
|
|
18
|
+
"LifecycleResult",
|
|
19
|
+
"PrairieLearnBackend",
|
|
20
|
+
"QuestionCase",
|
|
21
|
+
"QuestionHarness",
|
|
22
|
+
"__version__",
|
|
23
|
+
"discover_questions",
|
|
24
|
+
"load_config",
|
|
25
|
+
"run_parallel",
|
|
26
|
+
)
|
|
27
|
+
|
|
28
|
+
|
|
29
|
+
_LAZY_EXPORTS = {
|
|
30
|
+
"HarnessConfig": (".config", "HarnessConfig"),
|
|
31
|
+
"LifecycleResult": (".harness", "LifecycleResult"),
|
|
32
|
+
"PrairieLearnBackend": (".backend", "PrairieLearnBackend"),
|
|
33
|
+
"QuestionCase": (".discovery", "QuestionCase"),
|
|
34
|
+
"QuestionHarness": (".harness", "QuestionHarness"),
|
|
35
|
+
"discover_questions": (".discovery", "discover_questions"),
|
|
36
|
+
"load_config": (".config", "load_config"),
|
|
37
|
+
"run_parallel": (".parallel", "run_parallel"),
|
|
38
|
+
}
|
|
39
|
+
|
|
40
|
+
|
|
41
|
+
def __getattr__(name: str) -> Any:
|
|
42
|
+
"""Load PrairieLearn-dependent public objects only after runtime bootstrap."""
|
|
43
|
+
|
|
44
|
+
try:
|
|
45
|
+
module_name, attribute = _LAZY_EXPORTS[name]
|
|
46
|
+
except KeyError as exc:
|
|
47
|
+
raise AttributeError(name) from exc
|
|
48
|
+
value = getattr(import_module(module_name, __name__), attribute)
|
|
49
|
+
globals()[name] = value
|
|
50
|
+
return value
|