pl-e2e-test 0.1.8__tar.gz → 0.2.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (64) hide show
  1. {pl_e2e_test-0.1.8/src/pl_e2e_test.egg-info → pl_e2e_test-0.2.0}/PKG-INFO +64 -44
  2. {pl_e2e_test-0.1.8 → pl_e2e_test-0.2.0}/README.md +63 -43
  3. {pl_e2e_test-0.1.8 → pl_e2e_test-0.2.0}/pyproject.toml +1 -1
  4. {pl_e2e_test-0.1.8 → pl_e2e_test-0.2.0/src/pl_e2e_test.egg-info}/PKG-INFO +64 -44
  5. {pl_e2e_test-0.1.8 → pl_e2e_test-0.2.0}/src/pl_e2e_test.egg-info/SOURCES.txt +4 -0
  6. pl_e2e_test-0.2.0/src/prairielearn_e2e/__init__.py +50 -0
  7. {pl_e2e_test-0.1.8 → pl_e2e_test-0.2.0}/src/prairielearn_e2e/_git.py +47 -3
  8. {pl_e2e_test-0.1.8 → pl_e2e_test-0.2.0}/src/prairielearn_e2e/cli.py +37 -6
  9. {pl_e2e_test-0.1.8 → pl_e2e_test-0.2.0}/src/prairielearn_e2e/config.py +10 -0
  10. {pl_e2e_test-0.1.8 → pl_e2e_test-0.2.0}/src/prairielearn_e2e/fuzz.py +236 -42
  11. pl_e2e_test-0.2.0/src/prairielearn_e2e/instances.py +161 -0
  12. {pl_e2e_test-0.1.8 → pl_e2e_test-0.2.0}/src/prairielearn_e2e/parallel.py +96 -9
  13. {pl_e2e_test-0.1.8 → pl_e2e_test-0.2.0}/src/prairielearn_e2e/pytest_plugin.py +90 -1
  14. {pl_e2e_test-0.1.8 → pl_e2e_test-0.2.0}/src/prairielearn_e2e/reporting.py +48 -3
  15. pl_e2e_test-0.2.0/src/prairielearn_e2e/repository.py +602 -0
  16. pl_e2e_test-0.2.0/src/prairielearn_e2e/runner.py +48 -0
  17. pl_e2e_test-0.2.0/src/prairielearn_e2e/runtime.py +387 -0
  18. {pl_e2e_test-0.1.8 → pl_e2e_test-0.2.0}/tests/stub_git.py +33 -7
  19. {pl_e2e_test-0.1.8 → pl_e2e_test-0.2.0}/tests/test_cli.py +14 -4
  20. {pl_e2e_test-0.1.8 → pl_e2e_test-0.2.0}/tests/test_config.py +4 -0
  21. {pl_e2e_test-0.1.8 → pl_e2e_test-0.2.0}/tests/test_fuzz.py +211 -1
  22. {pl_e2e_test-0.1.8 → pl_e2e_test-0.2.0}/tests/test_git.py +25 -1
  23. {pl_e2e_test-0.1.8 → pl_e2e_test-0.2.0}/tests/test_harness.py +34 -1
  24. pl_e2e_test-0.2.0/tests/test_instances.py +222 -0
  25. {pl_e2e_test-0.1.8 → pl_e2e_test-0.2.0}/tests/test_parallel.py +186 -4
  26. pl_e2e_test-0.2.0/tests/test_publish_version.py +479 -0
  27. {pl_e2e_test-0.1.8 → pl_e2e_test-0.2.0}/tests/test_pytest_plugin.py +125 -0
  28. {pl_e2e_test-0.1.8 → pl_e2e_test-0.2.0}/tests/test_reporting.py +35 -2
  29. {pl_e2e_test-0.1.8 → pl_e2e_test-0.2.0}/tests/test_repository.py +240 -1
  30. pl_e2e_test-0.2.0/tests/test_runtime.py +473 -0
  31. pl_e2e_test-0.1.8/src/prairielearn_e2e/__init__.py +0 -23
  32. pl_e2e_test-0.1.8/src/prairielearn_e2e/repository.py +0 -298
  33. pl_e2e_test-0.1.8/src/prairielearn_e2e/runner.py +0 -28
  34. pl_e2e_test-0.1.8/tests/test_publish_version.py +0 -169
  35. {pl_e2e_test-0.1.8 → pl_e2e_test-0.2.0}/LICENSE +0 -0
  36. {pl_e2e_test-0.1.8 → pl_e2e_test-0.2.0}/MANIFEST.in +0 -0
  37. {pl_e2e_test-0.1.8 → pl_e2e_test-0.2.0}/setup.cfg +0 -0
  38. {pl_e2e_test-0.1.8 → pl_e2e_test-0.2.0}/src/pl_e2e_test.egg-info/dependency_links.txt +0 -0
  39. {pl_e2e_test-0.1.8 → pl_e2e_test-0.2.0}/src/pl_e2e_test.egg-info/entry_points.txt +0 -0
  40. {pl_e2e_test-0.1.8 → pl_e2e_test-0.2.0}/src/pl_e2e_test.egg-info/requires.txt +0 -0
  41. {pl_e2e_test-0.1.8 → pl_e2e_test-0.2.0}/src/pl_e2e_test.egg-info/top_level.txt +0 -0
  42. {pl_e2e_test-0.1.8 → pl_e2e_test-0.2.0}/src/prairielearn_e2e/__main__.py +0 -0
  43. {pl_e2e_test-0.1.8 → pl_e2e_test-0.2.0}/src/prairielearn_e2e/backend.py +0 -0
  44. {pl_e2e_test-0.1.8 → pl_e2e_test-0.2.0}/src/prairielearn_e2e/contract.py +0 -0
  45. {pl_e2e_test-0.1.8 → pl_e2e_test-0.2.0}/src/prairielearn_e2e/discovery.py +0 -0
  46. {pl_e2e_test-0.1.8 → pl_e2e_test-0.2.0}/src/prairielearn_e2e/errors.py +0 -0
  47. {pl_e2e_test-0.1.8 → pl_e2e_test-0.2.0}/src/prairielearn_e2e/expected_errors.py +0 -0
  48. {pl_e2e_test-0.1.8 → pl_e2e_test-0.2.0}/src/prairielearn_e2e/harness.py +0 -0
  49. {pl_e2e_test-0.1.8 → pl_e2e_test-0.2.0}/src/prairielearn_e2e/html_checks.py +0 -0
  50. {pl_e2e_test-0.1.8 → pl_e2e_test-0.2.0}/src/prairielearn_e2e/metadata.py +0 -0
  51. {pl_e2e_test-0.1.8 → pl_e2e_test-0.2.0}/src/prairielearn_e2e/question_tests.py +0 -0
  52. {pl_e2e_test-0.1.8 → pl_e2e_test-0.2.0}/src/prairielearn_e2e/regression.py +0 -0
  53. {pl_e2e_test-0.1.8 → pl_e2e_test-0.2.0}/src/prairielearn_e2e/server.py +0 -0
  54. {pl_e2e_test-0.1.8 → pl_e2e_test-0.2.0}/src/prairielearn_e2e/sharding.py +0 -0
  55. {pl_e2e_test-0.1.8 → pl_e2e_test-0.2.0}/tests/__init__.py +0 -0
  56. {pl_e2e_test-0.1.8 → pl_e2e_test-0.2.0}/tests/conftest.py +0 -0
  57. {pl_e2e_test-0.1.8 → pl_e2e_test-0.2.0}/tests/test_contract.py +0 -0
  58. {pl_e2e_test-0.1.8 → pl_e2e_test-0.2.0}/tests/test_discovery.py +0 -0
  59. {pl_e2e_test-0.1.8 → pl_e2e_test-0.2.0}/tests/test_html_checks.py +0 -0
  60. {pl_e2e_test-0.1.8 → pl_e2e_test-0.2.0}/tests/test_metadata.py +0 -0
  61. {pl_e2e_test-0.1.8 → pl_e2e_test-0.2.0}/tests/test_question_tests.py +0 -0
  62. {pl_e2e_test-0.1.8 → pl_e2e_test-0.2.0}/tests/test_regression.py +0 -0
  63. {pl_e2e_test-0.1.8 → pl_e2e_test-0.2.0}/tests/test_server.py +0 -0
  64. {pl_e2e_test-0.1.8 → pl_e2e_test-0.2.0}/tests/test_sharding.py +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: pl-e2e-test
3
- Version: 0.1.8
3
+ Version: 0.2.0
4
4
  Summary: Fuzz PrairieLearn question lifecycles against the real Python element runtime
5
5
  License-Expression: MIT
6
6
  Project-URL: Homepage, https://github.com/PrairieLearn/pl-e2e-test
@@ -38,6 +38,8 @@ The package installs the reusable `prairielearn_e2e` Python API and a single
38
38
  - `pl-e2e-test parallel` runs local shards and merges their results.
39
39
  - `pl-e2e-test add-seed` records a failing concrete variant seed.
40
40
  - `pl-e2e-test reports` merges parallel shard results into terminal and JUnit reports.
41
+ - `pl-e2e-test instances` shows configured, installed, cached, and local PrairieLearn
42
+ revisions.
41
43
  - `pl-e2e-test version` prints the installed version.
42
44
  - `pl-e2e-test help [COMMAND]` prints top-level help, or one command's help.
43
45
 
@@ -45,6 +47,7 @@ The package installs the reusable `prairielearn_e2e` Python API and a single
45
47
 
46
48
  - Python 3.13 or newer
47
49
  - Git 2.30 or newer
50
+ - uv 0.9 or newer
48
51
 
49
52
  The harness depends on pytest, Pyright, Ruff, Chevron, JSON Schema, lxml, NumPy,
50
53
  and PrairieLearn. PrairieLearn's Python package is currently consumed from its Git
@@ -241,19 +244,34 @@ jobs:
241
244
  ```
242
245
 
243
246
  The course must provide `.prairielearn/schemas/infoQuestion.json`. The harness checks
244
- question metadata against that schema and uses the exact PrairieLearn revision
245
- installed in the uv environment. It stores a bare, partial PrairieLearn repository in
246
- the platform-standard user cache directory. The first use of a new Git source or
247
- revision requires network access; later runs reuse that exact revision offline. The
248
- uv lockfile, rather than the cache, decides which revision is tested.
247
+ question metadata against that schema and resolves the configured PrairieLearn version
248
+ before each run. The default, `latest`, resolves the Git source's remote `HEAD`; a full
249
+ commit, branch, or unambiguous tag can be selected instead. Moving selectors require
250
+ network access and never silently fall back to stale cache state.
251
+
252
+ The selected revision's Python package and core elements are always used together. The
253
+ harness keeps a blob-filtered bare repository in the platform-standard user cache and
254
+ hydrates only `apps/prairielearn/python` and `apps/prairielearn/elements`. It launches
255
+ the selected Python package through a frozen uv overlay, without changing the course's
256
+ `pyproject.toml`, `uv.lock`, or virtual environment. A previously hydrated exact commit
257
+ can be reused offline when its uv dependencies are also cached.
249
258
 
250
259
  To use an existing PrairieLearn worktree or bare repository instead, pass
251
260
  `--prairielearn-path /path/to/PrairieLearn`. The repository must already contain the
252
- installed revision. An explicit path is authoritative: the harness does not fetch into
253
- it or fall back to the managed cache. In either mode, only the pinned revision's Python
254
- element controllers are archived into the test run's temporary directory. Standard
255
- cache environment settings such as `XDG_CACHE_HOME` can relocate the managed cache;
256
- CI systems can persist that directory between runs if desired.
261
+ selected revision. An explicit path is authoritative: the harness resolves selectors
262
+ against its local refs, does not fetch into it, and does not fall back to the managed
263
+ cache. For a local repository, `latest` means its current `HEAD`.
264
+
265
+ Inspect available revisions without contacting remotes:
266
+
267
+ ```sh
268
+ uv run pl-e2e-test instances
269
+ ```
270
+
271
+ Add `--refresh` to resolve the configured moving selector and verify remote access.
272
+ The table distinguishes fully hydrated runtime revisions from legacy element-only cache
273
+ entries. Standard cache environment settings such as `XDG_CACHE_HOME` can relocate the
274
+ managed cache; CI systems can persist that directory between runs if desired.
257
275
 
258
276
  ### Harness configuration
259
277
 
@@ -261,11 +279,17 @@ Selector settings are read automatically from the course's `pyproject.toml`:
261
279
 
262
280
  ```toml
263
281
  [tool.pl-e2e-test]
282
+ prairielearnVersion = "latest"
264
283
  elementSolutionSelectors = ["pl-sketch"]
265
284
  nonGradableSelectors = ['pl-sketch[read-only="true"]', 'pl-sketch[read-only="1"]']
266
285
  ```
267
286
 
268
- These are the defaults. `elementSolutionSelectors` identifies elements that provide
287
+ These are the defaults. Omitting `prairielearnVersion` is equivalent to `"latest"`.
288
+ Use a full commit for reproducible or offline runs, or a branch/tag when a named moving
289
+ target is intentional. `--prairielearn-version SELECTOR` overrides the TOML value for
290
+ one invocation, and replay commands always print the resolved full commit.
291
+
292
+ `elementSolutionSelectors` identifies elements that provide
269
293
  solutions without a `correct_answers` entry. It only relaxes that entry requirement;
270
294
  it does not generate a test submission. `nonGradableSelectors` excludes display-only
271
295
  answers from grading checks. Each supplied list replaces its default; omitted settings
@@ -455,12 +479,16 @@ course's TOML settings, or `config=load_config("settings.toml")` for an explicit
455
479
  `load_config` and `HarnessConfig` are exported from `prairielearn_e2e`. Omitting
456
480
  `config` uses the built-in defaults.
457
481
 
482
+ The command-line runner and high-level `run_parallel` API perform version resolution
483
+ and uv bootstrapping. Manually constructed `PrairieLearnBackend` and `QuestionHarness`
484
+ objects continue to use the PrairieLearn package already loaded by their interpreter.
485
+
458
486
  ## Development
459
487
 
460
488
  This project uses uv 0.9 or newer:
461
489
 
462
490
  ```sh
463
- uv sync
491
+ make install
464
492
  make test
465
493
  make format
466
494
  make smoke-dist
@@ -472,44 +500,36 @@ into an isolated target, and imports it using the locked development dependencie
472
500
 
473
501
  ## Publishing
474
502
 
475
- There are three supported publishing paths. Choose one for a release; do not run the
476
- local and GitHub paths for the same version because PyPI versions are immutable.
477
-
478
- ### GitHub-hosted trusted publishing
479
-
480
503
  Configure the PyPI project to trust the `publish.yml` workflow in the GitHub `pypi`
481
- environment. From a clean branch, run:
504
+ environment. Releases must start from a clean `main` that is synchronized with
505
+ `origin/main`, with `gh` authenticated and the publishing workflow available. To bump
506
+ the current version, run one of:
482
507
 
483
508
  ```sh
484
- make publish-version VERSION=0.2.0
509
+ make publish patch
510
+ make publish minor
511
+ make publish major
485
512
  ```
486
513
 
487
- This installs Python 3.13, performs the same locked dependency sync used in CI, runs
488
- the full checks, updates `pyproject.toml` and `uv.lock`, builds and smoke-tests the
489
- distributions, creates a release commit and annotated `v0.2.0` tag, and atomically
490
- pushes the branch and tag. The tag starts the workflow on `ubuntu-latest`; the workflow
491
- rechecks the tag/version match, tests, builds, generates attestations, and publishes
492
- with PyPI trusted-publishing OIDC credentials. Use `REMOTE=name` for a different Git
493
- remote.
494
-
495
- If an unpushed commit already contains the requested version, use the guarded bypass:
514
+ If the synchronized commit already contains the intended, not-yet-tagged version, use:
496
515
 
497
516
  ```sh
498
- make publish-version VERSION=0.2.0 ALLOW_PREBUMPED_VERSION=1
517
+ make publish current
499
518
  ```
500
519
 
501
- ### Direct local publishing
502
-
503
- To publish the version already present in `pyproject.toml` directly from this machine,
504
- configure a uv-supported PyPI credential such as `UV_PUBLISH_TOKEN`, then run:
520
+ The release command runs the shared `make release-check` gate, updates
521
+ `pyproject.toml` and `uv.lock` when needed, creates a release commit and annotated tag,
522
+ atomically pushes `main` and the tag, and watches the trusted-publishing workflow to
523
+ completion. Preview the full process without changing files, refs, remotes, or workflows
524
+ with:
505
525
 
506
526
  ```sh
507
- make publish-local
527
+ make publish patch DRY_RUN=1
508
528
  ```
509
529
 
510
- This runs all checks, builds and smoke-tests the current wheel and source archive, and
511
- passes only those two version-matched files to `uv publish`. It does not create a Git
512
- commit or tag.
530
+ If the release tag exists but trusted publishing needs to be dispatched again, run
531
+ `make publish-pypi`. This validates that the tag matches the current version and is
532
+ reachable from `origin/main`, then dispatches and watches a new workflow run.
513
533
 
514
534
  ### Local self-hosted GitHub runner
515
535
 
@@ -518,11 +538,11 @@ inside Docker. The container has no host mounts and does not receive the Docker
518
538
  Authenticate `gh` with repository administration permission, then run:
519
539
 
520
540
  ```sh
521
- make runner-start
522
- make publish-self-hosted RELEASE_TAG=v0.2.0
541
+ make publish-self-hosted
523
542
  ```
524
543
 
525
- The tag must already exist. This dispatches `publish.yml` from `main` with the
526
- `self-hosted` runner choice. `make runner-ensure` starts Docker Desktop on macOS when
527
- needed and repairs the runner before dispatch. Other lifecycle commands are
528
- `runner-status`, `runner-stop`, `runner-remove`, `runner-logs`, and `clean-runner`.
544
+ The current version's tag must already exist. This ensures the local runner is online,
545
+ dispatches `publish.yml` from `main` with the `self-hosted` runner choice, and watches
546
+ the run. `make runner-ensure` starts Docker Desktop on macOS when needed and repairs the
547
+ runner before dispatch. Other lifecycle commands are `runner-status`, `runner-stop`,
548
+ `runner-remove`, `runner-logs`, and `clean-runner`.
@@ -14,6 +14,8 @@ The package installs the reusable `prairielearn_e2e` Python API and a single
14
14
  - `pl-e2e-test parallel` runs local shards and merges their results.
15
15
  - `pl-e2e-test add-seed` records a failing concrete variant seed.
16
16
  - `pl-e2e-test reports` merges parallel shard results into terminal and JUnit reports.
17
+ - `pl-e2e-test instances` shows configured, installed, cached, and local PrairieLearn
18
+ revisions.
17
19
  - `pl-e2e-test version` prints the installed version.
18
20
  - `pl-e2e-test help [COMMAND]` prints top-level help, or one command's help.
19
21
 
@@ -21,6 +23,7 @@ The package installs the reusable `prairielearn_e2e` Python API and a single
21
23
 
22
24
  - Python 3.13 or newer
23
25
  - Git 2.30 or newer
26
+ - uv 0.9 or newer
24
27
 
25
28
  The harness depends on pytest, Pyright, Ruff, Chevron, JSON Schema, lxml, NumPy,
26
29
  and PrairieLearn. PrairieLearn's Python package is currently consumed from its Git
@@ -217,19 +220,34 @@ jobs:
217
220
  ```
218
221
 
219
222
  The course must provide `.prairielearn/schemas/infoQuestion.json`. The harness checks
220
- question metadata against that schema and uses the exact PrairieLearn revision
221
- installed in the uv environment. It stores a bare, partial PrairieLearn repository in
222
- the platform-standard user cache directory. The first use of a new Git source or
223
- revision requires network access; later runs reuse that exact revision offline. The
224
- uv lockfile, rather than the cache, decides which revision is tested.
223
+ question metadata against that schema and resolves the configured PrairieLearn version
224
+ before each run. The default, `latest`, resolves the Git source's remote `HEAD`; a full
225
+ commit, branch, or unambiguous tag can be selected instead. Moving selectors require
226
+ network access and never silently fall back to stale cache state.
227
+
228
+ The selected revision's Python package and core elements are always used together. The
229
+ harness keeps a blob-filtered bare repository in the platform-standard user cache and
230
+ hydrates only `apps/prairielearn/python` and `apps/prairielearn/elements`. It launches
231
+ the selected Python package through a frozen uv overlay, without changing the course's
232
+ `pyproject.toml`, `uv.lock`, or virtual environment. A previously hydrated exact commit
233
+ can be reused offline when its uv dependencies are also cached.
225
234
 
226
235
  To use an existing PrairieLearn worktree or bare repository instead, pass
227
236
  `--prairielearn-path /path/to/PrairieLearn`. The repository must already contain the
228
- installed revision. An explicit path is authoritative: the harness does not fetch into
229
- it or fall back to the managed cache. In either mode, only the pinned revision's Python
230
- element controllers are archived into the test run's temporary directory. Standard
231
- cache environment settings such as `XDG_CACHE_HOME` can relocate the managed cache;
232
- CI systems can persist that directory between runs if desired.
237
+ selected revision. An explicit path is authoritative: the harness resolves selectors
238
+ against its local refs, does not fetch into it, and does not fall back to the managed
239
+ cache. For a local repository, `latest` means its current `HEAD`.
240
+
241
+ Inspect available revisions without contacting remotes:
242
+
243
+ ```sh
244
+ uv run pl-e2e-test instances
245
+ ```
246
+
247
+ Add `--refresh` to resolve the configured moving selector and verify remote access.
248
+ The table distinguishes fully hydrated runtime revisions from legacy element-only cache
249
+ entries. Standard cache environment settings such as `XDG_CACHE_HOME` can relocate the
250
+ managed cache; CI systems can persist that directory between runs if desired.
233
251
 
234
252
  ### Harness configuration
235
253
 
@@ -237,11 +255,17 @@ Selector settings are read automatically from the course's `pyproject.toml`:
237
255
 
238
256
  ```toml
239
257
  [tool.pl-e2e-test]
258
+ prairielearnVersion = "latest"
240
259
  elementSolutionSelectors = ["pl-sketch"]
241
260
  nonGradableSelectors = ['pl-sketch[read-only="true"]', 'pl-sketch[read-only="1"]']
242
261
  ```
243
262
 
244
- These are the defaults. `elementSolutionSelectors` identifies elements that provide
263
+ These are the defaults. Omitting `prairielearnVersion` is equivalent to `"latest"`.
264
+ Use a full commit for reproducible or offline runs, or a branch/tag when a named moving
265
+ target is intentional. `--prairielearn-version SELECTOR` overrides the TOML value for
266
+ one invocation, and replay commands always print the resolved full commit.
267
+
268
+ `elementSolutionSelectors` identifies elements that provide
245
269
  solutions without a `correct_answers` entry. It only relaxes that entry requirement;
246
270
  it does not generate a test submission. `nonGradableSelectors` excludes display-only
247
271
  answers from grading checks. Each supplied list replaces its default; omitted settings
@@ -431,12 +455,16 @@ course's TOML settings, or `config=load_config("settings.toml")` for an explicit
431
455
  `load_config` and `HarnessConfig` are exported from `prairielearn_e2e`. Omitting
432
456
  `config` uses the built-in defaults.
433
457
 
458
+ The command-line runner and high-level `run_parallel` API perform version resolution
459
+ and uv bootstrapping. Manually constructed `PrairieLearnBackend` and `QuestionHarness`
460
+ objects continue to use the PrairieLearn package already loaded by their interpreter.
461
+
434
462
  ## Development
435
463
 
436
464
  This project uses uv 0.9 or newer:
437
465
 
438
466
  ```sh
439
- uv sync
467
+ make install
440
468
  make test
441
469
  make format
442
470
  make smoke-dist
@@ -448,44 +476,36 @@ into an isolated target, and imports it using the locked development dependencie
448
476
 
449
477
  ## Publishing
450
478
 
451
- There are three supported publishing paths. Choose one for a release; do not run the
452
- local and GitHub paths for the same version because PyPI versions are immutable.
453
-
454
- ### GitHub-hosted trusted publishing
455
-
456
479
  Configure the PyPI project to trust the `publish.yml` workflow in the GitHub `pypi`
457
- environment. From a clean branch, run:
480
+ environment. Releases must start from a clean `main` that is synchronized with
481
+ `origin/main`, with `gh` authenticated and the publishing workflow available. To bump
482
+ the current version, run one of:
458
483
 
459
484
  ```sh
460
- make publish-version VERSION=0.2.0
485
+ make publish patch
486
+ make publish minor
487
+ make publish major
461
488
  ```
462
489
 
463
- This installs Python 3.13, performs the same locked dependency sync used in CI, runs
464
- the full checks, updates `pyproject.toml` and `uv.lock`, builds and smoke-tests the
465
- distributions, creates a release commit and annotated `v0.2.0` tag, and atomically
466
- pushes the branch and tag. The tag starts the workflow on `ubuntu-latest`; the workflow
467
- rechecks the tag/version match, tests, builds, generates attestations, and publishes
468
- with PyPI trusted-publishing OIDC credentials. Use `REMOTE=name` for a different Git
469
- remote.
470
-
471
- If an unpushed commit already contains the requested version, use the guarded bypass:
490
+ If the synchronized commit already contains the intended, not-yet-tagged version, use:
472
491
 
473
492
  ```sh
474
- make publish-version VERSION=0.2.0 ALLOW_PREBUMPED_VERSION=1
493
+ make publish current
475
494
  ```
476
495
 
477
- ### Direct local publishing
478
-
479
- To publish the version already present in `pyproject.toml` directly from this machine,
480
- configure a uv-supported PyPI credential such as `UV_PUBLISH_TOKEN`, then run:
496
+ The release command runs the shared `make release-check` gate, updates
497
+ `pyproject.toml` and `uv.lock` when needed, creates a release commit and annotated tag,
498
+ atomically pushes `main` and the tag, and watches the trusted-publishing workflow to
499
+ completion. Preview the full process without changing files, refs, remotes, or workflows
500
+ with:
481
501
 
482
502
  ```sh
483
- make publish-local
503
+ make publish patch DRY_RUN=1
484
504
  ```
485
505
 
486
- This runs all checks, builds and smoke-tests the current wheel and source archive, and
487
- passes only those two version-matched files to `uv publish`. It does not create a Git
488
- commit or tag.
506
+ If the release tag exists but trusted publishing needs to be dispatched again, run
507
+ `make publish-pypi`. This validates that the tag matches the current version and is
508
+ reachable from `origin/main`, then dispatches and watches a new workflow run.
489
509
 
490
510
  ### Local self-hosted GitHub runner
491
511
 
@@ -494,11 +514,11 @@ inside Docker. The container has no host mounts and does not receive the Docker
494
514
  Authenticate `gh` with repository administration permission, then run:
495
515
 
496
516
  ```sh
497
- make runner-start
498
- make publish-self-hosted RELEASE_TAG=v0.2.0
517
+ make publish-self-hosted
499
518
  ```
500
519
 
501
- The tag must already exist. This dispatches `publish.yml` from `main` with the
502
- `self-hosted` runner choice. `make runner-ensure` starts Docker Desktop on macOS when
503
- needed and repairs the runner before dispatch. Other lifecycle commands are
504
- `runner-status`, `runner-stop`, `runner-remove`, `runner-logs`, and `clean-runner`.
520
+ The current version's tag must already exist. This ensures the local runner is online,
521
+ dispatches `publish.yml` from `main` with the `self-hosted` runner choice, and watches
522
+ the run. `make runner-ensure` starts Docker Desktop on macOS when needed and repairs the
523
+ runner before dispatch. Other lifecycle commands are `runner-status`, `runner-stop`,
524
+ `runner-remove`, `runner-logs`, and `clean-runner`.
@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
4
4
 
5
5
  [project]
6
6
  name = "pl-e2e-test"
7
- version = "0.1.8"
7
+ version = "0.2.0"
8
8
  description = "Fuzz PrairieLearn question lifecycles against the real Python element runtime"
9
9
  readme = "README.md"
10
10
  requires-python = ">=3.13"
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: pl-e2e-test
3
- Version: 0.1.8
3
+ Version: 0.2.0
4
4
  Summary: Fuzz PrairieLearn question lifecycles against the real Python element runtime
5
5
  License-Expression: MIT
6
6
  Project-URL: Homepage, https://github.com/PrairieLearn/pl-e2e-test
@@ -38,6 +38,8 @@ The package installs the reusable `prairielearn_e2e` Python API and a single
38
38
  - `pl-e2e-test parallel` runs local shards and merges their results.
39
39
  - `pl-e2e-test add-seed` records a failing concrete variant seed.
40
40
  - `pl-e2e-test reports` merges parallel shard results into terminal and JUnit reports.
41
+ - `pl-e2e-test instances` shows configured, installed, cached, and local PrairieLearn
42
+ revisions.
41
43
  - `pl-e2e-test version` prints the installed version.
42
44
  - `pl-e2e-test help [COMMAND]` prints top-level help, or one command's help.
43
45
 
@@ -45,6 +47,7 @@ The package installs the reusable `prairielearn_e2e` Python API and a single
45
47
 
46
48
  - Python 3.13 or newer
47
49
  - Git 2.30 or newer
50
+ - uv 0.9 or newer
48
51
 
49
52
  The harness depends on pytest, Pyright, Ruff, Chevron, JSON Schema, lxml, NumPy,
50
53
  and PrairieLearn. PrairieLearn's Python package is currently consumed from its Git
@@ -241,19 +244,34 @@ jobs:
241
244
  ```
242
245
 
243
246
  The course must provide `.prairielearn/schemas/infoQuestion.json`. The harness checks
244
- question metadata against that schema and uses the exact PrairieLearn revision
245
- installed in the uv environment. It stores a bare, partial PrairieLearn repository in
246
- the platform-standard user cache directory. The first use of a new Git source or
247
- revision requires network access; later runs reuse that exact revision offline. The
248
- uv lockfile, rather than the cache, decides which revision is tested.
247
+ question metadata against that schema and resolves the configured PrairieLearn version
248
+ before each run. The default, `latest`, resolves the Git source's remote `HEAD`; a full
249
+ commit, branch, or unambiguous tag can be selected instead. Moving selectors require
250
+ network access and never silently fall back to stale cache state.
251
+
252
+ The selected revision's Python package and core elements are always used together. The
253
+ harness keeps a blob-filtered bare repository in the platform-standard user cache and
254
+ hydrates only `apps/prairielearn/python` and `apps/prairielearn/elements`. It launches
255
+ the selected Python package through a frozen uv overlay, without changing the course's
256
+ `pyproject.toml`, `uv.lock`, or virtual environment. A previously hydrated exact commit
257
+ can be reused offline when its uv dependencies are also cached.
249
258
 
250
259
  To use an existing PrairieLearn worktree or bare repository instead, pass
251
260
  `--prairielearn-path /path/to/PrairieLearn`. The repository must already contain the
252
- installed revision. An explicit path is authoritative: the harness does not fetch into
253
- it or fall back to the managed cache. In either mode, only the pinned revision's Python
254
- element controllers are archived into the test run's temporary directory. Standard
255
- cache environment settings such as `XDG_CACHE_HOME` can relocate the managed cache;
256
- CI systems can persist that directory between runs if desired.
261
+ selected revision. An explicit path is authoritative: the harness resolves selectors
262
+ against its local refs, does not fetch into it, and does not fall back to the managed
263
+ cache. For a local repository, `latest` means its current `HEAD`.
264
+
265
+ Inspect available revisions without contacting remotes:
266
+
267
+ ```sh
268
+ uv run pl-e2e-test instances
269
+ ```
270
+
271
+ Add `--refresh` to resolve the configured moving selector and verify remote access.
272
+ The table distinguishes fully hydrated runtime revisions from legacy element-only cache
273
+ entries. Standard cache environment settings such as `XDG_CACHE_HOME` can relocate the
274
+ managed cache; CI systems can persist that directory between runs if desired.
257
275
 
258
276
  ### Harness configuration
259
277
 
@@ -261,11 +279,17 @@ Selector settings are read automatically from the course's `pyproject.toml`:
261
279
 
262
280
  ```toml
263
281
  [tool.pl-e2e-test]
282
+ prairielearnVersion = "latest"
264
283
  elementSolutionSelectors = ["pl-sketch"]
265
284
  nonGradableSelectors = ['pl-sketch[read-only="true"]', 'pl-sketch[read-only="1"]']
266
285
  ```
267
286
 
268
- These are the defaults. `elementSolutionSelectors` identifies elements that provide
287
+ These are the defaults. Omitting `prairielearnVersion` is equivalent to `"latest"`.
288
+ Use a full commit for reproducible or offline runs, or a branch/tag when a named moving
289
+ target is intentional. `--prairielearn-version SELECTOR` overrides the TOML value for
290
+ one invocation, and replay commands always print the resolved full commit.
291
+
292
+ `elementSolutionSelectors` identifies elements that provide
269
293
  solutions without a `correct_answers` entry. It only relaxes that entry requirement;
270
294
  it does not generate a test submission. `nonGradableSelectors` excludes display-only
271
295
  answers from grading checks. Each supplied list replaces its default; omitted settings
@@ -455,12 +479,16 @@ course's TOML settings, or `config=load_config("settings.toml")` for an explicit
455
479
  `load_config` and `HarnessConfig` are exported from `prairielearn_e2e`. Omitting
456
480
  `config` uses the built-in defaults.
457
481
 
482
+ The command-line runner and high-level `run_parallel` API perform version resolution
483
+ and uv bootstrapping. Manually constructed `PrairieLearnBackend` and `QuestionHarness`
484
+ objects continue to use the PrairieLearn package already loaded by their interpreter.
485
+
458
486
  ## Development
459
487
 
460
488
  This project uses uv 0.9 or newer:
461
489
 
462
490
  ```sh
463
- uv sync
491
+ make install
464
492
  make test
465
493
  make format
466
494
  make smoke-dist
@@ -472,44 +500,36 @@ into an isolated target, and imports it using the locked development dependencie
472
500
 
473
501
  ## Publishing
474
502
 
475
- There are three supported publishing paths. Choose one for a release; do not run the
476
- local and GitHub paths for the same version because PyPI versions are immutable.
477
-
478
- ### GitHub-hosted trusted publishing
479
-
480
503
  Configure the PyPI project to trust the `publish.yml` workflow in the GitHub `pypi`
481
- environment. From a clean branch, run:
504
+ environment. Releases must start from a clean `main` that is synchronized with
505
+ `origin/main`, with `gh` authenticated and the publishing workflow available. To bump
506
+ the current version, run one of:
482
507
 
483
508
  ```sh
484
- make publish-version VERSION=0.2.0
509
+ make publish patch
510
+ make publish minor
511
+ make publish major
485
512
  ```
486
513
 
487
- This installs Python 3.13, performs the same locked dependency sync used in CI, runs
488
- the full checks, updates `pyproject.toml` and `uv.lock`, builds and smoke-tests the
489
- distributions, creates a release commit and annotated `v0.2.0` tag, and atomically
490
- pushes the branch and tag. The tag starts the workflow on `ubuntu-latest`; the workflow
491
- rechecks the tag/version match, tests, builds, generates attestations, and publishes
492
- with PyPI trusted-publishing OIDC credentials. Use `REMOTE=name` for a different Git
493
- remote.
494
-
495
- If an unpushed commit already contains the requested version, use the guarded bypass:
514
+ If the synchronized commit already contains the intended, not-yet-tagged version, use:
496
515
 
497
516
  ```sh
498
- make publish-version VERSION=0.2.0 ALLOW_PREBUMPED_VERSION=1
517
+ make publish current
499
518
  ```
500
519
 
501
- ### Direct local publishing
502
-
503
- To publish the version already present in `pyproject.toml` directly from this machine,
504
- configure a uv-supported PyPI credential such as `UV_PUBLISH_TOKEN`, then run:
520
+ The release command runs the shared `make release-check` gate, updates
521
+ `pyproject.toml` and `uv.lock` when needed, creates a release commit and annotated tag,
522
+ atomically pushes `main` and the tag, and watches the trusted-publishing workflow to
523
+ completion. Preview the full process without changing files, refs, remotes, or workflows
524
+ with:
505
525
 
506
526
  ```sh
507
- make publish-local
527
+ make publish patch DRY_RUN=1
508
528
  ```
509
529
 
510
- This runs all checks, builds and smoke-tests the current wheel and source archive, and
511
- passes only those two version-matched files to `uv publish`. It does not create a Git
512
- commit or tag.
530
+ If the release tag exists but trusted publishing needs to be dispatched again, run
531
+ `make publish-pypi`. This validates that the tag matches the current version and is
532
+ reachable from `origin/main`, then dispatches and watches a new workflow run.
513
533
 
514
534
  ### Local self-hosted GitHub runner
515
535
 
@@ -518,11 +538,11 @@ inside Docker. The container has no host mounts and does not receive the Docker
518
538
  Authenticate `gh` with repository administration permission, then run:
519
539
 
520
540
  ```sh
521
- make runner-start
522
- make publish-self-hosted RELEASE_TAG=v0.2.0
541
+ make publish-self-hosted
523
542
  ```
524
543
 
525
- The tag must already exist. This dispatches `publish.yml` from `main` with the
526
- `self-hosted` runner choice. `make runner-ensure` starts Docker Desktop on macOS when
527
- needed and repairs the runner before dispatch. Other lifecycle commands are
528
- `runner-status`, `runner-stop`, `runner-remove`, `runner-logs`, and `clean-runner`.
544
+ The current version's tag must already exist. This ensures the local runner is online,
545
+ dispatches `publish.yml` from `main` with the `self-hosted` runner choice, and watches
546
+ the run. `make runner-ensure` starts Docker Desktop on macOS when needed and repairs the
547
+ runner before dispatch. Other lifecycle commands are `runner-status`, `runner-stop`,
548
+ `runner-remove`, `runner-logs`, and `clean-runner`.
@@ -21,6 +21,7 @@ src/prairielearn_e2e/expected_errors.py
21
21
  src/prairielearn_e2e/fuzz.py
22
22
  src/prairielearn_e2e/harness.py
23
23
  src/prairielearn_e2e/html_checks.py
24
+ src/prairielearn_e2e/instances.py
24
25
  src/prairielearn_e2e/metadata.py
25
26
  src/prairielearn_e2e/parallel.py
26
27
  src/prairielearn_e2e/pytest_plugin.py
@@ -29,6 +30,7 @@ src/prairielearn_e2e/regression.py
29
30
  src/prairielearn_e2e/reporting.py
30
31
  src/prairielearn_e2e/repository.py
31
32
  src/prairielearn_e2e/runner.py
33
+ src/prairielearn_e2e/runtime.py
32
34
  src/prairielearn_e2e/server.py
33
35
  src/prairielearn_e2e/sharding.py
34
36
  tests/__init__.py
@@ -42,6 +44,7 @@ tests/test_fuzz.py
42
44
  tests/test_git.py
43
45
  tests/test_harness.py
44
46
  tests/test_html_checks.py
47
+ tests/test_instances.py
45
48
  tests/test_metadata.py
46
49
  tests/test_parallel.py
47
50
  tests/test_publish_version.py
@@ -50,5 +53,6 @@ tests/test_question_tests.py
50
53
  tests/test_regression.py
51
54
  tests/test_reporting.py
52
55
  tests/test_repository.py
56
+ tests/test_runtime.py
53
57
  tests/test_server.py
54
58
  tests/test_sharding.py
@@ -0,0 +1,50 @@
1
+ """End-to-end contract tests for PrairieLearn question directories."""
2
+
3
+ from importlib import import_module
4
+ from importlib.metadata import version
5
+ from typing import TYPE_CHECKING, Any
6
+
7
+ if TYPE_CHECKING:
8
+ from .backend import PrairieLearnBackend
9
+ from .config import HarnessConfig, load_config
10
+ from .discovery import QuestionCase, discover_questions
11
+ from .harness import LifecycleResult, QuestionHarness
12
+ from .parallel import run_parallel
13
+
14
+ __version__ = version("pl-e2e-test")
15
+
16
+ __all__ = (
17
+ "HarnessConfig",
18
+ "LifecycleResult",
19
+ "PrairieLearnBackend",
20
+ "QuestionCase",
21
+ "QuestionHarness",
22
+ "__version__",
23
+ "discover_questions",
24
+ "load_config",
25
+ "run_parallel",
26
+ )
27
+
28
+
29
+ _LAZY_EXPORTS = {
30
+ "HarnessConfig": (".config", "HarnessConfig"),
31
+ "LifecycleResult": (".harness", "LifecycleResult"),
32
+ "PrairieLearnBackend": (".backend", "PrairieLearnBackend"),
33
+ "QuestionCase": (".discovery", "QuestionCase"),
34
+ "QuestionHarness": (".harness", "QuestionHarness"),
35
+ "discover_questions": (".discovery", "discover_questions"),
36
+ "load_config": (".config", "load_config"),
37
+ "run_parallel": (".parallel", "run_parallel"),
38
+ }
39
+
40
+
41
+ def __getattr__(name: str) -> Any:
42
+ """Load PrairieLearn-dependent public objects only after runtime bootstrap."""
43
+
44
+ try:
45
+ module_name, attribute = _LAZY_EXPORTS[name]
46
+ except KeyError as exc:
47
+ raise AttributeError(name) from exc
48
+ value = getattr(import_module(module_name, __name__), attribute)
49
+ globals()[name] = value
50
+ return value