plainproof 0.1.0__tar.gz → 0.1.1__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {plainproof-0.1.0 → plainproof-0.1.1}/CITATION.cff +7 -0
- {plainproof-0.1.0 → plainproof-0.1.1}/PKG-INFO +12 -8
- {plainproof-0.1.0 → plainproof-0.1.1}/README.md +11 -7
- {plainproof-0.1.0 → plainproof-0.1.1}/pyproject.toml +1 -1
- {plainproof-0.1.0 → plainproof-0.1.1}/scripts/fetch_heldout.sh +3 -2
- {plainproof-0.1.0 → plainproof-0.1.1}/src/plainproof/cli.py +1 -3
- {plainproof-0.1.0 → plainproof-0.1.1}/src/plainproof.egg-info/PKG-INFO +12 -8
- {plainproof-0.1.0 → plainproof-0.1.1}/.gitignore +0 -0
- {plainproof-0.1.0 → plainproof-0.1.1}/LICENSE +0 -0
- {plainproof-0.1.0 → plainproof-0.1.1}/LICENSE-docs.md +0 -0
- {plainproof-0.1.0 → plainproof-0.1.1}/MANIFEST.in +0 -0
- {plainproof-0.1.0 → plainproof-0.1.1}/data/edm-v2-e0/study/calibration-v2.json +0 -0
- {plainproof-0.1.0 → plainproof-0.1.1}/data/edm-v2-e0/study/calibration-v2.receipt.json +0 -0
- {plainproof-0.1.0 → plainproof-0.1.1}/data/edm-v2-e0/study/calibration.json +0 -0
- {plainproof-0.1.0 → plainproof-0.1.1}/data/edm-v2-e0/study/calibration.receipt.json +0 -0
- {plainproof-0.1.0 → plainproof-0.1.1}/data/edm-v2-e0/study/pilot.json +0 -0
- {plainproof-0.1.0 → plainproof-0.1.1}/data/edm-v2-e0/study/tune.json +0 -0
- {plainproof-0.1.0 → plainproof-0.1.1}/docs/PREREG_EDM_V2.md +0 -0
- {plainproof-0.1.0 → plainproof-0.1.1}/docs/PROTOCOL_E0.md +0 -0
- {plainproof-0.1.0 → plainproof-0.1.1}/reports/edm-v2-e0/error-vs-d.csv +0 -0
- {plainproof-0.1.0 → plainproof-0.1.1}/reports/edm-v2-e0/figures/error-vs-d.png +0 -0
- {plainproof-0.1.0 → plainproof-0.1.1}/reports/edm-v2-e0/freeze.json +0 -0
- {plainproof-0.1.0 → plainproof-0.1.1}/reports/edm-v2-e0/results-numbers.json +0 -0
- {plainproof-0.1.0 → plainproof-0.1.1}/reports/edm-v2-e0/signature-table.md +0 -0
- {plainproof-0.1.0 → plainproof-0.1.1}/reports/edm-v2-e0b/figures/probe-auc-vs-d.png +0 -0
- {plainproof-0.1.0 → plainproof-0.1.1}/reports/edm-v2-e0b/freeze.json +0 -0
- {plainproof-0.1.0 → plainproof-0.1.1}/reports/edm-v2-e0b/numbers.json +0 -0
- {plainproof-0.1.0 → plainproof-0.1.1}/reports/edm-v2-e0b/probe-auc-vs-d.csv +0 -0
- {plainproof-0.1.0 → plainproof-0.1.1}/reports/edm-v2-e0b/probe-table.md +0 -0
- {plainproof-0.1.0 → plainproof-0.1.1}/reports/receipts/assist2009-dev-2pl.receipt.json +0 -0
- {plainproof-0.1.0 → plainproof-0.1.1}/reports/receipts/bobcat-gate.json +0 -0
- {plainproof-0.1.0 → plainproof-0.1.1}/reports/receipts/bobcat-run-settings.json +0 -0
- {plainproof-0.1.0 → plainproof-0.1.1}/reports/receipts/catirt-install-receipt.txt +0 -0
- {plainproof-0.1.0 → plainproof-0.1.1}/requirements.txt +0 -0
- {plainproof-0.1.0 → plainproof-0.1.1}/scripts/bobcat/check_checkpoint.py +0 -0
- {plainproof-0.1.0 → plainproof-0.1.1}/scripts/bobcat/fetch_bobcat.sh +0 -0
- {plainproof-0.1.0 → plainproof-0.1.1}/scripts/bobcat/retrain.py +0 -0
- {plainproof-0.1.0 → plainproof-0.1.1}/scripts/edm_v2_e0/bobcat_gate.py +0 -0
- {plainproof-0.1.0 → plainproof-0.1.1}/scripts/edm_v2_e0/catirt_sprt_steps.R +0 -0
- {plainproof-0.1.0 → plainproof-0.1.1}/scripts/edm_v2_e0/fit_assist2009.py +0 -0
- {plainproof-0.1.0 → plainproof-0.1.1}/scripts/edm_v2_e0/heldout.py +0 -0
- {plainproof-0.1.0 → plainproof-0.1.1}/scripts/edm_v2_e0/install_catirt.R +0 -0
- {plainproof-0.1.0 → plainproof-0.1.1}/scripts/edm_v2_e0/pilot.py +0 -0
- {plainproof-0.1.0 → plainproof-0.1.1}/scripts/edm_v2_e0/plot_error_vs_d.R +0 -0
- {plainproof-0.1.0 → plainproof-0.1.1}/scripts/edm_v2_e0/r_env.sh +0 -0
- {plainproof-0.1.0 → plainproof-0.1.1}/scripts/edm_v2_e0/report.py +0 -0
- {plainproof-0.1.0 → plainproof-0.1.1}/scripts/edm_v2_e0/repro_paths.py +0 -0
- {plainproof-0.1.0 → plainproof-0.1.1}/scripts/edm_v2_e0/study.py +0 -0
- {plainproof-0.1.0 → plainproof-0.1.1}/scripts/edm_v2_e0b/plot_probe_auc.R +0 -0
- {plainproof-0.1.0 → plainproof-0.1.1}/scripts/edm_v2_e0b/report.py +0 -0
- {plainproof-0.1.0 → plainproof-0.1.1}/scripts/edm_v2_e0b/run.py +0 -0
- {plainproof-0.1.0 → plainproof-0.1.1}/scripts/published_cat_reference_v1/bobcat_support.py +0 -0
- {plainproof-0.1.0 → plainproof-0.1.1}/scripts/published_cat_reference_v1/compare_selection_v1.py +0 -0
- {plainproof-0.1.0 → plainproof-0.1.1}/scripts/rscript.sh +0 -0
- {plainproof-0.1.0 → plainproof-0.1.1}/scripts/verify_frozen.py +0 -0
- {plainproof-0.1.0 → plainproof-0.1.1}/setup.cfg +0 -0
- {plainproof-0.1.0 → plainproof-0.1.1}/src/plainproof/__init__.py +0 -0
- {plainproof-0.1.0 → plainproof-0.1.1}/src/plainproof/bank_from_fit.py +0 -0
- {plainproof-0.1.0 → plainproof-0.1.1}/src/plainproof/fit.py +0 -0
- {plainproof-0.1.0 → plainproof-0.1.1}/src/plainproof/irt.py +0 -0
- {plainproof-0.1.0 → plainproof-0.1.1}/src/plainproof/planted.py +0 -0
- {plainproof-0.1.0 → plainproof-0.1.1}/src/plainproof/reliability.py +0 -0
- {plainproof-0.1.0 → plainproof-0.1.1}/src/plainproof/signature.py +0 -0
- {plainproof-0.1.0 → plainproof-0.1.1}/src/plainproof/simulator.py +0 -0
- {plainproof-0.1.0 → plainproof-0.1.1}/src/plainproof/sprt.py +0 -0
- {plainproof-0.1.0 → plainproof-0.1.1}/src/plainproof/stopping.py +0 -0
- {plainproof-0.1.0 → plainproof-0.1.1}/src/plainproof.egg-info/SOURCES.txt +0 -0
- {plainproof-0.1.0 → plainproof-0.1.1}/src/plainproof.egg-info/dependency_links.txt +0 -0
- {plainproof-0.1.0 → plainproof-0.1.1}/src/plainproof.egg-info/entry_points.txt +0 -0
- {plainproof-0.1.0 → plainproof-0.1.1}/src/plainproof.egg-info/requires.txt +0 -0
- {plainproof-0.1.0 → plainproof-0.1.1}/src/plainproof.egg-info/top_level.txt +0 -0
- {plainproof-0.1.0 → plainproof-0.1.1}/substitutions.json +0 -0
- {plainproof-0.1.0 → plainproof-0.1.1}/tests/conftest.py +0 -0
- {plainproof-0.1.0 → plainproof-0.1.1}/tests/fixtures/edm_v2_catirt_sprt/bank.csv +0 -0
- {plainproof-0.1.0 → plainproof-0.1.1}/tests/fixtures/edm_v2_catirt_sprt/config.csv +0 -0
- {plainproof-0.1.0 → plainproof-0.1.1}/tests/fixtures/edm_v2_catirt_sprt/resp.csv +0 -0
- {plainproof-0.1.0 → plainproof-0.1.1}/tests/fixtures/edm_v2_catirt_sprt/steps.csv +0 -0
- {plainproof-0.1.0 → plainproof-0.1.1}/tests/fixtures/edm_v2_catirt_sprt/terminal.csv +0 -0
- {plainproof-0.1.0 → plainproof-0.1.1}/tests/test_edm_v2_e0.py +0 -0
- {plainproof-0.1.0 → plainproof-0.1.1}/tests/test_edm_v2_e0b.py +0 -0
|
@@ -5,3 +5,10 @@ authors:
|
|
|
5
5
|
- family-names: Vettrivel
|
|
6
6
|
given-names: Vishnu
|
|
7
7
|
license: MIT
|
|
8
|
+
version: 0.1.1
|
|
9
|
+
doi: 10.5281/zenodo.23647649
|
|
10
|
+
identifiers:
|
|
11
|
+
- type: doi
|
|
12
|
+
value: 10.5281/zenodo.23647650
|
|
13
|
+
description: version 0.1.0
|
|
14
|
+
repository-code: https://github.com/cloudronin/plainproof
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: plainproof
|
|
3
|
-
Version: 0.1.
|
|
3
|
+
Version: 0.1.1
|
|
4
4
|
Summary: Reproduction tool for the paper 'How many follow-up questions distinguish genuine understanding from apparent mastery?'. Not a general library. Its sealed checks can stop a run on purpose.
|
|
5
5
|
Author: Vishnu Vettrivel
|
|
6
6
|
License-Expression: MIT
|
|
@@ -19,6 +19,10 @@ Dynamic: license-file
|
|
|
19
19
|
|
|
20
20
|
# plainproof
|
|
21
21
|
|
|
22
|
+
[](https://doi.org/10.5281/zenodo.23647650)
|
|
23
|
+
|
|
24
|
+
Archived on Zenodo: version 0.1.0 is DOI [10.5281/zenodo.23647650](https://doi.org/10.5281/zenodo.23647650); all versions are DOI [10.5281/zenodo.23647649](https://doi.org/10.5281/zenodo.23647649). Also on PyPI: `pip install plainproof`.
|
|
25
|
+
|
|
22
26
|
Reproduction package for **"How many follow-up questions distinguish genuine understanding from apparent mastery?"** (EDM 2027 submission, simulation study). It asks how many adaptive follow-up questions a classification test needs to tell genuine understanding from apparent mastery at a calibrated confidence target, and how that changes as a learner outsources a share *d* of the follow-ups to an AI. A second, sealed study (E0b) tests whether planted hard probe items separate outsourcing learners from honest ones.
|
|
23
27
|
|
|
24
28
|
Everything here is CPU-only and needs no paid service. Code: MIT (`LICENSE`). Docs, figures and tables: CC BY 4.0 (`LICENSE-docs.md`).
|
|
@@ -37,8 +41,8 @@ python -m pytest -n auto -q tests
|
|
|
37
41
|
# 2. Check that the code here is the sealed code (undoes the documented path substitutions, then compares sha256)
|
|
38
42
|
plainproof verify
|
|
39
43
|
|
|
40
|
-
# 3. Fetch the sealed held-out outputs from
|
|
41
|
-
|
|
44
|
+
# 3. Fetch the sealed held-out outputs (from the v0.1.0 release assets) and check their hashes
|
|
45
|
+
plainproof fetch heldout
|
|
42
46
|
|
|
43
47
|
# 4. Rebuild the tables and figures from the held-out summaries (needs R 4.x for the plots; base graphics only)
|
|
44
48
|
python scripts/edm_v2_e0/report.py data/edm-v2-e0/heldout/summary.json
|
|
@@ -67,7 +71,7 @@ python scripts/edm_v2_e0/heldout.py run data/edm-v2-e0/heldout && python scripts
|
|
|
67
71
|
python scripts/edm_v2_e0b/run.py run data/edm-v2-e0b/heldout && python scripts/edm_v2_e0b/run.py summarize data/edm-v2-e0b/heldout
|
|
68
72
|
```
|
|
69
73
|
|
|
70
|
-
Two caveats, stated plainly. The sealed code checks the item fit's sha256, so a different numpy can stop step b; and it asserts that the BOBCAT checkpoint's best epoch is 105, so a retrain that is not bit-identical can stop step d. In either case, use the
|
|
74
|
+
Two caveats, stated plainly. The sealed code checks the item fit's sha256, so a different numpy can stop step b; and it asserts that the BOBCAT checkpoint's best epoch is 105, so a retrain that is not bit-identical can stop step d. In either case, use the published held-out outputs (`plainproof fetch heldout`). The development stages (`study.py tune|calibrate|calibrate_sprt_full|pilot`) are included for the record; their outputs (synthetic learners only) are in `data/edm-v2-e0/study/`.
|
|
71
75
|
|
|
72
76
|
## Fingerprints (sha256)
|
|
73
77
|
|
|
@@ -75,10 +79,10 @@ Two caveats, stated plainly. The sealed code checks the item fit's sha256, so a
|
|
|
75
79
|
|---|---|
|
|
76
80
|
| E0 sealed freeze `reports/edm-v2-e0/freeze.json` | `a0f8ce404f2b5f7e671e2503a33b4554040e80714f6053553507e0972acdbcdc` |
|
|
77
81
|
| E0b sealed freeze `reports/edm-v2-e0b/freeze.json` | `3fa0b8d457d813f59abd5255f9303e44deb2119ee79d6a9b92439a6c1cbf58ed` |
|
|
78
|
-
| E0 held-out summary (
|
|
79
|
-
| E0b held-out summary (
|
|
80
|
-
|
|
|
81
|
-
|
|
|
82
|
+
| E0 held-out summary (`e0-heldout.tar` → `heldout/summary.json`) | `aff9a8c49d48821c0e9343cd058c4e0e2f1659ffabcaa719824412f3b14abc90` |
|
|
83
|
+
| E0b held-out summary (`e0b-heldout.tar` → `heldout/summary.json`) | `617be427feac24df2e342da00d34e91c78d5de91592cee390ef3516ae9e2ff3f` |
|
|
84
|
+
| `e0-heldout.tar` (release asset) | `bd06a11eb7163da102cad6ea0df12ea79e52ff94aab1dc35d28ec3910af2be22` |
|
|
85
|
+
| `e0b-heldout.tar` (release asset) | `68219f228c41da972fe4c8f92c218e01f85f08eed864185f237cecda05314bbf` |
|
|
82
86
|
| ASSISTments 2009 file at BOBCAT e6b6245e | `c07ca9fa0f08207b06488cec527725938fe96e81e3c1762839e1ad0b84f974cc` |
|
|
83
87
|
| 2PL item fit (regenerated in step b) | `12a3b167cd9c9b78b3c3c7faf71b2e464b0943a454f6418faf0f99b1114cb7fc` |
|
|
84
88
|
| catIrt 0.5.1 CRAN binary used for the fixture | `f5418f447e91d184791e668bd2a135ea5da40c981d4a76826018295b6ea9f079` |
|
|
@@ -1,5 +1,9 @@
|
|
|
1
1
|
# plainproof
|
|
2
2
|
|
|
3
|
+
[](https://doi.org/10.5281/zenodo.23647650)
|
|
4
|
+
|
|
5
|
+
Archived on Zenodo: version 0.1.0 is DOI [10.5281/zenodo.23647650](https://doi.org/10.5281/zenodo.23647650); all versions are DOI [10.5281/zenodo.23647649](https://doi.org/10.5281/zenodo.23647649). Also on PyPI: `pip install plainproof`.
|
|
6
|
+
|
|
3
7
|
Reproduction package for **"How many follow-up questions distinguish genuine understanding from apparent mastery?"** (EDM 2027 submission, simulation study). It asks how many adaptive follow-up questions a classification test needs to tell genuine understanding from apparent mastery at a calibrated confidence target, and how that changes as a learner outsources a share *d* of the follow-ups to an AI. A second, sealed study (E0b) tests whether planted hard probe items separate outsourcing learners from honest ones.
|
|
4
8
|
|
|
5
9
|
Everything here is CPU-only and needs no paid service. Code: MIT (`LICENSE`). Docs, figures and tables: CC BY 4.0 (`LICENSE-docs.md`).
|
|
@@ -18,8 +22,8 @@ python -m pytest -n auto -q tests
|
|
|
18
22
|
# 2. Check that the code here is the sealed code (undoes the documented path substitutions, then compares sha256)
|
|
19
23
|
plainproof verify
|
|
20
24
|
|
|
21
|
-
# 3. Fetch the sealed held-out outputs from
|
|
22
|
-
|
|
25
|
+
# 3. Fetch the sealed held-out outputs (from the v0.1.0 release assets) and check their hashes
|
|
26
|
+
plainproof fetch heldout
|
|
23
27
|
|
|
24
28
|
# 4. Rebuild the tables and figures from the held-out summaries (needs R 4.x for the plots; base graphics only)
|
|
25
29
|
python scripts/edm_v2_e0/report.py data/edm-v2-e0/heldout/summary.json
|
|
@@ -48,7 +52,7 @@ python scripts/edm_v2_e0/heldout.py run data/edm-v2-e0/heldout && python scripts
|
|
|
48
52
|
python scripts/edm_v2_e0b/run.py run data/edm-v2-e0b/heldout && python scripts/edm_v2_e0b/run.py summarize data/edm-v2-e0b/heldout
|
|
49
53
|
```
|
|
50
54
|
|
|
51
|
-
Two caveats, stated plainly. The sealed code checks the item fit's sha256, so a different numpy can stop step b; and it asserts that the BOBCAT checkpoint's best epoch is 105, so a retrain that is not bit-identical can stop step d. In either case, use the
|
|
55
|
+
Two caveats, stated plainly. The sealed code checks the item fit's sha256, so a different numpy can stop step b; and it asserts that the BOBCAT checkpoint's best epoch is 105, so a retrain that is not bit-identical can stop step d. In either case, use the published held-out outputs (`plainproof fetch heldout`). The development stages (`study.py tune|calibrate|calibrate_sprt_full|pilot`) are included for the record; their outputs (synthetic learners only) are in `data/edm-v2-e0/study/`.
|
|
52
56
|
|
|
53
57
|
## Fingerprints (sha256)
|
|
54
58
|
|
|
@@ -56,10 +60,10 @@ Two caveats, stated plainly. The sealed code checks the item fit's sha256, so a
|
|
|
56
60
|
|---|---|
|
|
57
61
|
| E0 sealed freeze `reports/edm-v2-e0/freeze.json` | `a0f8ce404f2b5f7e671e2503a33b4554040e80714f6053553507e0972acdbcdc` |
|
|
58
62
|
| E0b sealed freeze `reports/edm-v2-e0b/freeze.json` | `3fa0b8d457d813f59abd5255f9303e44deb2119ee79d6a9b92439a6c1cbf58ed` |
|
|
59
|
-
| E0 held-out summary (
|
|
60
|
-
| E0b held-out summary (
|
|
61
|
-
|
|
|
62
|
-
|
|
|
63
|
+
| E0 held-out summary (`e0-heldout.tar` → `heldout/summary.json`) | `aff9a8c49d48821c0e9343cd058c4e0e2f1659ffabcaa719824412f3b14abc90` |
|
|
64
|
+
| E0b held-out summary (`e0b-heldout.tar` → `heldout/summary.json`) | `617be427feac24df2e342da00d34e91c78d5de91592cee390ef3516ae9e2ff3f` |
|
|
65
|
+
| `e0-heldout.tar` (release asset) | `bd06a11eb7163da102cad6ea0df12ea79e52ff94aab1dc35d28ec3910af2be22` |
|
|
66
|
+
| `e0b-heldout.tar` (release asset) | `68219f228c41da972fe4c8f92c218e01f85f08eed864185f237cecda05314bbf` |
|
|
63
67
|
| ASSISTments 2009 file at BOBCAT e6b6245e | `c07ca9fa0f08207b06488cec527725938fe96e81e3c1762839e1ad0b84f974cc` |
|
|
64
68
|
| 2PL item fit (regenerated in step b) | `12a3b167cd9c9b78b3c3c7faf71b2e464b0943a454f6418faf0f99b1114cb7fc` |
|
|
65
69
|
| catIrt 0.5.1 CRAN binary used for the fixture | `f5418f447e91d184791e668bd2a135ea5da40c981d4a76826018295b6ea9f079` |
|
|
@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
|
|
|
4
4
|
|
|
5
5
|
[project]
|
|
6
6
|
name = "plainproof"
|
|
7
|
-
version = "0.1.
|
|
7
|
+
version = "0.1.1"
|
|
8
8
|
description = "Reproduction tool for the paper 'How many follow-up questions distinguish genuine understanding from apparent mastery?'. Not a general library. Its sealed checks can stop a run on purpose."
|
|
9
9
|
readme = "README.md"
|
|
10
10
|
license = "MIT"
|
|
@@ -1,7 +1,8 @@
|
|
|
1
1
|
#!/bin/bash
|
|
2
|
-
# Download the sealed held-out outputs
|
|
2
|
+
# Download the sealed held-out outputs and check their sha256 before use. Default source: the v0.1.0 GitHub release assets
|
|
3
|
+
# (cited by DOI 10.5281/zenodo.23647650); set ZENODO_URL (or any mirror base URL) to fetch from elsewhere.
|
|
3
4
|
set -euo pipefail
|
|
4
|
-
DOI_URL=${ZENODO_URL
|
|
5
|
+
DOI_URL=${ZENODO_URL:-https://github.com/cloudronin/plainproof/releases/download/v0.1.0}
|
|
5
6
|
D=${REPRO_DATA:-data}; mkdir -p "$D/edm-v2-e0" "$D/edm-v2-e0b"
|
|
6
7
|
for f in e0-heldout.tar e0b-heldout.tar; do curl -fsSL -o "$D/$f" "$DOI_URL/$f"; done
|
|
7
8
|
cd "$D" && printf '%s %s\n%s %s\n' bd06a11eb7163da102cad6ea0df12ea79e52ff94aab1dc35d28ec3910af2be22 e0-heldout.tar 68219f228c41da972fe4c8f92c218e01f85f08eed864185f237cecda05314bbf e0b-heldout.tar | shasum -a 256 -c
|
|
@@ -3,7 +3,7 @@
|
|
|
3
3
|
It runs the repository's own scripts, so it needs a checkout or the unpacked sdist (the sealed scripts locate the
|
|
4
4
|
repository by its layout). Pass --root, or run it from the repository root.
|
|
5
5
|
|
|
6
|
-
plainproof fetch [bobcat|heldout|all] fetch BOBCAT + the ASSISTments file
|
|
6
|
+
plainproof fetch [bobcat|heldout|all] fetch BOBCAT + the ASSISTments file, and/or the held-out outputs (all hash-checked)
|
|
7
7
|
plainproof fit refit the 2PL item parameters on ASSISTments development learners
|
|
8
8
|
plainproof run e0|e0b held-out run and summary (refuses to overwrite an existing cell)
|
|
9
9
|
plainproof figures rebuild tables and figures from the held-out summaries (needs R for the plots)
|
|
@@ -54,8 +54,6 @@ def main(argv=None):
|
|
|
54
54
|
if a.what in ('bobcat', 'all'):
|
|
55
55
|
sh(root, 'bash', 'scripts/bobcat/fetch_bobcat.sh')
|
|
56
56
|
if a.what in ('heldout', 'all'):
|
|
57
|
-
if not os.environ.get('ZENODO_URL'):
|
|
58
|
-
raise SystemExit('set ZENODO_URL to the Zenodo record file URL to fetch the held-out outputs')
|
|
59
57
|
sh(root, 'bash', 'scripts/fetch_heldout.sh')
|
|
60
58
|
elif a.cmd == 'fit':
|
|
61
59
|
sh(root, py, 'scripts/edm_v2_e0/fit_assist2009.py', data / 'edm-v2-e0' / 'item-fit')
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: plainproof
|
|
3
|
-
Version: 0.1.
|
|
3
|
+
Version: 0.1.1
|
|
4
4
|
Summary: Reproduction tool for the paper 'How many follow-up questions distinguish genuine understanding from apparent mastery?'. Not a general library. Its sealed checks can stop a run on purpose.
|
|
5
5
|
Author: Vishnu Vettrivel
|
|
6
6
|
License-Expression: MIT
|
|
@@ -19,6 +19,10 @@ Dynamic: license-file
|
|
|
19
19
|
|
|
20
20
|
# plainproof
|
|
21
21
|
|
|
22
|
+
[](https://doi.org/10.5281/zenodo.23647650)
|
|
23
|
+
|
|
24
|
+
Archived on Zenodo: version 0.1.0 is DOI [10.5281/zenodo.23647650](https://doi.org/10.5281/zenodo.23647650); all versions are DOI [10.5281/zenodo.23647649](https://doi.org/10.5281/zenodo.23647649). Also on PyPI: `pip install plainproof`.
|
|
25
|
+
|
|
22
26
|
Reproduction package for **"How many follow-up questions distinguish genuine understanding from apparent mastery?"** (EDM 2027 submission, simulation study). It asks how many adaptive follow-up questions a classification test needs to tell genuine understanding from apparent mastery at a calibrated confidence target, and how that changes as a learner outsources a share *d* of the follow-ups to an AI. A second, sealed study (E0b) tests whether planted hard probe items separate outsourcing learners from honest ones.
|
|
23
27
|
|
|
24
28
|
Everything here is CPU-only and needs no paid service. Code: MIT (`LICENSE`). Docs, figures and tables: CC BY 4.0 (`LICENSE-docs.md`).
|
|
@@ -37,8 +41,8 @@ python -m pytest -n auto -q tests
|
|
|
37
41
|
# 2. Check that the code here is the sealed code (undoes the documented path substitutions, then compares sha256)
|
|
38
42
|
plainproof verify
|
|
39
43
|
|
|
40
|
-
# 3. Fetch the sealed held-out outputs from
|
|
41
|
-
|
|
44
|
+
# 3. Fetch the sealed held-out outputs (from the v0.1.0 release assets) and check their hashes
|
|
45
|
+
plainproof fetch heldout
|
|
42
46
|
|
|
43
47
|
# 4. Rebuild the tables and figures from the held-out summaries (needs R 4.x for the plots; base graphics only)
|
|
44
48
|
python scripts/edm_v2_e0/report.py data/edm-v2-e0/heldout/summary.json
|
|
@@ -67,7 +71,7 @@ python scripts/edm_v2_e0/heldout.py run data/edm-v2-e0/heldout && python scripts
|
|
|
67
71
|
python scripts/edm_v2_e0b/run.py run data/edm-v2-e0b/heldout && python scripts/edm_v2_e0b/run.py summarize data/edm-v2-e0b/heldout
|
|
68
72
|
```
|
|
69
73
|
|
|
70
|
-
Two caveats, stated plainly. The sealed code checks the item fit's sha256, so a different numpy can stop step b; and it asserts that the BOBCAT checkpoint's best epoch is 105, so a retrain that is not bit-identical can stop step d. In either case, use the
|
|
74
|
+
Two caveats, stated plainly. The sealed code checks the item fit's sha256, so a different numpy can stop step b; and it asserts that the BOBCAT checkpoint's best epoch is 105, so a retrain that is not bit-identical can stop step d. In either case, use the published held-out outputs (`plainproof fetch heldout`). The development stages (`study.py tune|calibrate|calibrate_sprt_full|pilot`) are included for the record; their outputs (synthetic learners only) are in `data/edm-v2-e0/study/`.
|
|
71
75
|
|
|
72
76
|
## Fingerprints (sha256)
|
|
73
77
|
|
|
@@ -75,10 +79,10 @@ Two caveats, stated plainly. The sealed code checks the item fit's sha256, so a
|
|
|
75
79
|
|---|---|
|
|
76
80
|
| E0 sealed freeze `reports/edm-v2-e0/freeze.json` | `a0f8ce404f2b5f7e671e2503a33b4554040e80714f6053553507e0972acdbcdc` |
|
|
77
81
|
| E0b sealed freeze `reports/edm-v2-e0b/freeze.json` | `3fa0b8d457d813f59abd5255f9303e44deb2119ee79d6a9b92439a6c1cbf58ed` |
|
|
78
|
-
| E0 held-out summary (
|
|
79
|
-
| E0b held-out summary (
|
|
80
|
-
|
|
|
81
|
-
|
|
|
82
|
+
| E0 held-out summary (`e0-heldout.tar` → `heldout/summary.json`) | `aff9a8c49d48821c0e9343cd058c4e0e2f1659ffabcaa719824412f3b14abc90` |
|
|
83
|
+
| E0b held-out summary (`e0b-heldout.tar` → `heldout/summary.json`) | `617be427feac24df2e342da00d34e91c78d5de91592cee390ef3516ae9e2ff3f` |
|
|
84
|
+
| `e0-heldout.tar` (release asset) | `bd06a11eb7163da102cad6ea0df12ea79e52ff94aab1dc35d28ec3910af2be22` |
|
|
85
|
+
| `e0b-heldout.tar` (release asset) | `68219f228c41da972fe4c8f92c218e01f85f08eed864185f237cecda05314bbf` |
|
|
82
86
|
| ASSISTments 2009 file at BOBCAT e6b6245e | `c07ca9fa0f08207b06488cec527725938fe96e81e3c1762839e1ad0b84f974cc` |
|
|
83
87
|
| 2PL item fit (regenerated in step b) | `12a3b167cd9c9b78b3c3c7faf71b2e464b0943a454f6418faf0f99b1114cb7fc` |
|
|
84
88
|
| catIrt 0.5.1 CRAN binary used for the fixture | `f5418f447e91d184791e668bd2a135ea5da40c981d4a76826018295b6ea9f079` |
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
{plainproof-0.1.0 → plainproof-0.1.1}/scripts/published_cat_reference_v1/compare_selection_v1.py
RENAMED
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|