lab-kit-cli 0.3.0__tar.gz → 0.5.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (134) hide show
  1. {lab_kit_cli-0.3.0 → lab_kit_cli-0.5.0}/CHANGELOG.md +14 -0
  2. {lab_kit_cli-0.3.0 → lab_kit_cli-0.5.0}/PKG-INFO +2 -2
  3. {lab_kit_cli-0.3.0 → lab_kit_cli-0.5.0}/README.md +1 -1
  4. {lab_kit_cli-0.3.0 → lab_kit_cli-0.5.0}/docs/spec/lab-model.md +13 -7
  5. {lab_kit_cli-0.3.0 → lab_kit_cli-0.5.0}/examples/monte-carlo-lab/.agents/skills/experiment/SKILL.md +4 -4
  6. {lab_kit_cli-0.3.0 → lab_kit_cli-0.5.0/examples/monte-carlo-lab/.agents}/skills/review/SKILL.md +2 -2
  7. {lab_kit_cli-0.3.0 → lab_kit_cli-0.5.0}/examples/monte-carlo-lab/.lab/method/DISCIPLINE.md +1 -1
  8. {lab_kit_cli-0.3.0 → lab_kit_cli-0.5.0}/examples/monte-carlo-lab/lab.yaml +1 -2
  9. {lab_kit_cli-0.3.0 → lab_kit_cli-0.5.0}/method/DISCIPLINE.md +1 -1
  10. {lab_kit_cli-0.3.0 → lab_kit_cli-0.5.0}/pyproject.toml +1 -1
  11. {lab_kit_cli-0.3.0 → lab_kit_cli-0.5.0}/skills/experiment/SKILL.md +4 -4
  12. {lab_kit_cli-0.3.0/examples/monte-carlo-lab/.agents → lab_kit_cli-0.5.0}/skills/review/SKILL.md +2 -2
  13. {lab_kit_cli-0.3.0 → lab_kit_cli-0.5.0}/src/lab_kit/__init__.py +1 -1
  14. {lab_kit_cli-0.3.0 → lab_kit_cli-0.5.0}/src/lab_kit/checks/files.py +1 -47
  15. {lab_kit_cli-0.3.0 → lab_kit_cli-0.5.0}/src/lab_kit/checks/locks.py +14 -0
  16. {lab_kit_cli-0.3.0 → lab_kit_cli-0.5.0}/src/lab_kit/checks/run.py +1 -1
  17. {lab_kit_cli-0.3.0 → lab_kit_cli-0.5.0}/src/lab_kit/cli.py +47 -0
  18. {lab_kit_cli-0.3.0 → lab_kit_cli-0.5.0}/src/lab_kit/commands/setup.py +0 -1
  19. {lab_kit_cli-0.3.0 → lab_kit_cli-0.5.0}/src/lab_kit/lab.py +10 -4
  20. lab_kit_cli-0.5.0/src/lab_kit/store.py +285 -0
  21. {lab_kit_cli-0.3.0 → lab_kit_cli-0.5.0}/src/lab_kit/supervise.py +22 -0
  22. {lab_kit_cli-0.3.0 → lab_kit_cli-0.5.0}/tests/test_checks.py +13 -5
  23. {lab_kit_cli-0.3.0 → lab_kit_cli-0.5.0}/tests/test_end_to_end.py +0 -5
  24. {lab_kit_cli-0.3.0 → lab_kit_cli-0.5.0}/tests/test_example.py +1 -1
  25. lab_kit_cli-0.5.0/tests/test_store.py +108 -0
  26. {lab_kit_cli-0.3.0 → lab_kit_cli-0.5.0}/.github/workflows/ci.yml +0 -0
  27. {lab_kit_cli-0.3.0 → lab_kit_cli-0.5.0}/.github/workflows/policy.yml +0 -0
  28. {lab_kit_cli-0.3.0 → lab_kit_cli-0.5.0}/.github/workflows/release.yml +0 -0
  29. {lab_kit_cli-0.3.0 → lab_kit_cli-0.5.0}/.gitignore +0 -0
  30. {lab_kit_cli-0.3.0 → lab_kit_cli-0.5.0}/AGENTS.md +0 -0
  31. {lab_kit_cli-0.3.0 → lab_kit_cli-0.5.0}/CLAUDE.md +0 -0
  32. {lab_kit_cli-0.3.0 → lab_kit_cli-0.5.0}/CONTRIBUTING.md +0 -0
  33. {lab_kit_cli-0.3.0 → lab_kit_cli-0.5.0}/LICENSE +0 -0
  34. {lab_kit_cli-0.3.0 → lab_kit_cli-0.5.0}/SETUP.md +0 -0
  35. {lab_kit_cli-0.3.0 → lab_kit_cli-0.5.0}/agents/reporter.md +0 -0
  36. {lab_kit_cli-0.3.0 → lab_kit_cli-0.5.0}/agents/reviewer.md +0 -0
  37. {lab_kit_cli-0.3.0 → lab_kit_cli-0.5.0}/agents/runner.md +0 -0
  38. {lab_kit_cli-0.3.0 → lab_kit_cli-0.5.0}/agents/scout.md +0 -0
  39. {lab_kit_cli-0.3.0 → lab_kit_cli-0.5.0}/docs/figures/how-lab-kit-works-dark.svg +0 -0
  40. {lab_kit_cli-0.3.0 → lab_kit_cli-0.5.0}/docs/figures/how-lab-kit-works-light.svg +0 -0
  41. {lab_kit_cli-0.3.0 → lab_kit_cli-0.5.0}/docs/figures/lab-loop-dark.svg +0 -0
  42. {lab_kit_cli-0.3.0 → lab_kit_cli-0.5.0}/docs/figures/lab-loop-light.svg +0 -0
  43. {lab_kit_cli-0.3.0 → lab_kit_cli-0.5.0}/docs/figures/number-chain-dark.svg +0 -0
  44. {lab_kit_cli-0.3.0 → lab_kit_cli-0.5.0}/docs/figures/number-chain-light.svg +0 -0
  45. {lab_kit_cli-0.3.0 → lab_kit_cli-0.5.0}/examples/monte-carlo-lab/.agents/agents/reporter.md +0 -0
  46. {lab_kit_cli-0.3.0 → lab_kit_cli-0.5.0}/examples/monte-carlo-lab/.agents/agents/reviewer.md +0 -0
  47. {lab_kit_cli-0.3.0 → lab_kit_cli-0.5.0}/examples/monte-carlo-lab/.agents/agents/runner.md +0 -0
  48. {lab_kit_cli-0.3.0 → lab_kit_cli-0.5.0}/examples/monte-carlo-lab/.agents/agents/scout.md +0 -0
  49. {lab_kit_cli-0.3.0 → lab_kit_cli-0.5.0}/examples/monte-carlo-lab/.agents/skills/address/SKILL.md +0 -0
  50. {lab_kit_cli-0.3.0 → lab_kit_cli-0.5.0}/examples/monte-carlo-lab/.agents/skills/configure/SKILL.md +0 -0
  51. {lab_kit_cli-0.3.0 → lab_kit_cli-0.5.0}/examples/monte-carlo-lab/.agents/skills/organise/SKILL.md +0 -0
  52. {lab_kit_cli-0.3.0 → lab_kit_cli-0.5.0}/examples/monte-carlo-lab/.agents/skills/plan-mission/SKILL.md +0 -0
  53. {lab_kit_cli-0.3.0 → lab_kit_cli-0.5.0}/examples/monte-carlo-lab/.agents/skills/publish/SKILL.md +0 -0
  54. {lab_kit_cli-0.3.0 → lab_kit_cli-0.5.0}/examples/monte-carlo-lab/.agents/skills/run/SKILL.md +0 -0
  55. {lab_kit_cli-0.3.0 → lab_kit_cli-0.5.0}/examples/monte-carlo-lab/.agents/skills/run-mission/SKILL.md +0 -0
  56. {lab_kit_cli-0.3.0 → lab_kit_cli-0.5.0}/examples/monte-carlo-lab/.agents/skills/set-up/SKILL.md +0 -0
  57. {lab_kit_cli-0.3.0 → lab_kit_cli-0.5.0}/examples/monte-carlo-lab/.agents/skills/set-up-lab/SKILL.md +0 -0
  58. {lab_kit_cli-0.3.0 → lab_kit_cli-0.5.0}/examples/monte-carlo-lab/.agents/skills/write/SKILL.md +0 -0
  59. {lab_kit_cli-0.3.0 → lab_kit_cli-0.5.0}/examples/monte-carlo-lab/.gitignore +0 -0
  60. {lab_kit_cli-0.3.0 → lab_kit_cli-0.5.0}/examples/monte-carlo-lab/.lab/frozen.sha256 +0 -0
  61. {lab_kit_cli-0.3.0 → lab_kit_cli-0.5.0}/examples/monte-carlo-lab/.lab/method/LADDER.md +0 -0
  62. {lab_kit_cli-0.3.0 → lab_kit_cli-0.5.0}/examples/monte-carlo-lab/AGENTS.md +0 -0
  63. {lab_kit_cli-0.3.0 → lab_kit_cli-0.5.0}/examples/monte-carlo-lab/CLAUDE.md +0 -0
  64. {lab_kit_cli-0.3.0 → lab_kit_cli-0.5.0}/examples/monte-carlo-lab/docs/.folio/backlinks.json +0 -0
  65. {lab_kit_cli-0.3.0 → lab_kit_cli-0.5.0}/examples/monte-carlo-lab/docs/.folio/cards.json +0 -0
  66. {lab_kit_cli-0.3.0 → lab_kit_cli-0.5.0}/examples/monte-carlo-lab/docs/.folio/catalog.json +0 -0
  67. {lab_kit_cli-0.3.0 → lab_kit_cli-0.5.0}/examples/monte-carlo-lab/docs/.folio/journal.json +0 -0
  68. {lab_kit_cli-0.3.0 → lab_kit_cli-0.5.0}/examples/monte-carlo-lab/docs/.folio/nav.json +0 -0
  69. {lab_kit_cli-0.3.0 → lab_kit_cli-0.5.0}/examples/monte-carlo-lab/docs/.folio/search.json +0 -0
  70. {lab_kit_cli-0.3.0 → lab_kit_cli-0.5.0}/examples/monte-carlo-lab/docs/.gitignore +0 -0
  71. {lab_kit_cli-0.3.0 → lab_kit_cli-0.5.0}/examples/monte-carlo-lab/docs/assets/data/.gitkeep +0 -0
  72. {lab_kit_cli-0.3.0 → lab_kit_cli-0.5.0}/examples/monte-carlo-lab/docs/assets/figures/.gitkeep +0 -0
  73. {lab_kit_cli-0.3.0 → lab_kit_cli-0.5.0}/examples/monte-carlo-lab/docs/assets/refs.bib +0 -0
  74. {lab_kit_cli-0.3.0 → lab_kit_cli-0.5.0}/examples/monte-carlo-lab/docs/content/claims/C-1.md +0 -0
  75. {lab_kit_cli-0.3.0 → lab_kit_cli-0.5.0}/examples/monte-carlo-lab/docs/content/index.html +0 -0
  76. {lab_kit_cli-0.3.0 → lab_kit_cli-0.5.0}/examples/monte-carlo-lab/docs/content/journal/2026/2026-10-04-concluded-the-pi-error-scaling-mission.md +0 -0
  77. {lab_kit_cli-0.3.0 → lab_kit_cli-0.5.0}/examples/monte-carlo-lab/docs/content/journal/2026/2026-10-04-lab-set-up.md +0 -0
  78. {lab_kit_cli-0.3.0 → lab_kit_cli-0.5.0}/examples/monte-carlo-lab/docs/content/journal/2026/2026-10-04-locked-the-pi-error-scaling-protocol.md +0 -0
  79. {lab_kit_cli-0.3.0 → lab_kit_cli-0.5.0}/examples/monte-carlo-lab/docs/content/journal/2026/2026-10-04-opened-the-pi-error-scaling-mission.md +0 -0
  80. {lab_kit_cli-0.3.0 → lab_kit_cli-0.5.0}/examples/monte-carlo-lab/docs/content/journal/2026/2026-10-04-r-1-fitted-the-mean-absolute-error.md +0 -0
  81. {lab_kit_cli-0.3.0 → lab_kit_cli-0.5.0}/examples/monte-carlo-lab/docs/content/journal/2026/2026-10-04-ran-the-pi-error-scaling-protocol.md +0 -0
  82. {lab_kit_cli-0.3.0 → lab_kit_cli-0.5.0}/examples/monte-carlo-lab/docs/content/journal/2026/2026-10-04-recorded-r-1.md +0 -0
  83. {lab_kit_cli-0.3.0 → lab_kit_cli-0.5.0}/examples/monte-carlo-lab/docs/content/journal/2026/2026-10-04-recorded-r-2.md +0 -0
  84. {lab_kit_cli-0.3.0 → lab_kit_cli-0.5.0}/examples/monte-carlo-lab/docs/content/journal/2026/2026-10-04-reported-the-pi-error-scaling-experiment.md +0 -0
  85. {lab_kit_cli-0.3.0 → lab_kit_cli-0.5.0}/examples/monte-carlo-lab/docs/content/journal/2026/2026-10-04-the-square-root-law-holds-per-size-predictions.md +0 -0
  86. {lab_kit_cli-0.3.0 → lab_kit_cli-0.5.0}/examples/monte-carlo-lab/docs/content/maps/research.html +0 -0
  87. {lab_kit_cli-0.3.0 → lab_kit_cli-0.5.0}/examples/monte-carlo-lab/docs/content/projects/lab/index.html +0 -0
  88. {lab_kit_cli-0.3.0 → lab_kit_cli-0.5.0}/examples/monte-carlo-lab/docs/content/protocols/pi-error-scaling.md +0 -0
  89. {lab_kit_cli-0.3.0 → lab_kit_cli-0.5.0}/examples/monte-carlo-lab/docs/content/questions/Q-1.md +0 -0
  90. {lab_kit_cli-0.3.0 → lab_kit_cli-0.5.0}/examples/monte-carlo-lab/docs/content/reports/pi-error-scaling-report/index.html +0 -0
  91. {lab_kit_cli-0.3.0 → lab_kit_cli-0.5.0}/examples/monte-carlo-lab/docs/content/results/R-1.md +0 -0
  92. {lab_kit_cli-0.3.0 → lab_kit_cli-0.5.0}/examples/monte-carlo-lab/docs/content/results/R-2.md +0 -0
  93. {lab_kit_cli-0.3.0 → lab_kit_cli-0.5.0}/examples/monte-carlo-lab/docs/folio.yaml +0 -0
  94. {lab_kit_cli-0.3.0 → lab_kit_cli-0.5.0}/examples/monte-carlo-lab/experiments/.gitkeep +0 -0
  95. {lab_kit_cli-0.3.0 → lab_kit_cli-0.5.0}/examples/monte-carlo-lab/experiments/pi-error-scaling/.gitignore +0 -0
  96. {lab_kit_cli-0.3.0 → lab_kit_cli-0.5.0}/examples/monte-carlo-lab/experiments/pi-error-scaling/bin/.gitkeep +0 -0
  97. {lab_kit_cli-0.3.0 → lab_kit_cli-0.5.0}/examples/monte-carlo-lab/experiments/pi-error-scaling/bin/fold.py +0 -0
  98. {lab_kit_cli-0.3.0 → lab_kit_cli-0.5.0}/examples/monte-carlo-lab/experiments/pi-error-scaling/bin/sample.py +0 -0
  99. {lab_kit_cli-0.3.0 → lab_kit_cli-0.5.0}/examples/monte-carlo-lab/experiments/pi-error-scaling/lock.json +0 -0
  100. {lab_kit_cli-0.3.0 → lab_kit_cli-0.5.0}/examples/monte-carlo-lab/experiments/pi-error-scaling/runs/.gitkeep +0 -0
  101. {lab_kit_cli-0.3.0 → lab_kit_cli-0.5.0}/examples/monte-carlo-lab/experiments/pi-error-scaling/runs/20261004-231629/MANIFEST.sha256 +0 -0
  102. {lab_kit_cli-0.3.0 → lab_kit_cli-0.5.0}/examples/monte-carlo-lab/experiments/pi-error-scaling/runs/20261004-231629/out/estimates.tsv +0 -0
  103. {lab_kit_cli-0.3.0 → lab_kit_cli-0.5.0}/examples/monte-carlo-lab/experiments/pi-error-scaling/runs/20261004-231629/run.json +0 -0
  104. {lab_kit_cli-0.3.0 → lab_kit_cli-0.5.0}/examples/monte-carlo-lab/experiments/pi-error-scaling/score.yaml +0 -0
  105. {lab_kit_cli-0.3.0 → lab_kit_cli-0.5.0}/examples/monte-carlo-lab/ops/STATE.md +0 -0
  106. {lab_kit_cli-0.3.0 → lab_kit_cli-0.5.0}/examples/monte-carlo-lab/ops/missions/.gitkeep +0 -0
  107. {lab_kit_cli-0.3.0 → lab_kit_cli-0.5.0}/examples/monte-carlo-lab/ops/missions/2026-10-04-pi-error-scaling.md +0 -0
  108. {lab_kit_cli-0.3.0 → lab_kit_cli-0.5.0}/examples/monte-carlo-lab/substrate/README.md +0 -0
  109. {lab_kit_cli-0.3.0 → lab_kit_cli-0.5.0}/examples/monte-carlo-lab/substrate/estimator.py +0 -0
  110. {lab_kit_cli-0.3.0 → lab_kit_cli-0.5.0}/method/LADDER.md +0 -0
  111. {lab_kit_cli-0.3.0 → lab_kit_cli-0.5.0}/skills/plan-mission/SKILL.md +0 -0
  112. {lab_kit_cli-0.3.0 → lab_kit_cli-0.5.0}/skills/run-mission/SKILL.md +0 -0
  113. {lab_kit_cli-0.3.0 → lab_kit_cli-0.5.0}/skills/set-up-lab/SKILL.md +0 -0
  114. {lab_kit_cli-0.3.0 → lab_kit_cli-0.5.0}/src/lab_kit/__main__.py +0 -0
  115. {lab_kit_cli-0.3.0 → lab_kit_cli-0.5.0}/src/lab_kit/budgets.py +0 -0
  116. {lab_kit_cli-0.3.0 → lab_kit_cli-0.5.0}/src/lab_kit/checks/__init__.py +0 -0
  117. {lab_kit_cli-0.3.0 → lab_kit_cli-0.5.0}/src/lab_kit/checks/base.py +0 -0
  118. {lab_kit_cli-0.3.0 → lab_kit_cli-0.5.0}/src/lab_kit/checks/results.py +0 -0
  119. {lab_kit_cli-0.3.0 → lab_kit_cli-0.5.0}/src/lab_kit/commands/__init__.py +0 -0
  120. {lab_kit_cli-0.3.0 → lab_kit_cli-0.5.0}/src/lab_kit/commands/experiments.py +0 -0
  121. {lab_kit_cli-0.3.0 → lab_kit_cli-0.5.0}/src/lab_kit/commands/runs.py +0 -0
  122. {lab_kit_cli-0.3.0 → lab_kit_cli-0.5.0}/src/lab_kit/commands/status.py +0 -0
  123. {lab_kit_cli-0.3.0 → lab_kit_cli-0.5.0}/src/lab_kit/data.py +0 -0
  124. {lab_kit_cli-0.3.0 → lab_kit_cli-0.5.0}/src/lab_kit/errors.py +0 -0
  125. {lab_kit_cli-0.3.0 → lab_kit_cli-0.5.0}/src/lab_kit/frozen.py +0 -0
  126. {lab_kit_cli-0.3.0 → lab_kit_cli-0.5.0}/src/lab_kit/gitlog.py +0 -0
  127. {lab_kit_cli-0.3.0 → lab_kit_cli-0.5.0}/src/lab_kit/ops.py +0 -0
  128. {lab_kit_cli-0.3.0 → lab_kit_cli-0.5.0}/src/lab_kit/protocol.py +0 -0
  129. {lab_kit_cli-0.3.0 → lab_kit_cli-0.5.0}/src/lab_kit/records.py +0 -0
  130. {lab_kit_cli-0.3.0 → lab_kit_cli-0.5.0}/src/lab_kit/rederive.py +0 -0
  131. {lab_kit_cli-0.3.0 → lab_kit_cli-0.5.0}/tests/conftest.py +0 -0
  132. {lab_kit_cli-0.3.0 → lab_kit_cli-0.5.0}/tests/test_budgets.py +0 -0
  133. {lab_kit_cli-0.3.0 → lab_kit_cli-0.5.0}/tests/test_instrument.py +0 -0
  134. {lab_kit_cli-0.3.0 → lab_kit_cli-0.5.0}/tests/test_runs.py +0 -0
@@ -1,5 +1,19 @@
1
1
  # Changelog
2
2
 
3
+ ## 0.5.0
4
+
5
+ - lab-kit no longer runs a lab's code to check it. The `lab-selftest` check is gone, and with it the `tools:` key in `lab.yaml`: a lab that still has the key gets an error saying to delete it. lab-kit checks the record; the lab tests its code in its own test suite, which its gate runs beside `lab-kit check`.
6
+ - The method (DISCIPLINE §7) and the experiment and review skills say so: an experiment's command calls the lab's own tested code, `bin/` holds only what is particular to one experiment and may be empty, and a fold is tested on a planted case in the lab's suite. The lock still hashes `bin/` and records the commit, which says which version of the lab's code a run used.
7
+ - Locks and run records are unchanged.
8
+
9
+ ## 0.4.0
10
+
11
+ - The run store. `store:` in `lab.yaml` names one shared filesystem, such as `/labs/<lab>`, where every finished run's raw output goes. When a run ends, `lab-kit run` publishes `work/` and `run.log` to the store at the run folder's own path: a copy into `.incoming/`, checked against `work.sha256` (nothing missing, extra or changed), renamed into place, made read-only. It then writes `store.json` beside `run.json` and deletes the local copy. A run in the store is never overwritten. A failed publish is noted in `run.log`, keeps the local copy and never changes the run's exit code.
12
+ - `lab-kit publish <slug>/<run id>... | --pending [--keep]`, `lab-kit fetch <slug>/<run id>` and `lab-kit verify <slug>/<run id>`. A machine that does not mount the store sets `LAB_STORE_HOST` to one that does (rsync over ssh, `sha256sum` there).
13
+ - `lab-work-stored` (warning): with a store set, every ended run with a `work.sha256` has a `store.json`.
14
+ - The experiment skill says to commit `store.json` with the run, and never to mount the store where the system under test runs.
15
+ - A lab without `store:` works as before.
16
+
3
17
  ## 0.3.0
4
18
 
5
19
  - The lock covers the instrument. `lab-kit lock` records, beside the protocol's hash, `instrument` (one sha256 over the files in the experiment's `bin/`, sorted by path, each fed as its path and its bytes, `__pycache__` left out), `commit` (the git head) and `lab-kit` (the version). `lab-lock-intact` fails when `bin/` no longer hashes to `instrument`, and `lab-kit run` refuses to start on such a lock, as it already did for a changed protocol.
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.5
2
2
  Name: lab-kit-cli
3
- Version: 0.3.0
3
+ Version: 0.5.0
4
4
  Summary: A research lab's method and machinery on top of folio: pre-register, lock, run, score, re-derive.
5
5
  Project-URL: Homepage, https://github.com/pierg/lab-kit
6
6
  Project-URL: Issues, https://github.com/pierg/lab-kit/issues
@@ -82,7 +82,7 @@ The gate, `lab-kit check`, runs folio's checks and then the lab's. It runs offli
82
82
  - **Runs.** Every run started after its lock, used exactly the pinned configuration, and its evidence matches its manifest (`lab-run-after-lock`, `lab-roster-frozen`, `lab-evidence-sealed`).
83
83
  - **Results.** Every live result re-derives exactly, rests on a finished run of a locked protocol, and passed an independent reviewer (`lab-rederive`, `lab-result-grounded`).
84
84
  - **Scores.** A scorecard scores exactly the ids the lock names (`lab-score-exact`), and a scored experiment is reported or explained (`lab-scored-reported`).
85
- - **Lab files.** Ids resolve, frozen surfaces are untouched, scripts pass `--selftest`, nothing runs under an unapproved mission, spend stays within its cap and its standing budget, records only grow, the method is current, and the state file stays a short pointer.
85
+ - **Lab files.** Ids resolve, frozen surfaces are untouched, nothing runs under an unapproved mission, spend stays within its cap and its standing budget, records only grow, the method is current, and the state file stays a short pointer.
86
86
 
87
87
  Together they hold a chain from every number on a page back to the lock:
88
88
 
@@ -66,7 +66,7 @@ The gate, `lab-kit check`, runs folio's checks and then the lab's. It runs offli
66
66
  - **Runs.** Every run started after its lock, used exactly the pinned configuration, and its evidence matches its manifest (`lab-run-after-lock`, `lab-roster-frozen`, `lab-evidence-sealed`).
67
67
  - **Results.** Every live result re-derives exactly, rests on a finished run of a locked protocol, and passed an independent reviewer (`lab-rederive`, `lab-result-grounded`).
68
68
  - **Scores.** A scorecard scores exactly the ids the lock names (`lab-score-exact`), and a scored experiment is reported or explained (`lab-scored-reported`).
69
- - **Lab files.** Ids resolve, frozen surfaces are untouched, scripts pass `--selftest`, nothing runs under an unapproved mission, spend stays within its cap and its standing budget, records only grow, the method is current, and the state file stays a short pointer.
69
+ - **Lab files.** Ids resolve, frozen surfaces are untouched, nothing runs under an unapproved mission, spend stays within its cap and its standing budget, records only grow, the method is current, and the state file stays a short pointer.
70
70
 
71
71
  Together they hold a chain from every number on a page back to the lock:
72
72
 
@@ -44,7 +44,7 @@ A lab is a repository. Its folio library sits at the root or in a folder such as
44
44
  journal/{yyyy}/ the library's one journal: one permanent file per entry, {date}-{slug}.md
45
45
  ... maps, concepts, the front door, papers
46
46
  experiments/{slug}/ one per protocol, same slug
47
- bin/ the lab's own scripts for this experiment, each with --selftest
47
+ bin/ what is particular to this experiment, if anything; may be empty
48
48
  lock.json the lock record: protocol id, its sha256 with the status field set aside, the lock time (UTC),
49
49
  the instrument (a hash of bin/), the git commit and the lab-kit version
50
50
  runs/{run-id}/
@@ -53,6 +53,7 @@ A lab is a repository. Its folio library sits at the root or in a folder such as
53
53
  MANIFEST.sha256 a hash of every file in out/, written when the run ends
54
54
  work/ run.log raw output and log, gitignored
55
55
  work.sha256 a hash of every file in work/, written when the run ends; committed
56
+ store.json where work/ and run.log were published, and when (when lab.yaml names a store); committed
56
57
  score.yaml the review's scorecard against the lock, and the results to record
57
58
  ops/
58
59
  STATE.md what is true now and what is next; a pointer, never a record
@@ -69,13 +70,13 @@ A lab is a repository. Its folio library sits at the root or in a folder such as
69
70
  `lab.yaml`:
70
71
 
71
72
  ```yaml
72
- lab-kit: 0.3.0 # the version this lab is checked with
73
+ lab-kit: 0.5.0 # the version this lab is checked with
73
74
  library: docs # where folio.yaml is; "." for the root
74
75
  front: lab # the front door's id: a project document
75
76
  frozen: # surfaces never edited in place
76
77
  - substrate/
77
- tools: [tools/fold.py] # lab-wide scripts; each must pass --selftest
78
78
  rederive_timeout: 120 # seconds per re-derive command
79
+ store: /labs/monte-carlo-lab # optional: where finished runs' raw output is published (below)
79
80
  ```
80
81
 
81
82
  Paths in `lab.yaml`, `run.json`, `score.yaml`, and a result's `evidence` and `rederive` fields, are relative to the lab root. folio resolves a result's paths from the charter's `root`, so a lab whose library is in `docs/` sets `root: ..` in `folio.yaml`. Both tools then read the same path the same way.
@@ -84,6 +85,8 @@ Paths in `lab.yaml`, `run.json`, `score.yaml`, and a result's `evidence` and `re
84
85
 
85
86
  `run.json` holds `protocol`, `lock` (the hash the run was checked against), `config` (the pinned configuration, copied from the protocol), `mission`, `spend`, `command`, `started`, `ended`, `exit`, `pid`, `spent`, `commit` (the git head the run started at; null outside git or before the first commit) and `lab-kit` (the version that started it). Times are UTC, to the second. A run's command finds its folders in the environment: `LAB_RUN_DIR`, `LAB_OUT` and `LAB_WORK`. A run that spends writes `out/spend.json` with `{"spent": <number>}`, in the unit of its mission's cap, and `lab-kit run` copies it into `spent` when the command exits. When the command exits, `lab-kit run` also writes `work.sha256` beside `MANIFEST.sha256`, in the same format, one line per file under `work/`, when that folder exists: the raw output stays out of git, and its hashes go in, so any copy of it can be verified.
86
87
 
88
+ **The store.** `store:` in `lab.yaml` is optional: an absolute path, such as `/labs/<lab>`, where every machine that runs the lab mounts one shared filesystem. When it is set, a run's raw output leaves the machine that ran it once the run ends: `lab-kit run` publishes `work/` and `run.log` to `<store>/<the run folder's path from the lab root>/`. Publishing copies into `<store>/.incoming/`, checks the copy against `work.sha256` (no file missing, none extra, none changed), renames it into place in one step, makes it read-only, writes `store.json` (`store`, `files`, `published`, `by`) beside `run.json`, and then deletes the local `work/` and `run.log`. A run already in the store is never overwritten: a copy there that verifies counts as published, one that does not is refused. A publish that fails leaves the run where it is, notes why in `run.log`, and never changes the run's exit code; `lab-kit publish --pending` tries again. The command a run starts is never told the store's path, and a lab's instrument never mounts it into the sandbox where the system under test runs: the store holds other runs' raw output, answers included. A machine that does not mount the store sets `LAB_STORE_HOST` to a host that does; lab-kit then copies with rsync over ssh and verifies there with `sha256sum`.
89
+
87
90
  `lock.json` holds `protocol`, `path`, `sha256` and `locked`, and from lab-kit 0.3.0 `instrument`, `commit` and `lab-kit`. `instrument` is one sha256 over every file under `experiments/<slug>/bin/` (a `__pycache__` folder left out), sorted by its path from `bin/`, each fed as its path, a newline, then its bytes. `commit` is the git head at the lock (null outside git or before the first commit), and `lab-kit` the version that took it. A lock or a run record without these fields predates them and is read as it is: records are never edited.
88
91
 
89
92
  A mission file starts with YAML front matter: `title`, `status` (`draft`, `active` or `concluded`), `rests_on` (the ids it rests on), `approved` (who approved the mission, and when, with the date as `YYYY-MM-DD`) and `cap` (a number, in the unit its runs report spend; `0` when nothing spends), and optionally `budget` (the name of a standing budget it draws on). A mission is `draft` until it is approved; `approved` is required from `active` on. Its log is everything under its `## Log` heading; entries are only added at the bottom. `ops/STATE.md` has one line `Active mission: <path>` or `Active mission: none`, and never names a draft.
@@ -106,7 +109,7 @@ question --> protocol (draft) --> review of the draft --> lock --> run --> evide
106
109
  4. **Lock.** The write skill sets the protocol's status to `locked`. The protocol's card lists `locked` in `frozen_in`, so folio's `frozen` check holds it from here: nothing changes but its status field. `lab-kit lock {slug}` writes `lock.json` with the file's hash, taken with the status field set aside, so the one change folio allows (to `abandoned`) does not break it. `folio journal add --title .. --description .. --body .. --kind lock --about {slug}` records it. The commit that holds the protocol, `lock.json`, that journal entry and `.folio/` is the lock; folio's `frozen` check compares with the first commit that set `locked`. From here lab-kit also checks the protocol's bytes, and the bytes of the experiment's `bin/`, so the instrument is committed before the lock.
107
110
  5. **Run.** `lab-kit run {slug} -- <command>` refuses unless the lock is intact, the instrument included. It writes `run.json`, starts the lab's own command in the background, and logs it. A run under a mission names it with `--mission`, and the mission must be approved and `active`. A run that spends tokens or money also needs a mission that records a cap. Launch, checkpoints and terminations each go in a journal entry of kind `run` about the protocol.
108
111
  6. **Evidence.** When the command exits, `lab-kit run` writes `MANIFEST.sha256` and the exit code. `out/` is committed once, at the end, in one commit. A run whose tool failed is invalid, and is recorded as invalid.
109
- 7. **Score.** The review skill reads the lock first. It folds `out/` with a selftested script. `lab-kit score {slug}` creates `score.yaml` with every prediction and rule id the lock names. The review fills each with a verdict, a value and the evidence it came from, and lists the numbers to record under `record`.
112
+ 7. **Score.** The review skill reads the lock first. It folds `out/` with a tested fold. `lab-kit score {slug}` creates `score.yaml` with every prediction and rule id the lock names. The review fills each with a verdict, a value and the evidence it came from, and lists the numbers to record under `record`.
110
113
  8. **Reviewer pass.** An independent reviewer re-derives the scored numbers and returns a verdict. The commander records it, and `score.yaml` points to it.
111
114
  9. **Result.** For each number worth citing, folio's run skill follows `record-a-result` with `from: experiments/{slug}/score.yaml#<name>`, and writes `R-n`. Its number, baseline, bound, evidence path, re-derive command and protocol come from that entry; its date is the day it is recorded. A result is permanent and writes no status. folio derives it: `live` until a newer result supersedes it or a retraction names it. A null or a fired kill rule is a result too.
112
115
  10. **Claim.** If the lab wants to say something, the write skill adds or revises a claim `C-n` citing its results.
@@ -134,17 +137,17 @@ A result that is not promoted is recorded too: a journal entry about the protoco
134
137
  | `lab-score-exact` | A `score.yaml` names the lock's hash. Its `scores` list exactly the locked protocol's prediction ids (`P1`...) and decision-rule ids (`D1`...), no more and no fewer. Each has an allowed verdict and an existing evidence path. |
135
138
  | `lab-ids-resolve` | Every id cited in a lab file (state, missions, lock records, runs, scores) resolves to a document in the library. |
136
139
  | `lab-frozen-intact` | Every surface listed under `frozen:` matches its hash in `.lab/frozen.sha256`. |
137
- | `lab-selftest` | Every script in `experiments/*/bin/` and under `tools:` passes `--selftest`. |
138
140
  | `lab-mission-approved` | Every mission file has status `draft`, `active` or `concluded`. An `active` or `concluded` mission records `approved`, naming who and a date. No run names a draft mission, and `ops/STATE.md` names no draft as the active mission. Every standing budget has grants, each with a positive `amount` and an `approved` naming who and a date. A mission's `budget` names a budget that exists; past draft, such a mission records a numeric `cap`; and what the missions under a budget commit fits inside its amount. |
139
141
  | `lab-spend-recorded` | A run marked as spending names a mission that records the operator's approval and a cap, and its recorded spend is within the cap. |
140
142
  | `lab-append-only` | Lab files only: committed mission log entries, standing budget grants and units, lock records and run records are unchanged in later commits; a mission file or a budget is never deleted. A run record committed while its run was going may later fill in `ended`, `exit` and `spent`, each once: nothing else in it changes, and a closing field once set never changes again. Journal entries and results are folio's `permanent` check. |
141
143
  | `lab-method-current` | `.lab/method/`, the five skills and the four roles match the lab-kit version in `lab.yaml`. |
142
144
  | `lab-state-pointer` | `ops/STATE.md` names an active mission file or "none", stays under 300 words, and cites ids for any number (warning). |
143
145
  | `lab-scored-reported` | Every scored experiment has a report citing its results, or a journal entry about its protocol saying why not (warning). |
146
+ | `lab-work-stored` | When `lab.yaml` names a `store:`, every ended run with a `work.sha256` has a `store.json`: its raw output was published (warning). |
144
147
 
145
- The gate checks that the record is consistent. It does not check that the record is honest. That is the reviewer's job.
148
+ The gate checks that the record is consistent. It does not check that the record is honest. That is the reviewer's job. Nor does it test the lab's code: the lab's own gate runs the lab's test suite beside `lab-kit check`, and an experiment's command calls that tested code.
146
149
 
147
- lab-kit runs every selftest, re-derive command and run from the lab root with `PYTHONDONTWRITEBYTECODE=1`, so no bytecode lands in a frozen surface. A run counts as ended for `lab-evidence-sealed` whatever its exit code; for `lab-result-grounded` it must have exited 0.
150
+ lab-kit runs every re-derive command and run from the lab root with `PYTHONDONTWRITEBYTECODE=1`, so no bytecode lands in a frozen surface. A run counts as ended for `lab-evidence-sealed` whatever its exit code; for `lab-result-grounded` it must have exited 0.
148
151
 
149
152
  ## 6. The `lab-kit` command
150
153
 
@@ -158,6 +161,9 @@ Only skills call it, the way an agent calls git. Every command that changes file
158
161
  | `lab-kit lock <slug>` | Writes `lock.json` for a protocol whose status is `locked` and whose gate is green, hashing the file with its status field set aside, and records the instrument's hash, the git commit and the lab-kit version. Refuses otherwise. |
159
162
  | `lab-kit run <slug> [--mission <file>] [--spend] [--wait] -- <command>` | Checks the lock, writes `run.json`, starts the command in the background from the lab root, records the git commit and the lab-kit version, and writes `MANIFEST.sha256` for `out/` and `work.sha256` for `work/` when it exits. It refuses when the protocol or `bin/` no longer hashes to the lock. With `--mission`, it refuses a mission whose status is not `active` or that records no approval. `--wait` waits for the end, for a short token-free stage. |
160
163
  | `lab-kit runs [--live]` | Lists runs and their state: running, finished, failed, orphaned. |
164
+ | `lab-kit publish <slug>/<run id>... \| --pending [--keep]` | Publishes ended runs' raw output to the store, as above. `--pending` takes every ended run whose `work/` is still here. `--keep` keeps the local copy. |
165
+ | `lab-kit fetch <slug>/<run id>` | Copies a published run's `work/` and `run.log` back into its folder, writable, and verifies them against `work.sha256`. |
166
+ | `lab-kit verify <slug>/<run id>` | Checks the store's copy of a run against its `work.sha256`. Changes nothing. |
161
167
  | `lab-kit score <slug>` | Creates `score.yaml` from the locked protocol's prediction and rule ids, with empty verdicts. |
162
168
  | `lab-kit rederive <R-n>` | Runs one result's re-derive command and compares its output with the number. |
163
169
  | `lab-kit freeze <path>` | Records a frozen surface's hash. Only the operator asks for this. |
@@ -30,7 +30,7 @@ You take one question from a draft protocol to a finished run with its evidence
30
30
  7. Fail loud. An arm whose tool fails is invalid, never downgraded. Undecidable is not a pass. Judged, not checked.
31
31
  8. Name every move the agent under test may make, under the protocol's allowed moves. A move the list does not name is one it may never use. The section is checked by the protocol genre's required parts; its completeness is judged.
32
32
  9. No live spend without the operator's word, recorded in a mission with a cap. Checked by `lab-spend-recorded`.
33
- 10. Every script in `bin/` has a `--selftest`. Checked by `lab-selftest`.
33
+ 10. The code a run executes is tested by the lab's own test suite. Prefer a command that calls the lab's own tested code (its package or its tools) over a script written for one experiment; keep in `bin/` only what is particular to this experiment, and test it there too. Judged, not checked.
34
34
  11. Evidence is committed once, at the end of the run. Scratch and logs stay out of git. Checked by `lab-evidence-sealed`; the rest is judged.
35
35
  12. Every document goes through the write skill. You never write one by hand. Checked by `folio check`.
36
36
 
@@ -45,14 +45,14 @@ You take one question from a draft protocol to a finished run with its evidence
45
45
  - the decision rules `D1`..., with the kill rule marked `(kill)`;
46
46
  - the predictions `P1`..., each with a confidence;
47
47
  - what it cannot show.
48
- 2. **Make its folder.** Run `lab-kit experiment <slug>`. Put the lab's scripts in `experiments/<slug>/bin/`, each with a `--selftest`. Run `lab-kit check --only lab-selftest`.
48
+ 2. **Make its folder.** Run `lab-kit experiment <slug>`. The run's command calls the lab's own tested code; anything particular to this experiment goes in `experiments/<slug>/bin/`, with its tests in the lab's suite. An experiment that needs nothing of its own leaves `bin/` empty. Run the lab's tests.
49
49
  3. **Review the draft.** Send a reviewer with fresh context. Fold its points into the draft through the write skill. A protocol is cheap to fix before the lock and impossible to fix after.
50
- 4. **Lock.** Commit `bin/` first: the lock hashes it, and `lab-lock-intact` fails if it changes after. Through the write skill, set the protocol's status to `locked`. Run `lab-kit lock <slug>`. Record it: `folio journal add --title .. --description .. --body .. --kind lock --about <slug>,<Q-n>`. Run `folio index`, then `lab-kit check`. Commit exactly the protocol, `lock.json`, the `lock` journal entry and `.folio/` together; that commit is the lock. From here folio's `frozen` check and `lab-lock-intact` hold it.
50
+ 4. **Lock.** Commit `bin/` and the code the command calls first: the lock hashes `bin/` and records the commit, and `lab-lock-intact` fails if `bin/` changes after. The lab's code outside `bin/` may change later; the lock's commit says which version the run used, so a protocol that needs it fixed pins it by commit. Through the write skill, set the protocol's status to `locked`. Run `lab-kit lock <slug>`. Record it: `folio journal add --title .. --description .. --body .. --kind lock --about <slug>,<Q-n>`. Run `folio index`, then `lab-kit check`. Commit exactly the protocol, `lock.json`, the `lock` journal entry and `.folio/` together; that commit is the lock. From here folio's `frozen` check and `lab-lock-intact` hold it.
51
51
  5. **De-risk.** Run the token-free stages first, with `lab-kit run`. Fix the apparatus only before the lock. After it, an apparatus fault ends the run as invalid.
52
52
  6. **Launch.** Run `lab-kit run <slug> -- <command>`. Add `--spend --mission <file>` for a run that spends. Watch it at once: arm one watcher on its log that fires on the end marker or an error signature. A silent launch failure can waste a whole night.
53
53
  7. **Supervise.** Write a journal entry of kind `run` about the protocol for each launch, checkpoint, surprise and termination, with `folio journal add`. Keep each entry to what changed. Record a deviation the moment you see it. Commit each entry with `.folio/` after `folio index`; never stage the run's working files.
54
54
  8. **On failure.** End the run. Start a fresh one with the same configuration. Disclose it in a `run` entry. Never swap a model or a tool mid-run.
55
- 9. **Conclude.** When the run ends, `lab-kit run` writes the manifest of `out/`. Commit `out/`, `MANIFEST.sha256`, `work.sha256` and `run.json` in one commit; `work/` stays out of git, and `work.sha256` is what any copy of it is verified against. Run `lab-kit check`. Hand the experiment to the review skill. Do not score your own run.
55
+ 9. **Conclude.** When the run ends, `lab-kit run` writes the manifest of `out/`. Commit `out/`, `MANIFEST.sha256`, `work.sha256` and `run.json` in one commit, with `store.json` when `lab.yaml` names a store; `work/` stays out of git, and `work.sha256` is what any copy of it is verified against. With a store, `lab-kit run` has already published `work/` there and deleted the local copy; if `run.log` says the publish failed, run `lab-kit publish --pending`. Never mount the store where the system under test runs: it holds other runs' answers. Run `lab-kit check`. Hand the experiment to the review skill. Do not score your own run.
56
56
 
57
57
  ## Stops
58
58
 
@@ -27,7 +27,7 @@ You turn a finished experiment's evidence into a scorecard against its lock, the
27
27
  3. Score exactly what the lock names: every prediction and every rule, no more and no fewer. Checked by `lab-score-exact`.
28
28
  4. A badly chosen rule is still the rule. Record the flaw as a lesson; never rescore. Judged, not checked.
29
29
  5. A miss is a result. Nulls, fired kill rules and invalid arms are recorded at the same length as wins. Checked in part by `lab-scored-reported`; the rest is judged.
30
- 6. Every fold script has a `--selftest` on a planted case with a known answer. Checked by `lab-selftest`.
30
+ 6. Every fold is tested against a planted case with a known answer, in the lab's own test suite. Judged, not checked.
31
31
  7. You score; the reviewer gates. Nothing is recorded as a result before a reviewer pass. Checked by `lab-result-grounded`.
32
32
  8. Every document goes through folio: results and reports through workflows that folio's run skill follows, the rest through the write skill. Checked by `folio check`.
33
33
  9. A report is frozen once `live`, and a result is permanent. A new result earns a new report, never a rewrite of an old one. folio shows a banner on the old report when a result it cites is superseded or retracted. Checked by folio's `frozen` and `permanent` checks.
@@ -36,7 +36,7 @@ You turn a finished experiment's evidence into a scorecard against its lock, the
36
36
 
37
37
  ## Steps
38
38
 
39
- 1. **Fold.** Run the experiment's fold script over each finished run's `out/`. If none exists, write one in `bin/` with a `--selftest` on a planted case, and land it as an instrument change. Never fold by eye.
39
+ 1. **Fold.** Run the experiment's fold script over each finished run's `out/`. If none exists, write one, with a test on a planted case in the lab's suite, and land it as an instrument change. Never fold by eye.
40
40
  2. **Score.** Run `lab-kit score <slug>`. Under `scores`, for every prediction `P1`..., fill HIT, MISS or INDETERMINATE, with the value and the evidence file it came from. For every decision rule `D1`..., say whether it fired, the kill rule included. Run `lab-kit check --only lab-score-exact`.
41
41
  3. **Gate.** Ask the commander for a reviewer pass, or send one yourself if you are the commander. The reviewer re-derives the scored numbers. Record its verdict where the mission keeps verdicts, and point `score.yaml` to it: `review.verdict` and `review.record`.
42
42
  4. **Choose what to record.** For each scored number, ask: is this a citable fact? A null counts. Add each one to `score.yaml` under `record`, keyed by a short name. Give it the result's fields: `title`, `protocol`, `number`, `baseline`, `bound`, `evidence` and `rederive`. The `number` is exactly what the re-derive command prints, and the evidence lies inside the run. Name every number you leave out.
@@ -55,7 +55,7 @@ No defensive fallbacks. No silent degradation. An arm whose tool fails is invali
55
55
 
56
56
  An apparatus failure is a result to report, not a thing to patch around. Judged, not checked.
57
57
 
58
- Every script ships a `--selftest`, and the gate runs it. Checked by `lab-selftest`.
58
+ Every piece of code a run executes is tested by the lab's own test suite, and the lab's gate runs it. lab-kit runs no lab code in its checks: it checks the record, the lab tests its code. Judged, not checked.
59
59
 
60
60
  Every fold is tested against a planted case with a known answer. A fold that looks fine can still be blind to a renamed field. Judged, not checked.
61
61
 
@@ -1,6 +1,5 @@
1
- lab-kit: 0.3.0 # the version this lab is checked with
1
+ lab-kit: 0.5.0 # the version this lab is checked with
2
2
  library: docs # where folio.yaml is; "." for the root
3
3
  front: lab # the front door's id: a project document
4
4
  frozen: [substrate/] # surfaces never edited in place; record each with `lab-kit freeze`
5
- tools: [] # lab-wide scripts; each must pass --selftest
6
5
  rederive_timeout: 120 # seconds per re-derive command
@@ -55,7 +55,7 @@ No defensive fallbacks. No silent degradation. An arm whose tool fails is invali
55
55
 
56
56
  An apparatus failure is a result to report, not a thing to patch around. Judged, not checked.
57
57
 
58
- Every script ships a `--selftest`, and the gate runs it. Checked by `lab-selftest`.
58
+ Every piece of code a run executes is tested by the lab's own test suite, and the lab's gate runs it. lab-kit runs no lab code in its checks: it checks the record, the lab tests its code. Judged, not checked.
59
59
 
60
60
  Every fold is tested against a planted case with a known answer. A fold that looks fine can still be blind to a renamed field. Judged, not checked.
61
61
 
@@ -5,7 +5,7 @@ build-backend = "hatchling.build"
5
5
  [project]
6
6
  dynamic = ["readme"]
7
7
  name = "lab-kit-cli"
8
- version = "0.3.0"
8
+ version = "0.5.0"
9
9
  description = "A research lab's method and machinery on top of folio: pre-register, lock, run, score, re-derive."
10
10
  license = "MIT"
11
11
  authors = [{ name = "Piergiuseppe Mallozzi" }]
@@ -30,7 +30,7 @@ You take one question from a draft protocol to a finished run with its evidence
30
30
  7. Fail loud. An arm whose tool fails is invalid, never downgraded. Undecidable is not a pass. Judged, not checked.
31
31
  8. Name every move the agent under test may make, under the protocol's allowed moves. A move the list does not name is one it may never use. The section is checked by the protocol genre's required parts; its completeness is judged.
32
32
  9. No live spend without the operator's word, recorded in a mission with a cap. Checked by `lab-spend-recorded`.
33
- 10. Every script in `bin/` has a `--selftest`. Checked by `lab-selftest`.
33
+ 10. The code a run executes is tested by the lab's own test suite. Prefer a command that calls the lab's own tested code (its package or its tools) over a script written for one experiment; keep in `bin/` only what is particular to this experiment, and test it there too. Judged, not checked.
34
34
  11. Evidence is committed once, at the end of the run. Scratch and logs stay out of git. Checked by `lab-evidence-sealed`; the rest is judged.
35
35
  12. Every document goes through the write skill. You never write one by hand. Checked by `folio check`.
36
36
 
@@ -45,14 +45,14 @@ You take one question from a draft protocol to a finished run with its evidence
45
45
  - the decision rules `D1`..., with the kill rule marked `(kill)`;
46
46
  - the predictions `P1`..., each with a confidence;
47
47
  - what it cannot show.
48
- 2. **Make its folder.** Run `lab-kit experiment <slug>`. Put the lab's scripts in `experiments/<slug>/bin/`, each with a `--selftest`. Run `lab-kit check --only lab-selftest`.
48
+ 2. **Make its folder.** Run `lab-kit experiment <slug>`. The run's command calls the lab's own tested code; anything particular to this experiment goes in `experiments/<slug>/bin/`, with its tests in the lab's suite. An experiment that needs nothing of its own leaves `bin/` empty. Run the lab's tests.
49
49
  3. **Review the draft.** Send a reviewer with fresh context. Fold its points into the draft through the write skill. A protocol is cheap to fix before the lock and impossible to fix after.
50
- 4. **Lock.** Commit `bin/` first: the lock hashes it, and `lab-lock-intact` fails if it changes after. Through the write skill, set the protocol's status to `locked`. Run `lab-kit lock <slug>`. Record it: `folio journal add --title .. --description .. --body .. --kind lock --about <slug>,<Q-n>`. Run `folio index`, then `lab-kit check`. Commit exactly the protocol, `lock.json`, the `lock` journal entry and `.folio/` together; that commit is the lock. From here folio's `frozen` check and `lab-lock-intact` hold it.
50
+ 4. **Lock.** Commit `bin/` and the code the command calls first: the lock hashes `bin/` and records the commit, and `lab-lock-intact` fails if `bin/` changes after. The lab's code outside `bin/` may change later; the lock's commit says which version the run used, so a protocol that needs it fixed pins it by commit. Through the write skill, set the protocol's status to `locked`. Run `lab-kit lock <slug>`. Record it: `folio journal add --title .. --description .. --body .. --kind lock --about <slug>,<Q-n>`. Run `folio index`, then `lab-kit check`. Commit exactly the protocol, `lock.json`, the `lock` journal entry and `.folio/` together; that commit is the lock. From here folio's `frozen` check and `lab-lock-intact` hold it.
51
51
  5. **De-risk.** Run the token-free stages first, with `lab-kit run`. Fix the apparatus only before the lock. After it, an apparatus fault ends the run as invalid.
52
52
  6. **Launch.** Run `lab-kit run <slug> -- <command>`. Add `--spend --mission <file>` for a run that spends. Watch it at once: arm one watcher on its log that fires on the end marker or an error signature. A silent launch failure can waste a whole night.
53
53
  7. **Supervise.** Write a journal entry of kind `run` about the protocol for each launch, checkpoint, surprise and termination, with `folio journal add`. Keep each entry to what changed. Record a deviation the moment you see it. Commit each entry with `.folio/` after `folio index`; never stage the run's working files.
54
54
  8. **On failure.** End the run. Start a fresh one with the same configuration. Disclose it in a `run` entry. Never swap a model or a tool mid-run.
55
- 9. **Conclude.** When the run ends, `lab-kit run` writes the manifest of `out/`. Commit `out/`, `MANIFEST.sha256`, `work.sha256` and `run.json` in one commit; `work/` stays out of git, and `work.sha256` is what any copy of it is verified against. Run `lab-kit check`. Hand the experiment to the review skill. Do not score your own run.
55
+ 9. **Conclude.** When the run ends, `lab-kit run` writes the manifest of `out/`. Commit `out/`, `MANIFEST.sha256`, `work.sha256` and `run.json` in one commit, with `store.json` when `lab.yaml` names a store; `work/` stays out of git, and `work.sha256` is what any copy of it is verified against. With a store, `lab-kit run` has already published `work/` there and deleted the local copy; if `run.log` says the publish failed, run `lab-kit publish --pending`. Never mount the store where the system under test runs: it holds other runs' answers. Run `lab-kit check`. Hand the experiment to the review skill. Do not score your own run.
56
56
 
57
57
  ## Stops
58
58
 
@@ -27,7 +27,7 @@ You turn a finished experiment's evidence into a scorecard against its lock, the
27
27
  3. Score exactly what the lock names: every prediction and every rule, no more and no fewer. Checked by `lab-score-exact`.
28
28
  4. A badly chosen rule is still the rule. Record the flaw as a lesson; never rescore. Judged, not checked.
29
29
  5. A miss is a result. Nulls, fired kill rules and invalid arms are recorded at the same length as wins. Checked in part by `lab-scored-reported`; the rest is judged.
30
- 6. Every fold script has a `--selftest` on a planted case with a known answer. Checked by `lab-selftest`.
30
+ 6. Every fold is tested against a planted case with a known answer, in the lab's own test suite. Judged, not checked.
31
31
  7. You score; the reviewer gates. Nothing is recorded as a result before a reviewer pass. Checked by `lab-result-grounded`.
32
32
  8. Every document goes through folio: results and reports through workflows that folio's run skill follows, the rest through the write skill. Checked by `folio check`.
33
33
  9. A report is frozen once `live`, and a result is permanent. A new result earns a new report, never a rewrite of an old one. folio shows a banner on the old report when a result it cites is superseded or retracted. Checked by folio's `frozen` and `permanent` checks.
@@ -36,7 +36,7 @@ You turn a finished experiment's evidence into a scorecard against its lock, the
36
36
 
37
37
  ## Steps
38
38
 
39
- 1. **Fold.** Run the experiment's fold script over each finished run's `out/`. If none exists, write one in `bin/` with a `--selftest` on a planted case, and land it as an instrument change. Never fold by eye.
39
+ 1. **Fold.** Run the experiment's fold script over each finished run's `out/`. If none exists, write one, with a test on a planted case in the lab's suite, and land it as an instrument change. Never fold by eye.
40
40
  2. **Score.** Run `lab-kit score <slug>`. Under `scores`, for every prediction `P1`..., fill HIT, MISS or INDETERMINATE, with the value and the evidence file it came from. For every decision rule `D1`..., say whether it fired, the kill rule included. Run `lab-kit check --only lab-score-exact`.
41
41
  3. **Gate.** Ask the commander for a reviewer pass, or send one yourself if you are the commander. The reviewer re-derives the scored numbers. Record its verdict where the mission keeps verdicts, and point `score.yaml` to it: `review.verdict` and `review.record`.
42
42
  4. **Choose what to record.** For each scored number, ask: is this a citable fact? A null counts. Add each one to `score.yaml` under `record`, keyed by a short name. Give it the result's fields: `title`, `protocol`, `number`, `baseline`, `bound`, `evidence` and `rederive`. The `number` is exactly what the re-derive command prints, and the evidence lies inside the run. Name every number you leave out.
@@ -1,3 +1,3 @@
1
1
  """lab-kit: a research lab's method and machinery on top of folio."""
2
2
 
3
- __version__ = "0.3.0"
3
+ __version__ = "0.5.0"
@@ -1,18 +1,15 @@
1
- """Checks on the lab's own files: ids, frozen surfaces, selftests, spend, history, method and state."""
1
+ """Checks on the lab's own files: ids, frozen surfaces, spend, history, method and state."""
2
2
 
3
3
  from __future__ import annotations
4
4
 
5
5
  import json
6
6
  import re
7
- import subprocess
8
- import sys
9
7
  from pathlib import Path
10
8
  from typing import Any
11
9
 
12
10
  from folio.checks.rules import measurements
13
11
 
14
12
  from .. import __version__, budgets, data, frozen, gitlog, ops, records
15
- from ..records import script_env
16
13
  from ..errors import LabError
17
14
  from ..lab import Lab
18
15
  from .base import Reporter
@@ -112,49 +109,6 @@ def frozen_intact(lab: Lab, rep: Reporter) -> None:
112
109
  rep.add(path, f"changed: frozen surface `{surface}` is never edited in place")
113
110
 
114
111
 
115
- def _command(path: Path) -> list[str] | None:
116
- if path.suffix == ".py":
117
- return [sys.executable, str(path)]
118
- if path.suffix == ".sh":
119
- return ["bash", str(path)]
120
- if path.stat().st_mode & 0o111:
121
- return [str(path)]
122
- return None
123
-
124
-
125
- def scripts(lab: Lab) -> list[Path]:
126
- found: list[Path] = []
127
- for slug in lab.experiment_slugs():
128
- folder = lab.experiments / slug / "bin"
129
- if folder.is_dir():
130
- found.extend(p for p in sorted(folder.iterdir())
131
- if p.is_file() and not p.name.startswith(".") and p.suffix not in (".pyc", ".md"))
132
- found.extend(lab.root / t for t in lab.settings.tools)
133
- return found
134
-
135
-
136
- def selftest(lab: Lab, rep: Reporter) -> None:
137
- """Every script in experiments/*/bin/ and under `tools:` passes --selftest."""
138
- for path in scripts(lab):
139
- where = lab.rel(path) if path.exists() else path.relative_to(lab.root).as_posix()
140
- if not path.is_file():
141
- rep.add("lab.yaml", f"tool `{where}` does not exist")
142
- continue
143
- command = _command(path)
144
- if command is None:
145
- rep.add(where, "is neither a .py or .sh script nor executable, so it cannot run --selftest")
146
- continue
147
- try:
148
- done = subprocess.run([*command, "--selftest"], cwd=lab.root, capture_output=True, text=True,
149
- timeout=lab.settings.rederive_timeout, check=False, env=script_env())
150
- except subprocess.TimeoutExpired:
151
- rep.add(where, f"--selftest ran past {lab.settings.rederive_timeout}s")
152
- continue
153
- if done.returncode != 0:
154
- tail = (done.stderr.strip() or done.stdout.strip()).splitlines()[-1:] or [""]
155
- rep.add(where, f"--selftest exited {done.returncode}: {tail[0]}".rstrip(": "))
156
-
157
-
158
112
  def mission_approved(lab: Lab, rep: Reporter) -> None:
159
113
  """Missions are draft, active or concluded; an active or concluded one records who approved it and when;
160
114
  nothing runs under a draft, and the state never names a draft as active. A mission under a standing
@@ -138,3 +138,17 @@ def evidence_sealed(lab: Lab, rep: Reporter) -> None:
138
138
  elif recorded[name] != actual[name]:
139
139
  rep.add(manifest_path, f"out/{name} changed after the run ended")
140
140
  rep.guard(run.path, one)
141
+
142
+
143
+ def work_stored(lab: Lab, rep: Reporter) -> None:
144
+ """With a store in lab.yaml, every ended run with a work.sha256 records that it was published (warning)."""
145
+ if not lab.settings.store:
146
+ return
147
+ from ..store import STORE_RECORD
148
+ for run in records.runs(lab):
149
+ if not run.ended or not (run.folder / records.WORK_MANIFEST).is_file():
150
+ continue
151
+ if not (run.folder / STORE_RECORD).is_file():
152
+ rep.warn(run.path.rsplit("/", 1)[0] + "/" + records.WORK_MANIFEST,
153
+ f"its raw output was never published to the store {lab.settings.store}; "
154
+ "`lab-kit publish --pending` publishes it, then commit store.json")
@@ -23,13 +23,13 @@ CHECKS: tuple[LabCheck, ...] = (
23
23
  LabCheck("lab-score-exact", "error", results.score_exact),
24
24
  LabCheck("lab-ids-resolve", "error", files.ids_resolve),
25
25
  LabCheck("lab-frozen-intact", "error", files.frozen_intact),
26
- LabCheck("lab-selftest", "error", files.selftest),
27
26
  LabCheck("lab-mission-approved", "error", files.mission_approved),
28
27
  LabCheck("lab-spend-recorded", "error", files.spend_recorded),
29
28
  LabCheck("lab-append-only", "error", files.append_only),
30
29
  LabCheck("lab-method-current", "error", files.method_current),
31
30
  LabCheck("lab-state-pointer", "warning", files.state_pointer),
32
31
  LabCheck("lab-scored-reported", "warning", results.scored_reported),
32
+ LabCheck("lab-work-stored", "warning", locks.work_stored),
33
33
  )
34
34
  BY_NAME = {c.name: c for c in CHECKS}
35
35
 
@@ -10,6 +10,7 @@ from pathlib import Path
10
10
  from . import __version__, frozen
11
11
  from . import lab as lab_mod
12
12
  from . import rederive as rederive_mod
13
+ from . import store as store_mod
13
14
  from .checks import run as checks_run
14
15
  from .commands import experiments, runs, setup, status
15
16
  from .errors import LabError
@@ -65,6 +66,37 @@ def cmd_runs(args: argparse.Namespace) -> int:
65
66
  return 0
66
67
 
67
68
 
69
+ def cmd_publish(args: argparse.Namespace) -> int:
70
+ lab = _lab()
71
+ if args.pending == bool(args.runs):
72
+ raise LabError("name runs as <slug>/<run id>, or pass --pending for every ended run still here")
73
+ store_mod.root(lab) # a lab with no store says so once
74
+ targets = store_mod.pending(lab) if args.pending else [store_mod.find_run(lab, r) for r in args.runs]
75
+ if not targets:
76
+ print("Nothing to publish.")
77
+ return 0
78
+ failed = 0
79
+ for run in targets:
80
+ try:
81
+ _print(store_mod.publish(lab, run, keep=args.keep))
82
+ except (LabError, OSError) as exc:
83
+ print(f"lab-kit: {lab.rel(run.folder)}: not published: {exc}", file=sys.stderr)
84
+ failed += 1
85
+ return 1 if failed else 0
86
+
87
+
88
+ def cmd_fetch(args: argparse.Namespace) -> int:
89
+ lab = _lab()
90
+ _print(store_mod.fetch(lab, store_mod.find_run(lab, args.run)))
91
+ return 0
92
+
93
+
94
+ def cmd_verify(args: argparse.Namespace) -> int:
95
+ lab = _lab()
96
+ _print(store_mod.verify(lab, store_mod.find_run(lab, args.run)))
97
+ return 0
98
+
99
+
68
100
  def cmd_score(args: argparse.Namespace) -> int:
69
101
  _print(experiments.score(_lab(), args.slug))
70
102
  return 0
@@ -131,6 +163,21 @@ def parser() -> argparse.ArgumentParser:
131
163
  s.add_argument("--live", action="store_true", help="only running and orphaned runs")
132
164
  s.set_defaults(func=cmd_runs)
133
165
 
166
+ s = sub.add_parser("publish", help="Copy ended runs' raw output to the store named in lab.yaml, verify it "
167
+ "against work.sha256, and delete the local copy.")
168
+ s.add_argument("runs", nargs="*", metavar="SLUG/RUN_ID")
169
+ s.add_argument("--pending", action="store_true", help="every ended run whose work/ is still here")
170
+ s.add_argument("--keep", action="store_true", help="keep the local work/ and run.log")
171
+ s.set_defaults(func=cmd_publish)
172
+
173
+ s = sub.add_parser("fetch", help="Copy a published run's raw output back into its folder, verified.")
174
+ s.add_argument("run", metavar="SLUG/RUN_ID")
175
+ s.set_defaults(func=cmd_fetch)
176
+
177
+ s = sub.add_parser("verify", help="Check the store's copy of a run against its work.sha256.")
178
+ s.add_argument("run", metavar="SLUG/RUN_ID")
179
+ s.set_defaults(func=cmd_verify)
180
+
134
181
  s = sub.add_parser("score", help="Create score.yaml from the lock's prediction and rule ids.")
135
182
  s.add_argument("slug")
136
183
  s.set_defaults(func=cmd_score)
@@ -26,7 +26,6 @@ lab-kit: {version} # the version this lab is checked with
26
26
  library: {library} # where folio.yaml is; "." for the root
27
27
  front: # the front door's id: a project document
28
28
  frozen: [] # surfaces never edited in place; record each with `lab-kit freeze`
29
- tools: [] # lab-wide scripts; each must pass --selftest
30
29
  rederive_timeout: 120 # seconds per re-derive command
31
30
  """
32
31
 
@@ -14,7 +14,7 @@ from folio.library import Library
14
14
  from .errors import LabError
15
15
 
16
16
  LAB_YAML = "lab.yaml"
17
- _KEYS = ("lab-kit", "library", "front", "frozen", "tools", "rederive_timeout")
17
+ _KEYS = ("lab-kit", "library", "front", "frozen", "rederive_timeout", "store")
18
18
 
19
19
 
20
20
  @dataclass
@@ -23,8 +23,8 @@ class Settings:
23
23
  library: str
24
24
  front: str | None
25
25
  frozen: list[str]
26
- tools: list[str]
27
26
  rederive_timeout: int
27
+ store: str | None = None
28
28
 
29
29
 
30
30
  @dataclass
@@ -91,6 +91,9 @@ def find_root(start: Path) -> Path:
91
91
  def parse_settings(raw: Any, where: str) -> Settings:
92
92
  if not isinstance(raw, dict):
93
93
  raise LabError(f"{where}: must be a mapping")
94
+ if "tools" in raw:
95
+ raise LabError(f"{where}: `tools` was removed in lab-kit 0.5.0, which no longer runs scripts' "
96
+ "--selftest; delete the key and test those scripts in the lab's own test suite")
94
97
  unknown = sorted(set(raw) - set(_KEYS))
95
98
  if unknown:
96
99
  raise LabError(f"{where}: unknown key(s) {', '.join(unknown)}; the keys are {', '.join(_KEYS)}")
@@ -104,7 +107,7 @@ def parse_settings(raw: Any, where: str) -> Settings:
104
107
  if front is not None and not isinstance(front, str):
105
108
  raise LabError(f"{where}: `front` must be a document id")
106
109
  lists = {}
107
- for key in ("frozen", "tools"):
110
+ for key in ("frozen",):
108
111
  value = raw.get(key) or []
109
112
  if not isinstance(value, list) or not all(isinstance(v, str) and v for v in value):
110
113
  raise LabError(f"{where}: `{key}` must be a list of paths from the lab root")
@@ -115,7 +118,10 @@ def parse_settings(raw: Any, where: str) -> Settings:
115
118
  timeout = raw.get("rederive_timeout", 120)
116
119
  if not isinstance(timeout, int) or isinstance(timeout, bool) or timeout <= 0:
117
120
  raise LabError(f"{where}: `rederive_timeout` must be a positive whole number of seconds")
118
- return Settings(version, library, front or None, lists["frozen"], lists["tools"], timeout)
121
+ store = raw.get("store")
122
+ if store is not None and (not isinstance(store, str) or not Path(store).is_absolute()):
123
+ raise LabError(f"{where}: `store` must be an absolute path, such as /labs/<lab>")
124
+ return Settings(version, library, front or None, lists["frozen"], timeout, store or None)
119
125
 
120
126
 
121
127
  def load_at(root: Path) -> Lab: