lab-kit-cli 0.3.0__tar.gz → 0.5.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {lab_kit_cli-0.3.0 → lab_kit_cli-0.5.0}/CHANGELOG.md +14 -0
- {lab_kit_cli-0.3.0 → lab_kit_cli-0.5.0}/PKG-INFO +2 -2
- {lab_kit_cli-0.3.0 → lab_kit_cli-0.5.0}/README.md +1 -1
- {lab_kit_cli-0.3.0 → lab_kit_cli-0.5.0}/docs/spec/lab-model.md +13 -7
- {lab_kit_cli-0.3.0 → lab_kit_cli-0.5.0}/examples/monte-carlo-lab/.agents/skills/experiment/SKILL.md +4 -4
- {lab_kit_cli-0.3.0 → lab_kit_cli-0.5.0/examples/monte-carlo-lab/.agents}/skills/review/SKILL.md +2 -2
- {lab_kit_cli-0.3.0 → lab_kit_cli-0.5.0}/examples/monte-carlo-lab/.lab/method/DISCIPLINE.md +1 -1
- {lab_kit_cli-0.3.0 → lab_kit_cli-0.5.0}/examples/monte-carlo-lab/lab.yaml +1 -2
- {lab_kit_cli-0.3.0 → lab_kit_cli-0.5.0}/method/DISCIPLINE.md +1 -1
- {lab_kit_cli-0.3.0 → lab_kit_cli-0.5.0}/pyproject.toml +1 -1
- {lab_kit_cli-0.3.0 → lab_kit_cli-0.5.0}/skills/experiment/SKILL.md +4 -4
- {lab_kit_cli-0.3.0/examples/monte-carlo-lab/.agents → lab_kit_cli-0.5.0}/skills/review/SKILL.md +2 -2
- {lab_kit_cli-0.3.0 → lab_kit_cli-0.5.0}/src/lab_kit/__init__.py +1 -1
- {lab_kit_cli-0.3.0 → lab_kit_cli-0.5.0}/src/lab_kit/checks/files.py +1 -47
- {lab_kit_cli-0.3.0 → lab_kit_cli-0.5.0}/src/lab_kit/checks/locks.py +14 -0
- {lab_kit_cli-0.3.0 → lab_kit_cli-0.5.0}/src/lab_kit/checks/run.py +1 -1
- {lab_kit_cli-0.3.0 → lab_kit_cli-0.5.0}/src/lab_kit/cli.py +47 -0
- {lab_kit_cli-0.3.0 → lab_kit_cli-0.5.0}/src/lab_kit/commands/setup.py +0 -1
- {lab_kit_cli-0.3.0 → lab_kit_cli-0.5.0}/src/lab_kit/lab.py +10 -4
- lab_kit_cli-0.5.0/src/lab_kit/store.py +285 -0
- {lab_kit_cli-0.3.0 → lab_kit_cli-0.5.0}/src/lab_kit/supervise.py +22 -0
- {lab_kit_cli-0.3.0 → lab_kit_cli-0.5.0}/tests/test_checks.py +13 -5
- {lab_kit_cli-0.3.0 → lab_kit_cli-0.5.0}/tests/test_end_to_end.py +0 -5
- {lab_kit_cli-0.3.0 → lab_kit_cli-0.5.0}/tests/test_example.py +1 -1
- lab_kit_cli-0.5.0/tests/test_store.py +108 -0
- {lab_kit_cli-0.3.0 → lab_kit_cli-0.5.0}/.github/workflows/ci.yml +0 -0
- {lab_kit_cli-0.3.0 → lab_kit_cli-0.5.0}/.github/workflows/policy.yml +0 -0
- {lab_kit_cli-0.3.0 → lab_kit_cli-0.5.0}/.github/workflows/release.yml +0 -0
- {lab_kit_cli-0.3.0 → lab_kit_cli-0.5.0}/.gitignore +0 -0
- {lab_kit_cli-0.3.0 → lab_kit_cli-0.5.0}/AGENTS.md +0 -0
- {lab_kit_cli-0.3.0 → lab_kit_cli-0.5.0}/CLAUDE.md +0 -0
- {lab_kit_cli-0.3.0 → lab_kit_cli-0.5.0}/CONTRIBUTING.md +0 -0
- {lab_kit_cli-0.3.0 → lab_kit_cli-0.5.0}/LICENSE +0 -0
- {lab_kit_cli-0.3.0 → lab_kit_cli-0.5.0}/SETUP.md +0 -0
- {lab_kit_cli-0.3.0 → lab_kit_cli-0.5.0}/agents/reporter.md +0 -0
- {lab_kit_cli-0.3.0 → lab_kit_cli-0.5.0}/agents/reviewer.md +0 -0
- {lab_kit_cli-0.3.0 → lab_kit_cli-0.5.0}/agents/runner.md +0 -0
- {lab_kit_cli-0.3.0 → lab_kit_cli-0.5.0}/agents/scout.md +0 -0
- {lab_kit_cli-0.3.0 → lab_kit_cli-0.5.0}/docs/figures/how-lab-kit-works-dark.svg +0 -0
- {lab_kit_cli-0.3.0 → lab_kit_cli-0.5.0}/docs/figures/how-lab-kit-works-light.svg +0 -0
- {lab_kit_cli-0.3.0 → lab_kit_cli-0.5.0}/docs/figures/lab-loop-dark.svg +0 -0
- {lab_kit_cli-0.3.0 → lab_kit_cli-0.5.0}/docs/figures/lab-loop-light.svg +0 -0
- {lab_kit_cli-0.3.0 → lab_kit_cli-0.5.0}/docs/figures/number-chain-dark.svg +0 -0
- {lab_kit_cli-0.3.0 → lab_kit_cli-0.5.0}/docs/figures/number-chain-light.svg +0 -0
- {lab_kit_cli-0.3.0 → lab_kit_cli-0.5.0}/examples/monte-carlo-lab/.agents/agents/reporter.md +0 -0
- {lab_kit_cli-0.3.0 → lab_kit_cli-0.5.0}/examples/monte-carlo-lab/.agents/agents/reviewer.md +0 -0
- {lab_kit_cli-0.3.0 → lab_kit_cli-0.5.0}/examples/monte-carlo-lab/.agents/agents/runner.md +0 -0
- {lab_kit_cli-0.3.0 → lab_kit_cli-0.5.0}/examples/monte-carlo-lab/.agents/agents/scout.md +0 -0
- {lab_kit_cli-0.3.0 → lab_kit_cli-0.5.0}/examples/monte-carlo-lab/.agents/skills/address/SKILL.md +0 -0
- {lab_kit_cli-0.3.0 → lab_kit_cli-0.5.0}/examples/monte-carlo-lab/.agents/skills/configure/SKILL.md +0 -0
- {lab_kit_cli-0.3.0 → lab_kit_cli-0.5.0}/examples/monte-carlo-lab/.agents/skills/organise/SKILL.md +0 -0
- {lab_kit_cli-0.3.0 → lab_kit_cli-0.5.0}/examples/monte-carlo-lab/.agents/skills/plan-mission/SKILL.md +0 -0
- {lab_kit_cli-0.3.0 → lab_kit_cli-0.5.0}/examples/monte-carlo-lab/.agents/skills/publish/SKILL.md +0 -0
- {lab_kit_cli-0.3.0 → lab_kit_cli-0.5.0}/examples/monte-carlo-lab/.agents/skills/run/SKILL.md +0 -0
- {lab_kit_cli-0.3.0 → lab_kit_cli-0.5.0}/examples/monte-carlo-lab/.agents/skills/run-mission/SKILL.md +0 -0
- {lab_kit_cli-0.3.0 → lab_kit_cli-0.5.0}/examples/monte-carlo-lab/.agents/skills/set-up/SKILL.md +0 -0
- {lab_kit_cli-0.3.0 → lab_kit_cli-0.5.0}/examples/monte-carlo-lab/.agents/skills/set-up-lab/SKILL.md +0 -0
- {lab_kit_cli-0.3.0 → lab_kit_cli-0.5.0}/examples/monte-carlo-lab/.agents/skills/write/SKILL.md +0 -0
- {lab_kit_cli-0.3.0 → lab_kit_cli-0.5.0}/examples/monte-carlo-lab/.gitignore +0 -0
- {lab_kit_cli-0.3.0 → lab_kit_cli-0.5.0}/examples/monte-carlo-lab/.lab/frozen.sha256 +0 -0
- {lab_kit_cli-0.3.0 → lab_kit_cli-0.5.0}/examples/monte-carlo-lab/.lab/method/LADDER.md +0 -0
- {lab_kit_cli-0.3.0 → lab_kit_cli-0.5.0}/examples/monte-carlo-lab/AGENTS.md +0 -0
- {lab_kit_cli-0.3.0 → lab_kit_cli-0.5.0}/examples/monte-carlo-lab/CLAUDE.md +0 -0
- {lab_kit_cli-0.3.0 → lab_kit_cli-0.5.0}/examples/monte-carlo-lab/docs/.folio/backlinks.json +0 -0
- {lab_kit_cli-0.3.0 → lab_kit_cli-0.5.0}/examples/monte-carlo-lab/docs/.folio/cards.json +0 -0
- {lab_kit_cli-0.3.0 → lab_kit_cli-0.5.0}/examples/monte-carlo-lab/docs/.folio/catalog.json +0 -0
- {lab_kit_cli-0.3.0 → lab_kit_cli-0.5.0}/examples/monte-carlo-lab/docs/.folio/journal.json +0 -0
- {lab_kit_cli-0.3.0 → lab_kit_cli-0.5.0}/examples/monte-carlo-lab/docs/.folio/nav.json +0 -0
- {lab_kit_cli-0.3.0 → lab_kit_cli-0.5.0}/examples/monte-carlo-lab/docs/.folio/search.json +0 -0
- {lab_kit_cli-0.3.0 → lab_kit_cli-0.5.0}/examples/monte-carlo-lab/docs/.gitignore +0 -0
- {lab_kit_cli-0.3.0 → lab_kit_cli-0.5.0}/examples/monte-carlo-lab/docs/assets/data/.gitkeep +0 -0
- {lab_kit_cli-0.3.0 → lab_kit_cli-0.5.0}/examples/monte-carlo-lab/docs/assets/figures/.gitkeep +0 -0
- {lab_kit_cli-0.3.0 → lab_kit_cli-0.5.0}/examples/monte-carlo-lab/docs/assets/refs.bib +0 -0
- {lab_kit_cli-0.3.0 → lab_kit_cli-0.5.0}/examples/monte-carlo-lab/docs/content/claims/C-1.md +0 -0
- {lab_kit_cli-0.3.0 → lab_kit_cli-0.5.0}/examples/monte-carlo-lab/docs/content/index.html +0 -0
- {lab_kit_cli-0.3.0 → lab_kit_cli-0.5.0}/examples/monte-carlo-lab/docs/content/journal/2026/2026-10-04-concluded-the-pi-error-scaling-mission.md +0 -0
- {lab_kit_cli-0.3.0 → lab_kit_cli-0.5.0}/examples/monte-carlo-lab/docs/content/journal/2026/2026-10-04-lab-set-up.md +0 -0
- {lab_kit_cli-0.3.0 → lab_kit_cli-0.5.0}/examples/monte-carlo-lab/docs/content/journal/2026/2026-10-04-locked-the-pi-error-scaling-protocol.md +0 -0
- {lab_kit_cli-0.3.0 → lab_kit_cli-0.5.0}/examples/monte-carlo-lab/docs/content/journal/2026/2026-10-04-opened-the-pi-error-scaling-mission.md +0 -0
- {lab_kit_cli-0.3.0 → lab_kit_cli-0.5.0}/examples/monte-carlo-lab/docs/content/journal/2026/2026-10-04-r-1-fitted-the-mean-absolute-error.md +0 -0
- {lab_kit_cli-0.3.0 → lab_kit_cli-0.5.0}/examples/monte-carlo-lab/docs/content/journal/2026/2026-10-04-ran-the-pi-error-scaling-protocol.md +0 -0
- {lab_kit_cli-0.3.0 → lab_kit_cli-0.5.0}/examples/monte-carlo-lab/docs/content/journal/2026/2026-10-04-recorded-r-1.md +0 -0
- {lab_kit_cli-0.3.0 → lab_kit_cli-0.5.0}/examples/monte-carlo-lab/docs/content/journal/2026/2026-10-04-recorded-r-2.md +0 -0
- {lab_kit_cli-0.3.0 → lab_kit_cli-0.5.0}/examples/monte-carlo-lab/docs/content/journal/2026/2026-10-04-reported-the-pi-error-scaling-experiment.md +0 -0
- {lab_kit_cli-0.3.0 → lab_kit_cli-0.5.0}/examples/monte-carlo-lab/docs/content/journal/2026/2026-10-04-the-square-root-law-holds-per-size-predictions.md +0 -0
- {lab_kit_cli-0.3.0 → lab_kit_cli-0.5.0}/examples/monte-carlo-lab/docs/content/maps/research.html +0 -0
- {lab_kit_cli-0.3.0 → lab_kit_cli-0.5.0}/examples/monte-carlo-lab/docs/content/projects/lab/index.html +0 -0
- {lab_kit_cli-0.3.0 → lab_kit_cli-0.5.0}/examples/monte-carlo-lab/docs/content/protocols/pi-error-scaling.md +0 -0
- {lab_kit_cli-0.3.0 → lab_kit_cli-0.5.0}/examples/monte-carlo-lab/docs/content/questions/Q-1.md +0 -0
- {lab_kit_cli-0.3.0 → lab_kit_cli-0.5.0}/examples/monte-carlo-lab/docs/content/reports/pi-error-scaling-report/index.html +0 -0
- {lab_kit_cli-0.3.0 → lab_kit_cli-0.5.0}/examples/monte-carlo-lab/docs/content/results/R-1.md +0 -0
- {lab_kit_cli-0.3.0 → lab_kit_cli-0.5.0}/examples/monte-carlo-lab/docs/content/results/R-2.md +0 -0
- {lab_kit_cli-0.3.0 → lab_kit_cli-0.5.0}/examples/monte-carlo-lab/docs/folio.yaml +0 -0
- {lab_kit_cli-0.3.0 → lab_kit_cli-0.5.0}/examples/monte-carlo-lab/experiments/.gitkeep +0 -0
- {lab_kit_cli-0.3.0 → lab_kit_cli-0.5.0}/examples/monte-carlo-lab/experiments/pi-error-scaling/.gitignore +0 -0
- {lab_kit_cli-0.3.0 → lab_kit_cli-0.5.0}/examples/monte-carlo-lab/experiments/pi-error-scaling/bin/.gitkeep +0 -0
- {lab_kit_cli-0.3.0 → lab_kit_cli-0.5.0}/examples/monte-carlo-lab/experiments/pi-error-scaling/bin/fold.py +0 -0
- {lab_kit_cli-0.3.0 → lab_kit_cli-0.5.0}/examples/monte-carlo-lab/experiments/pi-error-scaling/bin/sample.py +0 -0
- {lab_kit_cli-0.3.0 → lab_kit_cli-0.5.0}/examples/monte-carlo-lab/experiments/pi-error-scaling/lock.json +0 -0
- {lab_kit_cli-0.3.0 → lab_kit_cli-0.5.0}/examples/monte-carlo-lab/experiments/pi-error-scaling/runs/.gitkeep +0 -0
- {lab_kit_cli-0.3.0 → lab_kit_cli-0.5.0}/examples/monte-carlo-lab/experiments/pi-error-scaling/runs/20261004-231629/MANIFEST.sha256 +0 -0
- {lab_kit_cli-0.3.0 → lab_kit_cli-0.5.0}/examples/monte-carlo-lab/experiments/pi-error-scaling/runs/20261004-231629/out/estimates.tsv +0 -0
- {lab_kit_cli-0.3.0 → lab_kit_cli-0.5.0}/examples/monte-carlo-lab/experiments/pi-error-scaling/runs/20261004-231629/run.json +0 -0
- {lab_kit_cli-0.3.0 → lab_kit_cli-0.5.0}/examples/monte-carlo-lab/experiments/pi-error-scaling/score.yaml +0 -0
- {lab_kit_cli-0.3.0 → lab_kit_cli-0.5.0}/examples/monte-carlo-lab/ops/STATE.md +0 -0
- {lab_kit_cli-0.3.0 → lab_kit_cli-0.5.0}/examples/monte-carlo-lab/ops/missions/.gitkeep +0 -0
- {lab_kit_cli-0.3.0 → lab_kit_cli-0.5.0}/examples/monte-carlo-lab/ops/missions/2026-10-04-pi-error-scaling.md +0 -0
- {lab_kit_cli-0.3.0 → lab_kit_cli-0.5.0}/examples/monte-carlo-lab/substrate/README.md +0 -0
- {lab_kit_cli-0.3.0 → lab_kit_cli-0.5.0}/examples/monte-carlo-lab/substrate/estimator.py +0 -0
- {lab_kit_cli-0.3.0 → lab_kit_cli-0.5.0}/method/LADDER.md +0 -0
- {lab_kit_cli-0.3.0 → lab_kit_cli-0.5.0}/skills/plan-mission/SKILL.md +0 -0
- {lab_kit_cli-0.3.0 → lab_kit_cli-0.5.0}/skills/run-mission/SKILL.md +0 -0
- {lab_kit_cli-0.3.0 → lab_kit_cli-0.5.0}/skills/set-up-lab/SKILL.md +0 -0
- {lab_kit_cli-0.3.0 → lab_kit_cli-0.5.0}/src/lab_kit/__main__.py +0 -0
- {lab_kit_cli-0.3.0 → lab_kit_cli-0.5.0}/src/lab_kit/budgets.py +0 -0
- {lab_kit_cli-0.3.0 → lab_kit_cli-0.5.0}/src/lab_kit/checks/__init__.py +0 -0
- {lab_kit_cli-0.3.0 → lab_kit_cli-0.5.0}/src/lab_kit/checks/base.py +0 -0
- {lab_kit_cli-0.3.0 → lab_kit_cli-0.5.0}/src/lab_kit/checks/results.py +0 -0
- {lab_kit_cli-0.3.0 → lab_kit_cli-0.5.0}/src/lab_kit/commands/__init__.py +0 -0
- {lab_kit_cli-0.3.0 → lab_kit_cli-0.5.0}/src/lab_kit/commands/experiments.py +0 -0
- {lab_kit_cli-0.3.0 → lab_kit_cli-0.5.0}/src/lab_kit/commands/runs.py +0 -0
- {lab_kit_cli-0.3.0 → lab_kit_cli-0.5.0}/src/lab_kit/commands/status.py +0 -0
- {lab_kit_cli-0.3.0 → lab_kit_cli-0.5.0}/src/lab_kit/data.py +0 -0
- {lab_kit_cli-0.3.0 → lab_kit_cli-0.5.0}/src/lab_kit/errors.py +0 -0
- {lab_kit_cli-0.3.0 → lab_kit_cli-0.5.0}/src/lab_kit/frozen.py +0 -0
- {lab_kit_cli-0.3.0 → lab_kit_cli-0.5.0}/src/lab_kit/gitlog.py +0 -0
- {lab_kit_cli-0.3.0 → lab_kit_cli-0.5.0}/src/lab_kit/ops.py +0 -0
- {lab_kit_cli-0.3.0 → lab_kit_cli-0.5.0}/src/lab_kit/protocol.py +0 -0
- {lab_kit_cli-0.3.0 → lab_kit_cli-0.5.0}/src/lab_kit/records.py +0 -0
- {lab_kit_cli-0.3.0 → lab_kit_cli-0.5.0}/src/lab_kit/rederive.py +0 -0
- {lab_kit_cli-0.3.0 → lab_kit_cli-0.5.0}/tests/conftest.py +0 -0
- {lab_kit_cli-0.3.0 → lab_kit_cli-0.5.0}/tests/test_budgets.py +0 -0
- {lab_kit_cli-0.3.0 → lab_kit_cli-0.5.0}/tests/test_instrument.py +0 -0
- {lab_kit_cli-0.3.0 → lab_kit_cli-0.5.0}/tests/test_runs.py +0 -0
|
@@ -1,5 +1,19 @@
|
|
|
1
1
|
# Changelog
|
|
2
2
|
|
|
3
|
+
## 0.5.0
|
|
4
|
+
|
|
5
|
+
- lab-kit no longer runs a lab's code to check it. The `lab-selftest` check is gone, and with it the `tools:` key in `lab.yaml`: a lab that still has the key gets an error saying to delete it. lab-kit checks the record; the lab tests its code in its own test suite, which its gate runs beside `lab-kit check`.
|
|
6
|
+
- The method (DISCIPLINE §7) and the experiment and review skills say so: an experiment's command calls the lab's own tested code, `bin/` holds only what is particular to one experiment and may be empty, and a fold is tested on a planted case in the lab's suite. The lock still hashes `bin/` and records the commit, which says which version of the lab's code a run used.
|
|
7
|
+
- Locks and run records are unchanged.
|
|
8
|
+
|
|
9
|
+
## 0.4.0
|
|
10
|
+
|
|
11
|
+
- The run store. `store:` in `lab.yaml` names one shared filesystem, such as `/labs/<lab>`, where every finished run's raw output goes. When a run ends, `lab-kit run` publishes `work/` and `run.log` to the store at the run folder's own path: a copy into `.incoming/`, checked against `work.sha256` (nothing missing, extra or changed), renamed into place, made read-only. It then writes `store.json` beside `run.json` and deletes the local copy. A run in the store is never overwritten. A failed publish is noted in `run.log`, keeps the local copy and never changes the run's exit code.
|
|
12
|
+
- `lab-kit publish <slug>/<run id>... | --pending [--keep]`, `lab-kit fetch <slug>/<run id>` and `lab-kit verify <slug>/<run id>`. A machine that does not mount the store sets `LAB_STORE_HOST` to one that does (rsync over ssh, `sha256sum` there).
|
|
13
|
+
- `lab-work-stored` (warning): with a store set, every ended run with a `work.sha256` has a `store.json`.
|
|
14
|
+
- The experiment skill says to commit `store.json` with the run, and never to mount the store where the system under test runs.
|
|
15
|
+
- A lab without `store:` works as before.
|
|
16
|
+
|
|
3
17
|
## 0.3.0
|
|
4
18
|
|
|
5
19
|
- The lock covers the instrument. `lab-kit lock` records, beside the protocol's hash, `instrument` (one sha256 over the files in the experiment's `bin/`, sorted by path, each fed as its path and its bytes, `__pycache__` left out), `commit` (the git head) and `lab-kit` (the version). `lab-lock-intact` fails when `bin/` no longer hashes to `instrument`, and `lab-kit run` refuses to start on such a lock, as it already did for a changed protocol.
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.5
|
|
2
2
|
Name: lab-kit-cli
|
|
3
|
-
Version: 0.
|
|
3
|
+
Version: 0.5.0
|
|
4
4
|
Summary: A research lab's method and machinery on top of folio: pre-register, lock, run, score, re-derive.
|
|
5
5
|
Project-URL: Homepage, https://github.com/pierg/lab-kit
|
|
6
6
|
Project-URL: Issues, https://github.com/pierg/lab-kit/issues
|
|
@@ -82,7 +82,7 @@ The gate, `lab-kit check`, runs folio's checks and then the lab's. It runs offli
|
|
|
82
82
|
- **Runs.** Every run started after its lock, used exactly the pinned configuration, and its evidence matches its manifest (`lab-run-after-lock`, `lab-roster-frozen`, `lab-evidence-sealed`).
|
|
83
83
|
- **Results.** Every live result re-derives exactly, rests on a finished run of a locked protocol, and passed an independent reviewer (`lab-rederive`, `lab-result-grounded`).
|
|
84
84
|
- **Scores.** A scorecard scores exactly the ids the lock names (`lab-score-exact`), and a scored experiment is reported or explained (`lab-scored-reported`).
|
|
85
|
-
- **Lab files.** Ids resolve, frozen surfaces are untouched,
|
|
85
|
+
- **Lab files.** Ids resolve, frozen surfaces are untouched, nothing runs under an unapproved mission, spend stays within its cap and its standing budget, records only grow, the method is current, and the state file stays a short pointer.
|
|
86
86
|
|
|
87
87
|
Together they hold a chain from every number on a page back to the lock:
|
|
88
88
|
|
|
@@ -66,7 +66,7 @@ The gate, `lab-kit check`, runs folio's checks and then the lab's. It runs offli
|
|
|
66
66
|
- **Runs.** Every run started after its lock, used exactly the pinned configuration, and its evidence matches its manifest (`lab-run-after-lock`, `lab-roster-frozen`, `lab-evidence-sealed`).
|
|
67
67
|
- **Results.** Every live result re-derives exactly, rests on a finished run of a locked protocol, and passed an independent reviewer (`lab-rederive`, `lab-result-grounded`).
|
|
68
68
|
- **Scores.** A scorecard scores exactly the ids the lock names (`lab-score-exact`), and a scored experiment is reported or explained (`lab-scored-reported`).
|
|
69
|
-
- **Lab files.** Ids resolve, frozen surfaces are untouched,
|
|
69
|
+
- **Lab files.** Ids resolve, frozen surfaces are untouched, nothing runs under an unapproved mission, spend stays within its cap and its standing budget, records only grow, the method is current, and the state file stays a short pointer.
|
|
70
70
|
|
|
71
71
|
Together they hold a chain from every number on a page back to the lock:
|
|
72
72
|
|
|
@@ -44,7 +44,7 @@ A lab is a repository. Its folio library sits at the root or in a folder such as
|
|
|
44
44
|
journal/{yyyy}/ the library's one journal: one permanent file per entry, {date}-{slug}.md
|
|
45
45
|
... maps, concepts, the front door, papers
|
|
46
46
|
experiments/{slug}/ one per protocol, same slug
|
|
47
|
-
bin/
|
|
47
|
+
bin/ what is particular to this experiment, if anything; may be empty
|
|
48
48
|
lock.json the lock record: protocol id, its sha256 with the status field set aside, the lock time (UTC),
|
|
49
49
|
the instrument (a hash of bin/), the git commit and the lab-kit version
|
|
50
50
|
runs/{run-id}/
|
|
@@ -53,6 +53,7 @@ A lab is a repository. Its folio library sits at the root or in a folder such as
|
|
|
53
53
|
MANIFEST.sha256 a hash of every file in out/, written when the run ends
|
|
54
54
|
work/ run.log raw output and log, gitignored
|
|
55
55
|
work.sha256 a hash of every file in work/, written when the run ends; committed
|
|
56
|
+
store.json where work/ and run.log were published, and when (when lab.yaml names a store); committed
|
|
56
57
|
score.yaml the review's scorecard against the lock, and the results to record
|
|
57
58
|
ops/
|
|
58
59
|
STATE.md what is true now and what is next; a pointer, never a record
|
|
@@ -69,13 +70,13 @@ A lab is a repository. Its folio library sits at the root or in a folder such as
|
|
|
69
70
|
`lab.yaml`:
|
|
70
71
|
|
|
71
72
|
```yaml
|
|
72
|
-
lab-kit: 0.
|
|
73
|
+
lab-kit: 0.5.0 # the version this lab is checked with
|
|
73
74
|
library: docs # where folio.yaml is; "." for the root
|
|
74
75
|
front: lab # the front door's id: a project document
|
|
75
76
|
frozen: # surfaces never edited in place
|
|
76
77
|
- substrate/
|
|
77
|
-
tools: [tools/fold.py] # lab-wide scripts; each must pass --selftest
|
|
78
78
|
rederive_timeout: 120 # seconds per re-derive command
|
|
79
|
+
store: /labs/monte-carlo-lab # optional: where finished runs' raw output is published (below)
|
|
79
80
|
```
|
|
80
81
|
|
|
81
82
|
Paths in `lab.yaml`, `run.json`, `score.yaml`, and a result's `evidence` and `rederive` fields, are relative to the lab root. folio resolves a result's paths from the charter's `root`, so a lab whose library is in `docs/` sets `root: ..` in `folio.yaml`. Both tools then read the same path the same way.
|
|
@@ -84,6 +85,8 @@ Paths in `lab.yaml`, `run.json`, `score.yaml`, and a result's `evidence` and `re
|
|
|
84
85
|
|
|
85
86
|
`run.json` holds `protocol`, `lock` (the hash the run was checked against), `config` (the pinned configuration, copied from the protocol), `mission`, `spend`, `command`, `started`, `ended`, `exit`, `pid`, `spent`, `commit` (the git head the run started at; null outside git or before the first commit) and `lab-kit` (the version that started it). Times are UTC, to the second. A run's command finds its folders in the environment: `LAB_RUN_DIR`, `LAB_OUT` and `LAB_WORK`. A run that spends writes `out/spend.json` with `{"spent": <number>}`, in the unit of its mission's cap, and `lab-kit run` copies it into `spent` when the command exits. When the command exits, `lab-kit run` also writes `work.sha256` beside `MANIFEST.sha256`, in the same format, one line per file under `work/`, when that folder exists: the raw output stays out of git, and its hashes go in, so any copy of it can be verified.
|
|
86
87
|
|
|
88
|
+
**The store.** `store:` in `lab.yaml` is optional: an absolute path, such as `/labs/<lab>`, where every machine that runs the lab mounts one shared filesystem. When it is set, a run's raw output leaves the machine that ran it once the run ends: `lab-kit run` publishes `work/` and `run.log` to `<store>/<the run folder's path from the lab root>/`. Publishing copies into `<store>/.incoming/`, checks the copy against `work.sha256` (no file missing, none extra, none changed), renames it into place in one step, makes it read-only, writes `store.json` (`store`, `files`, `published`, `by`) beside `run.json`, and then deletes the local `work/` and `run.log`. A run already in the store is never overwritten: a copy there that verifies counts as published, one that does not is refused. A publish that fails leaves the run where it is, notes why in `run.log`, and never changes the run's exit code; `lab-kit publish --pending` tries again. The command a run starts is never told the store's path, and a lab's instrument never mounts it into the sandbox where the system under test runs: the store holds other runs' raw output, answers included. A machine that does not mount the store sets `LAB_STORE_HOST` to a host that does; lab-kit then copies with rsync over ssh and verifies there with `sha256sum`.
|
|
89
|
+
|
|
87
90
|
`lock.json` holds `protocol`, `path`, `sha256` and `locked`, and from lab-kit 0.3.0 `instrument`, `commit` and `lab-kit`. `instrument` is one sha256 over every file under `experiments/<slug>/bin/` (a `__pycache__` folder left out), sorted by its path from `bin/`, each fed as its path, a newline, then its bytes. `commit` is the git head at the lock (null outside git or before the first commit), and `lab-kit` the version that took it. A lock or a run record without these fields predates them and is read as it is: records are never edited.
|
|
88
91
|
|
|
89
92
|
A mission file starts with YAML front matter: `title`, `status` (`draft`, `active` or `concluded`), `rests_on` (the ids it rests on), `approved` (who approved the mission, and when, with the date as `YYYY-MM-DD`) and `cap` (a number, in the unit its runs report spend; `0` when nothing spends), and optionally `budget` (the name of a standing budget it draws on). A mission is `draft` until it is approved; `approved` is required from `active` on. Its log is everything under its `## Log` heading; entries are only added at the bottom. `ops/STATE.md` has one line `Active mission: <path>` or `Active mission: none`, and never names a draft.
|
|
@@ -106,7 +109,7 @@ question --> protocol (draft) --> review of the draft --> lock --> run --> evide
|
|
|
106
109
|
4. **Lock.** The write skill sets the protocol's status to `locked`. The protocol's card lists `locked` in `frozen_in`, so folio's `frozen` check holds it from here: nothing changes but its status field. `lab-kit lock {slug}` writes `lock.json` with the file's hash, taken with the status field set aside, so the one change folio allows (to `abandoned`) does not break it. `folio journal add --title .. --description .. --body .. --kind lock --about {slug}` records it. The commit that holds the protocol, `lock.json`, that journal entry and `.folio/` is the lock; folio's `frozen` check compares with the first commit that set `locked`. From here lab-kit also checks the protocol's bytes, and the bytes of the experiment's `bin/`, so the instrument is committed before the lock.
|
|
107
110
|
5. **Run.** `lab-kit run {slug} -- <command>` refuses unless the lock is intact, the instrument included. It writes `run.json`, starts the lab's own command in the background, and logs it. A run under a mission names it with `--mission`, and the mission must be approved and `active`. A run that spends tokens or money also needs a mission that records a cap. Launch, checkpoints and terminations each go in a journal entry of kind `run` about the protocol.
|
|
108
111
|
6. **Evidence.** When the command exits, `lab-kit run` writes `MANIFEST.sha256` and the exit code. `out/` is committed once, at the end, in one commit. A run whose tool failed is invalid, and is recorded as invalid.
|
|
109
|
-
7. **Score.** The review skill reads the lock first. It folds `out/` with a
|
|
112
|
+
7. **Score.** The review skill reads the lock first. It folds `out/` with a tested fold. `lab-kit score {slug}` creates `score.yaml` with every prediction and rule id the lock names. The review fills each with a verdict, a value and the evidence it came from, and lists the numbers to record under `record`.
|
|
110
113
|
8. **Reviewer pass.** An independent reviewer re-derives the scored numbers and returns a verdict. The commander records it, and `score.yaml` points to it.
|
|
111
114
|
9. **Result.** For each number worth citing, folio's run skill follows `record-a-result` with `from: experiments/{slug}/score.yaml#<name>`, and writes `R-n`. Its number, baseline, bound, evidence path, re-derive command and protocol come from that entry; its date is the day it is recorded. A result is permanent and writes no status. folio derives it: `live` until a newer result supersedes it or a retraction names it. A null or a fired kill rule is a result too.
|
|
112
115
|
10. **Claim.** If the lab wants to say something, the write skill adds or revises a claim `C-n` citing its results.
|
|
@@ -134,17 +137,17 @@ A result that is not promoted is recorded too: a journal entry about the protoco
|
|
|
134
137
|
| `lab-score-exact` | A `score.yaml` names the lock's hash. Its `scores` list exactly the locked protocol's prediction ids (`P1`...) and decision-rule ids (`D1`...), no more and no fewer. Each has an allowed verdict and an existing evidence path. |
|
|
135
138
|
| `lab-ids-resolve` | Every id cited in a lab file (state, missions, lock records, runs, scores) resolves to a document in the library. |
|
|
136
139
|
| `lab-frozen-intact` | Every surface listed under `frozen:` matches its hash in `.lab/frozen.sha256`. |
|
|
137
|
-
| `lab-selftest` | Every script in `experiments/*/bin/` and under `tools:` passes `--selftest`. |
|
|
138
140
|
| `lab-mission-approved` | Every mission file has status `draft`, `active` or `concluded`. An `active` or `concluded` mission records `approved`, naming who and a date. No run names a draft mission, and `ops/STATE.md` names no draft as the active mission. Every standing budget has grants, each with a positive `amount` and an `approved` naming who and a date. A mission's `budget` names a budget that exists; past draft, such a mission records a numeric `cap`; and what the missions under a budget commit fits inside its amount. |
|
|
139
141
|
| `lab-spend-recorded` | A run marked as spending names a mission that records the operator's approval and a cap, and its recorded spend is within the cap. |
|
|
140
142
|
| `lab-append-only` | Lab files only: committed mission log entries, standing budget grants and units, lock records and run records are unchanged in later commits; a mission file or a budget is never deleted. A run record committed while its run was going may later fill in `ended`, `exit` and `spent`, each once: nothing else in it changes, and a closing field once set never changes again. Journal entries and results are folio's `permanent` check. |
|
|
141
143
|
| `lab-method-current` | `.lab/method/`, the five skills and the four roles match the lab-kit version in `lab.yaml`. |
|
|
142
144
|
| `lab-state-pointer` | `ops/STATE.md` names an active mission file or "none", stays under 300 words, and cites ids for any number (warning). |
|
|
143
145
|
| `lab-scored-reported` | Every scored experiment has a report citing its results, or a journal entry about its protocol saying why not (warning). |
|
|
146
|
+
| `lab-work-stored` | When `lab.yaml` names a `store:`, every ended run with a `work.sha256` has a `store.json`: its raw output was published (warning). |
|
|
144
147
|
|
|
145
|
-
The gate checks that the record is consistent. It does not check that the record is honest. That is the reviewer's job.
|
|
148
|
+
The gate checks that the record is consistent. It does not check that the record is honest. That is the reviewer's job. Nor does it test the lab's code: the lab's own gate runs the lab's test suite beside `lab-kit check`, and an experiment's command calls that tested code.
|
|
146
149
|
|
|
147
|
-
lab-kit runs every
|
|
150
|
+
lab-kit runs every re-derive command and run from the lab root with `PYTHONDONTWRITEBYTECODE=1`, so no bytecode lands in a frozen surface. A run counts as ended for `lab-evidence-sealed` whatever its exit code; for `lab-result-grounded` it must have exited 0.
|
|
148
151
|
|
|
149
152
|
## 6. The `lab-kit` command
|
|
150
153
|
|
|
@@ -158,6 +161,9 @@ Only skills call it, the way an agent calls git. Every command that changes file
|
|
|
158
161
|
| `lab-kit lock <slug>` | Writes `lock.json` for a protocol whose status is `locked` and whose gate is green, hashing the file with its status field set aside, and records the instrument's hash, the git commit and the lab-kit version. Refuses otherwise. |
|
|
159
162
|
| `lab-kit run <slug> [--mission <file>] [--spend] [--wait] -- <command>` | Checks the lock, writes `run.json`, starts the command in the background from the lab root, records the git commit and the lab-kit version, and writes `MANIFEST.sha256` for `out/` and `work.sha256` for `work/` when it exits. It refuses when the protocol or `bin/` no longer hashes to the lock. With `--mission`, it refuses a mission whose status is not `active` or that records no approval. `--wait` waits for the end, for a short token-free stage. |
|
|
160
163
|
| `lab-kit runs [--live]` | Lists runs and their state: running, finished, failed, orphaned. |
|
|
164
|
+
| `lab-kit publish <slug>/<run id>... \| --pending [--keep]` | Publishes ended runs' raw output to the store, as above. `--pending` takes every ended run whose `work/` is still here. `--keep` keeps the local copy. |
|
|
165
|
+
| `lab-kit fetch <slug>/<run id>` | Copies a published run's `work/` and `run.log` back into its folder, writable, and verifies them against `work.sha256`. |
|
|
166
|
+
| `lab-kit verify <slug>/<run id>` | Checks the store's copy of a run against its `work.sha256`. Changes nothing. |
|
|
161
167
|
| `lab-kit score <slug>` | Creates `score.yaml` from the locked protocol's prediction and rule ids, with empty verdicts. |
|
|
162
168
|
| `lab-kit rederive <R-n>` | Runs one result's re-derive command and compares its output with the number. |
|
|
163
169
|
| `lab-kit freeze <path>` | Records a frozen surface's hash. Only the operator asks for this. |
|
{lab_kit_cli-0.3.0 → lab_kit_cli-0.5.0}/examples/monte-carlo-lab/.agents/skills/experiment/SKILL.md
RENAMED
|
@@ -30,7 +30,7 @@ You take one question from a draft protocol to a finished run with its evidence
|
|
|
30
30
|
7. Fail loud. An arm whose tool fails is invalid, never downgraded. Undecidable is not a pass. Judged, not checked.
|
|
31
31
|
8. Name every move the agent under test may make, under the protocol's allowed moves. A move the list does not name is one it may never use. The section is checked by the protocol genre's required parts; its completeness is judged.
|
|
32
32
|
9. No live spend without the operator's word, recorded in a mission with a cap. Checked by `lab-spend-recorded`.
|
|
33
|
-
10.
|
|
33
|
+
10. The code a run executes is tested by the lab's own test suite. Prefer a command that calls the lab's own tested code (its package or its tools) over a script written for one experiment; keep in `bin/` only what is particular to this experiment, and test it there too. Judged, not checked.
|
|
34
34
|
11. Evidence is committed once, at the end of the run. Scratch and logs stay out of git. Checked by `lab-evidence-sealed`; the rest is judged.
|
|
35
35
|
12. Every document goes through the write skill. You never write one by hand. Checked by `folio check`.
|
|
36
36
|
|
|
@@ -45,14 +45,14 @@ You take one question from a draft protocol to a finished run with its evidence
|
|
|
45
45
|
- the decision rules `D1`..., with the kill rule marked `(kill)`;
|
|
46
46
|
- the predictions `P1`..., each with a confidence;
|
|
47
47
|
- what it cannot show.
|
|
48
|
-
2. **Make its folder.** Run `lab-kit experiment <slug>`.
|
|
48
|
+
2. **Make its folder.** Run `lab-kit experiment <slug>`. The run's command calls the lab's own tested code; anything particular to this experiment goes in `experiments/<slug>/bin/`, with its tests in the lab's suite. An experiment that needs nothing of its own leaves `bin/` empty. Run the lab's tests.
|
|
49
49
|
3. **Review the draft.** Send a reviewer with fresh context. Fold its points into the draft through the write skill. A protocol is cheap to fix before the lock and impossible to fix after.
|
|
50
|
-
4. **Lock.** Commit `bin/` first: the lock hashes
|
|
50
|
+
4. **Lock.** Commit `bin/` and the code the command calls first: the lock hashes `bin/` and records the commit, and `lab-lock-intact` fails if `bin/` changes after. The lab's code outside `bin/` may change later; the lock's commit says which version the run used, so a protocol that needs it fixed pins it by commit. Through the write skill, set the protocol's status to `locked`. Run `lab-kit lock <slug>`. Record it: `folio journal add --title .. --description .. --body .. --kind lock --about <slug>,<Q-n>`. Run `folio index`, then `lab-kit check`. Commit exactly the protocol, `lock.json`, the `lock` journal entry and `.folio/` together; that commit is the lock. From here folio's `frozen` check and `lab-lock-intact` hold it.
|
|
51
51
|
5. **De-risk.** Run the token-free stages first, with `lab-kit run`. Fix the apparatus only before the lock. After it, an apparatus fault ends the run as invalid.
|
|
52
52
|
6. **Launch.** Run `lab-kit run <slug> -- <command>`. Add `--spend --mission <file>` for a run that spends. Watch it at once: arm one watcher on its log that fires on the end marker or an error signature. A silent launch failure can waste a whole night.
|
|
53
53
|
7. **Supervise.** Write a journal entry of kind `run` about the protocol for each launch, checkpoint, surprise and termination, with `folio journal add`. Keep each entry to what changed. Record a deviation the moment you see it. Commit each entry with `.folio/` after `folio index`; never stage the run's working files.
|
|
54
54
|
8. **On failure.** End the run. Start a fresh one with the same configuration. Disclose it in a `run` entry. Never swap a model or a tool mid-run.
|
|
55
|
-
9. **Conclude.** When the run ends, `lab-kit run` writes the manifest of `out/`. Commit `out/`, `MANIFEST.sha256`, `work.sha256` and `run.json` in one commit; `work/` stays out of git, and `work.sha256` is what any copy of it is verified against. Run `lab-kit check`. Hand the experiment to the review skill. Do not score your own run.
|
|
55
|
+
9. **Conclude.** When the run ends, `lab-kit run` writes the manifest of `out/`. Commit `out/`, `MANIFEST.sha256`, `work.sha256` and `run.json` in one commit, with `store.json` when `lab.yaml` names a store; `work/` stays out of git, and `work.sha256` is what any copy of it is verified against. With a store, `lab-kit run` has already published `work/` there and deleted the local copy; if `run.log` says the publish failed, run `lab-kit publish --pending`. Never mount the store where the system under test runs: it holds other runs' answers. Run `lab-kit check`. Hand the experiment to the review skill. Do not score your own run.
|
|
56
56
|
|
|
57
57
|
## Stops
|
|
58
58
|
|
{lab_kit_cli-0.3.0 → lab_kit_cli-0.5.0/examples/monte-carlo-lab/.agents}/skills/review/SKILL.md
RENAMED
|
@@ -27,7 +27,7 @@ You turn a finished experiment's evidence into a scorecard against its lock, the
|
|
|
27
27
|
3. Score exactly what the lock names: every prediction and every rule, no more and no fewer. Checked by `lab-score-exact`.
|
|
28
28
|
4. A badly chosen rule is still the rule. Record the flaw as a lesson; never rescore. Judged, not checked.
|
|
29
29
|
5. A miss is a result. Nulls, fired kill rules and invalid arms are recorded at the same length as wins. Checked in part by `lab-scored-reported`; the rest is judged.
|
|
30
|
-
6. Every fold
|
|
30
|
+
6. Every fold is tested against a planted case with a known answer, in the lab's own test suite. Judged, not checked.
|
|
31
31
|
7. You score; the reviewer gates. Nothing is recorded as a result before a reviewer pass. Checked by `lab-result-grounded`.
|
|
32
32
|
8. Every document goes through folio: results and reports through workflows that folio's run skill follows, the rest through the write skill. Checked by `folio check`.
|
|
33
33
|
9. A report is frozen once `live`, and a result is permanent. A new result earns a new report, never a rewrite of an old one. folio shows a banner on the old report when a result it cites is superseded or retracted. Checked by folio's `frozen` and `permanent` checks.
|
|
@@ -36,7 +36,7 @@ You turn a finished experiment's evidence into a scorecard against its lock, the
|
|
|
36
36
|
|
|
37
37
|
## Steps
|
|
38
38
|
|
|
39
|
-
1. **Fold.** Run the experiment's fold script over each finished run's `out/`. If none exists, write one
|
|
39
|
+
1. **Fold.** Run the experiment's fold script over each finished run's `out/`. If none exists, write one, with a test on a planted case in the lab's suite, and land it as an instrument change. Never fold by eye.
|
|
40
40
|
2. **Score.** Run `lab-kit score <slug>`. Under `scores`, for every prediction `P1`..., fill HIT, MISS or INDETERMINATE, with the value and the evidence file it came from. For every decision rule `D1`..., say whether it fired, the kill rule included. Run `lab-kit check --only lab-score-exact`.
|
|
41
41
|
3. **Gate.** Ask the commander for a reviewer pass, or send one yourself if you are the commander. The reviewer re-derives the scored numbers. Record its verdict where the mission keeps verdicts, and point `score.yaml` to it: `review.verdict` and `review.record`.
|
|
42
42
|
4. **Choose what to record.** For each scored number, ask: is this a citable fact? A null counts. Add each one to `score.yaml` under `record`, keyed by a short name. Give it the result's fields: `title`, `protocol`, `number`, `baseline`, `bound`, `evidence` and `rederive`. The `number` is exactly what the re-derive command prints, and the evidence lies inside the run. Name every number you leave out.
|
|
@@ -55,7 +55,7 @@ No defensive fallbacks. No silent degradation. An arm whose tool fails is invali
|
|
|
55
55
|
|
|
56
56
|
An apparatus failure is a result to report, not a thing to patch around. Judged, not checked.
|
|
57
57
|
|
|
58
|
-
Every
|
|
58
|
+
Every piece of code a run executes is tested by the lab's own test suite, and the lab's gate runs it. lab-kit runs no lab code in its checks: it checks the record, the lab tests its code. Judged, not checked.
|
|
59
59
|
|
|
60
60
|
Every fold is tested against a planted case with a known answer. A fold that looks fine can still be blind to a renamed field. Judged, not checked.
|
|
61
61
|
|
|
@@ -1,6 +1,5 @@
|
|
|
1
|
-
lab-kit: 0.
|
|
1
|
+
lab-kit: 0.5.0 # the version this lab is checked with
|
|
2
2
|
library: docs # where folio.yaml is; "." for the root
|
|
3
3
|
front: lab # the front door's id: a project document
|
|
4
4
|
frozen: [substrate/] # surfaces never edited in place; record each with `lab-kit freeze`
|
|
5
|
-
tools: [] # lab-wide scripts; each must pass --selftest
|
|
6
5
|
rederive_timeout: 120 # seconds per re-derive command
|
|
@@ -55,7 +55,7 @@ No defensive fallbacks. No silent degradation. An arm whose tool fails is invali
|
|
|
55
55
|
|
|
56
56
|
An apparatus failure is a result to report, not a thing to patch around. Judged, not checked.
|
|
57
57
|
|
|
58
|
-
Every
|
|
58
|
+
Every piece of code a run executes is tested by the lab's own test suite, and the lab's gate runs it. lab-kit runs no lab code in its checks: it checks the record, the lab tests its code. Judged, not checked.
|
|
59
59
|
|
|
60
60
|
Every fold is tested against a planted case with a known answer. A fold that looks fine can still be blind to a renamed field. Judged, not checked.
|
|
61
61
|
|
|
@@ -5,7 +5,7 @@ build-backend = "hatchling.build"
|
|
|
5
5
|
[project]
|
|
6
6
|
dynamic = ["readme"]
|
|
7
7
|
name = "lab-kit-cli"
|
|
8
|
-
version = "0.
|
|
8
|
+
version = "0.5.0"
|
|
9
9
|
description = "A research lab's method and machinery on top of folio: pre-register, lock, run, score, re-derive."
|
|
10
10
|
license = "MIT"
|
|
11
11
|
authors = [{ name = "Piergiuseppe Mallozzi" }]
|
|
@@ -30,7 +30,7 @@ You take one question from a draft protocol to a finished run with its evidence
|
|
|
30
30
|
7. Fail loud. An arm whose tool fails is invalid, never downgraded. Undecidable is not a pass. Judged, not checked.
|
|
31
31
|
8. Name every move the agent under test may make, under the protocol's allowed moves. A move the list does not name is one it may never use. The section is checked by the protocol genre's required parts; its completeness is judged.
|
|
32
32
|
9. No live spend without the operator's word, recorded in a mission with a cap. Checked by `lab-spend-recorded`.
|
|
33
|
-
10.
|
|
33
|
+
10. The code a run executes is tested by the lab's own test suite. Prefer a command that calls the lab's own tested code (its package or its tools) over a script written for one experiment; keep in `bin/` only what is particular to this experiment, and test it there too. Judged, not checked.
|
|
34
34
|
11. Evidence is committed once, at the end of the run. Scratch and logs stay out of git. Checked by `lab-evidence-sealed`; the rest is judged.
|
|
35
35
|
12. Every document goes through the write skill. You never write one by hand. Checked by `folio check`.
|
|
36
36
|
|
|
@@ -45,14 +45,14 @@ You take one question from a draft protocol to a finished run with its evidence
|
|
|
45
45
|
- the decision rules `D1`..., with the kill rule marked `(kill)`;
|
|
46
46
|
- the predictions `P1`..., each with a confidence;
|
|
47
47
|
- what it cannot show.
|
|
48
|
-
2. **Make its folder.** Run `lab-kit experiment <slug>`.
|
|
48
|
+
2. **Make its folder.** Run `lab-kit experiment <slug>`. The run's command calls the lab's own tested code; anything particular to this experiment goes in `experiments/<slug>/bin/`, with its tests in the lab's suite. An experiment that needs nothing of its own leaves `bin/` empty. Run the lab's tests.
|
|
49
49
|
3. **Review the draft.** Send a reviewer with fresh context. Fold its points into the draft through the write skill. A protocol is cheap to fix before the lock and impossible to fix after.
|
|
50
|
-
4. **Lock.** Commit `bin/` first: the lock hashes
|
|
50
|
+
4. **Lock.** Commit `bin/` and the code the command calls first: the lock hashes `bin/` and records the commit, and `lab-lock-intact` fails if `bin/` changes after. The lab's code outside `bin/` may change later; the lock's commit says which version the run used, so a protocol that needs it fixed pins it by commit. Through the write skill, set the protocol's status to `locked`. Run `lab-kit lock <slug>`. Record it: `folio journal add --title .. --description .. --body .. --kind lock --about <slug>,<Q-n>`. Run `folio index`, then `lab-kit check`. Commit exactly the protocol, `lock.json`, the `lock` journal entry and `.folio/` together; that commit is the lock. From here folio's `frozen` check and `lab-lock-intact` hold it.
|
|
51
51
|
5. **De-risk.** Run the token-free stages first, with `lab-kit run`. Fix the apparatus only before the lock. After it, an apparatus fault ends the run as invalid.
|
|
52
52
|
6. **Launch.** Run `lab-kit run <slug> -- <command>`. Add `--spend --mission <file>` for a run that spends. Watch it at once: arm one watcher on its log that fires on the end marker or an error signature. A silent launch failure can waste a whole night.
|
|
53
53
|
7. **Supervise.** Write a journal entry of kind `run` about the protocol for each launch, checkpoint, surprise and termination, with `folio journal add`. Keep each entry to what changed. Record a deviation the moment you see it. Commit each entry with `.folio/` after `folio index`; never stage the run's working files.
|
|
54
54
|
8. **On failure.** End the run. Start a fresh one with the same configuration. Disclose it in a `run` entry. Never swap a model or a tool mid-run.
|
|
55
|
-
9. **Conclude.** When the run ends, `lab-kit run` writes the manifest of `out/`. Commit `out/`, `MANIFEST.sha256`, `work.sha256` and `run.json` in one commit; `work/` stays out of git, and `work.sha256` is what any copy of it is verified against. Run `lab-kit check`. Hand the experiment to the review skill. Do not score your own run.
|
|
55
|
+
9. **Conclude.** When the run ends, `lab-kit run` writes the manifest of `out/`. Commit `out/`, `MANIFEST.sha256`, `work.sha256` and `run.json` in one commit, with `store.json` when `lab.yaml` names a store; `work/` stays out of git, and `work.sha256` is what any copy of it is verified against. With a store, `lab-kit run` has already published `work/` there and deleted the local copy; if `run.log` says the publish failed, run `lab-kit publish --pending`. Never mount the store where the system under test runs: it holds other runs' answers. Run `lab-kit check`. Hand the experiment to the review skill. Do not score your own run.
|
|
56
56
|
|
|
57
57
|
## Stops
|
|
58
58
|
|
{lab_kit_cli-0.3.0/examples/monte-carlo-lab/.agents → lab_kit_cli-0.5.0}/skills/review/SKILL.md
RENAMED
|
@@ -27,7 +27,7 @@ You turn a finished experiment's evidence into a scorecard against its lock, the
|
|
|
27
27
|
3. Score exactly what the lock names: every prediction and every rule, no more and no fewer. Checked by `lab-score-exact`.
|
|
28
28
|
4. A badly chosen rule is still the rule. Record the flaw as a lesson; never rescore. Judged, not checked.
|
|
29
29
|
5. A miss is a result. Nulls, fired kill rules and invalid arms are recorded at the same length as wins. Checked in part by `lab-scored-reported`; the rest is judged.
|
|
30
|
-
6. Every fold
|
|
30
|
+
6. Every fold is tested against a planted case with a known answer, in the lab's own test suite. Judged, not checked.
|
|
31
31
|
7. You score; the reviewer gates. Nothing is recorded as a result before a reviewer pass. Checked by `lab-result-grounded`.
|
|
32
32
|
8. Every document goes through folio: results and reports through workflows that folio's run skill follows, the rest through the write skill. Checked by `folio check`.
|
|
33
33
|
9. A report is frozen once `live`, and a result is permanent. A new result earns a new report, never a rewrite of an old one. folio shows a banner on the old report when a result it cites is superseded or retracted. Checked by folio's `frozen` and `permanent` checks.
|
|
@@ -36,7 +36,7 @@ You turn a finished experiment's evidence into a scorecard against its lock, the
|
|
|
36
36
|
|
|
37
37
|
## Steps
|
|
38
38
|
|
|
39
|
-
1. **Fold.** Run the experiment's fold script over each finished run's `out/`. If none exists, write one
|
|
39
|
+
1. **Fold.** Run the experiment's fold script over each finished run's `out/`. If none exists, write one, with a test on a planted case in the lab's suite, and land it as an instrument change. Never fold by eye.
|
|
40
40
|
2. **Score.** Run `lab-kit score <slug>`. Under `scores`, for every prediction `P1`..., fill HIT, MISS or INDETERMINATE, with the value and the evidence file it came from. For every decision rule `D1`..., say whether it fired, the kill rule included. Run `lab-kit check --only lab-score-exact`.
|
|
41
41
|
3. **Gate.** Ask the commander for a reviewer pass, or send one yourself if you are the commander. The reviewer re-derives the scored numbers. Record its verdict where the mission keeps verdicts, and point `score.yaml` to it: `review.verdict` and `review.record`.
|
|
42
42
|
4. **Choose what to record.** For each scored number, ask: is this a citable fact? A null counts. Add each one to `score.yaml` under `record`, keyed by a short name. Give it the result's fields: `title`, `protocol`, `number`, `baseline`, `bound`, `evidence` and `rederive`. The `number` is exactly what the re-derive command prints, and the evidence lies inside the run. Name every number you leave out.
|
|
@@ -1,18 +1,15 @@
|
|
|
1
|
-
"""Checks on the lab's own files: ids, frozen surfaces,
|
|
1
|
+
"""Checks on the lab's own files: ids, frozen surfaces, spend, history, method and state."""
|
|
2
2
|
|
|
3
3
|
from __future__ import annotations
|
|
4
4
|
|
|
5
5
|
import json
|
|
6
6
|
import re
|
|
7
|
-
import subprocess
|
|
8
|
-
import sys
|
|
9
7
|
from pathlib import Path
|
|
10
8
|
from typing import Any
|
|
11
9
|
|
|
12
10
|
from folio.checks.rules import measurements
|
|
13
11
|
|
|
14
12
|
from .. import __version__, budgets, data, frozen, gitlog, ops, records
|
|
15
|
-
from ..records import script_env
|
|
16
13
|
from ..errors import LabError
|
|
17
14
|
from ..lab import Lab
|
|
18
15
|
from .base import Reporter
|
|
@@ -112,49 +109,6 @@ def frozen_intact(lab: Lab, rep: Reporter) -> None:
|
|
|
112
109
|
rep.add(path, f"changed: frozen surface `{surface}` is never edited in place")
|
|
113
110
|
|
|
114
111
|
|
|
115
|
-
def _command(path: Path) -> list[str] | None:
|
|
116
|
-
if path.suffix == ".py":
|
|
117
|
-
return [sys.executable, str(path)]
|
|
118
|
-
if path.suffix == ".sh":
|
|
119
|
-
return ["bash", str(path)]
|
|
120
|
-
if path.stat().st_mode & 0o111:
|
|
121
|
-
return [str(path)]
|
|
122
|
-
return None
|
|
123
|
-
|
|
124
|
-
|
|
125
|
-
def scripts(lab: Lab) -> list[Path]:
|
|
126
|
-
found: list[Path] = []
|
|
127
|
-
for slug in lab.experiment_slugs():
|
|
128
|
-
folder = lab.experiments / slug / "bin"
|
|
129
|
-
if folder.is_dir():
|
|
130
|
-
found.extend(p for p in sorted(folder.iterdir())
|
|
131
|
-
if p.is_file() and not p.name.startswith(".") and p.suffix not in (".pyc", ".md"))
|
|
132
|
-
found.extend(lab.root / t for t in lab.settings.tools)
|
|
133
|
-
return found
|
|
134
|
-
|
|
135
|
-
|
|
136
|
-
def selftest(lab: Lab, rep: Reporter) -> None:
|
|
137
|
-
"""Every script in experiments/*/bin/ and under `tools:` passes --selftest."""
|
|
138
|
-
for path in scripts(lab):
|
|
139
|
-
where = lab.rel(path) if path.exists() else path.relative_to(lab.root).as_posix()
|
|
140
|
-
if not path.is_file():
|
|
141
|
-
rep.add("lab.yaml", f"tool `{where}` does not exist")
|
|
142
|
-
continue
|
|
143
|
-
command = _command(path)
|
|
144
|
-
if command is None:
|
|
145
|
-
rep.add(where, "is neither a .py or .sh script nor executable, so it cannot run --selftest")
|
|
146
|
-
continue
|
|
147
|
-
try:
|
|
148
|
-
done = subprocess.run([*command, "--selftest"], cwd=lab.root, capture_output=True, text=True,
|
|
149
|
-
timeout=lab.settings.rederive_timeout, check=False, env=script_env())
|
|
150
|
-
except subprocess.TimeoutExpired:
|
|
151
|
-
rep.add(where, f"--selftest ran past {lab.settings.rederive_timeout}s")
|
|
152
|
-
continue
|
|
153
|
-
if done.returncode != 0:
|
|
154
|
-
tail = (done.stderr.strip() or done.stdout.strip()).splitlines()[-1:] or [""]
|
|
155
|
-
rep.add(where, f"--selftest exited {done.returncode}: {tail[0]}".rstrip(": "))
|
|
156
|
-
|
|
157
|
-
|
|
158
112
|
def mission_approved(lab: Lab, rep: Reporter) -> None:
|
|
159
113
|
"""Missions are draft, active or concluded; an active or concluded one records who approved it and when;
|
|
160
114
|
nothing runs under a draft, and the state never names a draft as active. A mission under a standing
|
|
@@ -138,3 +138,17 @@ def evidence_sealed(lab: Lab, rep: Reporter) -> None:
|
|
|
138
138
|
elif recorded[name] != actual[name]:
|
|
139
139
|
rep.add(manifest_path, f"out/{name} changed after the run ended")
|
|
140
140
|
rep.guard(run.path, one)
|
|
141
|
+
|
|
142
|
+
|
|
143
|
+
def work_stored(lab: Lab, rep: Reporter) -> None:
|
|
144
|
+
"""With a store in lab.yaml, every ended run with a work.sha256 records that it was published (warning)."""
|
|
145
|
+
if not lab.settings.store:
|
|
146
|
+
return
|
|
147
|
+
from ..store import STORE_RECORD
|
|
148
|
+
for run in records.runs(lab):
|
|
149
|
+
if not run.ended or not (run.folder / records.WORK_MANIFEST).is_file():
|
|
150
|
+
continue
|
|
151
|
+
if not (run.folder / STORE_RECORD).is_file():
|
|
152
|
+
rep.warn(run.path.rsplit("/", 1)[0] + "/" + records.WORK_MANIFEST,
|
|
153
|
+
f"its raw output was never published to the store {lab.settings.store}; "
|
|
154
|
+
"`lab-kit publish --pending` publishes it, then commit store.json")
|
|
@@ -23,13 +23,13 @@ CHECKS: tuple[LabCheck, ...] = (
|
|
|
23
23
|
LabCheck("lab-score-exact", "error", results.score_exact),
|
|
24
24
|
LabCheck("lab-ids-resolve", "error", files.ids_resolve),
|
|
25
25
|
LabCheck("lab-frozen-intact", "error", files.frozen_intact),
|
|
26
|
-
LabCheck("lab-selftest", "error", files.selftest),
|
|
27
26
|
LabCheck("lab-mission-approved", "error", files.mission_approved),
|
|
28
27
|
LabCheck("lab-spend-recorded", "error", files.spend_recorded),
|
|
29
28
|
LabCheck("lab-append-only", "error", files.append_only),
|
|
30
29
|
LabCheck("lab-method-current", "error", files.method_current),
|
|
31
30
|
LabCheck("lab-state-pointer", "warning", files.state_pointer),
|
|
32
31
|
LabCheck("lab-scored-reported", "warning", results.scored_reported),
|
|
32
|
+
LabCheck("lab-work-stored", "warning", locks.work_stored),
|
|
33
33
|
)
|
|
34
34
|
BY_NAME = {c.name: c for c in CHECKS}
|
|
35
35
|
|
|
@@ -10,6 +10,7 @@ from pathlib import Path
|
|
|
10
10
|
from . import __version__, frozen
|
|
11
11
|
from . import lab as lab_mod
|
|
12
12
|
from . import rederive as rederive_mod
|
|
13
|
+
from . import store as store_mod
|
|
13
14
|
from .checks import run as checks_run
|
|
14
15
|
from .commands import experiments, runs, setup, status
|
|
15
16
|
from .errors import LabError
|
|
@@ -65,6 +66,37 @@ def cmd_runs(args: argparse.Namespace) -> int:
|
|
|
65
66
|
return 0
|
|
66
67
|
|
|
67
68
|
|
|
69
|
+
def cmd_publish(args: argparse.Namespace) -> int:
|
|
70
|
+
lab = _lab()
|
|
71
|
+
if args.pending == bool(args.runs):
|
|
72
|
+
raise LabError("name runs as <slug>/<run id>, or pass --pending for every ended run still here")
|
|
73
|
+
store_mod.root(lab) # a lab with no store says so once
|
|
74
|
+
targets = store_mod.pending(lab) if args.pending else [store_mod.find_run(lab, r) for r in args.runs]
|
|
75
|
+
if not targets:
|
|
76
|
+
print("Nothing to publish.")
|
|
77
|
+
return 0
|
|
78
|
+
failed = 0
|
|
79
|
+
for run in targets:
|
|
80
|
+
try:
|
|
81
|
+
_print(store_mod.publish(lab, run, keep=args.keep))
|
|
82
|
+
except (LabError, OSError) as exc:
|
|
83
|
+
print(f"lab-kit: {lab.rel(run.folder)}: not published: {exc}", file=sys.stderr)
|
|
84
|
+
failed += 1
|
|
85
|
+
return 1 if failed else 0
|
|
86
|
+
|
|
87
|
+
|
|
88
|
+
def cmd_fetch(args: argparse.Namespace) -> int:
|
|
89
|
+
lab = _lab()
|
|
90
|
+
_print(store_mod.fetch(lab, store_mod.find_run(lab, args.run)))
|
|
91
|
+
return 0
|
|
92
|
+
|
|
93
|
+
|
|
94
|
+
def cmd_verify(args: argparse.Namespace) -> int:
|
|
95
|
+
lab = _lab()
|
|
96
|
+
_print(store_mod.verify(lab, store_mod.find_run(lab, args.run)))
|
|
97
|
+
return 0
|
|
98
|
+
|
|
99
|
+
|
|
68
100
|
def cmd_score(args: argparse.Namespace) -> int:
|
|
69
101
|
_print(experiments.score(_lab(), args.slug))
|
|
70
102
|
return 0
|
|
@@ -131,6 +163,21 @@ def parser() -> argparse.ArgumentParser:
|
|
|
131
163
|
s.add_argument("--live", action="store_true", help="only running and orphaned runs")
|
|
132
164
|
s.set_defaults(func=cmd_runs)
|
|
133
165
|
|
|
166
|
+
s = sub.add_parser("publish", help="Copy ended runs' raw output to the store named in lab.yaml, verify it "
|
|
167
|
+
"against work.sha256, and delete the local copy.")
|
|
168
|
+
s.add_argument("runs", nargs="*", metavar="SLUG/RUN_ID")
|
|
169
|
+
s.add_argument("--pending", action="store_true", help="every ended run whose work/ is still here")
|
|
170
|
+
s.add_argument("--keep", action="store_true", help="keep the local work/ and run.log")
|
|
171
|
+
s.set_defaults(func=cmd_publish)
|
|
172
|
+
|
|
173
|
+
s = sub.add_parser("fetch", help="Copy a published run's raw output back into its folder, verified.")
|
|
174
|
+
s.add_argument("run", metavar="SLUG/RUN_ID")
|
|
175
|
+
s.set_defaults(func=cmd_fetch)
|
|
176
|
+
|
|
177
|
+
s = sub.add_parser("verify", help="Check the store's copy of a run against its work.sha256.")
|
|
178
|
+
s.add_argument("run", metavar="SLUG/RUN_ID")
|
|
179
|
+
s.set_defaults(func=cmd_verify)
|
|
180
|
+
|
|
134
181
|
s = sub.add_parser("score", help="Create score.yaml from the lock's prediction and rule ids.")
|
|
135
182
|
s.add_argument("slug")
|
|
136
183
|
s.set_defaults(func=cmd_score)
|
|
@@ -26,7 +26,6 @@ lab-kit: {version} # the version this lab is checked with
|
|
|
26
26
|
library: {library} # where folio.yaml is; "." for the root
|
|
27
27
|
front: # the front door's id: a project document
|
|
28
28
|
frozen: [] # surfaces never edited in place; record each with `lab-kit freeze`
|
|
29
|
-
tools: [] # lab-wide scripts; each must pass --selftest
|
|
30
29
|
rederive_timeout: 120 # seconds per re-derive command
|
|
31
30
|
"""
|
|
32
31
|
|
|
@@ -14,7 +14,7 @@ from folio.library import Library
|
|
|
14
14
|
from .errors import LabError
|
|
15
15
|
|
|
16
16
|
LAB_YAML = "lab.yaml"
|
|
17
|
-
_KEYS = ("lab-kit", "library", "front", "frozen", "
|
|
17
|
+
_KEYS = ("lab-kit", "library", "front", "frozen", "rederive_timeout", "store")
|
|
18
18
|
|
|
19
19
|
|
|
20
20
|
@dataclass
|
|
@@ -23,8 +23,8 @@ class Settings:
|
|
|
23
23
|
library: str
|
|
24
24
|
front: str | None
|
|
25
25
|
frozen: list[str]
|
|
26
|
-
tools: list[str]
|
|
27
26
|
rederive_timeout: int
|
|
27
|
+
store: str | None = None
|
|
28
28
|
|
|
29
29
|
|
|
30
30
|
@dataclass
|
|
@@ -91,6 +91,9 @@ def find_root(start: Path) -> Path:
|
|
|
91
91
|
def parse_settings(raw: Any, where: str) -> Settings:
|
|
92
92
|
if not isinstance(raw, dict):
|
|
93
93
|
raise LabError(f"{where}: must be a mapping")
|
|
94
|
+
if "tools" in raw:
|
|
95
|
+
raise LabError(f"{where}: `tools` was removed in lab-kit 0.5.0, which no longer runs scripts' "
|
|
96
|
+
"--selftest; delete the key and test those scripts in the lab's own test suite")
|
|
94
97
|
unknown = sorted(set(raw) - set(_KEYS))
|
|
95
98
|
if unknown:
|
|
96
99
|
raise LabError(f"{where}: unknown key(s) {', '.join(unknown)}; the keys are {', '.join(_KEYS)}")
|
|
@@ -104,7 +107,7 @@ def parse_settings(raw: Any, where: str) -> Settings:
|
|
|
104
107
|
if front is not None and not isinstance(front, str):
|
|
105
108
|
raise LabError(f"{where}: `front` must be a document id")
|
|
106
109
|
lists = {}
|
|
107
|
-
for key in ("frozen",
|
|
110
|
+
for key in ("frozen",):
|
|
108
111
|
value = raw.get(key) or []
|
|
109
112
|
if not isinstance(value, list) or not all(isinstance(v, str) and v for v in value):
|
|
110
113
|
raise LabError(f"{where}: `{key}` must be a list of paths from the lab root")
|
|
@@ -115,7 +118,10 @@ def parse_settings(raw: Any, where: str) -> Settings:
|
|
|
115
118
|
timeout = raw.get("rederive_timeout", 120)
|
|
116
119
|
if not isinstance(timeout, int) or isinstance(timeout, bool) or timeout <= 0:
|
|
117
120
|
raise LabError(f"{where}: `rederive_timeout` must be a positive whole number of seconds")
|
|
118
|
-
|
|
121
|
+
store = raw.get("store")
|
|
122
|
+
if store is not None and (not isinstance(store, str) or not Path(store).is_absolute()):
|
|
123
|
+
raise LabError(f"{where}: `store` must be an absolute path, such as /labs/<lab>")
|
|
124
|
+
return Settings(version, library, front or None, lists["frozen"], timeout, store or None)
|
|
119
125
|
|
|
120
126
|
|
|
121
127
|
def load_at(root: Path) -> Lab:
|