lab-kit-cli 0.2.0__tar.gz → 0.4.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {lab_kit_cli-0.2.0 → lab_kit_cli-0.4.0}/CHANGELOG.md +16 -0
- {lab_kit_cli-0.2.0 → lab_kit_cli-0.4.0}/PKG-INFO +1 -1
- {lab_kit_cli-0.2.0 → lab_kit_cli-0.4.0}/docs/spec/lab-model.md +22 -10
- {lab_kit_cli-0.2.0 → lab_kit_cli-0.4.0}/examples/monte-carlo-lab/.agents/skills/experiment/SKILL.md +2 -2
- {lab_kit_cli-0.2.0 → lab_kit_cli-0.4.0}/examples/monte-carlo-lab/lab.yaml +1 -1
- {lab_kit_cli-0.2.0 → lab_kit_cli-0.4.0}/pyproject.toml +1 -1
- {lab_kit_cli-0.2.0 → lab_kit_cli-0.4.0}/skills/experiment/SKILL.md +2 -2
- {lab_kit_cli-0.2.0 → lab_kit_cli-0.4.0}/src/lab_kit/__init__.py +1 -1
- {lab_kit_cli-0.2.0 → lab_kit_cli-0.4.0}/src/lab_kit/checks/locks.py +21 -1
- {lab_kit_cli-0.2.0 → lab_kit_cli-0.4.0}/src/lab_kit/checks/run.py +1 -0
- {lab_kit_cli-0.2.0 → lab_kit_cli-0.4.0}/src/lab_kit/cli.py +47 -0
- {lab_kit_cli-0.2.0 → lab_kit_cli-0.4.0}/src/lab_kit/commands/experiments.py +2 -0
- {lab_kit_cli-0.2.0 → lab_kit_cli-0.4.0}/src/lab_kit/commands/runs.py +3 -1
- {lab_kit_cli-0.2.0 → lab_kit_cli-0.4.0}/src/lab_kit/lab.py +6 -2
- {lab_kit_cli-0.2.0 → lab_kit_cli-0.4.0}/src/lab_kit/records.py +43 -2
- lab_kit_cli-0.4.0/src/lab_kit/store.py +285 -0
- {lab_kit_cli-0.2.0 → lab_kit_cli-0.4.0}/src/lab_kit/supervise.py +24 -1
- lab_kit_cli-0.4.0/tests/test_instrument.py +93 -0
- lab_kit_cli-0.4.0/tests/test_store.py +108 -0
- {lab_kit_cli-0.2.0 → lab_kit_cli-0.4.0}/.github/workflows/ci.yml +0 -0
- {lab_kit_cli-0.2.0 → lab_kit_cli-0.4.0}/.github/workflows/policy.yml +0 -0
- {lab_kit_cli-0.2.0 → lab_kit_cli-0.4.0}/.github/workflows/release.yml +0 -0
- {lab_kit_cli-0.2.0 → lab_kit_cli-0.4.0}/.gitignore +0 -0
- {lab_kit_cli-0.2.0 → lab_kit_cli-0.4.0}/AGENTS.md +0 -0
- {lab_kit_cli-0.2.0 → lab_kit_cli-0.4.0}/CLAUDE.md +0 -0
- {lab_kit_cli-0.2.0 → lab_kit_cli-0.4.0}/CONTRIBUTING.md +0 -0
- {lab_kit_cli-0.2.0 → lab_kit_cli-0.4.0}/LICENSE +0 -0
- {lab_kit_cli-0.2.0 → lab_kit_cli-0.4.0}/README.md +0 -0
- {lab_kit_cli-0.2.0 → lab_kit_cli-0.4.0}/SETUP.md +0 -0
- {lab_kit_cli-0.2.0 → lab_kit_cli-0.4.0}/agents/reporter.md +0 -0
- {lab_kit_cli-0.2.0 → lab_kit_cli-0.4.0}/agents/reviewer.md +0 -0
- {lab_kit_cli-0.2.0 → lab_kit_cli-0.4.0}/agents/runner.md +0 -0
- {lab_kit_cli-0.2.0 → lab_kit_cli-0.4.0}/agents/scout.md +0 -0
- {lab_kit_cli-0.2.0 → lab_kit_cli-0.4.0}/docs/figures/how-lab-kit-works-dark.svg +0 -0
- {lab_kit_cli-0.2.0 → lab_kit_cli-0.4.0}/docs/figures/how-lab-kit-works-light.svg +0 -0
- {lab_kit_cli-0.2.0 → lab_kit_cli-0.4.0}/docs/figures/lab-loop-dark.svg +0 -0
- {lab_kit_cli-0.2.0 → lab_kit_cli-0.4.0}/docs/figures/lab-loop-light.svg +0 -0
- {lab_kit_cli-0.2.0 → lab_kit_cli-0.4.0}/docs/figures/number-chain-dark.svg +0 -0
- {lab_kit_cli-0.2.0 → lab_kit_cli-0.4.0}/docs/figures/number-chain-light.svg +0 -0
- {lab_kit_cli-0.2.0 → lab_kit_cli-0.4.0}/examples/monte-carlo-lab/.agents/agents/reporter.md +0 -0
- {lab_kit_cli-0.2.0 → lab_kit_cli-0.4.0}/examples/monte-carlo-lab/.agents/agents/reviewer.md +0 -0
- {lab_kit_cli-0.2.0 → lab_kit_cli-0.4.0}/examples/monte-carlo-lab/.agents/agents/runner.md +0 -0
- {lab_kit_cli-0.2.0 → lab_kit_cli-0.4.0}/examples/monte-carlo-lab/.agents/agents/scout.md +0 -0
- {lab_kit_cli-0.2.0 → lab_kit_cli-0.4.0}/examples/monte-carlo-lab/.agents/skills/address/SKILL.md +0 -0
- {lab_kit_cli-0.2.0 → lab_kit_cli-0.4.0}/examples/monte-carlo-lab/.agents/skills/configure/SKILL.md +0 -0
- {lab_kit_cli-0.2.0 → lab_kit_cli-0.4.0}/examples/monte-carlo-lab/.agents/skills/organise/SKILL.md +0 -0
- {lab_kit_cli-0.2.0 → lab_kit_cli-0.4.0}/examples/monte-carlo-lab/.agents/skills/plan-mission/SKILL.md +0 -0
- {lab_kit_cli-0.2.0 → lab_kit_cli-0.4.0}/examples/monte-carlo-lab/.agents/skills/publish/SKILL.md +0 -0
- {lab_kit_cli-0.2.0 → lab_kit_cli-0.4.0}/examples/monte-carlo-lab/.agents/skills/review/SKILL.md +0 -0
- {lab_kit_cli-0.2.0 → lab_kit_cli-0.4.0}/examples/monte-carlo-lab/.agents/skills/run/SKILL.md +0 -0
- {lab_kit_cli-0.2.0 → lab_kit_cli-0.4.0}/examples/monte-carlo-lab/.agents/skills/run-mission/SKILL.md +0 -0
- {lab_kit_cli-0.2.0 → lab_kit_cli-0.4.0}/examples/monte-carlo-lab/.agents/skills/set-up/SKILL.md +0 -0
- {lab_kit_cli-0.2.0 → lab_kit_cli-0.4.0}/examples/monte-carlo-lab/.agents/skills/set-up-lab/SKILL.md +0 -0
- {lab_kit_cli-0.2.0 → lab_kit_cli-0.4.0}/examples/monte-carlo-lab/.agents/skills/write/SKILL.md +0 -0
- {lab_kit_cli-0.2.0 → lab_kit_cli-0.4.0}/examples/monte-carlo-lab/.gitignore +0 -0
- {lab_kit_cli-0.2.0 → lab_kit_cli-0.4.0}/examples/monte-carlo-lab/.lab/frozen.sha256 +0 -0
- {lab_kit_cli-0.2.0 → lab_kit_cli-0.4.0}/examples/monte-carlo-lab/.lab/method/DISCIPLINE.md +0 -0
- {lab_kit_cli-0.2.0 → lab_kit_cli-0.4.0}/examples/monte-carlo-lab/.lab/method/LADDER.md +0 -0
- {lab_kit_cli-0.2.0 → lab_kit_cli-0.4.0}/examples/monte-carlo-lab/AGENTS.md +0 -0
- {lab_kit_cli-0.2.0 → lab_kit_cli-0.4.0}/examples/monte-carlo-lab/CLAUDE.md +0 -0
- {lab_kit_cli-0.2.0 → lab_kit_cli-0.4.0}/examples/monte-carlo-lab/docs/.folio/backlinks.json +0 -0
- {lab_kit_cli-0.2.0 → lab_kit_cli-0.4.0}/examples/monte-carlo-lab/docs/.folio/cards.json +0 -0
- {lab_kit_cli-0.2.0 → lab_kit_cli-0.4.0}/examples/monte-carlo-lab/docs/.folio/catalog.json +0 -0
- {lab_kit_cli-0.2.0 → lab_kit_cli-0.4.0}/examples/monte-carlo-lab/docs/.folio/journal.json +0 -0
- {lab_kit_cli-0.2.0 → lab_kit_cli-0.4.0}/examples/monte-carlo-lab/docs/.folio/nav.json +0 -0
- {lab_kit_cli-0.2.0 → lab_kit_cli-0.4.0}/examples/monte-carlo-lab/docs/.folio/search.json +0 -0
- {lab_kit_cli-0.2.0 → lab_kit_cli-0.4.0}/examples/monte-carlo-lab/docs/.gitignore +0 -0
- {lab_kit_cli-0.2.0 → lab_kit_cli-0.4.0}/examples/monte-carlo-lab/docs/assets/data/.gitkeep +0 -0
- {lab_kit_cli-0.2.0 → lab_kit_cli-0.4.0}/examples/monte-carlo-lab/docs/assets/figures/.gitkeep +0 -0
- {lab_kit_cli-0.2.0 → lab_kit_cli-0.4.0}/examples/monte-carlo-lab/docs/assets/refs.bib +0 -0
- {lab_kit_cli-0.2.0 → lab_kit_cli-0.4.0}/examples/monte-carlo-lab/docs/content/claims/C-1.md +0 -0
- {lab_kit_cli-0.2.0 → lab_kit_cli-0.4.0}/examples/monte-carlo-lab/docs/content/index.html +0 -0
- {lab_kit_cli-0.2.0 → lab_kit_cli-0.4.0}/examples/monte-carlo-lab/docs/content/journal/2026/2026-10-04-concluded-the-pi-error-scaling-mission.md +0 -0
- {lab_kit_cli-0.2.0 → lab_kit_cli-0.4.0}/examples/monte-carlo-lab/docs/content/journal/2026/2026-10-04-lab-set-up.md +0 -0
- {lab_kit_cli-0.2.0 → lab_kit_cli-0.4.0}/examples/monte-carlo-lab/docs/content/journal/2026/2026-10-04-locked-the-pi-error-scaling-protocol.md +0 -0
- {lab_kit_cli-0.2.0 → lab_kit_cli-0.4.0}/examples/monte-carlo-lab/docs/content/journal/2026/2026-10-04-opened-the-pi-error-scaling-mission.md +0 -0
- {lab_kit_cli-0.2.0 → lab_kit_cli-0.4.0}/examples/monte-carlo-lab/docs/content/journal/2026/2026-10-04-r-1-fitted-the-mean-absolute-error.md +0 -0
- {lab_kit_cli-0.2.0 → lab_kit_cli-0.4.0}/examples/monte-carlo-lab/docs/content/journal/2026/2026-10-04-ran-the-pi-error-scaling-protocol.md +0 -0
- {lab_kit_cli-0.2.0 → lab_kit_cli-0.4.0}/examples/monte-carlo-lab/docs/content/journal/2026/2026-10-04-recorded-r-1.md +0 -0
- {lab_kit_cli-0.2.0 → lab_kit_cli-0.4.0}/examples/monte-carlo-lab/docs/content/journal/2026/2026-10-04-recorded-r-2.md +0 -0
- {lab_kit_cli-0.2.0 → lab_kit_cli-0.4.0}/examples/monte-carlo-lab/docs/content/journal/2026/2026-10-04-reported-the-pi-error-scaling-experiment.md +0 -0
- {lab_kit_cli-0.2.0 → lab_kit_cli-0.4.0}/examples/monte-carlo-lab/docs/content/journal/2026/2026-10-04-the-square-root-law-holds-per-size-predictions.md +0 -0
- {lab_kit_cli-0.2.0 → lab_kit_cli-0.4.0}/examples/monte-carlo-lab/docs/content/maps/research.html +0 -0
- {lab_kit_cli-0.2.0 → lab_kit_cli-0.4.0}/examples/monte-carlo-lab/docs/content/projects/lab/index.html +0 -0
- {lab_kit_cli-0.2.0 → lab_kit_cli-0.4.0}/examples/monte-carlo-lab/docs/content/protocols/pi-error-scaling.md +0 -0
- {lab_kit_cli-0.2.0 → lab_kit_cli-0.4.0}/examples/monte-carlo-lab/docs/content/questions/Q-1.md +0 -0
- {lab_kit_cli-0.2.0 → lab_kit_cli-0.4.0}/examples/monte-carlo-lab/docs/content/reports/pi-error-scaling-report/index.html +0 -0
- {lab_kit_cli-0.2.0 → lab_kit_cli-0.4.0}/examples/monte-carlo-lab/docs/content/results/R-1.md +0 -0
- {lab_kit_cli-0.2.0 → lab_kit_cli-0.4.0}/examples/monte-carlo-lab/docs/content/results/R-2.md +0 -0
- {lab_kit_cli-0.2.0 → lab_kit_cli-0.4.0}/examples/monte-carlo-lab/docs/folio.yaml +0 -0
- {lab_kit_cli-0.2.0 → lab_kit_cli-0.4.0}/examples/monte-carlo-lab/experiments/.gitkeep +0 -0
- {lab_kit_cli-0.2.0 → lab_kit_cli-0.4.0}/examples/monte-carlo-lab/experiments/pi-error-scaling/.gitignore +0 -0
- {lab_kit_cli-0.2.0 → lab_kit_cli-0.4.0}/examples/monte-carlo-lab/experiments/pi-error-scaling/bin/.gitkeep +0 -0
- {lab_kit_cli-0.2.0 → lab_kit_cli-0.4.0}/examples/monte-carlo-lab/experiments/pi-error-scaling/bin/fold.py +0 -0
- {lab_kit_cli-0.2.0 → lab_kit_cli-0.4.0}/examples/monte-carlo-lab/experiments/pi-error-scaling/bin/sample.py +0 -0
- {lab_kit_cli-0.2.0 → lab_kit_cli-0.4.0}/examples/monte-carlo-lab/experiments/pi-error-scaling/lock.json +0 -0
- {lab_kit_cli-0.2.0 → lab_kit_cli-0.4.0}/examples/monte-carlo-lab/experiments/pi-error-scaling/runs/.gitkeep +0 -0
- {lab_kit_cli-0.2.0 → lab_kit_cli-0.4.0}/examples/monte-carlo-lab/experiments/pi-error-scaling/runs/20261004-231629/MANIFEST.sha256 +0 -0
- {lab_kit_cli-0.2.0 → lab_kit_cli-0.4.0}/examples/monte-carlo-lab/experiments/pi-error-scaling/runs/20261004-231629/out/estimates.tsv +0 -0
- {lab_kit_cli-0.2.0 → lab_kit_cli-0.4.0}/examples/monte-carlo-lab/experiments/pi-error-scaling/runs/20261004-231629/run.json +0 -0
- {lab_kit_cli-0.2.0 → lab_kit_cli-0.4.0}/examples/monte-carlo-lab/experiments/pi-error-scaling/score.yaml +0 -0
- {lab_kit_cli-0.2.0 → lab_kit_cli-0.4.0}/examples/monte-carlo-lab/ops/STATE.md +0 -0
- {lab_kit_cli-0.2.0 → lab_kit_cli-0.4.0}/examples/monte-carlo-lab/ops/missions/.gitkeep +0 -0
- {lab_kit_cli-0.2.0 → lab_kit_cli-0.4.0}/examples/monte-carlo-lab/ops/missions/2026-10-04-pi-error-scaling.md +0 -0
- {lab_kit_cli-0.2.0 → lab_kit_cli-0.4.0}/examples/monte-carlo-lab/substrate/README.md +0 -0
- {lab_kit_cli-0.2.0 → lab_kit_cli-0.4.0}/examples/monte-carlo-lab/substrate/estimator.py +0 -0
- {lab_kit_cli-0.2.0 → lab_kit_cli-0.4.0}/method/DISCIPLINE.md +0 -0
- {lab_kit_cli-0.2.0 → lab_kit_cli-0.4.0}/method/LADDER.md +0 -0
- {lab_kit_cli-0.2.0 → lab_kit_cli-0.4.0}/skills/plan-mission/SKILL.md +0 -0
- {lab_kit_cli-0.2.0 → lab_kit_cli-0.4.0}/skills/review/SKILL.md +0 -0
- {lab_kit_cli-0.2.0 → lab_kit_cli-0.4.0}/skills/run-mission/SKILL.md +0 -0
- {lab_kit_cli-0.2.0 → lab_kit_cli-0.4.0}/skills/set-up-lab/SKILL.md +0 -0
- {lab_kit_cli-0.2.0 → lab_kit_cli-0.4.0}/src/lab_kit/__main__.py +0 -0
- {lab_kit_cli-0.2.0 → lab_kit_cli-0.4.0}/src/lab_kit/budgets.py +0 -0
- {lab_kit_cli-0.2.0 → lab_kit_cli-0.4.0}/src/lab_kit/checks/__init__.py +0 -0
- {lab_kit_cli-0.2.0 → lab_kit_cli-0.4.0}/src/lab_kit/checks/base.py +0 -0
- {lab_kit_cli-0.2.0 → lab_kit_cli-0.4.0}/src/lab_kit/checks/files.py +0 -0
- {lab_kit_cli-0.2.0 → lab_kit_cli-0.4.0}/src/lab_kit/checks/results.py +0 -0
- {lab_kit_cli-0.2.0 → lab_kit_cli-0.4.0}/src/lab_kit/commands/__init__.py +0 -0
- {lab_kit_cli-0.2.0 → lab_kit_cli-0.4.0}/src/lab_kit/commands/setup.py +0 -0
- {lab_kit_cli-0.2.0 → lab_kit_cli-0.4.0}/src/lab_kit/commands/status.py +0 -0
- {lab_kit_cli-0.2.0 → lab_kit_cli-0.4.0}/src/lab_kit/data.py +0 -0
- {lab_kit_cli-0.2.0 → lab_kit_cli-0.4.0}/src/lab_kit/errors.py +0 -0
- {lab_kit_cli-0.2.0 → lab_kit_cli-0.4.0}/src/lab_kit/frozen.py +0 -0
- {lab_kit_cli-0.2.0 → lab_kit_cli-0.4.0}/src/lab_kit/gitlog.py +0 -0
- {lab_kit_cli-0.2.0 → lab_kit_cli-0.4.0}/src/lab_kit/ops.py +0 -0
- {lab_kit_cli-0.2.0 → lab_kit_cli-0.4.0}/src/lab_kit/protocol.py +0 -0
- {lab_kit_cli-0.2.0 → lab_kit_cli-0.4.0}/src/lab_kit/rederive.py +0 -0
- {lab_kit_cli-0.2.0 → lab_kit_cli-0.4.0}/tests/conftest.py +0 -0
- {lab_kit_cli-0.2.0 → lab_kit_cli-0.4.0}/tests/test_budgets.py +0 -0
- {lab_kit_cli-0.2.0 → lab_kit_cli-0.4.0}/tests/test_checks.py +0 -0
- {lab_kit_cli-0.2.0 → lab_kit_cli-0.4.0}/tests/test_end_to_end.py +0 -0
- {lab_kit_cli-0.2.0 → lab_kit_cli-0.4.0}/tests/test_example.py +0 -0
- {lab_kit_cli-0.2.0 → lab_kit_cli-0.4.0}/tests/test_runs.py +0 -0
|
@@ -1,5 +1,21 @@
|
|
|
1
1
|
# Changelog
|
|
2
2
|
|
|
3
|
+
## 0.4.0
|
|
4
|
+
|
|
5
|
+
- The run store. `store:` in `lab.yaml` names one shared filesystem, such as `/labs/<lab>`, where every finished run's raw output goes. When a run ends, `lab-kit run` publishes `work/` and `run.log` to the store at the run folder's own path: a copy into `.incoming/`, checked against `work.sha256` (nothing missing, extra or changed), renamed into place, made read-only. It then writes `store.json` beside `run.json` and deletes the local copy. A run in the store is never overwritten. A failed publish is noted in `run.log`, keeps the local copy and never changes the run's exit code.
|
|
6
|
+
- `lab-kit publish <slug>/<run id>... | --pending [--keep]`, `lab-kit fetch <slug>/<run id>` and `lab-kit verify <slug>/<run id>`. A machine that does not mount the store sets `LAB_STORE_HOST` to one that does (rsync over ssh, `sha256sum` there).
|
|
7
|
+
- `lab-work-stored` (warning): with a store set, every ended run with a `work.sha256` has a `store.json`.
|
|
8
|
+
- The experiment skill says to commit `store.json` with the run, and never to mount the store where the system under test runs.
|
|
9
|
+
- A lab without `store:` works as before.
|
|
10
|
+
|
|
11
|
+
## 0.3.0
|
|
12
|
+
|
|
13
|
+
- The lock covers the instrument. `lab-kit lock` records, beside the protocol's hash, `instrument` (one sha256 over the files in the experiment's `bin/`, sorted by path, each fed as its path and its bytes, `__pycache__` left out), `commit` (the git head) and `lab-kit` (the version). `lab-lock-intact` fails when `bin/` no longer hashes to `instrument`, and `lab-kit run` refuses to start on such a lock, as it already did for a changed protocol.
|
|
14
|
+
- Records name their commit. `lab-kit run` writes `commit` and `lab-kit` into `run.json` when it starts.
|
|
15
|
+
- Raw output is sealed. When the command exits, `lab-kit run` writes `work.sha256` beside `MANIFEST.sha256`, one line per file under `work/`, when that folder exists. `work/` stays out of git; its hashes go in, so a copy kept elsewhere can be verified.
|
|
16
|
+
- A lock or a run record without the new fields predates them and passes: records are never edited. The example lab's records are unchanged and are that test.
|
|
17
|
+
- The experiment skill says to commit `bin/` before the lock and to commit `work.sha256` with the run.
|
|
18
|
+
|
|
3
19
|
## 0.2.0
|
|
4
20
|
|
|
5
21
|
- Standing budgets. The operator can grant money once for many missions: `ops/budgets/<name>.yaml` lists grants, each with an amount and who granted it and when. A mission names the budget it draws on in `budget:`. `lab-mission-approved` checks that what the missions under a budget commit (an active mission's cap, or its spend when that is more; a concluded mission's spend) fits inside its grants; `lab-append-only` keeps grants append-only and the unit fixed. `lab-kit status` shows each budget's granted, committed and left amounts.
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.5
|
|
2
2
|
Name: lab-kit-cli
|
|
3
|
-
Version: 0.
|
|
3
|
+
Version: 0.4.0
|
|
4
4
|
Summary: A research lab's method and machinery on top of folio: pre-register, lock, run, score, re-derive.
|
|
5
5
|
Project-URL: Homepage, https://github.com/pierg/lab-kit
|
|
6
6
|
Project-URL: Issues, https://github.com/pierg/lab-kit/issues
|
|
@@ -45,12 +45,15 @@ A lab is a repository. Its folio library sits at the root or in a folder such as
|
|
|
45
45
|
... maps, concepts, the front door, papers
|
|
46
46
|
experiments/{slug}/ one per protocol, same slug
|
|
47
47
|
bin/ the lab's own scripts for this experiment, each with --selftest
|
|
48
|
-
lock.json the lock record: protocol id, its sha256 with the status field set aside, the lock time (UTC)
|
|
48
|
+
lock.json the lock record: protocol id, its sha256 with the status field set aside, the lock time (UTC),
|
|
49
|
+
the instrument (a hash of bin/), the git commit and the lab-kit version
|
|
49
50
|
runs/{run-id}/
|
|
50
|
-
run.json protocol hash, frozen configuration, mission, start, end, exit code
|
|
51
|
+
run.json protocol hash, frozen configuration, mission, start, end, exit code, commit, lab-kit version
|
|
51
52
|
out/ the evidence, committed at the end of the run, never edited
|
|
52
53
|
MANIFEST.sha256 a hash of every file in out/, written when the run ends
|
|
53
|
-
work/ run.log
|
|
54
|
+
work/ run.log raw output and log, gitignored
|
|
55
|
+
work.sha256 a hash of every file in work/, written when the run ends; committed
|
|
56
|
+
store.json where work/ and run.log were published, and when (when lab.yaml names a store); committed
|
|
54
57
|
score.yaml the review's scorecard against the lock, and the results to record
|
|
55
58
|
ops/
|
|
56
59
|
STATE.md what is true now and what is next; a pointer, never a record
|
|
@@ -67,20 +70,25 @@ A lab is a repository. Its folio library sits at the root or in a folder such as
|
|
|
67
70
|
`lab.yaml`:
|
|
68
71
|
|
|
69
72
|
```yaml
|
|
70
|
-
lab-kit: 0.
|
|
73
|
+
lab-kit: 0.4.0 # the version this lab is checked with
|
|
71
74
|
library: docs # where folio.yaml is; "." for the root
|
|
72
75
|
front: lab # the front door's id: a project document
|
|
73
76
|
frozen: # surfaces never edited in place
|
|
74
77
|
- substrate/
|
|
75
78
|
tools: [tools/fold.py] # lab-wide scripts; each must pass --selftest
|
|
76
79
|
rederive_timeout: 120 # seconds per re-derive command
|
|
80
|
+
store: /labs/monte-carlo-lab # optional: where finished runs' raw output is published (below)
|
|
77
81
|
```
|
|
78
82
|
|
|
79
83
|
Paths in `lab.yaml`, `run.json`, `score.yaml`, and a result's `evidence` and `rederive` fields, are relative to the lab root. folio resolves a result's paths from the charter's `root`, so a lab whose library is in `docs/` sets `root: ..` in `folio.yaml`. Both tools then read the same path the same way.
|
|
80
84
|
|
|
81
85
|
`score.yaml` names its `protocol` and the lock's hash (`lock`), and has three parts. `review` holds the independent reviewer's `verdict` (`pass` or `fail`) and `record`, the path of the file where that verdict is recorded, usually the mission file. `scores` holds one entry per prediction and decision rule the lock names, keyed by its id (`P1`, `D1`): a verdict, the value and the evidence it came from. A prediction's verdict is `HIT`, `MISS` or `INDETERMINATE`; a rule's is `FIRED`, `NOT FIRED` or `INDETERMINATE`. `record` holds one entry per number to record, keyed by a short name, with the result's field names: `title`, `protocol`, `number`, `baseline`, `bound`, `evidence`, `rederive`. The review hands one entry to `record-a-result` as `from: experiments/<slug>/score.yaml#<name>`.
|
|
82
86
|
|
|
83
|
-
`run.json` holds `protocol`, `lock` (the hash the run was checked against), `config` (the pinned configuration, copied from the protocol), `mission`, `spend`, `command`, `started`, `ended`, `exit`, `pid` and `
|
|
87
|
+
`run.json` holds `protocol`, `lock` (the hash the run was checked against), `config` (the pinned configuration, copied from the protocol), `mission`, `spend`, `command`, `started`, `ended`, `exit`, `pid`, `spent`, `commit` (the git head the run started at; null outside git or before the first commit) and `lab-kit` (the version that started it). Times are UTC, to the second. A run's command finds its folders in the environment: `LAB_RUN_DIR`, `LAB_OUT` and `LAB_WORK`. A run that spends writes `out/spend.json` with `{"spent": <number>}`, in the unit of its mission's cap, and `lab-kit run` copies it into `spent` when the command exits. When the command exits, `lab-kit run` also writes `work.sha256` beside `MANIFEST.sha256`, in the same format, one line per file under `work/`, when that folder exists: the raw output stays out of git, and its hashes go in, so any copy of it can be verified.
|
|
88
|
+
|
|
89
|
+
**The store.** `store:` in `lab.yaml` is optional: an absolute path, such as `/labs/<lab>`, where every machine that runs the lab mounts one shared filesystem. When it is set, a run's raw output leaves the machine that ran it once the run ends: `lab-kit run` publishes `work/` and `run.log` to `<store>/<the run folder's path from the lab root>/`. Publishing copies into `<store>/.incoming/`, checks the copy against `work.sha256` (no file missing, none extra, none changed), renames it into place in one step, makes it read-only, writes `store.json` (`store`, `files`, `published`, `by`) beside `run.json`, and then deletes the local `work/` and `run.log`. A run already in the store is never overwritten: a copy there that verifies counts as published, one that does not is refused. A publish that fails leaves the run where it is, notes why in `run.log`, and never changes the run's exit code; `lab-kit publish --pending` tries again. The command a run starts is never told the store's path, and a lab's instrument never mounts it into the sandbox where the system under test runs: the store holds other runs' raw output, answers included. A machine that does not mount the store sets `LAB_STORE_HOST` to a host that does; lab-kit then copies with rsync over ssh and verifies there with `sha256sum`.
|
|
90
|
+
|
|
91
|
+
`lock.json` holds `protocol`, `path`, `sha256` and `locked`, and from lab-kit 0.3.0 `instrument`, `commit` and `lab-kit`. `instrument` is one sha256 over every file under `experiments/<slug>/bin/` (a `__pycache__` folder left out), sorted by its path from `bin/`, each fed as its path, a newline, then its bytes. `commit` is the git head at the lock (null outside git or before the first commit), and `lab-kit` the version that took it. A lock or a run record without these fields predates them and is read as it is: records are never edited.
|
|
84
92
|
|
|
85
93
|
A mission file starts with YAML front matter: `title`, `status` (`draft`, `active` or `concluded`), `rests_on` (the ids it rests on), `approved` (who approved the mission, and when, with the date as `YYYY-MM-DD`) and `cap` (a number, in the unit its runs report spend; `0` when nothing spends), and optionally `budget` (the name of a standing budget it draws on). A mission is `draft` until it is approved; `approved` is required from `active` on. Its log is everything under its `## Log` heading; entries are only added at the bottom. `ops/STATE.md` has one line `Active mission: <path>` or `Active mission: none`, and never names a draft.
|
|
86
94
|
|
|
@@ -99,8 +107,8 @@ question --> protocol (draft) --> review of the draft --> lock --> run --> evide
|
|
|
99
107
|
1. **Question.** The write skill adds or revises a question `Q-n`. It is ranked, and it says what would settle it.
|
|
100
108
|
2. **Protocol.** The write skill drafts `protocols/{slug}.md`. `lab-kit experiment {slug}` creates `experiments/{slug}/`. The protocol names its question, the intuition, the one variable and its arms, the pinned configuration, the allowed moves, the measures, the decision rules `D1`... with the kill rule marked, and the predictions `P1`... with confidences.
|
|
101
109
|
3. **Draft review.** A reviewer reads the draft before the lock. The commander records the verdict in the mission log.
|
|
102
|
-
4. **Lock.** The write skill sets the protocol's status to `locked`. The protocol's card lists `locked` in `frozen_in`, so folio's `frozen` check holds it from here: nothing changes but its status field. `lab-kit lock {slug}` writes `lock.json` with the file's hash, taken with the status field set aside, so the one change folio allows (to `abandoned`) does not break it. `folio journal add --title .. --description .. --body .. --kind lock --about {slug}` records it. The commit that holds the protocol, `lock.json`, that journal entry and `.folio/` is the lock; folio's `frozen` check compares with the first commit that set `locked`. From here lab-kit also checks the protocol's bytes.
|
|
103
|
-
5. **Run.** `lab-kit run {slug} -- <command>` refuses unless the lock is intact. It writes `run.json`, starts the lab's own command in the background, and logs it. A run under a mission names it with `--mission`, and the mission must be approved and `active`. A run that spends tokens or money also needs a mission that records a cap. Launch, checkpoints and terminations each go in a journal entry of kind `run` about the protocol.
|
|
110
|
+
4. **Lock.** The write skill sets the protocol's status to `locked`. The protocol's card lists `locked` in `frozen_in`, so folio's `frozen` check holds it from here: nothing changes but its status field. `lab-kit lock {slug}` writes `lock.json` with the file's hash, taken with the status field set aside, so the one change folio allows (to `abandoned`) does not break it. `folio journal add --title .. --description .. --body .. --kind lock --about {slug}` records it. The commit that holds the protocol, `lock.json`, that journal entry and `.folio/` is the lock; folio's `frozen` check compares with the first commit that set `locked`. From here lab-kit also checks the protocol's bytes, and the bytes of the experiment's `bin/`, so the instrument is committed before the lock.
|
|
111
|
+
5. **Run.** `lab-kit run {slug} -- <command>` refuses unless the lock is intact, the instrument included. It writes `run.json`, starts the lab's own command in the background, and logs it. A run under a mission names it with `--mission`, and the mission must be approved and `active`. A run that spends tokens or money also needs a mission that records a cap. Launch, checkpoints and terminations each go in a journal entry of kind `run` about the protocol.
|
|
104
112
|
6. **Evidence.** When the command exits, `lab-kit run` writes `MANIFEST.sha256` and the exit code. `out/` is committed once, at the end, in one commit. A run whose tool failed is invalid, and is recorded as invalid.
|
|
105
113
|
7. **Score.** The review skill reads the lock first. It folds `out/` with a selftested script. `lab-kit score {slug}` creates `score.yaml` with every prediction and rule id the lock names. The review fills each with a verdict, a value and the evidence it came from, and lists the numbers to record under `record`.
|
|
106
114
|
8. **Reviewer pass.** An independent reviewer re-derives the scored numbers and returns a verdict. The commander records it, and `score.yaml` points to it.
|
|
@@ -121,7 +129,7 @@ A result that is not promoted is recorded too: a journal entry about the protoco
|
|
|
121
129
|
| Id | What it requires |
|
|
122
130
|
| --- | --- |
|
|
123
131
|
| `lab-lock-recorded` | Every protocol with status `locked` has a `lock.json` naming it, and every `lock.json` names a protocol whose status is `locked` or `abandoned`. |
|
|
124
|
-
| `lab-lock-intact` | A locked protocol's bytes, with its status field set aside, hash to the value in its `lock.json`. Nothing but the status changes after the lock. |
|
|
132
|
+
| `lab-lock-intact` | A locked protocol's bytes, with its status field set aside, hash to the value in its `lock.json`. Nothing but the status changes after the lock. When the lock records `instrument`, the experiment's `bin/` hashes to it. A lock without `instrument` predates it and passes. |
|
|
125
133
|
| `lab-run-after-lock` | Every run belongs to a locked experiment, records the lock's hash, and started after the lock date. |
|
|
126
134
|
| `lab-roster-frozen` | Every run's configuration in `run.json` equals the `yaml` block under its locked protocol's `## Pinned configuration`. |
|
|
127
135
|
| `lab-evidence-sealed` | Every finished run's `out/` matches its `MANIFEST.sha256`. A run with no end and no live process is reported as orphaned (warning). |
|
|
@@ -137,6 +145,7 @@ A result that is not promoted is recorded too: a journal entry about the protoco
|
|
|
137
145
|
| `lab-method-current` | `.lab/method/`, the five skills and the four roles match the lab-kit version in `lab.yaml`. |
|
|
138
146
|
| `lab-state-pointer` | `ops/STATE.md` names an active mission file or "none", stays under 300 words, and cites ids for any number (warning). |
|
|
139
147
|
| `lab-scored-reported` | Every scored experiment has a report citing its results, or a journal entry about its protocol saying why not (warning). |
|
|
148
|
+
| `lab-work-stored` | When `lab.yaml` names a `store:`, every ended run with a `work.sha256` has a `store.json`: its raw output was published (warning). |
|
|
140
149
|
|
|
141
150
|
The gate checks that the record is consistent. It does not check that the record is honest. That is the reviewer's job.
|
|
142
151
|
|
|
@@ -151,9 +160,12 @@ Only skills call it, the way an agent calls git. Every command that changes file
|
|
|
151
160
|
| `lab-kit init [--library <dir>]` | Writes `lab.yaml`, `ops/`, `experiments/`, `.lab/`, the five skills and the four roles, and links `.claude/skills` to `.agents/skills` and `.claude/agents` to `.agents/agents`. With no `--library`, no `lab.yaml` and no `folio.yaml` at the root, it installs only the method, the skills and the roles, so the set-up-lab skill can be read before the library exists. With `--library`, the folder must hold a `folio.yaml`. In the library it only switches the lab pack on, if set-up has not. It refuses a library whose `root` is not the lab root. Run again in a lab, it restores the method, the skills and the roles to this version's, and sets `lab-kit:` in `lab.yaml` to it. |
|
|
152
161
|
| `lab-kit check [--only <id>] [--json]` | The lab gate: `folio check`, then the checks above. `--only` runs one check, and may repeat; `folio` names folio's gate. Exit 0 only when there is no error. |
|
|
153
162
|
| `lab-kit experiment <slug>` | Creates `experiments/<slug>/` with `bin/`, `runs/` and the gitignore for scratch. |
|
|
154
|
-
| `lab-kit lock <slug>` | Writes `lock.json` for a protocol whose status is `locked` and whose gate is green, hashing the file with its status field set aside. Refuses otherwise. |
|
|
155
|
-
| `lab-kit run <slug> [--mission <file>] [--spend] [--wait] -- <command>` | Checks the lock, writes `run.json`, starts the command in the background from the lab root, and writes `MANIFEST.sha256` for `out/` when it exits. With `--mission`, it refuses a mission whose status is not `active` or that records no approval. `--wait` waits for the end, for a short token-free stage. |
|
|
163
|
+
| `lab-kit lock <slug>` | Writes `lock.json` for a protocol whose status is `locked` and whose gate is green, hashing the file with its status field set aside, and records the instrument's hash, the git commit and the lab-kit version. Refuses otherwise. |
|
|
164
|
+
| `lab-kit run <slug> [--mission <file>] [--spend] [--wait] -- <command>` | Checks the lock, writes `run.json`, starts the command in the background from the lab root, records the git commit and the lab-kit version, and writes `MANIFEST.sha256` for `out/` and `work.sha256` for `work/` when it exits. It refuses when the protocol or `bin/` no longer hashes to the lock. With `--mission`, it refuses a mission whose status is not `active` or that records no approval. `--wait` waits for the end, for a short token-free stage. |
|
|
156
165
|
| `lab-kit runs [--live]` | Lists runs and their state: running, finished, failed, orphaned. |
|
|
166
|
+
| `lab-kit publish <slug>/<run id>... \| --pending [--keep]` | Publishes ended runs' raw output to the store, as above. `--pending` takes every ended run whose `work/` is still here. `--keep` keeps the local copy. |
|
|
167
|
+
| `lab-kit fetch <slug>/<run id>` | Copies a published run's `work/` and `run.log` back into its folder, writable, and verifies them against `work.sha256`. |
|
|
168
|
+
| `lab-kit verify <slug>/<run id>` | Checks the store's copy of a run against its `work.sha256`. Changes nothing. |
|
|
157
169
|
| `lab-kit score <slug>` | Creates `score.yaml` from the locked protocol's prediction and rule ids, with empty verdicts. |
|
|
158
170
|
| `lab-kit rederive <R-n>` | Runs one result's re-derive command and compares its output with the number. |
|
|
159
171
|
| `lab-kit freeze <path>` | Records a frozen surface's hash. Only the operator asks for this. |
|
{lab_kit_cli-0.2.0 → lab_kit_cli-0.4.0}/examples/monte-carlo-lab/.agents/skills/experiment/SKILL.md
RENAMED
|
@@ -47,12 +47,12 @@ You take one question from a draft protocol to a finished run with its evidence
|
|
|
47
47
|
- what it cannot show.
|
|
48
48
|
2. **Make its folder.** Run `lab-kit experiment <slug>`. Put the lab's scripts in `experiments/<slug>/bin/`, each with a `--selftest`. Run `lab-kit check --only lab-selftest`.
|
|
49
49
|
3. **Review the draft.** Send a reviewer with fresh context. Fold its points into the draft through the write skill. A protocol is cheap to fix before the lock and impossible to fix after.
|
|
50
|
-
4. **Lock.** Through the write skill, set the protocol's status to `locked`. Run `lab-kit lock <slug>`. Record it: `folio journal add --title .. --description .. --body .. --kind lock --about <slug>,<Q-n>`. Run `folio index`, then `lab-kit check`. Commit exactly the protocol, `lock.json`, the `lock` journal entry and `.folio/` together; that commit is the lock. From here folio's `frozen` check and `lab-lock-intact` hold it.
|
|
50
|
+
4. **Lock.** Commit `bin/` first: the lock hashes it, and `lab-lock-intact` fails if it changes after. Through the write skill, set the protocol's status to `locked`. Run `lab-kit lock <slug>`. Record it: `folio journal add --title .. --description .. --body .. --kind lock --about <slug>,<Q-n>`. Run `folio index`, then `lab-kit check`. Commit exactly the protocol, `lock.json`, the `lock` journal entry and `.folio/` together; that commit is the lock. From here folio's `frozen` check and `lab-lock-intact` hold it.
|
|
51
51
|
5. **De-risk.** Run the token-free stages first, with `lab-kit run`. Fix the apparatus only before the lock. After it, an apparatus fault ends the run as invalid.
|
|
52
52
|
6. **Launch.** Run `lab-kit run <slug> -- <command>`. Add `--spend --mission <file>` for a run that spends. Watch it at once: arm one watcher on its log that fires on the end marker or an error signature. A silent launch failure can waste a whole night.
|
|
53
53
|
7. **Supervise.** Write a journal entry of kind `run` about the protocol for each launch, checkpoint, surprise and termination, with `folio journal add`. Keep each entry to what changed. Record a deviation the moment you see it. Commit each entry with `.folio/` after `folio index`; never stage the run's working files.
|
|
54
54
|
8. **On failure.** End the run. Start a fresh one with the same configuration. Disclose it in a `run` entry. Never swap a model or a tool mid-run.
|
|
55
|
-
9. **Conclude.** When the run ends, `lab-kit run` writes the manifest of `out/`. Commit `out/`, `MANIFEST.sha256` and `run.json` in one commit. Run `lab-kit check`. Hand the experiment to the review skill. Do not score your own run.
|
|
55
|
+
9. **Conclude.** When the run ends, `lab-kit run` writes the manifest of `out/`. Commit `out/`, `MANIFEST.sha256`, `work.sha256` and `run.json` in one commit, with `store.json` when `lab.yaml` names a store; `work/` stays out of git, and `work.sha256` is what any copy of it is verified against. With a store, `lab-kit run` has already published `work/` there and deleted the local copy; if `run.log` says the publish failed, run `lab-kit publish --pending`. Never mount the store where the system under test runs: it holds other runs' answers. Run `lab-kit check`. Hand the experiment to the review skill. Do not score your own run.
|
|
56
56
|
|
|
57
57
|
## Stops
|
|
58
58
|
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
lab-kit: 0.
|
|
1
|
+
lab-kit: 0.4.0 # the version this lab is checked with
|
|
2
2
|
library: docs # where folio.yaml is; "." for the root
|
|
3
3
|
front: lab # the front door's id: a project document
|
|
4
4
|
frozen: [substrate/] # surfaces never edited in place; record each with `lab-kit freeze`
|
|
@@ -5,7 +5,7 @@ build-backend = "hatchling.build"
|
|
|
5
5
|
[project]
|
|
6
6
|
dynamic = ["readme"]
|
|
7
7
|
name = "lab-kit-cli"
|
|
8
|
-
version = "0.
|
|
8
|
+
version = "0.4.0"
|
|
9
9
|
description = "A research lab's method and machinery on top of folio: pre-register, lock, run, score, re-derive."
|
|
10
10
|
license = "MIT"
|
|
11
11
|
authors = [{ name = "Piergiuseppe Mallozzi" }]
|
|
@@ -47,12 +47,12 @@ You take one question from a draft protocol to a finished run with its evidence
|
|
|
47
47
|
- what it cannot show.
|
|
48
48
|
2. **Make its folder.** Run `lab-kit experiment <slug>`. Put the lab's scripts in `experiments/<slug>/bin/`, each with a `--selftest`. Run `lab-kit check --only lab-selftest`.
|
|
49
49
|
3. **Review the draft.** Send a reviewer with fresh context. Fold its points into the draft through the write skill. A protocol is cheap to fix before the lock and impossible to fix after.
|
|
50
|
-
4. **Lock.** Through the write skill, set the protocol's status to `locked`. Run `lab-kit lock <slug>`. Record it: `folio journal add --title .. --description .. --body .. --kind lock --about <slug>,<Q-n>`. Run `folio index`, then `lab-kit check`. Commit exactly the protocol, `lock.json`, the `lock` journal entry and `.folio/` together; that commit is the lock. From here folio's `frozen` check and `lab-lock-intact` hold it.
|
|
50
|
+
4. **Lock.** Commit `bin/` first: the lock hashes it, and `lab-lock-intact` fails if it changes after. Through the write skill, set the protocol's status to `locked`. Run `lab-kit lock <slug>`. Record it: `folio journal add --title .. --description .. --body .. --kind lock --about <slug>,<Q-n>`. Run `folio index`, then `lab-kit check`. Commit exactly the protocol, `lock.json`, the `lock` journal entry and `.folio/` together; that commit is the lock. From here folio's `frozen` check and `lab-lock-intact` hold it.
|
|
51
51
|
5. **De-risk.** Run the token-free stages first, with `lab-kit run`. Fix the apparatus only before the lock. After it, an apparatus fault ends the run as invalid.
|
|
52
52
|
6. **Launch.** Run `lab-kit run <slug> -- <command>`. Add `--spend --mission <file>` for a run that spends. Watch it at once: arm one watcher on its log that fires on the end marker or an error signature. A silent launch failure can waste a whole night.
|
|
53
53
|
7. **Supervise.** Write a journal entry of kind `run` about the protocol for each launch, checkpoint, surprise and termination, with `folio journal add`. Keep each entry to what changed. Record a deviation the moment you see it. Commit each entry with `.folio/` after `folio index`; never stage the run's working files.
|
|
54
54
|
8. **On failure.** End the run. Start a fresh one with the same configuration. Disclose it in a `run` entry. Never swap a model or a tool mid-run.
|
|
55
|
-
9. **Conclude.** When the run ends, `lab-kit run` writes the manifest of `out/`. Commit `out/`, `MANIFEST.sha256` and `run.json` in one commit. Run `lab-kit check`. Hand the experiment to the review skill. Do not score your own run.
|
|
55
|
+
9. **Conclude.** When the run ends, `lab-kit run` writes the manifest of `out/`. Commit `out/`, `MANIFEST.sha256`, `work.sha256` and `run.json` in one commit, with `store.json` when `lab.yaml` names a store; `work/` stays out of git, and `work.sha256` is what any copy of it is verified against. With a store, `lab-kit run` has already published `work/` there and deleted the local copy; if `run.log` says the publish failed, run `lab-kit publish --pending`. Never mount the store where the system under test runs: it holds other runs' answers. Run `lab-kit check`. Hand the experiment to the review skill. Do not score your own run.
|
|
56
56
|
|
|
57
57
|
## Stops
|
|
58
58
|
|
|
@@ -44,7 +44,8 @@ def lock_recorded(lab: Lab, rep: Reporter) -> None:
|
|
|
44
44
|
|
|
45
45
|
|
|
46
46
|
def lock_intact(lab: Lab, rep: Reporter) -> None:
|
|
47
|
-
"""A locked protocol, its status set aside, hashes to the value in its lock.json
|
|
47
|
+
"""A locked protocol, its status set aside, hashes to the value in its lock.json, and the experiment's
|
|
48
|
+
`bin/` hashes to the lock's `instrument` when the lock records one."""
|
|
48
49
|
for slug in lab.experiment_slugs():
|
|
49
50
|
path = f"experiments/{slug}/{records.LOCK}"
|
|
50
51
|
|
|
@@ -59,6 +60,11 @@ def lock_intact(lab: Lab, rep: Reporter) -> None:
|
|
|
59
60
|
if actual != lock.sha256:
|
|
60
61
|
rep.add(proto.path, f"changed since its lock: its hash, status set aside, is {actual[:12]}, "
|
|
61
62
|
f"and {path} records {lock.sha256[:12]}")
|
|
63
|
+
if lock.instrument is not None:
|
|
64
|
+
bin_hash = records.instrument_hash(lab.experiments / slug / "bin")
|
|
65
|
+
if bin_hash != lock.instrument:
|
|
66
|
+
rep.add(f"experiments/{slug}/bin", f"bin/ changed since its lock: it hashes to {bin_hash[:12]}, "
|
|
67
|
+
f"and {path} records {lock.instrument[:12]}")
|
|
62
68
|
rep.guard(path, one)
|
|
63
69
|
|
|
64
70
|
|
|
@@ -132,3 +138,17 @@ def evidence_sealed(lab: Lab, rep: Reporter) -> None:
|
|
|
132
138
|
elif recorded[name] != actual[name]:
|
|
133
139
|
rep.add(manifest_path, f"out/{name} changed after the run ended")
|
|
134
140
|
rep.guard(run.path, one)
|
|
141
|
+
|
|
142
|
+
|
|
143
|
+
def work_stored(lab: Lab, rep: Reporter) -> None:
|
|
144
|
+
"""With a store in lab.yaml, every ended run with a work.sha256 records that it was published (warning)."""
|
|
145
|
+
if not lab.settings.store:
|
|
146
|
+
return
|
|
147
|
+
from ..store import STORE_RECORD
|
|
148
|
+
for run in records.runs(lab):
|
|
149
|
+
if not run.ended or not (run.folder / records.WORK_MANIFEST).is_file():
|
|
150
|
+
continue
|
|
151
|
+
if not (run.folder / STORE_RECORD).is_file():
|
|
152
|
+
rep.warn(run.path.rsplit("/", 1)[0] + "/" + records.WORK_MANIFEST,
|
|
153
|
+
f"its raw output was never published to the store {lab.settings.store}; "
|
|
154
|
+
"`lab-kit publish --pending` publishes it, then commit store.json")
|
|
@@ -30,6 +30,7 @@ CHECKS: tuple[LabCheck, ...] = (
|
|
|
30
30
|
LabCheck("lab-method-current", "error", files.method_current),
|
|
31
31
|
LabCheck("lab-state-pointer", "warning", files.state_pointer),
|
|
32
32
|
LabCheck("lab-scored-reported", "warning", results.scored_reported),
|
|
33
|
+
LabCheck("lab-work-stored", "warning", locks.work_stored),
|
|
33
34
|
)
|
|
34
35
|
BY_NAME = {c.name: c for c in CHECKS}
|
|
35
36
|
|
|
@@ -10,6 +10,7 @@ from pathlib import Path
|
|
|
10
10
|
from . import __version__, frozen
|
|
11
11
|
from . import lab as lab_mod
|
|
12
12
|
from . import rederive as rederive_mod
|
|
13
|
+
from . import store as store_mod
|
|
13
14
|
from .checks import run as checks_run
|
|
14
15
|
from .commands import experiments, runs, setup, status
|
|
15
16
|
from .errors import LabError
|
|
@@ -65,6 +66,37 @@ def cmd_runs(args: argparse.Namespace) -> int:
|
|
|
65
66
|
return 0
|
|
66
67
|
|
|
67
68
|
|
|
69
|
+
def cmd_publish(args: argparse.Namespace) -> int:
|
|
70
|
+
lab = _lab()
|
|
71
|
+
if args.pending == bool(args.runs):
|
|
72
|
+
raise LabError("name runs as <slug>/<run id>, or pass --pending for every ended run still here")
|
|
73
|
+
store_mod.root(lab) # a lab with no store says so once
|
|
74
|
+
targets = store_mod.pending(lab) if args.pending else [store_mod.find_run(lab, r) for r in args.runs]
|
|
75
|
+
if not targets:
|
|
76
|
+
print("Nothing to publish.")
|
|
77
|
+
return 0
|
|
78
|
+
failed = 0
|
|
79
|
+
for run in targets:
|
|
80
|
+
try:
|
|
81
|
+
_print(store_mod.publish(lab, run, keep=args.keep))
|
|
82
|
+
except (LabError, OSError) as exc:
|
|
83
|
+
print(f"lab-kit: {lab.rel(run.folder)}: not published: {exc}", file=sys.stderr)
|
|
84
|
+
failed += 1
|
|
85
|
+
return 1 if failed else 0
|
|
86
|
+
|
|
87
|
+
|
|
88
|
+
def cmd_fetch(args: argparse.Namespace) -> int:
|
|
89
|
+
lab = _lab()
|
|
90
|
+
_print(store_mod.fetch(lab, store_mod.find_run(lab, args.run)))
|
|
91
|
+
return 0
|
|
92
|
+
|
|
93
|
+
|
|
94
|
+
def cmd_verify(args: argparse.Namespace) -> int:
|
|
95
|
+
lab = _lab()
|
|
96
|
+
_print(store_mod.verify(lab, store_mod.find_run(lab, args.run)))
|
|
97
|
+
return 0
|
|
98
|
+
|
|
99
|
+
|
|
68
100
|
def cmd_score(args: argparse.Namespace) -> int:
|
|
69
101
|
_print(experiments.score(_lab(), args.slug))
|
|
70
102
|
return 0
|
|
@@ -131,6 +163,21 @@ def parser() -> argparse.ArgumentParser:
|
|
|
131
163
|
s.add_argument("--live", action="store_true", help="only running and orphaned runs")
|
|
132
164
|
s.set_defaults(func=cmd_runs)
|
|
133
165
|
|
|
166
|
+
s = sub.add_parser("publish", help="Copy ended runs' raw output to the store named in lab.yaml, verify it "
|
|
167
|
+
"against work.sha256, and delete the local copy.")
|
|
168
|
+
s.add_argument("runs", nargs="*", metavar="SLUG/RUN_ID")
|
|
169
|
+
s.add_argument("--pending", action="store_true", help="every ended run whose work/ is still here")
|
|
170
|
+
s.add_argument("--keep", action="store_true", help="keep the local work/ and run.log")
|
|
171
|
+
s.set_defaults(func=cmd_publish)
|
|
172
|
+
|
|
173
|
+
s = sub.add_parser("fetch", help="Copy a published run's raw output back into its folder, verified.")
|
|
174
|
+
s.add_argument("run", metavar="SLUG/RUN_ID")
|
|
175
|
+
s.set_defaults(func=cmd_fetch)
|
|
176
|
+
|
|
177
|
+
s = sub.add_parser("verify", help="Check the store's copy of a run against its work.sha256.")
|
|
178
|
+
s.add_argument("run", metavar="SLUG/RUN_ID")
|
|
179
|
+
s.set_defaults(func=cmd_verify)
|
|
180
|
+
|
|
134
181
|
s = sub.add_parser("score", help="Create score.yaml from the lock's prediction and rule ids.")
|
|
135
182
|
s.add_argument("slug")
|
|
136
183
|
s.set_defaults(func=cmd_score)
|
|
@@ -52,6 +52,8 @@ def lock(lab: Lab, slug: str) -> list[str]:
|
|
|
52
52
|
"path": proto.path,
|
|
53
53
|
"sha256": protocol_mod.hash_text(proto.text()),
|
|
54
54
|
"locked": records.now(),
|
|
55
|
+
"instrument": records.instrument_hash(folder / "bin"),
|
|
56
|
+
**records.provenance(lab.root),
|
|
55
57
|
})
|
|
56
58
|
return [f"created experiments/{proto.slug}/{records.LOCK}"]
|
|
57
59
|
|
|
@@ -54,6 +54,8 @@ def start(lab: Lab, slug: str, command: list[str], mission: str | None = None, s
|
|
|
54
54
|
raise LabError(f"{proto.path} has status `{proto.status}`, not `locked`")
|
|
55
55
|
if protocol_mod.hash_text(proto.text()) != lock.sha256:
|
|
56
56
|
raise LabError(f"{proto.path} changed since its lock; no run starts against a broken lock")
|
|
57
|
+
if lock.instrument is not None and records.instrument_hash(lab.experiments / slug / "bin") != lock.instrument:
|
|
58
|
+
raise LabError(f"experiments/{slug}/bin/ changed since its lock; no run starts against a broken lock")
|
|
57
59
|
config = protocol_mod.pinned_config(proto.text(), proto.path)
|
|
58
60
|
mission_rel = _check_mission(lab, mission, spend)
|
|
59
61
|
run_id = _run_id(lab, slug)
|
|
@@ -63,7 +65,7 @@ def start(lab: Lab, slug: str, command: list[str], mission: str | None = None, s
|
|
|
63
65
|
records.write_json(run_dir / records.RUN, {
|
|
64
66
|
"protocol": lock.protocol, "lock": lock.sha256, "config": config, "mission": mission_rel,
|
|
65
67
|
"spend": spend, "command": command, "started": records.now(), "ended": None, "exit": None,
|
|
66
|
-
"pid": None, "spent": None,
|
|
68
|
+
"pid": None, "spent": None, **records.provenance(lab.root),
|
|
67
69
|
})
|
|
68
70
|
base = f"experiments/{slug}/runs/{run_id}"
|
|
69
71
|
changed = [f"created {base}/{records.RUN}", f"created {base}/out/", f"started {' '.join(command)}",
|
|
@@ -14,7 +14,7 @@ from folio.library import Library
|
|
|
14
14
|
from .errors import LabError
|
|
15
15
|
|
|
16
16
|
LAB_YAML = "lab.yaml"
|
|
17
|
-
_KEYS = ("lab-kit", "library", "front", "frozen", "tools", "rederive_timeout")
|
|
17
|
+
_KEYS = ("lab-kit", "library", "front", "frozen", "tools", "rederive_timeout", "store")
|
|
18
18
|
|
|
19
19
|
|
|
20
20
|
@dataclass
|
|
@@ -25,6 +25,7 @@ class Settings:
|
|
|
25
25
|
frozen: list[str]
|
|
26
26
|
tools: list[str]
|
|
27
27
|
rederive_timeout: int
|
|
28
|
+
store: str | None = None
|
|
28
29
|
|
|
29
30
|
|
|
30
31
|
@dataclass
|
|
@@ -115,7 +116,10 @@ def parse_settings(raw: Any, where: str) -> Settings:
|
|
|
115
116
|
timeout = raw.get("rederive_timeout", 120)
|
|
116
117
|
if not isinstance(timeout, int) or isinstance(timeout, bool) or timeout <= 0:
|
|
117
118
|
raise LabError(f"{where}: `rederive_timeout` must be a positive whole number of seconds")
|
|
118
|
-
|
|
119
|
+
store = raw.get("store")
|
|
120
|
+
if store is not None and (not isinstance(store, str) or not Path(store).is_absolute()):
|
|
121
|
+
raise LabError(f"{where}: `store` must be an absolute path, such as /labs/<lab>")
|
|
122
|
+
return Settings(version, library, front or None, lists["frozen"], lists["tools"], timeout, store or None)
|
|
119
123
|
|
|
120
124
|
|
|
121
125
|
def load_at(root: Path) -> Lab:
|
|
@@ -6,23 +6,26 @@ import datetime as _dt
|
|
|
6
6
|
import hashlib
|
|
7
7
|
import json
|
|
8
8
|
import os
|
|
9
|
+
import subprocess
|
|
9
10
|
from dataclasses import dataclass
|
|
10
11
|
from pathlib import Path
|
|
11
12
|
from typing import Any
|
|
12
13
|
|
|
13
14
|
import yaml
|
|
14
15
|
|
|
16
|
+
from . import __version__
|
|
15
17
|
from .errors import LabError
|
|
16
18
|
from .lab import Lab
|
|
17
19
|
|
|
18
20
|
LOCK = "lock.json"
|
|
19
21
|
RUN = "run.json"
|
|
20
22
|
MANIFEST = "MANIFEST.sha256"
|
|
23
|
+
WORK_MANIFEST = "work.sha256" # beside MANIFEST.sha256: the run's raw output under work/, which git ignores
|
|
21
24
|
SCORE = "score.yaml"
|
|
22
25
|
SPEND = "spend.json" # written by a spending run's own command into out/
|
|
23
|
-
LOCK_KEYS = ("protocol", "path", "sha256", "locked")
|
|
26
|
+
LOCK_KEYS = ("protocol", "path", "sha256", "locked") # required; `instrument`, `commit`, `lab-kit` since 0.3.0
|
|
24
27
|
RUN_KEYS = ("protocol", "lock", "config", "mission", "spend", "command", "started", "ended", "exit", "pid",
|
|
25
|
-
"spent")
|
|
28
|
+
"spent", "commit", "lab-kit")
|
|
26
29
|
PREDICTION_VERDICTS = ("HIT", "MISS", "INDETERMINATE")
|
|
27
30
|
RULE_VERDICTS = ("FIRED", "NOT FIRED", "INDETERMINATE")
|
|
28
31
|
REVIEW_VERDICTS = ("pass", "fail")
|
|
@@ -43,6 +46,31 @@ def script_env() -> dict[str, str]:
|
|
|
43
46
|
return env
|
|
44
47
|
|
|
45
48
|
|
|
49
|
+
def provenance(root: Path) -> dict[str, Any]:
|
|
50
|
+
"""What a lock or a run record names about where it was taken: the git head and this lab-kit.
|
|
51
|
+
|
|
52
|
+
`commit` is null outside git or before the first commit, so the record says there was none.
|
|
53
|
+
"""
|
|
54
|
+
result = subprocess.run(["git", "-C", str(root), "rev-parse", "--verify", "-q", "HEAD"],
|
|
55
|
+
capture_output=True, text=True, check=False)
|
|
56
|
+
head = result.stdout.strip() if result.returncode == 0 else None
|
|
57
|
+
return {"commit": head or None, "lab-kit": __version__}
|
|
58
|
+
|
|
59
|
+
|
|
60
|
+
def instrument_hash(folder: Path) -> str:
|
|
61
|
+
"""One sha256 over the instrument: every file under `folder` (an experiment's `bin/`), sorted by its
|
|
62
|
+
path from `folder`, each fed as its path, a newline, then its bytes. A `__pycache__` folder is not
|
|
63
|
+
the instrument and is left out."""
|
|
64
|
+
digest = hashlib.sha256()
|
|
65
|
+
if folder.is_dir():
|
|
66
|
+
files = sorted((p.relative_to(folder).as_posix(), p) for p in folder.rglob("*")
|
|
67
|
+
if p.is_file() and "__pycache__" not in p.relative_to(folder).parts)
|
|
68
|
+
for rel, path in files:
|
|
69
|
+
digest.update(rel.encode("utf-8") + b"\n")
|
|
70
|
+
digest.update(path.read_bytes())
|
|
71
|
+
return digest.hexdigest()
|
|
72
|
+
|
|
73
|
+
|
|
46
74
|
def parse_moment(value: Any, where: str) -> _dt.datetime:
|
|
47
75
|
if not isinstance(value, str):
|
|
48
76
|
raise LabError(f"{where}: expected an ISO 8601 time, not {value!r}")
|
|
@@ -98,6 +126,12 @@ class LockRecord:
|
|
|
98
126
|
def sha256(self) -> str:
|
|
99
127
|
return str(self.data.get("sha256", ""))
|
|
100
128
|
|
|
129
|
+
@property
|
|
130
|
+
def instrument(self) -> str | None:
|
|
131
|
+
"""The hash of `bin/` at the lock; None for a lock taken before lab-kit 0.3.0."""
|
|
132
|
+
value = self.data.get("instrument")
|
|
133
|
+
return value if isinstance(value, str) else None
|
|
134
|
+
|
|
101
135
|
@property
|
|
102
136
|
def locked(self) -> _dt.datetime:
|
|
103
137
|
return parse_moment(self.data.get("locked"), self.path)
|
|
@@ -238,6 +272,13 @@ def write_manifest(run: RunRecord) -> None:
|
|
|
238
272
|
run.manifest.write_text(format_hashes(tree_hashes(run.out, run.out)), encoding="utf-8")
|
|
239
273
|
|
|
240
274
|
|
|
275
|
+
def write_work_manifest(run: RunRecord) -> None:
|
|
276
|
+
"""`work.sha256` beside the manifest, one line per file under `work/`, when that folder exists."""
|
|
277
|
+
work = run.folder / "work"
|
|
278
|
+
if work.is_dir():
|
|
279
|
+
(run.folder / WORK_MANIFEST).write_text(format_hashes(tree_hashes(work, work)), encoding="utf-8")
|
|
280
|
+
|
|
281
|
+
|
|
241
282
|
# ---------------------------------------------------------------- scorecards
|
|
242
283
|
|
|
243
284
|
|