lab-kit-cli 0.3.0__tar.gz → 0.4.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (134) hide show
  1. {lab_kit_cli-0.3.0 → lab_kit_cli-0.4.0}/CHANGELOG.md +8 -0
  2. {lab_kit_cli-0.3.0 → lab_kit_cli-0.4.0}/PKG-INFO +1 -1
  3. {lab_kit_cli-0.3.0 → lab_kit_cli-0.4.0}/docs/spec/lab-model.md +9 -1
  4. {lab_kit_cli-0.3.0 → lab_kit_cli-0.4.0}/examples/monte-carlo-lab/.agents/skills/experiment/SKILL.md +1 -1
  5. {lab_kit_cli-0.3.0 → lab_kit_cli-0.4.0}/examples/monte-carlo-lab/lab.yaml +1 -1
  6. {lab_kit_cli-0.3.0 → lab_kit_cli-0.4.0}/pyproject.toml +1 -1
  7. {lab_kit_cli-0.3.0 → lab_kit_cli-0.4.0}/skills/experiment/SKILL.md +1 -1
  8. {lab_kit_cli-0.3.0 → lab_kit_cli-0.4.0}/src/lab_kit/__init__.py +1 -1
  9. {lab_kit_cli-0.3.0 → lab_kit_cli-0.4.0}/src/lab_kit/checks/locks.py +14 -0
  10. {lab_kit_cli-0.3.0 → lab_kit_cli-0.4.0}/src/lab_kit/checks/run.py +1 -0
  11. {lab_kit_cli-0.3.0 → lab_kit_cli-0.4.0}/src/lab_kit/cli.py +47 -0
  12. {lab_kit_cli-0.3.0 → lab_kit_cli-0.4.0}/src/lab_kit/lab.py +6 -2
  13. lab_kit_cli-0.4.0/src/lab_kit/store.py +285 -0
  14. {lab_kit_cli-0.3.0 → lab_kit_cli-0.4.0}/src/lab_kit/supervise.py +22 -0
  15. lab_kit_cli-0.4.0/tests/test_store.py +108 -0
  16. {lab_kit_cli-0.3.0 → lab_kit_cli-0.4.0}/.github/workflows/ci.yml +0 -0
  17. {lab_kit_cli-0.3.0 → lab_kit_cli-0.4.0}/.github/workflows/policy.yml +0 -0
  18. {lab_kit_cli-0.3.0 → lab_kit_cli-0.4.0}/.github/workflows/release.yml +0 -0
  19. {lab_kit_cli-0.3.0 → lab_kit_cli-0.4.0}/.gitignore +0 -0
  20. {lab_kit_cli-0.3.0 → lab_kit_cli-0.4.0}/AGENTS.md +0 -0
  21. {lab_kit_cli-0.3.0 → lab_kit_cli-0.4.0}/CLAUDE.md +0 -0
  22. {lab_kit_cli-0.3.0 → lab_kit_cli-0.4.0}/CONTRIBUTING.md +0 -0
  23. {lab_kit_cli-0.3.0 → lab_kit_cli-0.4.0}/LICENSE +0 -0
  24. {lab_kit_cli-0.3.0 → lab_kit_cli-0.4.0}/README.md +0 -0
  25. {lab_kit_cli-0.3.0 → lab_kit_cli-0.4.0}/SETUP.md +0 -0
  26. {lab_kit_cli-0.3.0 → lab_kit_cli-0.4.0}/agents/reporter.md +0 -0
  27. {lab_kit_cli-0.3.0 → lab_kit_cli-0.4.0}/agents/reviewer.md +0 -0
  28. {lab_kit_cli-0.3.0 → lab_kit_cli-0.4.0}/agents/runner.md +0 -0
  29. {lab_kit_cli-0.3.0 → lab_kit_cli-0.4.0}/agents/scout.md +0 -0
  30. {lab_kit_cli-0.3.0 → lab_kit_cli-0.4.0}/docs/figures/how-lab-kit-works-dark.svg +0 -0
  31. {lab_kit_cli-0.3.0 → lab_kit_cli-0.4.0}/docs/figures/how-lab-kit-works-light.svg +0 -0
  32. {lab_kit_cli-0.3.0 → lab_kit_cli-0.4.0}/docs/figures/lab-loop-dark.svg +0 -0
  33. {lab_kit_cli-0.3.0 → lab_kit_cli-0.4.0}/docs/figures/lab-loop-light.svg +0 -0
  34. {lab_kit_cli-0.3.0 → lab_kit_cli-0.4.0}/docs/figures/number-chain-dark.svg +0 -0
  35. {lab_kit_cli-0.3.0 → lab_kit_cli-0.4.0}/docs/figures/number-chain-light.svg +0 -0
  36. {lab_kit_cli-0.3.0 → lab_kit_cli-0.4.0}/examples/monte-carlo-lab/.agents/agents/reporter.md +0 -0
  37. {lab_kit_cli-0.3.0 → lab_kit_cli-0.4.0}/examples/monte-carlo-lab/.agents/agents/reviewer.md +0 -0
  38. {lab_kit_cli-0.3.0 → lab_kit_cli-0.4.0}/examples/monte-carlo-lab/.agents/agents/runner.md +0 -0
  39. {lab_kit_cli-0.3.0 → lab_kit_cli-0.4.0}/examples/monte-carlo-lab/.agents/agents/scout.md +0 -0
  40. {lab_kit_cli-0.3.0 → lab_kit_cli-0.4.0}/examples/monte-carlo-lab/.agents/skills/address/SKILL.md +0 -0
  41. {lab_kit_cli-0.3.0 → lab_kit_cli-0.4.0}/examples/monte-carlo-lab/.agents/skills/configure/SKILL.md +0 -0
  42. {lab_kit_cli-0.3.0 → lab_kit_cli-0.4.0}/examples/monte-carlo-lab/.agents/skills/organise/SKILL.md +0 -0
  43. {lab_kit_cli-0.3.0 → lab_kit_cli-0.4.0}/examples/monte-carlo-lab/.agents/skills/plan-mission/SKILL.md +0 -0
  44. {lab_kit_cli-0.3.0 → lab_kit_cli-0.4.0}/examples/monte-carlo-lab/.agents/skills/publish/SKILL.md +0 -0
  45. {lab_kit_cli-0.3.0 → lab_kit_cli-0.4.0}/examples/monte-carlo-lab/.agents/skills/review/SKILL.md +0 -0
  46. {lab_kit_cli-0.3.0 → lab_kit_cli-0.4.0}/examples/monte-carlo-lab/.agents/skills/run/SKILL.md +0 -0
  47. {lab_kit_cli-0.3.0 → lab_kit_cli-0.4.0}/examples/monte-carlo-lab/.agents/skills/run-mission/SKILL.md +0 -0
  48. {lab_kit_cli-0.3.0 → lab_kit_cli-0.4.0}/examples/monte-carlo-lab/.agents/skills/set-up/SKILL.md +0 -0
  49. {lab_kit_cli-0.3.0 → lab_kit_cli-0.4.0}/examples/monte-carlo-lab/.agents/skills/set-up-lab/SKILL.md +0 -0
  50. {lab_kit_cli-0.3.0 → lab_kit_cli-0.4.0}/examples/monte-carlo-lab/.agents/skills/write/SKILL.md +0 -0
  51. {lab_kit_cli-0.3.0 → lab_kit_cli-0.4.0}/examples/monte-carlo-lab/.gitignore +0 -0
  52. {lab_kit_cli-0.3.0 → lab_kit_cli-0.4.0}/examples/monte-carlo-lab/.lab/frozen.sha256 +0 -0
  53. {lab_kit_cli-0.3.0 → lab_kit_cli-0.4.0}/examples/monte-carlo-lab/.lab/method/DISCIPLINE.md +0 -0
  54. {lab_kit_cli-0.3.0 → lab_kit_cli-0.4.0}/examples/monte-carlo-lab/.lab/method/LADDER.md +0 -0
  55. {lab_kit_cli-0.3.0 → lab_kit_cli-0.4.0}/examples/monte-carlo-lab/AGENTS.md +0 -0
  56. {lab_kit_cli-0.3.0 → lab_kit_cli-0.4.0}/examples/monte-carlo-lab/CLAUDE.md +0 -0
  57. {lab_kit_cli-0.3.0 → lab_kit_cli-0.4.0}/examples/monte-carlo-lab/docs/.folio/backlinks.json +0 -0
  58. {lab_kit_cli-0.3.0 → lab_kit_cli-0.4.0}/examples/monte-carlo-lab/docs/.folio/cards.json +0 -0
  59. {lab_kit_cli-0.3.0 → lab_kit_cli-0.4.0}/examples/monte-carlo-lab/docs/.folio/catalog.json +0 -0
  60. {lab_kit_cli-0.3.0 → lab_kit_cli-0.4.0}/examples/monte-carlo-lab/docs/.folio/journal.json +0 -0
  61. {lab_kit_cli-0.3.0 → lab_kit_cli-0.4.0}/examples/monte-carlo-lab/docs/.folio/nav.json +0 -0
  62. {lab_kit_cli-0.3.0 → lab_kit_cli-0.4.0}/examples/monte-carlo-lab/docs/.folio/search.json +0 -0
  63. {lab_kit_cli-0.3.0 → lab_kit_cli-0.4.0}/examples/monte-carlo-lab/docs/.gitignore +0 -0
  64. {lab_kit_cli-0.3.0 → lab_kit_cli-0.4.0}/examples/monte-carlo-lab/docs/assets/data/.gitkeep +0 -0
  65. {lab_kit_cli-0.3.0 → lab_kit_cli-0.4.0}/examples/monte-carlo-lab/docs/assets/figures/.gitkeep +0 -0
  66. {lab_kit_cli-0.3.0 → lab_kit_cli-0.4.0}/examples/monte-carlo-lab/docs/assets/refs.bib +0 -0
  67. {lab_kit_cli-0.3.0 → lab_kit_cli-0.4.0}/examples/monte-carlo-lab/docs/content/claims/C-1.md +0 -0
  68. {lab_kit_cli-0.3.0 → lab_kit_cli-0.4.0}/examples/monte-carlo-lab/docs/content/index.html +0 -0
  69. {lab_kit_cli-0.3.0 → lab_kit_cli-0.4.0}/examples/monte-carlo-lab/docs/content/journal/2026/2026-10-04-concluded-the-pi-error-scaling-mission.md +0 -0
  70. {lab_kit_cli-0.3.0 → lab_kit_cli-0.4.0}/examples/monte-carlo-lab/docs/content/journal/2026/2026-10-04-lab-set-up.md +0 -0
  71. {lab_kit_cli-0.3.0 → lab_kit_cli-0.4.0}/examples/monte-carlo-lab/docs/content/journal/2026/2026-10-04-locked-the-pi-error-scaling-protocol.md +0 -0
  72. {lab_kit_cli-0.3.0 → lab_kit_cli-0.4.0}/examples/monte-carlo-lab/docs/content/journal/2026/2026-10-04-opened-the-pi-error-scaling-mission.md +0 -0
  73. {lab_kit_cli-0.3.0 → lab_kit_cli-0.4.0}/examples/monte-carlo-lab/docs/content/journal/2026/2026-10-04-r-1-fitted-the-mean-absolute-error.md +0 -0
  74. {lab_kit_cli-0.3.0 → lab_kit_cli-0.4.0}/examples/monte-carlo-lab/docs/content/journal/2026/2026-10-04-ran-the-pi-error-scaling-protocol.md +0 -0
  75. {lab_kit_cli-0.3.0 → lab_kit_cli-0.4.0}/examples/monte-carlo-lab/docs/content/journal/2026/2026-10-04-recorded-r-1.md +0 -0
  76. {lab_kit_cli-0.3.0 → lab_kit_cli-0.4.0}/examples/monte-carlo-lab/docs/content/journal/2026/2026-10-04-recorded-r-2.md +0 -0
  77. {lab_kit_cli-0.3.0 → lab_kit_cli-0.4.0}/examples/monte-carlo-lab/docs/content/journal/2026/2026-10-04-reported-the-pi-error-scaling-experiment.md +0 -0
  78. {lab_kit_cli-0.3.0 → lab_kit_cli-0.4.0}/examples/monte-carlo-lab/docs/content/journal/2026/2026-10-04-the-square-root-law-holds-per-size-predictions.md +0 -0
  79. {lab_kit_cli-0.3.0 → lab_kit_cli-0.4.0}/examples/monte-carlo-lab/docs/content/maps/research.html +0 -0
  80. {lab_kit_cli-0.3.0 → lab_kit_cli-0.4.0}/examples/monte-carlo-lab/docs/content/projects/lab/index.html +0 -0
  81. {lab_kit_cli-0.3.0 → lab_kit_cli-0.4.0}/examples/monte-carlo-lab/docs/content/protocols/pi-error-scaling.md +0 -0
  82. {lab_kit_cli-0.3.0 → lab_kit_cli-0.4.0}/examples/monte-carlo-lab/docs/content/questions/Q-1.md +0 -0
  83. {lab_kit_cli-0.3.0 → lab_kit_cli-0.4.0}/examples/monte-carlo-lab/docs/content/reports/pi-error-scaling-report/index.html +0 -0
  84. {lab_kit_cli-0.3.0 → lab_kit_cli-0.4.0}/examples/monte-carlo-lab/docs/content/results/R-1.md +0 -0
  85. {lab_kit_cli-0.3.0 → lab_kit_cli-0.4.0}/examples/monte-carlo-lab/docs/content/results/R-2.md +0 -0
  86. {lab_kit_cli-0.3.0 → lab_kit_cli-0.4.0}/examples/monte-carlo-lab/docs/folio.yaml +0 -0
  87. {lab_kit_cli-0.3.0 → lab_kit_cli-0.4.0}/examples/monte-carlo-lab/experiments/.gitkeep +0 -0
  88. {lab_kit_cli-0.3.0 → lab_kit_cli-0.4.0}/examples/monte-carlo-lab/experiments/pi-error-scaling/.gitignore +0 -0
  89. {lab_kit_cli-0.3.0 → lab_kit_cli-0.4.0}/examples/monte-carlo-lab/experiments/pi-error-scaling/bin/.gitkeep +0 -0
  90. {lab_kit_cli-0.3.0 → lab_kit_cli-0.4.0}/examples/monte-carlo-lab/experiments/pi-error-scaling/bin/fold.py +0 -0
  91. {lab_kit_cli-0.3.0 → lab_kit_cli-0.4.0}/examples/monte-carlo-lab/experiments/pi-error-scaling/bin/sample.py +0 -0
  92. {lab_kit_cli-0.3.0 → lab_kit_cli-0.4.0}/examples/monte-carlo-lab/experiments/pi-error-scaling/lock.json +0 -0
  93. {lab_kit_cli-0.3.0 → lab_kit_cli-0.4.0}/examples/monte-carlo-lab/experiments/pi-error-scaling/runs/.gitkeep +0 -0
  94. {lab_kit_cli-0.3.0 → lab_kit_cli-0.4.0}/examples/monte-carlo-lab/experiments/pi-error-scaling/runs/20261004-231629/MANIFEST.sha256 +0 -0
  95. {lab_kit_cli-0.3.0 → lab_kit_cli-0.4.0}/examples/monte-carlo-lab/experiments/pi-error-scaling/runs/20261004-231629/out/estimates.tsv +0 -0
  96. {lab_kit_cli-0.3.0 → lab_kit_cli-0.4.0}/examples/monte-carlo-lab/experiments/pi-error-scaling/runs/20261004-231629/run.json +0 -0
  97. {lab_kit_cli-0.3.0 → lab_kit_cli-0.4.0}/examples/monte-carlo-lab/experiments/pi-error-scaling/score.yaml +0 -0
  98. {lab_kit_cli-0.3.0 → lab_kit_cli-0.4.0}/examples/monte-carlo-lab/ops/STATE.md +0 -0
  99. {lab_kit_cli-0.3.0 → lab_kit_cli-0.4.0}/examples/monte-carlo-lab/ops/missions/.gitkeep +0 -0
  100. {lab_kit_cli-0.3.0 → lab_kit_cli-0.4.0}/examples/monte-carlo-lab/ops/missions/2026-10-04-pi-error-scaling.md +0 -0
  101. {lab_kit_cli-0.3.0 → lab_kit_cli-0.4.0}/examples/monte-carlo-lab/substrate/README.md +0 -0
  102. {lab_kit_cli-0.3.0 → lab_kit_cli-0.4.0}/examples/monte-carlo-lab/substrate/estimator.py +0 -0
  103. {lab_kit_cli-0.3.0 → lab_kit_cli-0.4.0}/method/DISCIPLINE.md +0 -0
  104. {lab_kit_cli-0.3.0 → lab_kit_cli-0.4.0}/method/LADDER.md +0 -0
  105. {lab_kit_cli-0.3.0 → lab_kit_cli-0.4.0}/skills/plan-mission/SKILL.md +0 -0
  106. {lab_kit_cli-0.3.0 → lab_kit_cli-0.4.0}/skills/review/SKILL.md +0 -0
  107. {lab_kit_cli-0.3.0 → lab_kit_cli-0.4.0}/skills/run-mission/SKILL.md +0 -0
  108. {lab_kit_cli-0.3.0 → lab_kit_cli-0.4.0}/skills/set-up-lab/SKILL.md +0 -0
  109. {lab_kit_cli-0.3.0 → lab_kit_cli-0.4.0}/src/lab_kit/__main__.py +0 -0
  110. {lab_kit_cli-0.3.0 → lab_kit_cli-0.4.0}/src/lab_kit/budgets.py +0 -0
  111. {lab_kit_cli-0.3.0 → lab_kit_cli-0.4.0}/src/lab_kit/checks/__init__.py +0 -0
  112. {lab_kit_cli-0.3.0 → lab_kit_cli-0.4.0}/src/lab_kit/checks/base.py +0 -0
  113. {lab_kit_cli-0.3.0 → lab_kit_cli-0.4.0}/src/lab_kit/checks/files.py +0 -0
  114. {lab_kit_cli-0.3.0 → lab_kit_cli-0.4.0}/src/lab_kit/checks/results.py +0 -0
  115. {lab_kit_cli-0.3.0 → lab_kit_cli-0.4.0}/src/lab_kit/commands/__init__.py +0 -0
  116. {lab_kit_cli-0.3.0 → lab_kit_cli-0.4.0}/src/lab_kit/commands/experiments.py +0 -0
  117. {lab_kit_cli-0.3.0 → lab_kit_cli-0.4.0}/src/lab_kit/commands/runs.py +0 -0
  118. {lab_kit_cli-0.3.0 → lab_kit_cli-0.4.0}/src/lab_kit/commands/setup.py +0 -0
  119. {lab_kit_cli-0.3.0 → lab_kit_cli-0.4.0}/src/lab_kit/commands/status.py +0 -0
  120. {lab_kit_cli-0.3.0 → lab_kit_cli-0.4.0}/src/lab_kit/data.py +0 -0
  121. {lab_kit_cli-0.3.0 → lab_kit_cli-0.4.0}/src/lab_kit/errors.py +0 -0
  122. {lab_kit_cli-0.3.0 → lab_kit_cli-0.4.0}/src/lab_kit/frozen.py +0 -0
  123. {lab_kit_cli-0.3.0 → lab_kit_cli-0.4.0}/src/lab_kit/gitlog.py +0 -0
  124. {lab_kit_cli-0.3.0 → lab_kit_cli-0.4.0}/src/lab_kit/ops.py +0 -0
  125. {lab_kit_cli-0.3.0 → lab_kit_cli-0.4.0}/src/lab_kit/protocol.py +0 -0
  126. {lab_kit_cli-0.3.0 → lab_kit_cli-0.4.0}/src/lab_kit/records.py +0 -0
  127. {lab_kit_cli-0.3.0 → lab_kit_cli-0.4.0}/src/lab_kit/rederive.py +0 -0
  128. {lab_kit_cli-0.3.0 → lab_kit_cli-0.4.0}/tests/conftest.py +0 -0
  129. {lab_kit_cli-0.3.0 → lab_kit_cli-0.4.0}/tests/test_budgets.py +0 -0
  130. {lab_kit_cli-0.3.0 → lab_kit_cli-0.4.0}/tests/test_checks.py +0 -0
  131. {lab_kit_cli-0.3.0 → lab_kit_cli-0.4.0}/tests/test_end_to_end.py +0 -0
  132. {lab_kit_cli-0.3.0 → lab_kit_cli-0.4.0}/tests/test_example.py +0 -0
  133. {lab_kit_cli-0.3.0 → lab_kit_cli-0.4.0}/tests/test_instrument.py +0 -0
  134. {lab_kit_cli-0.3.0 → lab_kit_cli-0.4.0}/tests/test_runs.py +0 -0
@@ -1,5 +1,13 @@
1
1
  # Changelog
2
2
 
3
+ ## 0.4.0
4
+
5
+ - The run store. `store:` in `lab.yaml` names one shared filesystem, such as `/labs/<lab>`, where every finished run's raw output goes. When a run ends, `lab-kit run` publishes `work/` and `run.log` to the store at the run folder's own path: a copy into `.incoming/`, checked against `work.sha256` (nothing missing, extra or changed), renamed into place, made read-only. It then writes `store.json` beside `run.json` and deletes the local copy. A run in the store is never overwritten. A failed publish is noted in `run.log`, keeps the local copy and never changes the run's exit code.
6
+ - `lab-kit publish <slug>/<run id>... | --pending [--keep]`, `lab-kit fetch <slug>/<run id>` and `lab-kit verify <slug>/<run id>`. A machine that does not mount the store sets `LAB_STORE_HOST` to one that does (rsync over ssh, `sha256sum` there).
7
+ - `lab-work-stored` (warning): with a store set, every ended run with a `work.sha256` has a `store.json`.
8
+ - The experiment skill says to commit `store.json` with the run, and never to mount the store where the system under test runs.
9
+ - A lab without `store:` works as before.
10
+
3
11
  ## 0.3.0
4
12
 
5
13
  - The lock covers the instrument. `lab-kit lock` records, beside the protocol's hash, `instrument` (one sha256 over the files in the experiment's `bin/`, sorted by path, each fed as its path and its bytes, `__pycache__` left out), `commit` (the git head) and `lab-kit` (the version). `lab-lock-intact` fails when `bin/` no longer hashes to `instrument`, and `lab-kit run` refuses to start on such a lock, as it already did for a changed protocol.
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.5
2
2
  Name: lab-kit-cli
3
- Version: 0.3.0
3
+ Version: 0.4.0
4
4
  Summary: A research lab's method and machinery on top of folio: pre-register, lock, run, score, re-derive.
5
5
  Project-URL: Homepage, https://github.com/pierg/lab-kit
6
6
  Project-URL: Issues, https://github.com/pierg/lab-kit/issues
@@ -53,6 +53,7 @@ A lab is a repository. Its folio library sits at the root or in a folder such as
53
53
  MANIFEST.sha256 a hash of every file in out/, written when the run ends
54
54
  work/ run.log raw output and log, gitignored
55
55
  work.sha256 a hash of every file in work/, written when the run ends; committed
56
+ store.json where work/ and run.log were published, and when (when lab.yaml names a store); committed
56
57
  score.yaml the review's scorecard against the lock, and the results to record
57
58
  ops/
58
59
  STATE.md what is true now and what is next; a pointer, never a record
@@ -69,13 +70,14 @@ A lab is a repository. Its folio library sits at the root or in a folder such as
69
70
  `lab.yaml`:
70
71
 
71
72
  ```yaml
72
- lab-kit: 0.3.0 # the version this lab is checked with
73
+ lab-kit: 0.4.0 # the version this lab is checked with
73
74
  library: docs # where folio.yaml is; "." for the root
74
75
  front: lab # the front door's id: a project document
75
76
  frozen: # surfaces never edited in place
76
77
  - substrate/
77
78
  tools: [tools/fold.py] # lab-wide scripts; each must pass --selftest
78
79
  rederive_timeout: 120 # seconds per re-derive command
80
+ store: /labs/monte-carlo-lab # optional: where finished runs' raw output is published (below)
79
81
  ```
80
82
 
81
83
  Paths in `lab.yaml`, `run.json`, `score.yaml`, and a result's `evidence` and `rederive` fields, are relative to the lab root. folio resolves a result's paths from the charter's `root`, so a lab whose library is in `docs/` sets `root: ..` in `folio.yaml`. Both tools then read the same path the same way.
@@ -84,6 +86,8 @@ Paths in `lab.yaml`, `run.json`, `score.yaml`, and a result's `evidence` and `re
84
86
 
85
87
  `run.json` holds `protocol`, `lock` (the hash the run was checked against), `config` (the pinned configuration, copied from the protocol), `mission`, `spend`, `command`, `started`, `ended`, `exit`, `pid`, `spent`, `commit` (the git head the run started at; null outside git or before the first commit) and `lab-kit` (the version that started it). Times are UTC, to the second. A run's command finds its folders in the environment: `LAB_RUN_DIR`, `LAB_OUT` and `LAB_WORK`. A run that spends writes `out/spend.json` with `{"spent": <number>}`, in the unit of its mission's cap, and `lab-kit run` copies it into `spent` when the command exits. When the command exits, `lab-kit run` also writes `work.sha256` beside `MANIFEST.sha256`, in the same format, one line per file under `work/`, when that folder exists: the raw output stays out of git, and its hashes go in, so any copy of it can be verified.
86
88
 
89
+ **The store.** `store:` in `lab.yaml` is optional: an absolute path, such as `/labs/<lab>`, where every machine that runs the lab mounts one shared filesystem. When it is set, a run's raw output leaves the machine that ran it once the run ends: `lab-kit run` publishes `work/` and `run.log` to `<store>/<the run folder's path from the lab root>/`. Publishing copies into `<store>/.incoming/`, checks the copy against `work.sha256` (no file missing, none extra, none changed), renames it into place in one step, makes it read-only, writes `store.json` (`store`, `files`, `published`, `by`) beside `run.json`, and then deletes the local `work/` and `run.log`. A run already in the store is never overwritten: a copy there that verifies counts as published, one that does not is refused. A publish that fails leaves the run where it is, notes why in `run.log`, and never changes the run's exit code; `lab-kit publish --pending` tries again. The command a run starts is never told the store's path, and a lab's instrument never mounts it into the sandbox where the system under test runs: the store holds other runs' raw output, answers included. A machine that does not mount the store sets `LAB_STORE_HOST` to a host that does; lab-kit then copies with rsync over ssh and verifies there with `sha256sum`.
90
+
87
91
  `lock.json` holds `protocol`, `path`, `sha256` and `locked`, and from lab-kit 0.3.0 `instrument`, `commit` and `lab-kit`. `instrument` is one sha256 over every file under `experiments/<slug>/bin/` (a `__pycache__` folder left out), sorted by its path from `bin/`, each fed as its path, a newline, then its bytes. `commit` is the git head at the lock (null outside git or before the first commit), and `lab-kit` the version that took it. A lock or a run record without these fields predates them and is read as it is: records are never edited.
88
92
 
89
93
  A mission file starts with YAML front matter: `title`, `status` (`draft`, `active` or `concluded`), `rests_on` (the ids it rests on), `approved` (who approved the mission, and when, with the date as `YYYY-MM-DD`) and `cap` (a number, in the unit its runs report spend; `0` when nothing spends), and optionally `budget` (the name of a standing budget it draws on). A mission is `draft` until it is approved; `approved` is required from `active` on. Its log is everything under its `## Log` heading; entries are only added at the bottom. `ops/STATE.md` has one line `Active mission: <path>` or `Active mission: none`, and never names a draft.
@@ -141,6 +145,7 @@ A result that is not promoted is recorded too: a journal entry about the protoco
141
145
  | `lab-method-current` | `.lab/method/`, the five skills and the four roles match the lab-kit version in `lab.yaml`. |
142
146
  | `lab-state-pointer` | `ops/STATE.md` names an active mission file or "none", stays under 300 words, and cites ids for any number (warning). |
143
147
  | `lab-scored-reported` | Every scored experiment has a report citing its results, or a journal entry about its protocol saying why not (warning). |
148
+ | `lab-work-stored` | When `lab.yaml` names a `store:`, every ended run with a `work.sha256` has a `store.json`: its raw output was published (warning). |
144
149
 
145
150
  The gate checks that the record is consistent. It does not check that the record is honest. That is the reviewer's job.
146
151
 
@@ -158,6 +163,9 @@ Only skills call it, the way an agent calls git. Every command that changes file
158
163
  | `lab-kit lock <slug>` | Writes `lock.json` for a protocol whose status is `locked` and whose gate is green, hashing the file with its status field set aside, and records the instrument's hash, the git commit and the lab-kit version. Refuses otherwise. |
159
164
  | `lab-kit run <slug> [--mission <file>] [--spend] [--wait] -- <command>` | Checks the lock, writes `run.json`, starts the command in the background from the lab root, records the git commit and the lab-kit version, and writes `MANIFEST.sha256` for `out/` and `work.sha256` for `work/` when it exits. It refuses when the protocol or `bin/` no longer hashes to the lock. With `--mission`, it refuses a mission whose status is not `active` or that records no approval. `--wait` waits for the end, for a short token-free stage. |
160
165
  | `lab-kit runs [--live]` | Lists runs and their state: running, finished, failed, orphaned. |
166
+ | `lab-kit publish <slug>/<run id>... \| --pending [--keep]` | Publishes ended runs' raw output to the store, as above. `--pending` takes every ended run whose `work/` is still here. `--keep` keeps the local copy. |
167
+ | `lab-kit fetch <slug>/<run id>` | Copies a published run's `work/` and `run.log` back into its folder, writable, and verifies them against `work.sha256`. |
168
+ | `lab-kit verify <slug>/<run id>` | Checks the store's copy of a run against its `work.sha256`. Changes nothing. |
161
169
  | `lab-kit score <slug>` | Creates `score.yaml` from the locked protocol's prediction and rule ids, with empty verdicts. |
162
170
  | `lab-kit rederive <R-n>` | Runs one result's re-derive command and compares its output with the number. |
163
171
  | `lab-kit freeze <path>` | Records a frozen surface's hash. Only the operator asks for this. |
@@ -52,7 +52,7 @@ You take one question from a draft protocol to a finished run with its evidence
52
52
  6. **Launch.** Run `lab-kit run <slug> -- <command>`. Add `--spend --mission <file>` for a run that spends. Watch it at once: arm one watcher on its log that fires on the end marker or an error signature. A silent launch failure can waste a whole night.
53
53
  7. **Supervise.** Write a journal entry of kind `run` about the protocol for each launch, checkpoint, surprise and termination, with `folio journal add`. Keep each entry to what changed. Record a deviation the moment you see it. Commit each entry with `.folio/` after `folio index`; never stage the run's working files.
54
54
  8. **On failure.** End the run. Start a fresh one with the same configuration. Disclose it in a `run` entry. Never swap a model or a tool mid-run.
55
- 9. **Conclude.** When the run ends, `lab-kit run` writes the manifest of `out/`. Commit `out/`, `MANIFEST.sha256`, `work.sha256` and `run.json` in one commit; `work/` stays out of git, and `work.sha256` is what any copy of it is verified against. Run `lab-kit check`. Hand the experiment to the review skill. Do not score your own run.
55
+ 9. **Conclude.** When the run ends, `lab-kit run` writes the manifest of `out/`. Commit `out/`, `MANIFEST.sha256`, `work.sha256` and `run.json` in one commit, with `store.json` when `lab.yaml` names a store; `work/` stays out of git, and `work.sha256` is what any copy of it is verified against. With a store, `lab-kit run` has already published `work/` there and deleted the local copy; if `run.log` says the publish failed, run `lab-kit publish --pending`. Never mount the store where the system under test runs: it holds other runs' answers. Run `lab-kit check`. Hand the experiment to the review skill. Do not score your own run.
56
56
 
57
57
  ## Stops
58
58
 
@@ -1,4 +1,4 @@
1
- lab-kit: 0.3.0 # the version this lab is checked with
1
+ lab-kit: 0.4.0 # the version this lab is checked with
2
2
  library: docs # where folio.yaml is; "." for the root
3
3
  front: lab # the front door's id: a project document
4
4
  frozen: [substrate/] # surfaces never edited in place; record each with `lab-kit freeze`
@@ -5,7 +5,7 @@ build-backend = "hatchling.build"
5
5
  [project]
6
6
  dynamic = ["readme"]
7
7
  name = "lab-kit-cli"
8
- version = "0.3.0"
8
+ version = "0.4.0"
9
9
  description = "A research lab's method and machinery on top of folio: pre-register, lock, run, score, re-derive."
10
10
  license = "MIT"
11
11
  authors = [{ name = "Piergiuseppe Mallozzi" }]
@@ -52,7 +52,7 @@ You take one question from a draft protocol to a finished run with its evidence
52
52
  6. **Launch.** Run `lab-kit run <slug> -- <command>`. Add `--spend --mission <file>` for a run that spends. Watch it at once: arm one watcher on its log that fires on the end marker or an error signature. A silent launch failure can waste a whole night.
53
53
  7. **Supervise.** Write a journal entry of kind `run` about the protocol for each launch, checkpoint, surprise and termination, with `folio journal add`. Keep each entry to what changed. Record a deviation the moment you see it. Commit each entry with `.folio/` after `folio index`; never stage the run's working files.
54
54
  8. **On failure.** End the run. Start a fresh one with the same configuration. Disclose it in a `run` entry. Never swap a model or a tool mid-run.
55
- 9. **Conclude.** When the run ends, `lab-kit run` writes the manifest of `out/`. Commit `out/`, `MANIFEST.sha256`, `work.sha256` and `run.json` in one commit; `work/` stays out of git, and `work.sha256` is what any copy of it is verified against. Run `lab-kit check`. Hand the experiment to the review skill. Do not score your own run.
55
+ 9. **Conclude.** When the run ends, `lab-kit run` writes the manifest of `out/`. Commit `out/`, `MANIFEST.sha256`, `work.sha256` and `run.json` in one commit, with `store.json` when `lab.yaml` names a store; `work/` stays out of git, and `work.sha256` is what any copy of it is verified against. With a store, `lab-kit run` has already published `work/` there and deleted the local copy; if `run.log` says the publish failed, run `lab-kit publish --pending`. Never mount the store where the system under test runs: it holds other runs' answers. Run `lab-kit check`. Hand the experiment to the review skill. Do not score your own run.
56
56
 
57
57
  ## Stops
58
58
 
@@ -1,3 +1,3 @@
1
1
  """lab-kit: a research lab's method and machinery on top of folio."""
2
2
 
3
- __version__ = "0.3.0"
3
+ __version__ = "0.4.0"
@@ -138,3 +138,17 @@ def evidence_sealed(lab: Lab, rep: Reporter) -> None:
138
138
  elif recorded[name] != actual[name]:
139
139
  rep.add(manifest_path, f"out/{name} changed after the run ended")
140
140
  rep.guard(run.path, one)
141
+
142
+
143
+ def work_stored(lab: Lab, rep: Reporter) -> None:
144
+ """With a store in lab.yaml, every ended run with a work.sha256 records that it was published (warning)."""
145
+ if not lab.settings.store:
146
+ return
147
+ from ..store import STORE_RECORD
148
+ for run in records.runs(lab):
149
+ if not run.ended or not (run.folder / records.WORK_MANIFEST).is_file():
150
+ continue
151
+ if not (run.folder / STORE_RECORD).is_file():
152
+ rep.warn(run.path.rsplit("/", 1)[0] + "/" + records.WORK_MANIFEST,
153
+ f"its raw output was never published to the store {lab.settings.store}; "
154
+ "`lab-kit publish --pending` publishes it, then commit store.json")
@@ -30,6 +30,7 @@ CHECKS: tuple[LabCheck, ...] = (
30
30
  LabCheck("lab-method-current", "error", files.method_current),
31
31
  LabCheck("lab-state-pointer", "warning", files.state_pointer),
32
32
  LabCheck("lab-scored-reported", "warning", results.scored_reported),
33
+ LabCheck("lab-work-stored", "warning", locks.work_stored),
33
34
  )
34
35
  BY_NAME = {c.name: c for c in CHECKS}
35
36
 
@@ -10,6 +10,7 @@ from pathlib import Path
10
10
  from . import __version__, frozen
11
11
  from . import lab as lab_mod
12
12
  from . import rederive as rederive_mod
13
+ from . import store as store_mod
13
14
  from .checks import run as checks_run
14
15
  from .commands import experiments, runs, setup, status
15
16
  from .errors import LabError
@@ -65,6 +66,37 @@ def cmd_runs(args: argparse.Namespace) -> int:
65
66
  return 0
66
67
 
67
68
 
69
+ def cmd_publish(args: argparse.Namespace) -> int:
70
+ lab = _lab()
71
+ if args.pending == bool(args.runs):
72
+ raise LabError("name runs as <slug>/<run id>, or pass --pending for every ended run still here")
73
+ store_mod.root(lab) # a lab with no store says so once
74
+ targets = store_mod.pending(lab) if args.pending else [store_mod.find_run(lab, r) for r in args.runs]
75
+ if not targets:
76
+ print("Nothing to publish.")
77
+ return 0
78
+ failed = 0
79
+ for run in targets:
80
+ try:
81
+ _print(store_mod.publish(lab, run, keep=args.keep))
82
+ except (LabError, OSError) as exc:
83
+ print(f"lab-kit: {lab.rel(run.folder)}: not published: {exc}", file=sys.stderr)
84
+ failed += 1
85
+ return 1 if failed else 0
86
+
87
+
88
+ def cmd_fetch(args: argparse.Namespace) -> int:
89
+ lab = _lab()
90
+ _print(store_mod.fetch(lab, store_mod.find_run(lab, args.run)))
91
+ return 0
92
+
93
+
94
+ def cmd_verify(args: argparse.Namespace) -> int:
95
+ lab = _lab()
96
+ _print(store_mod.verify(lab, store_mod.find_run(lab, args.run)))
97
+ return 0
98
+
99
+
68
100
  def cmd_score(args: argparse.Namespace) -> int:
69
101
  _print(experiments.score(_lab(), args.slug))
70
102
  return 0
@@ -131,6 +163,21 @@ def parser() -> argparse.ArgumentParser:
131
163
  s.add_argument("--live", action="store_true", help="only running and orphaned runs")
132
164
  s.set_defaults(func=cmd_runs)
133
165
 
166
+ s = sub.add_parser("publish", help="Copy ended runs' raw output to the store named in lab.yaml, verify it "
167
+ "against work.sha256, and delete the local copy.")
168
+ s.add_argument("runs", nargs="*", metavar="SLUG/RUN_ID")
169
+ s.add_argument("--pending", action="store_true", help="every ended run whose work/ is still here")
170
+ s.add_argument("--keep", action="store_true", help="keep the local work/ and run.log")
171
+ s.set_defaults(func=cmd_publish)
172
+
173
+ s = sub.add_parser("fetch", help="Copy a published run's raw output back into its folder, verified.")
174
+ s.add_argument("run", metavar="SLUG/RUN_ID")
175
+ s.set_defaults(func=cmd_fetch)
176
+
177
+ s = sub.add_parser("verify", help="Check the store's copy of a run against its work.sha256.")
178
+ s.add_argument("run", metavar="SLUG/RUN_ID")
179
+ s.set_defaults(func=cmd_verify)
180
+
134
181
  s = sub.add_parser("score", help="Create score.yaml from the lock's prediction and rule ids.")
135
182
  s.add_argument("slug")
136
183
  s.set_defaults(func=cmd_score)
@@ -14,7 +14,7 @@ from folio.library import Library
14
14
  from .errors import LabError
15
15
 
16
16
  LAB_YAML = "lab.yaml"
17
- _KEYS = ("lab-kit", "library", "front", "frozen", "tools", "rederive_timeout")
17
+ _KEYS = ("lab-kit", "library", "front", "frozen", "tools", "rederive_timeout", "store")
18
18
 
19
19
 
20
20
  @dataclass
@@ -25,6 +25,7 @@ class Settings:
25
25
  frozen: list[str]
26
26
  tools: list[str]
27
27
  rederive_timeout: int
28
+ store: str | None = None
28
29
 
29
30
 
30
31
  @dataclass
@@ -115,7 +116,10 @@ def parse_settings(raw: Any, where: str) -> Settings:
115
116
  timeout = raw.get("rederive_timeout", 120)
116
117
  if not isinstance(timeout, int) or isinstance(timeout, bool) or timeout <= 0:
117
118
  raise LabError(f"{where}: `rederive_timeout` must be a positive whole number of seconds")
118
- return Settings(version, library, front or None, lists["frozen"], lists["tools"], timeout)
119
+ store = raw.get("store")
120
+ if store is not None and (not isinstance(store, str) or not Path(store).is_absolute()):
121
+ raise LabError(f"{where}: `store` must be an absolute path, such as /labs/<lab>")
122
+ return Settings(version, library, front or None, lists["frozen"], lists["tools"], timeout, store or None)
119
123
 
120
124
 
121
125
  def load_at(root: Path) -> Lab:
@@ -0,0 +1,285 @@
1
+ """The run store: where a finished run's raw output lives once it leaves the machine that ran it.
2
+
3
+ `lab.yaml` names it with `store:`, an absolute path such as `/labs/<lab>`. A run's raw output
4
+ (`work/` and `run.log`, which git ignores) is published to `<store>/<the run's folder from the lab
5
+ root>/`, so the record in git and its raw data share one path. Publishing copies into
6
+ `<store>/.incoming/`, checks every file against the committed `work.sha256` (no file missing, none
7
+ extra, none changed), renames the copy into place in one step, makes it read-only, writes
8
+ `store.json` beside `run.json`, and only then deletes the local copy. A run id already in the store
9
+ is never overwritten: a copy that verifies is accepted as published, one that does not is refused.
10
+
11
+ Where the store path is not mounted, set `LAB_STORE_HOST` to a host that mounts it: lab-kit then
12
+ copies with rsync over ssh and verifies on that host with `sha256sum`. The path is the same either way.
13
+ """
14
+
15
+ from __future__ import annotations
16
+
17
+ import os
18
+ import shlex
19
+ import shutil
20
+ import socket
21
+ import stat
22
+ import subprocess
23
+ from pathlib import Path
24
+
25
+ from . import records
26
+ from .errors import LabError
27
+ from .lab import Lab
28
+
29
+ STORE_RECORD = "store.json" # beside run.json: where the raw output went, and when; committed
30
+ ENV_HOST = "LAB_STORE_HOST" # a host that mounts the store, for a machine that does not
31
+ RAW = ("work", "run.log") # what is published: the gitignored half of a run
32
+
33
+
34
+ def root(lab: Lab) -> Path:
35
+ if not lab.settings.store:
36
+ raise LabError("lab.yaml names no `store:`; set it to the store's path, such as /labs/<lab>")
37
+ return Path(lab.settings.store)
38
+
39
+
40
+ def find_run(lab: Lab, ref: str) -> records.RunRecord:
41
+ """A run named `<slug>/<run id>`."""
42
+ slug, sep, run_id = ref.strip("/").partition("/")
43
+ if not sep or "/" in run_id:
44
+ raise LabError(f"name a run as <slug>/<run id>, not `{ref}`")
45
+ for run in records.runs(lab, slug):
46
+ if run.run_id == run_id:
47
+ return run
48
+ raise LabError(f"experiments/{slug}/runs/{run_id}/ has no run.json")
49
+
50
+
51
+ def _rel(lab: Lab, run: records.RunRecord) -> str:
52
+ return lab.rel(run.folder)
53
+
54
+
55
+ def _expected(run: records.RunRecord) -> dict[str, str]:
56
+ path = run.folder / records.WORK_MANIFEST
57
+ if not path.is_file():
58
+ raise LabError(f"{run.path.rsplit('/', 1)[0]}/{records.WORK_MANIFEST} does not exist; "
59
+ "only a sealed run is published")
60
+ return records.parse_hashes(path.read_text(encoding="utf-8"), records.WORK_MANIFEST)
61
+
62
+
63
+ def _problems(expected: dict[str, str], found: dict[str, str]) -> list[str]:
64
+ out = [f"missing {p}" for p in sorted(set(expected) - set(found))]
65
+ out += [f"extra {p}" for p in sorted(set(found) - set(expected))]
66
+ out += [f"changed {p}" for p in sorted(set(expected) & set(found)) if expected[p] != found[p]]
67
+ return out
68
+
69
+
70
+ def verify_local(run: records.RunRecord, dest: Path) -> list[str]:
71
+ """What is wrong with the copy of `work/` under `dest`, against work.sha256: nothing when empty."""
72
+ return _problems(_expected(run), records.tree_hashes(dest / "work", dest / "work"))
73
+
74
+
75
+ def _read_only(folder: Path) -> None:
76
+ for path in sorted(folder.rglob("*"), reverse=True):
77
+ mode = path.stat().st_mode
78
+ path.chmod(mode & ~(stat.S_IWUSR | stat.S_IWGRP | stat.S_IWOTH))
79
+ folder.chmod(folder.stat().st_mode & ~(stat.S_IWUSR | stat.S_IWGRP | stat.S_IWOTH))
80
+
81
+
82
+ def _writable(folder: Path) -> None:
83
+ for path in [folder, *folder.rglob("*")]:
84
+ path.chmod(path.stat().st_mode | stat.S_IWUSR)
85
+
86
+
87
+ def _incoming_name(run: records.RunRecord) -> str:
88
+ return f"{run.slug}--{run.run_id}--{socket.gethostname()}--{os.getpid()}"
89
+
90
+
91
+ def _publish_local(lab: Lab, run: records.RunRecord, store: Path) -> str:
92
+ dest = store / _rel(lab, run)
93
+ if dest.exists():
94
+ wrong = verify_local(run, dest)
95
+ if wrong:
96
+ raise LabError(f"{dest} exists and does not match work.sha256 ({'; '.join(wrong[:3])}); "
97
+ "a run in the store is never overwritten")
98
+ return "already in the store, verified"
99
+ incoming = store / ".incoming" / _incoming_name(run)
100
+ if incoming.exists():
101
+ _writable(incoming)
102
+ shutil.rmtree(incoming)
103
+ incoming.mkdir(parents=True)
104
+ try:
105
+ for name in RAW:
106
+ src = run.folder / name
107
+ if src.is_dir():
108
+ shutil.copytree(src, incoming / name, symlinks=False) # a link's bytes, never the link
109
+ elif src.is_file():
110
+ shutil.copy2(src, incoming / name)
111
+ if not (incoming / "work").exists():
112
+ (incoming / "work").mkdir()
113
+ wrong = verify_local(run, incoming)
114
+ if wrong:
115
+ raise LabError(f"the copy in {incoming} does not match work.sha256 ({'; '.join(wrong[:3])}); "
116
+ "nothing was published")
117
+ dest.parent.mkdir(parents=True, exist_ok=True)
118
+ # A directory's own write bit is needed to rename it (its `..` changes), so lock after the move.
119
+ try:
120
+ os.rename(incoming, dest)
121
+ except OSError:
122
+ # Another publish of the same run got there first: accept its copy if it verifies.
123
+ if dest.is_dir() and not verify_local(run, dest):
124
+ _writable(incoming)
125
+ shutil.rmtree(incoming, ignore_errors=True)
126
+ return "already in the store, verified"
127
+ raise
128
+ _read_only(dest)
129
+ except BaseException:
130
+ if incoming.exists():
131
+ _writable(incoming)
132
+ shutil.rmtree(incoming, ignore_errors=True)
133
+ raise
134
+ return "copied and verified"
135
+
136
+
137
+ def _ssh(host: str, script: str, stdin: str | None = None) -> subprocess.CompletedProcess[str]:
138
+ return subprocess.run(["ssh", "-o", "BatchMode=yes", host, script], input=stdin, capture_output=True, text=True)
139
+
140
+
141
+ def _remote_check(folder: str, count: int) -> str:
142
+ """A shell line that verifies <folder>/work against the manifest on stdin: every file, none extra."""
143
+ q = shlex.quote
144
+ return (f"cd {q(folder)}/work && sha256sum --quiet --strict -c - >/dev/null && "
145
+ f"test \"$(find . -type f | wc -l)\" -eq {count} && test -z \"$(find . ! -type f ! -type d)\"")
146
+
147
+
148
+ def _publish_remote(lab: Lab, run: records.RunRecord, store: Path, host: str) -> str:
149
+ dest = (store / _rel(lab, run)).as_posix()
150
+ incoming = (store / ".incoming" / _incoming_name(run)).as_posix()
151
+ q = shlex.quote
152
+ manifest = (run.folder / records.WORK_MANIFEST).read_text(encoding="utf-8")
153
+ count = len(_expected(run))
154
+ if _ssh(host, f"test -e {q(dest)}").returncode == 0:
155
+ if _ssh(host, _remote_check(dest, count), manifest).returncode == 0:
156
+ return f"already in the store on {host}, verified"
157
+ raise LabError(f"{host}:{dest} exists and does not match work.sha256; a run in the store is never overwritten")
158
+ if _ssh(host, f"rm -rf {q(incoming)} && mkdir -p {q(incoming)}/work").returncode:
159
+ raise LabError(f"cannot reach the store on {host} (ssh {host})")
160
+ sources = [str(run.folder / n) for n in RAW if (run.folder / n).exists()]
161
+ copied = subprocess.run(["rsync", "-aL", *sources, f"{host}:{incoming}/"], capture_output=True, text=True)
162
+ if copied.returncode:
163
+ raise LabError(f"rsync to {host}:{incoming} failed: {copied.stderr.strip()[-200:]}")
164
+ if _ssh(host, _remote_check(incoming, count), manifest).returncode:
165
+ _ssh(host, f"chmod -R u+w {q(incoming)}; rm -rf {q(incoming)}")
166
+ raise LabError(f"the copy on {host} does not match work.sha256; nothing was published")
167
+ done = _ssh(host, f"mkdir -p {q(str(Path(dest).parent))} && mv -T {q(incoming)} {q(dest)} && chmod -R a-w {q(dest)}")
168
+ if done.returncode:
169
+ _ssh(host, f"chmod -R u+w {q(incoming)}; rm -rf {q(incoming)}")
170
+ raise LabError(f"cannot move the copy into place on {host}: {done.stderr.strip()[-200:]}")
171
+ return f"copied to {host} and verified"
172
+
173
+
174
+ def mounted(store: Path) -> bool:
175
+ """The store is here: its folder exists, or one of its ancestors (not `/`) is a mount point.
176
+
177
+ An empty mount point left by a mount that failed is not taken for the store, so nothing is
178
+ ever published onto the local disk under it.
179
+ """
180
+ try:
181
+ if store.is_dir():
182
+ return True
183
+ return any(os.path.ismount(p) for p in store.parents if p != Path(p.anchor))
184
+ except OSError:
185
+ return False
186
+
187
+
188
+ def publish(lab: Lab, run: records.RunRecord, keep: bool = False) -> list[str]:
189
+ """Publish one finished run's raw output; then, unless `keep`, delete the local copy."""
190
+ if not run.ended:
191
+ raise LabError(f"{_rel(lab, run)} has not ended; only a sealed run is published")
192
+ store = root(lab)
193
+ host = os.environ.get(ENV_HOST, "").strip()
194
+ if not (run.folder / "work").exists() and not (run.folder / STORE_RECORD).exists():
195
+ raise LabError(f"{_rel(lab, run)}/work/ is not here to publish")
196
+ if (run.folder / STORE_RECORD).exists() and not (run.folder / "work").exists():
197
+ return [f"{_rel(lab, run)}: already published"]
198
+ if not host and mounted(store):
199
+ store.mkdir(parents=True, exist_ok=True)
200
+ how = _publish_local(lab, run, store)
201
+ where = str(store / _rel(lab, run))
202
+ elif host:
203
+ how = _publish_remote(lab, run, store, host)
204
+ where = f"{host}:{store / _rel(lab, run)}"
205
+ else:
206
+ raise LabError(f"the store {store} is not mounted here; set {ENV_HOST} to a host that mounts it")
207
+ records.write_json(run.folder / STORE_RECORD, {
208
+ "store": str(store / _rel(lab, run)), "files": len(_expected(run)),
209
+ "published": records.now(), "by": socket.gethostname(),
210
+ })
211
+ out = [f"{_rel(lab, run)}: {how} at {where}", f"wrote {_rel(lab, run)}/{STORE_RECORD}"]
212
+ if not keep:
213
+ for name in RAW:
214
+ path = run.folder / name
215
+ if path.is_dir():
216
+ shutil.rmtree(path)
217
+ elif path.is_file():
218
+ path.unlink()
219
+ out.append(f"deleted the local {_rel(lab, run)}/work/ and run.log")
220
+ return out
221
+
222
+
223
+ def pending(lab: Lab) -> list[records.RunRecord]:
224
+ """Ended runs whose raw output is still only here."""
225
+ return [r for r in records.runs(lab) if r.ended and (r.folder / "work").exists()
226
+ and (r.folder / records.WORK_MANIFEST).exists()]
227
+
228
+
229
+ def fetch(lab: Lab, run: records.RunRecord) -> list[str]:
230
+ """Copy a published run's raw output back into its run folder, writable, and verify it."""
231
+ store = root(lab)
232
+ src = store / _rel(lab, run)
233
+ host = os.environ.get(ENV_HOST, "").strip()
234
+ if (run.folder / "work").exists() and not verify_local(run, run.folder):
235
+ return [f"{_rel(lab, run)}/work/ is here and verifies"]
236
+ tmp = run.folder / ".fetch"
237
+ if tmp.exists():
238
+ _writable(tmp)
239
+ shutil.rmtree(tmp)
240
+ tmp.mkdir()
241
+ if not host and mounted(store):
242
+ if not src.is_dir():
243
+ raise LabError(f"{src} does not exist: this run was never published")
244
+ for name in RAW:
245
+ if (src / name).is_dir():
246
+ shutil.copytree(src / name, tmp / name, symlinks=False)
247
+ elif (src / name).is_file():
248
+ shutil.copy2(src / name, tmp / name)
249
+ elif host:
250
+ got = subprocess.run(["rsync", "-a", f"{host}:{src.as_posix()}/", f"{tmp}/"], capture_output=True, text=True)
251
+ if got.returncode:
252
+ raise LabError(f"rsync from {host}:{src} failed: {got.stderr.strip()[-200:]}")
253
+ else:
254
+ raise LabError(f"the store {store} is not mounted here; set {ENV_HOST} to a host that mounts it")
255
+ _writable(tmp)
256
+ wrong = verify_local(run, tmp)
257
+ if wrong:
258
+ shutil.rmtree(tmp)
259
+ raise LabError(f"the copy from the store does not match work.sha256 ({'; '.join(wrong[:3])})")
260
+ for name in RAW:
261
+ target = run.folder / name
262
+ if target.is_dir():
263
+ shutil.rmtree(target)
264
+ if (tmp / name).exists():
265
+ os.rename(tmp / name, target)
266
+ shutil.rmtree(tmp)
267
+ return [f"{_rel(lab, run)}: fetched from the store and verified against work.sha256"]
268
+
269
+
270
+ def verify(lab: Lab, run: records.RunRecord) -> list[str]:
271
+ """Check the store's copy of a run against work.sha256. Raises when it does not match."""
272
+ store = root(lab)
273
+ host = os.environ.get(ENV_HOST, "").strip()
274
+ dest = store / _rel(lab, run)
275
+ if host:
276
+ manifest = (run.folder / records.WORK_MANIFEST).read_text(encoding="utf-8")
277
+ if _ssh(host, _remote_check(dest.as_posix(), len(_expected(run))), manifest).returncode:
278
+ raise LabError(f"{host}:{dest} is missing or does not match work.sha256")
279
+ return [f"{_rel(lab, run)}: the store's copy on {host} matches work.sha256"]
280
+ if not dest.is_dir():
281
+ raise LabError(f"{dest} does not exist: this run was never published")
282
+ wrong = verify_local(run, dest)
283
+ if wrong:
284
+ raise LabError(f"{dest} does not match work.sha256: {'; '.join(wrong[:5])}")
285
+ return [f"{_rel(lab, run)}: the store's copy matches work.sha256"]
@@ -4,6 +4,8 @@
4
4
  and writes its pid into `run.json`. This process waits for that, runs the command from the lab root with its output in
5
5
  `run.log`, and when the command exits writes `MANIFEST.sha256` for `out/`,
6
6
  `work.sha256` for `work/` when it exists, and the exit code and the end time into `run.json`.
7
+ When `lab.yaml` names a `store:`, it then publishes the run's raw output there (lab_kit.store); a
8
+ publish that fails is noted in `run.log` and leaves the run here for `lab-kit publish --pending`.
7
9
  """
8
10
 
9
11
  from __future__ import annotations
@@ -74,8 +76,28 @@ def supervise(run_dir: Path, lab_root: Path) -> int:
74
76
  if data.get("spend"):
75
77
  data["spent"] = _spent(run_dir / "out")
76
78
  records.write_json(run_json, data)
79
+ _publish(run_dir, lab_root)
77
80
  return code
78
81
 
79
82
 
83
+ def _publish(run_dir: Path, lab_root: Path) -> None:
84
+ """Publish the sealed run to the lab's store, if it names one. Never changes the run's exit code."""
85
+ import yaml
86
+
87
+ from . import lab as lab_mod
88
+ from . import store
89
+ try:
90
+ raw = yaml.safe_load((lab_root / lab_mod.LAB_YAML).read_text(encoding="utf-8")) or {}
91
+ if not isinstance(raw, dict) or not raw.get("store"):
92
+ return # a lab without a store: nothing more happens after the seal
93
+ lab = lab_mod.load_at(lab_root)
94
+ run = next(r for r in records.runs(lab, run_dir.parent.parent.name) if r.run_id == run_dir.name)
95
+ store.publish(lab, run)
96
+ except Exception as exc: # noqa: BLE001 - a failed publish never changes the run's outcome
97
+ if (run_dir / "run.log").exists():
98
+ with (run_dir / "run.log").open("a", encoding="utf-8") as log:
99
+ log.write(f"lab-kit: not published to the store: {exc}; `lab-kit publish --pending` retries\n")
100
+
101
+
80
102
  if __name__ == "__main__":
81
103
  raise SystemExit(supervise(Path(sys.argv[1]), Path(sys.argv[2])))
@@ -0,0 +1,108 @@
1
+ """The run store: publish, verify, fetch, and the lab-work-stored warning."""
2
+
3
+ from __future__ import annotations
4
+
5
+ import stat
6
+ from pathlib import Path
7
+
8
+ from lab_kit import records
9
+ from lab_kit.lab import load
10
+
11
+ WRITER = ("import os, pathlib; w = pathlib.Path(os.environ['LAB_WORK']); (w / 'a').mkdir(); "
12
+ "(w / 'a' / 'trace.txt').write_text('raw'); (w / 'b.txt').write_text('more'); "
13
+ "open(os.environ['LAB_OUT'] + '/n.txt', 'w').write('1')")
14
+
15
+
16
+ def _with_store(store: Path):
17
+ def before(root: Path) -> None:
18
+ text = (root / "lab.yaml").read_text(encoding="utf-8")
19
+ (root / "lab.yaml").write_text(text + f"store: {store}\n", encoding="utf-8")
20
+ return before
21
+
22
+
23
+ def test_a_run_publishes_to_the_store_and_drops_its_local_copy(make_lab, lab_kit, tmp_path) -> None:
24
+ store = tmp_path / "labs" / "mc"
25
+ store.mkdir(parents=True)
26
+ root = make_lab(_with_store(store))
27
+ code, out = lab_kit("run", "pi-error-scaling", "--wait", "--", "python3", "-c", WRITER)
28
+ assert code == 0, out
29
+ run = records.runs(load(root), "pi-error-scaling")[-1]
30
+ dest = store / "experiments" / "pi-error-scaling" / "runs" / run.run_id
31
+ assert (dest / "work" / "a" / "trace.txt").read_text() == "raw"
32
+ assert (dest / "run.log").is_file()
33
+ assert not (dest / "work" / "b.txt").stat().st_mode & stat.S_IWUSR
34
+ assert not (run.folder / "work").exists() and not (run.folder / "run.log").exists()
35
+ assert (run.folder / "store.json").is_file()
36
+ assert not any((store / ".incoming").iterdir())
37
+ ref = f"pi-error-scaling/{run.run_id}"
38
+ assert lab_kit("verify", ref)[0] == 0
39
+ code, out = lab_kit("fetch", ref)
40
+ assert code == 0 and (run.folder / "work" / "b.txt").read_text() == "more"
41
+ assert (run.folder / "work" / "b.txt").stat().st_mode & stat.S_IWUSR
42
+ assert lab_kit("check", "--only", "lab-work-stored") == (0, "lab-kit check: 0 errors, 0 warnings\n")
43
+
44
+
45
+ def test_a_tampered_store_copy_fails_verify_and_is_never_overwritten(make_lab, lab_kit, tmp_path) -> None:
46
+ store = tmp_path / "labs" / "mc"
47
+ store.mkdir(parents=True)
48
+ root = make_lab(_with_store(store))
49
+ assert lab_kit("run", "pi-error-scaling", "--wait", "--", "python3", "-c", WRITER)[0] == 0
50
+ run = records.runs(load(root), "pi-error-scaling")[-1]
51
+ ref = f"pi-error-scaling/{run.run_id}"
52
+ lab_kit("fetch", ref)
53
+ copy = store / "experiments" / "pi-error-scaling" / "runs" / run.run_id / "work" / "b.txt"
54
+ copy.chmod(0o644)
55
+ copy.write_text("edited")
56
+ code, out = lab_kit("verify", ref)
57
+ assert code == 2 and "changed b.txt" in out
58
+ (run.folder / "store.json").unlink()
59
+ code, out = lab_kit("publish", ref)
60
+ assert code == 1 and "never overwritten" in out
61
+ assert (run.folder / "work" / "b.txt").read_text() == "more"
62
+
63
+
64
+ def test_without_a_mounted_store_the_run_stays_here_and_check_warns(make_lab, lab_kit, tmp_path) -> None:
65
+ root = make_lab(_with_store(tmp_path / "not-mounted"))
66
+ assert lab_kit("run", "pi-error-scaling", "--wait", "--", "python3", "-c", WRITER)[0] == 0
67
+ run = records.runs(load(root), "pi-error-scaling")[-1]
68
+ assert (run.folder / "work" / "b.txt").is_file()
69
+ assert "not published to the store" in (run.folder / "run.log").read_text()
70
+ code, out = lab_kit("check", "--only", "lab-work-stored")
71
+ assert code == 0 and "never published" in out and "1 warnings" in out
72
+ (tmp_path / "not-mounted").mkdir()
73
+ code, out = lab_kit("publish", "--pending")
74
+ assert code == 0 and "copied and verified" in out
75
+ assert not (run.folder / "work").exists()
76
+
77
+
78
+ def test_a_lab_without_a_store_is_unchanged(make_lab, lab_kit) -> None:
79
+ root = make_lab()
80
+ assert lab_kit("run", "pi-error-scaling", "--wait", "--", "python3", "-c", WRITER)[0] == 0
81
+ run = records.runs(load(root), "pi-error-scaling")[-1]
82
+ assert (run.folder / "work" / "b.txt").is_file() and not (run.folder / "store.json").exists()
83
+ code, out = lab_kit("publish", "--pending")
84
+ assert code == 2 and "names no `store:`" in out
85
+
86
+
87
+ LINKER = ("import os, pathlib; w = pathlib.Path(os.environ['LAB_WORK']); "
88
+ "real = pathlib.Path(os.environ['LAB_RUN_DIR']).parent.parent / 'outside.bin'; real.write_text('bytes'); "
89
+ "(w / 'model.bin').symlink_to(real); (w / 'res_{seed}.json').write_text('{}')")
90
+
91
+
92
+ def test_a_linked_file_is_stored_as_its_bytes(make_lab, lab_kit, tmp_path) -> None:
93
+ store = tmp_path / "labs" / "mc"
94
+ store.mkdir(parents=True)
95
+ root = make_lab(_with_store(store))
96
+ assert lab_kit("run", "pi-error-scaling", "--wait", "--", "python3", "-c", LINKER)[0] == 0
97
+ run = records.runs(load(root), "pi-error-scaling")[-1]
98
+ copy = store / "experiments" / "pi-error-scaling" / "runs" / run.run_id / "work" / "model.bin"
99
+ assert not copy.is_symlink() and copy.read_text() == "bytes"
100
+ assert (copy.parent / "res_{seed}.json").is_file()
101
+
102
+
103
+ def test_an_empty_folder_where_the_store_should_be_is_not_the_store(make_lab, lab_kit, tmp_path) -> None:
104
+ root = make_lab(_with_store(tmp_path / "labs" / "mc"))
105
+ (tmp_path / "labs").mkdir() # a mount point whose mount failed: not a mount, no lab folder
106
+ assert lab_kit("run", "pi-error-scaling", "--wait", "--", "python3", "-c", WRITER)[0] == 0
107
+ run = records.runs(load(root), "pi-error-scaling")[-1]
108
+ assert (run.folder / "work" / "b.txt").is_file() and not (tmp_path / "labs" / "mc").exists()
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes
File without changes