lab-kit-cli 1.0.0__tar.gz → 1.0.1__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {lab_kit_cli-1.0.0 → lab_kit_cli-1.0.1}/CHANGELOG.md +22 -0
- {lab_kit_cli-1.0.0 → lab_kit_cli-1.0.1}/PKG-INFO +1 -1
- {lab_kit_cli-1.0.0 → lab_kit_cli-1.0.1}/docs/model.md +32 -5
- {lab_kit_cli-1.0.0 → lab_kit_cli-1.0.1}/pyproject.toml +1 -1
- {lab_kit_cli-1.0.0 → lab_kit_cli-1.0.1}/src/lab_kit/__init__.py +1 -1
- {lab_kit_cli-1.0.0 → lab_kit_cli-1.0.1}/src/lab_kit/checks.py +9 -1
- {lab_kit_cli-1.0.0 → lab_kit_cli-1.0.1}/src/lab_kit/history.py +83 -30
- {lab_kit_cli-1.0.0 → lab_kit_cli-1.0.1}/src/lab_kit/record.py +9 -5
- {lab_kit_cli-1.0.0 → lab_kit_cli-1.0.1}/src/lab_kit/runs.py +3 -0
- lab_kit_cli-1.0.1/tests/test_held.py +222 -0
- {lab_kit_cli-1.0.0 → lab_kit_cli-1.0.1}/tests/test_integrity.py +2 -1
- {lab_kit_cli-1.0.0 → lab_kit_cli-1.0.1}/.github/workflows/ci.yml +0 -0
- {lab_kit_cli-1.0.0 → lab_kit_cli-1.0.1}/.github/workflows/policy.yml +0 -0
- {lab_kit_cli-1.0.0 → lab_kit_cli-1.0.1}/.github/workflows/release.yml +0 -0
- {lab_kit_cli-1.0.0 → lab_kit_cli-1.0.1}/.gitignore +0 -0
- {lab_kit_cli-1.0.0 → lab_kit_cli-1.0.1}/AGENTS.md +0 -0
- {lab_kit_cli-1.0.0 → lab_kit_cli-1.0.1}/CLAUDE.md +0 -0
- {lab_kit_cli-1.0.0 → lab_kit_cli-1.0.1}/CONTRIBUTING.md +0 -0
- {lab_kit_cli-1.0.0 → lab_kit_cli-1.0.1}/LICENSE +0 -0
- {lab_kit_cli-1.0.0 → lab_kit_cli-1.0.1}/README.md +0 -0
- {lab_kit_cli-1.0.0 → lab_kit_cli-1.0.1}/agents/reviewer.md +0 -0
- {lab_kit_cli-1.0.0 → lab_kit_cli-1.0.1}/method/METHOD.md +0 -0
- {lab_kit_cli-1.0.0 → lab_kit_cli-1.0.1}/skills/lab-review/SKILL.md +0 -0
- {lab_kit_cli-1.0.0 → lab_kit_cli-1.0.1}/skills/lab-set-up/SKILL.md +0 -0
- {lab_kit_cli-1.0.0 → lab_kit_cli-1.0.1}/skills/lab-talk/SKILL.md +0 -0
- {lab_kit_cli-1.0.0 → lab_kit_cli-1.0.1}/skills/lab-work/SKILL.md +0 -0
- {lab_kit_cli-1.0.0 → lab_kit_cli-1.0.1}/src/lab_kit/cli.py +0 -0
- {lab_kit_cli-1.0.0 → lab_kit_cli-1.0.1}/src/lab_kit/site.py +0 -0
- {lab_kit_cli-1.0.0 → lab_kit_cli-1.0.1}/src/lab_kit/supervise.py +0 -0
- {lab_kit_cli-1.0.0 → lab_kit_cli-1.0.1}/src/lab_kit/views.py +0 -0
- {lab_kit_cli-1.0.0 → lab_kit_cli-1.0.1}/src/lab_kit/write.py +0 -0
- {lab_kit_cli-1.0.0 → lab_kit_cli-1.0.1}/tests/test_flow.py +0 -0
- {lab_kit_cli-1.0.0 → lab_kit_cli-1.0.1}/tests/test_refusals.py +0 -0
- {lab_kit_cli-1.0.0 → lab_kit_cli-1.0.1}/tests/test_site.py +0 -0
- {lab_kit_cli-1.0.0 → lab_kit_cli-1.0.1}/tools/import_run.py +0 -0
|
@@ -1,5 +1,27 @@
|
|
|
1
1
|
# Changelog
|
|
2
2
|
|
|
3
|
+
## 1.0.1
|
|
4
|
+
|
|
5
|
+
Three defects found by moving a real repository to 1.0.0, and one message.
|
|
6
|
+
|
|
7
|
+
- `lab check` reads the history of the lab's own record only: `log.md`, `resources.toml`,
|
|
8
|
+
`decisions/`, and each experiment's lock, runs, results and reviews under `goals/`. It used to
|
|
9
|
+
read every file with one of those names anywhere in the repository, so a repository that once held
|
|
10
|
+
an older layout, since removed, failed for ever with "was committed and is gone".
|
|
11
|
+
- A file of the record is known by the numbers in its path. A folder or a decision renamed after a
|
|
12
|
+
reworded title is the same record: the rename is no longer reported as a removal, and what was
|
|
13
|
+
committed under the old name is still held under the new one. A file moved to another number, or
|
|
14
|
+
out of the record's places, is reported as gone.
|
|
15
|
+
- The hash of `plan/` leaves out `__pycache__` folders and `*.pyc` files, as it always meant to.
|
|
16
|
+
Importing a script from `plan/` no longer makes `lab check` say the plan changed after the run.
|
|
17
|
+
The hash of a folder that holds neither is what it was, in both formats, so every existing lock,
|
|
18
|
+
run and review still verifies.
|
|
19
|
+
- A card's `see:` that names a page of another library is a note, not a problem, when that
|
|
20
|
+
library's folder is not on the machine, as in CI. With `--strict` it is a failure. When the folder
|
|
21
|
+
is there and the page is not, it is still a problem.
|
|
22
|
+
- `lab run` says that the resources changed when `uses:` is not what was locked. It used to say the
|
|
23
|
+
demand had changed.
|
|
24
|
+
|
|
3
25
|
## 1.0.0
|
|
4
26
|
|
|
5
27
|
A rewrite from first principles. Nothing of 0.6 carries over: a lab kept with 0.6 is not read by 1.0,
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.5
|
|
2
2
|
Name: lab-kit-cli
|
|
3
|
-
Version: 1.0.
|
|
3
|
+
Version: 1.0.1
|
|
4
4
|
Summary: A research lab's record, its method, and the command that keeps both.
|
|
5
5
|
Project-URL: Homepage, https://github.com/pierg/lab-kit
|
|
6
6
|
Project-URL: Issues, https://github.com/pierg/lab-kit/issues
|
|
@@ -117,7 +117,9 @@ Each arrow is a command that refuses when the step before it is missing. You see
|
|
|
117
117
|
one stays in the tree with its results, folded under the new one with that sentence, and off the
|
|
118
118
|
Now page. The lock also records the hash of `plan/` as it was locked, which is the plan the
|
|
119
119
|
reviewer read: the first run is held against it as every later run is held against the run before.
|
|
120
|
-
And it records the resources the experiment uses: `uses:` does not change after the lock.
|
|
120
|
+
And it records the resources the experiment uses: `uses:` does not change after the lock. `lab run`
|
|
121
|
+
refuses an experiment whose `uses:` is not what was locked, and says that it is the resources that
|
|
122
|
+
changed; `lab check` reports it.
|
|
121
123
|
- **Run.** One try at producing what the experiment demands. An experiment is run until one run
|
|
122
124
|
finishes. Each run pins the apparatus it ran, the configuration and the code in `plan/`, and is
|
|
123
125
|
detached, so it outlives the session; its output is sealed and kept outside git. A run that fails
|
|
@@ -147,6 +149,21 @@ Each arrow is a command that refuses when the step before it is missing. You see
|
|
|
147
149
|
`lab now` and `lab check` say which locks and lists of runs are not committed yet. With no commit
|
|
148
150
|
at all nothing is held, and `lab check` says that too; `--strict` turns each of these into a
|
|
149
151
|
failure.
|
|
152
|
+
**Where the record is.** The check reads the history of the lab's own record and of nothing else:
|
|
153
|
+
`log.md` and `resources.toml` at the lab's root, `decisions/D<n>-<name>.md`, and each experiment's
|
|
154
|
+
`lock.json`, `runs.tsv`, `results.tsv` and `reviews.tsv` in its folder,
|
|
155
|
+
`goals/G<n>-<name>/Q<n>-<name>/<n>-<name>/`. The lab's root may be a folder inside a larger
|
|
156
|
+
repository. What that repository holds or once held anywhere else, such as the folders of a lab
|
|
157
|
+
kept with an earlier kit, or a file with one of these names inside `plan/`, is not the lab's
|
|
158
|
+
record: its history is not read, and removing it is not a problem.
|
|
159
|
+
**A rename.** A file of the record is known by the numbers in its path and not by the words:
|
|
160
|
+
`goals/G1-<any name>/Q2-<any name>/3-<any name>/lock.json` is the lock of `G1.Q2.3`, and
|
|
161
|
+
`decisions/D4-<any name>.md` is `D4`. When a title is reworded, the folder or the file named after
|
|
162
|
+
it may be renamed to match. That is a rename and nothing else: what was committed under the old
|
|
163
|
+
name is held under the new one, so the lock is still the lock first committed, every committed
|
|
164
|
+
row is still there, and a settled decision still says what it said. A number never changes. A file
|
|
165
|
+
that is under another number, or outside these places, is not that file: the one that was
|
|
166
|
+
committed is gone, and `lab check` says so, as it does for a file deleted outright.
|
|
150
167
|
Two things keep this honest. The first run that finishes is the only source of results, and
|
|
151
168
|
nothing more runs under that experiment, so nobody chooses which numbers count; a different try
|
|
152
169
|
after that is a new experiment, because the numbers have been seen. And the apparatus of the run
|
|
@@ -300,9 +317,10 @@ file that is not text) is a problem it reports; no command stops on it.
|
|
|
300
317
|
|
|
301
318
|
`lab check` runs the command of every row in every `results.tsv`, as a shell command, each for at
|
|
302
319
|
most `rederive_timeout` seconds (`lab.toml`, 120 unless set). Run it only on a lab you trust, as
|
|
303
|
-
you would its code; `--no-rederive` runs none of them. Where the lab has no commit,
|
|
304
|
-
is not mounted,
|
|
305
|
-
that is a failure, which is
|
|
320
|
+
you would its code; `--no-rederive` runs none of them. Where the lab has no commit, the store
|
|
321
|
+
is not mounted, or another library that a card's `see:` points into is not on the machine, the check
|
|
322
|
+
says in a note what it could not check and still passes; with `--strict` that is a failure, which is
|
|
323
|
+
what CI wants.
|
|
306
324
|
|
|
307
325
|
Every command that writes validates everything first, so that a refusal leaves nothing half
|
|
308
326
|
written, and only one of them writes at a time.
|
|
@@ -426,6 +444,12 @@ for background the lab rests on and did not write. A card's prose may link the s
|
|
|
426
444
|
`[the words](target)`: a name the reader may not know, such as a paper or a method, is linked where
|
|
427
445
|
it is first used.
|
|
428
446
|
|
|
447
|
+
`lab check` asks that every page under `see:` is there. A page of another library is looked for in
|
|
448
|
+
that library's folder, where the charter says it is. When the folder itself is not on the machine,
|
|
449
|
+
as in CI, which checks out the lab alone, the page cannot be looked for: that is a note, and a
|
|
450
|
+
failure only with `--strict`. When the folder is there and the page is not, or the charter names no
|
|
451
|
+
such library, it is a problem.
|
|
452
|
+
|
|
429
453
|
**An arm** is a list item under Arms: its name, a colon, and what it is, so that a reader who has
|
|
430
454
|
never seen it knows what was run.
|
|
431
455
|
|
|
@@ -489,7 +513,10 @@ existed. A lock that names its kit and lacks one of these is not a lock the kit
|
|
|
489
513
|
**How a folder is hashed.** `plan/` is hashed file by file: the length of its path, the path, the
|
|
490
514
|
length of its content, the content. A lock or a run made this way says `"hash": 2`. One with no such
|
|
491
515
|
marker was made by an earlier kit, which wrote the path, a newline and the content; it is verified
|
|
492
|
-
the way it was made, and never hashed again.
|
|
516
|
+
the way it was made, and never hashed again. Either way the hash leaves out what Python writes when
|
|
517
|
+
it imports a script, which is no part of the plan: a folder named `__pycache__` with everything in
|
|
518
|
+
it, and any file whose name ends in `.pyc`, wherever it sits. Nothing else is left out; a hidden
|
|
519
|
+
file is hashed like any other.
|
|
493
520
|
|
|
494
521
|
**`runs.tsv`**, beside the lock and in git, is written by `lab run` and by the run as it ends: `run`
|
|
495
522
|
(its id), `kind` (run, trial), `what` (started, ended, lost), `at`, on the ended row `exit` and `seal`,
|
|
@@ -5,7 +5,7 @@ build-backend = "hatchling.build"
|
|
|
5
5
|
[project]
|
|
6
6
|
dynamic = ["readme"]
|
|
7
7
|
name = "lab-kit-cli"
|
|
8
|
-
version = "1.0.
|
|
8
|
+
version = "1.0.1"
|
|
9
9
|
description = "A research lab's record, its method, and the command that keeps both."
|
|
10
10
|
license = "MIT"
|
|
11
11
|
authors = [{ name = "Piergiuseppe Mallozzi" }]
|
|
@@ -1,2 +1,2 @@
|
|
|
1
1
|
"""lab-kit: a research lab's record, its method, and the command that keeps both."""
|
|
2
|
-
__version__ = "1.0.
|
|
2
|
+
__version__ = "1.0.1"
|
|
@@ -26,6 +26,7 @@ def run(lab, rederive=True):
|
|
|
26
26
|
notes.append("the store is not here: runs and re-derivation were not checked")
|
|
27
27
|
unchecked = True
|
|
28
28
|
|
|
29
|
+
absent = {} # the libraries the charter names whose folder is not on this machine
|
|
29
30
|
# cards: each has its slots, stays short, and writes only what is its to write
|
|
30
31
|
for n in lab.goals + lab.questions + lab.items + lab.decisions:
|
|
31
32
|
try:
|
|
@@ -54,10 +55,17 @@ def run(lab, rederive=True):
|
|
|
54
55
|
from . import site
|
|
55
56
|
name, _, path = str(page).partition(":")
|
|
56
57
|
lib = site.libraries(lab).get(name)
|
|
57
|
-
if not lib
|
|
58
|
+
if not lib:
|
|
58
59
|
bad("links", n["path"], f"see names {page}, which is not a page of a library the charter names")
|
|
60
|
+
elif not os.path.isdir(lib["path"]): # the other library is not on this machine, as in CI: nothing to look in
|
|
61
|
+
absent.setdefault(name, lib["path"])
|
|
62
|
+
elif not os.path.exists(os.path.join(lib["path"], path.lstrip("/"))):
|
|
63
|
+
bad("links", n["path"], f"see names {page}, and the library {name} holds no such page")
|
|
59
64
|
elif not os.path.exists(os.path.join(library, str(page).split("#")[0], "index.html")):
|
|
60
65
|
bad("links", n["path"], f"see names the page {page}, which is not in the library")
|
|
66
|
+
for name, path in sorted(absent.items()):
|
|
67
|
+
notes.append(f"the library {name} is not here ({path}): the pages cards name in it under `see:` could not be checked here")
|
|
68
|
+
unchecked = True
|
|
61
69
|
for d in {os.path.dirname(x["path"]) for x in lab.items}:
|
|
62
70
|
if sum(os.path.exists(os.path.join(root, d, f"{kind}.md")) for kind in R.STEPS) > 1:
|
|
63
71
|
bad("tree", d, "holds more than one of an experiment, a build and a reading; a folder is one of them")
|
|
@@ -3,21 +3,29 @@
|
|
|
3
3
|
What is held is held from the commit that first holds it: the record is tamper-evident through git, not tamper-proof.
|
|
4
4
|
Before that commit nothing is held, and `lab check` says so.
|
|
5
5
|
|
|
6
|
+
Only the lab's own record is read, at its places under the lab's root: `log.md`, `resources.toml`,
|
|
7
|
+
`decisions/D<n>-<name>.md`, and each experiment's `lock.json`, `runs.tsv`, `results.tsv` and `reviews.tsv` under
|
|
8
|
+
`goals/G<n>-<name>/Q<n>-<name>/<n>-<name>/`. What the repository holds or once held anywhere else is not the lab's.
|
|
9
|
+
A file is known by the numbers in its path and not by the words, so a folder renamed after a reworded title is the
|
|
10
|
+
same place: what was committed under the old name is held under the new one.
|
|
11
|
+
|
|
6
12
|
- A committed `lock.json` is byte for byte its first committed version.
|
|
7
13
|
- `log.md`, `reviews.tsv` and `runs.tsv` only grow: no committed line is removed or changed.
|
|
8
14
|
- A `results.tsv` that a review has passed keeps every committed row.
|
|
9
15
|
- A settled decision keeps its `decided:`, its `about:` and its Decided text as first committed.
|
|
10
16
|
- A resource, once committed, is never removed. A metered resource's `granted`, `per_run` and `unit` change only together
|
|
11
17
|
with `from`, which then names another settled decision that lists the resource. Its `other_use` rows only grow.
|
|
12
|
-
- None of these files is deleted.
|
|
18
|
+
- None of these files is deleted, and none changes its number.
|
|
13
19
|
"""
|
|
14
20
|
from __future__ import annotations
|
|
15
21
|
|
|
16
|
-
import os, re, subprocess
|
|
22
|
+
import glob, os, re, subprocess
|
|
17
23
|
|
|
18
24
|
from . import record as R
|
|
19
25
|
|
|
20
26
|
GROW = ("log.md", "reviews.tsv", "results.tsv", "runs.tsv")
|
|
27
|
+
STEP = re.compile(r"goals/G(\d+)-[^/]+/Q(\d+)-[^/]+/(\d+)-[^/]+/(lock\.json|reviews\.tsv|results\.tsv|runs\.tsv)")
|
|
28
|
+
DECISION = re.compile(r"decisions/D(\d+)-[^/]+\.md")
|
|
21
29
|
|
|
22
30
|
|
|
23
31
|
def git(root, *args):
|
|
@@ -39,6 +47,45 @@ def subsequence(old, new):
|
|
|
39
47
|
return None
|
|
40
48
|
|
|
41
49
|
|
|
50
|
+
def place(path):
|
|
51
|
+
"""Where a path, from the lab's root, sits in the lab's record: `log.md`, `D4`, `G1.Q2.3/lock.json`. None when it is
|
|
52
|
+
not a file of the record. The numbers place a file and the words do not, so a renamed folder is the same place."""
|
|
53
|
+
if path == "log.md":
|
|
54
|
+
return path
|
|
55
|
+
m = STEP.fullmatch(path)
|
|
56
|
+
if m:
|
|
57
|
+
return f"G{int(m.group(1))}.Q{int(m.group(2))}.{int(m.group(3))}/{m.group(4)}"
|
|
58
|
+
m = DECISION.fullmatch(path)
|
|
59
|
+
return f"D{int(m.group(1))}" if m else None
|
|
60
|
+
|
|
61
|
+
|
|
62
|
+
def held(root, *where):
|
|
63
|
+
"""Every committed version of the lab's record under these paths, by place: {place: [(commit, path)]}, oldest
|
|
64
|
+
first. A rename is read as what it is in git, the old path leaving and the new one arriving, so both names of
|
|
65
|
+
one place are in its list whether or not git would have paired them."""
|
|
66
|
+
out = git(root, "-c", "core.quotepath=off", "log", "--topo-order", "-m", "--no-renames", "--format=commit %H", "--name-status", "--relative", "--", *where) or ""
|
|
67
|
+
found, commit = {}, None
|
|
68
|
+
for line in out.split("\n"):
|
|
69
|
+
if line.startswith("commit "):
|
|
70
|
+
commit = line[7:].strip()
|
|
71
|
+
elif "\t" in line and not line.startswith("D"): # a path that left in this commit has no version in it
|
|
72
|
+
path = line.split("\t", 1)[1]
|
|
73
|
+
if place(path) and (commit, path) not in found.setdefault(place(path), []):
|
|
74
|
+
found[place(path)].append((commit, path))
|
|
75
|
+
return {k: v[::-1] for k, v in found.items()}
|
|
76
|
+
|
|
77
|
+
|
|
78
|
+
def present(root):
|
|
79
|
+
"""The files of the record that are in the lab now, by place: {place: [path]}."""
|
|
80
|
+
found = {}
|
|
81
|
+
for pattern in ("log.md", "decisions/*", "goals/*/*/*/*"):
|
|
82
|
+
for full in sorted(glob.glob(os.path.join(glob.escape(root), pattern))):
|
|
83
|
+
path = os.path.relpath(full, root).replace(os.sep, "/")
|
|
84
|
+
if place(path) and os.path.isfile(full):
|
|
85
|
+
found.setdefault(place(path), []).append(path)
|
|
86
|
+
return found
|
|
87
|
+
|
|
88
|
+
|
|
42
89
|
def tables(text):
|
|
43
90
|
try:
|
|
44
91
|
return {k: v for k, v in R.tomllib.loads(text or "").items() if isinstance(v, dict)}
|
|
@@ -74,14 +121,14 @@ def resources(lab):
|
|
|
74
121
|
return sorted(set(problems))
|
|
75
122
|
|
|
76
123
|
|
|
77
|
-
def decisions(lab,
|
|
124
|
+
def decisions(lab, versions=None, now=None):
|
|
78
125
|
"""The settled decisions that are no longer as they were first committed settled."""
|
|
79
126
|
root, problems = lab.root, []
|
|
80
127
|
if not committed(root):
|
|
81
128
|
return problems
|
|
82
129
|
bad = lambda where, what: problems.append(f"history: {where}: {what}")
|
|
83
|
-
if
|
|
84
|
-
|
|
130
|
+
versions = held(root, "decisions") if versions is None else versions
|
|
131
|
+
now = present(root) if now is None else now
|
|
85
132
|
|
|
86
133
|
def settled(text):
|
|
87
134
|
"""What a decision's card says was decided, and about what, or None while it says nothing."""
|
|
@@ -94,16 +141,18 @@ def decisions(lab, added=None):
|
|
|
94
141
|
about = fm.get("about")
|
|
95
142
|
return str(fm.get("decided")), sorted(str(i) for i in (about if isinstance(about, list) else [about] if about is not None else [])), R.flat(slots.get("Decided", ""))
|
|
96
143
|
|
|
97
|
-
for
|
|
98
|
-
|
|
99
|
-
|
|
100
|
-
|
|
144
|
+
for where, commits in sorted(versions.items()):
|
|
145
|
+
if not DECISION.fullmatch(commits[-1][1]):
|
|
146
|
+
continue
|
|
147
|
+
if where not in now:
|
|
148
|
+
bad(commits[-1][1], "was committed and is gone; nothing recorded is removed")
|
|
101
149
|
continue
|
|
102
|
-
for c in
|
|
150
|
+
for c, path in commits: # oldest first, under whichever name it had then
|
|
103
151
|
was = settled(git(root, "show", f"{c}:./{path}") or "")
|
|
104
152
|
if was:
|
|
105
|
-
if
|
|
106
|
-
|
|
153
|
+
mine = path if path in now[where] else now[where][0]
|
|
154
|
+
if settled(R.read(os.path.join(root, mine))) != was:
|
|
155
|
+
bad(mine, f"was settled in {c[:8]}" + ("" if mine == path else f", as {path}") + ", and its `decided:`, its `about:` or its Decided text has changed since; a change of mind is a new decision")
|
|
107
156
|
break
|
|
108
157
|
return problems
|
|
109
158
|
|
|
@@ -130,26 +179,30 @@ def check(lab):
|
|
|
130
179
|
bad = lambda where, what: problems.append(f"history: {where}: {what}")
|
|
131
180
|
if not committed(root):
|
|
132
181
|
return [], ["this lab is not in git, or has no commit: permanence against history was not checked, and nothing recorded is held yet"]
|
|
133
|
-
|
|
182
|
+
versions, now = held(root, "log.md", "decisions", "goals"), present(root)
|
|
134
183
|
reviewed = {os.path.join(x["dir"], "results.tsv") for x in lab.experiments if x["reviewed"]}
|
|
135
|
-
for
|
|
136
|
-
name
|
|
137
|
-
|
|
138
|
-
|
|
139
|
-
|
|
184
|
+
for where, commits in sorted(versions.items()):
|
|
185
|
+
name = os.path.basename(commits[-1][1])
|
|
186
|
+
if name not in GROW + ("lock.json",):
|
|
187
|
+
continue # a decision, held below
|
|
188
|
+
if where not in now:
|
|
189
|
+
bad(commits[-1][1], "was committed and is gone; nothing recorded is removed")
|
|
140
190
|
continue
|
|
141
|
-
|
|
191
|
+
mine = lambda path: path if path in now[where] else now[where][0] # the same file, under the name it has now
|
|
192
|
+
named = lambda path: "" if mine(path) == path else f", as {path}"
|
|
142
193
|
if name == "lock.json":
|
|
143
|
-
|
|
144
|
-
if
|
|
145
|
-
bad(path, f"differs from the lock first committed in {
|
|
146
|
-
|
|
147
|
-
|
|
148
|
-
|
|
149
|
-
|
|
150
|
-
|
|
151
|
-
|
|
152
|
-
|
|
153
|
-
|
|
194
|
+
c, path = commits[0]
|
|
195
|
+
if git(root, "show", f"{c}:./{path}") != R.read(os.path.join(root, mine(path))):
|
|
196
|
+
bad(mine(path), f"differs from the lock first committed in {c[:8]}{named(path)}; a changed plan is a new experiment")
|
|
197
|
+
continue
|
|
198
|
+
if name == "results.tsv": # rows may change until a review passes them
|
|
199
|
+
commits = commits[-1:] if mine(commits[-1][1]) in reviewed else []
|
|
200
|
+
for c, path in reversed(commits):
|
|
201
|
+
lost = subsequence((git(root, "show", f"{c}:./{path}") or "").splitlines(), R.read(os.path.join(root, mine(path))).splitlines())
|
|
202
|
+
if lost is not None:
|
|
203
|
+
bad(mine(path), f"a line committed in {c[:8]}{named(path)} is gone or changed: {lost[:70]!r}")
|
|
204
|
+
break
|
|
205
|
+
|
|
206
|
+
problems += decisions(lab, versions, now)
|
|
154
207
|
problems += resources(lab)
|
|
155
208
|
return sorted(set(problems)), []
|
|
@@ -153,11 +153,15 @@ def text_hash(text):
|
|
|
153
153
|
|
|
154
154
|
def tree_hash(folder, how=HASH):
|
|
155
155
|
"""One hash over a folder: every file's path and bytes, each preceded by its length, so that no two folders give
|
|
156
|
-
the same hash. The first way (`how` 1: the path, a newline, the bytes) is kept to verify what was made with it.
|
|
157
|
-
|
|
158
|
-
|
|
159
|
-
|
|
160
|
-
|
|
156
|
+
the same hash. The first way (`how` 1: the path, a newline, the bytes) is kept to verify what was made with it.
|
|
157
|
+
What Python leaves when it imports a script is not part of the folder, in either way: a `__pycache__` folder with
|
|
158
|
+
everything under it, and a `*.pyc` file wherever it sits."""
|
|
159
|
+
h, walked = hashlib.sha256(), []
|
|
160
|
+
for root, dirs, files in os.walk(folder):
|
|
161
|
+
dirs[:] = [d for d in dirs if d != "__pycache__"] # as the walk goes: sorting the walk first would read them all
|
|
162
|
+
walked.append((root, files))
|
|
163
|
+
for root, files in sorted(walked):
|
|
164
|
+
for name in sorted(f for f in files if not f.endswith(".pyc")):
|
|
161
165
|
p = os.path.join(root, name)
|
|
162
166
|
path, data = os.path.relpath(p, folder).encode(), open(p, "rb").read()
|
|
163
167
|
h.update(path + b"\n" + data if how == 1 else len(path).to_bytes(8, "big") + path + len(data).to_bytes(8, "big") + data)
|
|
@@ -126,6 +126,9 @@ def _start(lab, ident, command, trial, changed):
|
|
|
126
126
|
if not x["lock"]:
|
|
127
127
|
stop(f"{ident} is not locked")
|
|
128
128
|
if not lab.lock_holds(x):
|
|
129
|
+
if x["lock"].get("card") == lab.before_hash(x): # the demand is as locked, so it is `uses:` that moved
|
|
130
|
+
stop(f"{ident}: the resources it uses changed after the lock: it was locked on {', '.join(sorted(map(str, x['lock']['uses']))) or 'none'}, and `uses:` now names "
|
|
131
|
+
f"{', '.join(sorted(map(str, x['uses']))) or 'none'}. That never changes: replace the experiment")
|
|
129
132
|
stop(f"{ident}: what it demands (its comparison, arms, cases, bars, outcomes or limits) changed after the lock. That never changes: replace the experiment")
|
|
130
133
|
if x["finished"]:
|
|
131
134
|
stop(f"{ident} has a finished run, {x['finished']['id']}, and takes no more: it is the only source of its results. A repeat is a new experiment")
|
|
@@ -0,0 +1,222 @@
|
|
|
1
|
+
"""Where the lab's record is, what a rename is, what a plan's hash leaves out, and a page that cannot be looked for here."""
|
|
2
|
+
import glob, json, os, shutil, subprocess, sys
|
|
3
|
+
|
|
4
|
+
from test_flow import SRC, experiment, fill, front, git, lab, open_question
|
|
5
|
+
from test_integrity import WRITES, commit, done, under
|
|
6
|
+
|
|
7
|
+
# A fixed folder, hashed by the released 1.0.0 in both ways. A lock, a run or a review made by 1.0.0 holds a hash like
|
|
8
|
+
# these, so they must not move: the same folder gives the same hash for ever.
|
|
9
|
+
FIXED = {"a.py": b"print(1)\n", "b/c.txt": b"x\n", "a-b/d.cfg": b"k = 1\n", "b/e/f.json": b"{}\n", "Z.md": b"\xef\xbb\xbfcr\r\n", "b.py": b""}
|
|
10
|
+
FIRST_WAY = "11376786fde57ae78bf9a099b1a8f1e4085e40640489f90acafac5f48654119f"
|
|
11
|
+
AS_NOW = "9b4bebefd67a0699c57b4b5c2a431326b634f4d9a7a4d6f11a8087b34ee1d2fb"
|
|
12
|
+
|
|
13
|
+
|
|
14
|
+
def put(folder, files):
|
|
15
|
+
for path, data in files.items():
|
|
16
|
+
os.makedirs(os.path.dirname(f"{folder}/{path}"), exist_ok=True)
|
|
17
|
+
open(f"{folder}/{path}", "wb").write(data)
|
|
18
|
+
|
|
19
|
+
|
|
20
|
+
def finished(root):
|
|
21
|
+
"""A lab with one experiment locked, run, reviewed and committed."""
|
|
22
|
+
open_question(root)
|
|
23
|
+
ident, d, command = experiment(root, command=WRITES % "0.5")
|
|
24
|
+
lab(root, "lock", ident)
|
|
25
|
+
lab(root, "run", ident, "--wait", "--", *command)
|
|
26
|
+
done(root, ident, d)
|
|
27
|
+
commit(root)
|
|
28
|
+
assert "the record is sound" in lab(root, "check", "--strict")
|
|
29
|
+
return ident, d
|
|
30
|
+
|
|
31
|
+
|
|
32
|
+
def test_the_history_that_is_checked_is_the_labs_own_record(tmp_path):
|
|
33
|
+
# a repository that held an older layout, since removed, with the lab in a folder of it
|
|
34
|
+
repo = under(tmp_path, "repo")
|
|
35
|
+
root = f"{repo}/lab"
|
|
36
|
+
old = {"experiments/26-11-09-static/lock.json": b"{}\n", "experiments/26-11-09-static/log.md": b"- an old line\n", "experiments/26-11-09-static/results.tsv": b"a\tb\n",
|
|
37
|
+
"reviews.tsv": b"x\n", "runs.tsv": b"y\n", "log.md": b"- the repository's own log\n", "resources.toml": b"[old]\nwhat = 'gone'\n", "decisions/D1-old.md": b"old\n",
|
|
38
|
+
"lab/experiments/old/lock.json": b"{}\n", "lab/notes/log.md": b"- a note\n", "lab/goals/old-layout/lock.json": b"{}\n", "lab/goals/a/b/c/d/runs.tsv": b"z\n"}
|
|
39
|
+
put(repo, old)
|
|
40
|
+
commit(repo, "an older layout")
|
|
41
|
+
for path in old:
|
|
42
|
+
os.remove(f"{repo}/{path}")
|
|
43
|
+
os.makedirs(root, exist_ok=True)
|
|
44
|
+
open_question(root)
|
|
45
|
+
ident, d, command = experiment(root, command=WRITES % "0.5")
|
|
46
|
+
put(d, {"plan/results.tsv": b"what the apparatus reads\n", "plan/log.md": b"- its own notes\n"}) # the plan's files are the plan's
|
|
47
|
+
commit(repo, "the lab")
|
|
48
|
+
os.remove(f"{d}/plan/results.tsv")
|
|
49
|
+
open(f"{d}/plan/log.md", "w").write("- rewritten\n")
|
|
50
|
+
out = lab(root, "check", "--strict")
|
|
51
|
+
assert "the record is sound" in out and "history:" not in out
|
|
52
|
+
lab(root, "lock", ident)
|
|
53
|
+
lab(root, "run", ident, "--wait", "--", *command)
|
|
54
|
+
done(root, ident, d)
|
|
55
|
+
commit(repo, "the run")
|
|
56
|
+
assert "the record is sound" in lab(root, "check", "--strict")
|
|
57
|
+
|
|
58
|
+
# and the lab's own record is still held, each file at its place
|
|
59
|
+
def broken(path, change, *said):
|
|
60
|
+
full = f"{root}/{path}"
|
|
61
|
+
kept = open(full).read()
|
|
62
|
+
change(full, kept)
|
|
63
|
+
out = lab(root, "check", "--no-rederive", ok=False)
|
|
64
|
+
assert all(f"history: {path}: {s}" in out for s in said), out
|
|
65
|
+
open(full, "w").write(kept)
|
|
66
|
+
|
|
67
|
+
x = os.path.relpath(d, root)
|
|
68
|
+
rewrite = lambda full, kept: open(full, "w").write(kept.replace("\n", "\n\n", 1))
|
|
69
|
+
drop = lambda full, kept: open(full, "w").write("".join(kept.splitlines(True)[:-1]))
|
|
70
|
+
broken(f"{x}/lock.json", lambda full, kept: os.remove(full), "was committed and is gone")
|
|
71
|
+
broken(f"{x}/lock.json", rewrite, "differs from the lock first committed in")
|
|
72
|
+
for name in ("runs.tsv", "reviews.tsv", "results.tsv"):
|
|
73
|
+
broken(f"{x}/{name}", lambda full, kept: os.remove(full), "was committed and is gone")
|
|
74
|
+
if name != "results.tsv": # its rows are held once a review has passed them, and a row dropped is a result gone
|
|
75
|
+
broken(f"{x}/{name}", drop, "a line committed in")
|
|
76
|
+
broken("log.md", lambda full, kept: os.remove(full), "was committed and is gone")
|
|
77
|
+
broken("log.md", drop, "a line committed in")
|
|
78
|
+
broken(os.path.relpath(glob.glob(f"{root}/decisions/D1-*.md")[0], root), lambda full, kept: os.remove(full), "was committed and is gone")
|
|
79
|
+
assert "the record is sound" in lab(root, "check", "--strict")
|
|
80
|
+
|
|
81
|
+
|
|
82
|
+
def test_a_rename_is_the_same_record_under_another_name(tmp_path):
|
|
83
|
+
def renamed(name):
|
|
84
|
+
"""A finished lab whose goal, question, experiment and first decision are then renamed, as after a reworded title."""
|
|
85
|
+
root = under(tmp_path, name)
|
|
86
|
+
ident, d = finished(root)
|
|
87
|
+
goal = glob.glob(f"{root}/goals/G1-*")[0]
|
|
88
|
+
os.rename(d, f"{os.path.dirname(d)}/1-two-sample-sizes")
|
|
89
|
+
os.rename(os.path.dirname(d), f"{goal}/Q1-the-square-root-law")
|
|
90
|
+
os.rename(goal, f"{root}/goals/G1-the-law")
|
|
91
|
+
os.rename(glob.glob(f"{root}/decisions/D1-*.md")[0], f"{root}/decisions/D1-where-the-lab-starts.md")
|
|
92
|
+
return root, ident, f"{root}/goals/G1-the-law/Q1-the-square-root-law/1-two-sample-sizes", os.path.relpath(d, root)
|
|
93
|
+
|
|
94
|
+
# the rename alone: nothing was removed and nothing was rewritten
|
|
95
|
+
root, ident, d, was = renamed("renamed")
|
|
96
|
+
out = lab(root, "check")
|
|
97
|
+
assert "the record is sound" in out and "history:" not in out
|
|
98
|
+
commit(root, "renamed after the titles")
|
|
99
|
+
assert "the record is sound" in lab(root, "check", "--strict") and "(done, confirmed)" in lab(root, "show", ident)
|
|
100
|
+
x = os.path.relpath(d, root)
|
|
101
|
+
|
|
102
|
+
# what was committed under the old name is held under the new one, after the rename is committed too
|
|
103
|
+
lock = open(f"{d}/lock.json").read()
|
|
104
|
+
open(f"{d}/lock.json", "w").write(lock.replace("\n", "\n\n", 1))
|
|
105
|
+
assert f"history: {x}/lock.json: differs from the lock first committed in" in lab(root, "check", "--no-rederive", ok=False)
|
|
106
|
+
open(f"{d}/lock.json", "w").write(lock)
|
|
107
|
+
assert "the record is sound" in lab(root, "check", "--strict")
|
|
108
|
+
|
|
109
|
+
# a rename is not a way to rewrite a lock, drop a row, remove a file or change what was decided
|
|
110
|
+
def under_cover(name, change, said):
|
|
111
|
+
root, ident, d, was = renamed(name)
|
|
112
|
+
where = change(root, d, was)
|
|
113
|
+
for committed in (False, True): # before the rename is committed, and once it is, in the same commit
|
|
114
|
+
out = lab(root, "check", "--no-rederive", ok=False)
|
|
115
|
+
assert any(line.startswith(f"history: {where}: ") and said in line for line in out.splitlines()), out
|
|
116
|
+
if not committed:
|
|
117
|
+
commit(root, "renamed, and more")
|
|
118
|
+
|
|
119
|
+
def relock(root, d, was):
|
|
120
|
+
lock = json.load(open(f"{d}/lock.json"))
|
|
121
|
+
open(f"{d}/lock.json", "w").write(json.dumps({**lock, "locked": "2026-01-01T00:00:00+00:00"}, indent=1) + "\n")
|
|
122
|
+
return os.path.relpath(d, root) + "/lock.json"
|
|
123
|
+
|
|
124
|
+
def unlist(root, d, was):
|
|
125
|
+
rows = open(f"{d}/reviews.tsv").read().splitlines(True)
|
|
126
|
+
open(f"{d}/reviews.tsv", "w").write("".join(rows[:-1]))
|
|
127
|
+
return os.path.relpath(d, root) + "/reviews.tsv"
|
|
128
|
+
|
|
129
|
+
def remove(root, d, was):
|
|
130
|
+
os.remove(f"{d}/runs.tsv")
|
|
131
|
+
return f"{was}/runs.tsv"
|
|
132
|
+
|
|
133
|
+
def renumber(root, d, was): # a number never changes: under another number it is another experiment's
|
|
134
|
+
os.rename(d, f"{os.path.dirname(d)}/2-two-sample-sizes")
|
|
135
|
+
return f"{was}/lock.json"
|
|
136
|
+
|
|
137
|
+
def out_of_place(root, d, was):
|
|
138
|
+
os.makedirs(f"{root}/attic")
|
|
139
|
+
os.rename(d, f"{root}/attic/1-two-sample-sizes")
|
|
140
|
+
return f"{was}/results.tsv"
|
|
141
|
+
|
|
142
|
+
def redecide(root, d, was):
|
|
143
|
+
card = f"{root}/decisions/D1-where-the-lab-starts.md"
|
|
144
|
+
open(card, "w").write(open(card).read().replace("It runs on the laptop.", "It runs on the GPU."))
|
|
145
|
+
return "decisions/D1-where-the-lab-starts.md"
|
|
146
|
+
|
|
147
|
+
under_cover("relocked", relock, "differs from the lock first committed in")
|
|
148
|
+
under_cover("unlisted", unlist, "is gone or changed")
|
|
149
|
+
under_cover("removed", remove, "was committed and is gone; nothing recorded is removed")
|
|
150
|
+
under_cover("renumbered", renumber, "was committed and is gone; nothing recorded is removed")
|
|
151
|
+
under_cover("out-of-place", out_of_place, "was committed and is gone; nothing recorded is removed")
|
|
152
|
+
under_cover("redecided", redecide, "its `decided:`, its `about:` or its Decided text has changed since")
|
|
153
|
+
|
|
154
|
+
|
|
155
|
+
def test_what_python_leaves_in_a_plan_is_not_the_plan(tmp_path):
|
|
156
|
+
sys.path.insert(0, SRC)
|
|
157
|
+
from lab_kit import record as R
|
|
158
|
+
|
|
159
|
+
# a folder with nothing of Python's in it hashes as the released 1.0.0 hashed it, in both ways
|
|
160
|
+
plan = under(tmp_path, "fixed")
|
|
161
|
+
put(plan, FIXED)
|
|
162
|
+
assert (R.tree_hash(plan, 1), R.tree_hash(plan, 2), R.tree_hash(plan)) == (FIRST_WAY, AS_NOW, AS_NOW)
|
|
163
|
+
|
|
164
|
+
# what an import leaves is left out, wherever it sits and whatever is under it
|
|
165
|
+
put(plan, {"__pycache__/a.cpython-312.pyc": b"\x00a", "b/__pycache__/c.cpython-310.pyc": b"\x00b", "b/e/__pycache__/kept/notes.txt": b"under it\n",
|
|
166
|
+
"__pycache__/__pycache__/x.py": b"1\n", "b/stray.pyc": b"\x00c", "a-b/.hidden.pyc": b""})
|
|
167
|
+
assert (R.tree_hash(plan, 1), R.tree_hash(plan, 2)) == (FIRST_WAY, AS_NOW)
|
|
168
|
+
|
|
169
|
+
# and nothing else is: any other file is the plan's, a hidden one too
|
|
170
|
+
for path in ("b/new.py", ".env", "b/__pycache__x/c.py", "b/pyc", "a.pyc.txt"):
|
|
171
|
+
put(plan, {path: b""})
|
|
172
|
+
assert R.tree_hash(plan, 1) != FIRST_WAY and R.tree_hash(plan, 2) != AS_NOW, path
|
|
173
|
+
os.remove(f"{plan}/{path}")
|
|
174
|
+
assert (R.tree_hash(plan, 1), R.tree_hash(plan, 2)) == (FIRST_WAY, AS_NOW)
|
|
175
|
+
|
|
176
|
+
# a script of plan/ imported after the lock and again after the run: the plan did not change, and a change to it still shows
|
|
177
|
+
root = under(tmp_path, "lab")
|
|
178
|
+
open_question(root)
|
|
179
|
+
ident, d, command = experiment(root, command=WRITES % "0.5")
|
|
180
|
+
lab(root, "lock", ident)
|
|
181
|
+
env = {k: v for k, v in os.environ.items() if k not in ("PYTHONDONTWRITEBYTECODE", "PYTHONPYCACHEPREFIX")}
|
|
182
|
+
subprocess.run([sys.executable, "-m", "compileall", "-q", f"{d}/plan"], check=True, env=env)
|
|
183
|
+
assert glob.glob(f"{d}/plan/__pycache__/sample.*.pyc")
|
|
184
|
+
assert "finished" in lab(root, "run", ident, "--wait", "--", *command) # no `--changed` is asked for: nothing changed
|
|
185
|
+
done(root, ident, d)
|
|
186
|
+
shutil.copy(glob.glob(f"{d}/plan/__pycache__/sample.*.pyc")[0], f"{d}/plan/sample.pyc")
|
|
187
|
+
put(d, {"plan/__pycache__/more.cpython-312.pyc": b"\x00"})
|
|
188
|
+
out = lab(root, "check")
|
|
189
|
+
assert "the record is sound" in out and "plan/ changed" not in out
|
|
190
|
+
open(f"{d}/plan/sample.py", "a").write("# after the fact\n")
|
|
191
|
+
assert "plan/ changed after" in lab(root, "check", "--no-rederive", ok=False)
|
|
192
|
+
|
|
193
|
+
|
|
194
|
+
def test_a_page_of_another_library_is_looked_for_where_that_library_is(tmp_path):
|
|
195
|
+
root = under(tmp_path, "lab")
|
|
196
|
+
qdir = open_question(root)
|
|
197
|
+
os.makedirs(f"{root}/docs")
|
|
198
|
+
open(f"{root}/docs/folio.yaml", "w").write("title: lab\nlibraries:\n notes:\n path: ../../notes\n url: https://notes.example\n")
|
|
199
|
+
front(f"{qdir}/question.md", see="[notes:/content/readings/the-law.html]")
|
|
200
|
+
commit(root)
|
|
201
|
+
|
|
202
|
+
# the other library is not on this machine: the page could not be looked for, which is a note
|
|
203
|
+
out = lab(root, "check")
|
|
204
|
+
assert "note: the library notes is not here" in out and "could not be checked here" in out and "the record is sound" in out and "links:" not in out
|
|
205
|
+
out = lab(root, "check", "--strict", ok=False)
|
|
206
|
+
assert "not everything could be checked here" in out and "problem(s)" not in out
|
|
207
|
+
|
|
208
|
+
# it is here, and the page is not: a problem
|
|
209
|
+
os.makedirs(tmp_path / "notes" / "content")
|
|
210
|
+
out = lab(root, "check", ok=False)
|
|
211
|
+
assert "see names notes:/content/readings/the-law.html, and the library notes holds no such page" in out and "is not here" not in out
|
|
212
|
+
assert "holds no such page" in lab(root, "check", "--strict", ok=False)
|
|
213
|
+
|
|
214
|
+
# it is here with the page
|
|
215
|
+
os.makedirs(tmp_path / "notes" / "content" / "readings")
|
|
216
|
+
open(tmp_path / "notes" / "content" / "readings" / "the-law.html", "w").write("<p>the law</p>\n")
|
|
217
|
+
out = lab(root, "check", "--strict")
|
|
218
|
+
assert "the record is sound" in out and "note:" not in out
|
|
219
|
+
|
|
220
|
+
# a library the charter does not name is a problem wherever the check runs
|
|
221
|
+
front(f"{qdir}/question.md", see="[elsewhere:/content/readings/the-law.html]")
|
|
222
|
+
assert "which is not a page of a library the charter names" in lab(root, "check", ok=False)
|
|
@@ -483,7 +483,8 @@ def test_there_is_no_way_around_a_decision_to_a_larger_amount(tmp_path):
|
|
|
483
483
|
assert json.load(open(f"{d}/lock.json"))["uses"] == ["gpu"]
|
|
484
484
|
open(f"{root}/resources.toml", "w").write(spent + second)
|
|
485
485
|
front(f"{d}/experiment.md", uses="[gpu-b]")
|
|
486
|
-
|
|
486
|
+
refused = lab(root, "run", ident, "--wait", "--", "env", "RES=gpu-b", *command, ok=False) # the lock holds the resources it uses, and the refusal says so
|
|
487
|
+
assert "the resources it uses changed after the lock: it was locked on gpu, and `uses:` now names gpu-b" in refused and "what it demands" not in refused
|
|
487
488
|
out = lab(root, "check", "--no-rederive", ok=False)
|
|
488
489
|
assert "gpu-b: its amounts come out of a decision" in out and "or the resources it uses changed after the lock" in out
|
|
489
490
|
front(f"{d}/experiment.md", uses="[gpu]")
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|