task-pipeline-skill 1.97.1 → 1.99.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +283 -0
- package/HOW-IT-WORKS.md +17 -13
- package/README.md +13 -10
- package/SKILL-CARD.md +1 -1
- package/cursor/rules/task-pipeline.mdc +22 -14
- package/evals/task-pipeline.evals.json +3 -3
- package/package.json +1 -1
- package/plugins/task-pipeline/.claude-plugin/plugin.json +2 -2
- package/plugins/task-pipeline/commands/task-pipeline.md +14 -219
- package/plugins/task-pipeline/skills/evidence-docs/SKILL.md +8 -1
- package/plugins/task-pipeline/skills/evidence-docs/references/GENERATED.md +1 -1
- package/plugins/task-pipeline/skills/evidence-docs/references/learned.md +24 -1
- package/plugins/task-pipeline/skills/evidence-docs/references/retrospective.md +21 -576
- package/plugins/task-pipeline/skills/project-audit/SKILL.md +8 -1
- package/plugins/task-pipeline/skills/task-pipeline/SKILL.md +162 -230
- package/plugins/task-pipeline/skills/task-pipeline/graph.schema.json +1 -1
- package/plugins/task-pipeline/skills/task-pipeline/pipeline.example.json +9 -9
- package/plugins/task-pipeline/skills/task-pipeline/pipeline.schema.json +11 -4
- package/plugins/task-pipeline/skills/task-pipeline/references/acceptance.md +2 -2
- package/plugins/task-pipeline/skills/task-pipeline/references/brainstorm.md +3 -3
- package/plugins/task-pipeline/skills/task-pipeline/references/build.md +22 -7
- package/plugins/task-pipeline/skills/task-pipeline/references/certification.md +49 -44
- package/plugins/task-pipeline/skills/task-pipeline/references/continuity.md +6 -3
- package/plugins/task-pipeline/skills/task-pipeline/references/doctrine-map.md +1 -1
- package/plugins/task-pipeline/skills/task-pipeline/references/finish-line.md +13 -8
- package/plugins/task-pipeline/skills/task-pipeline/references/iteration.md +188 -32
- package/plugins/task-pipeline/skills/task-pipeline/references/knowledge-sources.md +5 -3
- package/plugins/task-pipeline/skills/task-pipeline/references/learned.md +24 -1
- package/plugins/task-pipeline/skills/task-pipeline/references/loop-guard.md +17 -7
- package/plugins/task-pipeline/skills/task-pipeline/references/retrospective.md +49 -2
- package/plugins/task-pipeline/skills/task-pipeline/references/spec.md +5 -4
- package/plugins/task-pipeline/skills/task-pipeline/references/stage-00-intake.md +39 -25
- package/plugins/task-pipeline/skills/task-pipeline/references/stage-05-dev.md +1 -1
- package/plugins/task-pipeline/skills/task-pipeline/references/stages.md +36 -28
- package/plugins/task-pipeline/skills/task-pipeline/references/waves.md +8 -5
- package/plugins/task-pipeline/skills/task-pipeline/scripts/graph.py +79 -22
- package/plugins/task-pipeline/skills/task-pipeline/scripts/lessons.py +1229 -110
- package/plugins/task-pipeline/skills/task-pipeline/scripts/retro_budget.py +305 -0
- package/plugins/task-pipeline/skills/task-pipeline/scripts/short_path.py +70 -0
- package/plugins/task-pipeline/skills/task-pipeline/scripts/waves.py +22 -10
- package/plugins/task-pipeline/skills/task-pipeline/templates/canvas.md +1 -1
- package/plugins/task-pipeline/skills/task-pipeline/templates/lesson.schema.json +38 -10
- package/plugins/task-pipeline/skills/task-pipeline/templates/retro-archive.md +9 -0
- package/plugins/task-pipeline/skills/task-pipeline/templates/retro.md +10 -1
|
@@ -9,8 +9,10 @@ lesson/1), and the index projects every such entry from every project on this ma
|
|
|
9
9
|
one file that stage 0 queries by the task's nouns.
|
|
10
10
|
|
|
11
11
|
**The retro stays canonical; the index is a projection.** It can be lost and rebuilt from
|
|
12
|
-
the retros (`rebuild`)
|
|
13
|
-
|
|
12
|
+
the retros (`rebuild`). An entry is a pure function of its origins, so a complete rebuild of
|
|
13
|
+
the same retros from the same checkouts, with nothing carried and no memory id recorded,
|
|
14
|
+
equals the incremental index byte for byte. Otherwise the rebuild also holds what only the
|
|
15
|
+
previous index knew: carried origins, their provenance and memory ids.
|
|
14
16
|
|
|
15
17
|
lessons.py add --retro FILE (--entry ANCHOR ... | --all) [--root DIR]
|
|
16
18
|
lessons.py add --json FILE|- [--root DIR]
|
|
@@ -18,9 +20,26 @@ entry is a pure function of its origins.
|
|
|
18
20
|
classKey makes it a candidate, a repeat from the same project revises, never mints
|
|
19
21
|
lessons.py query NOUN ... [--limit 5]
|
|
20
22
|
at most --limit lines: `[lesson L-<sha12>] title — trigger — check (seen N, projects K)`
|
|
21
|
-
lessons.py rebuild --roots DIR ... [--depth 3]
|
|
22
|
-
re-derive the
|
|
23
|
-
|
|
23
|
+
lessons.py rebuild --roots DIR ... [--depth 3] [--prune-carried [--prune-carried-reason R]]
|
|
24
|
+
re-derive the index from every retro found under the roots and their git worktrees.
|
|
25
|
+
A rebuild never removes or moves anything on its own: every previous origin it does
|
|
26
|
+
not re-derive is carried forward, marked `carried: <reason>`, and named, keeping its
|
|
27
|
+
memory id — except an origin v1.95.0 wrote (no provenance) that this run proves is the
|
|
28
|
+
same repository under a new key (its commit resolves in that checkout, descends from the
|
|
29
|
+
new key's root, and v1.95.0 would have keyed that checkout as the origin; its project is
|
|
30
|
+
not known to the index): it is re-keyed, its memory id kept and its source key recorded
|
|
31
|
+
(`rekeyedFrom`). When that source project is read later (rebuild or add) or records
|
|
32
|
+
the id, it takes the id back, named as `reclaimed:`, so the final owner never depends
|
|
33
|
+
on which verb ran first. One memory id stays on one live project (the earlier holder),
|
|
34
|
+
in rebuild and add; the other is named. `--prune-carried` (optionally one reason) is
|
|
35
|
+
the only way an origin leaves the index; --prune-carried-reason needs --prune-carried
|
|
36
|
+
and takes only a reason a rebuild produces. An origin whose every checkout now holds another project, without
|
|
37
|
+
the origin's own commit, is kept but not counted. A shallow clone — or one whose
|
|
38
|
+
shallow check does not answer — is never a source. Memory ids follow (project,
|
|
39
|
+
classKey). Messages — stdout and stderr, in every verb but `path` — name checkouts and
|
|
40
|
+
files, never an absolute path. Exit 1, touching nothing, only when nothing could be
|
|
41
|
+
read: every root missing or unreadable; a readable empty root exits 0 and writes the
|
|
42
|
+
index
|
|
24
43
|
lessons.py candidates [--min-projects 2]
|
|
25
44
|
lessons two or more projects hold: the `retro.publish` queue for learned.md
|
|
26
45
|
lessons.py show L-<sha12>
|
|
@@ -30,9 +49,16 @@ entry is a pure function of its origins.
|
|
|
30
49
|
session lists it, else `observatory_record` (the default; memoryId and expectedRevision
|
|
31
50
|
once recorded). The same pseudonymous project key, never a `project:<slug>` id; the
|
|
32
51
|
commit as `commit:` + 12 hex. The agent calls the tool
|
|
33
|
-
lessons.py record L-<sha12> --answer FILE|- [--ledger PATH] [--root DIR]
|
|
52
|
+
lessons.py record L-<sha12> --answer FILE|- [--ledger PATH] [--root DIR] [--replace]
|
|
34
53
|
keep the store's memory id beside this project's origin; a refusal becomes one ledger
|
|
35
|
-
line (`event: lessons — store refused — <code>`) and the local entry stays
|
|
54
|
+
line (`event: lessons — store refused — <code>`) and the local entry stays. Replacing a
|
|
55
|
+
different id this origin holds needs --replace; an id another project's or class's
|
|
56
|
+
origin holds is refused (exit 1) — unless that origin is carried as `checkout holds
|
|
57
|
+
another project` from this very checkout (the re-attach after a re-init), or took the
|
|
58
|
+
id by a re-key from this project (then this project reclaims it) — and one the
|
|
59
|
+
index cannot hold (`mem:<id>@<rev>`, no
|
|
60
|
+
quote, backslash or space) exits 2. add, emit-observatory and record refuse in a
|
|
61
|
+
shallow clone: its project key is not stable until it is deepened
|
|
36
62
|
lessons.py path
|
|
37
63
|
the index this invocation would use
|
|
38
64
|
|
|
@@ -65,7 +91,8 @@ SHA or a repository name in text no pattern can tell from an ordinary word — t
|
|
|
65
91
|
writer's duty, with names. A person's or a customer's
|
|
66
92
|
name a pattern cannot see; that stays the writer's duty (`references/retrospective.md` →
|
|
67
93
|
*What may leave the project*). The project is a pseudonymous key — `p-` and 12 hex digits
|
|
68
|
-
of the repository's root commit
|
|
94
|
+
of the repository's root commit, the first-parent root of its default branch (else the branch
|
|
95
|
+
init.defaultBranch names, then the main worktree's HEAD), shared by every worktree of it; never its name, and the evidence commit must resolve in
|
|
69
96
|
the repository it came from.
|
|
70
97
|
|
|
71
98
|
**The retro line it reads**, inside a `### ` entry of the retro or its archive:
|
|
@@ -76,9 +103,14 @@ with the entry's own `**Surfaced at:** stage N · **Owned by:** stage M`, `**Fix
|
|
|
76
103
|
`**The check:**` and `**Commit:** `<sha>`` lines. An entry with no Lesson line is private to
|
|
77
104
|
its project and is skipped, counted aloud.
|
|
78
105
|
|
|
79
|
-
Exit codes: 0 done (an absent index on `query` is a state, printed
|
|
80
|
-
|
|
81
|
-
|
|
106
|
+
Exit codes: 0 done (an absent index on `query` is a state, printed; a `rebuild` that named
|
|
107
|
+
what it skipped or carried); 1 a refusal — a lesson that fails lesson/1 or whose commit does
|
|
108
|
+
not resolve on `add`, a held lock, an unknown id, a `rebuild` whose every root was missing or
|
|
109
|
+
unreadable, a `record` naming an id another class or another project holds, an older
|
|
110
|
+
revision or (without --replace) a different id, and add/emit/record in a shallow clone; 2 usage
|
|
111
|
+
(--prune-carried-reason without --prune-carried, or a reason a rebuild never produces), a
|
|
112
|
+
memory id the index cannot hold, or an unreadable input. Stdlib only
|
|
113
|
+
(`references/portability.md`).
|
|
82
114
|
"""
|
|
83
115
|
from __future__ import annotations
|
|
84
116
|
|
|
@@ -188,17 +220,30 @@ def set_aside(path: pathlib.Path) -> pathlib.Path:
|
|
|
188
220
|
def open_index(idx: pathlib.Path, notes: list[str]) -> dict[str, dict]:
|
|
189
221
|
"""The index's entries — the same corrupt handling for the family index and the standalone
|
|
190
222
|
one: a file that does not parse is a DERIVED file gone bad, never a reason to exit 1. It is
|
|
191
|
-
set aside as `.corrupt-<date>[-n]
|
|
192
|
-
|
|
193
|
-
|
|
223
|
+
set aside whole as `.corrupt-<date>[-n]`; every line that still parses (memory ids and
|
|
224
|
+
carried origins included) is salvaged into the live index at once, and the run goes on from
|
|
225
|
+
that; `rebuild --roots` re-derives the rest. A file that cannot be READ (permissions) is the
|
|
226
|
+
one refusal: its contents may be good, so nothing replaces it."""
|
|
194
227
|
try:
|
|
195
|
-
|
|
228
|
+
mine: list[str] = []
|
|
229
|
+
out = load(idx, mine)
|
|
230
|
+
notes += mine
|
|
231
|
+
return out
|
|
196
232
|
except IndexCorrupt:
|
|
233
|
+
good, future, bad, only = _salvage(idx, notes)
|
|
197
234
|
dest = set_aside(idx)
|
|
198
|
-
notes.append(f"the index {idx} could not be parsed — kept as {dest.name}; "
|
|
199
|
-
"
|
|
200
|
-
|
|
201
|
-
|
|
235
|
+
notes.append(f"the index {idx} could not be parsed — kept whole as {dest.name}; "
|
|
236
|
+
f"{len(good)} lesson line{'' if len(good) == 1 else 's'} salvaged into the "
|
|
237
|
+
f"live index, {bad} bad line{'' if bad == 1 else 's'} left only in {dest.name}; "
|
|
238
|
+
"`lessons.py rebuild --roots <dir>` re-derives the rest from the retros")
|
|
239
|
+
if only:
|
|
240
|
+
notes.append(f"memory id{'' if len(only) == 1 else 's'} held only by a bad line, kept "
|
|
241
|
+
f"in {dest.name} and nowhere else: {', '.join(sorted(only))}")
|
|
242
|
+
_CORRUPT_ONLY[str(idx)] = (dest.name, only)
|
|
243
|
+
_FUTURE[str(idx)] = future
|
|
244
|
+
if good or future:
|
|
245
|
+
save(idx, good) # under the session's lock: a read-only verb keeps the salvage too
|
|
246
|
+
return good
|
|
202
247
|
|
|
203
248
|
|
|
204
249
|
def _merge_legacy(idx: pathlib.Path, notes: list[str]) -> None:
|
|
@@ -223,6 +268,9 @@ def _merge_legacy(idx: pathlib.Path, notes: list[str]) -> None:
|
|
|
223
268
|
old_future = list(_FUTURE.get(str(legacy), []))
|
|
224
269
|
both = idx.exists()
|
|
225
270
|
new = open_index(idx, notes) if both else {}
|
|
271
|
+
# A family file that did not parse was set aside by open_index: only one file was read.
|
|
272
|
+
set_aside_family = both and not idx.exists()
|
|
273
|
+
both = both and not set_aside_family
|
|
226
274
|
copied = [k for k in sorted(old) if k not in new]
|
|
227
275
|
for k in copied:
|
|
228
276
|
new[k] = old[k]
|
|
@@ -236,9 +284,12 @@ def _merge_legacy(idx: pathlib.Path, notes: list[str]) -> None:
|
|
|
236
284
|
except OSError as exc:
|
|
237
285
|
raise Refused(f"merged the old index into {idx} but could not rename {legacy} "
|
|
238
286
|
f"({type(exc).__name__}) — remove it by hand so one index is read")
|
|
239
|
-
what = "both index files existed — read both: merged" if both else
|
|
240
|
-
|
|
241
|
-
|
|
287
|
+
what = ("both index files existed — read both: merged" if both else
|
|
288
|
+
"the family index was corrupt and set aside, so only the local one was read: moved"
|
|
289
|
+
if set_aside_family else "moved the local index from")
|
|
290
|
+
kept = "" if set_aside_family else ", the family copy kept wherever both had one"
|
|
291
|
+
notes.append(f"{what} {legacy} into {idx} — {len(copied)} of {len(old)} lessons copied"
|
|
292
|
+
f"{kept}; the old file is kept as {dest.name}")
|
|
242
293
|
|
|
243
294
|
|
|
244
295
|
class session:
|
|
@@ -279,7 +330,7 @@ class session:
|
|
|
279
330
|
lk.__exit__(None, None, None)
|
|
280
331
|
self.locks = []
|
|
281
332
|
for n in self.notes:
|
|
282
|
-
print(f"lessons: {n}", file=sys.stderr)
|
|
333
|
+
print(f"lessons: {_say(n)}", file=sys.stderr)
|
|
283
334
|
self.notes = []
|
|
284
335
|
|
|
285
336
|
def __exit__(self, *exc):
|
|
@@ -312,19 +363,114 @@ def repo_root(start: pathlib.Path) -> pathlib.Path:
|
|
|
312
363
|
if not top:
|
|
313
364
|
raise Refused(f"{start} is not inside a git repository — a lesson carries its evidence "
|
|
314
365
|
"commit, and a commit needs a repository")
|
|
366
|
+
_show(pathlib.Path(top))
|
|
315
367
|
return pathlib.Path(top)
|
|
316
368
|
|
|
317
369
|
|
|
370
|
+
_KEYS: dict[str, str] = {}
|
|
371
|
+
# Absolute paths a message shows by name (round 12, E; round 13, O): `rebuild` fills it, and
|
|
372
|
+
# every line it prints — stdout, the session's notes, a refusal — goes through `_say`.
|
|
373
|
+
_SHOWN: dict[str, str] = {}
|
|
374
|
+
|
|
375
|
+
|
|
376
|
+
def _show(path: pathlib.Path, home_relative: bool = False) -> None:
|
|
377
|
+
"""Show `path` — as given and as resolved — by its name in every message (round 14, L3). An
|
|
378
|
+
index file is shown home-relative (`~/.sshlg-skills/lessons/index.jsonl`) where it lies
|
|
379
|
+
under the home directory, so the family's and the standalone index stay told apart."""
|
|
380
|
+
try:
|
|
381
|
+
real = path.resolve()
|
|
382
|
+
except OSError:
|
|
383
|
+
real = path
|
|
384
|
+
name = real.name or path.name
|
|
385
|
+
if home_relative:
|
|
386
|
+
try:
|
|
387
|
+
name = "~/" + real.relative_to(pathlib.Path.home().resolve()).as_posix()
|
|
388
|
+
except (ValueError, RuntimeError, OSError):
|
|
389
|
+
pass
|
|
390
|
+
for form in {str(path), str(real), str(path.expanduser())}:
|
|
391
|
+
if os.path.isabs(form) and name:
|
|
392
|
+
_SHOWN[form] = name
|
|
393
|
+
|
|
394
|
+
|
|
318
395
|
def project_key(root: pathlib.Path) -> str:
|
|
319
|
-
"""`p-` + 12 hex of the repository's root commit: stable across clones, names nothing.
|
|
320
|
-
|
|
321
|
-
|
|
322
|
-
|
|
323
|
-
|
|
396
|
+
"""`p-` + 12 hex of the repository's root commit: stable across clones, names nothing.
|
|
397
|
+
|
|
398
|
+
The REPOSITORY's root, not the checked-out branch's (round 12): the first-parent root of the
|
|
399
|
+
default branch: `origin/HEAD`, else `main`, else `master`, else the branch
|
|
400
|
+
`init.defaultBranch` names (round 17, L1: it names the default, wherever the main worktree
|
|
401
|
+
stands); without any of them the MAIN worktree's HEAD, read in the common directory (round
|
|
402
|
+
13: one answer for every worktree, so a linked worktree on an orphan branch is not a second
|
|
403
|
+
project), else this checkout's HEAD. A worktree on a branch that merged an unrelated history, or a
|
|
404
|
+
checkout sitting on an orphan `gh-pages`, is still its repository's project. Computed once
|
|
405
|
+
per `git rev-parse --git-common-dir`, so every worktree of one repository shares it."""
|
|
406
|
+
common = _git(root, "rev-parse", "--path-format=absolute", "--git-common-dir")
|
|
407
|
+
key = common or str(root)
|
|
408
|
+
if key in _KEYS:
|
|
409
|
+
return _KEYS[key]
|
|
410
|
+
refs: list[tuple[tuple[str, ...], str]] = []
|
|
411
|
+
head = _git(root, "symbolic-ref", "-q", "refs/remotes/origin/HEAD")
|
|
412
|
+
if head:
|
|
413
|
+
refs.append(((), head))
|
|
414
|
+
refs += [((), "refs/heads/main"), ((), "refs/heads/master")]
|
|
415
|
+
# The configured default before the main worktree's HEAD (see the docstring): a main
|
|
416
|
+
# worktree checked out on an orphan `pages` branch must not move the repository's key.
|
|
417
|
+
default = _git(root, "config", "--get", "init.defaultBranch")
|
|
418
|
+
if default:
|
|
419
|
+
refs.append(((), "refs/heads/" + default))
|
|
420
|
+
if common:
|
|
421
|
+
refs.append((("--git-dir", common), "HEAD")) # the main worktree's HEAD
|
|
422
|
+
refs.append(((), "HEAD"))
|
|
423
|
+
for extra, ref in refs:
|
|
424
|
+
if _git(root, *extra, "rev-parse", "-q", "--verify", ref + "^{commit}") is None:
|
|
425
|
+
continue
|
|
426
|
+
roots = _git(root, *extra, "rev-list", "--max-parents=0", "--first-parent", ref)
|
|
427
|
+
if roots:
|
|
428
|
+
_KEYS[key] = "p-" + roots.split()[-1][:12]
|
|
429
|
+
return _KEYS[key]
|
|
430
|
+
raise Refused(f"the repository {root.name} has no commit — a project key needs one")
|
|
431
|
+
|
|
432
|
+
|
|
433
|
+
_RESOLVES: dict[tuple[str, str], bool] = {}
|
|
324
434
|
|
|
325
435
|
|
|
326
436
|
def commit_resolves(root: pathlib.Path, sha: str) -> bool:
|
|
327
|
-
|
|
437
|
+
"""Does `sha` name a commit in this checkout? Asked of git once per (checkout, sha) in a
|
|
438
|
+
run (round 14, L2): a rebuild asks it for every carried origin in every checkout of its
|
|
439
|
+
provenance, and 150 origins over 5 checkouts doubled the run. Only an answer is cached; a
|
|
440
|
+
git that does not answer raises, uncached, every time."""
|
|
441
|
+
key = (os.path.realpath(root), sha)
|
|
442
|
+
if key not in _RESOLVES:
|
|
443
|
+
_RESOLVES[key] = _git(root, "cat-file", "-e", f"{sha}^{{commit}}") is not None
|
|
444
|
+
return _RESOLVES[key]
|
|
445
|
+
|
|
446
|
+
|
|
447
|
+
_LEGACY_KEYS: dict[str, str | None] = {}
|
|
448
|
+
_ROOTS: dict[tuple[str, str], frozenset] = {}
|
|
449
|
+
|
|
450
|
+
|
|
451
|
+
def _legacy_key(checkout: str) -> str | None:
|
|
452
|
+
"""The key v1.95.0 gave this checkout: `p-` + the SMALLEST root commit of HEAD (round 14,
|
|
453
|
+
F1). None when git cannot say. Cached per checkout."""
|
|
454
|
+
if checkout not in _LEGACY_KEYS:
|
|
455
|
+
try:
|
|
456
|
+
roots = _git(pathlib.Path(checkout), "rev-list", "--max-parents=0", "HEAD")
|
|
457
|
+
except Refused:
|
|
458
|
+
roots = None
|
|
459
|
+
_LEGACY_KEYS[checkout] = "p-" + sorted(roots.split())[0][:12] if roots else None
|
|
460
|
+
return _LEGACY_KEYS[checkout]
|
|
461
|
+
|
|
462
|
+
|
|
463
|
+
def _roots_of(checkout: str, sha: str) -> frozenset:
|
|
464
|
+
"""The 12-hex prefixes of the root commits `sha` descends from in this checkout; empty when
|
|
465
|
+
git cannot say. Cached per (checkout, sha)."""
|
|
466
|
+
key = (checkout, sha)
|
|
467
|
+
if key not in _ROOTS:
|
|
468
|
+
try:
|
|
469
|
+
roots = _git(pathlib.Path(checkout), "rev-list", "--max-parents=0", sha)
|
|
470
|
+
except Refused:
|
|
471
|
+
roots = None
|
|
472
|
+
_ROOTS[key] = frozenset(r[:12] for r in (roots or "").split())
|
|
473
|
+
return _ROOTS[key]
|
|
328
474
|
|
|
329
475
|
|
|
330
476
|
def slug(text: str) -> str:
|
|
@@ -350,8 +496,11 @@ def _check_text(name: str, value: object, lo: int, hi: int) -> list[str]:
|
|
|
350
496
|
errs = []
|
|
351
497
|
if not lo <= len(value) <= hi:
|
|
352
498
|
errs.append(f"{name} is {len(value)} characters; lesson/1 allows {lo}-{hi}")
|
|
353
|
-
|
|
354
|
-
|
|
499
|
+
# Every character str.splitlines() breaks on, not only \n and \r (round 12, B): a U+2028 in a
|
|
500
|
+
# title once made every later read of the index call it corrupt.
|
|
501
|
+
if value != "".join(value.splitlines()) or any(ch in value for ch in "\r\n"):
|
|
502
|
+
errs.append(f"{name} carries a line break (\\n, \\r, U+2028, U+2029, U+0085 or "
|
|
503
|
+
"another separator) — one line each")
|
|
355
504
|
for rx, what in FORBIDDEN:
|
|
356
505
|
if rx.search(value):
|
|
357
506
|
errs.append(f"{name} carries {what}")
|
|
@@ -382,6 +531,12 @@ def validate(lesson: object) -> list[str]:
|
|
|
382
531
|
or not all(isinstance(v, (int, str)) and not isinstance(v, bool) and str(v).strip()
|
|
383
532
|
for v in st.values())):
|
|
384
533
|
errs.append("stages is not {owned, surfaced}")
|
|
534
|
+
else:
|
|
535
|
+
# A stage id that is a string gets the text fields' checks (round 13, P): one line, at
|
|
536
|
+
# most 32 characters, no path, URL or credential shape.
|
|
537
|
+
for k in ("owned", "surfaced"):
|
|
538
|
+
if isinstance(st[k], str):
|
|
539
|
+
errs += _check_text(f"stages.{k}", st[k], 1, 32)
|
|
385
540
|
nouns = lesson["nouns"]
|
|
386
541
|
if (not isinstance(nouns, list) or not 3 <= len(nouns) <= 8
|
|
387
542
|
or len(set(map(str, nouns))) != len(nouns)):
|
|
@@ -436,8 +591,10 @@ def _relative(root: pathlib.Path, path: pathlib.Path) -> str:
|
|
|
436
591
|
raise Refused(f"{path} is outside the repository at {root}")
|
|
437
592
|
|
|
438
593
|
|
|
439
|
-
def parse_retro(path: pathlib.Path, root: pathlib.Path, project: str
|
|
440
|
-
|
|
594
|
+
def parse_retro(path: pathlib.Path, root: pathlib.Path, project: str,
|
|
595
|
+
refused: set[str] | None = None) -> tuple[list[dict], int, list[str]]:
|
|
596
|
+
"""Lessons from one retro file: (lessons, entries without a Lesson line, refusals).
|
|
597
|
+
`refused`, when given, collects the `retro#anchor` of every Lesson entry refused here."""
|
|
441
598
|
try:
|
|
442
599
|
text = path.read_text(encoding="utf-8")
|
|
443
600
|
except (OSError, UnicodeError) as exc:
|
|
@@ -481,6 +638,8 @@ def parse_retro(path: pathlib.Path, root: pathlib.Path, project: str) -> tuple[l
|
|
|
481
638
|
if missing:
|
|
482
639
|
problems.append(f"{rel}#{anchor}: the Lesson line needs the entry's "
|
|
483
640
|
+ ", ".join(missing))
|
|
641
|
+
if refused is not None:
|
|
642
|
+
refused.add(f"{rel}#{anchor}")
|
|
484
643
|
continue
|
|
485
644
|
|
|
486
645
|
def stage(v: str) -> int | str:
|
|
@@ -503,37 +662,57 @@ def _origin_key(o: dict) -> tuple:
|
|
|
503
662
|
|
|
504
663
|
|
|
505
664
|
def derive(origins: list[dict]) -> dict:
|
|
506
|
-
"""One index entry as a pure function of its origins
|
|
665
|
+
"""One index entry as a pure function of its origins (carried ones and their memory ids
|
|
666
|
+
included) — so a complete rebuild with nothing carried equals the incremental index.
|
|
667
|
+
|
|
668
|
+
`seen`, `projects` and `status` count re-derived origins and origins carried for a
|
|
669
|
+
temporary unavailability; an origin carried as `checkout holds another project` is kept but
|
|
670
|
+
NOT counted (round 11): a stranger at a path, or one checkout under a new root commit, never
|
|
671
|
+
makes a phantom second project. Applied at load too, so these fields are never trusted from
|
|
672
|
+
disk (round 11, #6)."""
|
|
507
673
|
origins = sorted(origins, key=lambda o: (_origin_key(o), json.dumps(o, sort_keys=True)))
|
|
508
674
|
first = origins[0]["lesson"]
|
|
509
675
|
nouns = sorted({n for o in origins for n in o["lesson"]["nouns"]}, key=str.casefold)
|
|
510
|
-
|
|
676
|
+
counted = [o for o in origins if o.get("carried") != CARRY_FOREIGN]
|
|
677
|
+
projects = len({o["project"] for o in counted})
|
|
511
678
|
return {"schema": "lesson-index/1", "id": lesson_id(first["classKey"]),
|
|
512
679
|
"classKey": first["classKey"], "title": first["title"], "trigger": first["trigger"],
|
|
513
680
|
"check": first["check"], "grade": min(o["lesson"]["grade"] for o in origins),
|
|
514
|
-
"stages": first["stages"], "nouns": nouns, "seen": len(
|
|
681
|
+
"stages": first["stages"], "nouns": nouns, "seen": len(counted),
|
|
515
682
|
"projects": projects, "status": "candidate" if projects >= 2 else "local",
|
|
516
683
|
"origins": origins}
|
|
517
684
|
|
|
518
685
|
|
|
519
|
-
def _origin(lesson: dict, memory: str | None = None) -> dict:
|
|
686
|
+
def _origin(lesson: dict, memory: str | None = None, checkouts: list[str] | None = None) -> dict:
|
|
520
687
|
o = lesson["origin"]
|
|
521
688
|
body = {k: lesson[k] for k in FIELDS if k not in ("schema", "origin")}
|
|
522
689
|
rec = {"project": o["project"], "commit": o["commit"], "retro": o["retro"], "lesson": body}
|
|
523
690
|
if memory:
|
|
524
691
|
rec["memory"] = memory
|
|
692
|
+
if checkouts:
|
|
693
|
+
rec["checkouts"] = sorted(set(checkouts))
|
|
525
694
|
return rec
|
|
526
695
|
|
|
527
696
|
|
|
528
|
-
def
|
|
529
|
-
"""
|
|
697
|
+
def checkout_id(root: pathlib.Path) -> str:
|
|
698
|
+
"""A checkout's provenance: its real top-level path. Stays in the local index; never emitted."""
|
|
699
|
+
return str(root.resolve())
|
|
700
|
+
|
|
701
|
+
|
|
702
|
+
def merge(entries: dict[str, dict], lessons: list[dict], checkout: str | None = None) -> int:
|
|
703
|
+
"""Merge lessons into {classKey: entry}; returns how many origins changed. `checkout` is the
|
|
704
|
+
checkout the lessons were read from: an origin's provenance is the union of the checkouts
|
|
705
|
+
that held it (round 9) — the only checkouts that may later show it is gone."""
|
|
530
706
|
changed = 0
|
|
531
707
|
for le in lessons:
|
|
532
708
|
ck = le["classKey"]
|
|
533
709
|
origins = list(entries[ck]["origins"]) if ck in entries else []
|
|
534
710
|
key = (le["origin"]["project"], le["origin"]["retro"])
|
|
535
711
|
old = next((o for o in origins if _origin_key(o) == key), None)
|
|
536
|
-
|
|
712
|
+
prov = list((old or {}).get("checkouts") or []) + ([checkout] if checkout else [])
|
|
713
|
+
new = _origin(le, old.get("memory") if old else None, prov)
|
|
714
|
+
if old and old.get("memory") and old.get("rekeyedFrom"):
|
|
715
|
+
new["rekeyedFrom"] = old["rekeyedFrom"] # the id's source key travels with it (round 16)
|
|
537
716
|
if old == new:
|
|
538
717
|
continue
|
|
539
718
|
origins = [o for o in origins if _origin_key(o) != key] + [new]
|
|
@@ -555,9 +734,167 @@ class IndexCorrupt(Refused):
|
|
|
555
734
|
# Lines in a NEWER format, per index path: skipped on read, counted, and written back on save,
|
|
556
735
|
# so an older script never destroys what a newer one wrote.
|
|
557
736
|
_FUTURE: dict[str, list[str]] = {}
|
|
737
|
+
# Per index path: the `.corrupt-<date>` file name and the memory ids only its bad lines held.
|
|
738
|
+
_CORRUPT_ONLY: dict[str, tuple[str, set[str]]] = {}
|
|
739
|
+
|
|
740
|
+
|
|
741
|
+
ENTRY_TEXT = ("id", "title", "trigger", "check")
|
|
742
|
+
LESSON_KEYS = tuple(k for k in FIELDS if k not in ("schema", "origin"))
|
|
743
|
+
MEM_RE = re.compile(r"mem:[^@\s\"\\]+@[1-9][0-9]*") # revisions start at 1 (round 11, #5)
|
|
744
|
+
|
|
745
|
+
|
|
746
|
+
def mem_base(memory: str) -> str:
|
|
747
|
+
"""A memory id without its revision: `mem:x@3` → `mem:x`. Ids compare on this (round 9, L3)."""
|
|
748
|
+
return memory.rsplit("@", 1)[0]
|
|
749
|
+
|
|
750
|
+
|
|
751
|
+
def mem_rev(memory: str) -> int:
|
|
752
|
+
try:
|
|
753
|
+
return int(memory.rsplit("@", 1)[1])
|
|
754
|
+
except (IndexError, ValueError):
|
|
755
|
+
return 0
|
|
756
|
+
|
|
757
|
+
|
|
758
|
+
def _entry_shape(e: object) -> bool:
|
|
759
|
+
"""A lesson-index/1 line every verb can use (rounds 8–11, L1/L2/H2): the entry's own fields
|
|
760
|
+
with their types — class key, id, title, trigger, check, nouns (strings), seen, projects —
|
|
761
|
+
and a non-empty list of origins, each with its project, retro, commit and a lesson that
|
|
762
|
+
lesson/1 itself accepts (`validate`, with that origin), of the entry's class. Optional
|
|
763
|
+
fields keep their types: memory `mem:<id>@<rev>` with rev from 1, checkouts non-empty
|
|
764
|
+
strings, carried one of CARRY_REASONS. Anything less is a corrupt line — set aside, its good
|
|
765
|
+
neighbours salvaged — never a TypeError in the middle of a verb. Derived fields (seen,
|
|
766
|
+
projects, status, id…) are recomputed by `_put` anyway."""
|
|
767
|
+
if not isinstance(e, dict) or not isinstance(e.get("classKey"), str):
|
|
768
|
+
return False
|
|
769
|
+
if not all(isinstance(e.get(k), str) for k in ENTRY_TEXT):
|
|
770
|
+
return False
|
|
771
|
+
if not isinstance(e.get("nouns"), list) or not all(isinstance(n, str) for n in e["nouns"]):
|
|
772
|
+
return False
|
|
773
|
+
if not all(isinstance(e.get(k), int) and not isinstance(e.get(k), bool) for k in ("seen", "projects")):
|
|
774
|
+
return False
|
|
775
|
+
origins = e.get("origins")
|
|
776
|
+
if not isinstance(origins, list) or not origins:
|
|
777
|
+
return False
|
|
778
|
+
for o in origins:
|
|
779
|
+
if not (isinstance(o, dict) and all(isinstance(o.get(k), str) for k in ("project", "retro", "commit"))
|
|
780
|
+
and isinstance(o.get("lesson"), dict) and all(k in o["lesson"] for k in LESSON_KEYS)):
|
|
781
|
+
return False
|
|
782
|
+
body = {k: o["lesson"][k] for k in LESSON_KEYS}
|
|
783
|
+
whole = dict(body, schema="lesson/1",
|
|
784
|
+
origin={"project": o["project"], "commit": o["commit"], "retro": o["retro"]})
|
|
785
|
+
if validate(whole) or body["classKey"] != e["classKey"]:
|
|
786
|
+
return False
|
|
787
|
+
if "memory" in o and not (isinstance(o["memory"], str) and MEM_RE.fullmatch(o["memory"])):
|
|
788
|
+
return False
|
|
789
|
+
if "checkouts" in o and not (isinstance(o["checkouts"], list)
|
|
790
|
+
and all(isinstance(c, str) and c for c in o["checkouts"])):
|
|
791
|
+
return False
|
|
792
|
+
if "carried" in o and o["carried"] not in CARRY_REASONS:
|
|
793
|
+
return False
|
|
794
|
+
if "rekeyedFrom" in o and not (isinstance(o["rekeyedFrom"], str)
|
|
795
|
+
and PROJECT.match(o["rekeyedFrom"]) and o.get("memory")):
|
|
796
|
+
return False
|
|
797
|
+
return True
|
|
798
|
+
|
|
799
|
+
|
|
800
|
+
def _stage_id(v: object) -> object:
|
|
801
|
+
"""A stage id lesson/1 accepts today, from one v1.95.0 accepted (round 14, L5): a string
|
|
802
|
+
that fails the text checks (longer than 32 characters, a line break, a path or credential
|
|
803
|
+
shape) is folded to one line and cut to 32 characters, and one that still fails becomes
|
|
804
|
+
`stage`. Anything else is returned as it is, for the shape check to judge."""
|
|
805
|
+
if not isinstance(v, str) or not v.strip() or not _check_text("s", v, 1, 32):
|
|
806
|
+
return v
|
|
807
|
+
folded = " ".join(" ".join(v.splitlines()).split())[:32].strip()
|
|
808
|
+
return folded if folded and not _check_text("s", folded, 1, 32) else "stage"
|
|
809
|
+
|
|
810
|
+
|
|
811
|
+
def _migrate_stages(e: object, notes: list[str] | None) -> None:
|
|
812
|
+
"""Normalise, in place, the stage ids of an index line written before lesson/1 checked them
|
|
813
|
+
(`_stage_id`), and say so — so the line is read, never set aside as corrupt, and no memory
|
|
814
|
+
id it holds is lost. The next write stores the normalised ids."""
|
|
815
|
+
if not isinstance(e, dict):
|
|
816
|
+
return
|
|
817
|
+
holders = [e] + [o["lesson"] for o in e.get("origins") or []
|
|
818
|
+
if isinstance(o, dict) and isinstance(o.get("lesson"), dict)]
|
|
819
|
+
changed = False
|
|
820
|
+
for h in holders:
|
|
821
|
+
st = h.get("stages")
|
|
822
|
+
if isinstance(st, dict):
|
|
823
|
+
for k in ("owned", "surfaced"):
|
|
824
|
+
if k in st and _stage_id(st[k]) != st[k]:
|
|
825
|
+
st[k] = _stage_id(st[k])
|
|
826
|
+
changed = True
|
|
827
|
+
if changed and notes is not None:
|
|
828
|
+
notes.append(f"{e.get('classKey', '?')}: a stage id written before lesson/1 checked stage "
|
|
829
|
+
"ids was normalised (one line, at most 32 characters)")
|
|
830
|
+
|
|
831
|
+
|
|
832
|
+
def _put(out: dict[str, dict], e: dict, notes: list[str] | None) -> None:
|
|
833
|
+
"""Every line read is normalised: one origin per (project, retro) — the one with the higher
|
|
834
|
+
memory revision winning (on a tie the lexically smaller id), its provenance the union of both copies' (round 10, G5) — and the
|
|
835
|
+
entry re-derived from its origins, so `seen`, `projects`, `status` and the rest are never
|
|
836
|
+
trusted from disk (round 11, #6). Two lines for one class key (a hand-concatenated index)
|
|
837
|
+
merge their origins, never last-one-wins (round 9, L4/D3), and that is said aloud — naming
|
|
838
|
+
every memory id the merge dropped (round 12: one origin holds one id)."""
|
|
839
|
+
ck = e["classKey"]
|
|
840
|
+
twice = ck in out
|
|
841
|
+
by_key: dict[tuple, dict] = {}
|
|
842
|
+
dropped: list[str] = []
|
|
843
|
+
for o in (out[ck]["origins"] if twice else []) + e["origins"]:
|
|
844
|
+
k = _origin_key(o)
|
|
845
|
+
if k in by_key:
|
|
846
|
+
# The higher revision wins; on equal revisions the lexically smaller id, then the
|
|
847
|
+
# smaller line — never the order the lines happen to be in (round 13, Q).
|
|
848
|
+
keep, other = sorted((o, by_key[k]), key=lambda x: (
|
|
849
|
+
-mem_rev(x.get("memory") or ""), x.get("memory") or "", json.dumps(x, sort_keys=True)))
|
|
850
|
+
if other.get("memory") and mem_base(other["memory"]) != mem_base(keep.get("memory") or ""):
|
|
851
|
+
dropped.append(other["memory"])
|
|
852
|
+
prov = sorted(set(keep.get("checkouts") or []) | set(other.get("checkouts") or []))
|
|
853
|
+
o = dict(keep, **({"checkouts": prov} if prov else {}))
|
|
854
|
+
by_key[k] = o
|
|
855
|
+
out[ck] = derive(list(by_key.values()))
|
|
856
|
+
if notes is not None and (twice or dropped):
|
|
857
|
+
lost = f"dropped {', '.join(dropped)}" if dropped else "none dropped"
|
|
858
|
+
notes.append(f"{'two lines' if twice else 'one line'} for {ck} merged — {len(by_key)} origin"
|
|
859
|
+
f"{'' if len(by_key) == 1 else 's'} kept, {lost}")
|
|
860
|
+
|
|
861
|
+
|
|
862
|
+
def _salvage(path: pathlib.Path, notes: list[str] | None = None
|
|
863
|
+
) -> tuple[dict[str, dict], list[str], int, set[str]]:
|
|
864
|
+
"""The lines of a corrupt index that still parse: (entries, newer-format lines, bad lines,
|
|
865
|
+
memory ids seen only in bad lines). A corrupt index is set aside whole; what parses is
|
|
866
|
+
carried into the live index, so one bad line never costs every memory id and carried origin
|
|
867
|
+
beside it (round 8, L2), and an id only a bad line held is named (round 9, D2)."""
|
|
868
|
+
out, future, bad, bad_mems = {}, [], 0, set()
|
|
869
|
+
try:
|
|
870
|
+
raw = path.read_bytes()
|
|
871
|
+
except OSError:
|
|
872
|
+
return out, future, 0, bad_mems
|
|
873
|
+
for chunk in raw.split(b"\n"): # "\n" only, as `dump` writes (round 12, B)
|
|
874
|
+
if not chunk.strip():
|
|
875
|
+
continue
|
|
876
|
+
try:
|
|
877
|
+
line = chunk.decode("utf-8")
|
|
878
|
+
e = json.loads(line)
|
|
879
|
+
except (UnicodeDecodeError, ValueError):
|
|
880
|
+
bad += 1
|
|
881
|
+
bad_mems |= set(MEM_RE.findall(chunk.decode("utf-8", "replace")))
|
|
882
|
+
continue
|
|
883
|
+
if isinstance(e, dict) and e.get("schema", "lesson-index/1") != "lesson-index/1":
|
|
884
|
+
future.append(line)
|
|
885
|
+
continue
|
|
886
|
+
_migrate_stages(e, notes)
|
|
887
|
+
if _entry_shape(e):
|
|
888
|
+
_put(out, e, notes)
|
|
889
|
+
else:
|
|
890
|
+
bad += 1
|
|
891
|
+
bad_mems |= set(MEM_RE.findall(line))
|
|
892
|
+
held = {o["memory"] for e in out.values() for o in e["origins"] if o.get("memory")}
|
|
893
|
+
return out, future, bad, {m for m in bad_mems
|
|
894
|
+
if mem_base(m) not in {mem_base(h) for h in held}}
|
|
558
895
|
|
|
559
896
|
|
|
560
|
-
def load(path: pathlib.Path) -> dict[str, dict]:
|
|
897
|
+
def load(path: pathlib.Path, notes: list[str] | None = None) -> dict[str, dict]:
|
|
561
898
|
if not path.exists():
|
|
562
899
|
return {}
|
|
563
900
|
try:
|
|
@@ -569,7 +906,10 @@ def load(path: pathlib.Path) -> dict[str, dict]:
|
|
|
569
906
|
raise IndexUnreadable(f"the index {path} cannot be read ({type(exc).__name__}) — fix "
|
|
570
907
|
"its permissions; nothing was written")
|
|
571
908
|
out, future = {}, []
|
|
572
|
-
|
|
909
|
+
# One index line per "\n", exactly as `dump` writes it — never str.splitlines(), which also
|
|
910
|
+
# breaks on U+2028/U+2029/U+0085 inside a JSON string written with ensure_ascii=False and
|
|
911
|
+
# made every later read call a good index corrupt (round 12, B).
|
|
912
|
+
for i, line in enumerate(text.split("\n"), 1):
|
|
573
913
|
if not line.strip():
|
|
574
914
|
continue
|
|
575
915
|
try:
|
|
@@ -580,14 +920,15 @@ def load(path: pathlib.Path) -> dict[str, dict]:
|
|
|
580
920
|
if isinstance(e, dict) and e.get("schema", "lesson-index/1") != "lesson-index/1":
|
|
581
921
|
future.append(line)
|
|
582
922
|
continue
|
|
583
|
-
|
|
923
|
+
_migrate_stages(e, notes)
|
|
924
|
+
if not _entry_shape(e):
|
|
584
925
|
raise IndexCorrupt(f"the index {path} has a line that is not a lesson (line {i}) — "
|
|
585
926
|
"`lessons.py rebuild` keeps it as .corrupt-<date> and re-derives it")
|
|
586
|
-
out
|
|
927
|
+
_put(out, e, notes)
|
|
587
928
|
_FUTURE[str(path)] = future
|
|
588
929
|
if future:
|
|
589
|
-
print(f"lessons: {len(future)} line{'' if len(future) == 1 else 's'} in a newer format "
|
|
590
|
-
|
|
930
|
+
print(_say(f"lessons: {len(future)} line{'' if len(future) == 1 else 's'} in a newer format "
|
|
931
|
+
f"skipped in {path} (kept on write; a newer lessons.py reads them)"), file=sys.stderr)
|
|
591
932
|
return out
|
|
592
933
|
|
|
593
934
|
|
|
@@ -665,7 +1006,32 @@ def _ensure_dir(d: pathlib.Path) -> None:
|
|
|
665
1006
|
"nothing was written")
|
|
666
1007
|
|
|
667
1008
|
|
|
1009
|
+
def _unloadable(entries: dict[str, dict]) -> list[str]:
|
|
1010
|
+
"""The class keys whose line, as `dump` writes it, the loader would refuse (round 17, F1):
|
|
1011
|
+
each line is read back exactly as `load` reads it (JSON, stage ids normalised, then
|
|
1012
|
+
`_entry_shape`). A writer that produced such a line once made the next command set the
|
|
1013
|
+
whole index aside as `.corrupt-<date>`, taking every lesson beside it out of the live index."""
|
|
1014
|
+
bad = []
|
|
1015
|
+
for ck in sorted(entries):
|
|
1016
|
+
try:
|
|
1017
|
+
e = json.loads(json.dumps(entries[ck], ensure_ascii=False, sort_keys=True))
|
|
1018
|
+
except (TypeError, ValueError):
|
|
1019
|
+
bad.append(str(ck))
|
|
1020
|
+
continue
|
|
1021
|
+
_migrate_stages(e, None)
|
|
1022
|
+
if not _entry_shape(e):
|
|
1023
|
+
bad.append(str(ck))
|
|
1024
|
+
return bad
|
|
1025
|
+
|
|
1026
|
+
|
|
668
1027
|
def save(path: pathlib.Path, entries: dict[str, dict]) -> None:
|
|
1028
|
+
# Never write a line the loader refuses (round 17, F1): fail loudly, the previous index
|
|
1029
|
+
# intact, rather than hand the next command an index it sets aside.
|
|
1030
|
+
bad = _unloadable(entries)
|
|
1031
|
+
if bad:
|
|
1032
|
+
raise Refused(f"the index would hold {len(bad)} line{'' if len(bad) == 1 else 's'} the "
|
|
1033
|
+
f"loader refuses ({', '.join(bad[:3])}); nothing was written, the previous "
|
|
1034
|
+
"index is intact; this is a defect, report it")
|
|
669
1035
|
_ensure_dir(path.parent)
|
|
670
1036
|
tmp = None
|
|
671
1037
|
try:
|
|
@@ -698,8 +1064,18 @@ def _resolve(entries: dict[str, dict], lid: str) -> dict:
|
|
|
698
1064
|
# ---------------------------------------------------------------------------- verbs
|
|
699
1065
|
|
|
700
1066
|
|
|
1067
|
+
def refuse_shallow(root: pathlib.Path) -> None:
|
|
1068
|
+
"""A shallow clone's project key — its root commit — moves whenever its boundary moves, so
|
|
1069
|
+
add, emit and record refuse there (round 11): nothing written, the store never sees a key
|
|
1070
|
+
that will not last."""
|
|
1071
|
+
if _git(root, "rev-parse", "--is-shallow-repository") == "true":
|
|
1072
|
+
raise Refused(f"{root.name} is a shallow clone: its project key changes when it is "
|
|
1073
|
+
"deepened — deepen it first (git fetch --unshallow); nothing was written")
|
|
1074
|
+
|
|
1075
|
+
|
|
701
1076
|
def cmd_add(a, idx: pathlib.Path) -> int:
|
|
702
1077
|
root = repo_root(pathlib.Path(a.root or "."))
|
|
1078
|
+
refuse_shallow(root)
|
|
703
1079
|
project = project_key(root)
|
|
704
1080
|
private, problems = 0, []
|
|
705
1081
|
if a.json is not None:
|
|
@@ -735,7 +1111,51 @@ def cmd_add(a, idx: pathlib.Path) -> int:
|
|
|
735
1111
|
raise Refused("refused, nothing written:\n " + "\n ".join(errors))
|
|
736
1112
|
with session(idx, a.default_index) as ss:
|
|
737
1113
|
entries = ss.entries
|
|
738
|
-
|
|
1114
|
+
before = json.loads(json.dumps(entries)) # the index as read: who held each id first
|
|
1115
|
+
changed = merge(entries, lessons, checkout_id(root))
|
|
1116
|
+
# The same FOREIGN rule at add time (round 12, repro G): an origin of another project key
|
|
1117
|
+
# that came from THIS checkout, whose commit no longer resolves here, is this checkout's
|
|
1118
|
+
# old identity (a re-init) — kept, marked, not counted, so add never adds a phantom.
|
|
1119
|
+
here = checkout_id(root)
|
|
1120
|
+
rekeyed: list[str] = []
|
|
1121
|
+
# The same live-project rule as rebuild (round 15): never re-key an origin into this
|
|
1122
|
+
# project when its key is a project the index already records with provenance.
|
|
1123
|
+
known = _known_projects(before)
|
|
1124
|
+
for ck in sorted({le["classKey"] for le in lessons}):
|
|
1125
|
+
origins = entries[ck]["origins"]
|
|
1126
|
+
marked = [dict(o, carried=CARRY_FOREIGN)
|
|
1127
|
+
if o["project"] != project and here in (o.get("checkouts") or [])
|
|
1128
|
+
and not o.get("carried") and not commit_resolves(root, o["commit"]) else o
|
|
1129
|
+
for o in origins]
|
|
1130
|
+
# An origin v1.95.0 wrote for this repository — no provenance, keyed on the smallest
|
|
1131
|
+
# root — whose commit resolves here is this checkout's origin under its old key
|
|
1132
|
+
# (round 13, K): it takes this key and gives its memory id to the origin just added.
|
|
1133
|
+
fresh = [o for o in marked if here in (o.get("checkouts") or []) and o["project"] == project]
|
|
1134
|
+
for o in list(marked):
|
|
1135
|
+
n = None if o["project"] in known else _legacy_twin(o, fresh)
|
|
1136
|
+
if n is None or (o.get("memory") and n.get("memory")
|
|
1137
|
+
and mem_base(o["memory"]) != mem_base(n["memory"])):
|
|
1138
|
+
continue
|
|
1139
|
+
memory = _newer_memory(o.get("memory"), n.get("memory"))
|
|
1140
|
+
# The source is o's project only when the id came from o (round 18, L1): a twin
|
|
1141
|
+
# keeping its own id keeps its own source, if any.
|
|
1142
|
+
source = ({"memory": memory} if not o.get("memory") else
|
|
1143
|
+
{"memory": memory, "rekeyedFrom": o.get("rekeyedFrom") or o["project"]}
|
|
1144
|
+
) if memory else {}
|
|
1145
|
+
marked = [dict(x, **source)
|
|
1146
|
+
if _origin_key(x) == _origin_key(n) else x for x in marked if x is not o]
|
|
1147
|
+
rekeyed.append(f"{ck} — {o['retro']}: {o['project']} is {project} (written before "
|
|
1148
|
+
"provenance; its commit resolves here)" + (f" · {memory}" if memory else ""))
|
|
1149
|
+
if marked != origins:
|
|
1150
|
+
entries[ck] = derive(marked)
|
|
1151
|
+
changed += 1
|
|
1152
|
+
# The project a re-key took an id from takes it back when it is read (round 16).
|
|
1153
|
+
reclaimed = _reclaim(entries)
|
|
1154
|
+
changed += len(reclaimed)
|
|
1155
|
+
# One memory id stays on one live project at add time too (round 14, F3): an add from
|
|
1156
|
+
# an old clone that brings a carried holder back to life must not leave its id on both.
|
|
1157
|
+
shared = _one_project_per_memory(entries, before)
|
|
1158
|
+
changed += len(shared)
|
|
739
1159
|
if changed:
|
|
740
1160
|
ss.save()
|
|
741
1161
|
n = len(lessons)
|
|
@@ -745,6 +1165,12 @@ def cmd_add(a, idx: pathlib.Path) -> int:
|
|
|
745
1165
|
for le in lessons:
|
|
746
1166
|
e = entries[le["classKey"]]
|
|
747
1167
|
print(f" {e['id']} {e['classKey']} — {e['status']} (seen {e['seen']}, projects {e['projects']})")
|
|
1168
|
+
for n in rekeyed:
|
|
1169
|
+
print(f" re-keyed: {n}")
|
|
1170
|
+
for n in reclaimed:
|
|
1171
|
+
print(f" reclaimed: {n}")
|
|
1172
|
+
for n in shared:
|
|
1173
|
+
print(f" memory: {n}")
|
|
748
1174
|
return 0
|
|
749
1175
|
|
|
750
1176
|
|
|
@@ -762,8 +1188,8 @@ def cmd_query(a, idx: pathlib.Path) -> int:
|
|
|
762
1188
|
with session(idx, a.default_index) as ss:
|
|
763
1189
|
entries = ss.entries
|
|
764
1190
|
if not entries and not idx.exists():
|
|
765
|
-
print(f"lessons: local index absent ({idx}) — nothing to read; the retro and "
|
|
766
|
-
|
|
1191
|
+
print(_say(f"lessons: local index absent ({idx}) — nothing to read; the retro and "
|
|
1192
|
+
"learned.md still run"))
|
|
767
1193
|
return 0
|
|
768
1194
|
query = [part.strip().casefold() for arg in a.nouns for part in arg.split(",") if part.strip()]
|
|
769
1195
|
qwords = set(query) | {w for q in query for w in _words(q)}
|
|
@@ -807,11 +1233,57 @@ def _source_stat(path: pathlib.Path):
|
|
|
807
1233
|
return info
|
|
808
1234
|
|
|
809
1235
|
|
|
810
|
-
|
|
811
|
-
|
|
812
|
-
|
|
1236
|
+
Problem = tuple # (owner checkout path or None, text)
|
|
1237
|
+
|
|
1238
|
+
|
|
1239
|
+
def find_retros(roots: list[str], depth: int) -> tuple[list[tuple[pathlib.Path, pathlib.Path]], list[Problem],
|
|
1240
|
+
dict[pathlib.Path, bool]]:
|
|
1241
|
+
"""Discovery and its unknowns, kept separate from a readable empty scope.
|
|
1242
|
+
|
|
1243
|
+
Returns (retro files, problems, checkouts). Each checkout discovered — keyed by its real
|
|
1244
|
+
path — maps to whether its retro sources were discovered in full. Each problem carries its
|
|
1245
|
+
OWNER: the checkout it belongs to, or None when no checkout owns it (an unreadable
|
|
1246
|
+
directory, a missing root). Ownership is structural, never a path-prefix match (round 9).
|
|
1247
|
+
|
|
1248
|
+
Every git worktree of a checkout found is discovered too (`git worktree list --porcelain`,
|
|
1249
|
+
round 8): the walk skips dot-directories, and `.worktrees/` or `.claude/worktrees/` is where
|
|
1250
|
+
a feature branch's retro usually lives. A listing git refuses — a non-zero exit or no answer —
|
|
1251
|
+
is a problem owned by that checkout (round 9, L1)."""
|
|
1252
|
+
found: list[tuple[pathlib.Path, pathlib.Path]] = []
|
|
1253
|
+
problems: list[Problem] = []
|
|
1254
|
+
repos: dict[pathlib.Path, bool] = {}
|
|
1255
|
+
|
|
813
1256
|
def unread(exc):
|
|
814
|
-
problems.append(f"{exc.filename}: cannot discover
|
|
1257
|
+
problems.append((None, f"{pathlib.Path(exc.filename or '?').name}: cannot discover "
|
|
1258
|
+
f"({type(exc).__name__})"))
|
|
1259
|
+
|
|
1260
|
+
def visit(here: pathlib.Path) -> None:
|
|
1261
|
+
try:
|
|
1262
|
+
here = here.resolve()
|
|
1263
|
+
except OSError:
|
|
1264
|
+
pass
|
|
1265
|
+
if here in repos:
|
|
1266
|
+
return
|
|
1267
|
+
repos[here] = False
|
|
1268
|
+
try:
|
|
1269
|
+
art = _artifact_root(here)
|
|
1270
|
+
archive = art / "retro"
|
|
1271
|
+
archive_info = _source_stat(archive)
|
|
1272
|
+
if archive_info is not None and not stat.S_ISDIR(archive_info.st_mode):
|
|
1273
|
+
raise Refused(f"{archive}: retro archive is not a directory")
|
|
1274
|
+
archived = sorted(archive.iterdir()) if archive_info is not None else []
|
|
1275
|
+
files = [art / "retro.md"] + [p for p in archived if p.suffix == ".md"]
|
|
1276
|
+
for f in files:
|
|
1277
|
+
info = _source_stat(f)
|
|
1278
|
+
if info is None:
|
|
1279
|
+
continue
|
|
1280
|
+
if not stat.S_ISREG(info.st_mode):
|
|
1281
|
+
raise Refused(f"{f}: retro source is not a regular file")
|
|
1282
|
+
found.append((here, f))
|
|
1283
|
+
repos[here] = True
|
|
1284
|
+
except (OSError, Refused) as exc:
|
|
1285
|
+
problems.append((here, f"{here.name}: {exc}"))
|
|
1286
|
+
|
|
815
1287
|
for r in roots:
|
|
816
1288
|
base = pathlib.Path(r).expanduser()
|
|
817
1289
|
try:
|
|
@@ -819,33 +1291,49 @@ def find_retros(roots: list[str], depth: int) -> tuple[list[tuple[pathlib.Path,
|
|
|
819
1291
|
if base_info is None or not stat.S_ISDIR(base_info.st_mode):
|
|
820
1292
|
raise Refused(f"{base}: root is missing or not a readable directory")
|
|
821
1293
|
except (OSError, Refused) as exc:
|
|
822
|
-
problems.append(str(exc))
|
|
1294
|
+
problems.append((None, str(exc)))
|
|
823
1295
|
continue
|
|
824
1296
|
for cur, dirs, _files in os.walk(base, followlinks=True, onerror=unread):
|
|
825
1297
|
here = pathlib.Path(cur)
|
|
826
1298
|
rel_depth = len(here.relative_to(base).parts)
|
|
827
1299
|
dirs[:] = sorted(d for d in dirs if d not in SKIP_DIRS and not d.startswith("."))
|
|
828
1300
|
try:
|
|
829
|
-
|
|
830
|
-
art = _artifact_root(here)
|
|
831
|
-
archive = art / "retro"
|
|
832
|
-
archive_info = _source_stat(archive)
|
|
833
|
-
if archive_info is not None and not stat.S_ISDIR(archive_info.st_mode):
|
|
834
|
-
raise Refused(f"{archive}: retro archive is not a directory")
|
|
835
|
-
archived = sorted(archive.iterdir()) if archive_info is not None else []
|
|
836
|
-
files = [art / "retro.md"] + [p for p in archived if p.suffix == ".md"]
|
|
837
|
-
for f in files:
|
|
838
|
-
info = _source_stat(f)
|
|
839
|
-
if info is None:
|
|
840
|
-
continue
|
|
841
|
-
if not stat.S_ISREG(info.st_mode):
|
|
842
|
-
raise Refused(f"{f}: retro source is not a regular file")
|
|
843
|
-
found.append((here, f))
|
|
1301
|
+
is_repo = _source_stat(here / ".git") is not None
|
|
844
1302
|
except (OSError, Refused) as exc:
|
|
845
|
-
problems.append(f"{here}: {exc}")
|
|
1303
|
+
problems.append((None, f"{here}: {exc}"))
|
|
1304
|
+
is_repo = False
|
|
1305
|
+
if is_repo:
|
|
1306
|
+
visit(here)
|
|
846
1307
|
if rel_depth >= depth:
|
|
847
1308
|
dirs[:] = []
|
|
848
|
-
|
|
1309
|
+
for repo in list(repos):
|
|
1310
|
+
for wt in _worktrees(repo, problems):
|
|
1311
|
+
visit(wt)
|
|
1312
|
+
return found, problems, repos
|
|
1313
|
+
|
|
1314
|
+
|
|
1315
|
+
def _worktrees(repo: pathlib.Path, problems: list[Problem]) -> list[pathlib.Path]:
|
|
1316
|
+
"""The live worktrees git lists for `repo` (bare and prunable ones skipped). A listing that
|
|
1317
|
+
cannot be read is a problem owned by `repo`: its siblings are unknown."""
|
|
1318
|
+
try:
|
|
1319
|
+
listing = _git(repo, "worktree", "list", "--porcelain")
|
|
1320
|
+
except Refused as exc:
|
|
1321
|
+
listing, why = None, str(exc)
|
|
1322
|
+
else:
|
|
1323
|
+
why = "git exited non-zero"
|
|
1324
|
+
if listing is None:
|
|
1325
|
+
problems.append((repo, f"{repo.name}: worktrees cannot be listed ({why})"))
|
|
1326
|
+
return []
|
|
1327
|
+
out = []
|
|
1328
|
+
for block in listing.split("\n\n"):
|
|
1329
|
+
lines = block.splitlines()
|
|
1330
|
+
head = next((ln[len("worktree "):] for ln in lines if ln.startswith("worktree ")), None)
|
|
1331
|
+
if head is None or any(ln == "bare" or ln.startswith("prunable") for ln in lines):
|
|
1332
|
+
continue
|
|
1333
|
+
path = pathlib.Path(head)
|
|
1334
|
+
if path.is_dir():
|
|
1335
|
+
out.append(path)
|
|
1336
|
+
return out
|
|
849
1337
|
|
|
850
1338
|
|
|
851
1339
|
def _artifact_root(repo: pathlib.Path) -> pathlib.Path:
|
|
@@ -881,66 +1369,617 @@ def _artifact_root(repo: pathlib.Path) -> pathlib.Path:
|
|
|
881
1369
|
return repo / ARTIFACT_KNOWN[0]
|
|
882
1370
|
|
|
883
1371
|
|
|
1372
|
+
CARRY_CHECKOUT = "checkout not read"
|
|
1373
|
+
CARRY_FILE = "retro file not read"
|
|
1374
|
+
CARRY_LEGACY = "no provenance"
|
|
1375
|
+
CARRY_FOREIGN = "checkout holds another project"
|
|
1376
|
+
CARRY_UNRESOLVED = "commit does not resolve"
|
|
1377
|
+
CARRY_CITES_UNRESOLVED = "retro cites a commit that does not resolve"
|
|
1378
|
+
CARRY_REFUSED = "lesson refused"
|
|
1379
|
+
CARRY_GONE = "no checkout read holds it"
|
|
1380
|
+
CARRY_SOURCE = "source not read" # the round-7 label, still accepted in an index line
|
|
1381
|
+
CARRY_REASONS = (CARRY_CHECKOUT, CARRY_FILE, CARRY_LEGACY, CARRY_FOREIGN, CARRY_UNRESOLVED,
|
|
1382
|
+
CARRY_CITES_UNRESOLVED, CARRY_REFUSED, CARRY_GONE, CARRY_SOURCE)
|
|
1383
|
+
# The reasons `_carry_reason` can produce today — the only ones --prune-carried-reason offers.
|
|
1384
|
+
PRUNE_REASONS = tuple(r for r in CARRY_REASONS if r != CARRY_SOURCE)
|
|
1385
|
+
|
|
1386
|
+
|
|
1387
|
+
def _redact(text: str, shown: dict[str, str]) -> str:
|
|
1388
|
+
"""`text` with every known absolute path replaced by its name, longest path first, so a
|
|
1389
|
+
checkout inside a root is shown as itself and never as the root plus a tail. Only absolute
|
|
1390
|
+
paths with a real name are replaced, and only as whole path components (round 13, N2): a
|
|
1391
|
+
root given as `.` once deleted every dot of every message (`repo.v2`, `mem:v1.2.3`)."""
|
|
1392
|
+
for path in sorted(shown, key=len, reverse=True):
|
|
1393
|
+
name = shown[path]
|
|
1394
|
+
if not os.path.isabs(path) or path == "/" or name in ("", ".", ".."):
|
|
1395
|
+
continue
|
|
1396
|
+
text = re.sub(r"(?<![\w.~/-])" + re.escape(path) + r"(?![\w.~-])", lambda _m: name, text)
|
|
1397
|
+
return text
|
|
1398
|
+
|
|
1399
|
+
|
|
1400
|
+
def _say(text: str) -> str:
|
|
1401
|
+
"""A message as printed: every path `rebuild` knows shown by its name (`_SHOWN`)."""
|
|
1402
|
+
return _redact(text, _SHOWN)
|
|
1403
|
+
|
|
1404
|
+
|
|
1405
|
+
def _legacy_twin(o: dict, candidates: list[dict]) -> dict | None:
|
|
1406
|
+
"""The re-derived origin a provenance-less origin is, or None (round 13, K).
|
|
1407
|
+
|
|
1408
|
+
v1.95.0 keyed a project on the SMALLEST root commit of HEAD and recorded no checkouts; a
|
|
1409
|
+
repository that merged an unrelated history whose root sorts lower is keyed differently
|
|
1410
|
+
now. Such an origin is the same repository's — re-keyed, never counted twice — only on
|
|
1411
|
+
positive evidence: it has no `checkouts`, exactly one candidate of another key holds the
|
|
1412
|
+
same `retro#anchor` under the same class key, and in a checkout that candidate was derived
|
|
1413
|
+
from (round 14, F1) the origin's own commit resolves AND descends from the root the
|
|
1414
|
+
candidate's key names (`_roots_of`: its own history, not a history it merged; a commit that
|
|
1415
|
+
does not resolve has no roots, and a git that does not answer is a no), and v1.95.0 would
|
|
1416
|
+
have computed exactly the origin's key there (`_legacy_key`). A monorepo that merged another
|
|
1417
|
+
repository's history holds that repository's commits too, and under v1.95.0 may even have
|
|
1418
|
+
shared its key; the commit's roots tell them apart."""
|
|
1419
|
+
# No `checkouts` check here (round 15): an origin with provenance names a project the
|
|
1420
|
+
# index knows (`_known_projects`), and both callers skip those before asking.
|
|
1421
|
+
|
|
1422
|
+
def same_repository(n: dict, c: str) -> bool:
|
|
1423
|
+
return (_legacy_key(c) == o["project"]
|
|
1424
|
+
and n["project"][2:] in _roots_of(c, o["commit"]))
|
|
1425
|
+
hits = [n for n in candidates if n.get("checkouts") and n["project"] != o["project"]
|
|
1426
|
+
and n["retro"] == o["retro"]
|
|
1427
|
+
and any(same_repository(n, c) for c in n["checkouts"])]
|
|
1428
|
+
return hits[0] if len(hits) == 1 else None
|
|
1429
|
+
|
|
1430
|
+
|
|
1431
|
+
def _newer_memory(a: str | None, b: str | None) -> str | None:
|
|
1432
|
+
"""Of two ids of ONE record (same base), the higher revision; either when one is absent."""
|
|
1433
|
+
if not a or not b:
|
|
1434
|
+
return a or b
|
|
1435
|
+
return a if mem_rev(a) >= mem_rev(b) else b
|
|
1436
|
+
|
|
1437
|
+
|
|
1438
|
+
def _known_projects(index: dict[str, dict]) -> set[str]:
|
|
1439
|
+
"""Every project key an index records with provenance — an origin with `checkouts` (round
|
|
1440
|
+
15). Only the current key derivation writes provenance, so such a key is a real project of
|
|
1441
|
+
this version, never v1.95.0's name for another one. Every verb treats it as live: a project
|
|
1442
|
+
absent from THIS run's roots is not thereby another repository's to take."""
|
|
1443
|
+
return {o["project"] for e in index.values() for o in e["origins"] if o.get("checkouts")}
|
|
1444
|
+
|
|
1445
|
+
|
|
1446
|
+
def _rekey_legacy(old: dict[str, dict], entries: dict[str, dict],
|
|
1447
|
+
live: set[str] | frozenset = frozenset()) -> list[str]:
|
|
1448
|
+
"""Re-key, in the previous index, every provenance-less origin `_legacy_twin` proves is
|
|
1449
|
+
the same repository as an origin this rebuild re-derived: it takes the new key, so
|
|
1450
|
+
`_attach_memory` carries its memory id to the re-derived origin and it is not carried as a
|
|
1451
|
+
second project. One that holds a different memory id than the previous index's copy under
|
|
1452
|
+
the new key is left as it is — carried, named — since two store records are not one. An
|
|
1453
|
+
origin whose own key this run re-derives is never re-keyed: it is that project's — nor is
|
|
1454
|
+
one whose PROJECT key any checkout read this run holds (round 14, F2), whatever its anchor:
|
|
1455
|
+
that project is here, and its retitled entry is not another repository's. `live` is this
|
|
1456
|
+
run's projects plus every project the previous index records with provenance
|
|
1457
|
+
(`_known_projects`, round 15): which project owns an id never depends on verb order or on
|
|
1458
|
+
which roots a rebuild was given."""
|
|
1459
|
+
said: list[str] = []
|
|
1460
|
+
for ck in sorted(old):
|
|
1461
|
+
if ck not in entries:
|
|
1462
|
+
continue
|
|
1463
|
+
origins, changed = list(old[ck]["origins"]), False
|
|
1464
|
+
for o in list(origins):
|
|
1465
|
+
if o["project"] in live:
|
|
1466
|
+
continue # its project is here this run (its own origin re-derived or not): never moved
|
|
1467
|
+
n = _legacy_twin(o, entries[ck]["origins"])
|
|
1468
|
+
if n is None:
|
|
1469
|
+
continue
|
|
1470
|
+
twin = next((x for x in origins if _origin_key(x) == _origin_key(n)), None)
|
|
1471
|
+
if (twin and twin.get("memory") and o.get("memory")
|
|
1472
|
+
and mem_base(twin["memory"]) != mem_base(o["memory"])):
|
|
1473
|
+
said.append(f"{ck} — {o['project']} {o['retro']}: the same repository as "
|
|
1474
|
+
f"{n['project']}, but each holds its own memory id — left carried")
|
|
1475
|
+
continue
|
|
1476
|
+
moved = {k: v for k, v in o.items() if k not in ("carried", "rekeyedFrom")}
|
|
1477
|
+
moved["project"] = n["project"]
|
|
1478
|
+
memory = _newer_memory(o.get("memory"), (twin or {}).get("memory"))
|
|
1479
|
+
if memory:
|
|
1480
|
+
moved["memory"] = memory
|
|
1481
|
+
# Where the id came from (round 16): the project whose key the legacy store
|
|
1482
|
+
# used may still be read later, and then it takes its id back (`_reclaim`).
|
|
1483
|
+
# Only when the id came from o (round 18, L1): the twin's own id keeps the
|
|
1484
|
+
# twin's own source, if it had one.
|
|
1485
|
+
if o.get("memory"):
|
|
1486
|
+
moved["rekeyedFrom"] = o.get("rekeyedFrom") or o["project"]
|
|
1487
|
+
elif (twin or {}).get("rekeyedFrom"):
|
|
1488
|
+
moved["rekeyedFrom"] = twin["rekeyedFrom"]
|
|
1489
|
+
origins = [x for x in origins if x is not o and x is not twin] + [moved]
|
|
1490
|
+
said.append(f"{ck} — {o['retro']}: {o['project']} is {n['project']} (written before "
|
|
1491
|
+
"provenance; its commit resolves in the checkout that re-derived it)"
|
|
1492
|
+
+ (f" · {memory}" if memory else ""))
|
|
1493
|
+
changed = True
|
|
1494
|
+
if changed:
|
|
1495
|
+
old[ck] = derive(origins)
|
|
1496
|
+
return said
|
|
1497
|
+
|
|
1498
|
+
|
|
1499
|
+
def _given(o: dict, memory: str) -> dict:
|
|
1500
|
+
"""o holding `memory`. Where the id it held came from (`rekeyedFrom`) stays only while the id
|
|
1501
|
+
is the same one (round 18, M1): an origin given another id (`record --replace`) holds a record
|
|
1502
|
+
of its own, which no other project may take back."""
|
|
1503
|
+
same = bool(o.get("memory")) and mem_base(o["memory"]) == mem_base(memory)
|
|
1504
|
+
return dict(o, memory=memory) if same else {
|
|
1505
|
+
k: v for k, v in dict(o, memory=memory).items() if k != "rekeyedFrom"}
|
|
1506
|
+
|
|
1507
|
+
|
|
1508
|
+
def _reclaim(entries: dict[str, dict]) -> list[str]:
|
|
1509
|
+
"""Give a re-keyed memory id back to the project it was taken from (round 16).
|
|
1510
|
+
|
|
1511
|
+
A re-key can only judge the projects it has seen: an `add` in X, or a rebuild over X's root
|
|
1512
|
+
alone, may re-key a v1.95.0 origin into X before the repository Y whose key the legacy store
|
|
1513
|
+
used has ever been read. So ownership is made recoverable instead of predicted: the re-keyed
|
|
1514
|
+
origin records `rekeyedFrom: <Y's key>`, and when Y is read with an origin of the same class
|
|
1515
|
+
key (the store's identity is (classKey, project)), the id moves to Y's first READ origin by
|
|
1516
|
+
(project, retro) and is dropped from X.
|
|
1517
|
+
|
|
1518
|
+
Y counts as read only by an origin that carries provenance (`checkouts`) and is not carried
|
|
1519
|
+
(round 17, F2): one this `add` merged, or this rebuild re-derived. A v1.95.0 origin under
|
|
1520
|
+
Y's key that could not be re-keyed (no provenance, carried `no provenance`) proves nothing
|
|
1521
|
+
about Y: it may be X's own entry whose twin X no longer holds. Round 16 counted any Y origin
|
|
1522
|
+
and reclaimed in the very run that re-keyed, to a project nobody had read. A project with
|
|
1523
|
+
provenance is known and never re-keyed (`_known_projects`), so a read Y origin here was read
|
|
1524
|
+
after the re-key. So whichever verb runs first, the final owner is the project the legacy
|
|
1525
|
+
store keyed, once that project is actually read. When Y already holds a different id for
|
|
1526
|
+
that class, two store records are not one: nothing moves, and that is said. Returns the
|
|
1527
|
+
lines to print."""
|
|
1528
|
+
said: list[str] = []
|
|
1529
|
+
for ck in sorted(entries):
|
|
1530
|
+
origins = entries[ck]["origins"]
|
|
1531
|
+
changed = False
|
|
1532
|
+
for o in sorted(origins, key=_origin_key):
|
|
1533
|
+
src = o.get("rekeyedFrom")
|
|
1534
|
+
if not src or not o.get("memory") or src == o["project"]:
|
|
1535
|
+
continue
|
|
1536
|
+
mine = sorted((y for y in origins if y["project"] == src and y.get("checkouts")
|
|
1537
|
+
and not y.get("carried")), key=_origin_key)
|
|
1538
|
+
if not mine:
|
|
1539
|
+
continue # Y not read this run: a carried or provenance-less origin proves nothing
|
|
1540
|
+
base = mem_base(o["memory"])
|
|
1541
|
+
other = next((y["memory"] for y in origins if y["project"] == src and y.get("memory")
|
|
1542
|
+
and mem_base(y["memory"]) != base), None)
|
|
1543
|
+
if other:
|
|
1544
|
+
said.append(f"{ck}: {base} was re-keyed from {src} into {o['project']}, but {src} now "
|
|
1545
|
+
f"holds its own {mem_base(other)} — nothing moved; two store records are not one")
|
|
1546
|
+
continue
|
|
1547
|
+
keep = _newer_memory(o["memory"], next((y["memory"] for y in origins if y["project"] == src
|
|
1548
|
+
and y.get("memory")), None))
|
|
1549
|
+
new = []
|
|
1550
|
+
for y in origins:
|
|
1551
|
+
if y is o or (y.get("memory") and mem_base(y["memory"]) == base and y["project"] != src):
|
|
1552
|
+
y = {k: v for k, v in y.items() if k not in ("memory", "rekeyedFrom")}
|
|
1553
|
+
elif y is mine[0]:
|
|
1554
|
+
y = dict({k: v for k, v in y.items() if k != "rekeyedFrom"}, memory=keep)
|
|
1555
|
+
new.append(y)
|
|
1556
|
+
said.append(f"{ck}: {keep} goes back to {src}, the project the legacy store keyed it under; "
|
|
1557
|
+
f"{o['project']} {o['retro']} keeps its place without it")
|
|
1558
|
+
origins = new
|
|
1559
|
+
changed = True
|
|
1560
|
+
if changed:
|
|
1561
|
+
entries[ck] = derive(origins)
|
|
1562
|
+
return said
|
|
1563
|
+
|
|
1564
|
+
|
|
1565
|
+
def _one_project_per_memory(entries: dict[str, dict], old: dict[str, dict]) -> list[str]:
|
|
1566
|
+
"""One memory id is one PROJECT's store record (round 13, M/M2), as `_one_class_per_memory`
|
|
1567
|
+
makes it one class's. Two projects may hold one id only while one of them is carried as
|
|
1568
|
+
`checkout holds another project` (the re-attach `record` allows at the same path). When two
|
|
1569
|
+
LIVE projects hold it — that origin's checkout came back — the id stays on the earlier
|
|
1570
|
+
holder: the project the previous index carried as `checkout holds another project` with it
|
|
1571
|
+
(it held the id before the re-attach), else the smaller project key. The other origin keeps
|
|
1572
|
+
its place without the id, and that is named; nothing is duplicated in silence."""
|
|
1573
|
+
first: dict[str, set] = {}
|
|
1574
|
+
for e in old.values():
|
|
1575
|
+
for o in e["origins"]:
|
|
1576
|
+
if o.get("memory") and o.get("carried") == CARRY_FOREIGN:
|
|
1577
|
+
first.setdefault(mem_base(o["memory"]), set()).add(o["project"])
|
|
1578
|
+
said: list[str] = []
|
|
1579
|
+
for ck in sorted(entries):
|
|
1580
|
+
origins = entries[ck]["origins"]
|
|
1581
|
+
live: dict[str, set] = {}
|
|
1582
|
+
for o in origins:
|
|
1583
|
+
if o.get("memory") and o.get("carried") != CARRY_FOREIGN:
|
|
1584
|
+
live.setdefault(mem_base(o["memory"]), set()).add(o["project"])
|
|
1585
|
+
changed = False
|
|
1586
|
+
for base, projects in sorted(live.items()):
|
|
1587
|
+
if len(projects) < 2:
|
|
1588
|
+
continue
|
|
1589
|
+
keep = (sorted(projects & first.get(base, set())) or sorted(projects))[0]
|
|
1590
|
+
out = []
|
|
1591
|
+
for o in origins:
|
|
1592
|
+
if (o.get("memory") and mem_base(o["memory"]) == base and o["project"] != keep
|
|
1593
|
+
and o.get("carried") != CARRY_FOREIGN):
|
|
1594
|
+
said.append(f"{base} is held by {keep} and {o['project']} for {ck}; one id is "
|
|
1595
|
+
f"one project's store record, so it stays on {keep}, which held it "
|
|
1596
|
+
f"first — {o['project']} {o['retro']} keeps its place without it "
|
|
1597
|
+
"(record that project's own id there)")
|
|
1598
|
+
# `rekeyedFrom` goes with the id (round 17, F1): the loader refuses a source
|
|
1599
|
+
# beside no memory id, and would set the whole index aside.
|
|
1600
|
+
o = {k: v for k, v in o.items() if k not in ("memory", "rekeyedFrom")}
|
|
1601
|
+
changed = True
|
|
1602
|
+
out.append(o)
|
|
1603
|
+
origins = out
|
|
1604
|
+
if changed:
|
|
1605
|
+
entries[ck] = derive(origins)
|
|
1606
|
+
return said
|
|
1607
|
+
|
|
1608
|
+
|
|
1609
|
+
def _resolves_in(checkout: str, sha: str) -> bool:
|
|
1610
|
+
"""Does `sha` resolve in this checkout? A git that does not answer counts as yes: the
|
|
1611
|
+
cautious side, which keeps the origin counted rather than calling its checkout a stranger."""
|
|
1612
|
+
try:
|
|
1613
|
+
return commit_resolves(pathlib.Path(checkout), sha)
|
|
1614
|
+
except Refused:
|
|
1615
|
+
return True
|
|
1616
|
+
|
|
1617
|
+
|
|
1618
|
+
def _carry_reason(ck: str, o: dict, run: dict) -> str:
|
|
1619
|
+
"""Why a previous origin this run did not re-derive is carried — every one is (round 11):
|
|
1620
|
+
a rebuild never removes or moves anything on its own. The reason is accurate and decides
|
|
1621
|
+
one thing only: an origin whose every checkout now holds ANOTHER project is not counted in
|
|
1622
|
+
`seen`/`projects`/`status` (`derive`); every other reason is a temporary unavailability.
|
|
1623
|
+
|
|
1624
|
+
Per checkout in the origin's provenance: not discovered, shallow, or owning a problem →
|
|
1625
|
+
`checkout not read`; holding another project AND not holding the origin's own commit →
|
|
1626
|
+
`checkout holds another project` (round 12: the same repository under another key is not a
|
|
1627
|
+
stranger); the
|
|
1628
|
+
entry refused there → `lesson refused`; its retro file not read → `retro file not read`;
|
|
1629
|
+
read, same project, entry absent → `no checkout read holds it`. The most available
|
|
1630
|
+
explanation wins; `checkout holds another project` only when every checkout says so."""
|
|
1631
|
+
k = _origin_key(o)
|
|
1632
|
+
if k in run["unresolved"]:
|
|
1633
|
+
return CARRY_UNRESOLVED if run["unresolved"][k] == o["commit"] else CARRY_CITES_UNRESOLVED
|
|
1634
|
+
prov = o.get("checkouts") or []
|
|
1635
|
+
if not prov:
|
|
1636
|
+
return CARRY_LEGACY
|
|
1637
|
+
states = set()
|
|
1638
|
+
rel = o["retro"].split("#", 1)[0]
|
|
1639
|
+
for c in prov:
|
|
1640
|
+
if c not in run["complete"] or c not in run["project"]:
|
|
1641
|
+
states.add(CARRY_CHECKOUT)
|
|
1642
|
+
elif run["project"][c] != o["project"] and not _resolves_in(c, o["commit"]):
|
|
1643
|
+
# Another project only when the origin's own commit is not there (round 12): the
|
|
1644
|
+
# same repository under another key — a branch, a lost default branch — still holds it.
|
|
1645
|
+
states.add(CARRY_FOREIGN)
|
|
1646
|
+
elif o["retro"] in run["refused"].get(c, ()):
|
|
1647
|
+
states.add(CARRY_REFUSED)
|
|
1648
|
+
elif rel not in run["files"].get(c, ()):
|
|
1649
|
+
states.add(CARRY_FILE)
|
|
1650
|
+
else:
|
|
1651
|
+
states.add(CARRY_GONE)
|
|
1652
|
+
for reason in (CARRY_REFUSED, CARRY_CHECKOUT, CARRY_FILE, CARRY_GONE):
|
|
1653
|
+
if reason in states:
|
|
1654
|
+
return reason
|
|
1655
|
+
return CARRY_FOREIGN
|
|
1656
|
+
|
|
1657
|
+
|
|
1658
|
+
def _attach_memory(entries: dict[str, dict], old: dict[str, dict], notes: list[str]) -> None:
|
|
1659
|
+
"""Memory ids follow the store's own identity, (project, classKey) — never a retro anchor
|
|
1660
|
+
(round 8, M1): a rotated archive or a reworded heading keeps its id, a class key changed
|
|
1661
|
+
inside one entry never inherits another key's id, and two entries under one heading keep one
|
|
1662
|
+
id each. Ids compare on the part before `@`; the highest revision wins (round 9, L3). Within
|
|
1663
|
+
(project, classKey) the id goes on the first origin by (project, retro); `emit-observatory`
|
|
1664
|
+
and `record` both read the origin of the project that holds the id (`_this_origin`), so the
|
|
1665
|
+
three agree. Only when one (project, classKey) held several ids does the exact
|
|
1666
|
+
(project, retro) decide, and only where that match is unique."""
|
|
1667
|
+
by_pc: dict[tuple, dict[str, set]] = {}
|
|
1668
|
+
best: dict[str, str] = {}
|
|
1669
|
+
# (memory base, project) → the key a re-key took that id from, into that project (round 16).
|
|
1670
|
+
# Keyed by the project too (round 17, F1): another project holding the same id was never
|
|
1671
|
+
# re-keyed, and must not be given a source it never had.
|
|
1672
|
+
source: dict[tuple[str, str], str] = {}
|
|
1673
|
+
for e in old.values():
|
|
1674
|
+
for o in e["origins"]:
|
|
1675
|
+
if o.get("memory") and o.get("rekeyedFrom"):
|
|
1676
|
+
source[(mem_base(o["memory"]), o["project"])] = o["rekeyedFrom"]
|
|
1677
|
+
for ck, e in old.items():
|
|
1678
|
+
for o in e["origins"]:
|
|
1679
|
+
m = o.get("memory")
|
|
1680
|
+
if m:
|
|
1681
|
+
by_pc.setdefault((o["project"], ck), {}).setdefault(mem_base(m), set()).add(o["retro"])
|
|
1682
|
+
if mem_base(m) not in best or mem_rev(m) > mem_rev(best[mem_base(m)]):
|
|
1683
|
+
best[mem_base(m)] = m # seeded unconditionally (round 11, #5)
|
|
1684
|
+
for ck, e in list(entries.items()):
|
|
1685
|
+
origins = sorted(e["origins"], key=_origin_key)
|
|
1686
|
+
changed = False
|
|
1687
|
+
for project in sorted({o["project"] for o in origins}):
|
|
1688
|
+
ids = by_pc.get((project, ck), {})
|
|
1689
|
+
mine = [o for o in origins if o["project"] == project]
|
|
1690
|
+
if len(ids) == 1:
|
|
1691
|
+
(base,) = ids
|
|
1692
|
+
mine[0]["memory"] = best[base]
|
|
1693
|
+
if (base, project) in source:
|
|
1694
|
+
mine[0]["rekeyedFrom"] = source[(base, project)]
|
|
1695
|
+
changed = True
|
|
1696
|
+
elif len(ids) > 1:
|
|
1697
|
+
for o in mine:
|
|
1698
|
+
exact = [b for b, retros in ids.items() if o["retro"] in retros]
|
|
1699
|
+
if len(exact) == 1:
|
|
1700
|
+
o["memory"] = best[exact[0]]
|
|
1701
|
+
if (exact[0], project) in source:
|
|
1702
|
+
o["rekeyedFrom"] = source[(exact[0], project)]
|
|
1703
|
+
changed = True
|
|
1704
|
+
if not any(o.get("memory") for o in mine):
|
|
1705
|
+
notes.append(f"{ck} — {project}: several memory ids and none matches a retro "
|
|
1706
|
+
"anchor uniquely; the previous origins keep them")
|
|
1707
|
+
if changed:
|
|
1708
|
+
entries[ck] = derive(origins)
|
|
1709
|
+
|
|
1710
|
+
|
|
1711
|
+
def _one_class_per_memory(entries: dict[str, dict], notes: list[str]) -> None:
|
|
1712
|
+
"""One memory id (compared before `@`) is one store record: it carries its highest revision
|
|
1713
|
+
everywhere it appears, and it belongs to one class key. An index that already holds an id on
|
|
1714
|
+
two keys (the round-7 defect) keeps it on the first and says so."""
|
|
1715
|
+
top: dict[str, str] = {}
|
|
1716
|
+
holders: dict[str, list[str]] = {}
|
|
1717
|
+
for ck in sorted(entries):
|
|
1718
|
+
for o in entries[ck]["origins"]:
|
|
1719
|
+
m = o.get("memory")
|
|
1720
|
+
if m:
|
|
1721
|
+
b = mem_base(m)
|
|
1722
|
+
if b not in top or mem_rev(m) > mem_rev(top[b]):
|
|
1723
|
+
top[b] = m # seeded unconditionally (round 11, #5)
|
|
1724
|
+
if ck not in holders.setdefault(b, []):
|
|
1725
|
+
holders[b].append(ck)
|
|
1726
|
+
for ck in list(entries):
|
|
1727
|
+
origins, changed = [], False
|
|
1728
|
+
for o in entries[ck]["origins"]:
|
|
1729
|
+
m = o.get("memory")
|
|
1730
|
+
if m and holders[mem_base(m)][0] != ck:
|
|
1731
|
+
# The id's source goes with it (round 17, F1), as in `_one_project_per_memory`.
|
|
1732
|
+
o = {k: v for k, v in o.items() if k not in ("memory", "rekeyedFrom")}
|
|
1733
|
+
changed = True
|
|
1734
|
+
notes.append(f"memory id {mem_base(m)} was on {holders[mem_base(m)][0]} and {ck}; one id "
|
|
1735
|
+
f"is one store record, so it stays on {holders[mem_base(m)][0]} only")
|
|
1736
|
+
elif m and m != top[mem_base(m)]:
|
|
1737
|
+
o = dict(o, memory=top[mem_base(m)])
|
|
1738
|
+
changed = True
|
|
1739
|
+
origins.append(o)
|
|
1740
|
+
if changed:
|
|
1741
|
+
entries[ck] = derive(origins)
|
|
1742
|
+
for ck, e in entries.items():
|
|
1743
|
+
for o in e["origins"]:
|
|
1744
|
+
if o.get("memory") and holders[mem_base(o["memory"])][0] != ck:
|
|
1745
|
+
raise Refused(f"memory id {o['memory']} is on two class keys — nothing was written")
|
|
1746
|
+
|
|
1747
|
+
|
|
884
1748
|
def cmd_rebuild(a, idx: pathlib.Path) -> int:
|
|
885
|
-
|
|
886
|
-
|
|
1749
|
+
"""A rebuild never removes or moves anything on its own (rounds 7–11 of release iteration 1;
|
|
1750
|
+
round 11 narrowed the contract under the loop guard).
|
|
1751
|
+
|
|
1752
|
+
The index becomes the lessons derived from every readable retro — under the roots, and in
|
|
1753
|
+
every git worktree of each checkout found — merged with EVERY origin of the previous index
|
|
1754
|
+
this run did not re-derive, CARRIED FORWARD. An origin is one project's copy of a lesson,
|
|
1755
|
+
`(project, retro#anchor)` under its class key; it records its provenance, `checkouts`, the
|
|
1756
|
+
real paths of the checkouts it was derived from (local; `show` and `emit` omit it).
|
|
1757
|
+
|
|
1758
|
+
No implicit removal: the only way an origin leaves the index is `--prune-carried`, narrowed
|
|
1759
|
+
to one reason by `--prune-carried-reason` (which needs `--prune-carried` and offers only the
|
|
1760
|
+
reasons below). No supersede, no move: a memory id stays on the origin that holds it;
|
|
1761
|
+
`_attach_memory` re-attaches it only to the same (project, classKey).
|
|
1762
|
+
|
|
1763
|
+
The project key is the repository's (round 12): the first-parent root of its default
|
|
1764
|
+
branch, computed once per git common dir, so a worktree on a branch that merged an
|
|
1765
|
+
unrelated history, or a checkout sitting on an orphan `gh-pages`, is still its project.
|
|
1766
|
+
|
|
1767
|
+
A carried origin keeps everything recorded on it and is marked `carried: <reason>`
|
|
1768
|
+
(`_carry_reason`): `checkout not read`, `retro file not read`, `no checkout read holds it`,
|
|
1769
|
+
`commit does not resolve`, `retro cites a commit that does not resolve`, `lesson refused`,
|
|
1770
|
+
`no provenance` (one this run proves is the same repository under a new key — no
|
|
1771
|
+
provenance, one re-derived origin of another key at its `retro#anchor`, its commit resolving
|
|
1772
|
+
in that checkout — is re-keyed instead, its memory id following: `_rekey_legacy`) —
|
|
1773
|
+
temporary unavailabilities, counted in `seen`/`projects`/`status` — or
|
|
1774
|
+
`checkout holds another project`, kept but NOT counted: every checkout in its provenance
|
|
1775
|
+
holds another project key AND the origin's own commit does not resolve there (a stranger at
|
|
1776
|
+
the path, a re-initialised history), so neither makes a phantom second project. `add`
|
|
1777
|
+
applies the same rule to this checkout's own old-key origin.
|
|
1778
|
+
|
|
1779
|
+
A shallow clone (`git rev-parse --is-shallow-repository`) is never a source: a problem owned
|
|
1780
|
+
by that checkout, nothing derived under its unstable key, its origins carried as `checkout
|
|
1781
|
+
not read`; a shallow check that does not answer is refused the same way (and, having taught
|
|
1782
|
+
nothing, does not count as something read). Known limit: a project whose root commit
|
|
1783
|
+
changes (re-init, unshallow) starts new store records; the operator re-attaches by
|
|
1784
|
+
`record`-ing the old id on the new origin of the same class key — allowed, because the
|
|
1785
|
+
origin holding it is carried as `checkout holds another project` from that same checkout.
|
|
1786
|
+
The re-key needs the commit to descend from the new key's root and v1.95.0's key for that
|
|
1787
|
+
checkout to equal the origin's, and never touches a project read this run (round 14).
|
|
1788
|
+
One memory id stays on one live project (`_one_project_per_memory`): the earlier holder keeps
|
|
1789
|
+
it, the other is named. Every message — stdout, the session's notes and a refusal on stderr —
|
|
1790
|
+
names checkouts and files by name, never by absolute path (`_redact`, `_say`).
|
|
1791
|
+
|
|
1792
|
+
Retro files are stamped when read and stamped again under the lock: a file that changed in
|
|
1793
|
+
between — an `add` landing mid-run — leaves its checkout not read for this run. Memory ids
|
|
1794
|
+
follow (project, classKey) and compare before `@`; the run refuses to write if a memory id
|
|
1795
|
+
of the previous index would vanish other than by `--prune-carried`, and names an id that only
|
|
1796
|
+
a corrupt line held.
|
|
1797
|
+
|
|
1798
|
+
Exit 0 whenever anything was read — a retro, or a checkout discovered in full — and for a
|
|
1799
|
+
readable root holding no checkout at all (the index is written, every origin carried). Exit 1
|
|
1800
|
+
only when nothing could be read because every root was missing or unreadable — before the
|
|
1801
|
+
session opens, so no index, lock directory or migration is touched — or on a lock or
|
|
1802
|
+
unreadable-index refusal. On the default path the one-time merge of the standalone index,
|
|
1803
|
+
and setting a corrupt index aside (its good lines salvaged), run first, as for every verb."""
|
|
1804
|
+
found, problems, repos = find_retros(a.roots, a.depth)
|
|
1805
|
+
# Messages name checkouts and files, never an absolute path (round 12, E): every root,
|
|
1806
|
+
# checkout and the index itself, as given and as resolved, is shown by its name.
|
|
1807
|
+
# The index's directory and the standalone index are named too: the session's notes and a
|
|
1808
|
+
# refusal go to stderr through the same `_say` (round 13, O).
|
|
1809
|
+
shown = _SHOWN
|
|
1810
|
+
legacy = LEGACY_INDEX()
|
|
1811
|
+
for p in [pathlib.Path(r).expanduser() for r in a.roots] + list(repos):
|
|
1812
|
+
_show(p)
|
|
1813
|
+
for p in (idx, idx.parent, legacy, legacy.parent):
|
|
1814
|
+
_show(p, home_relative=True)
|
|
1815
|
+
def redact(text: str) -> str:
|
|
1816
|
+
return _redact(text, shown)
|
|
1817
|
+
lessons: list[tuple[str, dict]] = []
|
|
1818
|
+
private, seen_files = 0, 0
|
|
1819
|
+
unresolved: list[str] = []
|
|
887
1820
|
keys: dict[pathlib.Path, str] = {}
|
|
1821
|
+
stats: dict[tuple, tuple] = {}
|
|
1822
|
+
run: dict = {"files": {}, "refused": {}, "unresolved": {}, "complete": set(), "project": {}}
|
|
1823
|
+
# A shallow clone is never a source (round 11): its root commit — the project key — moves
|
|
1824
|
+
# whenever its boundary does, so nothing is derived from it until it is deepened.
|
|
1825
|
+
shallow: set[pathlib.Path] = set()
|
|
1826
|
+
unchecked: set[pathlib.Path] = set()
|
|
1827
|
+
for repo in repos:
|
|
1828
|
+
try:
|
|
1829
|
+
if _git(repo, "rev-parse", "--is-shallow-repository") == "true":
|
|
1830
|
+
shallow.add(repo)
|
|
1831
|
+
problems.append((repo, f"{repo.name}: a shallow clone — never a source; deepen it "
|
|
1832
|
+
"(git fetch --unshallow) for its lessons to be read"))
|
|
1833
|
+
except Refused as exc:
|
|
1834
|
+
# A shallow check that does not answer is refused, not trusted (round 12): the
|
|
1835
|
+
# checkout is no source this run, and that is a problem it owns. Unlike a clone that
|
|
1836
|
+
# answered "shallow", nothing was learned from it: it does not count as read.
|
|
1837
|
+
unchecked.add(repo)
|
|
1838
|
+
problems.append((repo, f"{repo.name}: the shallow check did not answer ({exc}) — not "
|
|
1839
|
+
"a source this run"))
|
|
888
1840
|
for repo, retro in found:
|
|
889
|
-
|
|
890
|
-
|
|
1841
|
+
if repo in shallow or repo in unchecked:
|
|
1842
|
+
continue
|
|
1843
|
+
c = checkout_id(repo)
|
|
891
1844
|
try:
|
|
892
1845
|
if repo not in keys:
|
|
893
1846
|
keys[repo] = project_key(repo)
|
|
894
|
-
|
|
895
|
-
|
|
1847
|
+
stats[(repo, retro)] = _stamp(retro)
|
|
1848
|
+
refused: set[str] = set()
|
|
1849
|
+
got, priv, probs = parse_retro(retro, repo, keys[repo], refused)
|
|
1850
|
+
mine, unres = [], []
|
|
896
1851
|
for le in got:
|
|
897
1852
|
if validate(le):
|
|
898
1853
|
probs.append(f"{le['classKey']}: refused by lesson/1")
|
|
1854
|
+
refused.add(le["origin"]["retro"])
|
|
899
1855
|
elif not commit_resolves(repo, le["origin"]["commit"]):
|
|
900
|
-
unres
|
|
901
|
-
probs.append(f"{le['classKey']}: evidence commit does not resolve")
|
|
1856
|
+
unres.append(le)
|
|
902
1857
|
else:
|
|
903
1858
|
mine.append(le)
|
|
904
1859
|
except Refused as exc:
|
|
905
|
-
problems.append(f"{repo.name}: {exc}")
|
|
1860
|
+
problems.append((repo, f"{repo.name}: {exc}"))
|
|
906
1861
|
continue
|
|
907
1862
|
seen_files += 1
|
|
908
1863
|
private += priv
|
|
909
|
-
|
|
910
|
-
|
|
911
|
-
|
|
912
|
-
|
|
913
|
-
for
|
|
914
|
-
|
|
915
|
-
|
|
1864
|
+
run["files"].setdefault(c, set()).add(_relative(repo, retro))
|
|
1865
|
+
run["refused"].setdefault(c, set()).update(refused)
|
|
1866
|
+
for le in unres:
|
|
1867
|
+
run["unresolved"][_origin_key(le["origin"])] = le["origin"]["commit"]
|
|
1868
|
+
unresolved += [f"{repo.name}: {le['classKey']} (commit {le['origin']['commit']})" for le in unres]
|
|
1869
|
+
lessons += [(c, le) for le in mine]
|
|
1870
|
+
problems += [(repo, f"{repo.name}: {p}") for p in probs]
|
|
1871
|
+
# Which project each checkout holds: a path is not an identity (round 10, F1).
|
|
1872
|
+
for repo in repos:
|
|
1873
|
+
if repo in shallow or repo in unchecked:
|
|
1874
|
+
continue
|
|
1875
|
+
try:
|
|
1876
|
+
if repo not in keys:
|
|
1877
|
+
keys[repo] = project_key(repo)
|
|
1878
|
+
run["project"][checkout_id(repo)] = keys[repo]
|
|
1879
|
+
except Refused:
|
|
1880
|
+
continue
|
|
1881
|
+
owners = {owner for owner, _ in problems}
|
|
1882
|
+
run["complete"] = {checkout_id(r) for r, whole in repos.items() if whole and r not in owners}
|
|
1883
|
+
# A shallow checkout was found and read far enough to know it is shallow: something was read.
|
|
1884
|
+
if problems and not seen_files and not run["complete"] and not shallow:
|
|
1885
|
+
for _, problem in problems:
|
|
1886
|
+
print(f" not read: {redact(problem)}")
|
|
1887
|
+
print("lessons: rebuild refused — nothing could be read; the index, its lock directory "
|
|
1888
|
+
"and any migration were not touched", file=sys.stderr)
|
|
916
1889
|
return 1
|
|
917
1890
|
with session(idx, a.default_index) as ss:
|
|
918
|
-
#
|
|
919
|
-
# (
|
|
920
|
-
|
|
1891
|
+
# A retro that changed after it was read — an `add` landing between discovery and this
|
|
1892
|
+
# lock (round 10, H1) — leaves its checkout not read for this run.
|
|
1893
|
+
for (repo, retro), stamp in stats.items():
|
|
1894
|
+
if _stamp(retro) != stamp:
|
|
1895
|
+
problems.append((repo, f"{repo.name}: {_relative(repo, retro)} changed while the "
|
|
1896
|
+
"rebuild ran — read again next run"))
|
|
1897
|
+
run["complete"].discard(checkout_id(repo))
|
|
1898
|
+
# The previous index lends its memory ids and its carried origins. `session` already
|
|
1899
|
+
# refused an unreadable one (its contents may be good) and set a corrupt one aside, its
|
|
1900
|
+
# good lines salvaged; its notes go to stderr as for every verb, and into this report.
|
|
921
1901
|
old = ss.entries
|
|
922
|
-
|
|
923
|
-
|
|
924
|
-
|
|
1902
|
+
index_notes = list(ss.notes)
|
|
1903
|
+
corrupt_name, corrupt_only = _CORRUPT_ONLY.get(str(idx), ("", set()))
|
|
1904
|
+
notes: list[str] = []
|
|
925
1905
|
entries: dict[str, dict] = {}
|
|
926
|
-
# Worktrees of one project hold the same entries: one origin per (project, anchor),
|
|
927
|
-
#
|
|
928
|
-
|
|
929
|
-
|
|
930
|
-
|
|
931
|
-
|
|
932
|
-
|
|
933
|
-
|
|
1906
|
+
# Worktrees of one project hold the same entries: one origin per (project, anchor), its
|
|
1907
|
+
# provenance the checkouts that hold it in this run; where two copies differ the order
|
|
1908
|
+
# below decides, the same way every time.
|
|
1909
|
+
for c, le in sorted(lessons, key=lambda x: (x[0], json.dumps(x[1], sort_keys=True))):
|
|
1910
|
+
merge(entries, [le], c)
|
|
1911
|
+
# An origin v1.95.0 wrote — no provenance, keyed on the smallest root — that is the same
|
|
1912
|
+
# repository as one re-derived here takes its key first (round 13, K), so its memory id
|
|
1913
|
+
# follows and it is never a second project.
|
|
1914
|
+
rekeyed = _rekey_legacy(old, entries, set(run["project"].values()) | _known_projects(old))
|
|
1915
|
+
_attach_memory(entries, old, notes)
|
|
1916
|
+
derived = {(ck, _origin_key(o)) for ck, e in entries.items() for o in e["origins"]}
|
|
1917
|
+
carried, pruned = [], []
|
|
1918
|
+
for ck, e in sorted(old.items()):
|
|
1919
|
+
for o in e["origins"]:
|
|
1920
|
+
if (ck, _origin_key(o)) in derived:
|
|
1921
|
+
continue
|
|
1922
|
+
rec = dict(o, carried=_carry_reason(ck, o, run))
|
|
1923
|
+
if a.prune_carried and a.prune_carried_reason in (None, rec["carried"]):
|
|
1924
|
+
pruned.append((ck, rec))
|
|
1925
|
+
else:
|
|
1926
|
+
carried.append((ck, rec))
|
|
1927
|
+
for ck, o in carried:
|
|
1928
|
+
held = [x for x in entries[ck]["origins"] if _origin_key(x) != _origin_key(o)] if ck in entries else []
|
|
1929
|
+
entries[ck] = derive(held + [o])
|
|
1930
|
+
_one_class_per_memory(entries, notes)
|
|
1931
|
+
reclaimed = _reclaim(entries)
|
|
1932
|
+
shared = _one_project_per_memory(entries, old)
|
|
1933
|
+
# A memory id of the previous index vanishes only through --prune-carried (compared
|
|
1934
|
+
# before `@`). An id only a corrupt line held is counted and named.
|
|
1935
|
+
before = {mem_base(o["memory"]) for e in old.values() for o in e["origins"] if o.get("memory")}
|
|
1936
|
+
after = {mem_base(o["memory"]) for e in entries.values() for o in e["origins"] if o.get("memory")}
|
|
1937
|
+
gone = before - after - {mem_base(o["memory"]) for _, o in pruned if o.get("memory")}
|
|
1938
|
+
if gone:
|
|
1939
|
+
raise Refused(f"rebuild would lose {len(gone)} memory id(s) ({sorted(gone)[:3]}) — "
|
|
1940
|
+
"nothing was written; this is a defect, report it")
|
|
1941
|
+
lost_to_corrupt = sorted(m for m in corrupt_only if mem_base(m) not in after)
|
|
934
1942
|
ss.entries = entries
|
|
935
1943
|
ss.save()
|
|
936
1944
|
print(f"lessons: rebuilt from {seen_files} retro file{'s' if seen_files != 1 else ''} in "
|
|
937
|
-
f"{len(keys)} repositor{'y' if len(keys) == 1 else 'ies'} · {len(entries)}
|
|
938
|
-
f"{
|
|
939
|
-
|
|
940
|
-
|
|
1945
|
+
f"{len(keys)} repositor{'y' if len(keys) == 1 else 'ies'} · {len(entries)} "
|
|
1946
|
+
f"lesson{'' if len(entries) == 1 else 's'} · {private} private entries skipped · "
|
|
1947
|
+
f"{len(unresolved)} with a commit that does not resolve · {len(carried)} origin"
|
|
1948
|
+
f"{'' if len(carried) == 1 else 's'} carried forward (an origin leaves the index only "
|
|
1949
|
+
"by --prune-carried)")
|
|
1950
|
+
for ck, o in carried:
|
|
1951
|
+
print(f" carried ({o['carried']}): {ck} — {o['project']} {o['retro']}"
|
|
1952
|
+
+ (f" · {o['memory']}" if o.get("memory") else ""))
|
|
1953
|
+
for ck, o in pruned:
|
|
1954
|
+
print(f" pruned ({o['carried']}): {ck} — {o['project']} {o['retro']}")
|
|
1955
|
+
for u in unresolved:
|
|
1956
|
+
print(f" skipped, commit does not resolve: {u}")
|
|
1957
|
+
for n in rekeyed:
|
|
1958
|
+
print(f" re-keyed: {redact(n)}")
|
|
1959
|
+
for n in reclaimed:
|
|
1960
|
+
print(f" reclaimed: {redact(n)}")
|
|
1961
|
+
for n in shared:
|
|
1962
|
+
print(f" memory: {redact(n)}")
|
|
1963
|
+
for n in notes:
|
|
1964
|
+
print(f" memory: {redact(n)}")
|
|
1965
|
+
for m in lost_to_corrupt:
|
|
1966
|
+
print(f" memory: {m} is held only by a bad line, kept in {corrupt_name} — not in the live index")
|
|
1967
|
+
for n in index_notes:
|
|
1968
|
+
print(f" previous index: {redact(n)}")
|
|
1969
|
+
for _, p in problems:
|
|
1970
|
+
print(f" not read: {redact(p)}")
|
|
941
1971
|
return 0
|
|
942
1972
|
|
|
943
1973
|
|
|
1974
|
+
def _stamp(path: pathlib.Path) -> tuple:
|
|
1975
|
+
"""A retro file's (mtime_ns, size, inode) — compared under the lock to see it changed."""
|
|
1976
|
+
try:
|
|
1977
|
+
st = path.stat()
|
|
1978
|
+
except OSError:
|
|
1979
|
+
return ()
|
|
1980
|
+
return (st.st_mtime_ns, st.st_size, st.st_ino)
|
|
1981
|
+
|
|
1982
|
+
|
|
944
1983
|
def cmd_candidates(a, idx: pathlib.Path) -> int:
|
|
945
1984
|
with session(idx, a.default_index) as ss:
|
|
946
1985
|
entries = ss.entries
|
|
@@ -959,7 +1998,9 @@ def _this_origin(e: dict, root: pathlib.Path) -> dict:
|
|
|
959
1998
|
if not mine:
|
|
960
1999
|
raise Refused(f"{e['id']} has no origin in this repository — emit and record from the "
|
|
961
2000
|
"project that learned it")
|
|
962
|
-
|
|
2001
|
+
# The origin that holds this project's memory id first, then the first by (project, retro):
|
|
2002
|
+
# emit revises the record the store already has, and record writes where emit reads (round 9).
|
|
2003
|
+
return sorted(mine, key=lambda o: (not o.get("memory"), _origin_key(o)))[0]
|
|
963
2004
|
|
|
964
2005
|
|
|
965
2006
|
def cmd_emit(a, idx: pathlib.Path) -> int:
|
|
@@ -975,9 +2016,11 @@ def cmd_emit(a, idx: pathlib.Path) -> int:
|
|
|
975
2016
|
Either way the project travels as the same pseudonymous key the index holds, never a
|
|
976
2017
|
`project:<slug>` id, and the commit as `commit:` + 12 hex — the store's redactor removes a
|
|
977
2018
|
40-hex SHA as a credential shape and keeps exactly that short form."""
|
|
2019
|
+
root = repo_root(pathlib.Path(a.root or "."))
|
|
2020
|
+
refuse_shallow(root)
|
|
978
2021
|
with session(idx, a.default_index) as ss:
|
|
979
2022
|
e = _resolve(ss.entries, a.id)
|
|
980
|
-
o = _this_origin(e,
|
|
2023
|
+
o = _this_origin(e, root)
|
|
981
2024
|
le = o["lesson"]
|
|
982
2025
|
if a.tool == "observatory_lesson":
|
|
983
2026
|
args = {"owner": OWNER, "classKey": le["classKey"], "title": le["title"],
|
|
@@ -1015,6 +2058,7 @@ def _ledger_line(ledger: pathlib.Path | None, line: str) -> None:
|
|
|
1015
2058
|
|
|
1016
2059
|
def cmd_record(a, idx: pathlib.Path) -> int:
|
|
1017
2060
|
root = repo_root(pathlib.Path(a.root or "."))
|
|
2061
|
+
refuse_shallow(root)
|
|
1018
2062
|
ledger = pathlib.Path(a.ledger) if a.ledger else (root / DEFAULT_LEDGER)
|
|
1019
2063
|
ledger = ledger if ledger.exists() or a.ledger else None
|
|
1020
2064
|
try:
|
|
@@ -1049,10 +2093,63 @@ def cmd_record(a, idx: pathlib.Path) -> int:
|
|
|
1049
2093
|
file=sys.stderr)
|
|
1050
2094
|
return 2
|
|
1051
2095
|
memory = f"{mid}@{int(rev)}"
|
|
1052
|
-
|
|
2096
|
+
# Only an id the loader itself accepts is written (round 12, H): `mem:`, a quote or a
|
|
2097
|
+
# backslash would make the next read set the whole index aside.
|
|
2098
|
+
if not MEM_RE.fullmatch(memory):
|
|
2099
|
+
print(f"lessons: {mid!r} is not a memory id this index can hold (mem:<id>@<rev>, no "
|
|
2100
|
+
"quote, backslash or space) — nothing written", file=sys.stderr)
|
|
2101
|
+
return 2
|
|
2102
|
+
# One id is one PROJECT's record (round 12, C): an id another project's origin holds is
|
|
2103
|
+
# refused, naming that project — unless that origin is carried as `checkout holds another
|
|
2104
|
+
# project` FROM THIS CHECKOUT (round 13, M/M2): a re-init at the same path, the known
|
|
2105
|
+
# root-commit limit, where the operator re-attaches the old id on purpose. A stranger
|
|
2106
|
+
# elsewhere never takes it.
|
|
2107
|
+
here = checkout_id(root)
|
|
2108
|
+
# A holder that took the id by a re-key FROM this project gives it back (round 16).
|
|
2109
|
+
reclaim = [y for y in e["origins"] if y["project"] != o["project"]
|
|
2110
|
+
and mem_base(y.get("memory") or "") == mid and y.get("rekeyedFrom") == o["project"]]
|
|
2111
|
+
holders = sorted({y["project"] for y in e["origins"] if y["project"] != o["project"]
|
|
2112
|
+
and mem_base(y.get("memory") or "") == mid
|
|
2113
|
+
and not any(y is r for r in reclaim)
|
|
2114
|
+
and not (y.get("carried") == CARRY_FOREIGN and here in (y.get("checkouts") or []))})
|
|
2115
|
+
if holders:
|
|
2116
|
+
print(f"lessons: {mid} is the store record of project {', '.join(holders)} for "
|
|
2117
|
+
f"{e['classKey']} — refused for {o['project']}, nothing written", file=sys.stderr)
|
|
2118
|
+
return 1
|
|
2119
|
+
# One id is one store record of one class (round 10, G1): an answer naming an id another
|
|
2120
|
+
# class key holds is a stale or misplaced answer — refused, nothing written, or `emit`
|
|
2121
|
+
# for this class would revise the other class's record.
|
|
2122
|
+
others = sorted(ck for ck, x in entries.items() if ck != e["classKey"]
|
|
2123
|
+
and any(mem_base(y.get("memory") or "") == mid for y in x["origins"]))
|
|
2124
|
+
if others:
|
|
2125
|
+
print(f"lessons: {mid} is the store record of {', '.join(others)} — refused for "
|
|
2126
|
+
f"{e['classKey']}, nothing written", file=sys.stderr)
|
|
2127
|
+
return 1
|
|
2128
|
+
# A revision never goes back (G1b): a replayed older answer is refused.
|
|
2129
|
+
held = max((mem_rev(y["memory"]) for y in e["origins"]
|
|
2130
|
+
if y.get("memory") and mem_base(y["memory"]) == mid), default=0)
|
|
2131
|
+
if int(rev) < held:
|
|
2132
|
+
print(f"lessons: {mid} is at revision {held}; revision {int(rev)} is older — refused, "
|
|
2133
|
+
"nothing written", file=sys.stderr)
|
|
2134
|
+
return 1
|
|
2135
|
+
# Replacing the id this origin already holds is a deliberate act (round 11, R5): without
|
|
2136
|
+
# --replace the old store record would be orphaned in silence.
|
|
2137
|
+
if o.get("memory") and mem_base(o["memory"]) != mid and not a.replace:
|
|
2138
|
+
print(f"lessons: this origin holds {o['memory']}; {mid} would replace it — pass "
|
|
2139
|
+
"--replace to do that on purpose; nothing written", file=sys.stderr)
|
|
2140
|
+
return 1
|
|
2141
|
+
# The same id elsewhere in this entry takes the new revision too: one record, one revision.
|
|
2142
|
+
# A re-keyed holder this project reclaims drops it instead (round 16).
|
|
2143
|
+
origins = [{k: v for k, v in x.items() if k not in ("memory", "rekeyedFrom")}
|
|
2144
|
+
if any(x is r for r in reclaim) else
|
|
2145
|
+
_given(x, memory) if _origin_key(x) == _origin_key(o)
|
|
2146
|
+
or (x.get("memory") and mem_base(x["memory"]) == mid) else x
|
|
1053
2147
|
for x in e["origins"]]
|
|
1054
2148
|
entries[e["classKey"]] = derive(origins)
|
|
1055
2149
|
ss.save()
|
|
2150
|
+
for y in reclaim:
|
|
2151
|
+
print(f" reclaimed: {mid} goes back to {o['project']}, the project it was re-keyed from; "
|
|
2152
|
+
f"{y['project']} {y['retro']} keeps its place without it")
|
|
1056
2153
|
line = f"event: lessons — recorded {e['id']} as {memory} — {_now()}"
|
|
1057
2154
|
_ledger_line(ledger, line)
|
|
1058
2155
|
print(line)
|
|
@@ -1062,6 +2159,10 @@ def cmd_record(a, idx: pathlib.Path) -> int:
|
|
|
1062
2159
|
def cmd_show(a, idx: pathlib.Path) -> int:
|
|
1063
2160
|
with session(idx, a.default_index) as ss:
|
|
1064
2161
|
e = _resolve(ss.entries, a.id)
|
|
2162
|
+
# Provenance is local machine state — absolute checkout paths — and stays in the index file:
|
|
2163
|
+
# `show` prints the entry without it (round 10, G3), like everything that leaves the index.
|
|
2164
|
+
e = dict(e, origins=[{k: v for k, v in o.items() if k not in ("checkouts", "worktreeOf")}
|
|
2165
|
+
for o in e["origins"]])
|
|
1065
2166
|
print(json.dumps(e, ensure_ascii=True, indent=1, sort_keys=True))
|
|
1066
2167
|
return 0
|
|
1067
2168
|
|
|
@@ -1096,6 +2197,11 @@ def main(argv: list[str]) -> int:
|
|
|
1096
2197
|
rb = sub.add_parser("rebuild", parents=[common], help="re-derive the index from the retros")
|
|
1097
2198
|
rb.add_argument("--roots", nargs="+", required=True)
|
|
1098
2199
|
rb.add_argument("--depth", type=int, default=3, help="directory levels below each root")
|
|
2200
|
+
rb.add_argument("--prune-carried", action="store_true",
|
|
2201
|
+
help="remove, on purpose, every origin the rebuild would carry forward — the "
|
|
2202
|
+
"only way an origin leaves the index")
|
|
2203
|
+
rb.add_argument("--prune-carried-reason", choices=PRUNE_REASONS, default=None,
|
|
2204
|
+
help="with --prune-carried: prune only the origins carried for this reason")
|
|
1099
2205
|
c = sub.add_parser("candidates", parents=[common], help="lessons two or more projects hold")
|
|
1100
2206
|
c.add_argument("--min-projects", type=int, default=2)
|
|
1101
2207
|
sh = sub.add_parser("show", parents=[common], help="one entry with its origins")
|
|
@@ -1111,16 +2217,29 @@ def main(argv: list[str]) -> int:
|
|
|
1111
2217
|
rc.add_argument("--answer", required=True, help="the tool's JSON answer, a file or -")
|
|
1112
2218
|
rc.add_argument("--ledger", help=f"the run ledger (default {DEFAULT_LEDGER} when it exists)")
|
|
1113
2219
|
rc.add_argument("--root")
|
|
2220
|
+
rc.add_argument("--replace", action="store_true",
|
|
2221
|
+
help="replace a different memory id this origin already holds")
|
|
1114
2222
|
sub.add_parser("path", parents=[common], help="the index this invocation would use")
|
|
1115
2223
|
a = p.parse_args(argv)
|
|
1116
2224
|
if a.cmd == "add" and a.retro and not (a.all or a.entry):
|
|
1117
2225
|
p.error("add --retro needs --entry ANCHOR or --all")
|
|
1118
2226
|
if a.cmd == "rebuild" and a.depth < 0:
|
|
1119
2227
|
p.error("rebuild --depth must be non-negative")
|
|
2228
|
+
if a.cmd == "rebuild" and a.prune_carried_reason and not a.prune_carried:
|
|
2229
|
+
p.error("--prune-carried-reason narrows --prune-carried; it does nothing alone")
|
|
1120
2230
|
idx = index_path(a.index)
|
|
1121
2231
|
if a.cmd == "path":
|
|
1122
|
-
print(idx)
|
|
2232
|
+
print(idx) # the one verb whose answer IS the path
|
|
1123
2233
|
return 0
|
|
2234
|
+
# Every verb's messages name files and directories, never an absolute path (round 14, L3).
|
|
2235
|
+
legacy = LEGACY_INDEX()
|
|
2236
|
+
for known in (idx, idx.parent, legacy, legacy.parent):
|
|
2237
|
+
_show(known, home_relative=True)
|
|
2238
|
+
_show(pathlib.Path(getattr(a, "root", None) or "."))
|
|
2239
|
+
for given in (getattr(a, "retro", None), getattr(a, "json", None), getattr(a, "answer", None),
|
|
2240
|
+
getattr(a, "ledger", None)):
|
|
2241
|
+
if given and given != "-":
|
|
2242
|
+
_show(pathlib.Path(given))
|
|
1124
2243
|
# The default path (no --index, no $TASK_PIPELINE_LESSONS) is the one a standalone index
|
|
1125
2244
|
# may still need merging into: `session` does it, once, under both directories' locks.
|
|
1126
2245
|
a.default_index = not a.index and not os.environ.get(ENV_INDEX)
|
|
@@ -1129,7 +2248,7 @@ def main(argv: list[str]) -> int:
|
|
|
1129
2248
|
try:
|
|
1130
2249
|
return verbs[a.cmd](a, idx)
|
|
1131
2250
|
except Refused as exc:
|
|
1132
|
-
print(f"lessons: {exc}", file=sys.stderr)
|
|
2251
|
+
print(f"lessons: {_say(str(exc))}", file=sys.stderr)
|
|
1133
2252
|
return 1
|
|
1134
2253
|
|
|
1135
2254
|
|