spexcode 0.2.7 → 0.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +56 -44
- package/package.json +3 -3
- package/spec-cli/bin/spex.mjs +2 -2
- package/spec-cli/hooks/dispatch.sh +1 -1
- package/spec-cli/hooks/harness.sh +26 -6
- package/spec-cli/src/attach.ts +2 -2
- package/spec-cli/src/cli.ts +695 -536
- package/spec-cli/src/client.ts +31 -30
- package/spec-cli/src/contract-filter.ts +1 -1
- package/spec-cli/src/doctor.ts +40 -13
- package/spec-cli/src/gateway.ts +11 -7
- package/spec-cli/src/git.ts +2 -2
- package/spec-cli/src/{board.ts → graph.ts} +44 -14
- package/spec-cli/src/{boardCache.ts → graphCache.ts} +41 -21
- package/spec-cli/src/{boardDelta.ts → graphDelta.ts} +1 -1
- package/spec-cli/src/graphStream.ts +288 -0
- package/spec-cli/src/guide.ts +123 -96
- package/spec-cli/src/harness-select.ts +2 -2
- package/spec-cli/src/harness.ts +30 -14
- package/spec-cli/src/help.ts +289 -384
- package/spec-cli/src/hooks.ts +1 -1
- package/spec-cli/src/index.ts +130 -103
- package/spec-cli/src/init.ts +9 -9
- package/spec-cli/src/issues.ts +89 -33
- package/spec-cli/src/layout.ts +5 -5
- package/spec-cli/src/lint.ts +73 -8
- package/spec-cli/src/localIssues.ts +42 -60
- package/spec-cli/src/materialize.ts +1 -1
- package/spec-cli/src/mentions.ts +15 -15
- package/spec-cli/src/migrate-table.ts +397 -0
- package/spec-cli/src/migrate.ts +386 -0
- package/spec-cli/src/ranker.ts +30 -4
- package/spec-cli/src/reaper.ts +117 -0
- package/spec-cli/src/search.bench.mjs +10 -10
- package/spec-cli/src/search.ts +1 -1
- package/spec-cli/src/sessions.ts +244 -138
- package/spec-cli/src/specs.ts +25 -15
- package/spec-cli/src/supervise.ts +2 -2
- package/spec-cli/src/tree.ts +5 -5
- package/spec-cli/src/uninstall.ts +4 -4
- package/spec-cli/templates/hooks/post-checkout +1 -1
- package/spec-cli/templates/hooks/post-merge +3 -3
- package/spec-cli/templates/hooks/pre-commit +9 -9
- package/spec-cli/templates/hooks/prepare-commit-msg +1 -1
- package/spec-cli/templates/spec/project/{.config → .plugins}/core/idle/idle.sh +2 -2
- package/spec-cli/templates/spec/project/{.config → .plugins}/core/mark-active/mark-active.sh +7 -0
- package/spec-cli/templates/spec/project/{.config → .plugins}/core/mark-active/spec.md +2 -0
- package/spec-cli/templates/spec/project/{.config → .plugins}/core/session-fail/fail.sh +1 -1
- package/spec-cli/templates/spec/project/{.config → .plugins}/core/spec-first/spec-first.sh +1 -1
- package/spec-cli/templates/spec/project/{.config → .plugins}/core/spec-of-file/spec-of-file.sh +2 -2
- package/spec-cli/templates/spec/project/.plugins/core/spec.md +21 -0
- package/spec-cli/templates/spec/project/{.config → .plugins}/core/stop-gate/stop-gate.sh +21 -21
- package/spec-cli/templates/spec/project/{.config → .plugins}/distill/spec.md +2 -2
- package/spec-cli/templates/spec/project/{.config → .plugins}/extract/spec.md +4 -4
- package/spec-cli/templates/spec/project/{.config → .plugins}/forge-link/spec.md +1 -1
- package/spec-cli/templates/spec/project/{.config → .plugins}/regroup/spec.md +2 -2
- package/spec-cli/templates/spec/project/{.config → .plugins}/reproduce-before-fix/spec.md +3 -3
- package/spec-cli/templates/spec/project/{.config → .plugins}/spec.md +4 -4
- package/spec-cli/templates/spec/project/.plugins/supervisor/spec.md +8 -0
- package/spec-cli/templates/spec/project/{.config → .plugins}/tidy/spec.md +1 -1
- package/spec-cli/templates/spec/project/spec.md +1 -1
- package/spec-dashboard/dist/assets/{Dashboard-P0B9ukSG.js → Dashboard-C7Bzsv86.js} +9 -9
- package/spec-dashboard/dist/assets/EvalsPage-DKZZIdHq.js +2 -0
- package/spec-dashboard/dist/assets/{FoldToggle-BuQ0lokE.js → FoldToggle-D5iB4Ac2.js} +1 -1
- package/spec-dashboard/dist/assets/{IssuesPage-H-D8aHEl.js → IssuesPage-CMFTsQhg.js} +1 -1
- package/spec-dashboard/dist/assets/{MobileApp-oZXIeCPb.js → MobileApp-DwuTKgdP.js} +1 -1
- package/spec-dashboard/dist/assets/{SessionInterface-Blr_MEdU.js → SessionInterface-CBS5_cmK.js} +3 -3
- package/spec-dashboard/dist/assets/{SessionWindow-LcCzBMU7.js → SessionWindow-CqAnjWfI.js} +2 -2
- package/spec-dashboard/dist/assets/{Settings-_yOye-In.js → Settings-BW5f0OaW.js} +1 -1
- package/spec-dashboard/dist/assets/{index-uGs9v_9o.css → index-Cc26X4ce.css} +1 -1
- package/spec-dashboard/dist/assets/{index-BhIslAau.js → index-Ce0wDyQS.js} +8 -8
- package/spec-dashboard/dist/index.html +2 -2
- package/{spec-yatsu → spec-eval}/src/cache.ts +8 -5
- package/{spec-yatsu → spec-eval}/src/cli.ts +164 -95
- package/{spec-yatsu → spec-eval}/src/evaltab.ts +24 -24
- package/{spec-yatsu → spec-eval}/src/filing.ts +7 -5
- package/{spec-yatsu → spec-eval}/src/freshness.ts +50 -24
- package/{spec-yatsu → spec-eval}/src/scenariofresh.ts +22 -17
- package/{spec-yatsu/src/yatsu.ts → spec-eval/src/scenarios.ts} +41 -26
- package/{spec-yatsu/src/proof.ts → spec-eval/src/sessioneval.ts} +59 -59
- package/{spec-yatsu → spec-eval}/src/sidecar.ts +7 -1
- package/{spec-yatsu → spec-eval}/src/timeline.ts +1 -1
- package/spec-forge/src/__fixtures__/github-forge.json +9 -9
- package/spec-forge/src/cli.ts +14 -13
- package/spec-forge/src/{needs-yatsu-eval.ts → needs-eval.ts} +6 -6
- package/spec-cli/src/boardStream.ts +0 -179
- package/spec-cli/templates/spec/project/.config/core/spec.md +0 -17
- package/spec-cli/templates/spec/project/.config/supervisor/spec.md +0 -8
- package/spec-dashboard/dist/assets/EvalsPage-BrvAGyc4.js +0 -2
- /package/spec-cli/templates/presets/careful/{.config → .plugins}/clarify-before-code/spec.md +0 -0
- /package/spec-cli/templates/spec/project/{.config → .plugins}/core/idle/spec.md +0 -0
- /package/spec-cli/templates/spec/project/{.config → .plugins}/core/session-fail/spec.md +0 -0
- /package/spec-cli/templates/spec/project/{.config → .plugins}/core/spec-first/spec.md +0 -0
- /package/spec-cli/templates/spec/project/{.config → .plugins}/core/spec-of-file/spec.md +0 -0
- /package/spec-cli/templates/spec/project/{.config → .plugins}/core/stop-gate/spec.md +0 -0
- /package/spec-cli/templates/spec/project/{.config → .plugins}/distill/digest.mjs +0 -0
- /package/spec-cli/templates/spec/project/{.config → .plugins}/memory-hygiene/spec.md +0 -0
|
@@ -2,7 +2,7 @@ import { relative, dirname } from 'node:path'
|
|
|
2
2
|
import { repoRoot, driftIndex, historyIndex, type DriftIndex, type HistoryIndex } from '../../spec-cli/src/git.js'
|
|
3
3
|
import { loadSpecs } from '../../spec-cli/src/specs.js'
|
|
4
4
|
import { loadEvalRemarkTracks, trackKey, type RemarkTrack, type Issue, type Reply } from '../../spec-cli/src/issues.js'
|
|
5
|
-
import {
|
|
5
|
+
import { evalNodes, type EvalNode } from './scenarios.js'
|
|
6
6
|
import { readSidecar, applyRetractions, evidenceOf, isJsonBlob, type Verdict, type EvidenceKind, type Retraction } from './sidecar.js'
|
|
7
7
|
import { staleAxes, codeDrift, contentProbeFor, type StaleAxis } from './freshness.js'
|
|
8
8
|
import { scenarioIndex, type ScenarioIndex } from './scenariofresh.js'
|
|
@@ -18,7 +18,7 @@ export type EvidenceView = { hash: string; kind: EvidenceKind; state: 'present'
|
|
|
18
18
|
// per-reading display attachment.
|
|
19
19
|
export type RemarkView = {
|
|
20
20
|
rid: string
|
|
21
|
-
ref: string // `<thread-id>#<rid>` — the address `spex resolve`/`spex retract` take
|
|
21
|
+
ref: string // `<thread-id>#<rid>` — the address `spex remark resolve`/`spex remark retract` take
|
|
22
22
|
by: string
|
|
23
23
|
at: string
|
|
24
24
|
body: string
|
|
@@ -53,7 +53,7 @@ export type EvalEntry = {
|
|
|
53
53
|
fresh: boolean
|
|
54
54
|
staleAxes: StaleAxis[]
|
|
55
55
|
blobState: 'present' | 'miss' | 'none'
|
|
56
|
-
// the code axis's drift detail for a code-stale reading ([[
|
|
56
|
+
// the code axis's drift detail for a code-stale reading ([[eval-core]]'s codeDrift): each governed file
|
|
57
57
|
// that moved since this reading + how many commits behind, so the eval detail can EXPLAIN the staleness
|
|
58
58
|
// ("EvalsFeed.jsx +3") rather than just flag it. Absent when the reading isn't code-stale.
|
|
59
59
|
codeDrift?: { file: string; behind: number }[]
|
|
@@ -89,16 +89,16 @@ export type DanglingTrack = { scenario: string; threadId: string; thread: Issue;
|
|
|
89
89
|
|
|
90
90
|
export type ScenarioInfo = { name: string; expected: string; tags?: string[]; code?: string[] }
|
|
91
91
|
|
|
92
|
-
// `
|
|
92
|
+
// `hasEvalFile` distinguishes a node that declares no scenarios (no eval.md) from one that declares some but
|
|
93
93
|
// has no readings yet — the tab says different things for each. `scenarios` is the declared set; `readings`
|
|
94
94
|
// is NEWEST-FIRST (the sidecar is append-only oldest→newest; the tab leads with the latest measurement,
|
|
95
95
|
// like the history tab).
|
|
96
96
|
export type EvalTimeline = {
|
|
97
97
|
node: string
|
|
98
|
-
|
|
98
|
+
hasEvalFile: boolean
|
|
99
99
|
scenarios: ScenarioInfo[]
|
|
100
100
|
readings: EvalEntry[]
|
|
101
|
-
// retraction events ([[
|
|
101
|
+
// retraction events ([[eval-core]]'s retract verb), newest first — the sanctioned-undo TRACE. `readings`
|
|
102
102
|
// above is already the effective view (a retracted reading is dropped from the scoreboard everywhere);
|
|
103
103
|
// this list is how a surface still shows that the undo happened: which (scenario, ts) was withdrawn,
|
|
104
104
|
// by whom, why. Additive — a consumer that ignores it sees exactly the effective scoreboard.
|
|
@@ -115,13 +115,13 @@ export type EvalContext = {
|
|
|
115
115
|
idx: DriftIndex
|
|
116
116
|
hidx: HistoryIndex
|
|
117
117
|
scidx: ScenarioIndex // per-scenario block-change history ([[scenariofresh]]) — the SCENARIO axis, built ONCE per build
|
|
118
|
-
ynodes:
|
|
118
|
+
ynodes: EvalNode[]
|
|
119
119
|
// the trunk remark tracks ([[remark-teeth]]), keyed (node, scenario) — loaded ONCE per board/proof build
|
|
120
120
|
// and reused for every node, so the fold never re-reads the issue store per node.
|
|
121
121
|
remarks: Map<string, RemarkTrack>
|
|
122
122
|
}
|
|
123
123
|
|
|
124
|
-
// build the shared context with ONE
|
|
124
|
+
// build the shared context with ONE eval-file walk, reusing the caller's already-computed specs + the two
|
|
125
125
|
// HEAD-keyed git indices (drift for the code axis, history for the rename-safe scenario axis — both warm
|
|
126
126
|
// hits, loadSpecs already derived them). The remark tracks are the fourth, non-git freshness input
|
|
127
127
|
// ([[remark-teeth]]); a caller that omits them gets a live load, so a bare evalTimeline still has teeth.
|
|
@@ -131,28 +131,28 @@ export async function evalContext(
|
|
|
131
131
|
idx: DriftIndex,
|
|
132
132
|
hidx: HistoryIndex,
|
|
133
133
|
remarks?: Map<string, RemarkTrack>,
|
|
134
|
-
ynodes?:
|
|
134
|
+
ynodes?: EvalNode[], // the hot board build precomputes these off the event loop (evalNodesAsync); a bare caller walks sync
|
|
135
135
|
): Promise<EvalContext> {
|
|
136
|
-
const nodes = ynodes ??
|
|
137
|
-
const scidx = await scenarioIndex(root, nodes.map((n) => n.
|
|
136
|
+
const nodes = ynodes ?? evalNodes(root)
|
|
137
|
+
const scidx = await scenarioIndex(root, nodes.map((n) => n.evalPath))
|
|
138
138
|
return { root, specs, idx, hidx, scidx, ynodes: nodes, remarks: remarks ?? loadEvalRemarkTracks() }
|
|
139
139
|
}
|
|
140
140
|
|
|
141
141
|
export async function evalTimeline(id: string, ctx?: EvalContext): Promise<EvalTimeline> {
|
|
142
142
|
const root = ctx?.root ?? repoRoot()
|
|
143
|
-
// short-circuit a non-
|
|
144
|
-
// this is the common case and must stay cheap (a list the size of the few
|
|
145
|
-
const ynode = (ctx?.ynodes ??
|
|
146
|
-
if (!ynode) return { node: id,
|
|
143
|
+
// short-circuit a non-measurable node on the (short) eval-file walk — the board attaches `evals` to every node, so
|
|
144
|
+
// this is the common case and must stay cheap (a list the size of the few measurable nodes, not the whole tree).
|
|
145
|
+
const ynode = (ctx?.ynodes ?? evalNodes(root)).find((n) => n.id === id)
|
|
146
|
+
if (!ynode) return { node: id, hasEvalFile: false, scenarios: [], readings: [], retractions: [], dangling: [] }
|
|
147
147
|
// the governed `code:` files are the freshness CODE axis; read them from the canonical spec loader so a
|
|
148
|
-
// reparent/rename is seen the same way `spex lint` and `spex
|
|
148
|
+
// reparent/rename is seen the same way `spex spec lint` and `spex eval add` see it (joined by directory).
|
|
149
149
|
const specs = ctx?.specs ?? await loadSpecs()
|
|
150
150
|
const codeFiles = specs.find((s) => dirname(s.path) === relative(root, ynode.dir))?.code ?? []
|
|
151
151
|
const idx = ctx?.idx ?? await driftIndex(root)
|
|
152
152
|
const hidx = ctx?.hidx ?? await historyIndex(root)
|
|
153
153
|
// the SCENARIO axis: per-scenario block-change history, built once per HEAD (cached). A bare call builds it
|
|
154
|
-
// for the WHOLE
|
|
155
|
-
const scidx = ctx?.scidx ?? await scenarioIndex(root, (ctx?.ynodes ??
|
|
154
|
+
// for the WHOLE measurable set (not just this node) so the shared HEAD-keyed cache is complete for later callers.
|
|
155
|
+
const scidx = ctx?.scidx ?? await scenarioIndex(root, (ctx?.ynodes ?? evalNodes(root)).map((n) => n.evalPath))
|
|
156
156
|
const byName = new Map(ynode.scenarios.map((s) => [s.name, s])) // join each reading to its scenario's expected + code
|
|
157
157
|
// the trunk remark track per scenario ([[remark-teeth]]) — the non-git freshness input, fed to the teeth.
|
|
158
158
|
const tracks = ctx?.remarks ?? loadEvalRemarkTracks()
|
|
@@ -165,7 +165,7 @@ export async function evalTimeline(id: string, ctx?: EvalContext): Promise<EvalT
|
|
|
165
165
|
// one raw sidecar read: the effective readings feed the scoreboard rows below; the retraction events ride
|
|
166
166
|
// along as the undo trace (newest-first, like the readings).
|
|
167
167
|
const { readings: rawReadings, retractions } = readSidecar(ynode.sidecarPath)
|
|
168
|
-
// the off-history content fallback ([[
|
|
168
|
+
// the off-history content fallback ([[eval-core]]): fed to both git axes so a rebased/folded-away
|
|
169
169
|
// anchor with byte-identical governed content reads fresh. Lazy — an in-history reading never probes.
|
|
170
170
|
const probe = contentProbeFor(root)
|
|
171
171
|
const readings: EvalEntry[] = applyRetractions(rawReadings, retractions).map((r) => {
|
|
@@ -175,8 +175,8 @@ export async function evalTimeline(id: string, ctx?: EvalContext): Promise<EvalT
|
|
|
175
175
|
// remark makes it remark-stale (T1). Display attachment (which reading each remark pins to) is a separate
|
|
176
176
|
// read-time overlay below; freshness never depends on that pin.
|
|
177
177
|
const cf = sc?.code?.length ? sc.code : codeFiles
|
|
178
|
-
const axes = staleAxes(r, cf, ynode.
|
|
179
|
-
remarksFor(r.scenario).map((rm) => ({ resolved: !!rm.resolved, resolvedAt: rm.resolvedAt })), probe)
|
|
178
|
+
const axes = staleAxes(r, cf, ynode.evalPath, idx, scidx,
|
|
179
|
+
remarksFor(r.scenario).map((rm) => ({ resolved: !!rm.resolved, resolvedAt: rm.resolvedAt })), probe, sc)
|
|
180
180
|
// when the code axis is stale, explain it: which of THIS reading's governed files moved, by how many commits.
|
|
181
181
|
const drift = axes.includes('code') ? codeDrift(idx, r.codeSha, cf, probe) : []
|
|
182
182
|
// the reading's evidence list, each entry resolved to its live blob state; the primary (video-first, else
|
|
@@ -215,7 +215,7 @@ export async function evalTimeline(id: string, ctx?: EvalContext): Promise<EvalT
|
|
|
215
215
|
const rows = readings.filter((r) => r.scenario === track.scenario)
|
|
216
216
|
if (!rows.length) {
|
|
217
217
|
// no reading joins this track. If the scenario is still DECLARED it is just a blind spot (unmeasured),
|
|
218
|
-
// not orphaned — its remarks wait for a reading. Only a scenario that is BOTH gone from
|
|
218
|
+
// not orphaned — its remarks wait for a reading. Only a scenario that is BOTH gone from eval.md AND
|
|
219
219
|
// has no reading is truly dangling (renamed/deleted), and that is the one we surface at node level.
|
|
220
220
|
if (!declared.has(track.scenario)) {
|
|
221
221
|
dangling.push({
|
|
@@ -232,7 +232,7 @@ export async function evalTimeline(id: string, ctx?: EvalContext): Promise<EvalT
|
|
|
232
232
|
;(host.remarks ??= []).push(toRemarkView(rm, track.threadId, !target))
|
|
233
233
|
}
|
|
234
234
|
}
|
|
235
|
-
return { node: id,
|
|
235
|
+
return { node: id, hasEvalFile: true, scenarios, readings, retractions: [...retractions].reverse(), dangling }
|
|
236
236
|
}
|
|
237
237
|
|
|
238
238
|
const HEX64 = /^[0-9a-f]{64}$/
|
|
@@ -242,7 +242,7 @@ export type BlobResult =
|
|
|
242
242
|
| { ok: false; reason: 'invalid' | 'miss'; message: string }
|
|
243
243
|
|
|
244
244
|
export function readBlobByHash(hash: string, dir?: string): BlobResult {
|
|
245
|
-
if (!HEX64.test(hash)) return { ok: false, reason: 'invalid', message: 'bad
|
|
245
|
+
if (!HEX64.test(hash)) return { ok: false, reason: 'invalid', message: 'bad evidence hash' }
|
|
246
246
|
const bytes = getBlob(hash, dir) // undefined dir → the live cache (cache.ts default); a temp dir in tests
|
|
247
247
|
if (!bytes) return { ok: false, reason: 'miss', message: MISS_BLOB }
|
|
248
248
|
return { ok: true, bytes, mime: sniffBlobMime(bytes) }
|
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import { repoRoot, headSha } from '../../spec-cli/src/git.js'
|
|
2
|
-
import {
|
|
2
|
+
import { evalNodes, resolveEvalNode, scenarioHash } from './scenarios.js'
|
|
3
3
|
import { appendReading, readReadings, isJsonBlob, type Reading, type EvidenceKind } from './sidecar.js'
|
|
4
4
|
import { putBlob } from './cache.js'
|
|
5
5
|
|
|
@@ -7,7 +7,7 @@ export type FileResult = { ok: true; reading: Reading } | { ok: false; error: st
|
|
|
7
7
|
|
|
8
8
|
// the eval seam over DATA (no argv, no file paths): a caller with a verdict in hand — the HTTP eval route,
|
|
9
9
|
// a programmatic filer — appends through the SAME seam the CLI uses. Optional evidence arrives as text (a
|
|
10
|
-
// report referencing the clip by hash) → a transcript blob in the same content-addressed cache.
|
|
10
|
+
// report referencing the clip by hash) → a transcript blob in the same content-addressed cache. eval
|
|
11
11
|
// still runs nothing — this only records. The dashboard files nothing through this: [[event-detail]] is
|
|
12
12
|
// read-side on readings.
|
|
13
13
|
export function fileHumanReading(
|
|
@@ -15,9 +15,9 @@ export function fileHumanReading(
|
|
|
15
15
|
input: { scenario: string; status: 'pass' | 'fail'; note?: string; transcript?: string; by?: string },
|
|
16
16
|
): FileResult {
|
|
17
17
|
const root = repoRoot()
|
|
18
|
-
// the same loud resolution the CLI applies ([[
|
|
18
|
+
// the same loud resolution the CLI applies ([[eval-core]]): exact canonical id, else a unique bare
|
|
19
19
|
// leaf; an ambiguous leaf returns the candidate list instead of filing against an arbitrary node.
|
|
20
|
-
const res =
|
|
20
|
+
const res = resolveEvalNode(evalNodes(root), nodeId)
|
|
21
21
|
if (!res.ok) return { ok: false, error: res.error }
|
|
22
22
|
const node = res.node
|
|
23
23
|
const sc = node.scenarios.find((s) => s.name === input.scenario)
|
|
@@ -30,6 +30,8 @@ export function fileHumanReading(
|
|
|
30
30
|
const reading: Reading = {
|
|
31
31
|
scenario: sc.name,
|
|
32
32
|
codeSha: headSha(root),
|
|
33
|
+
// the contract this measurement was taken against — the HTTP filer stamps it exactly like the CLI
|
|
34
|
+
scenarioHash: scenarioHash(sc),
|
|
33
35
|
...(blob ? { evidence: [{ hash: blob, kind: (buf && isJsonBlob(buf) ? 'data' : 'transcript') as EvidenceKind }] } : {}),
|
|
34
36
|
// the filing session (caller-passed — the human annotator has no reachable session, so it stays absent
|
|
35
37
|
// there and the eval-comment loop-in is silent, per [[mentions]])
|
|
@@ -48,7 +50,7 @@ export function fileHumanReading(
|
|
|
48
50
|
// in-flight reading (filed on an unmerged branch) is invisible to the trunk sidecar — exactly the
|
|
49
51
|
// review-time case where the remark must still reach its filer ([[remark-polish]] strand 2).
|
|
50
52
|
export function evalReadingFiler(nodeId: string, scenario: string, root: string = repoRoot()): string | null {
|
|
51
|
-
const res =
|
|
53
|
+
const res = resolveEvalNode(evalNodes(root), nodeId)
|
|
52
54
|
if (!res.ok) return null
|
|
53
55
|
const forScenario = readReadings(res.node.sidecarPath).filter((r) => r.scenario === scenario)
|
|
54
56
|
return forScenario.length ? forScenario[forScenario.length - 1].by ?? null : null
|
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
import { git, headSha, ancestorsOf, inAncestors, type DriftIndex } from '../../spec-cli/src/git.js'
|
|
2
2
|
import type { Reading } from './sidecar.js'
|
|
3
|
+
import { scenarioHash, type Scenario } from './scenarios.js'
|
|
3
4
|
import { scenarioChangeCommits, scenarioBlocksAt, type ScenarioIndex } from './scenariofresh.js'
|
|
4
5
|
|
|
5
6
|
// the CODE axis is touch-based (DriftIndex), so a code-file rename is out of scope — the same blind spot lint's code-drift has
|
|
@@ -18,19 +19,23 @@ export type ContentProbe = {
|
|
|
18
19
|
// paths whose content differs between the anchor commit's tree and HEAD's; null = anchor object gone
|
|
19
20
|
changedPaths(anchorSha: string): Set<string> | null
|
|
20
21
|
// did THIS scenario's semantic block (description+expected) move between anchor and HEAD ([[scenariofresh]])
|
|
21
|
-
scenarioDiffers(anchorSha: string,
|
|
22
|
+
scenarioDiffers(anchorSha: string, evalPath: string, scenario: string): boolean
|
|
22
23
|
// codeDrift's display detail: commits in anchor..HEAD touching path (floored at 1 — the content differs)
|
|
23
24
|
behind(anchorSha: string, path: string): number
|
|
24
25
|
}
|
|
25
26
|
|
|
26
|
-
// (anchor, HEAD) name two immutable trees, so entries never invalidate; the LRU only bounds memory
|
|
27
|
+
// (anchor, HEAD) name two immutable trees, so entries never invalidate; the LRU only bounds memory,
|
|
28
|
+
// sized above the largest adopter reading corpus — one entry per (reading, path) worst case — so a
|
|
29
|
+
// repeat board build never thrashes back into forking (a bound below the corpus's distinct key count
|
|
30
|
+
// turns a fixed-order rebuild into sequential thrash: every pass evicts the whole memo before cycling
|
|
31
|
+
// back, re-forking one git child per key forever — scenariofresh's oidMemo sizing rule).
|
|
27
32
|
const diffMemo = new Map<string, Set<string> | null>()
|
|
28
33
|
const behindMemo = new Map<string, number>()
|
|
29
34
|
function memo<V>(m: Map<string, V>, k: string, build: () => V): V {
|
|
30
35
|
if (m.has(k)) { const v = m.get(k)!; m.delete(k); m.set(k, v); return v }
|
|
31
36
|
const v = build()
|
|
32
37
|
m.set(k, v)
|
|
33
|
-
if (m.size >
|
|
38
|
+
if (m.size > 4096) m.delete(m.keys().next().value!)
|
|
34
39
|
return v
|
|
35
40
|
}
|
|
36
41
|
|
|
@@ -46,10 +51,10 @@ export function contentProbeFor(root: string): ContentProbe {
|
|
|
46
51
|
} catch { return null } // the anchor commit object is gone — content can't testify
|
|
47
52
|
})
|
|
48
53
|
},
|
|
49
|
-
scenarioDiffers(sha,
|
|
50
|
-
const a = scenarioBlocksAt(root, sha,
|
|
51
|
-
if (!a) return true //
|
|
52
|
-
return a.get(scenario) !== scenarioBlocksAt(root, headOf(),
|
|
54
|
+
scenarioDiffers(sha, evalPath, scenario) {
|
|
55
|
+
const a = scenarioBlocksAt(root, sha, evalPath)
|
|
56
|
+
if (!a) return true // eval.md unreadable at the anchor (renamed/absent) → can't prove → stale
|
|
57
|
+
return a.get(scenario) !== scenarioBlocksAt(root, headOf(), evalPath)?.get(scenario)
|
|
53
58
|
},
|
|
54
59
|
behind(sha, path) {
|
|
55
60
|
return memo(behindMemo, `${root}\x1f${sha}\x1f${headOf()}\x1f${path}`, () => {
|
|
@@ -108,39 +113,59 @@ export function codeDrift(idx: DriftIndex, sinceSha: string, codeFiles: string[]
|
|
|
108
113
|
return out
|
|
109
114
|
}
|
|
110
115
|
|
|
111
|
-
// scenario
|
|
112
|
-
//
|
|
113
|
-
//
|
|
114
|
-
//
|
|
115
|
-
//
|
|
116
|
-
//
|
|
117
|
-
//
|
|
118
|
-
//
|
|
119
|
-
|
|
116
|
+
// @@@scenario axis decides by stored contract hash - a reading filed since #61 carries `scenarioHash`, the
|
|
117
|
+
// content hash of the semantic text it was measured against (scenarios.ts scenarioHash — normalized
|
|
118
|
+
// description+expected). For such a reading the scenario axis is a PURE TEXT COMPARE: recorded hash vs the
|
|
119
|
+
// CURRENT declaration's hash — no git walk, no chain, no ancestry. That is what makes it converge under
|
|
120
|
+
// fleet-parallel filing+merging: a sibling scenario's edit, a sidecar-only commit, a merge's textual shift,
|
|
121
|
+
// a whitespace re-wrap — none of them move THIS scenario's hash, so none can re-stale its reading; only the
|
|
122
|
+
// contract actually changing (or the scenario disappearing from eval.md — `current` undefined) does. A
|
|
123
|
+
// LEGACY reading without the hash is decided by the git-derived per-scenario rule below, unchanged — the
|
|
124
|
+
// one-shot degradation: exactly ONE track decides each reading (hash if present, else git), never both
|
|
125
|
+
// OR-ed together.
|
|
126
|
+
function scenarioStaleByHash(reading: Reading, current: Scenario | undefined): boolean | undefined {
|
|
127
|
+
if (reading.scenarioHash === undefined) return undefined // legacy → the git rule decides
|
|
128
|
+
return current ? scenarioHash(current) !== reading.scenarioHash : true
|
|
129
|
+
}
|
|
130
|
+
|
|
131
|
+
// LEGACY scenario freshness (readings filed before the stored contract hash) — PER-SCENARIO and SEMANTIC,
|
|
132
|
+
// not per-file: a reading stales only when ITS OWN scenario's semantic block (description+expected —
|
|
133
|
+
// [[scenariofresh]]'s blockContent projection) moved in scenarioSha..HEAD — never when a sibling in the
|
|
134
|
+
// same eval.md did, and never on a metadata-only edit (tags/test/code/related). Reads like the code axis's
|
|
135
|
+
// changedSince — the per-scenario change-commits ([[scenariofresh]], rename-followed so a bare git-mv
|
|
136
|
+
// reparent isn't a change) tested for ancestry — and an off-history codeSha takes the same content fallback
|
|
137
|
+
// at the same granularity and the same projection. Known limit (#61, why the hash replaced it for new
|
|
138
|
+
// readings): the change-commit chain is built off a LINEARIZED log walk, so parallel branches editing one
|
|
139
|
+
// eval.md cross-attribute each other's edits and can false-stale a sibling's reading across a merge.
|
|
140
|
+
function scenarioMoved(scIdx: ScenarioIndex, didx: DriftIndex, sinceSha: string, evalPath: string, scenario: string, probe?: ContentProbe): boolean {
|
|
120
141
|
const anc = ancestorsOf(didx, sinceSha)
|
|
121
|
-
if (anc) return scenarioChangeCommits(scIdx,
|
|
142
|
+
if (anc) return scenarioChangeCommits(scIdx, evalPath, scenario).some((h) => !inAncestors(didx, anc, h))
|
|
122
143
|
const diff = probe?.changedPaths(sinceSha)
|
|
123
144
|
if (!diff) return true
|
|
124
|
-
if (!diff.has(
|
|
125
|
-
return probe!.scenarioDiffers(sinceSha,
|
|
145
|
+
if (!diff.has(evalPath)) return false // whole file byte-identical → this block too
|
|
146
|
+
return probe!.scenarioDiffers(sinceSha, evalPath, scenario)
|
|
126
147
|
}
|
|
127
148
|
|
|
128
149
|
export function staleAxes(
|
|
129
150
|
reading: Reading,
|
|
130
151
|
codeFiles: string[],
|
|
131
|
-
|
|
152
|
+
evalPath: string,
|
|
132
153
|
didx: DriftIndex,
|
|
133
154
|
scIdx: ScenarioIndex,
|
|
134
155
|
remarks: RemarkSignal[] = [],
|
|
135
156
|
probe?: ContentProbe,
|
|
157
|
+
current?: Scenario, // the scenario's CURRENT declaration (undefined = gone from eval.md) — the hash compare's other side
|
|
136
158
|
): StaleAxis[] {
|
|
137
159
|
const axes: StaleAxis[] = []
|
|
160
|
+
const byHash = scenarioStaleByHash(reading, current)
|
|
138
161
|
if (probe && !ancestorsOf(didx, reading.codeSha) && probe.changedPaths(reading.codeSha) === null) {
|
|
139
|
-
// the anchor commit object is GONE — neither git axis can testify; say that, not "content changed"
|
|
162
|
+
// the anchor commit object is GONE — neither git axis can testify; say that, not "content changed".
|
|
163
|
+
// The stored contract hash needs no anchor, so it still decides the scenario axis when present.
|
|
140
164
|
axes.push('anchor')
|
|
165
|
+
if (byHash) axes.push('scenario')
|
|
141
166
|
} else {
|
|
142
167
|
if (codeFiles.some((f) => changedSince(didx, reading.codeSha, f, probe))) axes.push('code')
|
|
143
|
-
if (scenarioMoved(scIdx, didx, reading.codeSha,
|
|
168
|
+
if (byHash ?? scenarioMoved(scIdx, didx, reading.codeSha, evalPath, reading.scenario, probe)) axes.push('scenario')
|
|
144
169
|
}
|
|
145
170
|
if (remarkStale(reading, remarks)) axes.push('remark')
|
|
146
171
|
return axes
|
|
@@ -149,11 +174,12 @@ export function staleAxes(
|
|
|
149
174
|
export function isStale(
|
|
150
175
|
reading: Reading,
|
|
151
176
|
codeFiles: string[],
|
|
152
|
-
|
|
177
|
+
evalPath: string,
|
|
153
178
|
didx: DriftIndex,
|
|
154
179
|
scIdx: ScenarioIndex,
|
|
155
180
|
remarks: RemarkSignal[] = [],
|
|
156
181
|
probe?: ContentProbe,
|
|
182
|
+
current?: Scenario,
|
|
157
183
|
): boolean {
|
|
158
|
-
return staleAxes(reading, codeFiles,
|
|
184
|
+
return staleAxes(reading, codeFiles, evalPath, didx, scIdx, remarks, probe, current).length > 0
|
|
159
185
|
}
|
|
@@ -1,11 +1,11 @@
|
|
|
1
1
|
import { spawn } from 'node:child_process'
|
|
2
2
|
import { git, gitA, headSha } from '../../spec-cli/src/git.js'
|
|
3
|
-
import { parseScenarios } from './
|
|
3
|
+
import { parseScenarios } from './scenarios.js'
|
|
4
4
|
|
|
5
5
|
// @@@ per-scenario content freshness — the SCENARIO axis, sub-file
|
|
6
|
-
//
|
|
6
|
+
// An eval.md holds many scenarios, but a reading measures ONE. eval-core's contract says "a scenario is the
|
|
7
7
|
// unit of measurement, so its freshness is its OWN — two scenarios stale independently." Git has no sub-file
|
|
8
|
-
// history, so we build it: for each scenario NAME in
|
|
8
|
+
// history, so we build it: for each scenario NAME in an eval.md, the commits where THAT scenario's SEMANTIC
|
|
9
9
|
// block content (description + expected — see blockContent) changed (added / removed / edited),
|
|
10
10
|
// rename-followed. `scenarioMoved` then reads exactly like the code axis's `changedSince` — a pure ancestry
|
|
11
11
|
// lookup over this per-scenario commit list — so editing one scenario never re-stales its siblings (the
|
|
@@ -13,7 +13,7 @@ import { parseScenarios } from './yatsu.js'
|
|
|
13
13
|
|
|
14
14
|
const RS = '\x1e'
|
|
15
15
|
|
|
16
|
-
//
|
|
16
|
+
// evalPath (head path) -> scenario name -> commit hashes that changed that scenario's block (newest-first)
|
|
17
17
|
export type ScenarioIndex = Map<string, Map<string, string[]>>
|
|
18
18
|
|
|
19
19
|
// the block content that stales a reading: the scenario's SEMANTIC fields only — description (what to
|
|
@@ -42,7 +42,7 @@ const EMPTY: Map<string, string> = new Map()
|
|
|
42
42
|
// already parsed, which is what lets a worktree's freshness build stay cheap instead of re-reading everything.
|
|
43
43
|
const blockByOid = new Map<string, Map<string, string>>()
|
|
44
44
|
|
|
45
|
-
// ONE `git log --raw` walk over all
|
|
45
|
+
// ONE `git log --raw` walk over all eval.md → per head-path version chain [{hash, oid}], newest-first,
|
|
46
46
|
// rename-followed via the `alias` idiom (git.ts buildIndex): the newest sighting of a path IS its head path;
|
|
47
47
|
// an `R` row remaps the older `from` path onto that head. The raw row carries the new blob OID directly, so
|
|
48
48
|
// no per-version path resolution and no `git show`. --full-history is REQUIRED: default pathspec
|
|
@@ -53,8 +53,13 @@ const blockByOid = new Map<string, Map<string, string>>()
|
|
|
53
53
|
async function fileChains(root: string, wanted: Set<string>): Promise<Map<string, { hash: string; oid: string }[]>> {
|
|
54
54
|
const chains = new Map<string, { hash: string; oid: string }[]>()
|
|
55
55
|
const alias = new Map<string, string>()
|
|
56
|
+
// DUAL pathspec — the live name AND the archived one ([[eval-core]]): this walk reads IMMUTABLE history,
|
|
57
|
+
// and pre-rename commits touched files literally named yatsu.md, so an archive answers only to its
|
|
58
|
+
// archive name. A single '*eval.md' pathspec truncates every chain at the yatsu.md→eval.md rename commit,
|
|
59
|
+
// making the rename read as each scenario's birth — a change-commit newer than every pre-rename reading's
|
|
60
|
+
// codeSha — and false-stales whole adopter corpora (the scenariofresh.test.ts regression pins this).
|
|
56
61
|
const out = await gitA(['-C', root, '-c', 'core.quotePath=false', 'log',
|
|
57
|
-
'--raw', '--no-abbrev', '--full-history', '-M', `--format=${RS}%H`, '--', '*yatsu.md'])
|
|
62
|
+
'--raw', '--no-abbrev', '--full-history', '-M', `--format=${RS}%H`, '--', '*eval.md', '*yatsu.md']) // dead-words-ok: archived pathspec — immutable pre-rename history is read under its archived name
|
|
58
63
|
for (const rec of out.split(RS)) {
|
|
59
64
|
const nl = rec.indexOf('\n')
|
|
60
65
|
if (nl < 0) continue
|
|
@@ -115,7 +120,7 @@ function catFileBatch(root: string, oids: string[]): Promise<Map<string, string>
|
|
|
115
120
|
p.on('close', (_code, signal) => {
|
|
116
121
|
// a child that never exited was SIGKILLed at the timeout (a hung git must not pin this promise —
|
|
117
122
|
// same bound as git.ts's helpers); warn loudly and parse whatever arrived.
|
|
118
|
-
if (signal === 'SIGKILL') console.warn(`spec-
|
|
123
|
+
if (signal === 'SIGKILL') console.warn(`spec-eval: git cat-file --batch killed after timeout — child never exited`)
|
|
119
124
|
const buf = Buffer.concat(chunks)
|
|
120
125
|
let i = 0
|
|
121
126
|
while (i < buf.length) {
|
|
@@ -139,8 +144,8 @@ function catFileBatch(root: string, oids: string[]): Promise<Map<string, string>
|
|
|
139
144
|
|
|
140
145
|
// TWO git subprocesses for the whole index (was F logs + V shows): one `--raw` log for the rename-followed
|
|
141
146
|
// chains, one `cat-file --batch` for every distinct blob not already memoized. Then a pure in-memory diff.
|
|
142
|
-
async function build(root: string,
|
|
143
|
-
const chains = await fileChains(root, new Set(
|
|
147
|
+
async function build(root: string, evalPaths: string[]): Promise<ScenarioIndex> {
|
|
148
|
+
const chains = await fileChains(root, new Set(evalPaths))
|
|
144
149
|
const need = new Set<string>()
|
|
145
150
|
for (const chain of chains.values()) for (const v of chain) if (v.oid !== ZERO && !blockByOid.has(v.oid)) need.add(v.oid)
|
|
146
151
|
if (need.size) {
|
|
@@ -148,7 +153,7 @@ async function build(root: string, yatsuPaths: string[]): Promise<ScenarioIndex>
|
|
|
148
153
|
for (const [oid, src] of blobs) blockByOid.set(oid, blockContent(src))
|
|
149
154
|
}
|
|
150
155
|
const idx: ScenarioIndex = new Map()
|
|
151
|
-
for (const p of
|
|
156
|
+
for (const p of evalPaths) idx.set(p, scenarioCommits(chains.get(p) ?? [])) // every asked path gets an entry (empty if untracked)
|
|
152
157
|
return idx
|
|
153
158
|
}
|
|
154
159
|
|
|
@@ -156,20 +161,20 @@ async function build(root: string, yatsuPaths: string[]): Promise<ScenarioIndex>
|
|
|
156
161
|
// whatever the caller. Holds the in-flight promise so concurrent board builds share one build.
|
|
157
162
|
const SLOTS = 16
|
|
158
163
|
const cache = new Map<string, Promise<ScenarioIndex>>()
|
|
159
|
-
export function scenarioIndex(root: string,
|
|
164
|
+
export function scenarioIndex(root: string, evalPaths: string[]): Promise<ScenarioIndex> {
|
|
160
165
|
let head: string
|
|
161
|
-
try { head = headSha(root) } catch { return build(root,
|
|
166
|
+
try { head = headSha(root) } catch { return build(root, evalPaths) }
|
|
162
167
|
const hit = cache.get(head)
|
|
163
168
|
if (hit) { cache.delete(head); cache.set(head, hit); return hit }
|
|
164
|
-
const p = build(root,
|
|
169
|
+
const p = build(root, evalPaths)
|
|
165
170
|
p.catch(() => cache.delete(head))
|
|
166
171
|
cache.set(head, p)
|
|
167
172
|
while (cache.size > SLOTS) cache.delete(cache.keys().next().value!)
|
|
168
173
|
return p
|
|
169
174
|
}
|
|
170
175
|
|
|
171
|
-
export function scenarioChangeCommits(idx: ScenarioIndex,
|
|
172
|
-
return idx.get(
|
|
176
|
+
export function scenarioChangeCommits(idx: ScenarioIndex, evalPath: string, scenario: string): string[] {
|
|
177
|
+
return idx.get(evalPath)?.get(scenario) ?? []
|
|
173
178
|
}
|
|
174
179
|
|
|
175
180
|
// (root, sha, path) -> the blob oid at that commit ('' = unresolvable: path absent there, or the commit
|
|
@@ -194,7 +199,7 @@ function oidAt(root: string, rev: string, path: string): string {
|
|
|
194
199
|
}
|
|
195
200
|
|
|
196
201
|
// canonical per-scenario SEMANTIC blocks of `rev:path` (the blockContent projection), for the off-history
|
|
197
|
-
// content fallback ([[
|
|
202
|
+
// content fallback ([[eval-core]]'s ContentProbe): resolve the blob oid first (memoized above for a full
|
|
198
203
|
// sha) — oids are content-addressed, so an unchanged file usually hits blockByOid straight from the index
|
|
199
204
|
// build — and parse only on a genuine miss. null = the path is unreadable at that rev (absent, renamed
|
|
200
205
|
// since, or the rev itself is gone).
|
|
@@ -204,7 +209,7 @@ export function scenarioBlocksAt(root: string, rev: string, path: string): Map<s
|
|
|
204
209
|
const hit = blockByOid.get(oid)
|
|
205
210
|
if (hit) return hit
|
|
206
211
|
try {
|
|
207
|
-
const m = blockContent(git(['-C', root, 'cat-file', 'blob', oid]))
|
|
212
|
+
const m = blockContent(git(['-C', root, 'cat-file', 'blob', oid])) // dead-words-ok: git plumbing — 'blob' is git's object type, not our vocabulary
|
|
208
213
|
blockByOid.set(oid, m)
|
|
209
214
|
return m
|
|
210
215
|
} catch { return null }
|
|
@@ -1,10 +1,11 @@
|
|
|
1
1
|
import { readFileSync, readdirSync, existsSync } from 'node:fs'
|
|
2
2
|
import { readFile, readdir } from 'node:fs/promises'
|
|
3
|
+
import { createHash } from 'node:crypto'
|
|
3
4
|
import { join, relative, basename } from 'node:path'
|
|
4
5
|
import { mintIds } from '../../spec-cli/src/specs.js'
|
|
5
6
|
|
|
6
|
-
export const
|
|
7
|
-
export const SIDECAR_FILE = '
|
|
7
|
+
export const EVAL_FILE = 'eval.md'
|
|
8
|
+
export const SIDECAR_FILE = 'evals.ndjson'
|
|
8
9
|
|
|
9
10
|
export type Scenario = {
|
|
10
11
|
name: string
|
|
@@ -16,11 +17,11 @@ export type Scenario = {
|
|
|
16
17
|
related?: string[]
|
|
17
18
|
}
|
|
18
19
|
|
|
19
|
-
export type
|
|
20
|
+
export type EvalNode = {
|
|
20
21
|
id: string // the node's CANONICAL spec id (leaf name, '_'-disambiguated on a leaf collision)
|
|
21
22
|
dir: string // absolute node directory
|
|
22
|
-
|
|
23
|
-
sidecarPath: string // absolute path to
|
|
23
|
+
evalPath: string // repo-relative path to eval.md — the SCENARIO freshness axis
|
|
24
|
+
sidecarPath: string // absolute path to evals.ndjson
|
|
24
25
|
scenarios: Scenario[]
|
|
25
26
|
}
|
|
26
27
|
|
|
@@ -32,7 +33,7 @@ const LIST_KEYS: readonly ScenarioKey[] = ['tags', 'code', 'related']
|
|
|
32
33
|
// carried — kept (not dropped) so the validator can name a typo'd field instead of silently swallowing it.
|
|
33
34
|
type RawItem = { fields: Partial<Record<ScenarioKey, string>>; unknownKeys: string[] }
|
|
34
35
|
|
|
35
|
-
// tiny indentation parser for
|
|
36
|
+
// tiny indentation parser for eval.md's frontmatter `scenarios:` block (no YAML dep), shared by parseScenarios and validateScenarios so they can't disagree; reports hasFrontmatter/hasKey so the validator can tell "none declared" from "malformed"
|
|
36
37
|
function walkScenarios(src: string): { hasFrontmatter: boolean; hasKey: boolean; items: RawItem[] } {
|
|
37
38
|
const m = src.match(/^---\n([\s\S]*?)\n---/)
|
|
38
39
|
if (!m) return { hasFrontmatter: false, hasKey: false, items: [] }
|
|
@@ -117,6 +118,20 @@ function assignField(cur: RawItem, kv: string, lines: string[], idx: number, key
|
|
|
117
118
|
|
|
118
119
|
const unquote = (s: string) => s.replace(/^["'](.*)["']$/, '$1').trim()
|
|
119
120
|
|
|
121
|
+
// @@@scenario contract hash - the deterministic content hash of a scenario's SEMANTIC text, stamped on each
|
|
122
|
+
// reading at filing time ([[eval-core]]'s scenario freshness axis). Hashes ONLY the measurement contract —
|
|
123
|
+
// description (what to measure) + expected (what zero loss looks like) — never name/tags/test/code/related.
|
|
124
|
+
// Normalization (spec'd, don't drift): each field independently collapses every whitespace run (space, tab,
|
|
125
|
+
// CR, LF) to a single space and trims its ends, so a prose re-wrap, an indent shift, CRLF churn or trailing
|
|
126
|
+
// whitespace never changes the hash; the two normalized fields join with a single '\n' (neither can contain
|
|
127
|
+
// one after normalization, so the join is unambiguous) and sha256-hex the UTF-8 bytes. The hash is pure text
|
|
128
|
+
// over the parsed declaration — no git, no file position — so it is identical wherever and however the same
|
|
129
|
+
// contract text is read.
|
|
130
|
+
const normSemantic = (s: string) => s.replace(/\s+/g, ' ').trim()
|
|
131
|
+
export function scenarioHash(s: Pick<Scenario, 'description' | 'expected'>): string {
|
|
132
|
+
return createHash('sha256').update(`${normSemantic(s.description)}\n${normSemantic(s.expected)}`, 'utf8').digest('hex')
|
|
133
|
+
}
|
|
134
|
+
|
|
120
135
|
// a scenario's optional list field (`code:`/`related:`) is a comma-separated path list (a YAML flow list
|
|
121
136
|
// `[a, b]` or bare `a, b`, or a single path) — the tiny parser stays scalar-only, so it is split here.
|
|
122
137
|
function parseCodeList(raw: string): string[] {
|
|
@@ -148,7 +163,7 @@ export function parseScenarios(src: string): Scenario[] {
|
|
|
148
163
|
// disables only the membership check, never the ≥1-tag requirement.
|
|
149
164
|
export function validateScenarios(src: string, tagLibrary: string[] = []): string[] {
|
|
150
165
|
const { hasFrontmatter, hasKey, items } = walkScenarios(src)
|
|
151
|
-
if (!hasFrontmatter) return ['no frontmatter block —
|
|
166
|
+
if (!hasFrontmatter) return ['no frontmatter block — an eval.md must declare a `scenarios:` list']
|
|
152
167
|
if (!hasKey) return ['frontmatter has no `scenarios:` key — declare at least one scenario']
|
|
153
168
|
if (!items.length) return ['`scenarios:` declares no scenarios — add one (name + description + expected)']
|
|
154
169
|
const errs: string[] = []
|
|
@@ -170,17 +185,17 @@ export function validateScenarios(src: string, tagLibrary: string[] = []): strin
|
|
|
170
185
|
for (const u of it.unknownKeys) errs.push(`${label}: unknown field \`${u}\` (allowed: ${SCENARIO_KEYS.join(', ')})`)
|
|
171
186
|
if (it.fields.name) counts.set(it.fields.name, (counts.get(it.fields.name) ?? 0) + 1)
|
|
172
187
|
})
|
|
173
|
-
for (const [n, c] of counts) if (c > 1) errs.push(`duplicate scenario name '${n}' (${c}×) — names must be unique within
|
|
188
|
+
for (const [n, c] of counts) if (c > 1) errs.push(`duplicate scenario name '${n}' (${c}×) — names must be unique within an eval.md`)
|
|
174
189
|
return errs
|
|
175
190
|
}
|
|
176
191
|
|
|
177
|
-
// walk `.spec` for every dir holding
|
|
192
|
+
// walk `.spec` for every dir holding an eval.md; the node id is its CANONICAL spec id ([[id-url-safe]]) —
|
|
178
193
|
// minted by the SAME rule as specs.ts's loader (mintIds: the leaf dir name, or on a leaf collision the
|
|
179
194
|
// shortest globally-unique '_'-joined trailing suffix) over the SAME universe (every dir holding a spec.md,
|
|
180
|
-
// not just the
|
|
181
|
-
// them measures). So the id
|
|
182
|
-
// second, diverging bare-leaf scheme.
|
|
183
|
-
function assembleNodes(root: string, specDirs: string[], hits: { dir: string; src: string }[]):
|
|
195
|
+
// not just the eval subset — a leaf that collides among spec nodes is disambiguated even when only one of
|
|
196
|
+
// them measures). So the id eval verbs answer to is exactly the id board/scan/search already print — never a
|
|
197
|
+
// second, diverging bare-leaf scheme. An eval.md beside no spec.md keeps its leaf name (no spec id to align with).
|
|
198
|
+
function assembleNodes(root: string, specDirs: string[], hits: { dir: string; src: string }[]): EvalNode[] {
|
|
184
199
|
const specBase = join(root, '.spec')
|
|
185
200
|
const ids = mintIds(specDirs.map((d) => relative(specBase, d).split(/[/\\]/)))
|
|
186
201
|
const idByDir = new Map(specDirs.map((d, i) => [d, ids[i]]))
|
|
@@ -188,14 +203,14 @@ function assembleNodes(root: string, specDirs: string[], hits: { dir: string; sr
|
|
|
188
203
|
.map(({ dir, src }) => ({
|
|
189
204
|
id: idByDir.get(dir) ?? basename(dir),
|
|
190
205
|
dir,
|
|
191
|
-
|
|
206
|
+
evalPath: relative(root, join(dir, EVAL_FILE)),
|
|
192
207
|
sidecarPath: join(dir, SIDECAR_FILE),
|
|
193
208
|
scenarios: parseScenarios(src),
|
|
194
209
|
}))
|
|
195
210
|
.sort((a, b) => a.id.localeCompare(b.id))
|
|
196
211
|
}
|
|
197
212
|
|
|
198
|
-
export function
|
|
213
|
+
export function evalNodes(root: string): EvalNode[] {
|
|
199
214
|
const specDir = join(root, '.spec')
|
|
200
215
|
const specDirs: string[] = []
|
|
201
216
|
const hits: { dir: string; src: string }[] = []
|
|
@@ -205,17 +220,17 @@ export function yatsuNodes(root: string): YatsuNode[] {
|
|
|
205
220
|
let ents
|
|
206
221
|
try { ents = readdirSync(dir, { withFileTypes: true }) } catch { continue }
|
|
207
222
|
if (existsSync(join(dir, 'spec.md'))) specDirs.push(dir)
|
|
208
|
-
if (existsSync(join(dir,
|
|
223
|
+
if (existsSync(join(dir, EVAL_FILE))) hits.push({ dir, src: readFileSync(join(dir, EVAL_FILE), 'utf8') })
|
|
209
224
|
for (const e of ents) if (e.isDirectory()) stack.push(join(dir, e.name))
|
|
210
225
|
}
|
|
211
226
|
return assembleNodes(root, specDirs, hits)
|
|
212
227
|
}
|
|
213
228
|
|
|
214
|
-
// async twin of
|
|
229
|
+
// async twin of evalNodes for the HOT board build ([[graph-cache]]): reading each eval.md through
|
|
215
230
|
// fs/promises YIELDS the event loop between files, so the walk no longer stalls a `/health` probe in one
|
|
216
|
-
// ~600ms uninterrupted stretch. Same output (canonical ids, id-sorted) as
|
|
231
|
+
// ~600ms uninterrupted stretch. Same output (canonical ids, id-sorted) as evalNodes; only buildBoard uses
|
|
217
232
|
// it, other callers keep the sync form.
|
|
218
|
-
export async function
|
|
233
|
+
export async function evalNodesAsync(root: string): Promise<EvalNode[]> {
|
|
219
234
|
const specDir = join(root, '.spec')
|
|
220
235
|
const specDirs: string[] = []
|
|
221
236
|
const hits: { dir: string; src: string }[] = []
|
|
@@ -225,25 +240,25 @@ export async function yatsuNodesAsync(root: string): Promise<YatsuNode[]> {
|
|
|
225
240
|
let ents
|
|
226
241
|
try { ents = await readdir(dir, { withFileTypes: true }) } catch { continue }
|
|
227
242
|
if (existsSync(join(dir, 'spec.md'))) specDirs.push(dir)
|
|
228
|
-
if (existsSync(join(dir,
|
|
243
|
+
if (existsSync(join(dir, EVAL_FILE))) hits.push({ dir, src: await readFile(join(dir, EVAL_FILE), 'utf8') })
|
|
229
244
|
for (const e of ents) if (e.isDirectory()) stack.push(join(dir, e.name))
|
|
230
245
|
}
|
|
231
246
|
return assembleNodes(root, specDirs, hits)
|
|
232
247
|
}
|
|
233
248
|
|
|
234
|
-
export type
|
|
249
|
+
export type EvalResolution<T> = { ok: true; node: T } | { ok: false; ambiguous: boolean; error: string }
|
|
235
250
|
|
|
236
|
-
// resolve a user-supplied node ref against the
|
|
237
|
-
// name stays the convenience it always was while it names exactly ONE
|
|
251
|
+
// resolve a user-supplied node ref against the measurable set: an EXACT canonical id always wins; a bare leaf
|
|
252
|
+
// name stays the convenience it always was while it names exactly ONE measurable node; a leaf several nodes
|
|
238
253
|
// share fails LOUD listing the candidate canonical ids — never an arbitrary first hit, so a reading can
|
|
239
254
|
// only land on the node the caller actually named.
|
|
240
|
-
export function
|
|
255
|
+
export function resolveEvalNode<T extends Pick<EvalNode, 'id' | 'dir'>>(nodes: T[], ref: string): EvalResolution<T> {
|
|
241
256
|
const exact = nodes.find((n) => n.id === ref)
|
|
242
257
|
if (exact) return { ok: true, node: exact }
|
|
243
258
|
const byLeaf = nodes.filter((n) => basename(n.dir) === ref)
|
|
244
259
|
if (byLeaf.length === 1) return { ok: true, node: byLeaf[0] }
|
|
245
260
|
if (byLeaf.length > 1) {
|
|
246
|
-
return { ok: false, ambiguous: true, error: `'${ref}' is ambiguous — ${byLeaf.length}
|
|
261
|
+
return { ok: false, ambiguous: true, error: `'${ref}' is ambiguous — ${byLeaf.length} measurable nodes share that leaf name; use a canonical id: ${byLeaf.map((n) => n.id).sort().join(', ')}` }
|
|
247
262
|
}
|
|
248
|
-
return { ok: false, ambiguous: false, error: `no
|
|
263
|
+
return { ok: false, ambiguous: false, error: `no measurable node '${ref}' (a node needs an eval.md)` }
|
|
249
264
|
}
|