spexcode 0.6.0 → 0.6.2

package/spec-eval/src/scenariofresh.ts

@@ -1,5 +1,5 @@
 import { spawn } from 'node:child_process'
-import { git, gitA, gitTry, headSha } from '../../spec-cli/src/git.js'
+import { batchBlobTexts, batchRevisionOids, git, gitA, gitTry, headSha } from '../../spec-cli/src/git.js'
 import { parseScenarios } from './scenarios.js'
 import { rootSlots, touchRoot as touchRootLru } from '../../spec-cli/src/root-lru.js'
 
@@ -159,25 +159,75 @@ function oidAt(root: string, rev: string, path: string): string {
   return v
 }
 
+// @@@ the memo holds SETTLED oids, so concurrent askers need a flight to join - primes run concurrently
+// across the whole scope, and every one of them missed the memo while the first child was still running:
+// measured on adopter-a's 415-node scope, 2695 `rev-parse` children resolved 1212 distinct (rev, path) pairs.
+// `rev:path` under a full sha names an immutable object, so joining answers the same question; the LRU
+// write still happens once, on settle, exactly as the sync path does it.
+const oidFlight = new Map<string, Promise<string>>()
 async function oidAtAsync(root: string, rev: string, path: string): Promise<string> {
   if (!FULL_SHA.test(rev)) return (await gitTry(['-C', root, 'rev-parse', `${rev}:${path}`])).stdout.trim()
   const k = `${root}\x1f${rev}\x1f${path}`
   const hit = oidMemo.get(k)
   if (hit !== undefined) { oidMemo.delete(k); oidMemo.set(k, hit); return hit }
-  const result = await gitTry(['-C', root, 'rev-parse', `${rev}:${path}`])
-  const oid = result.ok ? result.stdout.trim() : ''
-  oidMemo.set(k, oid)
-  if (oidMemo.size > 4096) oidMemo.delete(oidMemo.keys().next().value!)
-  return oid
+  const joined = oidFlight.get(k)
+  if (joined) return joined
+  const run = async (): Promise<string> => {
+    const result = await gitTry(['-C', root, 'rev-parse', `${rev}:${path}`])
+    const oid = result.ok ? result.stdout.trim() : ''
+    oidMemo.set(k, oid)
+    if (oidMemo.size > 4096) oidMemo.delete(oidMemo.keys().next().value!)
+    return oid
+  }
+  const flight = run().finally(() => { if (oidFlight.get(k) === flight) oidFlight.delete(k) })
+  oidFlight.set(k, flight)
+  return flight
 }
 
 
+// @@@ the PLURAL prime, mirroring the anchor probe's - a caller hands over every (rev, path) its whole read
+// will ask about and TWO children answer them all: `cat-file --batch-check` for the object ids, then
+// `cat-file --batch` for the blobs. The singular form below is one demand's worth of the same thing, kept
+// for callers that genuinely learn their demand one at a time. Priming one at a time is what this replaces:
+// on adopter-a's 415-node scope it forked 1212 `rev-parse` children plus a `cat-file` each, and the batch
+// answers the identical set — same oids, same blocks, same verdicts, two children.
+export async function primeScenarioBlocks(root: string, demands: readonly { rev: string; path: string }[]): Promise<void> {
+  const wanted = new Map<string, { rev: string; path: string }>()
+  for (const d of demands) {
+    if (!FULL_SHA.test(d.rev) || !d.path) continue   // a symbolic rev is never memoized, so it can't ride a batch
+    const k = `${root}\x1f${d.rev}\x1f${d.path}`
+    if (!oidMemo.has(k) && !wanted.has(k)) wanted.set(k, d)
+  }
+  if (!wanted.size) return
+  const keys = [...wanted.keys()]
+  const rows = [...wanted.values()]
+  const oids = await batchRevisionOids(root, rows.map((d) => `${d.rev}:${d.path}`))
+  oids.forEach((oid, i) => {
+    oidMemo.set(keys[i], oid ?? '')
+    if (oidMemo.size > 4096) oidMemo.delete(oidMemo.keys().next().value!)
+  })
+  const missing = [...new Set(oids.filter((o): o is string => !!o && !blockByOid.has(o)))]
+  if (!missing.length) return
+  const texts = await batchBlobTexts(root, missing)
+  for (const [oid, src] of texts) if (src) blockByOid.set(oid, blockContent(src))
+}
+
+// the blob read needs the same flight as the oid lookup above it: an oid is content, so two primes that
+// resolved the same one must not each read it.
+const blockFlight = new Map<string, Promise<void>>()
 export async function primeScenarioBlocksAt(root: string, revs: string[], path: string): Promise<void> {
   for (const rev of revs) {
     const oid = await oidAtAsync(root, rev, path)
     if (!oid || blockByOid.has(oid)) continue
-    const src = await gitA(['-C', root, 'cat-file', 'blob', oid]) // dead-words-ok: git plumbing
-    if (src) blockByOid.set(oid, blockContent(src))
+    const joined = blockFlight.get(oid)
+    if (joined) { await joined; continue }
+    const run = async (): Promise<void> => {
+      const src = await gitA(['-C', root, 'cat-file', 'blob', oid]) // dead-words-ok: git plumbing
+      if (src) blockByOid.set(oid, blockContent(src))
+    }
+    const flight = run().finally(() => { if (blockFlight.get(oid) === flight) blockFlight.delete(oid) })
+    blockFlight.set(oid, flight)
+    await flight
   }
 }
 

package/spec-eval/src/scenarios.ts

@@ -2,7 +2,7 @@ import { readFileSync, readdirSync, existsSync } from 'node:fs'
 import { readFile, readdir } from 'node:fs/promises'
 import { createHash } from 'node:crypto'
 import { join, relative, basename } from 'node:path'
-import { mintIds } from '../../spec-cli/src/specs.js'
+import { mintIds, parseFrontmatter } from '../../spec-cli/src/specs.js'
 import { parseRelation, type RelationEntry } from '../../spec-cli/src/anchors.js'
 import { treeTextFiles } from '../../spec-cli/src/git.js'
 
@@ -41,13 +41,20 @@ export type ScenarioProjectionRow = {
   semantic: ScenarioSemanticRow
   measurement: ScenarioMeasurementRow
 }
+export type ScenarioProjectionNode = {
+  id: string
+  code: RelationEntry[]
+  related: RelationEntry[]
+}
 export type ScenarioProjectionProvenance = { head: string | null; treeSha: string | null }
 export type ScenarioProjection = {
   projection: typeof SCENARIO_PROJECTION
   schemaVersion: typeof SCENARIO_SCHEMA_VERSION
   provenance: ScenarioProjectionProvenance
+  nodes: ScenarioProjectionNode[]
   semanticIndexHash: string
   fullIndexHash: string
+  planningIndexHash: string
   rows: ScenarioProjectionRow[]
 }
 
@@ -58,6 +65,7 @@ export type EvalNode = {
   sidecarPath: string
   scenarios: Scenario[]
   evalSource?: string
+  specSource?: string
 }
 
 const SCENARIO_KEYS = ['name', 'description', 'expected', 'tags', 'test', 'code', 'related'] as const
@@ -349,11 +357,29 @@ const hashProjection = (value: unknown): string =>
   createHash('sha256').update(JSON.stringify(value), 'utf8').digest('hex')
 
 const semanticOnly = (row: ScenarioProjectionRow): ScenarioSemanticRow => row.semantic
+const frontmatterList = (value: string | string[] | undefined): string[] =>
+  Array.isArray(value) ? value : value ? [value] : []
+
+function projectionNode(node: Pick<EvalNode, 'id' | 'specSource'>): ScenarioProjectionNode {
+  const fm = node.specSource ? parseFrontmatter(node.specSource).fm : {}
+  const code = parseRelation(frontmatterList(fm.code), 'code')
+  const related = parseRelation(frontmatterList(fm.related), 'related')
+  const problems = [...code.problems, ...related.problems]
+  if (problems.length) {
+    throw new Error(`node '${node.id}' has malformed spec relations:\n${problems.map((e) => `  - ${e}`).join('\n')}`)
+  }
+  return {
+    id: node.id,
+    code: code.entries.map((entry) => ({ path: entry.path, selectors: [...entry.selectors] })),
+    related: related.entries.map((entry) => ({ path: entry.path, selectors: [...entry.selectors] })),
+  }
+}
 
 export function scenarioProjection(
-  nodes: readonly Pick<EvalNode, 'id' | 'scenarios' | 'evalSource'>[],
+  nodes: readonly Pick<EvalNode, 'id' | 'scenarios' | 'evalSource' | 'specSource'>[],
   provenance: Partial<ScenarioProjectionProvenance> = {},
 ): ScenarioProjection {
+  const nodeRows = nodes.map(projectionNode).sort((a, b) => compareStable(a.id, b.id))
   const rows: ScenarioProjectionRow[] = []
   for (const node of nodes) {
     if ('evalSource' in node && node.evalSource !== undefined) {
@@ -388,8 +414,10 @@ export function scenarioProjection(
     projection: SCENARIO_PROJECTION,
     schemaVersion: SCENARIO_SCHEMA_VERSION,
     provenance: { head: provenance.head ?? null, treeSha: provenance.treeSha ?? null },
+    nodes: nodeRows,
     semanticIndexHash: hashProjection(semanticRows),
     fullIndexHash: hashProjection(rows),
+    planningIndexHash: hashProjection({ nodes: nodeRows, rows }),
     rows,
   }
 }
@@ -543,18 +571,19 @@ export function writeScenarioMeasurementMetadata(source: string, request: unknow
   return proposed
 }
 
-function assembleNodes(root: string, specDirs: string[], hits: { dir: string; src: string }[]): EvalNode[] {
+function assembleNodes(root: string, specDirs: string[], hits: { dir: string; src: string; specSource?: string }[]): EvalNode[] {
   const specBase = join(root, '.spec')
   const ids = mintIds(specDirs.map((d) => relative(specBase, d).split(/[/\\]/)))
   const idByDir = new Map(specDirs.map((d, i) => [d, ids[i]]))
   return hits
-    .map(({ dir, src }) => ({
+    .map(({ dir, src, specSource }) => ({
       id: idByDir.get(dir) ?? basename(dir),
       dir,
       evalPath: relative(root, join(dir, EVAL_FILE)),
       sidecarPath: join(dir, SIDECAR_FILE),
       scenarios: parseScenarios(src),
       evalSource: src,
+      ...(specSource !== undefined ? { specSource } : {}),
     }))
     .sort((a, b) => a.id.localeCompare(b.id))
 }
@@ -562,14 +591,21 @@ function assembleNodes(root: string, specDirs: string[], hits: { dir: string; sr
 export function evalNodes(root: string): EvalNode[] {
   const specDir = join(root, '.spec')
   const specDirs: string[] = []
-  const hits: { dir: string; src: string }[] = []
+  const hits: { dir: string; src: string; specSource?: string }[] = []
   const stack = existsSync(specDir) ? [specDir] : []
   while (stack.length) {
     const dir = stack.pop()!
     let ents
     try { ents = readdirSync(dir, { withFileTypes: true }) } catch { continue }
     if (existsSync(join(dir, 'spec.md'))) specDirs.push(dir)
-    if (existsSync(join(dir, EVAL_FILE))) hits.push({ dir, src: readFileSync(join(dir, EVAL_FILE), 'utf8') })
+    if (existsSync(join(dir, EVAL_FILE))) {
+      const specPath = join(dir, 'spec.md')
+      hits.push({
+        dir,
+        src: readFileSync(join(dir, EVAL_FILE), 'utf8'),
+        ...(existsSync(specPath) ? { specSource: readFileSync(specPath, 'utf8') } : {}),
+      })
+    }
     for (const e of ents) if (e.isDirectory()) stack.push(join(dir, e.name))
   }
   return assembleNodes(root, specDirs, hits)
@@ -583,10 +619,15 @@ export function evalNodesAt(root: string, tip: string): EvalNode[] {
     .map((path) => join(root, path.slice(0, -'/spec.md'.length)))
   const hits = paths
     .filter((path) => path.endsWith(`/${EVAL_FILE}`))
-    .map((path) => ({
-      dir: join(root, path.slice(0, -`/${EVAL_FILE}`.length)),
-      src: files.get(path)!,
-    }))
+    .map((path) => {
+      const relDir = path.slice(0, -`/${EVAL_FILE}`.length)
+      const specSource = files.get(`${relDir}/spec.md`)
+      return {
+        dir: join(root, relDir),
+        src: files.get(path)!,
+        ...(specSource !== undefined ? { specSource } : {}),
+      }
+    })
   return assembleNodes(root, specDirs, hits)
 }
 
@@ -597,14 +638,21 @@ export function evalNodesAt(root: string, tip: string): EvalNode[] {
 export async function evalNodesAsync(root: string): Promise<EvalNode[]> {
   const specDir = join(root, '.spec')
   const specDirs: string[] = []
-  const hits: { dir: string; src: string }[] = []
+  const hits: { dir: string; src: string; specSource?: string }[] = []
   const stack = existsSync(specDir) ? [specDir] : []
   while (stack.length) {
     const dir = stack.pop()!
     let ents
     try { ents = await readdir(dir, { withFileTypes: true }) } catch { continue }
     if (existsSync(join(dir, 'spec.md'))) specDirs.push(dir)
-    if (existsSync(join(dir, EVAL_FILE))) hits.push({ dir, src: await readFile(join(dir, EVAL_FILE), 'utf8') })
+    if (existsSync(join(dir, EVAL_FILE))) {
+      const specPath = join(dir, 'spec.md')
+      hits.push({
+        dir,
+        src: await readFile(join(dir, EVAL_FILE), 'utf8'),
+        ...(existsSync(specPath) ? { specSource: await readFile(specPath, 'utf8') } : {}),
+      })
+    }
     for (const e of ents) if (e.isDirectory()) stack.push(join(dir, e.name))
   }
   return assembleNodes(root, specDirs, hits)