rvlib-shipkit 0.32.0 → 0.34.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +27 -0
- package/package.json +4 -1
- package/src/asks/asks-queue.ts +15 -2
- package/src/cli/cmd_Asks.ts +2 -1
- package/src/cli/cmd_Keys.ts +107 -0
- package/src/cli.ts +2 -0
- package/src/core/canonical-scripts.ts +3 -1
- package/src/core/check-suite.ts +95 -33
- package/src/core/discover-checks.ts +30 -1
- package/src/core/floor.ts +21 -2
- package/src/core/locked-write.ts +170 -1
- package/src/core/one-way-in-policy.ts +183 -0
- package/src/core/run-ci.ts +42 -3
- package/src/core/run-fixers.ts +61 -18
- package/src/core/task-runner.ts +55 -6
- package/src/hooks/one-way-in-guard.ts +39 -0
- package/src/hooks/run-hooks.ts +23 -2
- package/src/init-repo.ts +19 -1
- package/src/ledger/gate-units.ts +58 -0
- package/src/ledger/git-key.ts +330 -0
- package/src/ledger/git-scan.ts +314 -0
- package/src/ledger/toolchain-identity.ts +15 -8
- package/src/release/run-bump.ts +50 -5
- package/src/release/unit-gate.ts +17 -4
- package/src/rules/SK001-package-json-shape.ts +10 -3
- package/src/rules/SK002-contracts-fulfillness.ts +4 -0
- package/src/rules/SK003-claude-md-shape.ts +7 -2
- package/src/rules/SK004-gitignore-policy.ts +11 -2
- package/src/rules/SK005-rvlib-versions.ts +28 -1
- package/src/rules/SK006-npm-publish-safety.ts +3 -0
- package/src/rules/SK008-reviewer-agent.ts +6 -2
- package/src/rules/SK009-commit-ai-tells.ts +3 -0
- package/src/rules/SK010-undeclared-release-unit.ts +2 -0
- package/src/rules/SK011-sibling-unit-ranges.ts +2 -0
- package/src/rules/SK012-config-barrel-import.ts +6 -2
- package/src/rules/SK013-one-way-in-policy.ts +156 -0
- package/src/rules/rule-registry.ts +2 -0
- package/src/rules/rule.ts +22 -1
- package/src/rules/run-fix.ts +5 -3
- package/src/scaffold/files/claude-settings-local.json +0 -3
- package/src/scaffold/files/verify-skill.md +1 -1
- package/src/ts/config.ts +66 -6
- package/src/ts/cycles.ts +7 -92
- package/src/ts/source-graph.ts +158 -0
- package/src/ts/tsconfig-projects.ts +101 -0
- package/src/ts/typecheck-cli.ts +46 -0
package/src/core/task-runner.ts
CHANGED
|
@@ -4,7 +4,7 @@
|
|
|
4
4
|
// - a dep must PASS to unblock; a filtered-out dep counts as satisfied;
|
|
5
5
|
// a failed/skipped dep skips dependents transitively (note says why).
|
|
6
6
|
// Deps double as serialization guards (document racy edges at the def).
|
|
7
|
-
// - subprocess tasks: full log streamed to <logDir>/<id>.log, last non-empty
|
|
7
|
+
// - subprocess tasks: full log streamed to <logDir>/run-<pid>-<ts>/<id>.log, last non-empty
|
|
8
8
|
// line kept for the FAIL summary; SIGTERM at timeoutMs, SIGKILL +2s.
|
|
9
9
|
// - logic tasks run in-process against a CheckCtx: WARN never flips the
|
|
10
10
|
// exit code, FAIL does. Their lines print under the task line.
|
|
@@ -13,8 +13,8 @@
|
|
|
13
13
|
// - sharp edge, by design: `onlyIds` on a task whose race-guard dep is
|
|
14
14
|
// optIn/skipped-group drops that dep (filtered = satisfied) — explicit
|
|
15
15
|
// filters win over guards.
|
|
16
|
-
import { createWriteStream, mkdirSync, rmSync, type WriteStream } from 'fs'
|
|
17
|
-
import { join } from 'path'
|
|
16
|
+
import { createWriteStream, mkdirSync, readdirSync, rmSync, type WriteStream } from 'fs'
|
|
17
|
+
import { basename, join } from 'path'
|
|
18
18
|
import { CheckCtx } from './check-ctx.ts'
|
|
19
19
|
import { spawnWithTimeout } from './spawn-timeout.ts'
|
|
20
20
|
import { formatLine, type Line } from './wire.ts'
|
|
@@ -30,6 +30,19 @@ export type TaskDef = {
|
|
|
30
30
|
/** excluded unless listed in RunOpts.withIds */
|
|
31
31
|
optIn?: boolean
|
|
32
32
|
timeoutMs?: number
|
|
33
|
+
/** CONTENT population: the repo-relative paths this task reads. Declaring
|
|
34
|
+
* them is what makes the task keyable (ledger/git-key.ts). Undeclared is
|
|
35
|
+
* the DEFAULT and means never cached, so forgetting costs speed and never
|
|
36
|
+
* correctness. Declaring both this and `ttl` is a registration failure. */
|
|
37
|
+
inputs?: string[]
|
|
38
|
+
/** WORLD population: this task reads the network, a keychain, $PATH or the
|
|
39
|
+
* clock, so nothing on disk describes its answer and a clean pass is
|
|
40
|
+
* remembered for this long (e.g. '24h'). */
|
|
41
|
+
ttl?: string
|
|
42
|
+
/** a DECIDED never-cacheable, with the reason. Says somebody looked and
|
|
43
|
+
* nothing can describe this task, which takes it off the burn-down list
|
|
44
|
+
* that undeclared units land on. */
|
|
45
|
+
never?: string
|
|
33
46
|
}
|
|
34
47
|
|
|
35
48
|
export type TaskResult = {
|
|
@@ -113,6 +126,43 @@ export function selectTasks(o: RunOpts): { selected: TaskDef[]; errors: string[]
|
|
|
113
126
|
return { selected, errors }
|
|
114
127
|
}
|
|
115
128
|
|
|
129
|
+
/** THIS run's log dir. It used to be `logDir` itself, wiped with rmSync at the
|
|
130
|
+
* top of every run: a second agent starting a gate deleted the log files the
|
|
131
|
+
* first was still streaming into, so the surviving FAIL line pointed at a
|
|
132
|
+
* path that no longer existed. Now each run owns `run-<pid>-<started>/` and
|
|
133
|
+
* writes nowhere else.
|
|
134
|
+
*
|
|
135
|
+
* Old run dirs are swept on the way in, and only ones nothing can be using:
|
|
136
|
+
* the pid must not be running AND the dir must be older than the grace
|
|
137
|
+
* window, so a recycled pid cannot cost a live run its logs. */
|
|
138
|
+
function openRunDir(base: string): string {
|
|
139
|
+
// the pid plus the millisecond separates two PROCESSES; the counter
|
|
140
|
+
// separates two runTasks calls inside one (the TUI can start a second
|
|
141
|
+
// before the clock ticks), so no run ever writes into another's dir
|
|
142
|
+
const dir = join(base, `run-${process.pid}-${Date.now()}-${runSeq++}`)
|
|
143
|
+
mkdirSync(dir, { recursive: true })
|
|
144
|
+
const cutoff = Date.now() - RUN_DIR_KEEP_MS
|
|
145
|
+
try {
|
|
146
|
+
for (const name of readdirSync(base)) {
|
|
147
|
+
// run dirs AND the per-run release survey files, which a killed
|
|
148
|
+
// run leaves behind: both carry the pid and the start time, so one
|
|
149
|
+
// sweep covers both rather than one path leaking forever
|
|
150
|
+
const m = /^(?:run|releases)-(\d+)-(\d+)(?:-\d+)?(?:\.json(?:\.lock)?)?$/.exec(name)
|
|
151
|
+
if (m == null || name === basename(dir)) continue
|
|
152
|
+
if (Number(m[2]) > cutoff) continue
|
|
153
|
+
try { process.kill(Number(m[1]), 0); continue } catch (e) { if ((e as { code?: string }).code === 'EPERM') continue }
|
|
154
|
+
rmSync(join(base, name), { recursive: true, force: true })
|
|
155
|
+
}
|
|
156
|
+
} catch {
|
|
157
|
+
// an unreadable log base is a housekeeping miss, never a gate failure
|
|
158
|
+
}
|
|
159
|
+
return dir
|
|
160
|
+
}
|
|
161
|
+
|
|
162
|
+
/** how long a finished run's logs stay readable, and the pid-reuse guard */
|
|
163
|
+
const RUN_DIR_KEEP_MS = 6 * 60 * 60 * 1000
|
|
164
|
+
let runSeq = 0
|
|
165
|
+
|
|
116
166
|
export async function runTasks(o: RunOpts): Promise<RunSummary> {
|
|
117
167
|
const print = o.print ?? ((l: string) => console.log(l))
|
|
118
168
|
const color = o.color ?? (process.stdout.isTTY && process.env.NO_COLOR == null)
|
|
@@ -122,13 +172,12 @@ export async function runTasks(o: RunOpts): Promise<RunSummary> {
|
|
|
122
172
|
return { ok: false, results: [], interrupted: false }
|
|
123
173
|
}
|
|
124
174
|
const selectedIds = new Set(sel.selected.map((t) => t.id))
|
|
125
|
-
|
|
126
|
-
mkdirSync(o.logDir, { recursive: true })
|
|
175
|
+
const runDir = openRunDir(o.logDir)
|
|
127
176
|
|
|
128
177
|
const tasks: Live[] = sel.selected.map((t) => ({
|
|
129
178
|
...t,
|
|
130
179
|
status: 'skipped', exitCode: null, durationMs: 0, startedAt: 0,
|
|
131
|
-
logPath: t.cmd != null ? join(
|
|
180
|
+
logPath: t.cmd != null ? join(runDir, `${t.id.replace(/[^a-zA-Z0-9-]/g, '_')}.log`) : null,
|
|
132
181
|
lastLine: '', note: '', lines: [],
|
|
133
182
|
}))
|
|
134
183
|
const byId = new Map(tasks.map((t) => [t.id, t]))
|
|
@@ -0,0 +1,39 @@
|
|
|
1
|
+
#!/usr/bin/env bun
|
|
2
|
+
// the PreToolUse guard for ONE WAY IN (core/one-way-in-policy.ts owns the
|
|
3
|
+
// table and the wording). Its ONE job is to refuse a raw runner with the
|
|
4
|
+
// shipkit command to type instead: `permissions.deny` already blocks these,
|
|
5
|
+
// but a bare deny tells the model only that it was denied, and a refusal with
|
|
6
|
+
// no replacement just sends it looking for another way in.
|
|
7
|
+
//
|
|
8
|
+
// It runs before EVERY Bash call, so it stays a leaf: one tiny import, one
|
|
9
|
+
// small json read, no clipanion, no repo config. Anything it cannot decide it
|
|
10
|
+
// waves through, because a guard that fails closed on its own bugs would wedge
|
|
11
|
+
// a repo with no way to tell why, and `permissions.deny` is still the floor
|
|
12
|
+
// underneath it. Measured at ~10ms per call.
|
|
13
|
+
import { deniedHeads, rawRunnerHit, refusal } from '../core/one-way-in-policy.ts'
|
|
14
|
+
|
|
15
|
+
const input = await Bun.stdin.text()
|
|
16
|
+
let command = ''
|
|
17
|
+
try {
|
|
18
|
+
const ev = JSON.parse(input) as { tool_name?: string; tool_input?: { command?: string } }
|
|
19
|
+
if (ev.tool_name !== 'Bash') process.exit(0)
|
|
20
|
+
command = ev.tool_input?.command ?? ''
|
|
21
|
+
} catch {
|
|
22
|
+
process.exit(0)
|
|
23
|
+
}
|
|
24
|
+
// the settings file is the authority on WHICH runners this repo refuses, so
|
|
25
|
+
// deleting one `Bash(...)` line lifts the deny and this refusal together
|
|
26
|
+
const settingsPath = `${process.env['CLAUDE_PROJECT_DIR'] ?? process.cwd()}/.claude/settings.json`
|
|
27
|
+
let denied: Set<string>
|
|
28
|
+
try {
|
|
29
|
+
denied = deniedHeads(await Bun.file(settingsPath).json())
|
|
30
|
+
} catch {
|
|
31
|
+
process.exit(0)
|
|
32
|
+
}
|
|
33
|
+
const hit = command === '' || denied.size === 0 ? null : rawRunnerHit(command, denied)
|
|
34
|
+
if (hit == null) process.exit(0)
|
|
35
|
+
// exit 2 is the block: stderr becomes the reason the model reads
|
|
36
|
+
// naming the segment earns its place only when the command had MORE in it:
|
|
37
|
+
// on a bare call it repeats the head back at the reader
|
|
38
|
+
console.error(refusal(hit.runner, hit.segment.trim() === command.trim() ? undefined : hit.segment))
|
|
39
|
+
process.exit(2)
|
package/src/hooks/run-hooks.ts
CHANGED
|
@@ -75,6 +75,27 @@ export function armHooks(root: string, config: RepoConfig): ArmHooksResult {
|
|
|
75
75
|
return { state: 'armed', detail: `wrote ${r.written.join(', ') || 'nothing'}, ${r.note}` }
|
|
76
76
|
}
|
|
77
77
|
|
|
78
|
+
/** `git config` takes `.git/config.lock` for the write, and a linked worktree
|
|
79
|
+
* shares that file with its main checkout: two agents arming at the same
|
|
80
|
+
* moment, which is the normal case here, make one of them exit non zero and
|
|
81
|
+
* report "hooks will not fire" on a repo whose hooks are perfectly fine. The
|
|
82
|
+
* contention window is one file write, so a short backoff clears it. The
|
|
83
|
+
* retries are bounded and the last failure is reported with git's own words,
|
|
84
|
+
* never swallowed. */
|
|
85
|
+
function setHooksPath(root: string): { code: number; detail: string } {
|
|
86
|
+
let last = { code: -1, detail: '' }
|
|
87
|
+
for (let attempt = 0; attempt < 5; attempt++) {
|
|
88
|
+
if (attempt > 0) Bun.sleepSync(20 * 2 ** (attempt - 1))
|
|
89
|
+
const p = Bun.spawnSync(['git', 'config', 'core.hooksPath', HOOK_DIR], { cwd: root })
|
|
90
|
+
if (p.exitCode === 0) return { code: 0, detail: '' }
|
|
91
|
+
last = { code: p.exitCode, detail: p.stderr.toString().trim().split('\n')[0]?.slice(0, 120) ?? '' }
|
|
92
|
+
// only lock contention is worth retrying; anything else is a real
|
|
93
|
+
// refusal and retrying it just makes the failure slower
|
|
94
|
+
if (!/could not lock|file exists|another git process/i.test(last.detail)) return last
|
|
95
|
+
}
|
|
96
|
+
return last
|
|
97
|
+
}
|
|
98
|
+
|
|
78
99
|
/** wire `core.hooksPath` and write the two hook bodies. Idempotent.
|
|
79
100
|
* `keywordFile` = the repo's configured leak-guard file, seeding target. */
|
|
80
101
|
export function installHooks(root: string, binInvocation: string, keywordFile?: string): { written: string[]; note: string } {
|
|
@@ -89,12 +110,12 @@ export function installHooks(root: string, binInvocation: string, keywordFile?:
|
|
|
89
110
|
const prev = existsSync(path) ? readFileSync(path, 'utf8') : ''
|
|
90
111
|
if (prev !== body) { writeFileSync(path, body, { mode: 0o755 }); written.push(`${HOOK_DIR}/${h}`) }
|
|
91
112
|
}
|
|
92
|
-
const cfg =
|
|
113
|
+
const cfg = setHooksPath(root)
|
|
93
114
|
const seeded = seedKeywordFile(root, keywordFile)
|
|
94
115
|
if (seeded != null) written.push(seeded)
|
|
95
116
|
return {
|
|
96
117
|
written,
|
|
97
|
-
note: cfg.
|
|
118
|
+
note: cfg.code === 0 ? `core.hooksPath=${HOOK_DIR}` : `WARN could not set core.hooksPath (exit ${cfg.code}) — hooks will not fire${cfg.detail === '' ? '' : `: ${cfg.detail}`}`,
|
|
98
119
|
}
|
|
99
120
|
}
|
|
100
121
|
|
package/src/init-repo.ts
CHANGED
|
@@ -9,7 +9,8 @@
|
|
|
9
9
|
import { existsSync, readFileSync, writeFileSync } from 'fs'
|
|
10
10
|
import { join } from 'path'
|
|
11
11
|
import { canonicalBodies } from './core/canonical-scripts.ts'
|
|
12
|
-
import { hasTestFiles } from './rules/SK001-package-json-shape.ts'
|
|
12
|
+
import { hasTestFiles, packageJsonShape } from './rules/SK001-package-json-shape.ts'
|
|
13
|
+
import { oneWayInPolicy } from './rules/SK013-one-way-in-policy.ts'
|
|
13
14
|
import { reportToMotherhome } from './core/motherhome.ts'
|
|
14
15
|
import { ensureGitRepo, publishRepo, type Run } from './git-bootstrap.ts'
|
|
15
16
|
import { defineRepo, loadRepoConfig, resolveFloor } from './repo-config.ts'
|
|
@@ -67,6 +68,9 @@ export async function initRepo(root: string, opts: InitOpts = {}): Promise<numbe
|
|
|
67
68
|
const cfg = await loadRepoConfig(root)
|
|
68
69
|
const g = await gitignorePolicy.fix!({ root, config: cfg })
|
|
69
70
|
p(`ok: .gitignore policy block ${g.changed ? 'written' : 'already current'}`)
|
|
71
|
+
// the repos that need ONE WAY IN most are the ones that already exist,
|
|
72
|
+
// so a re-run REPAIRS rather than only seeding a fresh checkout
|
|
73
|
+
await repairOneWayIn(root, cfg, p)
|
|
70
74
|
const mh = reportToMotherhome(root, cfg)
|
|
71
75
|
if (mh.note != null) p(mh.note)
|
|
72
76
|
if (wired && opts.noPublish !== true) await publishRepo(root, { github: opts.github, gh: opts.gh, print: p })
|
|
@@ -99,6 +103,7 @@ export async function initRepo(root: string, opts: InitOpts = {}): Promise<numbe
|
|
|
99
103
|
// `shipkit check --fix SK004`, one code path
|
|
100
104
|
const g = await gitignorePolicy.fix!({ root, config: defineRepo({ name }) })
|
|
101
105
|
p(`ok: .gitignore policy block ${g.changed ? 'written' : 'already current'}`)
|
|
106
|
+
await repairOneWayIn(root, defineRepo({ name }), p)
|
|
102
107
|
// what the gate actually runs, in ONE line: a thin floor must be seen at
|
|
103
108
|
// init time rather than discovered after a bad commit slipped through.
|
|
104
109
|
// The per-task resolution is diagnostics, and a `skip:` naming a script
|
|
@@ -128,3 +133,16 @@ export async function initRepo(root: string, opts: InitOpts = {}): Promise<numbe
|
|
|
128
133
|
if (opts.noPublish !== true) p('next: `shipkit ci` is the gate — run it to check your work')
|
|
129
134
|
return 0
|
|
130
135
|
}
|
|
136
|
+
|
|
137
|
+
/** ONE WAY IN, both halves, through the SAME fixers `shipkit check --fix` runs
|
|
138
|
+
* (build/shipkit.md). Nothing here is a second implementation: rule SK001
|
|
139
|
+
* routes the package.json scripts, rule SK013 writes the harness policy, and
|
|
140
|
+
* init just calls them so a wired repo is born enforced and a re-run repairs
|
|
141
|
+
* one that was not. The deny is a RESTRICTION, which is why it lands
|
|
142
|
+
* unconditionally where the scaffold's allow list stays opt in. */
|
|
143
|
+
async function repairOneWayIn(root: string, config: Parameters<typeof defineRepo>[0], p: (line: string) => void): Promise<void> {
|
|
144
|
+
const pkg = await packageJsonShape.fix!({ root, config })
|
|
145
|
+
if (pkg.changed) p(`ok: package.json — ${pkg.summary}`)
|
|
146
|
+
const policy = await oneWayInPolicy.fix!({ root, config })
|
|
147
|
+
p(policy.changed ? `ok: ${policy.summary}` : `ok: .claude/settings.json already closes every raw runner`)
|
|
148
|
+
}
|
|
@@ -0,0 +1,58 @@
|
|
|
1
|
+
// EVERY UNIT THE GATE RUNS, with what it declared about its own cacheability.
|
|
2
|
+
//
|
|
3
|
+
// Three registries feed one list, because a key layer that can only see tasks
|
|
4
|
+
// would leave the 25 checks and the 32 rules uncached forever, and those are
|
|
5
|
+
// most of the wall time. The declaration lives with the unit (a `TaskDef`
|
|
6
|
+
// field, a `Rule` field, an `export const inputs` in a check file), never in a
|
|
7
|
+
// table over here: a table beside the registry is a second source of truth
|
|
8
|
+
// that drifts the first time somebody adds a check.
|
|
9
|
+
import { relative } from 'path'
|
|
10
|
+
import { allTasks, type RepoConfig } from '../repo-config.ts'
|
|
11
|
+
import { discoverChecks } from '../core/discover-checks.ts'
|
|
12
|
+
import { RULES } from '../rules/rule-registry.ts'
|
|
13
|
+
import type { UnitDecl } from './git-key.ts'
|
|
14
|
+
|
|
15
|
+
/** every task, rule and discovered check, in the order the gate meets them. */
|
|
16
|
+
export function gateUnits(root: string, c: RepoConfig): UnitDecl[] {
|
|
17
|
+
const out: UnitDecl[] = []
|
|
18
|
+
for (const t of allTasks(c, root).tasks) {
|
|
19
|
+
out.push({
|
|
20
|
+
id: t.id,
|
|
21
|
+
kind: 'task',
|
|
22
|
+
inputs: t.inputs,
|
|
23
|
+
ttl: t.ttl,
|
|
24
|
+
never: t.never,
|
|
25
|
+
argv: t.cmd,
|
|
26
|
+
// the env a task SETS is part of what it runs, not something read
|
|
27
|
+
// off the ambient environment, so it rides as an identity string
|
|
28
|
+
extra: t.env == null ? undefined : JSON.stringify(Object.entries(t.env).sort()),
|
|
29
|
+
})
|
|
30
|
+
}
|
|
31
|
+
for (const r of [...RULES, ...(c.rules ?? [])]) {
|
|
32
|
+
out.push({
|
|
33
|
+
id: r.id,
|
|
34
|
+
kind: 'rule',
|
|
35
|
+
inputs: r.inputs,
|
|
36
|
+
code: r.code,
|
|
37
|
+
ttl: r.ttl,
|
|
38
|
+
never: r.never,
|
|
39
|
+
extra: r.cacheKey?.({ root, config: c }),
|
|
40
|
+
})
|
|
41
|
+
}
|
|
42
|
+
for (const d of discoverChecks(root)) {
|
|
43
|
+
const rel = relative(root, d.path).replace(/\\/g, '/')
|
|
44
|
+
out.push({
|
|
45
|
+
id: d.name,
|
|
46
|
+
kind: 'check',
|
|
47
|
+
inputs: d.inputs ?? undefined,
|
|
48
|
+
ttl: d.ttl ?? undefined,
|
|
49
|
+
never: d.never ?? undefined,
|
|
50
|
+
// a check IS a file, so its own code needs no declaration: the
|
|
51
|
+
// module graph walks from here through every helper it imports,
|
|
52
|
+
// which is what makes fixing a shared helper move the check's key
|
|
53
|
+
code: [rel],
|
|
54
|
+
argv: ['bun', rel],
|
|
55
|
+
})
|
|
56
|
+
}
|
|
57
|
+
return out
|
|
58
|
+
}
|
|
@@ -0,0 +1,330 @@
|
|
|
1
|
+
// THE KEY. What question did this unit answer, and is that still the same
|
|
2
|
+
// question? Nothing in this file caches anything: it produces the identity a
|
|
3
|
+
// later ledger will store results under, and it refuses to produce one
|
|
4
|
+
// whenever the answer would be a guess.
|
|
5
|
+
//
|
|
6
|
+
// THREE POPULATIONS, declared at registration, never inferred:
|
|
7
|
+
//
|
|
8
|
+
// CONTENT declares `inputs` — the paths it reads. Keyed on their git object
|
|
9
|
+
// ids, plus the unit's own code (module graph), plus the toolchain
|
|
10
|
+
// (which content-hashes the gate binary, so editing shipkit moves
|
|
11
|
+
// every key in every repo), plus argv and declared env. No TTL: a
|
|
12
|
+
// content key is either the same question or a different one.
|
|
13
|
+
// WORLD declares `ttl` — it reads the network, the keychain, $PATH, a git
|
|
14
|
+
// remote or the clock. Nothing on disk describes its answer, so it
|
|
15
|
+
// keeps the existing clean-pass TTL and its `cacheKey`, unchanged.
|
|
16
|
+
// NEITHER declares nothing. Never cached. This is the DEFAULT, and it is
|
|
17
|
+
// the whole safety story: forgetting to declare costs speed, never
|
|
18
|
+
// correctness, so the burn-down list below is a to-do rather than a
|
|
19
|
+
// hazard. Declaring BOTH is a registration failure, loud, because
|
|
20
|
+
// the two answers contradict each other.
|
|
21
|
+
//
|
|
22
|
+
// CROSS WORKTREE. Every component here is content addressed or a version
|
|
23
|
+
// string; not one of them is an absolute path. Two worktrees of one repo
|
|
24
|
+
// holding the same content and the same install therefore produce the same
|
|
25
|
+
// key, and a worktree whose node_modules differs produces a different one.
|
|
26
|
+
// Measured on this disk: main and two worktrees, main's install id
|
|
27
|
+
// e55a7fd4ae252e5a against fd0e2908d222fe2e shared by both worktrees, which
|
|
28
|
+
// differ from main by two top-level packages. That is what makes a shared
|
|
29
|
+
// ledger safe rather than a stale green in a checkout nobody looked at.
|
|
30
|
+
import { createHash } from 'crypto'
|
|
31
|
+
import { loadConfig, type RvtConfig } from '../ts/config.ts'
|
|
32
|
+
import { importClosure } from '../ts/source-graph.ts'
|
|
33
|
+
import { GitScanError, hashFiles, normalizeInput, scanPaths, type ScanResult } from './git-scan.ts'
|
|
34
|
+
import { toolchainIdentity } from './toolchain-identity.ts'
|
|
35
|
+
|
|
36
|
+
/** the key format. Bumped when a component is added or its rendering changes,
|
|
37
|
+
* so an old ledger can never answer a new question. */
|
|
38
|
+
export const KEY_VERSION = 1
|
|
39
|
+
|
|
40
|
+
export type Population = 'content' | 'world' | 'none'
|
|
41
|
+
|
|
42
|
+
/** what one gate unit (a task, a rule, a discovered check) declares about its
|
|
43
|
+
* own cacheability. `gate-units.ts` builds these from the live registries. */
|
|
44
|
+
export type UnitDecl = {
|
|
45
|
+
/** the id an operator types: a task id, a rule id, a check name */
|
|
46
|
+
id: string
|
|
47
|
+
kind: 'task' | 'rule' | 'check'
|
|
48
|
+
/** CONTENT: repo-relative paths this unit reads. Presence declares the population. */
|
|
49
|
+
inputs?: string[]
|
|
50
|
+
/** WORLD: clean-pass duration, e.g. '24h'. Presence declares the population. */
|
|
51
|
+
ttl?: string
|
|
52
|
+
/** repo-relative entry files of the unit's OWN code — the module-graph
|
|
53
|
+
* roots. A discovered check gets its own file for free. */
|
|
54
|
+
code?: string[]
|
|
55
|
+
/** the argv a subprocess unit runs */
|
|
56
|
+
argv?: string[]
|
|
57
|
+
/** env var NAMES whose values belong in the key */
|
|
58
|
+
env?: string[]
|
|
59
|
+
/** anything else that identifies the question (a rule's `cacheKey`) */
|
|
60
|
+
extra?: string
|
|
61
|
+
/** a DECIDED never-cacheable, with the reason. Not the same thing as an
|
|
62
|
+
* undeclared unit: this one has been looked at and cannot be described
|
|
63
|
+
* (it reads git history, a path outside the repo, the wall clock), so it
|
|
64
|
+
* leaves the burn-down list. Declaring it beside inputs or ttl is a
|
|
65
|
+
* registration failure, because those say the opposite. */
|
|
66
|
+
never?: string
|
|
67
|
+
}
|
|
68
|
+
|
|
69
|
+
export type UnitKey = {
|
|
70
|
+
id: string
|
|
71
|
+
kind: UnitDecl['kind']
|
|
72
|
+
population: Population
|
|
73
|
+
/** the content key, or null for world/none/refused */
|
|
74
|
+
key: string | null
|
|
75
|
+
/** why there is no content key. Always set when key is null. */
|
|
76
|
+
reason: string | null
|
|
77
|
+
/** every component that went into the key, in key order — this is what a
|
|
78
|
+
* human reads when a key moved and nobody knows why */
|
|
79
|
+
components: string[]
|
|
80
|
+
/** the input paths this unit was keyed on, with their object ids */
|
|
81
|
+
inputOids: [string, string][]
|
|
82
|
+
/** the identity of the unit's OWN code, module graph included. Set for
|
|
83
|
+
* every unit that has code, WORLD ones too: a TTL'd probe whose source
|
|
84
|
+
* changed must not replay its previous answer, and this is the value that
|
|
85
|
+
* busts it. null when the unit has no code entry or git could not hash it. */
|
|
86
|
+
codeKey: string | null
|
|
87
|
+
}
|
|
88
|
+
|
|
89
|
+
export type Registration = {
|
|
90
|
+
/** loud, blocking: a unit that declared two contradictory populations */
|
|
91
|
+
errors: string[]
|
|
92
|
+
/** the default-deny burn-down, one line per undeclared unit */
|
|
93
|
+
burnDown: string[]
|
|
94
|
+
/** every unit that was classified */
|
|
95
|
+
units: number
|
|
96
|
+
/** units in a cacheable population (content or world) */
|
|
97
|
+
declared: number
|
|
98
|
+
}
|
|
99
|
+
|
|
100
|
+
/** which population a declaration puts a unit in, before any git work.
|
|
101
|
+
* `inputs` on a `ttl` unit is a registration FAIL: a unit cannot both be
|
|
102
|
+
* described by files on disk and be un-describable by them. */
|
|
103
|
+
export function classify(d: UnitDecl): { population: Population; error?: string; reason?: string; decided?: boolean } {
|
|
104
|
+
const hasInputs = (d.inputs ?? []).length > 0
|
|
105
|
+
const hasTtl = d.ttl != null && d.ttl !== ''
|
|
106
|
+
const hasNever = d.never != null && d.never !== ''
|
|
107
|
+
if (hasInputs && hasTtl)
|
|
108
|
+
return {
|
|
109
|
+
population: 'none',
|
|
110
|
+
error: `${d.kind} '${d.id}' declares BOTH inputs (CONTENT) and ttl (WORLD) — a unit reads files or it reads the world, never both`,
|
|
111
|
+
}
|
|
112
|
+
if (hasNever && (hasInputs || hasTtl))
|
|
113
|
+
return {
|
|
114
|
+
population: 'none',
|
|
115
|
+
error: `${d.kind} '${d.id}' declares never:'${d.never}' AND ${hasInputs ? 'inputs' : 'ttl'} — never means nothing can describe this unit, which the other declaration contradicts`,
|
|
116
|
+
}
|
|
117
|
+
if (hasInputs) return { population: 'content' }
|
|
118
|
+
if (hasTtl) return { population: 'world' }
|
|
119
|
+
if (hasNever) return { population: 'none', reason: d.never, decided: true }
|
|
120
|
+
return { population: 'none', reason: 'declares neither inputs nor ttl' }
|
|
121
|
+
}
|
|
122
|
+
|
|
123
|
+
export function validateRegistration(decls: UnitDecl[]): Registration {
|
|
124
|
+
const errors: string[] = []
|
|
125
|
+
const burnDown: string[] = []
|
|
126
|
+
let declared = 0
|
|
127
|
+
for (const d of decls) {
|
|
128
|
+
const c = classify(d)
|
|
129
|
+
if (c.error != null) errors.push(c.error)
|
|
130
|
+
// a DECIDED never leaves the list: the burn-down is what nobody has
|
|
131
|
+
// looked at yet, and mixing the two turns it into wallpaper
|
|
132
|
+
else if (c.population !== 'none') declared++
|
|
133
|
+
else if (c.decided !== true) burnDown.push(`${d.kind} ${d.id} — ${c.reason ?? 'undeclared'}`)
|
|
134
|
+
// a declaration that cannot even be turned into a repo-relative path
|
|
135
|
+
// is a registration bug, not a cache miss: it would key the unit on a
|
|
136
|
+
// path in another repo, or on the repo root
|
|
137
|
+
for (const raw of [...(d.inputs ?? []), ...(d.code ?? [])]) {
|
|
138
|
+
try {
|
|
139
|
+
normalizeInput(raw)
|
|
140
|
+
} catch (e) {
|
|
141
|
+
errors.push(`${d.kind} '${d.id}': ${e instanceof Error ? e.message : String(e)}`)
|
|
142
|
+
}
|
|
143
|
+
}
|
|
144
|
+
}
|
|
145
|
+
return { errors, burnDown, units: decls.length, declared }
|
|
146
|
+
}
|
|
147
|
+
|
|
148
|
+
export type KeyLayerOpts = {
|
|
149
|
+
/** the cli entry actually driving this run — toolchain identity needs to
|
|
150
|
+
* know whether the gate binary lives inside the repo being gated */
|
|
151
|
+
cliEntry: string
|
|
152
|
+
/** module-graph config; loaded from the repo when omitted */
|
|
153
|
+
cfg?: RvtConfig
|
|
154
|
+
}
|
|
155
|
+
|
|
156
|
+
/**
|
|
157
|
+
* one scan, N keys. Build it once per run: the git work is shared across every
|
|
158
|
+
* unit, and building two of them in one run would pay for the scan twice and
|
|
159
|
+
* could straddle an edit.
|
|
160
|
+
*/
|
|
161
|
+
export class KeyLayer {
|
|
162
|
+
private constructor(
|
|
163
|
+
readonly root: string,
|
|
164
|
+
readonly decls: UnitDecl[],
|
|
165
|
+
readonly registration: Registration,
|
|
166
|
+
private readonly scan: ScanResult | null,
|
|
167
|
+
private readonly scanError: string | null,
|
|
168
|
+
private readonly closureByUnit: Map<string, string[]>,
|
|
169
|
+
private readonly codeOids: Map<string, string>,
|
|
170
|
+
private readonly toolchain: { id: string; uncacheable: string[] },
|
|
171
|
+
private readonly scanPathsUsed: string[],
|
|
172
|
+
/** wall time of the WHOLE layer: toolchain identity, the module-graph
|
|
173
|
+
* closures, the git scan and the closure hashing. Reported rather
|
|
174
|
+
* than the scan alone, because the scan alone is the flattering
|
|
175
|
+
* number and the caller pays all of it. */
|
|
176
|
+
readonly openMs: number,
|
|
177
|
+
) {}
|
|
178
|
+
|
|
179
|
+
/** wall time the GIT half alone cost, ms — `openMs` is what a run pays */
|
|
180
|
+
get scanMs(): number {
|
|
181
|
+
return this.scan?.ms ?? 0
|
|
182
|
+
}
|
|
183
|
+
|
|
184
|
+
get failure(): string | null {
|
|
185
|
+
return this.scanError
|
|
186
|
+
}
|
|
187
|
+
|
|
188
|
+
static open(root: string, decls: UnitDecl[], o: KeyLayerOpts): KeyLayer {
|
|
189
|
+
const started = performance.now()
|
|
190
|
+
const registration = validateRegistration(decls)
|
|
191
|
+
const cfg = o.cfg ?? loadConfig(root)
|
|
192
|
+
// a gate binary that cannot be resolved is not a crash, it is a run
|
|
193
|
+
// whose program nobody can name — and the only safe answer to that is
|
|
194
|
+
// no key at all
|
|
195
|
+
let toolchain: { id: string; uncacheable: string[] }
|
|
196
|
+
try {
|
|
197
|
+
toolchain = toolchainIdentity(root, o.cliEntry)
|
|
198
|
+
} catch (e) {
|
|
199
|
+
toolchain = { id: 'unresolved', uncacheable: [`gate binary unreadable at ${o.cliEntry}: ${e instanceof Error ? e.message : String(e)}`] }
|
|
200
|
+
}
|
|
201
|
+
|
|
202
|
+
// the unit's OWN code, resolved through the same module graph the
|
|
203
|
+
// `cycles` task walks. A check that imports a shared helper carries
|
|
204
|
+
// that helper, so fixing the helper moves the check's key.
|
|
205
|
+
const closureByUnit = new Map<string, string[]>()
|
|
206
|
+
// computed for EVERY unit with code, not only the content ones: a
|
|
207
|
+
// world unit's TTL cache must break when the probe itself is rewritten
|
|
208
|
+
for (const d of decls) {
|
|
209
|
+
const entries = d.code ?? []
|
|
210
|
+
if (entries.length > 0) closureByUnit.set(d.id, importClosure(cfg, entries))
|
|
211
|
+
}
|
|
212
|
+
|
|
213
|
+
// ONE scan for the declared inputs plus shipkit's own source: those
|
|
214
|
+
// are mostly directories, mostly unchanged, and git's stat cache
|
|
215
|
+
// answers them without reading a byte.
|
|
216
|
+
const paths = new Set<string>()
|
|
217
|
+
for (const d of decls) {
|
|
218
|
+
if (classify(d).population !== 'content') continue
|
|
219
|
+
for (const p of d.inputs ?? []) paths.add(p)
|
|
220
|
+
}
|
|
221
|
+
|
|
222
|
+
// the closure is INDIVIDUAL FILES belonging to no declared path, so it
|
|
223
|
+
// goes through one `hash-object --stdin-paths` instead of one `git
|
|
224
|
+
// add` spawn per file (measured: 15ms for the whole set)
|
|
225
|
+
const closureFiles = new Set<string>()
|
|
226
|
+
for (const files of closureByUnit.values()) for (const f of files) closureFiles.add(f)
|
|
227
|
+
|
|
228
|
+
let scan: ScanResult | null = null
|
|
229
|
+
let codeOids = new Map<string, string>()
|
|
230
|
+
let scanError: string | null = null
|
|
231
|
+
try {
|
|
232
|
+
scan = scanPaths(root, [...paths])
|
|
233
|
+
codeOids = hashFiles(root, [...closureFiles]).oids
|
|
234
|
+
} catch (e) {
|
|
235
|
+
// a scan that failed may have left a STALE tree behind, so nothing
|
|
236
|
+
// downstream may key on it. Loud, and the whole run goes uncached.
|
|
237
|
+
scanError = e instanceof GitScanError ? e.message : String(e)
|
|
238
|
+
}
|
|
239
|
+
return new KeyLayer(root, decls, registration, scan, scanError, closureByUnit, codeOids, toolchain, [...paths].sort(), performance.now() - started)
|
|
240
|
+
}
|
|
241
|
+
|
|
242
|
+
/** the key for one unit, or the reason there is none. */
|
|
243
|
+
keyFor(id: string): UnitKey {
|
|
244
|
+
const d = this.decls.find((u) => u.id === id)
|
|
245
|
+
if (d == null) throw new Error(`keyFor: no unit '${id}' in this layer`)
|
|
246
|
+
const c = classify(d)
|
|
247
|
+
const own = (this.closureByUnit.get(d.id) ?? []).slice().sort()
|
|
248
|
+
const ownReadable = own.every((p) => this.codeOids.get(p) != null)
|
|
249
|
+
const codeLines = own.map((p) => `code:${p}=${this.codeOids.get(p)!}`)
|
|
250
|
+
const base: UnitKey = {
|
|
251
|
+
id: d.id, kind: d.kind, population: c.population, key: null, reason: null, components: [], inputOids: [],
|
|
252
|
+
codeKey: own.length === 0 || !ownReadable ? null : createHash('sha256').update(codeLines.join('\n')).digest('hex').slice(0, 16),
|
|
253
|
+
}
|
|
254
|
+
if (c.error != null) return { ...base, reason: c.error }
|
|
255
|
+
if (c.population === 'world') return { ...base, reason: `world unit — TTL ${d.ttl}, no content key` }
|
|
256
|
+
if (c.population === 'none') return { ...base, reason: c.reason ?? 'undeclared' }
|
|
257
|
+
if (this.scanError != null) return { ...base, reason: `git scan failed, nothing may be keyed: ${this.scanError}` }
|
|
258
|
+
const scan = this.scan!
|
|
259
|
+
if (this.toolchain.uncacheable.length > 0)
|
|
260
|
+
return { ...base, reason: `toolchain not describable: ${this.toolchain.uncacheable.join('; ')}` }
|
|
261
|
+
|
|
262
|
+
let inputs: string[]
|
|
263
|
+
try {
|
|
264
|
+
inputs = (d.inputs ?? []).map(normalizeInput).sort()
|
|
265
|
+
} catch (e) {
|
|
266
|
+
return { ...base, reason: e instanceof Error ? e.message : String(e) }
|
|
267
|
+
}
|
|
268
|
+
const stale = inputs.filter((p) => scan.missing.includes(p))
|
|
269
|
+
if (stale.length > 0) return { ...base, reason: `declared input(s) not on disk: ${stale.join(', ')}` }
|
|
270
|
+
const hidden = inputs.flatMap((p) => scan.excluded.get(p) ?? [])
|
|
271
|
+
const inputOids: [string, string][] = inputs.map((p) => [p, scan.oidByPath.get(p) ?? ''])
|
|
272
|
+
if (inputOids.some(([, oid]) => oid === '')) return { ...base, reason: 'git could not name one of the declared inputs' }
|
|
273
|
+
|
|
274
|
+
if (!ownReadable) return { ...base, reason: `git could not hash this unit's own source: ${own.filter((p) => this.codeOids.get(p) == null).join(', ')}` }
|
|
275
|
+
const components: string[] = [
|
|
276
|
+
`v=${KEY_VERSION}`,
|
|
277
|
+
`unit=${d.kind}:${d.id}`,
|
|
278
|
+
`bun=${Bun.version}`,
|
|
279
|
+
`toolchain=${this.toolchain.id}`,
|
|
280
|
+
`argv=${JSON.stringify(d.argv ?? [])}`,
|
|
281
|
+
...(d.env ?? []).slice().sort().map((n) => `env:${n}=${process.env[n] ?? '<unset>'}`),
|
|
282
|
+
...inputOids.map(([p, oid]) => `in:${p}=${oid}`),
|
|
283
|
+
...codeLines,
|
|
284
|
+
...(d.extra != null ? [`extra=${d.extra}`] : []),
|
|
285
|
+
// ignored content nobody keyed on is part of the key's MEANING:
|
|
286
|
+
// two runs that excluded different things answered different
|
|
287
|
+
// questions, and a silently narrowed key is the false green
|
|
288
|
+
...hidden.sort().map((p) => `excluded=${p}`),
|
|
289
|
+
]
|
|
290
|
+
const key = createHash('sha256').update(components.join('\n')).digest('hex').slice(0, 32)
|
|
291
|
+
return { ...base, key, components, inputOids }
|
|
292
|
+
}
|
|
293
|
+
|
|
294
|
+
keys(): UnitKey[] {
|
|
295
|
+
return this.decls.map((d) => this.keyFor(d.id))
|
|
296
|
+
}
|
|
297
|
+
|
|
298
|
+
/**
|
|
299
|
+
* POST RUN RE HASH. Re-scan exactly what this layer keyed on and report
|
|
300
|
+
* every path whose object id moved while the units were running.
|
|
301
|
+
*
|
|
302
|
+
* This is the answer to the poisoning class: a unit reads a file, an agent
|
|
303
|
+
* in the next window edits it, and the result gets stored under a key that
|
|
304
|
+
* describes the file as it was BEFORE the edit. Any drift here means the
|
|
305
|
+
* run's results describe a tree that no longer exists, and the caller
|
|
306
|
+
* discards them rather than storing a lie. Measured at ~140ms, which is
|
|
307
|
+
* the price of never having to explain a hit that was wrong.
|
|
308
|
+
*/
|
|
309
|
+
rescan(): { drifted: string[]; error: string | null; ms: number } {
|
|
310
|
+
if (this.scanError != null) return { drifted: [], error: this.scanError, ms: 0 }
|
|
311
|
+
const started = performance.now()
|
|
312
|
+
try {
|
|
313
|
+
const after = scanPaths(this.root, this.scanPathsUsed)
|
|
314
|
+
const drifted: string[] = []
|
|
315
|
+
for (const [p, oid] of this.scan!.oidByPath) if (after.oidByPath.get(p) !== oid) drifted.push(p)
|
|
316
|
+
for (const [p] of after.oidByPath) if (!this.scan!.oidByPath.has(p)) drifted.push(p)
|
|
317
|
+
// the unit's own code drifts the same way a declared input does: an
|
|
318
|
+
// agent fixing a check WHILE that check runs would otherwise store
|
|
319
|
+
// the old verdict under the new code's key
|
|
320
|
+
const files = [...this.codeOids.keys()]
|
|
321
|
+
if (files.length > 0) {
|
|
322
|
+
const now = hashFiles(this.root, files)
|
|
323
|
+
for (const [p, oid] of this.codeOids) if (now.oids.get(p) !== oid) drifted.push(p)
|
|
324
|
+
}
|
|
325
|
+
return { drifted: [...new Set(drifted)].sort(), error: null, ms: performance.now() - started }
|
|
326
|
+
} catch (e) {
|
|
327
|
+
return { drifted: [], error: e instanceof GitScanError ? e.message : String(e), ms: performance.now() - started }
|
|
328
|
+
}
|
|
329
|
+
}
|
|
330
|
+
}
|