rvlib-shipkit 1.5.0 → 1.7.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +54 -0
- package/package.json +1 -1
- package/src/assets/gen/comfyts-gen.ts +12 -8
- package/src/assets/server/studio-server.ts +14 -2
- package/src/check.ts +5 -8
- package/src/checks/SK001-package-json-shape.check.ts +5 -4
- package/src/checks/SK003-claude-md-shape.check.ts +2 -2
- package/src/checks/SK030-cache-audit.check.ts +34 -0
- package/src/checks/SK031-trace-audit.check.ts +39 -0
- package/src/checks/SK032-ignored-inputs.check.ts +79 -0
- package/src/cli/cmd_Assets.ts +10 -6
- package/src/cli/cmd_CacheAudit.ts +43 -0
- package/src/cli/cmd_CrossRepoActivity.ts +2 -1
- package/src/cli/cmd_GlobalProcesses.ts +4 -3
- package/src/cli/cmd_Home.ts +3 -2
- package/src/cli/cmd_Insights.ts +78 -0
- package/src/cli/cmd_Keys.ts +12 -1
- package/src/cli/cmd_Overview.ts +1 -1
- package/src/cli/cmd_Serve.ts +2 -1
- package/src/cli/cmd_TraceAudit.ts +62 -0
- package/src/cli.ts +6 -0
- package/src/collections/collection-types.ts +3 -0
- package/src/collections/providers.ts +5 -3
- package/src/core/app-icon.ts +154 -0
- package/src/core/audit-report.ts +21 -0
- package/src/core/cache-audit.ts +72 -0
- package/src/core/canonical-scripts.ts +1 -1
- package/src/core/check-ctx.ts +14 -5
- package/src/core/check-runner.ts +6 -3
- package/src/core/floor-resolve.ts +5 -7
- package/src/core/git-root.ts +13 -0
- package/src/core/ignored-block.ts +94 -0
- package/src/core/ignored-inputs.ts +47 -0
- package/src/core/jobs.ts +9 -2
- package/src/core/one-lib.ts +59 -4
- package/src/core/read-set.ts +97 -9
- package/src/core/run-ci.ts +6 -1
- package/src/core/run-fixers.ts +8 -2
- package/src/core/trace-audit.ts +129 -0
- package/src/home/gui-app/ShipkitGui.swift +125 -41
- package/src/home/gui-app.ts +24 -27
- package/src/home/home-routes.ts +15 -0
- package/src/home/home-start.ts +11 -5
- package/src/home/insights-shape.ts +124 -0
- package/src/home/insights.ts +90 -0
- package/src/home/open-app.ts +2 -2
- package/src/home/serve-web.ts +3 -3
- package/src/home/via-hub.ts +14 -0
- package/src/home/widgets.ts +12 -2
- package/src/ledger/db.ts +2 -13
- package/src/ledger/gate-checks.ts +21 -3
- package/src/ledger/git-key.ts +148 -30
- package/src/ledger/git-scan.ts +5 -2
- package/src/ledger/ledger.ts +131 -16
- package/src/ledger/store.ts +14 -0
- package/src/ledger/toolchain-identity.ts +210 -25
- package/src/problems/extract.ts +3 -0
- package/src/repo-config.ts +18 -1
- package/src/serve/hub-app/admin-svc.ts +36 -3
- package/src/serve/hub-app/admin-view.tsx +33 -1
- package/src/serve/hub-app/col-card.tsx +1 -44
- package/src/serve/hub-app/col-tables-view.tsx +71 -5
- package/src/serve/hub-app/col-tables.ts +27 -0
- package/src/serve/hub-app/home-st.ts +13 -6
- package/src/serve/hub-app/home-strip.css +7 -9
- package/src/serve/hub-app/home-strip.tsx +8 -17
- package/src/serve/hub-app/home.html +1 -0
- package/src/serve/hub-app/hub-app.tsx +20 -4
- package/src/serve/hub-app/hub.css +24 -20
- package/src/serve/hub-app/icons.tsx +1 -0
- package/src/serve/hub-app/index.html +1 -0
- package/src/serve/hub-app/insights-st.ts +111 -0
- package/src/serve/hub-app/insights-view.tsx +112 -0
- package/src/serve/hub-app/insights.css +24 -0
- package/src/serve/hub-app/keys.ts +11 -3
- package/src/serve/hub-app/machine-st.ts +23 -0
- package/src/serve/hub-app/palette-st.ts +2 -1
- package/src/serve/hub-app/processes-view.tsx +16 -9
- package/src/serve/hub-app/project-list.css +4 -1
- package/src/serve/hub-app/project-list.tsx +9 -5
- package/src/serve/hub-app/projects-svc.ts +11 -1
- package/src/serve/hub-app/repo-sections.ts +31 -17
- package/src/serve/hub-app/repo-st.ts +25 -6
- package/src/serve/hub-app/sidebar.tsx +5 -5
- package/src/serve/hub-app/tree-build.ts +2 -0
- package/src/serve/hub-app/tree-row.tsx +18 -5
- package/src/serve/hub-app/tree-svc.ts +27 -7
- package/src/serve/hub-runtime.ts +4 -2
- package/src/serve/project-gutter-route.ts +22 -1
- package/src/shipkit-files.ts +3 -2
- package/src/suite/judge-suites.ts +6 -0
- package/src/trace/interpose.c +655 -0
- package/src/trace/read-trace.ts +485 -0
- package/src/trace/tracer.ts +282 -0
|
@@ -0,0 +1,485 @@
|
|
|
1
|
+
// A TRACE, TURNED INTO A READ SET, or into the reason no read set can be
|
|
2
|
+
// trusted. trace/interpose.c writes the lines, one log per process; this reads
|
|
3
|
+
// them after the task exited and decides what its answer depended on.
|
|
4
|
+
//
|
|
5
|
+
// Every rule here fails toward "not kept". A hole in the trace (a child that
|
|
6
|
+
// ran untraced), a socket to a service the task did not start, anything the
|
|
7
|
+
// task looked at that moved while it ran: each one keeps the answer out of
|
|
8
|
+
// the store, with the reason, and the next run measures again.
|
|
9
|
+
//
|
|
10
|
+
// What the read set holds:
|
|
11
|
+
// - a file the task opened: in the repo by content (with its stat as a fast
|
|
12
|
+
// path), outside it by stat.
|
|
13
|
+
// - a file it only stat'ed, by stat: a tool with an mtime cache (tsc -b,
|
|
14
|
+
// eslint --cache) decides from that alone whether to read the file at all.
|
|
15
|
+
// - every image it ran or loaded (its executables, their libraries, a
|
|
16
|
+
// dlopen'd addon), like a file it opened.
|
|
17
|
+
// - every path it probed, present or absent, and every dir it listed.
|
|
18
|
+
// - a local port it found closed, re-checked before a replay.
|
|
19
|
+
// A file the task READ and also wrote (`eslint --fix`, a tsbuildinfo, a
|
|
20
|
+
// sqlite file opened read-write) stays an input, by its content AFTER the run:
|
|
21
|
+
// the answer holds for the state the task left, and any later edit re-runs
|
|
22
|
+
// it. Only its mid-run guard is off, since the task moved it itself.
|
|
23
|
+
//
|
|
24
|
+
// What is NOT an input, by design:
|
|
25
|
+
// - what the task wrote without ever reading it: its pure outputs.
|
|
26
|
+
// - the OS and the tool caches: /System, /usr, /dev, the dyld and timezone
|
|
27
|
+
// databases, the temp dirs, ~/Library/Caches, ~/.bun/install/cache and
|
|
28
|
+
// ~/.cache. A verdict that changes with a tool's cache is that tool's bug.
|
|
29
|
+
// - `.git/objects/`: an object never changes under its name, and the refs
|
|
30
|
+
// and the index that choose among them are recorded like any file.
|
|
31
|
+
// - what the gate itself writes: `.shipkit/tmp/` (its run logs and scratch,
|
|
32
|
+
// which the suite runner writes and sweeps while it runs), the status
|
|
33
|
+
// files every gate rewrites at its end, and its store under git's dir (the
|
|
34
|
+
// ledger db the suite runner records timings in, the scan indexes). A check that reads them reads the
|
|
35
|
+
// previous gate's output, which moves on every run by construction.
|
|
36
|
+
// - a listing of an ancestor of the repo root: a resolver walks up to `/`,
|
|
37
|
+
// and a new sibling repo in ~/dev must not re-run anything. What it looked
|
|
38
|
+
// for up there (a node_modules, a package.json) is kept as an existence
|
|
39
|
+
// probe, so one appearing still counts.
|
|
40
|
+
//
|
|
41
|
+
// Known limits, the same as for every content key: environment variables,
|
|
42
|
+
// the clock, randomness, UDP sent without a connect, and Mach or XPC
|
|
43
|
+
// services, none of which a file trace can see.
|
|
44
|
+
import { readdirSync, readFileSync, realpathSync, statSync } from 'fs'
|
|
45
|
+
import { homedir, tmpdir } from 'os'
|
|
46
|
+
import { dirname, isAbsolute, join, relative, resolve, sep } from 'path'
|
|
47
|
+
import { dirEntries, entriesHash, kindOf, linkTarget, oidOnDisk, statSig, type PathKind, type ReadSet } from '../core/read-set.ts'
|
|
48
|
+
import { LAYOUT } from '../core/layout.ts'
|
|
49
|
+
import { probeGitRoot } from '../core/git-root.ts'
|
|
50
|
+
import { shipkitRoot } from '../ledger/db.ts'
|
|
51
|
+
|
|
52
|
+
/** a traced run, read back */
|
|
53
|
+
export type TraceResult = {
|
|
54
|
+
/** what the answer depended on, null when it cannot be kept */
|
|
55
|
+
reads: ReadSet | null
|
|
56
|
+
/** why it cannot be kept, null when it can */
|
|
57
|
+
why: string | null
|
|
58
|
+
/** executables that ran untraced, absolute: what tracer.ts may be able
|
|
59
|
+
* to make traceable for the next run */
|
|
60
|
+
blind: string[]
|
|
61
|
+
/** how many processes wrote a trace, and how many lines */
|
|
62
|
+
processes: number
|
|
63
|
+
lines: number
|
|
64
|
+
}
|
|
65
|
+
|
|
66
|
+
const O_WRONLY = 0x1
|
|
67
|
+
const O_RDWR = 0x2
|
|
68
|
+
const O_APPEND = 0x8
|
|
69
|
+
const O_CREAT = 0x200
|
|
70
|
+
const O_TRUNC = 0x400
|
|
71
|
+
const WRITES = O_WRONLY | O_RDWR | O_APPEND | O_CREAT | O_TRUNC
|
|
72
|
+
/** a failed call that says nothing is there */
|
|
73
|
+
const ABSENT = new Set([2, 20])
|
|
74
|
+
|
|
75
|
+
const real = (p: string): string => { try { return realpathSync(p) } catch { return p } }
|
|
76
|
+
|
|
77
|
+
/** path prefixes that are never an input: the OS, and the tool caches */
|
|
78
|
+
export function ignoredPrefixes(): string[] {
|
|
79
|
+
const home = homedir()
|
|
80
|
+
const t = real(tmpdir())
|
|
81
|
+
return [
|
|
82
|
+
'/System/', '/usr/lib/', '/usr/libexec/', '/usr/share/', '/usr/bin/', '/usr/sbin/', '/bin/', '/sbin/', '/dev/',
|
|
83
|
+
'/private/var/db/', '/var/db/', '/private/var/run/', '/var/run/', '/Library/Apple/', '/Library/Preferences/Logging/',
|
|
84
|
+
'/private/var/folders/', '/var/folders/', '/private/tmp/', '/tmp/', `${t}/`,
|
|
85
|
+
`${home}/Library/Caches/`, `${home}/.bun/install/cache/`, `${home}/.cache/`,
|
|
86
|
+
]
|
|
87
|
+
}
|
|
88
|
+
|
|
89
|
+
/** what the task saw at a path when it probed it: its kind, and for a file
|
|
90
|
+
* its size, mtime and inode. `mixed` when two looks disagreed */
|
|
91
|
+
type Seen = { kind: PathKind | '?'; sig: string | null; mixed: boolean }
|
|
92
|
+
|
|
93
|
+
type Parsed = {
|
|
94
|
+
/** pid → how many images it started traced, and the parent of the first */
|
|
95
|
+
started: Map<number, { n: number; ppid: number }>
|
|
96
|
+
/** pids a traced parent spawned, with what it asked to run */
|
|
97
|
+
spawned: Map<number, string>
|
|
98
|
+
/** pid → execs that went through, and the last image it asked for */
|
|
99
|
+
execs: Map<number, { n: number; exe: string }>
|
|
100
|
+
images: Set<string>
|
|
101
|
+
reads: Map<string, { size: number; mtime: string }>
|
|
102
|
+
seen: Map<string, Seen>
|
|
103
|
+
/** dir → its mtime when the task listed it (`mixed` when two listings saw
|
|
104
|
+
* two), and the wall clock of the first listing, ms */
|
|
105
|
+
dirs: Map<string, { mtime: string; at: number }>
|
|
106
|
+
/** dir → every entry name a listing of it returned */
|
|
107
|
+
names: Map<string, Set<string>>
|
|
108
|
+
/** symlink → the target the task read, `\0mixed` when two reads disagreed */
|
|
109
|
+
targets: Map<string, string>
|
|
110
|
+
/** file → its stat when the task last closed it after writing */
|
|
111
|
+
closed: Map<string, string>
|
|
112
|
+
/** an exec whose new image never wrote its start line */
|
|
113
|
+
unstarted: string[]
|
|
114
|
+
written: Set<string>
|
|
115
|
+
/** every connect, with how it ended: a non-blocking one takes the errno of
|
|
116
|
+
* the next SO_ERROR read on its fd, in its own process's log order */
|
|
117
|
+
connects: { family: string; addr: string; port: number; errno: number }[]
|
|
118
|
+
binds: Set<string>
|
|
119
|
+
processes: number
|
|
120
|
+
lines: number
|
|
121
|
+
}
|
|
122
|
+
|
|
123
|
+
/** `size:mtimeNs:ino` from an S or O line's fields, the form statSig gives */
|
|
124
|
+
const sigOf = (size: string, mtime: string, ino: string): string => {
|
|
125
|
+
const [sec, nsec] = mtime.split('.')
|
|
126
|
+
return `${size}:${BigInt(sec!) * 1_000_000_000n + BigInt(nsec ?? '0')}:${ino}`
|
|
127
|
+
}
|
|
128
|
+
|
|
129
|
+
function parse(dir: string): Parsed {
|
|
130
|
+
const p: Parsed = { started: new Map(), spawned: new Map(), execs: new Map(), images: new Set(), reads: new Map(), seen: new Map(), dirs: new Map(), names: new Map(), targets: new Map(), closed: new Map(), unstarted: [], written: new Set(), connects: [], binds: new Set(), processes: 0, lines: 0 }
|
|
131
|
+
const abs = (base: string, path: string): string => resolve(isAbsolute(path) || base === '' ? '/' : base, path)
|
|
132
|
+
const look = (path: string, kind: PathKind | '?', sig: string | null): void => {
|
|
133
|
+
const was = p.seen.get(path)
|
|
134
|
+
if (was == null) { p.seen.set(path, { kind, sig, mixed: false }); return }
|
|
135
|
+
if (kind !== '?' && was.kind !== '?' && (was.kind !== kind || (sig != null && was.sig != null && sig !== was.sig))) was.mixed = true
|
|
136
|
+
if (was.kind === '?') was.kind = kind
|
|
137
|
+
was.sig ??= sig
|
|
138
|
+
}
|
|
139
|
+
for (const f of readdirSync(dir)) {
|
|
140
|
+
if (!f.endsWith('.log')) continue
|
|
141
|
+
p.processes++
|
|
142
|
+
// a connect left in progress, by fd: fds are reused, so only the next
|
|
143
|
+
// answer on that fd in this log belongs to it
|
|
144
|
+
const pending = new Map<number, number>()
|
|
145
|
+
// an exec writes its line into its own pid's log, and the image it
|
|
146
|
+
// becomes writes its start line into the same log, after it
|
|
147
|
+
let awaiting: string | null = null
|
|
148
|
+
// a rename logs its two paths as two lines in a row
|
|
149
|
+
let renamed: string | null = null
|
|
150
|
+
for (const line of readFileSync(join(dir, f), 'utf8').split('\n')) {
|
|
151
|
+
if (line === '') continue
|
|
152
|
+
p.lines++
|
|
153
|
+
const c = line.split('\t')
|
|
154
|
+
if (c[0] !== 'W') renamed = null
|
|
155
|
+
switch (c[0]) {
|
|
156
|
+
case 'H': {
|
|
157
|
+
awaiting = null
|
|
158
|
+
const pid = Number(c[1])
|
|
159
|
+
const s = p.started.get(pid) ?? { n: 0, ppid: Number(c[2]) }
|
|
160
|
+
s.n++
|
|
161
|
+
p.started.set(pid, s)
|
|
162
|
+
if (c[3] != null && c[3] !== '') p.images.add(c[3])
|
|
163
|
+
break
|
|
164
|
+
}
|
|
165
|
+
case 'L': if (c[1] != null && c[1] !== '') p.images.add(c[1]); break
|
|
166
|
+
case 'P': {
|
|
167
|
+
const [, op, r, pidText, , path] = c
|
|
168
|
+
const pid = Number(pidText)
|
|
169
|
+
// a relative target names no file without the spawner's cwd; the
|
|
170
|
+
// child's own H and L lines give it absolute, and a child that
|
|
171
|
+
// wrote none is refused anyway
|
|
172
|
+
if (path != null && isAbsolute(path) && (op !== 'spawn' || r === '0')) p.images.add(path)
|
|
173
|
+
if (op === 'exec') { if (awaiting != null) p.unstarted.push(awaiting); awaiting = path ?? '' }
|
|
174
|
+
if (op === 'execfail') awaiting = null
|
|
175
|
+
if (op === 'spawn' && r === '0') p.spawned.set(pid, path ?? '')
|
|
176
|
+
else if (op === 'exec' || op === 'execfail') {
|
|
177
|
+
const e = p.execs.get(pid) ?? { n: 0, exe: path ?? '' }
|
|
178
|
+
e.n += op === 'exec' ? 1 : -1
|
|
179
|
+
if (op === 'exec') e.exe = path ?? e.exe
|
|
180
|
+
p.execs.set(pid, e)
|
|
181
|
+
}
|
|
182
|
+
break
|
|
183
|
+
}
|
|
184
|
+
case 'O': {
|
|
185
|
+
if (c.length < 7) break
|
|
186
|
+
const path = abs(c[3]!, c[4]!)
|
|
187
|
+
const flags = Number(c[1])
|
|
188
|
+
if ((flags & WRITES) !== 0) p.written.add(path)
|
|
189
|
+
// a read-write open reads too
|
|
190
|
+
const reads = (flags & WRITES) === 0 || (flags & O_RDWR) !== 0
|
|
191
|
+
if (c[2] === 'd') look(path, 'dir', null)
|
|
192
|
+
else if (reads && !p.reads.has(path)) p.reads.set(path, { size: Number(c[5]), mtime: c[6]! })
|
|
193
|
+
break
|
|
194
|
+
}
|
|
195
|
+
case 'S': {
|
|
196
|
+
if (c.length < 5) break
|
|
197
|
+
const kind = c[2] === 'f' ? 'file' : c[2] === 'd' ? 'dir' : '?'
|
|
198
|
+
const at = abs(c[3]!, c[4]!)
|
|
199
|
+
look(at, kind, c.length >= 8 ? sigOf(c[5]!, c[6]!, c[7]!) : null)
|
|
200
|
+
// an lstat that found a link learned that a link stands
|
|
201
|
+
// there: its target is recorded, with nothing to compare
|
|
202
|
+
if (c[2] === 'l' && !p.targets.has(at)) p.targets.set(at, '\0lstat')
|
|
203
|
+
break
|
|
204
|
+
}
|
|
205
|
+
case 'E': if (c.length >= 5) look(abs(c[3]!, c[4]!), ABSENT.has(Number(c[2])) ? null : '?', null); break
|
|
206
|
+
case 'D': {
|
|
207
|
+
if (c[1] == null || c[1] === '') break
|
|
208
|
+
const was = p.dirs.get(c[1])
|
|
209
|
+
const at = Number(c[3] ?? 0) * 1000
|
|
210
|
+
if (was == null) p.dirs.set(c[1], { mtime: c[2] ?? '', at })
|
|
211
|
+
else { if (was.mtime !== c[2]) was.mtime = 'mixed'; was.at = Math.min(was.at, at) }
|
|
212
|
+
break
|
|
213
|
+
}
|
|
214
|
+
case 'W': {
|
|
215
|
+
if (c.length < 4) break
|
|
216
|
+
const w = abs(c[2]!, c[3]!)
|
|
217
|
+
p.written.add(w)
|
|
218
|
+
// a file written under one name and renamed into place keeps
|
|
219
|
+
// the stat its close left
|
|
220
|
+
if (c[1] === 'rename' && renamed != null) {
|
|
221
|
+
const was = p.closed.get(norm(renamed))
|
|
222
|
+
if (was != null) p.closed.set(norm(w), was)
|
|
223
|
+
renamed = null
|
|
224
|
+
} else renamed = c[1] === 'rename' ? w : null
|
|
225
|
+
break
|
|
226
|
+
}
|
|
227
|
+
case 'M': {
|
|
228
|
+
if (c[1] == null || c[1] === '') break
|
|
229
|
+
const set = p.names.get(c[1]) ?? new Set<string>()
|
|
230
|
+
for (const n of (c[2] ?? '').split('\x1f')) if (n !== '') set.add(n)
|
|
231
|
+
p.names.set(c[1], set)
|
|
232
|
+
break
|
|
233
|
+
}
|
|
234
|
+
case 'K': {
|
|
235
|
+
if (c.length < 4) break
|
|
236
|
+
const at = abs(c[1]!, c[2]!)
|
|
237
|
+
const was = p.targets.get(at)
|
|
238
|
+
// a real read replaces what an lstat alone learned
|
|
239
|
+
p.targets.set(at, was == null || was === '\0lstat' || was === c[3] ? c[3] ?? '' : '\0mixed')
|
|
240
|
+
break
|
|
241
|
+
}
|
|
242
|
+
case 'C': if (c.length >= 5) p.closed.set(norm(c[1]!), sigOf(c[2]!, c[3]!, c[4]!)); break
|
|
243
|
+
case 'N': {
|
|
244
|
+
const errno = Number(c[5] ?? 0)
|
|
245
|
+
p.connects.push({ family: c[1] ?? '', addr: c[2] ?? '', port: Number(c[3] ?? 0), errno })
|
|
246
|
+
if (errno === EINPROGRESS) pending.set(Number(c[6] ?? -1), p.connects.length - 1)
|
|
247
|
+
break
|
|
248
|
+
}
|
|
249
|
+
case 'R': {
|
|
250
|
+
const i = pending.get(Number(c[1]))
|
|
251
|
+
if (i != null) { p.connects[i]!.errno = Number(c[2] ?? 0); pending.delete(Number(c[1])) }
|
|
252
|
+
break
|
|
253
|
+
}
|
|
254
|
+
case 'B': p.binds.add(c[1] === 'unix' ? `unix:${c[2]}` : `port:${c[3]}`); break
|
|
255
|
+
}
|
|
256
|
+
}
|
|
257
|
+
if (awaiting != null) p.unstarted.push(awaiting)
|
|
258
|
+
}
|
|
259
|
+
return p
|
|
260
|
+
}
|
|
261
|
+
|
|
262
|
+
/** a path with its dir's links resolved: /var and /private/var name one file,
|
|
263
|
+
* and a file the task removed has no path of its own left to resolve */
|
|
264
|
+
const norm = (p: string): string => join(real(dirname(p)), p.slice(dirname(p).length + 1))
|
|
265
|
+
|
|
266
|
+
/** the images that should have written a start line and did not. A pid owes
|
|
267
|
+
* one for a spawn a traced parent made, one per exec, and one more when it
|
|
268
|
+
* is a ROOT (started by the untraced gate): the root's own start line must
|
|
269
|
+
* not pay for an image it later exec'd into */
|
|
270
|
+
function blindOf(p: Parsed): string[] {
|
|
271
|
+
const blind: string[] = [...p.unstarted]
|
|
272
|
+
for (const pid of new Set([...p.started.keys(), ...p.spawned.keys(), ...p.execs.keys()])) {
|
|
273
|
+
const s = p.started.get(pid)
|
|
274
|
+
const root = s != null && !p.spawned.has(pid) && !p.started.has(s.ppid)
|
|
275
|
+
const owed = (p.spawned.has(pid) ? 1 : 0) + (p.execs.get(pid)?.n ?? 0) + (root ? 1 : 0)
|
|
276
|
+
if ((s?.n ?? 0) < owed) blind.push(p.execs.get(pid)?.exe ?? p.spawned.get(pid) ?? `pid ${pid}`)
|
|
277
|
+
}
|
|
278
|
+
return blind
|
|
279
|
+
}
|
|
280
|
+
|
|
281
|
+
/** repo paths the gate writes itself: never an input of what it runs */
|
|
282
|
+
const GATE_WRITES: string[] = [LAYOUT.tmp, LAYOUT.status, LAYOUT.statusFull, `${LAYOUT.status}.lock`, `${LAYOUT.statusFull}.lock`]
|
|
283
|
+
|
|
284
|
+
const LOOPBACK = /^(127\.|::1$|::ffff:127\.|0\.0\.0\.0$|::$)/
|
|
285
|
+
const ECONNREFUSED = 61
|
|
286
|
+
const EINPROGRESS = 36
|
|
287
|
+
/** AF_SYSTEM: kernel event sockets the runtime opens for itself */
|
|
288
|
+
const KERNEL = 'family32'
|
|
289
|
+
|
|
290
|
+
/** the connect that reached outside the task, in words, or null, and the
|
|
291
|
+
* local ports it found closed. A local port the task bound itself is inside
|
|
292
|
+
* it. A local port nothing listens on is an answer about this machine, so it
|
|
293
|
+
* joins the read set and a replay asks again. Any other connect reads a
|
|
294
|
+
* service the trace cannot see into.
|
|
295
|
+
*
|
|
296
|
+
* A non-blocking connect ends after the call, and the runtime may learn how
|
|
297
|
+
* from its event loop rather than through SO_ERROR, which the trace sees.
|
|
298
|
+
* Such a local port is asked again now: nothing listening means the task
|
|
299
|
+
* met a refusal too */
|
|
300
|
+
async function networkOf(p: Parsed): Promise<{ world: string | null; closed: string[] }> {
|
|
301
|
+
const unsure = new Map<number, string>()
|
|
302
|
+
const closed = new Set<string>()
|
|
303
|
+
for (const n of p.connects) {
|
|
304
|
+
if (n.family === KERNEL) continue
|
|
305
|
+
if (n.family === 'unix') { if (!p.binds.has(`unix:${n.addr}`)) return { world: `it connected to the socket ${n.addr}`, closed: [] }; continue }
|
|
306
|
+
if (n.family === 'inet' || n.family === 'inet6') {
|
|
307
|
+
if (LOOPBACK.test(n.addr) && p.binds.has(`port:${n.port}`)) continue
|
|
308
|
+
if (LOOPBACK.test(n.addr) && n.errno === ECONNREFUSED) { closed.add(String(n.port)); continue }
|
|
309
|
+
if (LOOPBACK.test(n.addr) && n.errno === EINPROGRESS) { unsure.set(n.port, n.addr); continue }
|
|
310
|
+
return { world: `it connected to ${n.addr}:${n.port}`, closed: [] }
|
|
311
|
+
}
|
|
312
|
+
return { world: `it opened a ${n.family} connection`, closed: [] }
|
|
313
|
+
}
|
|
314
|
+
for (const [port, addr] of unsure) {
|
|
315
|
+
if (await listening(port)) return { world: `it connected to ${addr}:${port}, where a service listens`, closed: [] }
|
|
316
|
+
closed.add(String(port))
|
|
317
|
+
}
|
|
318
|
+
return { world: null, closed: [...closed].sort() }
|
|
319
|
+
}
|
|
320
|
+
|
|
321
|
+
/** does anything accept on this local port right now? */
|
|
322
|
+
function listening(port: number): Promise<boolean> {
|
|
323
|
+
return new Promise((done) => {
|
|
324
|
+
const t = setTimeout(() => done(true), 1_000)
|
|
325
|
+
Bun.connect({ hostname: '127.0.0.1', port, socket: { open(s) { clearTimeout(t); s.end(); done(true) }, data() {}, connectError() { clearTimeout(t); done(false) } } }).catch(() => { clearTimeout(t); done(false) })
|
|
326
|
+
})
|
|
327
|
+
}
|
|
328
|
+
|
|
329
|
+
/** the stat a file had when it was opened still holds, as size and mtime */
|
|
330
|
+
function heldStill(abs: string, at: { size: number; mtime: string }): boolean {
|
|
331
|
+
const sig = statSig(abs)
|
|
332
|
+
if (sig === '-') return false
|
|
333
|
+
const [size, mtimeNs] = sig.split(':')
|
|
334
|
+
const [sec, nsec] = at.mtime.split('.')
|
|
335
|
+
return Number(size) === at.size && BigInt(mtimeNs!) === BigInt(sec!) * 1_000_000_000n + BigInt(nsec ?? '0')
|
|
336
|
+
}
|
|
337
|
+
|
|
338
|
+
/** an entry of `dir` the task did not write that came into being after
|
|
339
|
+
* `at` (ms), or null. APFS keeps a birth time per file */
|
|
340
|
+
function bornAfter(dir: string, at: number, written: (abs: string) => boolean): string | null {
|
|
341
|
+
let names: string[]
|
|
342
|
+
try { names = readdirSync(dir) } catch { return null }
|
|
343
|
+
for (const n of names) {
|
|
344
|
+
const abs = join(dir, n)
|
|
345
|
+
if (written(abs)) continue
|
|
346
|
+
try { if (statSync(abs).birthtimeMs > at) return abs } catch { /* gone: nothing was added */ }
|
|
347
|
+
}
|
|
348
|
+
return null
|
|
349
|
+
}
|
|
350
|
+
|
|
351
|
+
/** a dir's mtime as the D lines write it */
|
|
352
|
+
function dirMtime(abs: string): string | null {
|
|
353
|
+
try {
|
|
354
|
+
const s = statSync(abs, { bigint: true })
|
|
355
|
+
return `${s.mtimeNs / 1_000_000_000n}.${String(s.mtimeNs % 1_000_000_000n).padStart(9, '0')}`
|
|
356
|
+
} catch {
|
|
357
|
+
return null
|
|
358
|
+
}
|
|
359
|
+
}
|
|
360
|
+
|
|
361
|
+
/**
|
|
362
|
+
* read one trace dir. `root` is the repo; a path under it is recorded
|
|
363
|
+
* repo-relative and by content, one outside by stat.
|
|
364
|
+
*/
|
|
365
|
+
export async function readTrace(dir: string, root: string): Promise<TraceResult> {
|
|
366
|
+
const p = parse(dir)
|
|
367
|
+
const done = (why: string, blind: string[] = []): TraceResult => ({ reads: null, why, blind, processes: p.processes, lines: p.lines })
|
|
368
|
+
if (p.processes === 0) return done('the trace is empty: the task ran without the tracer')
|
|
369
|
+
|
|
370
|
+
const blind = blindOf(p)
|
|
371
|
+
if (blind.length > 0) {
|
|
372
|
+
const names = [...new Set(blind)]
|
|
373
|
+
return done(`\`${names[0]}\` ran untraced${names.length > 1 ? `, and ${names.length - 1} more` : ''}`, names)
|
|
374
|
+
}
|
|
375
|
+
const net = await networkOf(p)
|
|
376
|
+
if (net.world != null) return done(net.world)
|
|
377
|
+
|
|
378
|
+
const roots = [...new Set([resolve(root), real(root)])]
|
|
379
|
+
const inRepo = (abs: string): string | null => {
|
|
380
|
+
for (const r of roots) {
|
|
381
|
+
const rel = relative(r, abs)
|
|
382
|
+
if (rel === '') return '.'
|
|
383
|
+
if (!rel.startsWith('..') && !isAbsolute(rel)) return rel.split(sep).join('/')
|
|
384
|
+
}
|
|
385
|
+
return null
|
|
386
|
+
}
|
|
387
|
+
const skip = ignoredPrefixes()
|
|
388
|
+
const ancestors = new Set<string>()
|
|
389
|
+
for (const r of roots) for (let d = r; ; d = resolve(d, '..')) { ancestors.add(d); if (d === '/') break }
|
|
390
|
+
// the gate's own store: the ledger db (shared by every worktree) and this
|
|
391
|
+
// checkout's scan indexes
|
|
392
|
+
const gitDir = probeGitRoot(root).root?.gitDir
|
|
393
|
+
const state = [...new Set([shipkitRoot(root), ...(gitDir == null ? [] : [join(gitDir, 'shipkit')])].flatMap((d) => [resolve(d), real(d)]))].map((d) => `${d}/`)
|
|
394
|
+
const writtenSet = new Set([...p.written].map(norm))
|
|
395
|
+
const written = (abs: string): boolean => writtenSet.has(norm(abs))
|
|
396
|
+
// the dirs the task itself added to or removed from: their mtime moved
|
|
397
|
+
// for the task's own reasons
|
|
398
|
+
const touchedDirs = new Set([...p.written].map((w) => dirname(w)))
|
|
399
|
+
/** the key a path is recorded under, null when it is not an input */
|
|
400
|
+
const keyOf = (abs: string): string | null => {
|
|
401
|
+
if (state.some((d) => abs.startsWith(d))) return null
|
|
402
|
+
const rel = inRepo(abs)
|
|
403
|
+
if (rel != null) return rel.startsWith('.git/objects/') || GATE_WRITES.some((w) => rel === w || rel.startsWith(`${w}/`)) ? null : rel
|
|
404
|
+
return skip.some((s) => abs.startsWith(s) || `${abs}/` === s) ? null : abs
|
|
405
|
+
}
|
|
406
|
+
const moved = (k: string): TraceResult => done(`${k} changed while it ran`)
|
|
407
|
+
|
|
408
|
+
const set: ReadSet = { files: {}, exists: {}, lists: {}, dirs: {}, sigs: {}, targets: {} }
|
|
409
|
+
/** a file by stat, and by content in the repo */
|
|
410
|
+
const byFile = (abs: string, k: string): void => {
|
|
411
|
+
set.sigs![k] = statSig(abs)
|
|
412
|
+
if (!isAbsolute(k)) set.files[k] = oidOnDisk(abs)
|
|
413
|
+
}
|
|
414
|
+
for (const [abs, at] of p.reads) {
|
|
415
|
+
const k = keyOf(abs)
|
|
416
|
+
if (k == null) continue
|
|
417
|
+
// a file another process changed while this task read it: the answer
|
|
418
|
+
// describes content that is gone, and nothing on disk says which
|
|
419
|
+
if (!written(abs) && !heldStill(abs, at)) return moved(k)
|
|
420
|
+
// one the task rewrote must still be what its last close left
|
|
421
|
+
const left = p.closed.get(norm(abs))
|
|
422
|
+
if (written(abs) && left != null && left !== statSig(abs)) return moved(k)
|
|
423
|
+
byFile(abs, k)
|
|
424
|
+
}
|
|
425
|
+
for (const abs of p.images) {
|
|
426
|
+
const k = keyOf(abs)
|
|
427
|
+
if (k != null && !(k in set.sigs!)) byFile(abs, k)
|
|
428
|
+
}
|
|
429
|
+
for (const [abs, seen] of p.seen) {
|
|
430
|
+
const k = keyOf(abs)
|
|
431
|
+
if (k == null || k in set.sigs!) continue
|
|
432
|
+
const now = kindOf(abs)
|
|
433
|
+
if (!written(abs)) {
|
|
434
|
+
// what the task saw then, against what stands there now: a file
|
|
435
|
+
// appearing after the task looked for it, or edited after it was
|
|
436
|
+
// stat'ed, changed the answer under it
|
|
437
|
+
if (seen.mixed) return moved(k)
|
|
438
|
+
if ((seen.kind === 'file' || seen.kind === 'dir' || seen.kind === null) && seen.kind !== now) return moved(k)
|
|
439
|
+
if (seen.sig != null && seen.sig !== statSig(abs)) return moved(k)
|
|
440
|
+
}
|
|
441
|
+
if (now === 'file' && seen.sig != null) set.sigs![k] = statSig(abs)
|
|
442
|
+
else set.exists[k] = now
|
|
443
|
+
}
|
|
444
|
+
for (const [abs, { mtime, at }] of p.dirs) {
|
|
445
|
+
if (ancestors.has(abs)) continue
|
|
446
|
+
const k = keyOf(abs)
|
|
447
|
+
if (k == null) continue
|
|
448
|
+
// an entry another process added, removed or renamed while the task
|
|
449
|
+
// listed the dir: a test file created mid-run that the runner never
|
|
450
|
+
// saw. The names the listing returned, against the names there now,
|
|
451
|
+
// the ones the task wrote itself set aside
|
|
452
|
+
const names = p.names.get(abs)
|
|
453
|
+
if (names != null) {
|
|
454
|
+
let now: string[] = []
|
|
455
|
+
try { now = readdirSync(abs) } catch { now = [] }
|
|
456
|
+
const others = (n: string): boolean => !written(join(abs, n))
|
|
457
|
+
if (now.some((n) => others(n) && !names.has(n)) || [...names].some((n) => others(n) && !now.includes(n))) return moved(`${k}/`)
|
|
458
|
+
} else {
|
|
459
|
+
// a listing with no names (getattrlistbulk): its mtime, and in a dir
|
|
460
|
+
// the task wrote into, no entry born after the listing
|
|
461
|
+
if (!touchedDirs.has(abs) && (mtime === 'mixed' || mtime !== dirMtime(abs))) return moved(`${k}/`)
|
|
462
|
+
if (touchedDirs.has(abs) && bornAfter(abs, at, written) != null) return moved(`${k}/`)
|
|
463
|
+
}
|
|
464
|
+
set.dirs![k] = entriesHash(dirEntries(abs))
|
|
465
|
+
}
|
|
466
|
+
for (const [abs, target] of p.targets) {
|
|
467
|
+
const k = keyOf(abs)
|
|
468
|
+
if (k == null) continue
|
|
469
|
+
const now = linkTarget(abs)
|
|
470
|
+
if (!written(abs) && target !== '\0lstat' && (target === '\0mixed' || target !== now)) return moved(k)
|
|
471
|
+
set.targets![k] = now
|
|
472
|
+
}
|
|
473
|
+
const sorted = <V>(o: Record<string, V>): Record<string, V> => Object.fromEntries(Object.entries(o).sort(([a], [b]) => (a < b ? -1 : a > b ? 1 : 0)))
|
|
474
|
+
return {
|
|
475
|
+
reads: {
|
|
476
|
+
files: sorted(set.files), exists: sorted(set.exists), lists: {}, dirs: sorted(set.dirs!), sigs: sorted(set.sigs!),
|
|
477
|
+
...(Object.keys(set.targets!).length > 0 ? { targets: sorted(set.targets!) } : {}),
|
|
478
|
+
...(net.closed.length > 0 ? { ports: Object.fromEntries(net.closed.map((port) => [port, 'closed' as const])) } : {}),
|
|
479
|
+
},
|
|
480
|
+
why: null,
|
|
481
|
+
blind: [],
|
|
482
|
+
processes: p.processes,
|
|
483
|
+
lines: p.lines,
|
|
484
|
+
}
|
|
485
|
+
}
|