rvlib-shipkit 1.5.0 → 1.6.0

package/src/trace/tracer.ts

@@ -0,0 +1,282 @@
+// TRACING A TASK: the interposer built once per machine and version, the env a
+// traced spawn carries, and the traceable copies of the binaries macOS would
+// otherwise run blind.
+//
+// Why a copy works: dyld ignores DYLD_INSERT_LIBRARIES in a binary signed with
+// the hardened runtime or library validation (Microsoft's native tsc, Xcode's
+// git), and SIP drops it for everything under /usr and /bin. An ad hoc signed
+// COPY of the same bytes is neither, so it runs traced. It sits beside the
+// original, because a tool finds its own files next to itself (tsc's lib
+// files), or for git in a prefix of links to the original's dirs. The system
+// shells cannot be copied: they are arm64e, which macOS runs only when Apple
+// signed them, so a script that needs /bin/sh stays untraced and says so.
+//
+// The tracer only AUDITS (core/trace-audit.ts): no verdict depends on a trace.
+// It is on wherever it can run (Apple silicon macOS and a C compiler). A
+// machine opts out with `sk trace-audit --disable`, which the daemon's health
+// pass sees, and one run with SHIPKIT_TRACE=0.
+import { createHash } from 'crypto'
+import { chmodSync, copyFileSync, existsSync, lstatSync, mkdirSync, mkdtempSync, readdirSync, readFileSync, renameSync, rmSync, statSync, symlinkSync, writeFileSync } from 'fs'
+import { homedir, tmpdir } from 'os'
+import { basename, dirname, join } from 'path'
+import { machineDelete, machineGet, machineSet } from '../core/machine-db.ts'
+
+/** the env var that turns tracing on (1) or off (0) for one run, over the
+ *  machine's setting */
+export const TRACE_ENV = 'SHIPKIT_TRACE'
+
+/** the machine setting `sk trace-audit --disable` writes (false) */
+export const TRACE_SETTING = 'trace.enabled'
+
+let settingMemo: { at: number; on: boolean } | null = null
+/** whether this machine left tracing on, asked at most every few seconds:
+ *  the composer asks it for every repo */
+export function traceEnabledHere(now = Date.now()): boolean {
+    if (settingMemo != null && now - settingMemo.at < 5_000) return settingMemo.on
+    settingMemo = { at: now, on: machineGet<boolean>(TRACE_SETTING) !== false }
+    return settingMemo.on
+}
+
+/** turn tracing back on for this machine (the default), or off */
+export function setTraceEnabled(on: boolean): void {
+    if (on) machineDelete(TRACE_SETTING)
+    else machineSet(TRACE_SETTING, false)
+    settingMemo = null
+}
+
+const SOURCE = join(import.meta.dir, 'interpose.c')
+
+/** where the built interposer and the traceable copies live */
+export const traceCache = (): string => process.env['SHIPKIT_TRACE_CACHE'] ?? join(homedir(), '.cache', 'shipkit', 'trace')
+
+/** why this machine cannot trace, null when it can. Cheap: no build */
+export function traceUnavailable(env: Record<string, string | undefined> = process.env): string | null {
+    const on = env[TRACE_ENV] === '1' || (env[TRACE_ENV] !== '0' && traceEnabledHere())
+    if (!on) return env[TRACE_ENV] === '0' ? 'tracing is off for this run (SHIPKIT_TRACE=0)' : 'tracing is off on this machine: sk trace-audit --enable turns it back on'
+    if (process.platform !== 'darwin' || process.arch !== 'arm64') return 'tracing runs on Apple silicon macOS only'
+    if (compiler() == null) return 'no C compiler to build the tracer (xcode-select --install)'
+    return null
+}
+
+const compiler = (): string | null => (existsSync('/usr/bin/clang') ? '/usr/bin/clang' : Bun.which('clang'))
+
+let built: { path: string } | { reason: string } | null = null
+
+/** the interposer for this source, built on first use. A build that fails is
+ *  a reason, never a throw: the task then runs untraced and is not kept */
+export function tracerLib(env: Record<string, string | undefined> = process.env): { path: string } | { reason: string } {
+    // the switch is asked every time: only the build is remembered
+    const off = traceUnavailable(env)
+    if (off != null) return { reason: off }
+    if (built != null) return built
+    let src: Buffer
+    try { src = readFileSync(SOURCE) } catch (e) { return (built = { reason: `the tracer source is missing: ${(e as Error).message}` }) }
+    const id = createHash('sha256').update(src).digest('hex').slice(0, 16)
+    const out = join(traceCache(), `interpose-${id}.dylib`)
+    if (existsSync(out)) return (built = { path: out })
+    mkdirSync(traceCache(), { recursive: true })
+    // built beside its final name and renamed in, so a parallel gate never
+    // loads a half-written library
+    const tmp = `${out}.${process.pid}.tmp`
+    const r = Bun.spawnSync([compiler()!, '-dynamiclib', '-O2', '-arch', 'arm64', '-arch', 'arm64e', '-Wno-deprecated-declarations', '-o', tmp, SOURCE], { stdout: 'pipe', stderr: 'pipe' })
+    if (r.exitCode !== 0) {
+        rmSync(tmp, { force: true })
+        return (built = { reason: `the tracer did not build: ${r.stderr.toString().trim().split('\n')[0] ?? `exit ${r.exitCode}`}` })
+    }
+    renameSync(tmp, out)
+    return (built = { path: out })
+}
+
+/** the remap file the interposer reads: original path, copy, and the size and
+ *  mtime the copy was made from, so a replaced original is never swapped */
+const remapFile = (): string => join(traceCache(), 'remap.tsv')
+
+type Remap = { from: string; to: string; size: number; mtime: number; env: string }
+
+function readRemaps(): Remap[] {
+    try {
+        return readFileSync(remapFile(), 'utf8').split('\n').filter((l) => l !== '' && !l.startsWith('#')).map((l) => {
+            const [from, to, size, mtime, env] = l.split('\t')
+            return { from: from!, to: to!, size: Number(size), mtime: Number(mtime), env: env ?? '' }
+        })
+    } catch {
+        return []
+    }
+}
+
+function writeRemaps(rows: Remap[]): void {
+    mkdirSync(traceCache(), { recursive: true })
+    const tmp = `${remapFile()}.${process.pid}.tmp`
+    writeFileSync(tmp, rows.map((r) => [r.from, r.to, r.size, r.mtime, r.env].join('\t')).join('\n') + '\n')
+    renameSync(tmp, remapFile())
+}
+
+const fresh = (r: Remap): boolean => {
+    try {
+        const s = statSync(r.from)
+        return s.size === r.size && Math.floor(s.mtimeMs / 1000) === r.mtime && existsSync(r.to)
+    } catch {
+        return false
+    }
+}
+
+function upsert(rows: Remap[], from: string, to: string, env = ''): Remap[] {
+    const s = statSync(from)
+    return [...rows.filter((r) => r.from !== from), { from, to, size: s.size, mtime: Math.floor(s.mtimeMs / 1000), env }]
+}
+
+/** an ad hoc signed copy of `bin` at `to`: the same bytes, no hardened runtime */
+function signedCopy(bin: string, to: string): string | null {
+    mkdirSync(dirname(to), { recursive: true })
+    const tmp = `${to}.${process.pid}.tmp`
+    try {
+        copyFileSync(bin, tmp)
+        chmodSync(tmp, 0o755)
+        const r = Bun.spawnSync(['/usr/bin/codesign', '-f', '-s', '-', tmp], { stdout: 'pipe', stderr: 'pipe' })
+        if (r.exitCode !== 0) { rmSync(tmp, { force: true }); return `codesign refused it: ${r.stderr.toString().trim().split('\n').at(-1)}` }
+        renameSync(tmp, to)
+        return null
+    } catch (e) {
+        rmSync(tmp, { force: true })
+        return (e as Error).message
+    }
+}
+
+/** /usr/bin/git is a shim into Xcode's git, and both are blind. The copy
+ *  lives in a prefix whose other dirs link to Xcode's, so it finds its
+ *  exec-path, templates and system config exactly where the original does */
+function gitRemap(rows: Remap[]): Remap[] {
+    if (!existsSync('/usr/bin/git')) return rows
+    const found = Bun.spawnSync(['/usr/bin/xcrun', '-f', 'git'], { stdout: 'pipe', stderr: 'pipe' })
+    const dev = found.exitCode === 0 ? found.stdout.toString().trim() : ''
+    if (dev === '' || !existsSync(dev)) return rows
+    const s = statSync(dev)
+    const prefix = join(traceCache(), `git-${createHash('sha256').update(`${dev}\0${s.size}\0${s.mtimeMs}`).digest('hex').slice(0, 12)}`)
+    const copy = join(prefix, 'usr', 'bin', 'git')
+    if (!existsSync(copy)) {
+        const devUsr = dirname(dirname(dev))
+        mkdirSync(join(prefix, 'usr'), { recursive: true })
+        for (const d of readdirSync(devUsr)) if (d !== 'bin' && !existsSync(join(prefix, 'usr', d))) symlinkSync(join(devUsr, d), join(prefix, 'usr', d))
+        if (signedCopy(dev, copy) != null) return rows
+    }
+    return upsert(upsert(rows, '/usr/bin/git', copy), dev, copy)
+}
+
+/** the remap file for this run, with every entry still describing its
+ *  original. Rebuilt when one went stale (a `bun install` replaced a copy) */
+export function remaps(): string {
+    let rows = readRemaps()
+    const git = rows.find((r) => r.from === '/usr/bin/git')
+    if (git == null || !fresh(git)) rows = gitRemap(rows)
+    const kept = rows.filter(fresh)
+    if (kept.length !== rows.length || !existsSync(remapFile())) writeRemaps(kept)
+    return remapFile()
+}
+
+/** SIP paths, where no copy runs traced */
+const PROTECTED = /^\/(usr\/(bin|sbin|libexec|lib)|bin|sbin|System)\//
+
+/** a binary that ran blind, made traceable for the next run when it can be:
+ *  a hardened binary a repo installed. Returns what happened, in words, for
+ *  the line that says the answer was not kept */
+export function learnBlind(paths: string[]): string | null {
+    const said: string[] = []
+    let rows = readRemaps()
+    for (const p of paths) {
+        if (PROTECTED.test(p)) { said.push(`macOS lets no tool trace ${p}`); continue }
+        let head: Buffer
+        try { head = readFileSync(p).subarray(0, 256) } catch { continue }
+        if (head.subarray(0, 2).toString() === '#!') {
+            const interp = head.toString().slice(2).split('\n')[0]!.trim().split(/\s+/)[0] ?? ''
+            said.push(`${basename(p)} runs through ${interp}${PROTECTED.test(interp) ? ', which macOS lets no tool trace' : ''}`)
+            continue
+        }
+        const magic = head.readUInt32BE(0)
+        if (![0xcffaedfe, 0xcafebabe, 0xfeedfacf].includes(magic)) continue
+        // a copy goes beside the original, so only into an install the repo
+        // owns, and only for a binary whose signature is what blinds it
+        const sign = Bun.spawnSync(['/usr/bin/codesign', '-dv', p], { stdout: 'pipe', stderr: 'pipe' }).stderr.toString()
+        if (!p.includes('/node_modules/') || !/flags=0x[0-9a-f]+\((?:[^)]*)(runtime|library-validation)/.test(sign)) { said.push(`${p} runs untraced and cannot be made traceable`); continue }
+        const copy = join(dirname(p), `.${basename(p)}.shipkit-traced`)
+        const failed = signedCopy(p, copy)
+        if (failed != null) { said.push(`${p} cannot be copied: ${failed}`); continue }
+        rows = upsert(rows, p, copy)
+        said.push(`a traceable copy of ${basename(p)} is ready for the next run`)
+    }
+    writeRemaps(rows)
+    return said.length === 0 ? null : said.join('; ')
+}
+
+/** a plain script: commands, arguments, quotes, `&&`, `||`, `;` and `|`.
+ *  Bun's shell runs exactly that the way sh does, in process, so a traced
+ *  task keeps its trace; anything else is left to the system shell */
+export function plainScript(body: string): boolean {
+    if (!/^[\w\s@%+=:,./'"&|;-]*$/.test(body)) return false
+    // an unclosed quote runs the rest of the line into one word
+    if ((body.match(/'/g) ?? []).length % 2 === 1 || (body.match(/"/g) ?? []).length % 2 === 1) return false
+    const words = body.split(/[\s&|;]+/).filter((w) => w !== '')
+    const shellOnly = new Set(['export', 'set', 'source', '.', 'eval', 'exec', 'trap', 'if', 'then', 'else', 'fi', 'for', 'do', 'done', 'while', 'case', 'esac', 'function', 'unset', 'shift', 'read'])
+    return !words.some((w) => shellOnly.has(w))
+}
+
+/** what a traced task actually runs, where its trace goes, and the env that
+ *  traces it, or why it cannot be traced. `bun run <script>` runs a plain
+ *  script through bun's shell, the only one macOS lets a trace into */
+export function tracedCommand(cmd: string[], cwd: string, env: Record<string, string | undefined> = process.env): { cmd: string[]; dir: string; env: Record<string, string>; note: string | null } | { reason: string } {
+    const lib = tracerLib(env)
+    if ('reason' in lib) return lib
+    // the copies first: the root command may be one of the binaries they stand for
+    const remap = remaps()
+    let run = rootCommand(cmd)
+    let note: string | null = null
+    if (cmd.length === 3 && basename(cmd[0]!) === 'bun' && cmd[1] === 'run') {
+        let body: string | null = null
+        try { body = (JSON.parse(readFileSync(join(cwd, 'package.json'), 'utf8')) as { scripts?: Record<string, string> }).scripts?.[cmd[2]!] ?? null } catch { body = null }
+        if (body != null && plainScript(body)) run = [cmd[0]!, 'run', '--shell=bun', cmd[2]!]
+        else if (body != null) note = `its script \`${cmd[2]}\` uses shell syntax only /bin/sh runs`
+    }
+    return { cmd: run, ...traceSpawn(lib.path, remap), note }
+}
+
+/** the task's own command, spawned the way the interposer spawns a child:
+ *  the gate itself runs untraced, so nobody else rewrites a
+ *  `#!/usr/bin/env node` launcher (a floor slot's `node_modules/.bin/tsc`)
+ *  or swaps a hardened binary for its traceable copy */
+export function rootCommand(cmd: string[]): string[] {
+    if (cmd.length === 0) return cmd
+    const first = cmd[0]!
+    const path = first.includes('/') ? first : Bun.which(first) ?? first
+    let head = ''
+    try { head = readFileSync(path).subarray(0, 512).toString() } catch { return cmd }
+    if (head.startsWith('#!')) {
+        const [interp, prog, ...rest] = head.slice(2).split('\n')[0]!.trim().split(/\s+/)
+        if (interp === '/usr/bin/env' && prog != null && !prog.startsWith('-') && rest.length === 0) {
+            const resolved = Bun.which(prog)
+            if (resolved != null) return [resolved, path, ...cmd.slice(1)]
+        }
+        return cmd
+    }
+    const copy = readRemaps().find((r) => r.from === path && fresh(r))
+    return copy == null ? cmd : [copy.to, ...cmd.slice(1)]
+}
+
+/** a fresh dir for one traced spawn, and the env that traces it. A gate
+ *  running inside a traced task keeps the outer dir, so both see the run */
+export function traceSpawn(lib: string, remap: string = remaps()): { dir: string; env: Record<string, string> } {
+    const dir = mkdtempSync(join(tmpdir(), 'shipkit-trace-'))
+    const outer = process.env['SHIPKIT_TRACE_DIR']
+    const inserted = process.env['DYLD_INSERT_LIBRARIES']
+    return {
+        dir,
+        env: {
+            SHIPKIT_TRACE_DIR: outer == null || outer === '' ? dir : `${outer}:${dir}`,
+            DYLD_INSERT_LIBRARIES: inserted == null || inserted === '' ? lib : inserted.split(':').includes(lib) ? inserted : `${lib}:${inserted}`,
+            SHIPKIT_TRACE_REMAP: remap,
+        },
+    }
+}
+
+/** a trace dir goes once it is read */
+export function dropTrace(dir: string): void {
+    try { if (lstatSync(dir).isDirectory()) rmSync(dir, { recursive: true, force: true }) } catch { /* gone either way */ }
+}