rvlib-shipkit 1.5.0 → 1.6.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +46 -0
- package/package.json +1 -1
- package/src/assets/gen/comfyts-gen.ts +12 -8
- package/src/assets/server/studio-server.ts +14 -2
- package/src/check.ts +5 -8
- package/src/checks/SK001-package-json-shape.check.ts +5 -4
- package/src/checks/SK003-claude-md-shape.check.ts +2 -2
- package/src/checks/SK030-cache-audit.check.ts +34 -0
- package/src/checks/SK031-trace-audit.check.ts +39 -0
- package/src/checks/SK032-ignored-inputs.check.ts +79 -0
- package/src/cli/cmd_Assets.ts +10 -6
- package/src/cli/cmd_CacheAudit.ts +43 -0
- package/src/cli/cmd_CrossRepoActivity.ts +2 -1
- package/src/cli/cmd_GlobalProcesses.ts +4 -3
- package/src/cli/cmd_Home.ts +3 -2
- package/src/cli/cmd_Keys.ts +12 -1
- package/src/cli/cmd_Overview.ts +1 -1
- package/src/cli/cmd_Serve.ts +2 -1
- package/src/cli/cmd_TraceAudit.ts +62 -0
- package/src/cli.ts +4 -0
- package/src/collections/collection-types.ts +3 -0
- package/src/collections/providers.ts +5 -3
- package/src/core/app-icon.ts +154 -0
- package/src/core/audit-report.ts +21 -0
- package/src/core/cache-audit.ts +72 -0
- package/src/core/canonical-scripts.ts +1 -1
- package/src/core/check-ctx.ts +14 -5
- package/src/core/check-runner.ts +6 -3
- package/src/core/floor-resolve.ts +5 -7
- package/src/core/git-root.ts +13 -0
- package/src/core/ignored-block.ts +94 -0
- package/src/core/ignored-inputs.ts +47 -0
- package/src/core/jobs.ts +9 -2
- package/src/core/one-lib.ts +59 -4
- package/src/core/read-set.ts +97 -9
- package/src/core/run-ci.ts +6 -1
- package/src/core/run-fixers.ts +8 -2
- package/src/core/trace-audit.ts +120 -0
- package/src/home/gui-app.ts +24 -27
- package/src/home/home-start.ts +11 -5
- package/src/home/open-app.ts +2 -2
- package/src/home/via-hub.ts +14 -0
- package/src/ledger/db.ts +2 -13
- package/src/ledger/gate-checks.ts +21 -3
- package/src/ledger/git-key.ts +148 -30
- package/src/ledger/git-scan.ts +5 -2
- package/src/ledger/ledger.ts +131 -16
- package/src/ledger/store.ts +14 -0
- package/src/ledger/toolchain-identity.ts +210 -25
- package/src/problems/extract.ts +3 -0
- package/src/repo-config.ts +18 -1
- package/src/serve/hub-app/admin-svc.ts +36 -3
- package/src/serve/hub-app/admin-view.tsx +33 -1
- package/src/serve/hub-app/col-card.tsx +1 -44
- package/src/serve/hub-app/col-tables-view.tsx +71 -5
- package/src/serve/hub-app/col-tables.ts +27 -0
- package/src/serve/hub-app/home-st.ts +1 -1
- package/src/serve/hub-app/home-strip.css +5 -9
- package/src/serve/hub-app/home-strip.tsx +4 -17
- package/src/serve/hub-app/hub-app.tsx +18 -4
- package/src/serve/hub-app/hub.css +24 -20
- package/src/serve/hub-app/keys.ts +10 -2
- package/src/serve/hub-app/processes-view.tsx +16 -9
- package/src/serve/hub-app/project-list.css +4 -1
- package/src/serve/hub-app/project-list.tsx +6 -3
- package/src/serve/hub-app/projects-svc.ts +10 -0
- package/src/serve/hub-app/repo-sections.ts +31 -17
- package/src/serve/hub-app/repo-st.ts +24 -5
- package/src/serve/hub-app/sidebar.tsx +5 -5
- package/src/serve/hub-app/tree-build.ts +2 -0
- package/src/serve/hub-app/tree-row.tsx +18 -5
- package/src/serve/hub-app/tree-svc.ts +27 -7
- package/src/serve/hub-runtime.ts +4 -2
- package/src/serve/project-gutter-route.ts +22 -1
- package/src/suite/judge-suites.ts +6 -0
- package/src/trace/interpose.c +655 -0
- package/src/trace/read-trace.ts +485 -0
- package/src/trace/tracer.ts +282 -0
package/src/home/home-start.ts
CHANGED
|
@@ -6,7 +6,11 @@ import { homedir } from 'os'
|
|
|
6
6
|
import { dirname, join, resolve } from 'path'
|
|
7
7
|
import { gitRoot } from '../core/git-root.ts'
|
|
8
8
|
import { homeRepo, houseIndex } from '../serve/project-gutter.ts'
|
|
9
|
-
import {
|
|
9
|
+
import type { WrapperOf } from './gui-app.ts'
|
|
10
|
+
|
|
11
|
+
// the app builder draws icons with canvas: loaded when a window opens, never
|
|
12
|
+
// at the startup of every command that imports this module
|
|
13
|
+
const gui = (): Promise<typeof import('./gui-app.ts')> => import('./gui-app.ts')
|
|
10
14
|
import { installedHouseRoot, LABEL, plistPath } from './home-launchd.ts'
|
|
11
15
|
import { childLog, daemonPort, daemonUp, homeDir, homePort, homeUp, pmLog } from './home-paths.ts'
|
|
12
16
|
|
|
@@ -88,10 +92,11 @@ export function windowCmd(url: string, o: { chrome?: string; platform?: string }
|
|
|
88
92
|
/** one more home window: the native app where it can be built (its own
|
|
89
93
|
* dock icon, the house mark), else a Chrome app window. `warn` hears why the
|
|
90
94
|
* native one was not used */
|
|
91
|
-
export function openWindow(url: string, o: { root?: string | null; version?: string; warn?: (l: string) => void } = {}): string[] {
|
|
95
|
+
export async function openWindow(url: string, o: { root?: string | null; version?: string; warn?: (l: string) => void } = {}): Promise<string[]> {
|
|
96
|
+
const { BUNDLE_ID, canBuildGui, ensureGuiApp, quitCmd } = await gui()
|
|
92
97
|
if (canBuildGui()) {
|
|
93
98
|
try {
|
|
94
|
-
const { app, rebuilt } = ensureGuiApp({ root: o.root ?? guiRoot(process.cwd()),version: o.version ?? '0' })
|
|
99
|
+
const { app, rebuilt } = await ensureGuiApp({ root: o.root ?? guiRoot(process.cwd()),version: o.version ?? '0' })
|
|
95
100
|
// a running copy is the old build: it quits, and the new one opens the window
|
|
96
101
|
if (rebuilt) Bun.spawnSync(quitCmd(BUNDLE_ID), { stdout: 'ignore', stderr: 'ignore' })
|
|
97
102
|
const cmd = ['open', '-a', app, url]
|
|
@@ -108,11 +113,12 @@ export function openWindow(url: string, o: { root?: string | null; version?: str
|
|
|
108
113
|
* `.shipkit/wrapper/`, then opened with no url, so a running copy comes to
|
|
109
114
|
* the front instead of stacking windows. Where no app can be built, a
|
|
110
115
|
* Chrome app window on its url */
|
|
111
|
-
export function openProcessApp(d: WrapperOf, o: { version: string; warn?: (l: string) => void }): string[] {
|
|
116
|
+
export async function openProcessApp(d: WrapperOf, o: { version: string; warn?: (l: string) => void }): Promise<string[]> {
|
|
112
117
|
if (d.url == null) throw new Error(`${d.id} declares no url: there is no page to open as an app`)
|
|
118
|
+
const { canBuildGui, ensureWrapperApp, quitCmd } = await gui()
|
|
113
119
|
if (canBuildGui()) {
|
|
114
120
|
try {
|
|
115
|
-
const { app, rebuilt, bundleId } = ensureWrapperApp(d, { version: o.version })
|
|
121
|
+
const { app, rebuilt, bundleId } = await ensureWrapperApp(d, { version: o.version })
|
|
116
122
|
if (rebuilt) Bun.spawnSync(quitCmd(bundleId), { stdout: 'ignore', stderr: 'ignore' })
|
|
117
123
|
const cmd = ['open', '-a', app]
|
|
118
124
|
Bun.spawnSync(cmd, { stdout: 'ignore', stderr: 'ignore' })
|
package/src/home/open-app.ts
CHANGED
|
@@ -13,10 +13,10 @@ if (import.meta.main) {
|
|
|
13
13
|
try {
|
|
14
14
|
if (first === '--process' && second != null) {
|
|
15
15
|
const d = JSON.parse(second) as WrapperOf
|
|
16
|
-
const cmd = openProcessApp(d, { version: LIB_VERSION, warn })
|
|
16
|
+
const cmd = await openProcessApp(d, { version: LIB_VERSION, warn })
|
|
17
17
|
console.log(`shipkit: opened ${d.id} as its own app: ${cmd.join(' ')}`)
|
|
18
18
|
} else if (first != null && first !== '--process') {
|
|
19
|
-
const cmd = openWindow(first, { version: LIB_VERSION, warn })
|
|
19
|
+
const cmd = await openWindow(first, { version: LIB_VERSION, warn })
|
|
20
20
|
console.log(`shipkit: opened ${first} as an app window: ${cmd.join(' ')}`)
|
|
21
21
|
} else {
|
|
22
22
|
console.error('usage: bun open-app.ts <url> | --process <json>')
|
package/src/home/via-hub.ts
CHANGED
|
@@ -17,6 +17,18 @@ import { daemonUp, daemonUrl, hubBase, WORKER_ENV } from './home-paths.ts'
|
|
|
17
17
|
import { homeWanted, startHome } from './home-start.ts'
|
|
18
18
|
import { RUN_EXIT, RUN_JOB } from './home-words.ts'
|
|
19
19
|
import { followJob, type FollowOpts } from '../core/job-follow.ts'
|
|
20
|
+
import { TRACE_ENV } from '../trace/tracer.ts'
|
|
21
|
+
import { FRESH_ENV } from '../ledger/store.ts'
|
|
22
|
+
import { LEDGER_ENV } from '../ledger/ledger.ts'
|
|
23
|
+
import { PARALLEL_ENV } from '../core/parallel.ts'
|
|
24
|
+
|
|
25
|
+
/** the env switches a gate reads, which a terminal may set for one run: the
|
|
26
|
+
* job the home starts runs with the daemon's env, so they travel with it */
|
|
27
|
+
export const FORWARDED_ENV = [TRACE_ENV, FRESH_ENV, LEDGER_ENV, PARALLEL_ENV] as const
|
|
28
|
+
|
|
29
|
+
/** the forwarded switches this env sets */
|
|
30
|
+
export const forwardedEnv = (env: Env = process.env): Record<string, string> =>
|
|
31
|
+
Object.fromEntries(FORWARDED_ENV.filter((k) => env[k] != null && env[k] !== '').map((k) => [k, env[k]!]))
|
|
20
32
|
|
|
21
33
|
type Env = Record<string, string | undefined>
|
|
22
34
|
const set = (v: string | undefined): boolean => v != null && v !== ''
|
|
@@ -46,10 +58,12 @@ export async function runViaHub(root: string, kind: 'ci' | 'check', opts: Record
|
|
|
46
58
|
}
|
|
47
59
|
// what the terminal knows and the hub does not: its tier and whether it is a terminal
|
|
48
60
|
const full = process.env.SHIPKIT_TEST_FULL === '1'
|
|
61
|
+
const env = forwardedEnv()
|
|
49
62
|
const sent = {
|
|
50
63
|
...opts,
|
|
51
64
|
...(full ? (kind === 'ci' ? { tests: opts.tests ?? 'full' } : { tier: opts.tier ?? 'full' }) : {}),
|
|
52
65
|
color: process.stdout.isTTY === true && process.env.NO_COLOR == null,
|
|
66
|
+
...(Object.keys(env).length > 0 ? { env } : {}),
|
|
53
67
|
}
|
|
54
68
|
// the home waits for a hub still loading before it answers
|
|
55
69
|
const post = (): Promise<Response> => fetch(`${daemonUrl()}${hubBase(name)}/api/run`, { method: 'POST', headers: { 'content-type': 'application/json' }, body: JSON.stringify({ kind, opts: sent }) })
|
package/src/ledger/db.ts
CHANGED
|
@@ -26,20 +26,9 @@
|
|
|
26
26
|
import { Database } from 'bun:sqlite'
|
|
27
27
|
import { existsSync, mkdirSync } from 'fs'
|
|
28
28
|
import { dirname, join } from 'path'
|
|
29
|
-
import { gitRoot } from '../core/git-root.ts'
|
|
30
|
-
import { LAYOUT } from '../core/layout.ts'
|
|
31
29
|
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
* `--git-common-dir` rather than `--git-dir`: a linked worktree has its own
|
|
35
|
-
* git dir but shares the common one, and that sharing is safe because every
|
|
36
|
-
* key component is content or a version string. A tree that is not a git
|
|
37
|
-
* repo at all still gets one, under `.shipkit/cache/`, because a scaffold in
|
|
38
|
-
* progress must not crash on it. */
|
|
39
|
-
export function shipkitRoot(root: string): string {
|
|
40
|
-
const dir = gitRoot(root)?.commonDir
|
|
41
|
-
return dir == null ? join(root, LAYOUT.cache) : join(dir, 'shipkit')
|
|
42
|
-
}
|
|
30
|
+
import { shipkitRoot } from '../core/git-root.ts'
|
|
31
|
+
export { shipkitRoot }
|
|
43
32
|
|
|
44
33
|
export const DB_FILE = 'shipkit.sqlite'
|
|
45
34
|
|
|
@@ -11,9 +11,11 @@
|
|
|
11
11
|
// check script, a `*.check.ts` with no check object). The declaration always lives WITH the check, never in a table
|
|
12
12
|
// over here: a table beside the registry is a second source of truth that
|
|
13
13
|
// drifts the first time somebody adds one.
|
|
14
|
-
import {
|
|
14
|
+
import { existsSync } from 'fs'
|
|
15
|
+
import { join, relative, resolve } from 'path'
|
|
15
16
|
import type { CheckCtx } from '../core/check-ctx.ts'
|
|
16
17
|
import { type RepoConfig } from '../repo-config.ts'
|
|
18
|
+
import { auditedExtras, ignoredFor, readAuditReport } from '../core/audit-report.ts'
|
|
17
19
|
import { listRepoFiles, resolveFloor, slotDescription, type FloorResolution } from '../core/floor.ts'
|
|
18
20
|
import { discoverLegacyChecks } from '../core/discover-legacy-checks.ts'
|
|
19
21
|
import { checkFileRunBy } from '../core/legacy-shapes.ts'
|
|
@@ -25,7 +27,7 @@ import { FOREVER, ttlMsOf } from '../core/wire.ts'
|
|
|
25
27
|
import { budgetOf } from '../core/budget.ts'
|
|
26
28
|
import { repoOff, withRepoOff, whenSummary, type WhenDecl } from '../core/when.ts'
|
|
27
29
|
import { cacheText, filesText, NOWHERE, sourced, type CacheDecl, type ResolvedRuns, type Runs } from '../core/runs.ts'
|
|
28
|
-
import type
|
|
30
|
+
import { isNamesInput, type CheckDecl } from './git-key.ts'
|
|
29
31
|
import { populationOf } from '../problems/problem-line.ts'
|
|
30
32
|
import { rawAccessInClosure, rawAccessLine } from '../core/raw-access.ts'
|
|
31
33
|
import type { RvtConfig } from '../ts/config.ts'
|
|
@@ -290,10 +292,23 @@ export async function composeChecks(root: string, c: RepoConfig, o: ComposeOpts
|
|
|
290
292
|
// what it likes, and a gate that guesses which one is the tests will miss
|
|
291
293
|
// one.
|
|
292
294
|
const rootTest = rootTestKey(root, c, floor)
|
|
295
|
+
// a task that declares nothing is keyed on the whole work tree, plus the
|
|
296
|
+
// gitignored paths it reads. The work tree sees no ignored file, so those
|
|
297
|
+
// are named: by the config, and by the kept traced audit (SK031), whose
|
|
298
|
+
// findings join the key of the task that read them before a config write
|
|
299
|
+
// carries them to other clones. One missing from disk is left out, and
|
|
300
|
+
// its appearing moves the key
|
|
301
|
+
const onDisk = (p: string): boolean => existsSync(join(root, isNamesInput(p) ? p.slice(0, -3) : p))
|
|
302
|
+
const audit = readAuditReport(root)
|
|
303
|
+
const extrasOf = (id: string): string[] => [...new Set([...ignoredFor(c.ignoredInputs, id), ...auditedExtras(audit, id)])].filter(onDisk).sort()
|
|
293
304
|
for (const raw of [...floor.tasks, ...(c.tasks ?? []).filter((t) => foldedInto.get(t.id) !== t)]) {
|
|
294
305
|
const t: TaskDef = full ? { ...raw, env: { ...raw.env, [FULL_TESTS_ENV]: '1' } } : raw
|
|
295
306
|
const own = t.id === 'test' && floor.tasks.includes(raw) ? rootTest : null
|
|
296
|
-
const
|
|
307
|
+
const declared = own?.inputs ?? t.inputs
|
|
308
|
+
// the declared extras join a declared list too: a list names what the
|
|
309
|
+
// task reads in the tree, and the audit found these outside it
|
|
310
|
+
const extras = extrasOf(t.id)
|
|
311
|
+
const inputs = declared == null && t.ttl == null && t.never == null && t.cmd != null ? ['.', ...extras] : declared == null ? declared : [...declared, ...extras]
|
|
297
312
|
const kind = floor.tasks.includes(raw) ? 'a floor slot' : 'a repo.config.ts task'
|
|
298
313
|
const runs = withRepoOff(derivedRuns({ gate: t.optIn !== true, build: true, inputs, ttl: t.ttl, kind, repair: false }), repoOff(c, t.id))
|
|
299
314
|
const row: CheckRow = {
|
|
@@ -382,6 +397,9 @@ export async function composeChecks(root: string, c: RepoConfig, o: ComposeOpts
|
|
|
382
397
|
id: u.id,
|
|
383
398
|
inputs: cache.inputs,
|
|
384
399
|
code: u.rel == null ? undefined : [u.rel],
|
|
400
|
+
// a built-in's code is the toolkit's: keyed on its own closure
|
|
401
|
+
// there, never on the toolkit as a whole
|
|
402
|
+
libCode: u.rel == null ? [u.path] : undefined,
|
|
385
403
|
ttl: cache.ttl,
|
|
386
404
|
never: cache.never,
|
|
387
405
|
argv: u.def.cmd,
|
package/src/ledger/git-key.ts
CHANGED
|
@@ -6,10 +6,10 @@
|
|
|
6
6
|
// FOUR POPULATIONS, set at registration, never inferred from source:
|
|
7
7
|
//
|
|
8
8
|
// CONTENT declares `inputs` — the paths it reads. Keyed on their git object
|
|
9
|
-
// ids, plus the check's own code (module graph), plus the
|
|
10
|
-
//
|
|
11
|
-
//
|
|
12
|
-
//
|
|
9
|
+
// ids, plus the check's own code (module graph), plus the runner
|
|
10
|
+
// and the install (toolchain-identity.ts), plus argv and declared
|
|
11
|
+
// env. No TTL: a content key is either the same question or a
|
|
12
|
+
// different one.
|
|
13
13
|
// WORLD declares `ttl` — it reads the network, the keychain, $PATH, a git
|
|
14
14
|
// remote or the clock. Nothing on disk describes its answer, so it
|
|
15
15
|
// keeps the existing clean-pass TTL and its `cacheKey`, unchanged.
|
|
@@ -24,11 +24,11 @@
|
|
|
24
24
|
// check that says `never`. Never cached. For those it is the
|
|
25
25
|
// DEFAULT, and it is the whole safety story: forgetting to declare
|
|
26
26
|
// costs speed, never correctness, so the burn-down list below is a
|
|
27
|
-
// to-do rather than a hazard.
|
|
28
|
-
// the
|
|
29
|
-
//
|
|
30
|
-
//
|
|
31
|
-
//
|
|
27
|
+
// to-do rather than a hazard. A TASK that declares nothing is not
|
|
28
|
+
// here: the composer keys it on the whole work tree plus the repo's
|
|
29
|
+
// ignoredInputs (ledger/gate-checks.ts). Declaring BOTH is a
|
|
30
|
+
// registration failure, loud, because the two answers contradict
|
|
31
|
+
// each other.
|
|
32
32
|
//
|
|
33
33
|
// CROSS WORKTREE. Every component here is content addressed or a version
|
|
34
34
|
// string; not one of them is an absolute path. Two worktrees of one repo
|
|
@@ -39,15 +39,31 @@
|
|
|
39
39
|
// differ from main by two top-level packages. That is what makes a shared
|
|
40
40
|
// ledger safe rather than a stale green in a checkout nobody looked at.
|
|
41
41
|
import { createHash } from 'crypto'
|
|
42
|
-
import {
|
|
42
|
+
import { existsSync, readdirSync, readFileSync, type Dirent } from 'fs'
|
|
43
|
+
import { isAbsolute, join, relative } from 'path'
|
|
44
|
+
import { gitBin } from '../core/git-bin.ts'
|
|
43
45
|
import { loadConfig, type RvtConfig } from '../ts/config.ts'
|
|
44
|
-
import { importClosure } from '../ts/source-graph.ts'
|
|
46
|
+
import { extractSpecs, importClosure } from '../ts/source-graph.ts'
|
|
45
47
|
import { GitScanError, hashFiles, normalizeInput, scanPaths, WHOLE_TREE, type ScanResult } from './git-scan.ts'
|
|
46
|
-
import { installIdentity, toolchainIdentity } from './toolchain-identity.ts'
|
|
48
|
+
import { installIdentity, toolchainIdentity, type LibSource } from './toolchain-identity.ts'
|
|
47
49
|
|
|
48
50
|
/** the key format. Bumped when a component is added or its rendering changes,
|
|
49
51
|
* so an old ledger can never answer a new question. */
|
|
50
|
-
export const KEY_VERSION =
|
|
52
|
+
export const KEY_VERSION = 4
|
|
53
|
+
|
|
54
|
+
/** `dir/**` names a dir a task only LISTS: keyed on the names under it */
|
|
55
|
+
export const isNamesInput = (p: string): boolean => p.endsWith('/**')
|
|
56
|
+
|
|
57
|
+
let nodeMemo: string | null = null
|
|
58
|
+
/** the node a task's tools run under, from PATH: a runtime upgrade is a
|
|
59
|
+
* different program, like a bun upgrade */
|
|
60
|
+
function nodeVersion(): string {
|
|
61
|
+
if (nodeMemo == null) {
|
|
62
|
+
const node = Bun.which('node')
|
|
63
|
+
nodeMemo = node == null ? 'none' : Bun.spawnSync([node, '--version'], { stdout: 'pipe', stderr: 'pipe' }).stdout.toString().trim() || 'unknown'
|
|
64
|
+
}
|
|
65
|
+
return nodeMemo
|
|
66
|
+
}
|
|
51
67
|
|
|
52
68
|
export type Population = 'content' | 'world' | 'reads' | 'none'
|
|
53
69
|
|
|
@@ -63,6 +79,10 @@ export type CheckDecl = {
|
|
|
63
79
|
/** repo-relative entry files of the check's OWN code — the module-graph
|
|
64
80
|
* roots. A discovered check gets its own file for free. */
|
|
65
81
|
code?: string[]
|
|
82
|
+
/** absolute entry files of the check's own code INSIDE the toolkit: a
|
|
83
|
+
* built-in check's file. A task whose argv runs a toolkit module gets
|
|
84
|
+
* that module's closure without declaring it */
|
|
85
|
+
libCode?: string[]
|
|
66
86
|
/** the argv a subprocess check runs */
|
|
67
87
|
argv?: string[]
|
|
68
88
|
/** env var NAMES whose values belong in the key */
|
|
@@ -102,10 +122,14 @@ export type CheckKey = {
|
|
|
102
122
|
components: string[]
|
|
103
123
|
/** the input paths this check was keyed on, with their object ids */
|
|
104
124
|
inputOids: [string, string][]
|
|
105
|
-
/** the
|
|
106
|
-
*
|
|
107
|
-
|
|
108
|
-
|
|
125
|
+
/** the toolkit files behind its `lib=` and `libused=` components,
|
|
126
|
+
* package-relative: what `shipkit keys --why` lists */
|
|
127
|
+
libFiles?: string[]
|
|
128
|
+
/** the identity of the check's OWN code, module graph included, in the
|
|
129
|
+
* repo and in the toolkit. Set for every check that has code, WORLD ones
|
|
130
|
+
* too: a TTL'd probe whose source changed must not replay its previous
|
|
131
|
+
* answer, and this is the value that busts it. null when the check has no
|
|
132
|
+
* code entry or its code could not be hashed. */
|
|
109
133
|
codeKey: string | null
|
|
110
134
|
}
|
|
111
135
|
|
|
@@ -196,7 +220,7 @@ export class KeyLayer {
|
|
|
196
220
|
private readonly scanError: string | null,
|
|
197
221
|
private readonly closureByCheck: Map<string, string[]>,
|
|
198
222
|
private readonly codeOids: Map<string, string>,
|
|
199
|
-
private readonly toolchain: { id: string; uncacheable: string[] },
|
|
223
|
+
private readonly toolchain: { id: string; install: string; lib: LibSource | null; uncacheable: string[] },
|
|
200
224
|
private readonly scanPathsUsed: string[],
|
|
201
225
|
private readonly scope: ReadonlySet<string> | null,
|
|
202
226
|
/** wall time of the WHOLE layer: toolchain identity, the module-graph
|
|
@@ -208,12 +232,11 @@ export class KeyLayer {
|
|
|
208
232
|
|
|
209
233
|
private readonly installIds = new Map<string, string>()
|
|
210
234
|
|
|
211
|
-
/** the
|
|
212
|
-
*
|
|
213
|
-
*
|
|
214
|
-
* different answer, and the TTL alone cannot see that. */
|
|
235
|
+
/** the runner and the install a WORLD answer is remembered under: a TTL'd
|
|
236
|
+
* answer measured by a different shipkit is a different answer, and the
|
|
237
|
+
* TTL alone cannot see that. The probe's own code rides in `codeKey` */
|
|
215
238
|
get toolchainId(): string {
|
|
216
|
-
return this.toolchain.id
|
|
239
|
+
return `${this.toolchain.id}:${this.toolchain.install}`
|
|
217
240
|
}
|
|
218
241
|
|
|
219
242
|
/** wall time the GIT half alone cost, ms — `openMs` is what a run pays */
|
|
@@ -232,11 +255,11 @@ export class KeyLayer {
|
|
|
232
255
|
// a gate binary that cannot be resolved is not a crash, it is a run
|
|
233
256
|
// whose program nobody can name — and the only safe answer to that is
|
|
234
257
|
// no key at all
|
|
235
|
-
let toolchain: { id: string; uncacheable: string[] }
|
|
258
|
+
let toolchain: { id: string; install: string; lib: LibSource | null; uncacheable: string[] }
|
|
236
259
|
try {
|
|
237
260
|
toolchain = { ...toolchainIdentity(root, o.cliEntry), uncacheable: [] }
|
|
238
261
|
} catch (e) {
|
|
239
|
-
toolchain = { id: 'unresolved', uncacheable: [`gate binary unreadable at ${o.cliEntry}: ${e instanceof Error ? e.message : String(e)}`] }
|
|
262
|
+
toolchain = { id: 'unresolved', install: 'unresolved', lib: null, uncacheable: [`gate binary unreadable at ${o.cliEntry}: ${e instanceof Error ? e.message : String(e)}`] }
|
|
240
263
|
}
|
|
241
264
|
|
|
242
265
|
// the check's OWN code, resolved through the same module graph the
|
|
@@ -257,7 +280,7 @@ export class KeyLayer {
|
|
|
257
280
|
const paths = new Set<string>()
|
|
258
281
|
for (const d of scoped) {
|
|
259
282
|
if (classify(d).population !== 'content') continue
|
|
260
|
-
for (const p of d.inputs ?? []) paths.add(p)
|
|
283
|
+
for (const p of d.inputs ?? []) if (!isNamesInput(p)) paths.add(p)
|
|
261
284
|
}
|
|
262
285
|
|
|
263
286
|
// the closure is INDIVIDUAL FILES belonging to no declared path, so it
|
|
@@ -287,9 +310,12 @@ export class KeyLayer {
|
|
|
287
310
|
const own = (this.closureByCheck.get(d.id) ?? []).slice().sort()
|
|
288
311
|
const ownReadable = own.every((p) => this.codeOids.get(p) != null)
|
|
289
312
|
const codeLines = own.map((p) => `code:${p}=${this.codeOids.get(p)!}`)
|
|
313
|
+
const lib = this.libLines(d)
|
|
290
314
|
const base: CheckKey = {
|
|
291
|
-
id: d.id, population: c.population, key: null, reason: null, components: [], inputOids: [],
|
|
292
|
-
codeKey: own.length === 0 || !ownReadable
|
|
315
|
+
id: d.id, population: c.population, key: null, reason: null, components: [], inputOids: [], libFiles: this.libFilesOf(d),
|
|
316
|
+
codeKey: typeof lib === 'string' || (own.length === 0 || !ownReadable) && lib.length === 0
|
|
317
|
+
? null
|
|
318
|
+
: createHash('sha256').update([...(ownReadable ? codeLines : []), ...lib].join('\n')).digest('hex').slice(0, 16),
|
|
293
319
|
}
|
|
294
320
|
if (c.error != null) return { ...base, reason: c.error }
|
|
295
321
|
if (this.scope != null && !this.scope.has(d.id)) return { ...base, reason: 'not one of the checks this run executes, so nothing was scanned for it' }
|
|
@@ -299,10 +325,10 @@ export class KeyLayer {
|
|
|
299
325
|
const scan = this.scan!
|
|
300
326
|
if (this.toolchain.uncacheable.length > 0)
|
|
301
327
|
return { ...base, reason: `toolchain not describable: ${this.toolchain.uncacheable.join('; ')}` }
|
|
328
|
+
if (typeof lib === 'string') return { ...base, reason: lib }
|
|
302
329
|
// the question alone: what it read is stored with each answer and
|
|
303
330
|
// re-checked before a replay, so no read belongs in this key
|
|
304
331
|
if (c.population === 'reads') {
|
|
305
|
-
// a built-in has no code entry: the toolchain id hashes its source
|
|
306
332
|
if (!ownReadable) return { ...base, reason: `git could not hash this check's own source: ${own.filter((p) => this.codeOids.get(p) == null).join(', ')}` }
|
|
307
333
|
const components = [
|
|
308
334
|
`v=${KEY_VERSION}`,
|
|
@@ -312,14 +338,17 @@ export class KeyLayer {
|
|
|
312
338
|
`toolchain=${this.toolchain.id}`,
|
|
313
339
|
'reads=recorded',
|
|
314
340
|
...codeLines,
|
|
341
|
+
...lib,
|
|
315
342
|
...(d.extra != null ? [`extra=${d.extra}`] : []),
|
|
316
343
|
]
|
|
317
344
|
return { ...base, key: createHash('sha256').update(components.join('\n')).digest('hex').slice(0, 32), components }
|
|
318
345
|
}
|
|
319
346
|
|
|
320
347
|
let inputs: string[]
|
|
348
|
+
let names: string[]
|
|
321
349
|
try {
|
|
322
|
-
inputs = (d.inputs ?? []).map(normalizeInput).sort()
|
|
350
|
+
inputs = (d.inputs ?? []).filter((p) => !isNamesInput(p)).map(normalizeInput).sort()
|
|
351
|
+
names = (d.inputs ?? []).filter(isNamesInput).map((p) => normalizeInput(p.slice(0, -3))).sort()
|
|
323
352
|
} catch (e) {
|
|
324
353
|
return { ...base, reason: e instanceof Error ? e.message : String(e) }
|
|
325
354
|
}
|
|
@@ -341,10 +370,17 @@ export class KeyLayer {
|
|
|
341
370
|
// frozen run and a normal one measure it twice.
|
|
342
371
|
...(d.writes === true ? [`frozen=${process.env['SHIPKIT_FROZEN'] === '1'}`] : []),
|
|
343
372
|
`toolchain=${this.toolchain.id}`,
|
|
373
|
+
// the repo's own tools run here, and what is installed is part of
|
|
374
|
+
// what they are
|
|
375
|
+
`install=${this.toolchain.install}`,
|
|
344
376
|
`argv=${JSON.stringify(d.argv ?? [])}`,
|
|
345
377
|
...(d.env ?? []).slice().sort().map((n) => `env:${n}=${process.env[n] ?? '<unset>'}`),
|
|
346
378
|
...inputOids.map(([p, oid]) => `in:${p}=${oid}`),
|
|
379
|
+
// a dir the task only lists: the names under it, never its bytes
|
|
380
|
+
...names.map((p) => `names:${p}=${this.namesOf(p)}`),
|
|
381
|
+
`node=${nodeVersion()}`,
|
|
347
382
|
...codeLines,
|
|
383
|
+
...lib,
|
|
348
384
|
...(d.extra != null ? [`extra=${d.extra}`] : []),
|
|
349
385
|
// ignored content nobody keyed on is part of the key's MEANING:
|
|
350
386
|
// two runs that excluded different things answered different
|
|
@@ -362,6 +398,88 @@ export class KeyLayer {
|
|
|
362
398
|
return { ...base, key, components, inputOids }
|
|
363
399
|
}
|
|
364
400
|
|
|
401
|
+
/** the toolkit code a check's answer comes from, as key lines, or why it
|
|
402
|
+
* cannot be keyed. A built-in names its file, a task running a toolkit
|
|
403
|
+
* module names it in its argv. Everything else runs the REPO's code, and
|
|
404
|
+
* in a repo that does not host the toolkit that code imports it by
|
|
405
|
+
* package name, through an install the install id leaves the toolkit out
|
|
406
|
+
* of: those carry the closure of every toolkit module the repo imports */
|
|
407
|
+
private libLines(d: CheckDecl): string[] | string {
|
|
408
|
+
const lib = this.toolchain.lib
|
|
409
|
+
if (lib == null) return []
|
|
410
|
+
const entries = [...(d.libCode ?? []), ...(d.argv ?? []).filter((a) => isAbsolute(a) && lib.owns(a))]
|
|
411
|
+
try {
|
|
412
|
+
const used = d.libCode == null ? this.libUsed() : []
|
|
413
|
+
if (typeof used === 'string') return used
|
|
414
|
+
return [...(entries.length > 0 ? [`lib=${lib.closureKey(entries)}`] : []), ...used]
|
|
415
|
+
} catch (e) {
|
|
416
|
+
return `the toolkit source this check runs could not be read: ${e instanceof Error ? e.message : String(e)}`
|
|
417
|
+
}
|
|
418
|
+
}
|
|
419
|
+
|
|
420
|
+
private libUsedMemo: string[] | string | null = null
|
|
421
|
+
private libUsedEntries: string[] = []
|
|
422
|
+
|
|
423
|
+
/** the files behind libLines, for a reader; empty when there are none */
|
|
424
|
+
private libFilesOf(d: CheckDecl): string[] {
|
|
425
|
+
const lib = this.toolchain.lib
|
|
426
|
+
if (lib == null) return []
|
|
427
|
+
const entries = [...(d.libCode ?? []), ...(d.argv ?? []).filter((a) => isAbsolute(a) && lib.owns(a)), ...(d.libCode == null ? this.libUsedEntries : [])]
|
|
428
|
+
try { return [...new Set(entries.flatMap((e) => lib.closureOf(e)))].sort() } catch { return [] }
|
|
429
|
+
}
|
|
430
|
+
|
|
431
|
+
/** the closure of every toolkit module this repo's own files import by
|
|
432
|
+
* package name, found with one `git grep` over tracked and untracked
|
|
433
|
+
* files. Empty in the repo that hosts the toolkit, whose own resolver
|
|
434
|
+
* follows those imports into the code closure */
|
|
435
|
+
private libUsed(): string[] | string {
|
|
436
|
+
if (this.libUsedMemo != null) return this.libUsedMemo
|
|
437
|
+
const lib = this.toolchain.lib
|
|
438
|
+
if (lib == null || lib.name == null || !relative(this.root, lib.pkgRoot).startsWith('..')) return (this.libUsedMemo = [])
|
|
439
|
+
const p = Bun.spawnSync([gitBin(), 'grep', '-l', '--untracked', '-F', '-e', `${lib.name}/`, '--', '*.ts', '*.tsx', '*.mts', '*.cts', '*.js', '*.mjs'], { cwd: this.root, stdout: 'pipe', stderr: 'pipe' })
|
|
440
|
+
// exit 1 is "no match", anything above it is git failing
|
|
441
|
+
if (p.exitCode > 1) return (this.libUsedMemo = `git grep could not list the files importing ${lib.name}: ${p.stderr.toString().trim().split('\n')[0]}`)
|
|
442
|
+
const entries: string[] = []
|
|
443
|
+
for (const f of p.stdout.toString().split('\n').filter((l) => l !== '')) {
|
|
444
|
+
let text = ''
|
|
445
|
+
try { text = readFileSync(join(this.root, f), 'utf8') } catch { continue }
|
|
446
|
+
for (const spec of [...extractSpecs(text, { dynamic: true }), ...[...text.matchAll(/import\.meta\.resolve\(\s*['"]([^'"\n]+)['"]\s*\)/g)].map((m) => m[1]!)]) {
|
|
447
|
+
const abs = lib.fileOf(spec)
|
|
448
|
+
if (abs != null) entries.push(abs)
|
|
449
|
+
}
|
|
450
|
+
}
|
|
451
|
+
this.libUsedEntries = [...new Set(entries)].sort()
|
|
452
|
+
return (this.libUsedMemo = entries.length === 0 ? [] : [`libused=${lib.closureKey(this.libUsedEntries)}`])
|
|
453
|
+
}
|
|
454
|
+
|
|
455
|
+
private readonly namesMemo = new Map<string, string>()
|
|
456
|
+
|
|
457
|
+
/** every relative path under a dir, sorted, as one hash: what a task that
|
|
458
|
+
* lists the dir without reading its files sees. `absent` for no dir */
|
|
459
|
+
private namesOf(rel: string): string {
|
|
460
|
+
let h = this.namesMemo.get(rel)
|
|
461
|
+
if (h == null) {
|
|
462
|
+
const abs = join(this.root, rel)
|
|
463
|
+
if (!existsSync(abs)) h = 'absent'
|
|
464
|
+
else {
|
|
465
|
+
const all: string[] = []
|
|
466
|
+
const walk = (d: string, prefix: string): void => {
|
|
467
|
+
let entries: Dirent[] = []
|
|
468
|
+
try { entries = readdirSync(d, { withFileTypes: true }) } catch { return }
|
|
469
|
+
for (const e of entries) {
|
|
470
|
+
const r = `${prefix}${e.name}`
|
|
471
|
+
all.push(e.isDirectory() ? `${r}/` : r)
|
|
472
|
+
if (e.isDirectory()) walk(join(d, e.name), `${r}/`)
|
|
473
|
+
}
|
|
474
|
+
}
|
|
475
|
+
walk(abs, '')
|
|
476
|
+
h = createHash('sha256').update(all.sort().join('\n')).digest('hex').slice(0, 16)
|
|
477
|
+
}
|
|
478
|
+
this.namesMemo.set(rel, h)
|
|
479
|
+
}
|
|
480
|
+
return h
|
|
481
|
+
}
|
|
482
|
+
|
|
365
483
|
/** an excluded install's identity, once per layer: several checks can name
|
|
366
484
|
* the same one, and hashing a `node_modules` twice in a run is pure cost. */
|
|
367
485
|
private installOf(rel: string): string {
|
package/src/ledger/git-scan.ts
CHANGED
|
@@ -290,10 +290,13 @@ export function scanPaths(root: string, rawPaths: string[], prior?: ScanResult,
|
|
|
290
290
|
// force-adding that one entry stages the node_modules inside it.
|
|
291
291
|
const declared = new Set(wanted)
|
|
292
292
|
const denied = ignored.filter(underNodeModules)
|
|
293
|
-
|
|
293
|
+
// a declared path INSIDE a wholly ignored dir comes back as that dir, so
|
|
294
|
+
// it is named by the collapsed ancestor rather than by itself
|
|
295
|
+
const underIgnored = present.filter((p) => p !== WHOLE_TREE && !ignored.includes(p) && ignored.some((d) => p.startsWith(`${d}/`)))
|
|
296
|
+
const forceAdd = [...ignored, ...underIgnored].filter((p) => declared.has(p) && !underNodeModules(p) && !denied.some((d) => d.startsWith(`${p}/`)))
|
|
294
297
|
// a path git itself calls ignored must NOT get the plain add: `git add --
|
|
295
298
|
// <ignored>` exits 1 and takes the whole scan down with it
|
|
296
|
-
const ignoredSet = new Set(ignored)
|
|
299
|
+
const ignoredSet = new Set([...ignored, ...underIgnored])
|
|
297
300
|
const plainAdds = toAdd.filter((p) => !ignoredSet.has(p))
|
|
298
301
|
if (plainAdds.length > 0) gitOrThrow(root, ['add', '--', ...plainAdds], indexFile)
|
|
299
302
|
// force-adds land in their OWN index, so a path one check named can never
|