rvlib-shipkit 1.7.0 → 1.9.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +48 -0
- package/package.json +1 -1
- package/src/assets/gen/comfyts-gen.ts +31 -14
- package/src/assets/gen/generate-roll.ts +20 -7
- package/src/assets/kit-config.ts +9 -0
- package/src/assets/kits/identity-kit.ts +7 -1
- package/src/assets/run-assets.ts +3 -3
- package/src/assets/server/studio-server.ts +3 -3
- package/src/checks/SK027-agent-wiki-links.check.ts +6 -2
- package/src/checks/SK033-global-contribution.check.ts +47 -0
- package/src/cli/cmd_Ci.ts +5 -4
- package/src/cli/cmd_GlobalContext.ts +47 -0
- package/src/cli/cmd_GlobalProcesses.ts +38 -1
- package/src/cli.ts +4 -1
- package/src/collections/providers.ts +1 -1
- package/src/core/check-runner.ts +13 -4
- package/src/core/discover-checks.ts +37 -8
- package/src/core/fix-scope.ts +5 -0
- package/src/core/floor-resolve.ts +2 -1
- package/src/core/git-root.ts +22 -12
- package/src/core/global-contribution.ts +42 -0
- package/src/core/one-lib.ts +4 -2
- package/src/core/run-ci.ts +12 -11
- package/src/core/slots.ts +36 -6
- package/src/home/global-context.ts +145 -0
- package/src/home/gui-app/ShipkitGui.swift +96 -0
- package/src/home/home-routes.ts +10 -0
- package/src/home/resources.ts +3 -0
- package/src/plan/planner.ts +21 -4
- package/src/plan/repo-source.ts +7 -3
- package/src/release/run-release.ts +17 -8
- package/src/release/unit-gate.ts +12 -5
- package/src/repo-config.ts +7 -0
- package/src/serve/hub-runtime.ts +34 -23
- package/src/serve/project-gutter.ts +5 -2
- package/src/serve/run-lanes.ts +138 -0
- package/src/smoke/run-smoke.ts +13 -6
- package/src/suite/judge-suites.ts +79 -11
- package/src/suite/reads-env.ts +8 -0
- package/src/suite/select.ts +510 -0
- package/src/suite/suite-cli.ts +213 -23
- package/src/suite/suite-reads-preload.ts +335 -0
- package/src/ts/typecheck-cache.ts +11 -0
- package/src/ts/typecheck-cli.ts +2 -1
|
@@ -158,13 +158,28 @@ function unloadable(path: string, message: string): CheckDef {
|
|
|
158
158
|
}
|
|
159
159
|
}
|
|
160
160
|
|
|
161
|
+
/** a built-in that does not load, as a repo that only CONSUMES the toolkit
|
|
162
|
+
* sees it: one warning, never a red gate, since only the toolkit's own repo
|
|
163
|
+
* can fix the file, and its gate stays red on it */
|
|
164
|
+
function brokenBuiltin(path: string, message: string): CheckDef {
|
|
165
|
+
return {
|
|
166
|
+
...unloadable(path, message),
|
|
167
|
+
blocking: false,
|
|
168
|
+
run: (c) => c.warn(`${basename(path)} does not load, so this gate skips it: ${message.replace(`${path}: `, '')}`, 'the toolkit repo owns the file, and its own gate fails on it'),
|
|
169
|
+
}
|
|
170
|
+
}
|
|
171
|
+
|
|
172
|
+
const BUILTIN_OWNER = 'build/shipkit/built-in-checks.md'
|
|
173
|
+
|
|
161
174
|
/** every check this repo can run: its own files, plus the toolkit's. Throws
|
|
162
175
|
* on a registration nobody can act on (a duplicate id, a repo check squatting
|
|
163
176
|
* the reserved prefix, a built-in with no owner page), so a broken
|
|
164
|
-
* declaration dies at load rather than half way through a gate.
|
|
165
|
-
|
|
177
|
+
* declaration dies at load rather than half way through a gate. A broken
|
|
178
|
+
* built-in is that only in the toolkit's own repo: elsewhere it warns.
|
|
179
|
+
* `builtinDir` is the toolkit's checks dir, a parameter for tests only */
|
|
180
|
+
export async function discoverChecks(root: string, builtinDir: string = BUILTIN_CHECK_DIR): Promise<DiscoveredCheck[]> {
|
|
166
181
|
const builtins: string[] = []
|
|
167
|
-
walk(
|
|
182
|
+
walk(builtinDir, builtins)
|
|
168
183
|
const found: string[] = []
|
|
169
184
|
walk(root, found)
|
|
170
185
|
const units = repoUnitFiles(root)
|
|
@@ -173,6 +188,9 @@ export async function discoverChecks(root: string): Promise<DiscoveredCheck[]> {
|
|
|
173
188
|
// because a consumer reaches the same files through a linked node_modules
|
|
174
189
|
const real = (f: string): string => { try { return realpathSync(f) } catch { return f } }
|
|
175
190
|
const shipped = new Set(builtins.map(real))
|
|
191
|
+
// its own source, never an install of it
|
|
192
|
+
const inRoot = relative(real(root), real(builtinDir))
|
|
193
|
+
const hostsToolkit = !inRoot.startsWith('..') && !inRoot.split(/[\\/]/).includes('node_modules')
|
|
176
194
|
const all = found.filter((f) => !shipped.has(real(f))).sort()
|
|
177
195
|
builtins.sort()
|
|
178
196
|
const names = fallbackNames(root, all)
|
|
@@ -185,23 +203,34 @@ export async function discoverChecks(root: string): Promise<DiscoveredCheck[]> {
|
|
|
185
203
|
for (const f of [...builtins, ...own, ...units]) {
|
|
186
204
|
const isBuiltin = builtins.includes(f)
|
|
187
205
|
let def: CheckDef | null
|
|
206
|
+
let loadError: string | null = null
|
|
188
207
|
try {
|
|
189
208
|
def = await load(f)
|
|
190
209
|
} catch (e) {
|
|
191
210
|
// a file that does not load is ONE red row, never a dead inventory:
|
|
192
211
|
// every other check still runs, and this one says why in every run
|
|
193
|
-
|
|
212
|
+
loadError = (e as Error).message
|
|
213
|
+
def = unloadable(f, loadError)
|
|
194
214
|
}
|
|
195
215
|
if (def == null) continue
|
|
196
216
|
const rel = isBuiltin ? null : relative(root, f).replace(/\\/g, '/')
|
|
197
217
|
const fallback = isBuiltin
|
|
198
218
|
? (BUILTIN_ID_RE.exec(base(f))?.[1] ?? base(f))
|
|
199
219
|
: f.endsWith(UNIT_SUFFIX) ? unitBase(f) : names.get(f)!
|
|
200
|
-
|
|
220
|
+
let id = def.id ?? fallback
|
|
201
221
|
if (isBuiltin) {
|
|
202
|
-
|
|
203
|
-
|
|
204
|
-
|
|
222
|
+
const refused = loadError != null ? null
|
|
223
|
+
: id !== fallback ? `${f}: declares id '${id}' but its file name says '${fallback}' — a built-in id is its file name`
|
|
224
|
+
: (def.owner ?? '') === '' ? `${f}: a built-in check names the page that OWNS the invariant`
|
|
225
|
+
: RETIRED_CHECK_IDS[id] != null ? `${f}: id '${id}' is retired (${RETIRED_CHECK_IDS[id]}), and an id is never reused`
|
|
226
|
+
: null
|
|
227
|
+
// the toolkit's own repo dies on a refused built-in, a consumer
|
|
228
|
+
// only warns: it cannot fix the file, and must not lose its gate
|
|
229
|
+
if (refused != null && hostsToolkit) throw new Error(refused)
|
|
230
|
+
if (refused != null || loadError != null) {
|
|
231
|
+
def = hostsToolkit ? { ...def, owner: BUILTIN_OWNER } : { ...brokenBuiltin(f, refused ?? loadError!), owner: BUILTIN_OWNER }
|
|
232
|
+
id = fallback
|
|
233
|
+
}
|
|
205
234
|
} else if (id.startsWith('SK')) {
|
|
206
235
|
throw new Error(`${rel}: id '${id}' uses the reserved SK prefix — a repo check picks its own (e.g. RV${id.slice(-3)}) or plain words`)
|
|
207
236
|
}
|
|
@@ -0,0 +1,5 @@
|
|
|
1
|
+
// does a gate run its repair pass (core/run-fixers.ts). A leaf, so the hub's
|
|
2
|
+
// run lanes read the same answer the gate acts on without loading the gate.
|
|
3
|
+
|
|
4
|
+
/** `fix` named wins. Unnamed: a whole gate repairs, a narrowed run does not */
|
|
5
|
+
export const fixPassRuns = (o: { fix?: boolean; onlyIds?: readonly string[] }): boolean => o.fix ?? (o.onlyIds ?? []).length === 0
|
|
@@ -20,6 +20,7 @@ import type { CheckCtx } from './check-ctx.ts'
|
|
|
20
20
|
import { checkUnitReader, type UnitReader } from '../release/unit-resolve.ts'
|
|
21
21
|
import { parseJsonc } from '../ts/jsonc.ts'
|
|
22
22
|
import { resolveTypecheckProjects } from '../ts/tsconfig-resolve.ts'
|
|
23
|
+
import { gateBuildInfoArgs } from '../ts/typecheck-cache.ts'
|
|
23
24
|
import { parseBunTestScript, suiteFlags } from '../suite/judge-suites.ts'
|
|
24
25
|
import { declaredTestUnits, siblingUnits, testCheckId, unitIgnores, unitTestBody, unitTsconfig, type TestUnitDecl } from '../suite/test-units.ts'
|
|
25
26
|
import { describeFirst, type TaskDef } from '../check.ts'
|
|
@@ -194,7 +195,7 @@ function builtinFloorTask(id: string, root: string, io: FloorReader): { task: Ta
|
|
|
194
195
|
const projects = resolveTypecheckProjects(tsconfig, io, root)
|
|
195
196
|
plain = projects.projects.length === 1 && projects.projects[0] === tsconfig && projects.missing.length === 0
|
|
196
197
|
} catch { plain = false }
|
|
197
|
-
if (plain) return { task: { id, cmd: [tsc, '--noEmit'] }, source: 'built-in (tsc)' }
|
|
198
|
+
if (plain) return { task: { id, cmd: [tsc, '--noEmit', ...gateBuildInfoArgs(tsconfig)] }, source: 'built-in (tsc)' }
|
|
198
199
|
const cli = fileURLToPath(import.meta.resolve('rvlib-shipkit/src/ts/typecheck-cli.ts'))
|
|
199
200
|
return { task: { id, cmd: ['bun', cli, tsc, tsconfig] }, source: 'built-in (tsc)' }
|
|
200
201
|
}
|
package/src/core/git-root.ts
CHANGED
|
@@ -58,14 +58,23 @@ export function probeGitRoot(at: string): Probe {
|
|
|
58
58
|
return p
|
|
59
59
|
}
|
|
60
60
|
|
|
61
|
-
/** the variables that let git find a repo with no `.git` in reach
|
|
61
|
+
/** the variables that let git find a repo with no `.git` in reach. They name
|
|
62
|
+
* ONE repo for the whole process (git sets GIT_DIR for every hook in a linked
|
|
63
|
+
* worktree), so they only answer for a directory with no `.git` of its own:
|
|
64
|
+
* any other directory is asked with them stripped, or the hook's repo would
|
|
65
|
+
* claim the house and every repo the process looks at */
|
|
62
66
|
const LOCATING_ENV = ['GIT_DIR', 'GIT_WORK_TREE', 'GIT_COMMON_DIR']
|
|
67
|
+
const locatingEnvSet = (): boolean => LOCATING_ENV.some((k) => (process.env[k] ?? '') !== '')
|
|
68
|
+
const discoveryEnv = (): Record<string, string | undefined> => {
|
|
69
|
+
const env = { ...process.env }
|
|
70
|
+
for (const k of LOCATING_ENV) delete env[k]
|
|
71
|
+
return env
|
|
72
|
+
}
|
|
63
73
|
|
|
64
|
-
/** git's discovery walks up looking for `.git`. With none at or above
|
|
65
|
-
*
|
|
66
|
-
*
|
|
74
|
+
/** git's discovery walks up looking for `.git`. With none at or above, its
|
|
75
|
+
* answer is already known, and a gate or a test suite that probes fresh temp
|
|
76
|
+
* dirs would spawn a process per dir for it */
|
|
67
77
|
function nothingInReach(dir: string): boolean {
|
|
68
|
-
if (LOCATING_ENV.some((k) => (process.env[k] ?? '') !== '')) return false
|
|
69
78
|
let cur: string
|
|
70
79
|
try {
|
|
71
80
|
cur = realpathSync(dir)
|
|
@@ -81,12 +90,11 @@ function nothingInReach(dir: string): boolean {
|
|
|
81
90
|
}
|
|
82
91
|
|
|
83
92
|
/** the common case answered from the disk: `.git` is a plain directory right
|
|
84
|
-
* at `dir`, which git describes as exactly that. Anything unusual (a
|
|
85
|
-
*
|
|
86
|
-
*
|
|
87
|
-
*
|
|
93
|
+
* at `dir`, which git describes as exactly that. Anything unusual (a gitfile
|
|
94
|
+
* or a symlinked `.git`, a shared common dir, a config that relocates the
|
|
95
|
+
* work tree, marks it bare or includes another file, a `.git` owned by
|
|
96
|
+
* someone else) is left to git */
|
|
88
97
|
function plainRoot(dir: string): GitRoot | null {
|
|
89
|
-
if (LOCATING_ENV.some((k) => (process.env[k] ?? '') !== '')) return null
|
|
90
98
|
try {
|
|
91
99
|
const top = realpathSync(dir)
|
|
92
100
|
const gitDir = join(top, '.git')
|
|
@@ -102,8 +110,9 @@ function plainRoot(dir: string): GitRoot | null {
|
|
|
102
110
|
}
|
|
103
111
|
|
|
104
112
|
function ask(dir: string): Probe {
|
|
105
|
-
|
|
106
|
-
|
|
113
|
+
const none = nothingInReach(dir)
|
|
114
|
+
if (none && !locatingEnvSet()) return { root: null, error: 'fatal: not a git repository (or any of the parent directories): .git' }
|
|
115
|
+
const plain = none ? null : plainRoot(dir)
|
|
107
116
|
if (plain != null) return { root: plain, error: '' }
|
|
108
117
|
let p: ReturnType<typeof Bun.spawnSync>
|
|
109
118
|
try {
|
|
@@ -111,6 +120,7 @@ function ask(dir: string): Probe {
|
|
|
111
120
|
cwd: dir,
|
|
112
121
|
stdout: 'pipe',
|
|
113
122
|
stderr: 'pipe',
|
|
123
|
+
env: none ? { ...process.env } : discoveryEnv(),
|
|
114
124
|
})
|
|
115
125
|
} catch (e) {
|
|
116
126
|
return { root: null, error: e instanceof Error ? e.message : String(e) }
|
|
@@ -0,0 +1,42 @@
|
|
|
1
|
+
// a house repo's global contribution: `agent/global-contribution.md`, what the
|
|
2
|
+
// repo offers every agent of every house repo (commands, how to run them).
|
|
3
|
+
// The house's generated context (home/global-context.ts) concatenates them,
|
|
4
|
+
// and every session of every house repo loads it, so each line costs tokens
|
|
5
|
+
// everywhere: CONTRIBUTION_MAX_LINES and CONTRIBUTION_MAX_BYTES bound it.
|
|
6
|
+
|
|
7
|
+
import { readFileSync } from 'fs'
|
|
8
|
+
|
|
9
|
+
export const CONTRIBUTION_FILE = 'agent/global-contribution.md'
|
|
10
|
+
/** the line a house index carries to load the generated context */
|
|
11
|
+
export const CONTEXT_INCLUDE = '@generated/global-context.md'
|
|
12
|
+
export const CONTRIBUTION_MAX_LINES = 40
|
|
13
|
+
export const CONTRIBUTION_MAX_BYTES = 3_000
|
|
14
|
+
|
|
15
|
+
/** what a new repo gets: a comment only, so it contributes nothing yet */
|
|
16
|
+
export const CONTRIBUTION_STUB = `<!-- global contribution: what this repo offers every agent in every house repo, such as the commands another repo can run and what they produce. The house concatenates every repo's file into one context loaded in every session, so keep it to what another repo's agent can act on: at most ${CONTRIBUTION_MAX_LINES} lines, no heading above ####. Empty contributes nothing. This comment is dropped. -->
|
|
17
|
+
`
|
|
18
|
+
|
|
19
|
+
/** the text that joins the context: html comments dropped, trimmed */
|
|
20
|
+
export const contributionBody = (text: string): string => text.replace(/<!--[\s\S]*?-->/g, '').trim()
|
|
21
|
+
|
|
22
|
+
/** whether a house index loads the generated context: only then does a
|
|
23
|
+
* contribution reach anyone */
|
|
24
|
+
export function loadsContext(index: string): boolean {
|
|
25
|
+
try {
|
|
26
|
+
return readFileSync(index, 'utf8').split('\n').some((l) => l.trim() === CONTEXT_INCLUDE)
|
|
27
|
+
} catch {
|
|
28
|
+
return false
|
|
29
|
+
}
|
|
30
|
+
}
|
|
31
|
+
|
|
32
|
+
/** why a body breaks the house context, or null: over its bounds, or a
|
|
33
|
+
* heading that would outrank the `### <repo>` it sits under */
|
|
34
|
+
export function contributionOver(body: string): string | null {
|
|
35
|
+
const lines = body === '' ? 0 : body.split('\n').length
|
|
36
|
+
const heading = body.replace(/^```[\s\S]*?^```/gm, '').split('\n').find((l) => /^#{1,3}\s/.test(l))
|
|
37
|
+
if (heading != null) return `holding the heading "${heading.slice(0, 40)}", above ####`
|
|
38
|
+
const bytes = Buffer.byteLength(body)
|
|
39
|
+
if (lines > CONTRIBUTION_MAX_LINES) return `${lines} lines, over the ${CONTRIBUTION_MAX_LINES} line cap`
|
|
40
|
+
if (bytes > CONTRIBUTION_MAX_BYTES) return `${bytes} bytes, over the ${CONTRIBUTION_MAX_BYTES} byte cap`
|
|
41
|
+
return null
|
|
42
|
+
}
|
package/src/core/one-lib.ts
CHANGED
|
@@ -119,7 +119,9 @@ export function useOneLib(root: string = LIB_ROOT): void {
|
|
|
119
119
|
build.onLoad({ filter: /\/(node_modules\/rvlib-shipkit|install\/cache\/rvlib-shipkit@[^/]*)\// }, (a) => {
|
|
120
120
|
const loader = LOADERS[extname(a.path)] ?? 'ts'
|
|
121
121
|
const sub = foreignLibPath(a.path, root)
|
|
122
|
-
|
|
122
|
+
// the host copy itself lives under node_modules when it is an
|
|
123
|
+
// install: its path can carry importFresh's `?v=` stamp
|
|
124
|
+
if (sub == null) return { contents: readFileSync(a.path.replace(/\?.*$/, ''), 'utf8'), loader }
|
|
123
125
|
return reexport(join(root, sub))
|
|
124
126
|
})
|
|
125
127
|
for (const name of SHARED) {
|
|
@@ -131,7 +133,7 @@ export function useOneLib(root: string = LIB_ROOT): void {
|
|
|
131
133
|
const skip = [...new Set([host, real(host)])].map((h) => `(?!${esc(h)}/)`).join('')
|
|
132
134
|
build.onLoad({ filter: new RegExp(`^${skip}.*/node_modules/${esc(name)}/`) }, (a) => {
|
|
133
135
|
const sub = foreignSharedPath(a.path, name, host)
|
|
134
|
-
if (sub == null) return { contents: readFileSync(a.path, 'utf8'), loader: LOADERS[extname(a.path)] ?? 'js' }
|
|
136
|
+
if (sub == null) return { contents: readFileSync(a.path.replace(/\?.*$/, ''), 'utf8'), loader: LOADERS[extname(a.path)] ?? 'js' }
|
|
135
137
|
return reexport(sharedTarget(a.path, sub, name, host))
|
|
136
138
|
})
|
|
137
139
|
}
|
package/src/core/run-ci.ts
CHANGED
|
@@ -10,6 +10,7 @@ import { loadRepoConfig, type RepoConfig } from '../repo-config.ts'
|
|
|
10
10
|
import { reportToMotherhome } from './motherhome.ts'
|
|
11
11
|
import { passScope, runCheckPass, type CheckOpts } from './run-check.ts'
|
|
12
12
|
import { poolLines } from './slots.ts'
|
|
13
|
+
import { fixPassRuns } from './fix-scope.ts'
|
|
13
14
|
import { runFixPass } from './run-fixers.ts'
|
|
14
15
|
import { formatLine } from './wire.ts'
|
|
15
16
|
import { outcomeCounts } from './outcome-words.ts'
|
|
@@ -29,8 +30,9 @@ export type CiOpts = {
|
|
|
29
30
|
onlyIds?: string[]
|
|
30
31
|
/** refresh STATUS.md after the run (default true — O4: every repo gets it) */
|
|
31
32
|
status?: boolean
|
|
32
|
-
/** repair the mechanical slots before gating
|
|
33
|
-
*
|
|
33
|
+
/** repair the mechanical slots before gating, verify mode under CI.
|
|
34
|
+
* Unset: on for a whole gate, off for a narrowed one (core/fix-scope.ts).
|
|
35
|
+
* core/run-fixers.ts owns the scope. */
|
|
34
36
|
fix?: boolean
|
|
35
37
|
/** also apply the repairs of the checks declaring `fix: 'ci'` during the
|
|
36
38
|
* fix pass */
|
|
@@ -138,15 +140,14 @@ export async function runCi(root: string, o: CiOpts = {}): Promise<number> {
|
|
|
138
140
|
let c = o.config ?? (await loadRepoConfig(root))
|
|
139
141
|
// mechanical failures are not information, so they never cost a round trip
|
|
140
142
|
let fixFailed = false
|
|
141
|
-
// a NARROWED run repairs
|
|
142
|
-
//
|
|
143
|
-
//
|
|
144
|
-
//
|
|
145
|
-
//
|
|
146
|
-
//
|
|
147
|
-
// back as a test failure.
|
|
143
|
+
// a NARROWED run repairs nothing unless --fix asks, and even then only the
|
|
144
|
+
// mechanical slots. `bun run test` routes here and must answer the one
|
|
145
|
+
// question its name promises without leaving files in the agent's diff
|
|
146
|
+
// that it never wrote: in a shared tree the formatter rewrites every other
|
|
147
|
+
// agent's files too. The structural rule fixers rewrite package.json and
|
|
148
|
+
// repo config, and arming rewrites hooks/: the full gate's price alone
|
|
148
149
|
const narrowed = (o.onlyIds ?? []).length > 0
|
|
149
|
-
if (o
|
|
150
|
+
if (fixPassRuns(o)) {
|
|
150
151
|
const pass = await runFixPass(root, c, { rules: !narrowed && o.fixRules !== false, armHooks: !narrowed && o.armHooks === true, print: sink, detail })
|
|
151
152
|
fixFailed = pass.failed
|
|
152
153
|
// a fixer that rewrote the config or moved a check file leaves the
|
|
@@ -158,7 +159,7 @@ export async function runCi(root: string, o: CiOpts = {}): Promise<number> {
|
|
|
158
159
|
return 1
|
|
159
160
|
}
|
|
160
161
|
}
|
|
161
|
-
} else detail('skip: fix pass (--no-fix)')
|
|
162
|
+
} else detail(o.fix === false ? 'skip: fix pass (--no-fix)' : 'skip: fix pass (a --only run leaves the tree alone, --fix repairs first)')
|
|
162
163
|
const tier: TestTier = o.tests ?? envTier()
|
|
163
164
|
// announced BEFORE the run (under --verbose, and in every release log),
|
|
164
165
|
// so a log that died mid-way still says which tier it was asking for
|
package/src/core/slots.ts
CHANGED
|
@@ -4,6 +4,11 @@
|
|
|
4
4
|
// mutate lease's non blocking form (locked-write.ts tryLease): a slot whose
|
|
5
5
|
// holder died is free at once, and a live holder heartbeats it.
|
|
6
6
|
//
|
|
7
|
+
// A slot is a check, and a check is often a whole multi core tool (bun test
|
|
8
|
+
// workers, oxlint threads, tsc), so slots alone never bound the CPU. Past the
|
|
9
|
+
// cores the pool admits fewer slots in proportion to the 1 minute load, never
|
|
10
|
+
// fewer than one (admittedSlots). A slot already held is never taken back.
|
|
11
|
+
//
|
|
7
12
|
// The one constraint a reader breaks first: a process started BY a check
|
|
8
13
|
// (SHIPKIT_SLOT_HELD=1 in its environment) takes no slot. It already runs on
|
|
9
14
|
// its parent's, and a parent waiting on a child that waits for a slot the
|
|
@@ -12,6 +17,8 @@ import { mkdirSync, readdirSync } from 'fs'
|
|
|
12
17
|
import { homedir } from 'os'
|
|
13
18
|
import { join } from 'path'
|
|
14
19
|
import { leaseHolder, tryLease } from './locked-write.ts'
|
|
20
|
+
import { machineLoad } from './budget.ts'
|
|
21
|
+
import type { MachineLoad } from '../suite/judge-suites.ts'
|
|
15
22
|
import { PARALLEL_ENV, parallelCap, parallelCapInfo } from './parallel.ts'
|
|
16
23
|
|
|
17
24
|
/** where the slots live, for a test that must not touch the machine's pool */
|
|
@@ -19,7 +26,9 @@ export const SLOTS_DIR_ENV = 'SHIPKIT_SLOTS_DIR'
|
|
|
19
26
|
/** set for every process a check spawns: it runs on its parent's slot */
|
|
20
27
|
export const SLOT_HELD_ENV = 'SHIPKIT_SLOT_HELD'
|
|
21
28
|
|
|
22
|
-
|
|
29
|
+
/** `load`: the reading that scales what the pool admits. Omitted: the whole
|
|
30
|
+
* size, whatever the machine runs (a test's own pool) */
|
|
31
|
+
export type SlotPool = { dir: string; size: number; load?: () => MachineLoad }
|
|
23
32
|
|
|
24
33
|
export type SlotHolder = { slot: number; pid: number; repo: string; check: string; since: number }
|
|
25
34
|
|
|
@@ -37,9 +46,18 @@ export const autoGatePool = (): SlotPool => ({ dir: `${poolDir()}-auto-gate`, si
|
|
|
37
46
|
|
|
38
47
|
/** this process's pool, or null when it runs inside a check that holds one */
|
|
39
48
|
export function machinePool(): SlotPool | null {
|
|
40
|
-
return process.env[SLOT_HELD_ENV] === '1' ? null : { dir: poolDir(), size: parallelCap() }
|
|
49
|
+
return process.env[SLOT_HELD_ENV] === '1' ? null : { dir: poolDir(), size: parallelCap(), load: machineLoad }
|
|
41
50
|
}
|
|
42
51
|
|
|
52
|
+
/** how many of `size` slots a machine at this load admits: all of them up to
|
|
53
|
+
* a load of one per core, then size × cores / load, never below one. At 20
|
|
54
|
+
* on 8 cores a pool of 8 admits 3, at 81 it admits 1 */
|
|
55
|
+
export const admittedSlots = (size: number, m: MachineLoad): number =>
|
|
56
|
+
Math.max(1, Math.min(size, Math.floor((size * m.cores) / Math.max(m.cores, m.load1))))
|
|
57
|
+
|
|
58
|
+
const admitWords = (size: number, admitted: number, m: MachineLoad): string =>
|
|
59
|
+
`the pool admits ${admitted} of its ${size} slots at load ${m.load1.toFixed(1)} on ${m.cores} cores`
|
|
60
|
+
|
|
43
61
|
/** who holds which slot right now, dead holders left out. Every slot file in
|
|
44
62
|
* the dir is read, so a slot taken under a larger size still shows. */
|
|
45
63
|
export function poolOccupancy(pool: SlotPool = { dir: poolDir(), size: parallelCap() }): { size: number; dir: string; held: SlotHolder[] } {
|
|
@@ -64,25 +82,37 @@ export function poolLines(): string[] {
|
|
|
64
82
|
...(info.problem == null ? [] : [`WARN ${info.problem}`]),
|
|
65
83
|
pool == null
|
|
66
84
|
? `ok: parallel: inside a check that holds a machine slot, so this run takes none (up to ${info.cap} at once in this process)`
|
|
67
|
-
: `ok: parallel: one machine pool of ${info.cap} slot(s) shared by every shipkit process, ${poolOccupancy(pool).held.length} held now (${why})`,
|
|
85
|
+
: `ok: parallel: one machine pool of ${info.cap} slot(s) shared by every shipkit process, ${poolOccupancy(pool).held.length} held now (${why})${loadNote(pool)}`,
|
|
68
86
|
]
|
|
69
87
|
}
|
|
70
88
|
|
|
89
|
+
const loadNote = (pool: SlotPool): string => {
|
|
90
|
+
const m = pool.load?.()
|
|
91
|
+
if (m == null) return ''
|
|
92
|
+
const admitted = admittedSlots(pool.size, m)
|
|
93
|
+
return admitted < pool.size ? `; ${admitWords(pool.size, admitted, m)}` : ''
|
|
94
|
+
}
|
|
95
|
+
|
|
71
96
|
/** wait for a free slot and hold it until release(). null = abort said stop. */
|
|
72
97
|
export async function takeSlot(
|
|
73
98
|
pool: SlotPool,
|
|
74
99
|
who: { repo: string; check: string },
|
|
75
|
-
o: { abort?: () => boolean; pollMs?: number; onWait?: (held: SlotHolder[]) => void } = {},
|
|
100
|
+
o: { abort?: () => boolean; pollMs?: number; onWait?: (held: SlotHolder[], why: string) => void } = {},
|
|
76
101
|
): Promise<{ slot: number; release: () => void } | null> {
|
|
77
102
|
mkdirSync(pool.dir, { recursive: true })
|
|
78
103
|
let heard = false
|
|
79
104
|
for (;;) {
|
|
80
|
-
|
|
105
|
+
const m = pool.load?.() ?? null
|
|
106
|
+
const admitted = m == null ? pool.size : admittedSlots(pool.size, m)
|
|
107
|
+
for (let slot = 0; slot < admitted; slot++) {
|
|
81
108
|
const lease = tryLease(join(pool.dir, `slot-${slot}`), who.check, { repo: who.repo, check: who.check })
|
|
82
109
|
if (lease != null) return { slot, release: lease.release }
|
|
83
110
|
}
|
|
84
111
|
if (o.abort?.() === true) return null
|
|
85
|
-
if (!heard) {
|
|
112
|
+
if (!heard) {
|
|
113
|
+
heard = true
|
|
114
|
+
o.onWait?.(poolOccupancy(pool).held, m != null && admitted < pool.size ? admitWords(pool.size, admitted, m) : `all ${pool.size} slots are held`)
|
|
115
|
+
}
|
|
86
116
|
await Bun.sleep(o.pollMs ?? SLOT_POLL_MS)
|
|
87
117
|
}
|
|
88
118
|
}
|
|
@@ -0,0 +1,145 @@
|
|
|
1
|
+
// the house's generated context: every global process with the commands that
|
|
2
|
+
// drive it, every house repo, and each repo's global contribution, as ONE md
|
|
3
|
+
// file the house index includes, so every agent of every house repo loads it.
|
|
4
|
+
// It lives beside the index (`generated/global-context.md`), gitignored: its
|
|
5
|
+
// sources are the repos on this disk, so it is this machine's, never history.
|
|
6
|
+
//
|
|
7
|
+
// the one constraint a reader breaks first: it is in every session's context,
|
|
8
|
+
// so a contribution over its cap is cut here, and SK033 fails in its repo.
|
|
9
|
+
import { existsSync, mkdirSync, readFileSync, writeFileSync } from 'fs'
|
|
10
|
+
import { homedir } from 'os'
|
|
11
|
+
import { dirname, join } from 'path'
|
|
12
|
+
import { CONTRIBUTION_FILE, CONTRIBUTION_MAX_BYTES, CONTRIBUTION_MAX_LINES, contributionBody, contributionOver, loadsContext } from '../core/global-contribution.ts'
|
|
13
|
+
import { KEYWORD_FILE } from '../core/layout.ts'
|
|
14
|
+
import { evaluateLeak, globalKeywordFile, maskPattern, parsePatterns } from '../hooks/leak-guard.ts'
|
|
15
|
+
import { homeRepo, houseRepos } from '../serve/project-gutter.ts'
|
|
16
|
+
import { scanGlobalUnits, type GlobalProcessDecl } from './global-units.ts'
|
|
17
|
+
|
|
18
|
+
/** how often the daemon looks whether a source moved */
|
|
19
|
+
export const GLOBAL_CONTEXT_EVERY_MS = 60_000
|
|
20
|
+
|
|
21
|
+
export type ContextRepo = { name: string; path: string; description: string | null; contribution: string }
|
|
22
|
+
export type ContextProcess = Pick<GlobalProcessDecl, 'id' | 'label' | 'repo' | 'url' | 'description'>
|
|
23
|
+
|
|
24
|
+
/** the banned keyword a text matches, masked, or null: every item is
|
|
25
|
+
* scanned before it joins, since every session everywhere loads the file */
|
|
26
|
+
export type Leak = (text: string) => string | null
|
|
27
|
+
|
|
28
|
+
/** a repo description's longest form here: the rest is the repo's own */
|
|
29
|
+
export const DESCRIPTION_MAX = 160
|
|
30
|
+
|
|
31
|
+
/** where a house keeps it: beside its index */
|
|
32
|
+
export const globalContextPath = (index: string): string => join(dirname(index), 'generated', 'global-context.md')
|
|
33
|
+
|
|
34
|
+
/** a body cut to the caps, with the line that says so */
|
|
35
|
+
function capped(r: ContextRepo, body: string): string {
|
|
36
|
+
const over = contributionOver(body)
|
|
37
|
+
if (over == null) return body
|
|
38
|
+
// a heading above #### would outrank the `### <repo>` it sits under
|
|
39
|
+
let cut = body.replace(/^#{1,3}(?=\s)/gm, '####').split('\n').slice(0, CONTRIBUTION_MAX_LINES).join('\n')
|
|
40
|
+
while (Buffer.byteLength(cut) > CONTRIBUTION_MAX_BYTES) cut = cut.slice(0, cut.lastIndexOf('\n'))
|
|
41
|
+
return `${cut}\n\n(cut: ${r.name}/${CONTRIBUTION_FILE} is ${over}, and SK033 fails there until it fits)`
|
|
42
|
+
}
|
|
43
|
+
|
|
44
|
+
/** a description's first line, at most DESCRIPTION_MAX characters */
|
|
45
|
+
const short = (d: string): string => {
|
|
46
|
+
const l = d.split('\n')[0]!.trim()
|
|
47
|
+
return l.length <= DESCRIPTION_MAX ? l : `${l.slice(0, DESCRIPTION_MAX - 1)}…`
|
|
48
|
+
}
|
|
49
|
+
|
|
50
|
+
const cell = (s: string): string => s.replace(/\|/g, '\\|').replace(/\n/g, ' ')
|
|
51
|
+
|
|
52
|
+
/** a description's first line only: the rest (a login, a how to) is the
|
|
53
|
+
* hub's to show, never every session's */
|
|
54
|
+
const firstLine = (d: string | null): string => {
|
|
55
|
+
const l = d?.split('\n')[0]?.trim() ?? ''
|
|
56
|
+
return l === '' ? '' : `: ${cell(l)}`
|
|
57
|
+
}
|
|
58
|
+
|
|
59
|
+
/** the file's text. Pure: the same sources always give the same bytes, so a
|
|
60
|
+
* write happens only when a source moved */
|
|
61
|
+
export function globalContextText(o: { repos: ContextRepo[]; processes: ContextProcess[]; errors: string[]; leak?: Leak }): string {
|
|
62
|
+
const left: string[] = []
|
|
63
|
+
/** the item when it matches no banned keyword, else null and a line saying so */
|
|
64
|
+
const clean = (where: string, text: string): string | null => {
|
|
65
|
+
const hit = o.leak?.(text) ?? null
|
|
66
|
+
if (hit == null) return text
|
|
67
|
+
left.push(`- ${where}: matches the banned keyword ${hit}`)
|
|
68
|
+
return null
|
|
69
|
+
}
|
|
70
|
+
const out: string[] = [
|
|
71
|
+
'# global context: this machine, for every agent',
|
|
72
|
+
'',
|
|
73
|
+
`Generated by \`sk global-context --write\` from every house repo (the daemon rewrites it when a source moves). Never edit it: a repo adds to it through its own \`${CONTRIBUTION_FILE}\`.`,
|
|
74
|
+
'',
|
|
75
|
+
'## Global processes',
|
|
76
|
+
'',
|
|
77
|
+
]
|
|
78
|
+
if (o.processes.length === 0) out.push('No house repo declares one.')
|
|
79
|
+
else {
|
|
80
|
+
out.push('Drive each by id: `sk gp start|stop|restart <id>` waits for its probe, `sk gp open <id> [--chrome|--app]` opens its url and starts it first when it is down, `sk gp` shows where each stands.', '')
|
|
81
|
+
out.push('| id | process | repo | url |', '|---|---|---|---|')
|
|
82
|
+
for (const p of [...o.processes].sort((a, b) => a.repo.localeCompare(b.repo) || a.id.localeCompare(b.id))) {
|
|
83
|
+
const row = clean(`${p.repo} process ${p.id}`, `| \`${p.id}\` | ${cell(p.label)}${firstLine(p.description)} | ${p.repo} | ${p.url ?? 'no url: start, stop, restart only'} |`)
|
|
84
|
+
if (row != null) out.push(row)
|
|
85
|
+
}
|
|
86
|
+
}
|
|
87
|
+
out.push('', '## House repos', '')
|
|
88
|
+
for (const r of o.repos) {
|
|
89
|
+
const line = clean(`${r.name} repo line`, `- **${r.name}** \`${r.path}\`${r.description == null ? '' : `: ${short(r.description)}`}`)
|
|
90
|
+
if (line != null) out.push(line)
|
|
91
|
+
}
|
|
92
|
+
const given = o.repos
|
|
93
|
+
.map((r) => ({ r, body: contributionBody(r.contribution) }))
|
|
94
|
+
.filter((x) => x.body !== '' && clean(`${x.r.name}/${CONTRIBUTION_FILE}`, x.body) != null)
|
|
95
|
+
out.push('', '## What each repo offers', '')
|
|
96
|
+
if (given.length === 0) out.push(`No repo contributes yet: each writes its own \`${CONTRIBUTION_FILE}\`.`)
|
|
97
|
+
for (const { r, body } of given) out.push(`### ${r.name}`, '', capped(r, body), '')
|
|
98
|
+
if (left.length > 0) out.push('', '## Left out', '', ...left)
|
|
99
|
+
if (o.errors.length > 0) out.push('', '## Could not read', '', ...o.errors.map((e) => `- ${e}`))
|
|
100
|
+
return `${out.join('\n').trimEnd()}\n`
|
|
101
|
+
}
|
|
102
|
+
|
|
103
|
+
/** one repo's line and contribution, read from disk */
|
|
104
|
+
export function contextRepo(r: { name: string; path: string }, home: string = homedir()): ContextRepo {
|
|
105
|
+
let description: string | null = null
|
|
106
|
+
try {
|
|
107
|
+
const d = (JSON.parse(readFileSync(join(r.path, 'package.json'), 'utf8')) as { description?: unknown }).description
|
|
108
|
+
if (typeof d === 'string' && d.trim() !== '') description = d.trim()
|
|
109
|
+
} catch { /* no package.json, or one that does not parse: no line */ }
|
|
110
|
+
const f = join(r.path, CONTRIBUTION_FILE)
|
|
111
|
+
return { name: r.name, path: r.path.startsWith(`${home}/`) ? `~${r.path.slice(home.length)}` : r.path, description, contribution: existsSync(f) ? readFileSync(f, 'utf8') : '' }
|
|
112
|
+
}
|
|
113
|
+
|
|
114
|
+
/** writes the text when it differs from what the file holds; true when it wrote */
|
|
115
|
+
export function writeIfChanged(path: string, text: string): boolean {
|
|
116
|
+
if (existsSync(path) && readFileSync(path, 'utf8') === text) return false
|
|
117
|
+
mkdirSync(dirname(path), { recursive: true })
|
|
118
|
+
writeFileSync(path, text)
|
|
119
|
+
return true
|
|
120
|
+
}
|
|
121
|
+
|
|
122
|
+
/** the leak scan over the machine's banned keywords and the house repo's own,
|
|
123
|
+
* the lists the commit leak guard reads; null when neither exists */
|
|
124
|
+
export function houseLeak(houseRoot: string): Leak | null {
|
|
125
|
+
const files = [globalKeywordFile(), join(houseRoot, KEYWORD_FILE)].filter((f): f is string => f != null && existsSync(f))
|
|
126
|
+
if (files.length === 0) return null
|
|
127
|
+
const patterns = parsePatterns(files.map((f) => readFileSync(f, 'utf8')).join('\n'))
|
|
128
|
+
return (text) => {
|
|
129
|
+
const hit = evaluateLeak([{ where: 'global context', content: text }], patterns, []).hits[0]
|
|
130
|
+
return hit == null ? null : maskPattern(hit.pattern)
|
|
131
|
+
}
|
|
132
|
+
}
|
|
133
|
+
|
|
134
|
+
/** THE computation, for the house the repo at `root` belongs to: where the
|
|
135
|
+
* file goes and what it holds. The cli and the daemon both call it. Null
|
|
136
|
+
* when `root` includes no house index, or one that does not load the file */
|
|
137
|
+
export async function houseContext(root: string, home: string = homedir()): Promise<{ path: string; text: string } | null> {
|
|
138
|
+
const h = houseRepos(homeRepo(root), home)
|
|
139
|
+
if (h.index == null || !loadsContext(h.index)) return null
|
|
140
|
+
const units = await scanGlobalUnits(h.repos.map((r) => ({ name: r.name, path: r.path })))
|
|
141
|
+
return {
|
|
142
|
+
path: globalContextPath(h.index),
|
|
143
|
+
text: globalContextText({ repos: h.repos.map((r) => contextRepo(r, home)), processes: units.processes, errors: units.errors, leak: houseLeak(h.home) ?? undefined }),
|
|
144
|
+
}
|
|
145
|
+
}
|