skillgauntlet 1.1.1 → 1.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +201 -201
- package/README.md +44 -1
- package/bin/skillgauntlet.js +380 -131
- package/lib/automerge.js +107 -0
- package/lib/browse.js +58 -0
- package/lib/chart.js +160 -0
- package/lib/depth.js +4 -1
- package/lib/discover.js +27 -2
- package/lib/findhost.js +83 -0
- package/lib/i18n.js +405 -39
- package/lib/models.js +46 -0
- package/lib/report.js +1 -1
- package/lib/schema.js +90 -0
- package/lib/scrub.js +186 -0
- package/lib/static.js +24 -13
- package/lib/submission.js +325 -0
- package/lib/submit.js +208 -0
- package/lib/ticker.js +39 -0
- package/lib/trial.js +65 -13
- package/lib/ui.js +40 -2
- package/lib/update.js +73 -0
- package/package.json +4 -2
- package/schemas/submission-1.json +210 -0
package/lib/automerge.js
ADDED
|
@@ -0,0 +1,107 @@
|
|
|
1
|
+
// The decision rules of the submission auto-merge, as pure functions: no network, no files, no clock.
|
|
2
|
+
// scripts/automerge.mjs gathers the facts (through the GitHub API, as data) and applies the verdict.
|
|
3
|
+
//
|
|
4
|
+
// A pull request is merged automatically only when every rule here holds. Anything else is not an
|
|
5
|
+
// error: it is sent to a maintainer with the reasons written on the pull request.
|
|
6
|
+
import { FILES, LIMITS } from './submission.js';
|
|
7
|
+
|
|
8
|
+
export const POLICY = {
|
|
9
|
+
fileBytes: LIMITS.fileBytes, // per file (the same cap the validator enforces)
|
|
10
|
+
totalBytes: 10 * 1024 * 1024, // all four files of one submission
|
|
11
|
+
maxFiles: FILES.length,
|
|
12
|
+
perAccountPerDay: 5, // auto-merged submissions per account in the last 24 hours
|
|
13
|
+
minAccountAgeDays: 30, // younger accounts go to manual review
|
|
14
|
+
trustedAssociations: ['OWNER', 'MEMBER', 'COLLABORATOR'], // skip the account-age and recent-failure rules only
|
|
15
|
+
claContext: 'license/cla',
|
|
16
|
+
label: 'needs-review',
|
|
17
|
+
};
|
|
18
|
+
|
|
19
|
+
const DIR_RE = new RegExp(`^(submissions/[a-z0-9][a-z0-9._-]*/[a-z0-9][a-z0-9._-]*/[0-9]{8}T[0-9]{6}Z-[A-Za-z0-9][A-Za-z0-9-]*)/(${FILES.map((f) => f.replace('.', '\\.')).join('|')})$`);
|
|
20
|
+
|
|
21
|
+
// The text goes into a pull-request comment: no @mentions, one line, bounded.
|
|
22
|
+
const safe = (s) => String(s).replace(/@/g, '@').replace(/[\r\n`]+/g, ' ').slice(0, 200);
|
|
23
|
+
|
|
24
|
+
// changed: [{ filename, status }] from the pull request's file list.
|
|
25
|
+
// entries: { [path]: { type, size } } from the contents API at the head commit (type is 'file' for a plain
|
|
26
|
+
// file, 'symlink' / 'submodule' / 'dir' otherwise). Returns { ok, reasons, dir }.
|
|
27
|
+
export function gateFiles(changed, entries = {}) {
|
|
28
|
+
const reasons = [];
|
|
29
|
+
const dirs = new Set();
|
|
30
|
+
if (!changed.length) reasons.push('the pull request lists no changed files');
|
|
31
|
+
if (changed.length > POLICY.maxFiles) reasons.push(`it changes ${changed.length} files; a submission is exactly ${POLICY.maxFiles}`);
|
|
32
|
+
let total = 0;
|
|
33
|
+
const seen = new Set();
|
|
34
|
+
for (const { filename, status } of changed) {
|
|
35
|
+
const name = String(filename);
|
|
36
|
+
if (name.split('/').some((seg) => seg === '..' || seg === '.' || seg === '') || name.includes('\\')) {
|
|
37
|
+
reasons.push(`\`${safe(name)}\` is not a plain relative path`);
|
|
38
|
+
continue;
|
|
39
|
+
}
|
|
40
|
+
const m = DIR_RE.exec(name);
|
|
41
|
+
if (!m) { reasons.push(`\`${safe(name)}\` is outside a submission directory or is not one of ${FILES.join(', ')}`); continue; }
|
|
42
|
+
dirs.add(m[1]);
|
|
43
|
+
seen.add(m[2]);
|
|
44
|
+
if (status !== 'added') { reasons.push(`\`${safe(name)}\` is ${safe(status)}, not added`); continue; }
|
|
45
|
+
const e = entries[name];
|
|
46
|
+
if (!e) { reasons.push(`\`${safe(name)}\` could not be read at the head commit`); continue; }
|
|
47
|
+
if (e.type !== 'file') { reasons.push(`\`${safe(name)}\` is a ${safe(e.type)}, not a regular file`); continue; }
|
|
48
|
+
if (!Number.isInteger(e.size) || e.size < 0) { reasons.push(`\`${safe(name)}\` has no usable size`); continue; }
|
|
49
|
+
if (e.size > POLICY.fileBytes) reasons.push(`\`${safe(name)}\` is ${e.size} bytes; the limit is ${POLICY.fileBytes}`);
|
|
50
|
+
total += e.size;
|
|
51
|
+
}
|
|
52
|
+
if (dirs.size > 1) reasons.push(`it touches ${dirs.size} submission directories; automatic merging takes exactly one`);
|
|
53
|
+
if (dirs.size === 1 && !reasons.length) {
|
|
54
|
+
const missing = FILES.filter((f) => !seen.has(f));
|
|
55
|
+
if (missing.length) reasons.push(`missing file(s): ${missing.join(', ')}`);
|
|
56
|
+
}
|
|
57
|
+
if (total > POLICY.totalBytes) reasons.push(`the files total ${total} bytes; the limit is ${POLICY.totalBytes}`);
|
|
58
|
+
return { ok: reasons.length === 0 && dirs.size === 1, reasons, dir: dirs.size === 1 ? [...dirs][0] : null };
|
|
59
|
+
}
|
|
60
|
+
|
|
61
|
+
// account: { createdAt, association, mergedLastDay, flaggedLastDay } (counts exclude this pull request).
|
|
62
|
+
export function gateAccount(account, now, policy = POLICY) {
|
|
63
|
+
const reasons = [];
|
|
64
|
+
if (account.mergedLastDay >= policy.perAccountPerDay) {
|
|
65
|
+
reasons.push(`this account already has ${account.mergedLastDay} auto-merged submissions in the last 24 hours (limit ${policy.perAccountPerDay})`);
|
|
66
|
+
}
|
|
67
|
+
if (!policy.trustedAssociations.includes(account.association)) {
|
|
68
|
+
const created = Date.parse(account.createdAt);
|
|
69
|
+
if (!Number.isFinite(created)) reasons.push('the account creation date is unknown');
|
|
70
|
+
else if ((now - created) / 86400000 < policy.minAccountAgeDays) reasons.push(`the account is younger than ${policy.minAccountAgeDays} days`);
|
|
71
|
+
if (account.flaggedLastDay > 0) reasons.push('a submission from this account was refused in the last 24 hours');
|
|
72
|
+
}
|
|
73
|
+
return { ok: reasons.length === 0, reasons };
|
|
74
|
+
}
|
|
75
|
+
|
|
76
|
+
// facts: {
|
|
77
|
+
// validatedSha, currentSha – the head commit the validation read, and the head right now
|
|
78
|
+
// files: gateFiles() result, validation: { ok, errors }, cla: 'success' | other | null,
|
|
79
|
+
// account: gateAccount() result
|
|
80
|
+
// }
|
|
81
|
+
// Returns { merge, reasons, waitForCla }. waitForCla marks a refusal that is only "not yet": the maintainer or
|
|
82
|
+
// contributor fixes it by signing, and no needs-review label is warranted.
|
|
83
|
+
export function decide({ validatedSha, currentSha, files, validation, cla, account }) {
|
|
84
|
+
const reasons = [];
|
|
85
|
+
if (!validatedSha || validatedSha !== currentSha) {
|
|
86
|
+
return { merge: false, reasons: ['the pull request changed after it was checked; the next run will check the new commit'], stale: true, waitForCla: false };
|
|
87
|
+
}
|
|
88
|
+
if (!files.ok) reasons.push(...files.reasons);
|
|
89
|
+
if (files.ok && !validation.ok) reasons.push(...validation.errors.map((e) => `check: ${e}`));
|
|
90
|
+
if (files.ok && validation.ok && validation.dir && validation.dir !== files.dir) reasons.push('the validated directory differs from the changed directory');
|
|
91
|
+
const claOk = cla === 'success';
|
|
92
|
+
if (!claOk) reasons.push(`the \`${POLICY.claContext}\` check is ${cla || 'missing'}; sign the CLA (see the comment from the CLA bot) and a maintainer can re-run this`);
|
|
93
|
+
if (!account.ok) reasons.push(...account.reasons);
|
|
94
|
+
const onlyCla = !claOk && reasons.length === 1;
|
|
95
|
+
return { merge: reasons.length === 0, reasons, stale: false, waitForCla: onlyCla };
|
|
96
|
+
}
|
|
97
|
+
|
|
98
|
+
export function renderVerdict({ merge, reasons, stale }, { dir } = {}) {
|
|
99
|
+
const marker = '<!-- skillgauntlet-automerge -->';
|
|
100
|
+
if (merge) return `${marker}\n### ✅ Merged automatically\n\n\`${safe(dir)}\` passed every check and was merged as a **community single sample**.\n`;
|
|
101
|
+
if (stale) return null;
|
|
102
|
+
return `${marker}\n### ⏸️ Not merged automatically\n\n${reasons.map((r) => `- ${r}`).join('\n')}\n\nA maintainer will look at this by hand. Nothing is wrong with asking: this is a safety rule, not a verdict on your results.\n`;
|
|
103
|
+
}
|
|
104
|
+
|
|
105
|
+
export function mergeTitle(dir) {
|
|
106
|
+
return `Add submission ${dir.replace(/^submissions\//, '')}`;
|
|
107
|
+
}
|
package/lib/browse.js
ADDED
|
@@ -0,0 +1,58 @@
|
|
|
1
|
+
// Picking a directory with the arrow keys: the list shows the folders under the current one, a way up, and a way
|
|
2
|
+
// to type a path instead. `select` is injected so the walk can be tested without a terminal.
|
|
3
|
+
import fs from 'node:fs';
|
|
4
|
+
import os from 'node:os';
|
|
5
|
+
import path from 'node:path';
|
|
6
|
+
import { t } from './i18n.js';
|
|
7
|
+
|
|
8
|
+
const DOT_KEPT = new Set(['.claude', '.agents', '.codex', '.codebuddy']);
|
|
9
|
+
|
|
10
|
+
function subdirs(dir) {
|
|
11
|
+
let entries = [];
|
|
12
|
+
try { entries = fs.readdirSync(dir, { withFileTypes: true }); } catch { return []; }
|
|
13
|
+
return entries
|
|
14
|
+
.filter((e) => e.isDirectory() && (!e.name.startsWith('.') || DOT_KEPT.has(e.name)) && e.name !== 'node_modules')
|
|
15
|
+
.map((e) => e.name)
|
|
16
|
+
.sort((a, b) => a.localeCompare(b));
|
|
17
|
+
}
|
|
18
|
+
|
|
19
|
+
const holdsSkill = (dir) => fs.existsSync(path.join(dir, 'SKILL.md'));
|
|
20
|
+
|
|
21
|
+
function drives() {
|
|
22
|
+
const found = [];
|
|
23
|
+
for (let c = 65; c <= 90; c += 1) {
|
|
24
|
+
const root = `${String.fromCharCode(c)}:\\`;
|
|
25
|
+
if (fs.existsSync(root)) found.push(root);
|
|
26
|
+
}
|
|
27
|
+
return found;
|
|
28
|
+
}
|
|
29
|
+
|
|
30
|
+
// A long path would wrap and break the in-place redraw, so only its tail is shown.
|
|
31
|
+
const tail = (p, n = 36) => (p.length > n ? `…${p.slice(-(n - 1))}` : p);
|
|
32
|
+
|
|
33
|
+
// Returns the chosen directory, or null when the user asks to type a path instead.
|
|
34
|
+
export async function browseDir(select, start = process.cwd()) {
|
|
35
|
+
let dir = path.resolve(start);
|
|
36
|
+
for (;;) {
|
|
37
|
+
const parent = path.dirname(dir);
|
|
38
|
+
const atRoot = parent === dir;
|
|
39
|
+
const options = [{ label: t('browseUse'), hint: holdsSkill(dir) ? t('browseHasSkill') : '', value: { use: dir } }];
|
|
40
|
+
if (!atRoot) options.push({ label: t('browseUp'), hint: tail(parent), value: { go: parent } });
|
|
41
|
+
else if (process.platform === 'win32') options.push({ label: t('browseDrives'), value: { drives: true } });
|
|
42
|
+
if (dir !== os.homedir()) options.push({ label: t('browseHome'), hint: tail(os.homedir()), value: { go: os.homedir() } });
|
|
43
|
+
for (const name of subdirs(dir)) {
|
|
44
|
+
const full = path.join(dir, name);
|
|
45
|
+
options.push({ label: `${name}${path.sep}`, hint: holdsSkill(full) ? t('browseHasSkill') : '', value: { go: full } });
|
|
46
|
+
}
|
|
47
|
+
options.push({ label: t('browseType'), value: { type: true } });
|
|
48
|
+
|
|
49
|
+
const pick = await select(t('browseTitle', { dir: tail(dir) }), options, 0);
|
|
50
|
+
if (pick.use) return pick.use;
|
|
51
|
+
if (pick.type) return null;
|
|
52
|
+
if (pick.go) dir = pick.go;
|
|
53
|
+
if (pick.drives) {
|
|
54
|
+
const root = await select(t('browseDrives'), drives().map((d) => ({ label: d, value: d })), 0);
|
|
55
|
+
dir = root;
|
|
56
|
+
}
|
|
57
|
+
}
|
|
58
|
+
}
|
package/lib/chart.js
ADDED
|
@@ -0,0 +1,160 @@
|
|
|
1
|
+
// A result drawn for the terminal: bars for rates, one mark per question, a verdict in words. The Markdown report
|
|
2
|
+
// stays the full record; this is what a person reads first, so it carries the few numbers that decide the reading.
|
|
3
|
+
import { stdout } from 'node:process';
|
|
4
|
+
import { gradeTrigger } from './submission.js';
|
|
5
|
+
import { t, lang } from './i18n.js';
|
|
6
|
+
|
|
7
|
+
const color = () => stdout.isTTY && !process.env.NO_COLOR;
|
|
8
|
+
const paint = (code, s) => (color() ? `\x1b[${code}m${s}\x1b[0m` : s);
|
|
9
|
+
const green = (s) => paint(32, s);
|
|
10
|
+
const yellow = (s) => paint(33, s);
|
|
11
|
+
const red = (s) => paint(31, s);
|
|
12
|
+
const dim = (s) => paint(2, s);
|
|
13
|
+
const bold = (s) => paint(1, s);
|
|
14
|
+
|
|
15
|
+
const pct = (x) => (x == null ? '—' : `${Math.round(x * 100)}%`);
|
|
16
|
+
|
|
17
|
+
// width cells; filled share = value (0..1)
|
|
18
|
+
export function bar(value, width = 24, tone = (s) => s) {
|
|
19
|
+
const v = Math.min(1, Math.max(0, value ?? 0));
|
|
20
|
+
const n = Math.round(v * width);
|
|
21
|
+
return `${tone('█'.repeat(n))}${dim('░'.repeat(width - n))}`;
|
|
22
|
+
}
|
|
23
|
+
|
|
24
|
+
const good = (v) => (v >= 0.8 ? green : v >= 0.5 ? yellow : red); // higher is better
|
|
25
|
+
const bad = (v) => (v <= 0.1 ? green : v <= 0.25 ? yellow : red); // lower is better
|
|
26
|
+
|
|
27
|
+
const GRADE_TONE = { well: green, fair: yellow, poorly: red, insufficient: dim };
|
|
28
|
+
|
|
29
|
+
function staticLines(result) {
|
|
30
|
+
const L = [bold(t('chart.static'))];
|
|
31
|
+
for (const g of result.groups) {
|
|
32
|
+
const mark = g.verdict === 'pass' ? green('✓') : g.verdict === 'warn' ? yellow('!') : red('✗');
|
|
33
|
+
L.push(` ${mark} ${g.id} ${t(g.titleKey)}`);
|
|
34
|
+
// Every check of the group with its reason: a letter and a mark alone cannot be acted on. Clean ones are dimmed.
|
|
35
|
+
for (const c of g.checks) {
|
|
36
|
+
const flagged = c.status === 'warn' || c.status === 'fail';
|
|
37
|
+
const sym = c.status === 'fail' ? red('✗') : c.status === 'warn' ? yellow('!') : c.status === 'info' ? dim('·') : green('✓');
|
|
38
|
+
const text = `${c.id} ${t(c.titleKey)} — ${cut(t(c.detailKey, c.detailVars), 70)}`;
|
|
39
|
+
L.push(` ${sym} ${flagged ? text : dim(text)}`);
|
|
40
|
+
for (const e of (c.evidence || []).slice(0, 2)) L.push(` ${dim(`${cut(`${e.file || ''}:${e.line}`, 56)} ${cut(e.match, 40)}`)}`);
|
|
41
|
+
}
|
|
42
|
+
}
|
|
43
|
+
L.push(` ${dim(t('chart.staticLegend'))}`);
|
|
44
|
+
const st = result.stats;
|
|
45
|
+
if (st) L.push(` ${dim(t('chart.stats', { tokens: st.skillMdTokens, desc: st.descriptionChars, files: st.files }))}`);
|
|
46
|
+
return L;
|
|
47
|
+
}
|
|
48
|
+
|
|
49
|
+
function triggerLines(trial, records) {
|
|
50
|
+
const pos = trial.questions.filter((q) => q.shouldTrigger);
|
|
51
|
+
const neg = trial.questions.filter((q) => !q.shouldTrigger);
|
|
52
|
+
const nPos = pos.reduce((a, q) => a + q.n, 0);
|
|
53
|
+
const nNeg = neg.reduce((a, q) => a + q.n, 0);
|
|
54
|
+
const grade = gradeTrigger({ recall: trial.recall, fpr: trial.fpr, nPos, nNeg });
|
|
55
|
+
const ci = (c) => (c?.[0] == null ? '' : dim(` ${t('chart.ci', { lo: pct(c[0]), hi: pct(c[1]) })}`));
|
|
56
|
+
const L = [bold(t('chart.trigger')), ''];
|
|
57
|
+
L.push(` ${dim(t('chart.conditions', { host: trial.host || '?', model: trial.modelResolved || trial.model || '?', trials: trial.trials, failed: trial.failedTrials || 0 }))}`, '');
|
|
58
|
+
L.push(` ${t('chart.verdict')} ${(GRADE_TONE[grade] || ((s) => s))(bold(t(`chart.grade.${grade}`)))}`);
|
|
59
|
+
L.push('');
|
|
60
|
+
L.push(` ${padCells(t('chart.recall'), pad(t('chart.recall'), t('chart.fpr')))}${bar(trial.recall, 24, good(trial.recall ?? 0))} ${pct(trial.recall)}${ci(trial.recallCi)}`);
|
|
61
|
+
L.push(` ${padCells(t('chart.fpr'), pad(t('chart.recall'), t('chart.fpr')))}${bar(trial.fpr, 24, bad(trial.fpr ?? 0))} ${pct(trial.fpr)}${ci(trial.fprCi)}`);
|
|
62
|
+
L.push('');
|
|
63
|
+
L.push(` ${dim(t('chart.legend'))}`);
|
|
64
|
+
const strip = (qs) => qs.map((q) => (q.hits * 2 > q.n ? green('●') : q.hits > 0 ? yellow('◐') : dim('○'))).join('');
|
|
65
|
+
L.push(` ${padCells(t('chart.should'), pad(t('chart.should'), t('chart.shouldNot')))}${strip(pos)} ${dim(`${pos.filter((q) => q.hits * 2 > q.n).length}/${pos.length}`)}`);
|
|
66
|
+
L.push(` ${padCells(t('chart.shouldNot'), pad(t('chart.should'), t('chart.shouldNot')))}${strip(neg)} ${dim(`${neg.filter((q) => q.hits > 0).length}/${neg.length}`)}`);
|
|
67
|
+
if (trial.incomplete) L.push(` ${yellow(t('chart.incomplete', { n: trial.incomplete }))}`);
|
|
68
|
+
if (trial.recall === 0 && trial.fpr === 0) L.push('', ` ${yellow(t('chart.allZero'))}`);
|
|
69
|
+
if (records?.length) L.push(...detailLines(records));
|
|
70
|
+
return L;
|
|
71
|
+
}
|
|
72
|
+
|
|
73
|
+
// Width of the wider of two labels in terminal cells: CJK characters take two.
|
|
74
|
+
function cells(s) {
|
|
75
|
+
let n = 0;
|
|
76
|
+
for (const ch of String(s)) n += /[ᄀ-ᅟ⺀-가-힣豈-︰--⦆¢-₩]/.test(ch) ? 2 : 1;
|
|
77
|
+
return n;
|
|
78
|
+
}
|
|
79
|
+
function pad(a, b) { return Math.max(cells(a), cells(b)) + 2; }
|
|
80
|
+
// String.padEnd counts code units, not cells, so columns are padded by hand.
|
|
81
|
+
const padCells = (str, width) => str + ' '.repeat(Math.max(0, width - cells(str)));
|
|
82
|
+
|
|
83
|
+
// What the model did instead when the skill was not used, then the questions behind the numbers.
|
|
84
|
+
function detailLines(records) {
|
|
85
|
+
const L = [];
|
|
86
|
+
const missed = records.filter((r) => r.shouldTrigger && r.verdict !== 'triggered');
|
|
87
|
+
if (missed.length) {
|
|
88
|
+
const kinds = new Map();
|
|
89
|
+
for (const r of missed) {
|
|
90
|
+
const k = r.verdict === 'other_skill' ? t('chart.otherSkill', { name: r.invokedSkill || '?' }) : (r.firstTool || t('chart.noTool'));
|
|
91
|
+
kinds.set(k, (kinds.get(k) || 0) + 1);
|
|
92
|
+
}
|
|
93
|
+
const rows = [...kinds].sort((a, b) => b[1] - a[1]);
|
|
94
|
+
const w = Math.max(...rows.map(([k]) => cells(k))) + 2;
|
|
95
|
+
L.push('', ` ${bold(t('chart.missedTitle', { n: missed.length }))}`);
|
|
96
|
+
for (const [k, n] of rows) L.push(` ${padCells(k, w)}${bar(n / missed.length, 16, yellow)} ${n}`);
|
|
97
|
+
const shown = missed.slice(0, 8);
|
|
98
|
+
L.push('', ` ${dim(t('chart.missedList'))}`);
|
|
99
|
+
for (const r of shown) L.push(` ${dim('○')} ${cut(r.query, 46)} ${dim(`→ ${r.firstTool || t('chart.noTool')}`)}`);
|
|
100
|
+
if (missed.length > shown.length) L.push(` ${dim(t('chart.more', { n: missed.length - shown.length }))}`);
|
|
101
|
+
}
|
|
102
|
+
const wrong = records.filter((r) => !r.shouldTrigger && r.verdict === 'triggered');
|
|
103
|
+
if (wrong.length) {
|
|
104
|
+
L.push('', ` ${bold(t('chart.wrongTitle', { n: wrong.length }))}`);
|
|
105
|
+
for (const r of wrong.slice(0, 8)) L.push(` ${red('●')} ${cut(r.query, 46)}`);
|
|
106
|
+
}
|
|
107
|
+
return L;
|
|
108
|
+
}
|
|
109
|
+
|
|
110
|
+
// Cut to n terminal cells, with an ellipsis when something was removed.
|
|
111
|
+
function cut(str, n) {
|
|
112
|
+
let out = '';
|
|
113
|
+
let w = 0;
|
|
114
|
+
for (const ch of String(str).replace(/\s+/g, ' ')) {
|
|
115
|
+
const c = cells(ch);
|
|
116
|
+
if (w + c > n - 1) return `${out}…`;
|
|
117
|
+
out += ch;
|
|
118
|
+
w += c;
|
|
119
|
+
}
|
|
120
|
+
return out;
|
|
121
|
+
}
|
|
122
|
+
|
|
123
|
+
function probeLines(probe) {
|
|
124
|
+
const same = (b) => (b ? green(t('probeSame')) : yellow(t('probeDiffer')));
|
|
125
|
+
const L = [bold(t('chart.probe')), ` ${dim(t('chart.probeAsk'))}`, ''];
|
|
126
|
+
const w = Math.max(cells(t('probeWith')), cells(t('probeWithout'))) + 2;
|
|
127
|
+
for (const [label, side] of [[t('probeWith'), probe.withSkill], [t('probeWithout'), probe.without]]) {
|
|
128
|
+
if (!side) continue;
|
|
129
|
+
const act = side.firstAction ? cut(`${side.firstAction.kind}: ${side.firstAction.detail}`, 80) : t('chart.noAction');
|
|
130
|
+
const files = side.createdFiles?.length ? cut(side.createdFiles.join(', '), 80) : t('chart.noFiles');
|
|
131
|
+
L.push(` ${bold(padCells(label, w))}${t('chart.firstAction')} ${act}`);
|
|
132
|
+
L.push(` ${padCells('', w)}${t('chart.filesMade')} ${files}`);
|
|
133
|
+
}
|
|
134
|
+
L.push('', ` ${t('probeActionKind')} ${same(probe.actionKindIdentical)}`, ` ${t('probeArtifacts')} ${same(probe.artifactsIdentical)}`);
|
|
135
|
+
const reading = probe.actionKindIdentical && probe.artifactsIdentical ? 'chart.probeNoEffect'
|
|
136
|
+
: probe.artifactsIdentical === false ? 'chart.probeEffect' : 'chart.probeMixed';
|
|
137
|
+
L.push('', ` ${t(reading)}`, ` ${dim(t('chart.probeNote'))}`);
|
|
138
|
+
return L;
|
|
139
|
+
}
|
|
140
|
+
|
|
141
|
+
function gainLines(gain) {
|
|
142
|
+
const L = [bold(t('chart.gain')), ` ${dim(t('chart.gainLegend'))}`];
|
|
143
|
+
for (const c of gain.contracts) {
|
|
144
|
+
L.push(` ${String(c.descI18n?.[lang()] || c.desc).slice(0, 60)}`);
|
|
145
|
+
L.push(` ${padCells(t('gain.withHeader'), 10)}${bar(c.with, 20, green)} ${pct(c.with)}`);
|
|
146
|
+
L.push(` ${padCells(t('gain.withoutHeader'), 10)}${bar(c.without, 20, (s) => s)} ${pct(c.without)}`);
|
|
147
|
+
}
|
|
148
|
+
return L;
|
|
149
|
+
}
|
|
150
|
+
|
|
151
|
+
// One block per tested skill. Any part may be absent (static only, probe only, no gain layer).
|
|
152
|
+
export function renderResult({ name, result, trial, records, gain, probe, reportPath }) {
|
|
153
|
+
const L = ['', bold(`════ ${name} ════`), ''];
|
|
154
|
+
if (result) L.push(...staticLines(result), '');
|
|
155
|
+
if (trial) L.push(...triggerLines(trial, records), '');
|
|
156
|
+
if (probe) L.push(...probeLines(probe), '');
|
|
157
|
+
if (gain?.contracts?.length) L.push(...gainLines(gain), '');
|
|
158
|
+
if (reportPath) L.push(dim(t('chart.full', { path: reportPath })));
|
|
159
|
+
return L.join('\n');
|
|
160
|
+
}
|
package/lib/depth.js
CHANGED
|
@@ -15,7 +15,7 @@ export const DEPTHS = {
|
|
|
15
15
|
id: 'quick',
|
|
16
16
|
sets: 1,
|
|
17
17
|
perClause: 2,
|
|
18
|
-
negatives:
|
|
18
|
+
negatives: 10, // the grade needs at least 10 on each side, so fewer would make every quick result "too few"
|
|
19
19
|
runs: 1,
|
|
20
20
|
maxExplore: 0,
|
|
21
21
|
layers: ['static', 'trigger'],
|
|
@@ -53,3 +53,6 @@ export function estimate(depth, clauses = 3) {
|
|
|
53
53
|
minutes: Math.max(1, Math.round((trials * 4) / 2 / 60)),
|
|
54
54
|
};
|
|
55
55
|
}
|
|
56
|
+
|
|
57
|
+
// Minutes for a number of trials, on the same assumption as estimate(): 2 in parallel at ~4 s each.
|
|
58
|
+
export const minutesFor = (trials) => Math.max(1, Math.round((trials * 4) / 2 / 60));
|
package/lib/discover.js
CHANGED
|
@@ -5,6 +5,9 @@ import os from 'node:os';
|
|
|
5
5
|
import path from 'node:path';
|
|
6
6
|
|
|
7
7
|
const SKIP = new Set(['.git', 'node_modules', '.cache', 'dist', 'build', 'vendor', '__pycache__']);
|
|
8
|
+
// Dot directories are skipped, except the ones agent hosts install skills into: a repository's own skill
|
|
9
|
+
// often lives only under .claude/skills.
|
|
10
|
+
const SKILL_DOT_DIRS = new Set(['.claude', '.agents', '.codex', '.codebuddy']);
|
|
8
11
|
|
|
9
12
|
export function findSkills(root, { maxDepth = 6 } = {}) {
|
|
10
13
|
const found = [];
|
|
@@ -17,12 +20,25 @@ export function findSkills(root, { maxDepth = 6 } = {}) {
|
|
|
17
20
|
return; // a skill dir does not contain nested skills
|
|
18
21
|
}
|
|
19
22
|
for (const e of entries) {
|
|
20
|
-
if (!e.isDirectory() || SKIP.has(e.name) || e.name.startsWith('.')) continue;
|
|
23
|
+
if (!e.isDirectory() || SKIP.has(e.name) || (e.name.startsWith('.') && !SKILL_DOT_DIRS.has(e.name))) continue;
|
|
21
24
|
walk(path.join(dir, e.name), depth + 1);
|
|
22
25
|
}
|
|
23
26
|
};
|
|
24
27
|
walk(path.resolve(root), 0);
|
|
25
|
-
return found.sort();
|
|
28
|
+
return dropCopies(found.sort());
|
|
29
|
+
}
|
|
30
|
+
|
|
31
|
+
// Repositories often ship a second copy of a skill (a packaged template next to the installed one). A copy with
|
|
32
|
+
// the same name and byte-identical SKILL.md adds nothing to a test, so only the first path is kept.
|
|
33
|
+
function dropCopies(dirs) {
|
|
34
|
+
const seen = new Set();
|
|
35
|
+
return dirs.filter((d) => {
|
|
36
|
+
let key = d;
|
|
37
|
+
try { key = [path.basename(d), fs.readFileSync(path.join(d, 'SKILL.md'), 'utf8')].join(' | '); } catch { /* keep it */ }
|
|
38
|
+
if (seen.has(key)) return false;
|
|
39
|
+
seen.add(key);
|
|
40
|
+
return true;
|
|
41
|
+
});
|
|
26
42
|
}
|
|
27
43
|
|
|
28
44
|
export function readFrontmatter(skillDir) {
|
|
@@ -81,3 +97,12 @@ export function resolveSpec(spec) {
|
|
|
81
97
|
} catch { /* keep empty */ }
|
|
82
98
|
return { kind: 'git', dir, source: { repo: `https://github.com/${owner}/${repo}`, commit, ref: ref || null } };
|
|
83
99
|
}
|
|
100
|
+
|
|
101
|
+
// The name a host knows a skill by is the `name` in its SKILL.md frontmatter; the folder name is only a fallback.
|
|
102
|
+
// A skill at the root of a repository sits in a folder named after the repository (or its cache slug), so using
|
|
103
|
+
// the folder name made every trial of such a skill look like "another skill was used".
|
|
104
|
+
export function skillNameOf(skillDir) {
|
|
105
|
+
const fm = readFrontmatter(skillDir).fm;
|
|
106
|
+
const name = String(fm.name || '').trim();
|
|
107
|
+
return /^[\w.-]+$/.test(name) ? name : path.basename(skillDir);
|
|
108
|
+
}
|
package/lib/findhost.js
ADDED
|
@@ -0,0 +1,83 @@
|
|
|
1
|
+
// A host CLI that is not on PATH is often still on the machine: in npm's global folder, in Downloads under the
|
|
2
|
+
// name the release page gave it, or in a tools folder. Look there before telling the user to install anything,
|
|
3
|
+
// and where the host is an npm package offer to install it.
|
|
4
|
+
import { execSync } from 'node:child_process';
|
|
5
|
+
import fs from 'node:fs';
|
|
6
|
+
import os from 'node:os';
|
|
7
|
+
import path from 'node:path';
|
|
8
|
+
|
|
9
|
+
// The command name each host answers to, and (when there is one) the npm package that provides it.
|
|
10
|
+
export const HOST_CLI = {
|
|
11
|
+
'claude-code': { bin: 'claude', pkg: '@anthropic-ai/claude-code' },
|
|
12
|
+
codebuddy: { bin: 'codebuddy', pkg: '@tencent-ai/codebuddy-code' },
|
|
13
|
+
codewhale: { bin: 'codewhale', pkg: 'codewhale' },
|
|
14
|
+
};
|
|
15
|
+
|
|
16
|
+
const isWin = process.platform === 'win32';
|
|
17
|
+
const RUNNABLE = isWin ? /\.(exe|cmd|bat)$/i : /^[^.]*$/;
|
|
18
|
+
|
|
19
|
+
function npmGlobalDir() {
|
|
20
|
+
try {
|
|
21
|
+
const prefix = execSync('npm prefix -g', { encoding: 'utf8', timeout: 15000, windowsHide: true, stdio: ['ignore', 'pipe', 'ignore'] }).trim();
|
|
22
|
+
return isWin ? prefix : path.join(prefix, 'bin');
|
|
23
|
+
} catch { return null; }
|
|
24
|
+
}
|
|
25
|
+
|
|
26
|
+
function listDir(dir) {
|
|
27
|
+
try { return fs.readdirSync(dir, { withFileTypes: true }); } catch { return []; }
|
|
28
|
+
}
|
|
29
|
+
|
|
30
|
+
// Files named like the host (codewhale.exe, codewhale-windows-x64.exe, codewhale.cmd) directly in dir.
|
|
31
|
+
function inDir(dir, bin) {
|
|
32
|
+
const out = [];
|
|
33
|
+
for (const e of listDir(dir)) {
|
|
34
|
+
if (!e.isFile()) continue;
|
|
35
|
+
const base = e.name.toLowerCase();
|
|
36
|
+
const exact = base === bin || base.startsWith(`${bin}.`);
|
|
37
|
+
// a Windows release asset keeps its platform suffix (codewhale-windows-x64.exe); only a real exe counts for those.
|
|
38
|
+
// Elsewhere an executable has no extension and only the exact name is trusted.
|
|
39
|
+
const variant = isWin && (base.startsWith(`${bin}-`) || base.startsWith(`${bin}_`)) && base.endsWith('.exe');
|
|
40
|
+
if ((exact || variant) && RUNNABLE.test(e.name)) out.push(path.join(dir, e.name));
|
|
41
|
+
}
|
|
42
|
+
return out;
|
|
43
|
+
}
|
|
44
|
+
|
|
45
|
+
// Directories worth looking in, nothing recursive beyond one level of subfolders for the download-like ones.
|
|
46
|
+
export function searchDirs() {
|
|
47
|
+
const home = os.homedir();
|
|
48
|
+
const dirs = [npmGlobalDir(), ...(process.env.PATH || '').split(path.delimiter)].filter(Boolean);
|
|
49
|
+
for (const sub of ['.local/bin', '.cargo/bin', 'bin', 'tools', 'Downloads', 'Desktop', 'Documents']) dirs.push(path.join(home, sub));
|
|
50
|
+
if (process.env.LOCALAPPDATA) dirs.push(path.join(process.env.LOCALAPPDATA, 'Programs'));
|
|
51
|
+
return dirs;
|
|
52
|
+
}
|
|
53
|
+
|
|
54
|
+
// Absolute paths of candidates for a host, best first (a real .exe before a .cmd shim). `dirs` is injectable.
|
|
55
|
+
export function searchHost(host, dirs = searchDirs()) {
|
|
56
|
+
const spec = HOST_CLI[host];
|
|
57
|
+
if (!spec) return [];
|
|
58
|
+
const seen = new Set();
|
|
59
|
+
const found = [];
|
|
60
|
+
const add = (f) => { if (!seen.has(f.toLowerCase())) { seen.add(f.toLowerCase()); found.push(f); } };
|
|
61
|
+
for (const dir of dirs) {
|
|
62
|
+
inDir(dir, spec.bin).forEach(add);
|
|
63
|
+
// one level down: Downloads\codewhale\codewhale.exe, Programs\codewhale\...
|
|
64
|
+
for (const e of listDir(dir).slice(0, 400)) {
|
|
65
|
+
if (e.isDirectory() && e.name.toLowerCase().includes(spec.bin)) inDir(path.join(dir, e.name), spec.bin).forEach(add);
|
|
66
|
+
}
|
|
67
|
+
}
|
|
68
|
+
const exactName = (f) => { const b = path.basename(f).toLowerCase(); return b === spec.bin || b.startsWith(`${spec.bin}.`); };
|
|
69
|
+
const rank = (f) => (exactName(f) ? 0 : 10) + (/\.exe$/i.test(f) ? 0 : /\.(cmd|bat)$/i.test(f) ? 1 : 2);
|
|
70
|
+
return found.sort((a, b) => rank(a) - rank(b));
|
|
71
|
+
}
|
|
72
|
+
|
|
73
|
+
// Codewhale's package has an install script, and npm only runs install scripts it has been told to allow. The
|
|
74
|
+
// allowance is given for this one command (--allow-scripts) rather than written into the user's npm config.
|
|
75
|
+
const INSTALL_FLAGS = { codewhale: ' --allow-scripts=codewhale' };
|
|
76
|
+
export const installCommand = (host) => (HOST_CLI[host]?.pkg ? `npm install -g ${HOST_CLI[host].pkg}${INSTALL_FLAGS[host] || ''}` : null);
|
|
77
|
+
|
|
78
|
+
// Runs the npm install with its output on the user's terminal. Throws with a short reason on failure.
|
|
79
|
+
export function installHost(host, run = (cmd) => execSync(cmd, { stdio: 'inherit' })) {
|
|
80
|
+
const cmd = installCommand(host);
|
|
81
|
+
if (!cmd) throw new Error('no installer for this host');
|
|
82
|
+
try { run(cmd); } catch (err) { throw new Error(String(err.message || err).split(/\r?\n/)[0]); }
|
|
83
|
+
}
|