skillgauntlet 1.0.0 → 1.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,43 @@
1
+ // Where a skill can already be installed. Claude Code, Codex and the shared .agents convention all
2
+ // read a directory of skill folders, each with a SKILL.md inside.
3
+ import fs from 'node:fs';
4
+ import os from 'node:os';
5
+ import path from 'node:path';
6
+
7
+ // Ordered: earlier roots win when the same skill name appears twice.
8
+ const ROOTS = [
9
+ () => path.join(os.homedir(), '.agents', 'skills'),
10
+ () => path.join(process.cwd(), '.agents', 'skills'),
11
+ () => path.join(os.homedir(), '.claude', 'skills'),
12
+ () => path.join(process.cwd(), '.claude', 'skills'),
13
+ () => path.join(os.homedir(), '.codex', 'skills'),
14
+ () => path.join(process.cwd(), '.codex', 'skills'),
15
+ ];
16
+
17
+ function skillName(dir) {
18
+ try {
19
+ const text = fs.readFileSync(path.join(dir, 'SKILL.md'), 'utf8');
20
+ const m = text.match(/^---\s*\r?\n([\s\S]*?)\r?\n---/);
21
+ const n = m && m[1].match(/^name\s*:\s*(.+)$/m);
22
+ if (n) return n[1].trim().replace(/^["']|["']$/g, '');
23
+ } catch { /* fall through */ }
24
+ return path.basename(dir);
25
+ }
26
+
27
+ export function installedSkills() {
28
+ const seen = new Map();
29
+ for (const rootFn of ROOTS) {
30
+ const root = rootFn();
31
+ let entries = [];
32
+ try { entries = fs.readdirSync(root, { withFileTypes: true }); } catch { continue; }
33
+ for (const e of entries) {
34
+ if (!e.isDirectory() || e.name.startsWith('.')) continue;
35
+ const dir = path.join(root, e.name);
36
+ if (!fs.existsSync(path.join(dir, 'SKILL.md'))) continue;
37
+ const name = skillName(dir);
38
+ if (seen.has(name)) continue;
39
+ seen.set(name, { name, dir, root });
40
+ }
41
+ }
42
+ return [...seen.values()].sort((a, b) => a.name.localeCompare(b.name));
43
+ }
package/lib/queries.js ADDED
@@ -0,0 +1,37 @@
1
+ // Question sets cost one model call to draft, so they are cached and reused while the skill is
2
+ // unchanged. The fingerprint is the SKILL.md hash: edit the description and the cache goes stale,
3
+ // which is exactly when fresh questions are needed.
4
+ import crypto from 'node:crypto';
5
+ import fs from 'node:fs';
6
+ import path from 'node:path';
7
+
8
+ export function fingerprint(skillDir) {
9
+ try {
10
+ return crypto.createHash('sha256').update(fs.readFileSync(path.join(skillDir, 'SKILL.md'))).digest('hex').slice(0, 16);
11
+ } catch { return null; }
12
+ }
13
+
14
+ export function cacheFileFor(skillDir, depth) {
15
+ return path.join(skillDir, '.skillgauntlet', `queries-${depth.perClause}x${depth.negatives}.json`);
16
+ }
17
+
18
+ export function readCachedQueries(file, want) {
19
+ try {
20
+ const j = JSON.parse(fs.readFileSync(file, 'utf8'));
21
+ const same = j.perClause === want.perClause
22
+ && j.negatives === want.negatives
23
+ && j.skillFingerprint === want.skillFingerprint
24
+ && Array.isArray(j.queries) && j.queries.length > 0;
25
+ return same ? j : null;
26
+ } catch { return null; }
27
+ }
28
+
29
+ export function writeCachedQueries(file, payload) {
30
+ try {
31
+ fs.mkdirSync(path.dirname(file), { recursive: true });
32
+ fs.writeFileSync(file, `${JSON.stringify(payload, null, 2)}\n`);
33
+ return true;
34
+ } catch {
35
+ return false; // a read-only skill directory must not break the run
36
+ }
37
+ }
package/lib/report.js ADDED
@@ -0,0 +1,145 @@
1
+ // Rendering. Two artefacts: a machine-readable scorecard and a page a human can read.
2
+ // Every sentence comes from the i18n table, so the report follows the language chosen in the CLI.
3
+ import { t, lang } from './i18n.js';
4
+
5
+ const ICON = { pass: '✓', warn: '!', fail: '✗', info: '·' };
6
+ const pct = (x) => (x == null ? '—' : `${Math.round(x * 100)}%`);
7
+
8
+ export function staticMarkdown(result) {
9
+ const s = result.stats;
10
+ const L = [];
11
+ L.push(`# ${result.skill}`);
12
+ L.push('');
13
+ L.push(`- ${t('report.path')}\`${result.path}\``);
14
+ L.push(`- ${t('report.skillMd', { chars: s.skillMdChars, tokens: s.skillMdTokens, desc: s.descriptionChars })}`);
15
+ L.push(`- ${t('report.package', { files: s.files, code: s.codeFiles, size: s.size })}`);
16
+ L.push(`- ${t('report.permissions', { tools: s.allowedTools || t('report.notDeclared') })}`);
17
+ L.push('');
18
+ L.push(`| ${t('report.groupHeader')} | ${t('report.verdictHeader')} | ${t('report.checkHeader')} |`);
19
+ L.push('|---|---|---|');
20
+ for (const g of result.groups) {
21
+ const bad = g.checks
22
+ .filter((c) => c.status === 'warn' || c.status === 'fail')
23
+ .map((c) => `${ICON[c.status]} ${t(c.titleKey)}`).join('; ');
24
+ L.push(`| ${g.id} ${t(g.titleKey)} | ${g.verdict} | ${bad || t('report.allPass')} |`);
25
+ }
26
+ L.push('');
27
+ L.push(t('report.legend'));
28
+ L.push('');
29
+ for (const g of result.groups) {
30
+ L.push(`## ${g.id} ${t(g.titleKey)}`);
31
+ L.push('');
32
+ for (const c of g.checks) {
33
+ L.push(`- ${ICON[c.status]} **${c.id} ${t(c.titleKey)}** — ${t(c.detailKey, c.detailVars)}`);
34
+ for (const e of c.evidence.slice(0, 3)) L.push(` - \`L${e.line}\` ${e.match}`);
35
+ }
36
+ L.push('');
37
+ }
38
+ L.push(t('report.noComposite'));
39
+ return L.join('\n');
40
+ }
41
+
42
+ export function trialSection(trial) {
43
+ if (!trial) return '';
44
+ const L = ['', `## ${t('trial.title')}`, ''];
45
+ L.push(`- ${t('trial.modelLine', {
46
+ model: trial.modelResolved || trial.model,
47
+ backend: trial.backend,
48
+ host: trial.host || t('trial.hostUnknown'),
49
+ runs: trial.runs,
50
+ trials: trial.trials,
51
+ })}`);
52
+ L.push(`- ${t('trial.recall', { p: pct(trial.recall), lo: pct(trial.recallCi?.[0]), hi: pct(trial.recallCi?.[1]) })}`);
53
+ L.push(`- ${t('trial.fpr', { p: pct(trial.fpr), lo: pct(trial.fprCi?.[0]), hi: pct(trial.fprCi?.[1]) })}`);
54
+ if (trial.noToolRate != null) {
55
+ L.push(`- ${t('trial.noTool', { p: pct(trial.noToolRate) })}${trial.noToolRate >= 0.5 ? t('trial.noToolWarn') : ''}`);
56
+ }
57
+ L.push('');
58
+ L.push(`| ${t('trial.stateHeader')} | ${t('trial.queryHeader')} | ${t('trial.hitsHeader')} |`);
59
+ L.push('|---|---|---|');
60
+ for (const q of trial.questions) {
61
+ L.push(`| ${q.shouldTrigger ? t('trial.should') : t('trial.shouldNot')} | ${q.query.replace(/\|/g, '\\|')} | ${q.hits}/${q.n} |`);
62
+ }
63
+ return L.join('\n');
64
+ }
65
+
66
+ export function gainSection(gain) {
67
+ if (!gain) return '';
68
+ const L = ['', `## ${t('gain.title')}`, ''];
69
+ L.push(t('gain.intro'));
70
+ L.push(t('gain.local'));
71
+ L.push('');
72
+ L.push(`| ${t('gain.contractHeader')} | ${t('gain.withHeader')} | ${t('gain.withoutHeader')} | ${t('gain.dirHeader')} |`);
73
+ L.push('|---|---|---|---|');
74
+ for (const c of gain.contracts) {
75
+ const delta = c.with - c.without;
76
+ const dir = delta > 0.1 ? t('gain.up') : delta < -0.1 ? t('gain.down') : t('gain.same');
77
+ const desc = c.descI18n?.[lang()] || c.desc;
78
+ L.push(`| ${desc} | ${pct(c.with)} | ${pct(c.without)} | ${dir} |`);
79
+ }
80
+ if (gain.tasks?.length) {
81
+ L.push('');
82
+ L.push(`### ${t('gain.perTask')}`);
83
+ for (const task of gain.tasks) {
84
+ L.push('');
85
+ L.push(`**${task.id}** ${task.prompt.replace(/\n/g, ' ').slice(0, 120)}`);
86
+ for (const [label, side] of [[t('gain.withSkill'), task.with], [t('gain.withoutSkill'), task.without]]) {
87
+ const files = side.added.length ? side.added.slice(0, 6).join(', ') : t('gain.noFiles');
88
+ const ok = side.checks.filter((c) => c.ok).length;
89
+ L.push(`- **${label}**: ${t('gain.side', { files, ok, n: side.checks.length })}`);
90
+ }
91
+ }
92
+ }
93
+ L.push('');
94
+ L.push(t('gain.footnote1'));
95
+ L.push(t('gain.footnote2'));
96
+ return L.join('\n');
97
+ }
98
+
99
+ export function probeSection(probe) {
100
+ if (!probe) return '';
101
+ const same = (b) => (b ? t('probeSame') : t('probeDiffer'));
102
+ const L = ['', `## ${t('probeTitle')}`, ''];
103
+ L.push(t('probeIntro'));
104
+ L.push('');
105
+ L.push(`| | ${t('probeAction')} | ${t('probeFiles')} |`);
106
+ L.push('|---|---|---|');
107
+ for (const [label, side] of [[t('probeWith'), probe.withSkill], [t('probeWithout'), probe.without]]) {
108
+ const action = side.firstAction ? `${side.firstAction.kind} — ${side.firstAction.detail}`.slice(0, 160) : '—';
109
+ L.push(`| ${label} | ${action} | ${side.createdFiles?.join(', ') || '—'} |`);
110
+ }
111
+ L.push('');
112
+ L.push(`- ${t('probeActionKind')}${t('report.colon')}${same(probe.actionKindIdentical)}`);
113
+ L.push(`- ${t('probeArtifacts')}${t('report.colon')}${same(probe.artifactsIdentical)}`);
114
+ L.push('');
115
+ L.push(`> ${t('probeNoVerdict')}`);
116
+ L.push(`> ${t('probeNoTrigger')}`);
117
+ return L.join('\n');
118
+ }
119
+
120
+ export function scorecard(result) {
121
+ return {
122
+ schema: 'skillgauntlet/scorecard/2',
123
+ skill: result.skill,
124
+ path: result.path,
125
+ stats: result.stats,
126
+ groups: result.groups.map((g) => ({
127
+ id: g.id,
128
+ titleKey: g.titleKey,
129
+ verdict: g.verdict,
130
+ checks: g.checks.map((c) => ({
131
+ id: c.id, status: c.status, detailKey: c.detailKey, detailVars: c.detailVars,
132
+ })),
133
+ })),
134
+ };
135
+ }
136
+
137
+ // A published report ends with one question: which host does the reader use. It is a poll, not a
138
+ // feature -- one line, answered with a digit -- and it exists because a share count says how many
139
+ // people read a report but not whether its trigger numbers apply to them. Codex has no skill item in
140
+ // its event stream, so a Codex reader gets nothing from a trigger table; finding out that the readers
141
+ // are mostly Codex users is cheap now and expensive to discover after building the wrong layer.
142
+ // Deliberately absent from per-skill reports: the person reading one of those usually ran it.
143
+ export function readerPollSection() {
144
+ return ['', '---', '', `## ${t('poll.title')}`, '', t('poll.body'), '', `> ${t('poll.why')}`].join('\n');
145
+ }
package/lib/runs.js ADDED
@@ -0,0 +1,167 @@
1
+ // Trial records are the only artefact that cannot be regenerated cheaply.
2
+ //
3
+ // The report is a rendering of them, a scorecard is a rendering of the directory, but the 110
4
+ // records behind a trigger number cost ten minutes and real money. Until now they lived in memory
5
+ // and died with the process, so nothing could be recomputed later, two runs could not be compared,
6
+ // and a community submission had no file to submit.
7
+ //
8
+ // The shape copies the Python runner deliberately: meta.json first, then one JSON line per finished
9
+ // trial, then summary.json. A run killed by Ctrl-C, a quota limit or a closed laptop keeps
10
+ // everything it already finished, and a reader can tell from the files exactly what conditions
11
+ // produced them.
12
+ import fs from 'node:fs';
13
+ import os from 'node:os';
14
+ import path from 'node:path';
15
+ import { createHash } from 'node:crypto';
16
+ import { fileURLToPath } from 'node:url';
17
+
18
+ const here = path.dirname(fileURLToPath(import.meta.url));
19
+ const RUNNER_VERSION = JSON.parse(fs.readFileSync(path.join(here, '..', 'package.json'), 'utf8')).version;
20
+
21
+ export const RUN_SCHEMA = 'skillgauntlet/run/1';
22
+ export { RUNNER_VERSION };
23
+
24
+ export function runsRoot(skillDir) {
25
+ return path.join(skillDir, '.skillgauntlet', 'runs');
26
+ }
27
+
28
+ function stamp(d = new Date()) {
29
+ const p = (n) => String(n).padStart(2, '0');
30
+ return `${d.getFullYear()}${p(d.getMonth() + 1)}${p(d.getDate())}-`
31
+ + `${p(d.getHours())}${p(d.getMinutes())}${p(d.getSeconds())}`;
32
+ }
33
+
34
+ // rename over an existing file can fail on Windows, so fall back to remove-then-rename.
35
+ function writeAtomic(file, text) {
36
+ const tmp = `${file}.tmp`;
37
+ fs.writeFileSync(tmp, text);
38
+ try {
39
+ fs.renameSync(tmp, file);
40
+ } catch {
41
+ fs.rmSync(file, { force: true });
42
+ fs.renameSync(tmp, file);
43
+ }
44
+ }
45
+
46
+ export function querySetHash(queries) {
47
+ return createHash('sha256').update(JSON.stringify(queries)).digest('hex').slice(0, 16);
48
+ }
49
+
50
+ // A second run started within the same second would otherwise reuse the first one's directory and
51
+ // overwrite its records without a word. Fast failures and a re-run from a script do exactly that.
52
+ function uniqueDir(root, base) {
53
+ let id = base;
54
+ let n = 1;
55
+ while (fs.existsSync(path.join(root, id))) {
56
+ n += 1;
57
+ id = `${base}-${n}`;
58
+ }
59
+ return id;
60
+ }
61
+
62
+ // Open a run before the first trial, so an interrupted run still has its conditions on disk.
63
+ export function openRun({ skillDir, skillName, host, backend, model, runs, maxExplore, depth, querySets, queries, skillFingerprint }) {
64
+ const root = runsRoot(skillDir);
65
+ fs.mkdirSync(root, { recursive: true });
66
+ const runId = uniqueDir(root, stamp());
67
+ const dir = path.join(root, runId);
68
+ fs.mkdirSync(dir);
69
+
70
+ const meta = {
71
+ schema: RUN_SCHEMA,
72
+ runId,
73
+ runnerVersion: RUNNER_VERSION,
74
+ skill: skillName,
75
+ host,
76
+ backend,
77
+ model,
78
+ runs,
79
+ maxExplore,
80
+ depth,
81
+ querySets,
82
+ skillFingerprint: skillFingerprint || null,
83
+ platform: `${os.platform()} ${os.release()} / Node ${process.version}`,
84
+ startedAt: new Date().toISOString(),
85
+ finishedAt: null,
86
+ trials: 0,
87
+ interrupted: false,
88
+ };
89
+ const metaFile = path.join(dir, 'meta.json');
90
+ const rawFile = path.join(dir, 'raw.jsonl');
91
+ writeAtomic(metaFile, `${JSON.stringify(meta, null, 2)}\n`);
92
+ fs.writeFileSync(rawFile, '');
93
+ // The questions themselves, so a resume works from exactly what this run asked and not from a cache
94
+ // that may have been redrafted since. Its hash is meta.querySets[0].sha256.
95
+ if (queries) writeAtomic(path.join(dir, 'queries.json'), `${JSON.stringify(queries, null, 2)}\n`);
96
+ return handle(dir, meta);
97
+ }
98
+
99
+ // Shared by a new run and a resumed one: appending, refreshing and closing behave the same.
100
+ function handle(dir, meta) {
101
+ const metaFile = path.join(dir, 'meta.json');
102
+ const rawFile = path.join(dir, 'raw.jsonl');
103
+ const flushMeta = () => writeAtomic(metaFile, `${JSON.stringify(meta, null, 2)}\n`);
104
+
105
+ return {
106
+ dir,
107
+ meta,
108
+ // Merge run-level facts learned mid-run (the tool list from the first init event, a stop reason).
109
+ patch(fields) {
110
+ Object.assign(meta, fields);
111
+ flushMeta();
112
+ },
113
+ get file() { return rawFile; },
114
+ // Called from the worker pool: appending is synchronous, so two in-flight trials cannot interleave
115
+ // a half line. Refreshing meta every trial would be wasteful; every 25 matches the Python runner.
116
+ append(row) {
117
+ fs.appendFileSync(rawFile, `${JSON.stringify(row)}\n`);
118
+ meta.trials += 1;
119
+ if (meta.trials % 25 === 0) flushMeta();
120
+ },
121
+ close(summary, { interrupted = false } = {}) {
122
+ meta.finishedAt = new Date().toISOString();
123
+ meta.interrupted = interrupted;
124
+ flushMeta();
125
+ if (summary) writeAtomic(path.join(dir, 'summary.json'), `${JSON.stringify(summary, null, 2)}\n`);
126
+ return dir;
127
+ },
128
+ };
129
+ }
130
+
131
+ // Open an existing run directory to continue it. The caller has already checked that the conditions
132
+ // match (see resumeRun in execute.js); this only reopens the files and notes the resume.
133
+ export function reopenRun(dir) {
134
+ const { meta, records } = readRun(dir);
135
+ meta.trials = records.length;
136
+ meta.finishedAt = null;
137
+ meta.interrupted = false;
138
+ delete meta.stoppedAt;
139
+ delete meta.stoppedReason;
140
+ meta.resumedAt = [...(meta.resumedAt || []), new Date().toISOString()];
141
+ fs.rmSync(path.join(dir, 'summary.json'), { force: true });
142
+ const h = handle(dir, meta);
143
+ h.patch({});
144
+ return h;
145
+ }
146
+
147
+ // One valid record per (query, run index): the latest that is not an error or timeout. Failed trials
148
+ // stay in raw.jsonl as evidence but are retried on resume and never counted.
149
+ export function validRecords(records) {
150
+ const byKey = new Map();
151
+ for (const r of records) {
152
+ if (r.verdict === 'error' || r.verdict === 'timeout') continue;
153
+ byKey.set(`${r.queryId}#${r.run}`, r);
154
+ }
155
+ return [...byKey.values()];
156
+ }
157
+
158
+ // Rebuilding a report from a saved run is the reason the records are kept: the statistics can be
159
+ // corrected without spending another token.
160
+ export function readRun(dir) {
161
+ const meta = JSON.parse(fs.readFileSync(path.join(dir, 'meta.json'), 'utf8'));
162
+ const raw = fs.readFileSync(path.join(dir, 'raw.jsonl'), 'utf8')
163
+ .split(/\r?\n/).filter(Boolean).map((line) => JSON.parse(line));
164
+ let summary = null;
165
+ try { summary = JSON.parse(fs.readFileSync(path.join(dir, 'summary.json'), 'utf8')); } catch { /* unfinished */ }
166
+ return { meta, records: raw, summary };
167
+ }
package/lib/spawn.js ADDED
@@ -0,0 +1,84 @@
1
+ // Launching a CLI on Windows. Three traps, all of which produce confusing errors rather than loud
2
+ // ones: `where` returns the extension-less shim first (Node cannot exec it), `shell: true` re-splits
3
+ // arguments on spaces (a prompt becomes several arguments), and cmd.exe treats & | < > ^ % specially.
4
+ //
5
+ // The way out is to not use a shell: run the package's JS entry with the current node binary. Only
6
+ // when that cannot be found do we fall back to the .cmd shim, and then every argument is quoted.
7
+ import fs from 'node:fs';
8
+ import path from 'node:path';
9
+
10
+ function looksLikeJs(p) {
11
+ return /\.(c|m)?js$/i.test(p);
12
+ }
13
+
14
+ // An npm .cmd shim names the script it runs (`"%_prog%" "%dp0%\<relative path>" %*`). Reading it is exact;
15
+ // guessing from the package layout can land on a dependency's index.js instead of the CLI.
16
+ function entryFromShim(bin) {
17
+ if (!/\.cmd$/i.test(bin)) return null;
18
+ let text;
19
+ try { text = fs.readFileSync(bin, 'utf8'); } catch { return null; }
20
+ const hits = [...text.matchAll(/"%_prog%"\s+"%dp0%\\([^"]+)"/g)];
21
+ if (!hits.length) return null;
22
+ const target = path.join(path.dirname(bin), ...hits[hits.length - 1][1].split('\\'));
23
+ try { return fs.statSync(target).isFile() ? target : null; } catch { return null; }
24
+ }
25
+
26
+ // Global npm layouts put the entry at node_modules/<pkg>/bin/<name>.js or
27
+ // node_modules/@scope/<pkg>/<entry>.js. Look a bounded distance, in that order.
28
+ function findJsEntry(bin, name) {
29
+ const fromShim = entryFromShim(bin);
30
+ if (fromShim) return fromShim;
31
+ const root = path.dirname(bin);
32
+ // A project-local install keeps its shims in node_modules/.bin, so the packages are one level up.
33
+ const nm = path.basename(root) === '.bin' ? path.dirname(root) : path.join(root, 'node_modules');
34
+ const candidates = [
35
+ path.join(nm, name, 'bin', `${name}.js`),
36
+ path.join(nm, name, `${name}.js`),
37
+ path.join(nm, name, 'cli.js'),
38
+ path.join(nm, name, 'index.js'),
39
+ ];
40
+ let entries = [];
41
+ try { entries = fs.readdirSync(nm, { withFileTypes: true }); } catch { return null; }
42
+ for (const e of entries) {
43
+ if (!e.isDirectory()) continue;
44
+ if (e.name.startsWith('@')) {
45
+ let scoped = [];
46
+ try { scoped = fs.readdirSync(path.join(nm, e.name), { withFileTypes: true }); } catch { continue; }
47
+ for (const s of scoped) {
48
+ if (!s.isDirectory()) continue;
49
+ candidates.push(path.join(nm, e.name, s.name, 'bin', `${name}.js`));
50
+ candidates.push(path.join(nm, e.name, s.name, 'bin', name));
51
+ candidates.push(path.join(nm, e.name, s.name, `${name}.js`));
52
+ candidates.push(path.join(nm, e.name, s.name, 'cli.js'));
53
+ candidates.push(path.join(nm, e.name, s.name, 'index.js'));
54
+ }
55
+ }
56
+ }
57
+ return candidates.find((p) => { try { return fs.statSync(p).isFile(); } catch { return false; } });
58
+ }
59
+
60
+ export function runnerFor(bin, name = path.basename(bin).replace(/\.(cmd|bat|exe)$/i, '')) {
61
+ if (looksLikeJs(bin)) return { cmd: process.execPath, prefix: [bin], shell: false };
62
+ if (/\.exe$/i.test(bin)) return { cmd: bin, prefix: [], shell: false };
63
+ const entry = findJsEntry(bin, name);
64
+ if (entry) return { cmd: process.execPath, prefix: [entry], shell: false };
65
+ return { cmd: bin, prefix: [], shell: true };
66
+ }
67
+
68
+ // Only used on the shell fallback. Wraps each argument and neutralises cmd.exe metacharacters.
69
+ export function quoteForCmd(arg) {
70
+ const s = String(arg);
71
+ if (!/[\s"&|<>^%()]/.test(s)) return s;
72
+ return `"${s.replace(/"/g, '""')}"`;
73
+ }
74
+
75
+ // Command, argument list and shell flag for one launch. Callers that used `shell: /\.(cmd|bat)$/` handed
76
+ // the prompt to cmd.exe, which re-splits it on spaces; this routes every launch through runnerFor instead.
77
+ export function launchSpec(bin, argv, name) {
78
+ const runner = runnerFor(bin, name);
79
+ return {
80
+ cmd: runner.cmd,
81
+ args: runner.shell ? [...runner.prefix, ...argv].map(quoteForCmd) : [...runner.prefix, ...argv],
82
+ shell: runner.shell,
83
+ };
84
+ }
package/lib/static.js ADDED
@@ -0,0 +1,187 @@
1
+ // Layer 0: static analysis. No model calls, no cost, reproducible: the same directory always
2
+ // yields the same result. Reports facts and flags, never a single composite score -- a weighted
3
+ // "87/100" cannot be audited or appealed, which is the one thing a rating must allow.
4
+ //
5
+ // Results carry message keys, not prose, so a scorecard is language-neutral and the report decides
6
+ // how to word it.
7
+ import fs from 'node:fs';
8
+ import path from 'node:path';
9
+ import { readFrontmatter } from './discover.js';
10
+
11
+ const CODE_EXT = new Set(['.py', '.sh', '.bash', '.zsh', '.js', '.mjs', '.cjs', '.ts', '.ps1', '.rb', '.pl', '.exe', '.bat', '.cmd']);
12
+ const SKIP_DIRS = new Set(['.git', 'node_modules', '__pycache__', '.cache', 'vendor', 'dist', 'build']);
13
+
14
+ const RE = {
15
+ network: /\b(curl|wget|nc\s|ncat|requests\.(get|post|put)|urllib\.request|httpx\.|axios|fetch\s*\(|XMLHttpRequest|Invoke-WebRequest)\b/gi,
16
+ exec: /\b(os\.system|os\.popen|subprocess\.(run|call|Popen|check_output)|child_process|execSync|spawnSync|eval\s*\(|exec\s*\(|Invoke-Expression)\b/g,
17
+ secrets: /(ANTHROPIC_API_KEY|OPENAI_API_KEY|DEEPSEEK_API_KEY|AWS_SECRET|GITHUB_TOKEN|NPM_TOKEN|process\.env|os\.environ|~\/\.ssh|\/\.ssh\/|id_rsa|\.aws\/credentials|\.npmrc|\.netrc)/g,
18
+ obfusc: /(base64\s+-d|atob\s*\(|fromCharCode|\\x[0-9a-fA-F]{2}\\x[0-9a-fA-F]{2}\\x[0-9a-fA-F]{2}|eval\(Buffer\.from)/g,
19
+ install: /(postinstall|preinstall|prepare"\s*:)/g,
20
+ hijack: /(ignore\s+(all\s+)?(previous|prior|above)\s+instructions|disregard\s+.{0,20}instructions|forget\s+(all\s+)?(previous|prior)|忽略(之前|以上|前面)的?(所有)?(指令|说明)|无视(之前|以上))/gi,
21
+ exfil: /(send|post|upload|exfiltrat|上传|发送|回传).{0,40}(https?:\/\/|webhook|\burl\b)/gi,
22
+ wide: /(\bany\b|\banything\b|\ball\s+tasks\b|\beverything\b|任何|所有任务|各种)/gi,
23
+ triggerWords: /\b(use when|use this when|use for|should be used|designed for|trigger|when)\b|当用户|使用时机|适用于|用于/gi,
24
+ extImage: /!\[[^\]]*\]\((https?:\/\/[^)\s]+)\)/g,
25
+ zeroWidth: /[\u200B-\u200F\u202A-\u202E\u2060\uFEFF]/g,
26
+ };
27
+
28
+ function lineOf(text, index) {
29
+ return text.slice(0, index).split(/\r?\n/).length;
30
+ }
31
+
32
+ function hits(text, re, limit = 5) {
33
+ const out = [];
34
+ re.lastIndex = 0;
35
+ let m;
36
+ while ((m = re.exec(text)) && out.length < limit) {
37
+ out.push({ line: lineOf(text, m.index), match: m[0].trim().slice(0, 60) });
38
+ }
39
+ return out;
40
+ }
41
+
42
+ export function estTokens(text) {
43
+ let cjk = 0;
44
+ let other = 0;
45
+ for (const ch of text) {
46
+ if (/[\u3400-\u9FFF\u3040-\u30FF\uAC00-\uD7AF]/.test(ch)) cjk += 1;
47
+ else other += 1;
48
+ }
49
+ return Math.round(cjk * 1 + other / 4);
50
+ }
51
+
52
+ function collect(skillDir) {
53
+ const files = [];
54
+ const walk = (dir, depth) => {
55
+ if (depth > 6 || files.length > 3000) return;
56
+ let entries;
57
+ try { entries = fs.readdirSync(dir, { withFileTypes: true }); } catch { return; }
58
+ for (const e of entries) {
59
+ const p = path.join(dir, e.name);
60
+ if (e.isDirectory()) {
61
+ if (!SKIP_DIRS.has(e.name)) walk(p, depth + 1);
62
+ } else if (e.isFile()) {
63
+ let size = 0;
64
+ try { size = fs.statSync(p).size; } catch { /* ignore */ }
65
+ files.push({ path: p, rel: path.relative(skillDir, p).replace(/\\/g, '/'), ext: path.extname(e.name).toLowerCase(), size });
66
+ }
67
+ }
68
+ };
69
+ walk(skillDir, 0);
70
+ return files;
71
+ }
72
+
73
+ // A check carries its message keys and the variables to fill them.
74
+ const check = (id, status, detailKey, detailVars, evidence) => ({
75
+ id,
76
+ titleKey: `check.${id}.title`,
77
+ status,
78
+ detailKey,
79
+ detailVars: detailVars || {},
80
+ evidence: evidence || [],
81
+ });
82
+
83
+ const total = (list) => list.reduce((a, c) => a + (c.size || 0), 0);
84
+ const humanSize = (n) => (n >= 1048576 ? `${(n / 1048576).toFixed(2)} MB` : `${(n / 1024).toFixed(1)} KB`);
85
+
86
+ export function analyze(skillDir) {
87
+ const dir = path.resolve(skillDir);
88
+ const { fm, text } = readFrontmatter(dir);
89
+ const files = collect(dir);
90
+ const codeFiles = files.filter((f) => CODE_EXT.has(f.ext));
91
+ const codeText = codeFiles.map((f) => {
92
+ try { return fs.readFileSync(f.path, 'utf8'); } catch { return ''; }
93
+ }).join('\n');
94
+ const body = text.replace(/^---[\s\S]*?---/, '');
95
+ const totalBytes = total(files);
96
+ const size = humanSize(totalBytes);
97
+ const tokens = estTokens(text);
98
+ const desc = fm.description || '';
99
+ const allowed = (fm['allowed-tools'] || fm.allowed_tools || '').trim();
100
+ const dmi = String(fm['disable-model-invocation'] || '').toLowerCase() === 'true';
101
+
102
+ // A. context cost -- what the skill adds to the window, and how much of it is always on
103
+ const A = [];
104
+ A.push(check('A1', tokens > 5000 ? 'fail' : tokens > 2000 ? 'warn' : 'pass',
105
+ tokens > 2000 ? 'check.A1.heavy' : 'check.A1.detail', { chars: text.length, tokens }));
106
+ A.push(check('A2', desc.length > 800 ? 'warn' : 'pass',
107
+ desc ? 'check.A2.detail' : 'check.A2.missing', { chars: desc.length }));
108
+ const refs = (body.match(/\b(references|reference|assets|scripts)\/[\w./-]+/g) || []).length;
109
+ A.push(check('A3', refs > 0 ? 'pass' : 'info', refs > 0 ? 'check.A3.detail' : 'check.A3.none', { n: refs }));
110
+ A.push(check('A4', totalBytes > 2 * 1048576 ? 'warn' : 'pass', 'check.A4.detail', { size, files: files.length }));
111
+
112
+ // B. permission surface -- what the host has to grant it
113
+ const B = [];
114
+ B.push(check('B1', allowed ? 'pass' : (codeFiles.length ? 'fail' : 'warn'),
115
+ allowed ? 'check.B1.declared' : 'check.B1.missing', { tools: allowed.slice(0, 120) }));
116
+ if (allowed) {
117
+ const wide = /Bash\(\s*\*?\s*\)|Bash\s*(,|$)|Write\(\*\*\)|Edit\(\*\*\)/.test(allowed);
118
+ B.push(check('B2', wide ? 'fail' : 'pass', wide ? 'check.B2.wide' : 'check.B2.narrow'));
119
+ }
120
+ B.push(check('B3', dmi ? 'info' : 'pass', dmi ? 'check.B3.manual' : 'check.B3.auto'));
121
+ B.push(check('B4', codeFiles.length ? 'info' : 'pass',
122
+ codeFiles.length ? 'check.B4.detail' : 'check.B4.none',
123
+ { n: codeFiles.length, list: codeFiles.slice(0, 6).map((f) => f.rel).join(', ') }));
124
+
125
+ // C. code risk -- facts, escalated only when the permission surface is undeclared
126
+ const undeclared = !allowed && codeFiles.length > 0;
127
+ const C = [];
128
+ for (const [id, re] of [['C1', RE.network], ['C2', RE.exec], ['C3', RE.secrets], ['C4', RE.obfusc], ['C5', RE.install]]) {
129
+ const h = hits(codeText, re);
130
+ const detailKey = h.length ? (undeclared ? 'check.C.escalated' : 'check.C.hits') : 'check.C.none';
131
+ C.push(check(id, h.length ? (undeclared ? 'warn' : 'info') : 'pass', detailKey, { n: h.length }, h));
132
+ }
133
+
134
+ // D. instruction hijacking -- aimed at the reader, not the shell
135
+ const D = [];
136
+ const dh = hits(text, RE.hijack);
137
+ D.push(check('D1', dh.length ? 'fail' : 'pass', dh.length ? 'check.D1.hit' : 'check.D.none', {}, dh));
138
+ const sec = hits(text, RE.secrets);
139
+ D.push(check('D2', sec.length ? 'warn' : 'pass', sec.length ? 'check.D2.hit' : 'check.D.none', {}, sec));
140
+ const ex = hits(text, RE.exfil);
141
+ D.push(check('D3', ex.length ? 'warn' : 'pass', ex.length ? 'check.D3.hit' : 'check.D.none', {}, ex));
142
+ const zw = hits(text, RE.zeroWidth);
143
+ D.push(check('D4', zw.length ? 'fail' : 'pass', zw.length ? 'check.D4.hit' : 'check.D.none', {}, zw));
144
+ const img = hits(text, RE.extImage);
145
+ D.push(check('D5', img.length ? 'warn' : 'pass', img.length ? 'check.D5.hit' : 'check.D.none', {}, img));
146
+
147
+ // E. description quality -- the only layer that is also the trigger layer's input
148
+ const E = [];
149
+ E.push(check('E1', desc ? 'pass' : 'fail', desc ? 'check.E1.detail' : 'check.E1.missing', { chars: desc.length }));
150
+ const tw = hits(desc, RE.triggerWords, 1);
151
+ E.push(check('E2', tw.length ? 'pass' : 'warn', tw.length ? 'check.E2.hit' : 'check.E2.miss'));
152
+ const wide = hits(desc, RE.wide, 1);
153
+ E.push(check('E3', wide.length ? 'warn' : 'pass', wide.length ? 'check.E3.hit' : 'check.E3.none', {}, wide));
154
+ const clauses = (desc.match(/\bor\b|、|,|,|\//g) || []).length + 1;
155
+ E.push(check('E4', 'info', 'check.E4.detail', { n: clauses }));
156
+ const dirName = path.basename(dir);
157
+ E.push(check('E5', fm.name && fm.name !== dirName ? 'warn' : 'pass',
158
+ fm.name ? 'check.E5.detail' : 'check.E5.noname', { name: fm.name || '', dir: dirName }));
159
+
160
+ const groups = [
161
+ { id: 'A', titleKey: 'group.A', checks: A },
162
+ { id: 'B', titleKey: 'group.B', checks: B },
163
+ { id: 'C', titleKey: 'group.C', checks: C },
164
+ { id: 'D', titleKey: 'group.D', checks: D },
165
+ { id: 'E', titleKey: 'group.E', checks: E },
166
+ ];
167
+ for (const g of groups) {
168
+ g.verdict = g.checks.some((c) => c.status === 'fail') ? 'fail'
169
+ : g.checks.some((c) => c.status === 'warn') ? 'warn' : 'pass';
170
+ }
171
+ return {
172
+ skill: fm.name || path.basename(dir),
173
+ path: dir,
174
+ groups,
175
+ stats: {
176
+ files: files.length,
177
+ codeFiles: codeFiles.length,
178
+ bytes: totalBytes,
179
+ size,
180
+ skillMdChars: text.length,
181
+ skillMdTokens: tokens,
182
+ descriptionChars: desc.length,
183
+ allowedTools: allowed || null,
184
+ disableModelInvocation: dmi,
185
+ },
186
+ };
187
+ }