skilldiff 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +40 -0
- package/LICENSE +21 -0
- package/README.md +158 -0
- package/dist/scripts/adapters.d.ts +14 -0
- package/dist/scripts/adapters.js +63 -0
- package/dist/scripts/freebuff-adapter.d.ts +24 -0
- package/dist/scripts/freebuff-adapter.js +94 -0
- package/dist/scripts/freebuff-spike.d.ts +1 -0
- package/dist/scripts/freebuff-spike.js +72 -0
- package/dist/scripts/recorded-spike.d.ts +1 -0
- package/dist/scripts/recorded-spike.js +48 -0
- package/dist/scripts/spike.d.ts +1 -0
- package/dist/scripts/spike.js +176 -0
- package/dist/src/assertions.d.ts +33 -0
- package/dist/src/assertions.js +101 -0
- package/dist/src/baseline.d.ts +14 -0
- package/dist/src/baseline.js +58 -0
- package/dist/src/cli.d.ts +2 -0
- package/dist/src/cli.js +69 -0
- package/dist/src/discover.d.ts +22 -0
- package/dist/src/discover.js +104 -0
- package/dist/src/init.d.ts +9 -0
- package/dist/src/init.js +66 -0
- package/dist/src/report.d.ts +7 -0
- package/dist/src/report.js +52 -0
- package/dist/src/runner.d.ts +29 -0
- package/dist/src/runner.js +118 -0
- package/dist/src/scenario.d.ts +36 -0
- package/dist/src/scenario.js +71 -0
- package/dist/src/trace-utils.d.ts +27 -0
- package/dist/src/trace-utils.js +54 -0
- package/package.json +56 -0
- package/scripts/adapters.ts +77 -0
- package/scripts/freebuff-adapter.ts +109 -0
- package/scripts/freebuff-spike.ts +86 -0
- package/scripts/recorded-spike.ts +54 -0
- package/scripts/spike.ts +195 -0
|
@@ -0,0 +1,176 @@
|
|
|
1
|
+
// T1 SPIKE — go/no-go for skilldiff (harness-adapter edition)
|
|
2
|
+
//
|
|
3
|
+
// Verifies the two feasibility gates from the design doc before any real code:
|
|
4
|
+
// 1. An installed harness CLI (claude / cursor-agent / codex), run headless,
|
|
5
|
+
// emits a parseable JSON event stream with tool-use events from a fixture repo.
|
|
6
|
+
// 2. `git show <base>:<skill path>` fetches the old skill version (baseline).
|
|
7
|
+
//
|
|
8
|
+
// Run: npm run spike [path-to-fixture]
|
|
9
|
+
// Requires: at least one harness CLI installed + logged in (its own subscription
|
|
10
|
+
// is fine — no ANTHROPIC_API_KEY needed).
|
|
11
|
+
import { spawn } from "node:child_process";
|
|
12
|
+
import { mkdtemp, mkdir, writeFile } from "node:fs/promises";
|
|
13
|
+
import { tmpdir } from "node:os";
|
|
14
|
+
import { join, resolve } from "node:path";
|
|
15
|
+
import { execFileSync } from "node:child_process";
|
|
16
|
+
import { ADAPTERS } from "./adapters.js";
|
|
17
|
+
function which(cmd) {
|
|
18
|
+
try {
|
|
19
|
+
return execFileSync("which", [cmd], { encoding: "utf8" }).trim() || null;
|
|
20
|
+
}
|
|
21
|
+
catch {
|
|
22
|
+
return null;
|
|
23
|
+
}
|
|
24
|
+
}
|
|
25
|
+
function pickAdapter() {
|
|
26
|
+
const forEnv = process.env.SKILLDIFF_HARNESS;
|
|
27
|
+
if (forEnv) {
|
|
28
|
+
const found = ADAPTERS.find((a) => a.name === forEnv);
|
|
29
|
+
if (!found) {
|
|
30
|
+
console.error(`SKILLDIFF_HARNESS=${forEnv} is not one of: ${ADAPTERS.map((a) => a.name).join(", ")}`);
|
|
31
|
+
process.exit(2);
|
|
32
|
+
}
|
|
33
|
+
if (!which(found.cmd)) {
|
|
34
|
+
console.error(`${found.cmd} not found in PATH`);
|
|
35
|
+
process.exit(2);
|
|
36
|
+
}
|
|
37
|
+
return found;
|
|
38
|
+
}
|
|
39
|
+
for (const a of ADAPTERS) {
|
|
40
|
+
if (which(a.cmd))
|
|
41
|
+
return a;
|
|
42
|
+
}
|
|
43
|
+
console.error("No harness CLI found. Install one of: claude, cursor-agent, codex");
|
|
44
|
+
process.exit(2);
|
|
45
|
+
}
|
|
46
|
+
function runHarness(adapter, cwd, prompt) {
|
|
47
|
+
return new Promise((resolvePromise) => {
|
|
48
|
+
const child = spawn(adapter.cmd, [...adapter.baseArgs, prompt], { cwd, stdio: ["ignore", "pipe", "pipe"] });
|
|
49
|
+
const traces = [];
|
|
50
|
+
let stderr = "";
|
|
51
|
+
let buf = "";
|
|
52
|
+
child.stdout.on("data", (chunk) => {
|
|
53
|
+
buf += chunk.toString();
|
|
54
|
+
let idx;
|
|
55
|
+
while ((idx = buf.indexOf("\n")) >= 0) {
|
|
56
|
+
const line = buf.slice(0, idx).trim();
|
|
57
|
+
buf = buf.slice(idx + 1);
|
|
58
|
+
if (!line)
|
|
59
|
+
continue;
|
|
60
|
+
let parsed;
|
|
61
|
+
try {
|
|
62
|
+
parsed = JSON.parse(line);
|
|
63
|
+
}
|
|
64
|
+
catch {
|
|
65
|
+
continue; // non-JSON noise lines are expected from some harnesses
|
|
66
|
+
}
|
|
67
|
+
traces.push(...adapter.extractToolUses(parsed));
|
|
68
|
+
}
|
|
69
|
+
});
|
|
70
|
+
child.stderr.on("data", (chunk) => {
|
|
71
|
+
stderr += chunk.toString();
|
|
72
|
+
});
|
|
73
|
+
child.on("close", (code) => resolvePromise({ traces, stderr, code }));
|
|
74
|
+
child.on("error", (err) => {
|
|
75
|
+
stderr += String(err);
|
|
76
|
+
resolvePromise({ traces, stderr, code: -1 });
|
|
77
|
+
});
|
|
78
|
+
});
|
|
79
|
+
}
|
|
80
|
+
// Gate 2: fetch the old skill version from a base ref into a temp copy
|
|
81
|
+
async function baselineFetch(repoPath, baseRef, skillPaths) {
|
|
82
|
+
try {
|
|
83
|
+
execFileSync("git", ["rev-parse", "--verify", baseRef], { cwd: repoPath });
|
|
84
|
+
const dest = await mkdtemp(join(tmpdir(), "skilldiff-baseline-"));
|
|
85
|
+
execFileSync("git", ["checkout", baseRef, "--", ...skillPaths], { cwd: repoPath });
|
|
86
|
+
// restore repo to HEAD immediately — we only wanted the files
|
|
87
|
+
execFileSync("git", ["checkout", "HEAD", "--", ...skillPaths], { cwd: repoPath });
|
|
88
|
+
void dest;
|
|
89
|
+
console.log(` baseline fetch OK: ${skillPaths.length} path(s) from ${baseRef}`);
|
|
90
|
+
return true;
|
|
91
|
+
}
|
|
92
|
+
catch (err) {
|
|
93
|
+
console.error(` baseline fetch FAILED: ${err.message}`);
|
|
94
|
+
return false;
|
|
95
|
+
}
|
|
96
|
+
}
|
|
97
|
+
export async function runSpike(fixturePathArg) {
|
|
98
|
+
const adapter = pickAdapter();
|
|
99
|
+
console.log(`Spike harness: ${adapter.name} (${which(adapter.cmd)})`);
|
|
100
|
+
const fixtureRepo = resolve(fixturePathArg ?? (await makeMinimalFixture()));
|
|
101
|
+
console.log(`Spike fixture: ${fixtureRepo}`);
|
|
102
|
+
// ---- Gate 1: tool trace capture ------------------------------------
|
|
103
|
+
console.log(`\n[Gate 1] headless ${adapter.name} run + tool-use trace capture`);
|
|
104
|
+
const prompt = "Read the file NOTES.md in this repo and follow the instructions in it exactly.";
|
|
105
|
+
try {
|
|
106
|
+
const { traces, stderr, code } = await runHarness(adapter, fixtureRepo, prompt);
|
|
107
|
+
if (stderr.trim())
|
|
108
|
+
console.log(` harness stderr: ${stderr.trim().slice(0, 500)}`);
|
|
109
|
+
if (code !== 0 && traces.length === 0) {
|
|
110
|
+
console.error(` Gate 1: FAIL — harness exited ${code} with no tool events captured`);
|
|
111
|
+
if (stderr.trim())
|
|
112
|
+
console.error(` stderr: ${stderr.trim().slice(0, 1000)}`);
|
|
113
|
+
process.exit(3);
|
|
114
|
+
}
|
|
115
|
+
const readCall = traces.find((t) => t.tool.toLowerCase().includes("read"));
|
|
116
|
+
const writeCall = traces.find((t) => /write|edit/i.test(t.tool));
|
|
117
|
+
console.log(` captured ${traces.length} tool events:`, traces.map((t) => t.tool));
|
|
118
|
+
if (readCall && writeCall) {
|
|
119
|
+
console.log(" Gate 1: PASS (Read + Write observed — assertions are feasible)");
|
|
120
|
+
}
|
|
121
|
+
else {
|
|
122
|
+
console.log(" Gate 1: PARTIAL — traces captured but expected tools missing");
|
|
123
|
+
process.exit(3);
|
|
124
|
+
}
|
|
125
|
+
}
|
|
126
|
+
catch (err) {
|
|
127
|
+
console.error(` Gate 1: FAIL — ${err.message}`);
|
|
128
|
+
process.exit(3);
|
|
129
|
+
}
|
|
130
|
+
// ---- Gate 2: baseline fetch ----------------------------------------
|
|
131
|
+
console.log("\n[Gate 2] git baseline fetch (old skill version)");
|
|
132
|
+
try {
|
|
133
|
+
execFileSync("git", ["init"], { cwd: fixtureRepo });
|
|
134
|
+
execFileSync("git", ["add", "-A"], { cwd: fixtureRepo });
|
|
135
|
+
execFileSync("git", ["-c", "user.email=spike@skilldiff", "-c", "user.name=spike", "commit", "-m", "init"], { cwd: fixtureRepo });
|
|
136
|
+
// modify the skill, then try fetching the old one from HEAD~1
|
|
137
|
+
const skillPath = join(fixtureRepo, ".claude", "skills", "notes-helper", "SKILL.md");
|
|
138
|
+
await writeFile(skillPath, "---\nname: notes-helper\n---\n\nCHANGED instructions.");
|
|
139
|
+
execFileSync("git", ["add", "-A"], { cwd: fixtureRepo });
|
|
140
|
+
execFileSync("git", ["-c", "user.email=spike@skilldiff", "-c", "user.name=spike", "commit", "-m", "change skill"], { cwd: fixtureRepo });
|
|
141
|
+
const oldContent = execFileSync("git", ["show", "HEAD~1:.claude/skills/notes-helper/SKILL.md"], { cwd: fixtureRepo }).toString();
|
|
142
|
+
const ok = oldContent.includes("ORIGINAL");
|
|
143
|
+
console.log(` git show of old skill: ${ok ? "OK (original content retrieved)" : "MISMATCH"}`);
|
|
144
|
+
console.log(ok ? " Gate 2: PASS" : " Gate 2: FAIL");
|
|
145
|
+
process.exit(ok ? 0 : 4);
|
|
146
|
+
}
|
|
147
|
+
catch (err) {
|
|
148
|
+
console.error(` Gate 2: FAIL — ${err.message}`);
|
|
149
|
+
process.exit(4);
|
|
150
|
+
}
|
|
151
|
+
}
|
|
152
|
+
async function makeMinimalFixture() {
|
|
153
|
+
const dir = await mkdtemp(join(tmpdir(), "skilldiff-fixture-"));
|
|
154
|
+
const skillDir = join(dir, ".claude", "skills", "notes-helper");
|
|
155
|
+
await mkdir(skillDir, { recursive: true });
|
|
156
|
+
await writeFile(join(skillDir, "SKILL.md"), [
|
|
157
|
+
"---",
|
|
158
|
+
"name: notes-helper",
|
|
159
|
+
"description: Helps take and organize notes",
|
|
160
|
+
"---",
|
|
161
|
+
"",
|
|
162
|
+
"# notes-helper",
|
|
163
|
+
"",
|
|
164
|
+
"When asked to take a note: create or update NOTES.md with the content,",
|
|
165
|
+
"then confirm what you wrote.",
|
|
166
|
+
"",
|
|
167
|
+
"ORIGINAL instructions for baseline comparison.",
|
|
168
|
+
].join("\n"));
|
|
169
|
+
await writeFile(join(dir, "NOTES.md"), "Instructions: use the notes-helper skill to append 'SPIKE RAN OK' to this file.");
|
|
170
|
+
console.log(` created minimal fixture at ${dir}`);
|
|
171
|
+
return dir;
|
|
172
|
+
}
|
|
173
|
+
// direct execution support: `tsx scripts/spike.ts`
|
|
174
|
+
if (import.meta.url === `file://${process.argv[1]}`) {
|
|
175
|
+
await runSpike(process.argv[2]);
|
|
176
|
+
}
|
|
@@ -0,0 +1,33 @@
|
|
|
1
|
+
import type { Expectations } from "./scenario.js";
|
|
2
|
+
/** Normalized tool name — harnesses name the same tool differently. */
|
|
3
|
+
export declare function normalizeToolName(tool: string): string;
|
|
4
|
+
/** Everything observable from one agent run. Produced by the runner. */
|
|
5
|
+
export interface RunTrace {
|
|
6
|
+
toolCalls: Array<{
|
|
7
|
+
tool: string;
|
|
8
|
+
input: Record<string, unknown>;
|
|
9
|
+
}>;
|
|
10
|
+
/** File paths changed/created relative to fixture root. */
|
|
11
|
+
filesChanged: string[];
|
|
12
|
+
/** Full command strings that were executed (from bash-like tool calls). */
|
|
13
|
+
commandsRun: string[];
|
|
14
|
+
/** The agent's final text output. */
|
|
15
|
+
output: string;
|
|
16
|
+
/** Non-fatal problems encountered during the run (e.g. harness truncation). */
|
|
17
|
+
warnings: string[];
|
|
18
|
+
}
|
|
19
|
+
export interface AssertionResult {
|
|
20
|
+
kind: AssertionKind;
|
|
21
|
+
/** What was being asserted (the expected value or pattern). */
|
|
22
|
+
expected: string;
|
|
23
|
+
pass: boolean;
|
|
24
|
+
/** Human-readable explanation of what actually happened. */
|
|
25
|
+
actual: string;
|
|
26
|
+
}
|
|
27
|
+
type AssertionKind = "files_changed" | "commands_run" | "tool_calls" | "must_not" | "output_contains";
|
|
28
|
+
export declare function evaluateAssertions(trace: RunTrace, expect: Expectations): AssertionResult[];
|
|
29
|
+
export declare function summarize(results: AssertionResult[]): {
|
|
30
|
+
passed: number;
|
|
31
|
+
failed: number;
|
|
32
|
+
};
|
|
33
|
+
export {};
|
|
@@ -0,0 +1,101 @@
|
|
|
1
|
+
// skilldiff v0.1 — the 5 assertion kinds, evaluated against a run trace.
|
|
2
|
+
// All matching is partial (substring / set membership) per design-decisions.md.
|
|
3
|
+
/** Normalized tool name — harnesses name the same tool differently. */
|
|
4
|
+
export function normalizeToolName(tool) {
|
|
5
|
+
const t = tool.toLowerCase();
|
|
6
|
+
if (/read|view_file|open_file/.test(t))
|
|
7
|
+
return "read";
|
|
8
|
+
if (/write|create_file|new_file|edit|str_replace|update_file|multiedit/.test(t))
|
|
9
|
+
return "write";
|
|
10
|
+
if (/bash|exec_command|shell|command_execution/.test(t))
|
|
11
|
+
return "bash";
|
|
12
|
+
if (/glob|ls_dir|list_dir|find/.test(t))
|
|
13
|
+
return "glob";
|
|
14
|
+
if (/grep|search/.test(t))
|
|
15
|
+
return "grep";
|
|
16
|
+
if (/spawn|task|subagent/.test(t))
|
|
17
|
+
return "spawn";
|
|
18
|
+
return t;
|
|
19
|
+
}
|
|
20
|
+
function partialMatch(haystacks, needle) {
|
|
21
|
+
const n = needle.toLowerCase();
|
|
22
|
+
return haystacks.some((h) => h.toLowerCase().includes(n));
|
|
23
|
+
}
|
|
24
|
+
export function evaluateAssertions(trace, expect) {
|
|
25
|
+
const results = [];
|
|
26
|
+
// files_changed
|
|
27
|
+
for (const expected of expect.files_changed ?? []) {
|
|
28
|
+
const pass = partialMatch(trace.filesChanged, expected);
|
|
29
|
+
results.push({
|
|
30
|
+
kind: "files_changed",
|
|
31
|
+
expected,
|
|
32
|
+
pass,
|
|
33
|
+
actual: trace.filesChanged.length > 0 ? `changed: [${trace.filesChanged.join(", ")}]` : "no files changed",
|
|
34
|
+
});
|
|
35
|
+
}
|
|
36
|
+
// commands_run
|
|
37
|
+
for (const expected of expect.commands_run ?? []) {
|
|
38
|
+
const pass = partialMatch(trace.commandsRun, expected);
|
|
39
|
+
results.push({
|
|
40
|
+
kind: "commands_run",
|
|
41
|
+
expected,
|
|
42
|
+
pass,
|
|
43
|
+
actual: trace.commandsRun.length > 0 ? `ran: [${trace.commandsRun.join("; ")}]` : "no commands run",
|
|
44
|
+
});
|
|
45
|
+
}
|
|
46
|
+
// tool_calls
|
|
47
|
+
for (const expected of expect.tool_calls ?? []) {
|
|
48
|
+
const called = trace.toolCalls.map((tc) => normalizeToolName(tc.tool));
|
|
49
|
+
const pass = called.includes(normalizeToolName(expected));
|
|
50
|
+
results.push({
|
|
51
|
+
kind: "tool_calls",
|
|
52
|
+
expected,
|
|
53
|
+
pass,
|
|
54
|
+
actual: called.length > 0 ? `called: [${called.join(", ")}]` : "no tool calls",
|
|
55
|
+
});
|
|
56
|
+
}
|
|
57
|
+
// must_not — inverted lookups over the same observables
|
|
58
|
+
const mn = expect.must_not ?? {};
|
|
59
|
+
for (const forbidden of mn.files_changed ?? []) {
|
|
60
|
+
const violated = partialMatch(trace.filesChanged, forbidden);
|
|
61
|
+
results.push({
|
|
62
|
+
kind: "must_not",
|
|
63
|
+
expected: `files_changed does not include ${forbidden}`,
|
|
64
|
+
pass: !violated,
|
|
65
|
+
actual: violated ? `VIOLATED — ${forbidden} was changed` : `changed: [${trace.filesChanged.join(", ") || "none"}]`,
|
|
66
|
+
});
|
|
67
|
+
}
|
|
68
|
+
for (const forbidden of mn.commands_run ?? []) {
|
|
69
|
+
const violated = partialMatch(trace.commandsRun, forbidden);
|
|
70
|
+
results.push({
|
|
71
|
+
kind: "must_not",
|
|
72
|
+
expected: `commands_run does not include ${forbidden}`,
|
|
73
|
+
pass: !violated,
|
|
74
|
+
actual: violated ? `VIOLATED — ran: ${forbidden}` : `ran: [${trace.commandsRun.join("; ") || "none"}]`,
|
|
75
|
+
});
|
|
76
|
+
}
|
|
77
|
+
for (const forbidden of mn.tool_calls ?? []) {
|
|
78
|
+
const violated = trace.toolCalls.map((tc) => normalizeToolName(tc.tool)).includes(normalizeToolName(forbidden));
|
|
79
|
+
results.push({
|
|
80
|
+
kind: "must_not",
|
|
81
|
+
expected: `tool_calls does not include ${forbidden}`,
|
|
82
|
+
pass: !violated,
|
|
83
|
+
actual: violated ? `VIOLATED — called ${forbidden}` : "ok",
|
|
84
|
+
});
|
|
85
|
+
}
|
|
86
|
+
// output_contains
|
|
87
|
+
for (const expected of expect.output_contains ?? []) {
|
|
88
|
+
const pass = trace.output.toLowerCase().includes(expected.toLowerCase());
|
|
89
|
+
results.push({
|
|
90
|
+
kind: "output_contains",
|
|
91
|
+
expected,
|
|
92
|
+
pass,
|
|
93
|
+
actual: trace.output ? `output: "${trace.output.slice(0, 200)}"` : "(no output)",
|
|
94
|
+
});
|
|
95
|
+
}
|
|
96
|
+
return results;
|
|
97
|
+
}
|
|
98
|
+
export function summarize(results) {
|
|
99
|
+
const passed = results.filter((r) => r.pass).length;
|
|
100
|
+
return { passed, failed: results.length - passed };
|
|
101
|
+
}
|
|
@@ -0,0 +1,14 @@
|
|
|
1
|
+
export interface SkillVersion {
|
|
2
|
+
/** Temp directory containing the skill files at this version. */
|
|
3
|
+
dir: string;
|
|
4
|
+
/** The files successfully retrieved (repo-relative paths). */
|
|
5
|
+
files: string[];
|
|
6
|
+
}
|
|
7
|
+
export declare function gitRefExists(repoPath: string, ref: string): boolean;
|
|
8
|
+
/**
|
|
9
|
+
* Fetch skill files as of `baseRef` into a temp directory.
|
|
10
|
+
* Missing files at that ref are skipped and reported in `files` (only found ones listed).
|
|
11
|
+
*/
|
|
12
|
+
export declare function fetchSkillVersion(repoPath: string, baseRef: string, skillPaths: string[]): Promise<SkillVersion>;
|
|
13
|
+
/** Clean up a fetched version's temp dir (best effort). */
|
|
14
|
+
export declare function cleanupSkillVersion(version: SkillVersion): Promise<void>;
|
|
@@ -0,0 +1,58 @@
|
|
|
1
|
+
// skilldiff v0.1 — baseline fetch: retrieve the OLD version of skill files
|
|
2
|
+
// from a git ref (base branch / commit) without touching the working tree.
|
|
3
|
+
//
|
|
4
|
+
// Design doc decision: `git show <base>:<path>` into a temp copy.
|
|
5
|
+
// Gate 2 of the spike validated exactly this mechanism.
|
|
6
|
+
import { execFileSync } from "node:child_process";
|
|
7
|
+
import { mkdtemp, mkdir, writeFile, rm } from "node:fs/promises";
|
|
8
|
+
import { tmpdir } from "node:os";
|
|
9
|
+
import { join, dirname } from "node:path";
|
|
10
|
+
function gitShow(repoPath, ref, filePath) {
|
|
11
|
+
try {
|
|
12
|
+
return execFileSync("git", ["show", `${ref}:${filePath}`], {
|
|
13
|
+
cwd: repoPath,
|
|
14
|
+
maxBuffer: 10 * 1024 * 1024,
|
|
15
|
+
});
|
|
16
|
+
}
|
|
17
|
+
catch {
|
|
18
|
+
return null; // file did not exist at that ref
|
|
19
|
+
}
|
|
20
|
+
}
|
|
21
|
+
export function gitRefExists(repoPath, ref) {
|
|
22
|
+
try {
|
|
23
|
+
execFileSync("git", ["rev-parse", "--verify", ref], { cwd: repoPath });
|
|
24
|
+
return true;
|
|
25
|
+
}
|
|
26
|
+
catch {
|
|
27
|
+
return false;
|
|
28
|
+
}
|
|
29
|
+
}
|
|
30
|
+
/**
|
|
31
|
+
* Fetch skill files as of `baseRef` into a temp directory.
|
|
32
|
+
* Missing files at that ref are skipped and reported in `files` (only found ones listed).
|
|
33
|
+
*/
|
|
34
|
+
export async function fetchSkillVersion(repoPath, baseRef, skillPaths) {
|
|
35
|
+
if (!gitRefExists(repoPath, baseRef)) {
|
|
36
|
+
throw new Error(`git ref '${baseRef}' not found in ${repoPath}`);
|
|
37
|
+
}
|
|
38
|
+
const dir = await mkdtemp(join(tmpdir(), "skilldiff-baseline-"));
|
|
39
|
+
const files = [];
|
|
40
|
+
for (const skillPath of skillPaths) {
|
|
41
|
+
const content = gitShow(repoPath, baseRef, skillPath);
|
|
42
|
+
if (content === null)
|
|
43
|
+
continue;
|
|
44
|
+
const dest = join(dir, skillPath);
|
|
45
|
+
await mkdir(dirname(dest), { recursive: true });
|
|
46
|
+
await writeFile(dest, content);
|
|
47
|
+
files.push(skillPath);
|
|
48
|
+
}
|
|
49
|
+
if (files.length === 0) {
|
|
50
|
+
await rm(dir, { recursive: true, force: true });
|
|
51
|
+
throw new Error(`none of the skill paths exist at ref '${baseRef}': ${skillPaths.join(", ")}`);
|
|
52
|
+
}
|
|
53
|
+
return { dir, files };
|
|
54
|
+
}
|
|
55
|
+
/** Clean up a fetched version's temp dir (best effort). */
|
|
56
|
+
export async function cleanupSkillVersion(version) {
|
|
57
|
+
await rm(version.dir, { recursive: true, force: true }).catch(() => { });
|
|
58
|
+
}
|
package/dist/src/cli.js
ADDED
|
@@ -0,0 +1,69 @@
|
|
|
1
|
+
#!/usr/bin/env node
|
|
2
|
+
// skilldiff CLI — entry point
|
|
3
|
+
// Commands:
|
|
4
|
+
// spike go/no-go harness validation
|
|
5
|
+
// run <scenario> run a behavior-diff scenario (recorded or live)
|
|
6
|
+
import { resolve } from "node:path";
|
|
7
|
+
function usage() {
|
|
8
|
+
console.error(`Usage:
|
|
9
|
+
skilldiff init [dir] discover skills, emit starter scenarios
|
|
10
|
+
skilldiff run <scenario.yaml> [--old recorded.json] [--new recorded.json]
|
|
11
|
+
[--live] [--base <git-ref>]
|
|
12
|
+
skilldiff spike [fixture-path]
|
|
13
|
+
skilldiff --version`);
|
|
14
|
+
process.exit(1);
|
|
15
|
+
}
|
|
16
|
+
const [command, ...args] = process.argv.slice(2);
|
|
17
|
+
switch (command) {
|
|
18
|
+
case "spike": {
|
|
19
|
+
const { runSpike } = await import("../scripts/spike.js");
|
|
20
|
+
await runSpike(args[0]);
|
|
21
|
+
break;
|
|
22
|
+
}
|
|
23
|
+
case "run": {
|
|
24
|
+
const scenarioPath = args[0];
|
|
25
|
+
if (!scenarioPath)
|
|
26
|
+
usage();
|
|
27
|
+
let oldTrace;
|
|
28
|
+
let newTrace;
|
|
29
|
+
let live = false;
|
|
30
|
+
let base;
|
|
31
|
+
for (let i = 1; i < args.length; i++) {
|
|
32
|
+
if (args[i] === "--old")
|
|
33
|
+
oldTrace = args[++i];
|
|
34
|
+
else if (args[i] === "--new")
|
|
35
|
+
newTrace = args[++i];
|
|
36
|
+
else if (args[i] === "--live")
|
|
37
|
+
live = true;
|
|
38
|
+
else if (args[i] === "--base")
|
|
39
|
+
base = args[++i];
|
|
40
|
+
else
|
|
41
|
+
usage();
|
|
42
|
+
}
|
|
43
|
+
const { loadScenario } = await import("./scenario.js");
|
|
44
|
+
const { runScenario } = await import("./runner.js");
|
|
45
|
+
const scenario = await loadScenario(scenarioPath);
|
|
46
|
+
const result = await runScenario(scenario, {
|
|
47
|
+
oldTrace: oldTrace ? resolve(oldTrace) : undefined,
|
|
48
|
+
newTrace: newTrace ? resolve(newTrace) : undefined,
|
|
49
|
+
live,
|
|
50
|
+
base,
|
|
51
|
+
});
|
|
52
|
+
console.log(result.report);
|
|
53
|
+
process.exit(result.passed ? 0 : 1);
|
|
54
|
+
break;
|
|
55
|
+
}
|
|
56
|
+
case "init": {
|
|
57
|
+
const { runInit, printInitSummary } = await import("./init.js");
|
|
58
|
+
const result = await runInit(resolve(args[0] ?? process.cwd()));
|
|
59
|
+
printInitSummary(result);
|
|
60
|
+
break;
|
|
61
|
+
}
|
|
62
|
+
case "--version":
|
|
63
|
+
case "-v":
|
|
64
|
+
console.log("skilldiff 0.1.0");
|
|
65
|
+
break;
|
|
66
|
+
default:
|
|
67
|
+
console.error(`Unknown command: ${command ?? "(none)"}`);
|
|
68
|
+
usage();
|
|
69
|
+
}
|
|
@@ -0,0 +1,22 @@
|
|
|
1
|
+
export interface DiscoveredSkill {
|
|
2
|
+
/** Repo-relative path to SKILL.md */
|
|
3
|
+
path: string;
|
|
4
|
+
/** Skill name from frontmatter, or directory name. */
|
|
5
|
+
name: string;
|
|
6
|
+
/** Description from frontmatter, if present. */
|
|
7
|
+
description: string;
|
|
8
|
+
}
|
|
9
|
+
/** Discover SKILL.md files in conventional locations under repoRoot. */
|
|
10
|
+
export declare function discoverSkills(repoRoot: string): Promise<DiscoveredSkill[]>;
|
|
11
|
+
/**
|
|
12
|
+
* Infer a starter prompt for a skill.
|
|
13
|
+
* Strategy: ask the agent to read the skill's own SKILL.md and follow it —
|
|
14
|
+
* works generically for any skill, exercises the read path by construction.
|
|
15
|
+
*/
|
|
16
|
+
export declare function inferPrompt(skill: DiscoveredSkill): string;
|
|
17
|
+
/**
|
|
18
|
+
* Build a starter scenario object for one discovered skill.
|
|
19
|
+
* Expectations start minimal (tool_calls: [read]) and grow as the user
|
|
20
|
+
* learns the skill's real behavior — init prints guidance on this.
|
|
21
|
+
*/
|
|
22
|
+
export declare function buildStarterScenario(skill: DiscoveredSkill): Record<string, unknown>;
|
|
@@ -0,0 +1,104 @@
|
|
|
1
|
+
// skilldiff v0.1 — skill auto-discovery for `skilldiff init`.
|
|
2
|
+
// Scans a repo for SKILL.md files in conventional locations and infers a
|
|
3
|
+
// starter scenario (name, skillPaths, prompt, expect) from each one.
|
|
4
|
+
import { readFile, readdir, stat } from "node:fs/promises";
|
|
5
|
+
import { join, relative } from "node:path";
|
|
6
|
+
function parseFrontmatter(text) {
|
|
7
|
+
const fm = {};
|
|
8
|
+
const match = /^---\n([\s\S]*?)\n---/.exec(text);
|
|
9
|
+
if (!match)
|
|
10
|
+
return fm;
|
|
11
|
+
for (const line of match[1].split("\n")) {
|
|
12
|
+
const idx = line.indexOf(":");
|
|
13
|
+
if (idx > 0) {
|
|
14
|
+
fm[line.slice(0, idx).trim()] = line.slice(idx + 1).trim();
|
|
15
|
+
}
|
|
16
|
+
}
|
|
17
|
+
return fm;
|
|
18
|
+
}
|
|
19
|
+
async function findSkillFilesInDir(baseDir, skillsDir, depth = 0, out = []) {
|
|
20
|
+
if (depth > 3)
|
|
21
|
+
return out; // skills are shallow: <skills>/<name>/SKILL.md
|
|
22
|
+
let entries;
|
|
23
|
+
try {
|
|
24
|
+
entries = await readdir(skillsDir, { withFileTypes: true });
|
|
25
|
+
}
|
|
26
|
+
catch {
|
|
27
|
+
return out;
|
|
28
|
+
}
|
|
29
|
+
for (const entry of entries) {
|
|
30
|
+
if (!entry.isDirectory() || entry.name.startsWith("."))
|
|
31
|
+
continue;
|
|
32
|
+
const skillMd = join(skillsDir, entry.name, "SKILL.md");
|
|
33
|
+
try {
|
|
34
|
+
const s = await stat(skillMd);
|
|
35
|
+
if (s.isFile())
|
|
36
|
+
out.push(skillMd);
|
|
37
|
+
else
|
|
38
|
+
await findSkillFilesInDir(baseDir, join(skillsDir, entry.name), depth + 1, out);
|
|
39
|
+
}
|
|
40
|
+
catch {
|
|
41
|
+
// not a skill dir; recurse one level (handles skills/<category>/<name>/SKILL.md)
|
|
42
|
+
await findSkillFilesInDir(baseDir, join(skillsDir, entry.name), depth + 1, out);
|
|
43
|
+
}
|
|
44
|
+
}
|
|
45
|
+
return out;
|
|
46
|
+
}
|
|
47
|
+
/** Discover SKILL.md files in conventional locations under repoRoot. */
|
|
48
|
+
export async function discoverSkills(repoRoot) {
|
|
49
|
+
const candidateRoots = [
|
|
50
|
+
join(repoRoot, ".claude", "skills"),
|
|
51
|
+
join(repoRoot, "skills"),
|
|
52
|
+
join(repoRoot, ".agents", "skills"),
|
|
53
|
+
];
|
|
54
|
+
const found = new Map();
|
|
55
|
+
for (const root of candidateRoots) {
|
|
56
|
+
const files = await findSkillFilesInDir(repoRoot, root);
|
|
57
|
+
for (const file of files) {
|
|
58
|
+
const rel = relative(repoRoot, file).split("\\").join("/");
|
|
59
|
+
if (found.has(rel))
|
|
60
|
+
continue;
|
|
61
|
+
let text = "";
|
|
62
|
+
try {
|
|
63
|
+
text = await readFile(file, "utf8");
|
|
64
|
+
}
|
|
65
|
+
catch {
|
|
66
|
+
continue;
|
|
67
|
+
}
|
|
68
|
+
const fm = parseFrontmatter(text);
|
|
69
|
+
const dirName = rel.split("/").slice(-2, -1)[0] ?? "skill";
|
|
70
|
+
found.set(rel, {
|
|
71
|
+
path: rel,
|
|
72
|
+
name: fm.name || dirName,
|
|
73
|
+
description: fm.description || "",
|
|
74
|
+
});
|
|
75
|
+
}
|
|
76
|
+
}
|
|
77
|
+
return [...found.values()];
|
|
78
|
+
}
|
|
79
|
+
/**
|
|
80
|
+
* Infer a starter prompt for a skill.
|
|
81
|
+
* Strategy: ask the agent to read the skill's own SKILL.md and follow it —
|
|
82
|
+
* works generically for any skill, exercises the read path by construction.
|
|
83
|
+
*/
|
|
84
|
+
export function inferPrompt(skill) {
|
|
85
|
+
return (`You have a skill named "${skill.name}". Read the file ${skill.path} in this repo ` +
|
|
86
|
+
`and follow its instructions exactly.` +
|
|
87
|
+
(skill.description ? ` (${skill.description})` : ""));
|
|
88
|
+
}
|
|
89
|
+
/**
|
|
90
|
+
* Build a starter scenario object for one discovered skill.
|
|
91
|
+
* Expectations start minimal (tool_calls: [read]) and grow as the user
|
|
92
|
+
* learns the skill's real behavior — init prints guidance on this.
|
|
93
|
+
*/
|
|
94
|
+
export function buildStarterScenario(skill) {
|
|
95
|
+
return {
|
|
96
|
+
name: skill.name,
|
|
97
|
+
skillPaths: [skill.path],
|
|
98
|
+
fixture: "./", // TODO: point at a real fixture repo; "." works for self-testing
|
|
99
|
+
prompt: inferPrompt(skill),
|
|
100
|
+
expect: {
|
|
101
|
+
tool_calls: ["read"],
|
|
102
|
+
},
|
|
103
|
+
};
|
|
104
|
+
}
|
|
@@ -0,0 +1,9 @@
|
|
|
1
|
+
import { type DiscoveredSkill } from "./discover.js";
|
|
2
|
+
export declare function runInit(repoRoot: string, outDir?: string): Promise<{
|
|
3
|
+
created: string[];
|
|
4
|
+
skills: DiscoveredSkill[];
|
|
5
|
+
}>;
|
|
6
|
+
export declare function printInitSummary(result: {
|
|
7
|
+
created: string[];
|
|
8
|
+
skills: DiscoveredSkill[];
|
|
9
|
+
}): void;
|
package/dist/src/init.js
ADDED
|
@@ -0,0 +1,66 @@
|
|
|
1
|
+
// skilldiff v0.1 — `skilldiff init`: scan the repo for skills and emit
|
|
2
|
+
// starter scenario YAMLs so users get value in under a minute.
|
|
3
|
+
import { writeFile, mkdir } from "node:fs/promises";
|
|
4
|
+
import { join, resolve } from "node:path";
|
|
5
|
+
import { discoverSkills, buildStarterScenario } from "./discover.js";
|
|
6
|
+
function scenarioToYaml(scenario) {
|
|
7
|
+
const lines = [];
|
|
8
|
+
lines.push(`name: ${scenario.name}`);
|
|
9
|
+
lines.push(`skillPaths:`);
|
|
10
|
+
for (const p of scenario.skillPaths)
|
|
11
|
+
lines.push(` - ${p}`);
|
|
12
|
+
lines.push(`fixture: ${scenario.fixture}`);
|
|
13
|
+
lines.push(`prompt: >-`);
|
|
14
|
+
lines.push(` ${scenario.prompt.replace(/\n+/g, " ")}`);
|
|
15
|
+
lines.push(`expect:`);
|
|
16
|
+
const expect = scenario.expect;
|
|
17
|
+
lines.push(` tool_calls:`);
|
|
18
|
+
for (const t of expect.tool_calls)
|
|
19
|
+
lines.push(` - ${t}`);
|
|
20
|
+
return lines.join("\n") + "\n";
|
|
21
|
+
}
|
|
22
|
+
export async function runInit(repoRoot, outDir = "skilldiff") {
|
|
23
|
+
const skills = await discoverSkills(repoRoot);
|
|
24
|
+
if (skills.length === 0) {
|
|
25
|
+
return { created: [], skills };
|
|
26
|
+
}
|
|
27
|
+
const absOutDir = resolve(outDir); // resolve against cwd, NOT repoRoot
|
|
28
|
+
await mkdir(absOutDir, { recursive: true });
|
|
29
|
+
const created = [];
|
|
30
|
+
for (const skill of skills) {
|
|
31
|
+
const file = join(absOutDir, `${skill.name}.scenario.yaml`);
|
|
32
|
+
const yaml = scenarioToYaml(buildStarterScenario(skill));
|
|
33
|
+
await writeFile(file, yaml);
|
|
34
|
+
created.push(file);
|
|
35
|
+
}
|
|
36
|
+
return { created, skills };
|
|
37
|
+
}
|
|
38
|
+
export function printInitSummary(result) {
|
|
39
|
+
if (result.skills.length === 0) {
|
|
40
|
+
console.log("No SKILL.md files found in conventional locations:");
|
|
41
|
+
console.log(" .claude/skills/, skills/, .agents/skills/");
|
|
42
|
+
console.log("\nTo test a skill elsewhere, write a scenario manually:");
|
|
43
|
+
console.log(" https://github.com/scs0209/skilldiff#quickstart");
|
|
44
|
+
return;
|
|
45
|
+
}
|
|
46
|
+
console.log(`Found ${result.skills.length} skill(s):\n`);
|
|
47
|
+
for (const skill of result.skills) {
|
|
48
|
+
console.log(` ${skill.name}${skill.description ? ` — ${skill.description}` : ""}`);
|
|
49
|
+
console.log(` ${skill.path}`);
|
|
50
|
+
}
|
|
51
|
+
console.log(`\nCreated starter scenarios:`);
|
|
52
|
+
for (const file of result.created) {
|
|
53
|
+
console.log(` ${file}`);
|
|
54
|
+
}
|
|
55
|
+
console.log(`
|
|
56
|
+
Next steps:
|
|
57
|
+
1. Point 'fixture' in each scenario at a small repo the skill can safely
|
|
58
|
+
operate on (a temp copy of a real project works well).
|
|
59
|
+
2. Strengthen 'expect' once you've seen the skill's real behavior — start
|
|
60
|
+
with what it MUST do, then add must_not for what it must NEVER do.
|
|
61
|
+
3. Try a live run:
|
|
62
|
+
skilldiff run <scenario>.yaml --live
|
|
63
|
+
4. Capture traces for CI (recorded mode):
|
|
64
|
+
skilldiff run <scenario>.yaml --live --save-trace traces/
|
|
65
|
+
`);
|
|
66
|
+
}
|