uni-harness 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.claude/agents/TEMPLATE.md +64 -0
- package/.claude/hooks/guard-pre-bash.sh +46 -0
- package/.claude/hooks/observe-log.sh +88 -0
- package/.claude/hooks/sensor-post-edit.sh +62 -0
- package/.claude/hooks/session-start.sh +86 -0
- package/.claude/hooks/stop-gate.sh +70 -0
- package/.claude/settings.json +76 -0
- package/.claude/skills/checkpoint/SKILL.md +49 -0
- package/.claude/skills/guide-audit/SKILL.md +50 -0
- package/.claude/skills/harness-init/SKILL.md +70 -0
- package/.claude/skills/ratchet/SKILL.md +72 -0
- package/.harness/commands.env +13 -0
- package/CLAUDE.md +68 -0
- package/LICENSE +21 -0
- package/README.md +212 -0
- package/bin/cli.js +371 -0
- package/harness/harness_report.py +90 -0
- package/harness/tests/test_hooks.sh +129 -0
- package/harness/tests/test_installer.sh +112 -0
- package/package.json +35 -0
package/bin/cli.js
ADDED
|
@@ -0,0 +1,371 @@
|
|
|
1
|
+
#!/usr/bin/env node
|
|
2
|
+
/**
|
|
3
|
+
* uni-harness — installer for the Claude Code agent harness kit
|
|
4
|
+
*
|
|
5
|
+
* Usage:
|
|
6
|
+
* npx uni-harness init [dir] [--force] install the harness
|
|
7
|
+
* npx uni-harness update [dir] [--force] update machinery files
|
|
8
|
+
* npx uni-harness doctor [dir] diagnose the installation
|
|
9
|
+
* npx uni-harness uninstall [dir] --yes remove machinery files
|
|
10
|
+
*
|
|
11
|
+
* File boundary (the core contract of this installer):
|
|
12
|
+
* machinery (managed) — .claude/hooks/, .claude/skills/, .claude/agents/,
|
|
13
|
+
* harness/ → init installs them, update refreshes
|
|
14
|
+
* them. Hashes are recorded in
|
|
15
|
+
* .harness/kit-manifest.json so files the user has
|
|
16
|
+
* modified are NEVER overwritten (unless --force).
|
|
17
|
+
* project-owned — CLAUDE.md, .harness/commands.env,
|
|
18
|
+
* .claude/settings.json → created by init when
|
|
19
|
+
* absent; update/uninstall never touch them.
|
|
20
|
+
*/
|
|
21
|
+
'use strict';
|
|
22
|
+
|
|
23
|
+
const fs = require('fs');
|
|
24
|
+
const path = require('path');
|
|
25
|
+
const crypto = require('crypto');
|
|
26
|
+
const { spawnSync } = require('child_process');
|
|
27
|
+
|
|
28
|
+
const PKG_ROOT = path.resolve(__dirname, '..');
|
|
29
|
+
const MACHINERY_DIRS = ['.claude/hooks', '.claude/skills', '.claude/agents', 'harness'];
|
|
30
|
+
const MANIFEST = '.harness/kit-manifest.json';
|
|
31
|
+
const KIT_VERSION = require(path.join(PKG_ROOT, 'package.json')).version;
|
|
32
|
+
|
|
33
|
+
// ── helpers ──────────────────────────────────────────────────────
|
|
34
|
+
const sha256 = (file) =>
|
|
35
|
+
crypto.createHash('sha256').update(fs.readFileSync(file)).digest('hex');
|
|
36
|
+
|
|
37
|
+
function walk(dir) {
|
|
38
|
+
if (!fs.existsSync(dir)) return [];
|
|
39
|
+
const out = [];
|
|
40
|
+
for (const e of fs.readdirSync(dir, { withFileTypes: true })) {
|
|
41
|
+
const p = path.join(dir, e.name);
|
|
42
|
+
if (e.isDirectory()) {
|
|
43
|
+
if (e.name === '__pycache__' || e.name === 'node_modules') continue;
|
|
44
|
+
out.push(...walk(p));
|
|
45
|
+
} else if (e.isFile() && e.name !== '.DS_Store' && !e.name.endsWith('.pyc')) {
|
|
46
|
+
out.push(p);
|
|
47
|
+
}
|
|
48
|
+
}
|
|
49
|
+
return out;
|
|
50
|
+
}
|
|
51
|
+
|
|
52
|
+
function machineryFiles() {
|
|
53
|
+
const files = [];
|
|
54
|
+
for (const d of MACHINERY_DIRS) {
|
|
55
|
+
for (const f of walk(path.join(PKG_ROOT, d))) {
|
|
56
|
+
files.push(path.relative(PKG_ROOT, f));
|
|
57
|
+
}
|
|
58
|
+
}
|
|
59
|
+
return files.sort();
|
|
60
|
+
}
|
|
61
|
+
|
|
62
|
+
function copyInto(target, rel) {
|
|
63
|
+
const dst = path.join(target, rel);
|
|
64
|
+
fs.mkdirSync(path.dirname(dst), { recursive: true });
|
|
65
|
+
fs.copyFileSync(path.join(PKG_ROOT, rel), dst);
|
|
66
|
+
if (rel.endsWith('.sh')) {
|
|
67
|
+
try { fs.chmodSync(dst, 0o755); } catch { /* e.g. Windows */ }
|
|
68
|
+
}
|
|
69
|
+
}
|
|
70
|
+
|
|
71
|
+
function readManifest(target) {
|
|
72
|
+
const p = path.join(target, MANIFEST);
|
|
73
|
+
if (!fs.existsSync(p)) return null;
|
|
74
|
+
try { return JSON.parse(fs.readFileSync(p, 'utf8')); } catch { return null; }
|
|
75
|
+
}
|
|
76
|
+
|
|
77
|
+
function writeManifest(target, files) {
|
|
78
|
+
const entries = {};
|
|
79
|
+
for (const rel of files) {
|
|
80
|
+
const p = path.join(target, rel);
|
|
81
|
+
if (fs.existsSync(p)) entries[rel] = sha256(p);
|
|
82
|
+
}
|
|
83
|
+
const p = path.join(target, MANIFEST);
|
|
84
|
+
fs.mkdirSync(path.dirname(p), { recursive: true });
|
|
85
|
+
fs.writeFileSync(p, JSON.stringify(
|
|
86
|
+
{ kitVersion: KIT_VERSION, installedAt: new Date().toISOString(), files: entries },
|
|
87
|
+
null, 2) + '\n');
|
|
88
|
+
}
|
|
89
|
+
|
|
90
|
+
function runHookTests(target) {
|
|
91
|
+
const t = path.join(target, 'harness/tests/test_hooks.sh');
|
|
92
|
+
if (!fs.existsSync(t)) return null;
|
|
93
|
+
const r = spawnSync('bash', [t], { cwd: target, encoding: 'utf8' });
|
|
94
|
+
if (r.error) return null; // no bash (e.g. Windows) — skip
|
|
95
|
+
return { ok: r.status === 0, out: (r.stdout || '') + (r.stderr || '') };
|
|
96
|
+
}
|
|
97
|
+
|
|
98
|
+
const log = (s) => process.stdout.write(s + '\n');
|
|
99
|
+
|
|
100
|
+
// ── settings.json merge (init only) ──────────────────────────────
|
|
101
|
+
// Hooks: append only the entries whose hook script is not yet registered
|
|
102
|
+
// for that event. Everything the user already has is left untouched.
|
|
103
|
+
// If the template ships a permissions block, it is used only when the
|
|
104
|
+
// target has none at all — we never silently widen an existing one; missing
|
|
105
|
+
// entries are printed as suggestions instead.
|
|
106
|
+
function mergeSettings(target) {
|
|
107
|
+
const tplPath = path.join(PKG_ROOT, '.claude/settings.json');
|
|
108
|
+
const dstPath = path.join(target, '.claude/settings.json');
|
|
109
|
+
const tpl = JSON.parse(fs.readFileSync(tplPath, 'utf8'));
|
|
110
|
+
|
|
111
|
+
if (!fs.existsSync(dstPath)) {
|
|
112
|
+
fs.mkdirSync(path.dirname(dstPath), { recursive: true });
|
|
113
|
+
fs.copyFileSync(tplPath, dstPath);
|
|
114
|
+
log(' + .claude/settings.json (new)');
|
|
115
|
+
return;
|
|
116
|
+
}
|
|
117
|
+
|
|
118
|
+
let cur;
|
|
119
|
+
try { cur = JSON.parse(fs.readFileSync(dstPath, 'utf8')); } catch {
|
|
120
|
+
log(' ! .claude/settings.json failed to parse — skipping merge. Register the hooks manually.');
|
|
121
|
+
return;
|
|
122
|
+
}
|
|
123
|
+
|
|
124
|
+
let changed = false;
|
|
125
|
+
cur.hooks = cur.hooks || {};
|
|
126
|
+
for (const [event, entries] of Object.entries(tpl.hooks || {})) {
|
|
127
|
+
cur.hooks[event] = cur.hooks[event] || [];
|
|
128
|
+
for (const entry of entries) {
|
|
129
|
+
const cmd = entry.hooks && entry.hooks[0] && entry.hooks[0].command;
|
|
130
|
+
const present = JSON.stringify(cur.hooks[event]).includes(path.basename(cmd || ''));
|
|
131
|
+
if (!present) { cur.hooks[event].push(entry); changed = true; }
|
|
132
|
+
}
|
|
133
|
+
}
|
|
134
|
+
|
|
135
|
+
const missing = [];
|
|
136
|
+
if (tpl.permissions) {
|
|
137
|
+
if (!cur.permissions) {
|
|
138
|
+
cur.permissions = tpl.permissions;
|
|
139
|
+
changed = true;
|
|
140
|
+
} else {
|
|
141
|
+
for (const key of ['allow', 'ask', 'deny']) {
|
|
142
|
+
for (const rule of (tpl.permissions[key] || [])) {
|
|
143
|
+
if (!(cur.permissions[key] || []).includes(rule)) missing.push(`${key}: ${rule}`);
|
|
144
|
+
}
|
|
145
|
+
}
|
|
146
|
+
}
|
|
147
|
+
}
|
|
148
|
+
|
|
149
|
+
if (changed) {
|
|
150
|
+
fs.writeFileSync(dstPath, JSON.stringify(cur, null, 2) + '\n');
|
|
151
|
+
log(' ~ .claude/settings.json (hooks merged; existing settings preserved)');
|
|
152
|
+
}
|
|
153
|
+
if (missing.length) {
|
|
154
|
+
log(' ! settings.json permissions is missing kit-recommended entries (NOT added automatically):');
|
|
155
|
+
for (const m of missing) log(` ${m}`);
|
|
156
|
+
}
|
|
157
|
+
}
|
|
158
|
+
|
|
159
|
+
// ── init ─────────────────────────────────────────────────────────
|
|
160
|
+
function init(target, force) {
|
|
161
|
+
const files = machineryFiles();
|
|
162
|
+
const manifest = readManifest(target);
|
|
163
|
+
const skipped = [];
|
|
164
|
+
|
|
165
|
+
for (const rel of files) {
|
|
166
|
+
const dst = path.join(target, rel);
|
|
167
|
+
if (fs.existsSync(dst)) {
|
|
168
|
+
const dstHash = sha256(dst);
|
|
169
|
+
if (dstHash === sha256(path.join(PKG_ROOT, rel))) continue; // already identical
|
|
170
|
+
const recorded = manifest && manifest.files && manifest.files[rel];
|
|
171
|
+
if (!force && dstHash !== recorded) { skipped.push(rel); continue; }
|
|
172
|
+
}
|
|
173
|
+
copyInto(target, rel);
|
|
174
|
+
log(' + ' + rel);
|
|
175
|
+
}
|
|
176
|
+
|
|
177
|
+
let claudeMdExisted = false;
|
|
178
|
+
for (const rel of ['CLAUDE.md', '.harness/commands.env']) {
|
|
179
|
+
const dst = path.join(target, rel);
|
|
180
|
+
if (fs.existsSync(dst)) {
|
|
181
|
+
log(' = ' + rel + ' (existing file kept)');
|
|
182
|
+
if (rel === 'CLAUDE.md') claudeMdExisted = true;
|
|
183
|
+
continue;
|
|
184
|
+
}
|
|
185
|
+
copyInto(target, rel);
|
|
186
|
+
log(' + ' + rel);
|
|
187
|
+
}
|
|
188
|
+
mergeSettings(target);
|
|
189
|
+
|
|
190
|
+
fs.mkdirSync(path.join(target, '.harness/logs'), { recursive: true });
|
|
191
|
+
const gi = path.join(target, '.gitignore');
|
|
192
|
+
const giBody = fs.existsSync(gi) ? fs.readFileSync(gi, 'utf8') : '';
|
|
193
|
+
if (!giBody.split('\n').some((l) => l.trim().replace(/\/$/, '') === '.harness/logs')) {
|
|
194
|
+
fs.writeFileSync(gi, giBody + (giBody && !giBody.endsWith('\n') ? '\n' : '') + '.harness/logs/\n');
|
|
195
|
+
log(' ~ .gitignore (+.harness/logs/)');
|
|
196
|
+
}
|
|
197
|
+
|
|
198
|
+
writeManifest(target, files);
|
|
199
|
+
|
|
200
|
+
if (skipped.length) {
|
|
201
|
+
log('\nSkipped files with local modifications (use --force to overwrite):');
|
|
202
|
+
for (const s of skipped) log(' ! ' + s);
|
|
203
|
+
}
|
|
204
|
+
|
|
205
|
+
const t = runHookTests(target);
|
|
206
|
+
if (t) log(t.ok ? '\nHook tests passed.' : '\nHook tests FAILED:\n' + t.out);
|
|
207
|
+
else log('\nbash not found — skipped hook tests.');
|
|
208
|
+
|
|
209
|
+
log('\nNext step: open Claude Code in the project and run /harness-init');
|
|
210
|
+
log('to fill in the BUILD/TEST/LINT commands — the verification sensors');
|
|
211
|
+
log('stay inactive until then.');
|
|
212
|
+
if (claudeMdExisted) {
|
|
213
|
+
log('Your existing CLAUDE.md was kept as-is. /harness-init will propose');
|
|
214
|
+
log('adding the harness sections (PROJECT / RULES / Work Loop /');
|
|
215
|
+
log('Checkpoints) to it — nothing is changed without your approval.');
|
|
216
|
+
}
|
|
217
|
+
return t && !t.ok ? 1 : 0;
|
|
218
|
+
}
|
|
219
|
+
|
|
220
|
+
// ── update ───────────────────────────────────────────────────────
|
|
221
|
+
function update(target, force) {
|
|
222
|
+
const manifest = readManifest(target);
|
|
223
|
+
if (!manifest) {
|
|
224
|
+
log('No kit-manifest.json found. Run `uni-harness init` first.');
|
|
225
|
+
return 1;
|
|
226
|
+
}
|
|
227
|
+
const files = machineryFiles();
|
|
228
|
+
const skipped = [];
|
|
229
|
+
let changed = 0;
|
|
230
|
+
|
|
231
|
+
for (const rel of files) {
|
|
232
|
+
const src = path.join(PKG_ROOT, rel);
|
|
233
|
+
const dst = path.join(target, rel);
|
|
234
|
+
const recorded = manifest.files[rel];
|
|
235
|
+
if (!fs.existsSync(dst)) { copyInto(target, rel); log(' + ' + rel); changed++; continue; }
|
|
236
|
+
const dstHash = sha256(dst);
|
|
237
|
+
if (dstHash === sha256(src)) continue; // up to date
|
|
238
|
+
if (!force && recorded && dstHash !== recorded) { skipped.push(rel); continue; }
|
|
239
|
+
copyInto(target, rel);
|
|
240
|
+
log(' ~ ' + rel);
|
|
241
|
+
changed++;
|
|
242
|
+
}
|
|
243
|
+
|
|
244
|
+
// Files removed from the kit: clean up unmodified copies only
|
|
245
|
+
for (const rel of Object.keys(manifest.files)) {
|
|
246
|
+
if (files.includes(rel)) continue;
|
|
247
|
+
const dst = path.join(target, rel);
|
|
248
|
+
if (fs.existsSync(dst) && sha256(dst) === manifest.files[rel]) {
|
|
249
|
+
fs.unlinkSync(dst);
|
|
250
|
+
log(' - ' + rel + ' (removed from kit)');
|
|
251
|
+
changed++;
|
|
252
|
+
}
|
|
253
|
+
}
|
|
254
|
+
|
|
255
|
+
writeManifest(target, files);
|
|
256
|
+
log(`\n${changed} file(s) updated (v${manifest.kitVersion} -> v${KIT_VERSION}). ` +
|
|
257
|
+
'CLAUDE.md / commands.env / settings.json were not touched.');
|
|
258
|
+
if (skipped.length) {
|
|
259
|
+
log('Skipped files with local modifications (use --force to overwrite):');
|
|
260
|
+
for (const s of skipped) log(' ! ' + s);
|
|
261
|
+
}
|
|
262
|
+
|
|
263
|
+
const t = runHookTests(target);
|
|
264
|
+
if (t) log(t.ok ? 'Hook tests passed.' : 'Hook tests FAILED:\n' + t.out);
|
|
265
|
+
return t && !t.ok ? 1 : 0;
|
|
266
|
+
}
|
|
267
|
+
|
|
268
|
+
// ── doctor ───────────────────────────────────────────────────────
|
|
269
|
+
function doctor(target) {
|
|
270
|
+
let hard = 0;
|
|
271
|
+
const check = (ok, name, hint) => {
|
|
272
|
+
log(` ${ok ? 'ok ' : 'FAIL'} ${name}${!ok && hint ? ' — ' + hint : ''}`);
|
|
273
|
+
if (!ok) hard++;
|
|
274
|
+
};
|
|
275
|
+
const warn = (name) => log(` warn ${name}`);
|
|
276
|
+
|
|
277
|
+
const has = (cmd) => !spawnSync(cmd, ['--version'], { encoding: 'utf8' }).error;
|
|
278
|
+
check(has('bash'), 'bash');
|
|
279
|
+
check(has('python3'), 'python3', 'the hooks use python3 for JSON parsing');
|
|
280
|
+
|
|
281
|
+
const manifest = readManifest(target);
|
|
282
|
+
check(!!manifest, MANIFEST, 'run `uni-harness init` first');
|
|
283
|
+
if (manifest) log(` kit version: ${manifest.kitVersion} (package: ${KIT_VERSION})`);
|
|
284
|
+
|
|
285
|
+
for (const rel of machineryFiles()) {
|
|
286
|
+
const p = path.join(target, rel);
|
|
287
|
+
if (!fs.existsSync(p)) { check(false, rel, 'missing — run `uni-harness update`'); continue; }
|
|
288
|
+
if (rel.endsWith('.sh')) {
|
|
289
|
+
try { fs.accessSync(p, fs.constants.X_OK); } catch { check(false, rel, 'not executable'); }
|
|
290
|
+
}
|
|
291
|
+
}
|
|
292
|
+
|
|
293
|
+
const sp = path.join(target, '.claude/settings.json');
|
|
294
|
+
try {
|
|
295
|
+
const body = fs.readFileSync(sp, 'utf8');
|
|
296
|
+
JSON.parse(body);
|
|
297
|
+
check(true, '.claude/settings.json parses');
|
|
298
|
+
for (const h of ['guard-pre-bash.sh', 'sensor-post-edit.sh', 'stop-gate.sh', 'observe-log.sh', 'session-start.sh']) {
|
|
299
|
+
if (!body.includes(h)) check(false, `hook registered: ${h}`, 'not in settings.json');
|
|
300
|
+
}
|
|
301
|
+
} catch { check(false, '.claude/settings.json', 'missing or failed to parse'); }
|
|
302
|
+
|
|
303
|
+
const envp = path.join(target, '.harness/commands.env');
|
|
304
|
+
if (fs.existsSync(envp)) {
|
|
305
|
+
const env = fs.readFileSync(envp, 'utf8');
|
|
306
|
+
if (/^(LINT_CMD|TEST_CMD)=".+"/m.test(env)) check(true, 'commands.env sensor commands');
|
|
307
|
+
else warn('commands.env is empty — sensors inactive. Run /harness-init in Claude Code');
|
|
308
|
+
} else check(false, '.harness/commands.env', 'missing');
|
|
309
|
+
|
|
310
|
+
const cm = path.join(target, 'CLAUDE.md');
|
|
311
|
+
if (fs.existsSync(cm) && fs.readFileSync(cm, 'utf8').includes('[project name]')) {
|
|
312
|
+
warn('CLAUDE.md PROJECT section is still a placeholder — run /harness-init');
|
|
313
|
+
}
|
|
314
|
+
|
|
315
|
+
const t = runHookTests(target);
|
|
316
|
+
if (t) check(t.ok, 'hook tests (harness/tests/test_hooks.sh)');
|
|
317
|
+
|
|
318
|
+
log(hard === 0 ? '\nDiagnosis passed.' : `\n${hard} failure(s).`);
|
|
319
|
+
return hard === 0 ? 0 : 1;
|
|
320
|
+
}
|
|
321
|
+
|
|
322
|
+
// ── uninstall ────────────────────────────────────────────────────
|
|
323
|
+
function uninstall(target, yes) {
|
|
324
|
+
const manifest = readManifest(target);
|
|
325
|
+
if (!manifest) { log('No kit-manifest.json — nothing installed here.'); return 1; }
|
|
326
|
+
if (!yes) { log('This removes the machinery files. Add --yes to confirm.'); return 1; }
|
|
327
|
+
|
|
328
|
+
for (const [rel, hash] of Object.entries(manifest.files)) {
|
|
329
|
+
const p = path.join(target, rel);
|
|
330
|
+
if (!fs.existsSync(p)) continue;
|
|
331
|
+
if (sha256(p) === hash) { fs.unlinkSync(p); log(' - ' + rel); }
|
|
332
|
+
else log(' ! ' + rel + ' (locally modified — kept)');
|
|
333
|
+
}
|
|
334
|
+
fs.unlinkSync(path.join(target, MANIFEST));
|
|
335
|
+
for (const d of ['.claude/hooks', '.claude/skills', '.claude/agents', 'harness/tests', 'harness']) {
|
|
336
|
+
try { fs.rmdirSync(path.join(target, d)); } catch { /* keep if not empty */ }
|
|
337
|
+
}
|
|
338
|
+
log('Uninstalled. CLAUDE.md / commands.env / settings.json / logs were kept.');
|
|
339
|
+
log('Remove the hook registrations from settings.json manually (missing');
|
|
340
|
+
log('hook files are ignored silently, but the entries are now dead).');
|
|
341
|
+
return 0;
|
|
342
|
+
}
|
|
343
|
+
|
|
344
|
+
// ── main ─────────────────────────────────────────────────────────
|
|
345
|
+
function main() {
|
|
346
|
+
const args = process.argv.slice(2);
|
|
347
|
+
const cmd = args[0];
|
|
348
|
+
const flags = new Set(args.filter((a) => a.startsWith('--')));
|
|
349
|
+
const dirArg = args.slice(1).find((a) => !a.startsWith('--'));
|
|
350
|
+
const target = path.resolve(dirArg || process.cwd());
|
|
351
|
+
|
|
352
|
+
if (!cmd || flags.has('--help')) {
|
|
353
|
+
log('Usage: uni-harness <init|update|doctor|uninstall> [dir] [--force|--yes]');
|
|
354
|
+
return 0;
|
|
355
|
+
}
|
|
356
|
+
if (path.resolve(target) === PKG_ROOT) {
|
|
357
|
+
log('Refusing to install into the kit repository itself. Pass a target project path.');
|
|
358
|
+
return 1;
|
|
359
|
+
}
|
|
360
|
+
if (!fs.existsSync(target)) { log(`Target path does not exist: ${target}`); return 1; }
|
|
361
|
+
|
|
362
|
+
switch (cmd) {
|
|
363
|
+
case 'init': return init(target, flags.has('--force'));
|
|
364
|
+
case 'update': return update(target, flags.has('--force'));
|
|
365
|
+
case 'doctor': return doctor(target);
|
|
366
|
+
case 'uninstall': return uninstall(target, flags.has('--yes'));
|
|
367
|
+
default: log(`Unknown command: ${cmd}`); return 1;
|
|
368
|
+
}
|
|
369
|
+
}
|
|
370
|
+
|
|
371
|
+
process.exit(main());
|
|
@@ -0,0 +1,90 @@
|
|
|
1
|
+
#!/usr/bin/env python3
|
|
2
|
+
"""Harness health scorecard.
|
|
3
|
+
|
|
4
|
+
Reads .harness/logs/tool_calls.jsonl and prints health metrics.
|
|
5
|
+
|
|
6
|
+
Usage:
|
|
7
|
+
python3 harness/harness_report.py [--days 7]
|
|
8
|
+
"""
|
|
9
|
+
import argparse
|
|
10
|
+
import datetime as dt
|
|
11
|
+
import json
|
|
12
|
+
import sys
|
|
13
|
+
from collections import Counter, defaultdict
|
|
14
|
+
from pathlib import Path
|
|
15
|
+
|
|
16
|
+
|
|
17
|
+
def load(log_path: Path, since: dt.datetime):
|
|
18
|
+
rows = []
|
|
19
|
+
if not log_path.exists():
|
|
20
|
+
return rows
|
|
21
|
+
with log_path.open() as f:
|
|
22
|
+
for line in f:
|
|
23
|
+
try:
|
|
24
|
+
e = json.loads(line)
|
|
25
|
+
except json.JSONDecodeError:
|
|
26
|
+
continue
|
|
27
|
+
try:
|
|
28
|
+
ts = dt.datetime.fromisoformat(e.get("ts", ""))
|
|
29
|
+
except ValueError:
|
|
30
|
+
continue
|
|
31
|
+
if ts >= since:
|
|
32
|
+
e["_ts"] = ts
|
|
33
|
+
rows.append(e)
|
|
34
|
+
return rows
|
|
35
|
+
|
|
36
|
+
|
|
37
|
+
def main() -> int:
|
|
38
|
+
ap = argparse.ArgumentParser()
|
|
39
|
+
ap.add_argument("--days", type=int, default=7)
|
|
40
|
+
args = ap.parse_args()
|
|
41
|
+
|
|
42
|
+
root = Path(__file__).resolve().parent.parent
|
|
43
|
+
log_path = root / ".harness" / "logs" / "tool_calls.jsonl"
|
|
44
|
+
since = dt.datetime.now() - dt.timedelta(days=args.days)
|
|
45
|
+
rows = load(log_path, since)
|
|
46
|
+
|
|
47
|
+
print(f"═══ Harness Health Scorecard (last {args.days} days) ═══")
|
|
48
|
+
if not rows:
|
|
49
|
+
print("No logs found. Check that the hooks are registered and a session has run.")
|
|
50
|
+
return 0
|
|
51
|
+
|
|
52
|
+
sessions = defaultdict(list)
|
|
53
|
+
for e in rows:
|
|
54
|
+
sessions[e.get("session_id", "?")].append(e)
|
|
55
|
+
|
|
56
|
+
total = len(rows)
|
|
57
|
+
failures = [e for e in rows if e.get("event") == "PostToolUseFailure"]
|
|
58
|
+
fail_rate = len(failures) / total * 100 if total else 0.0
|
|
59
|
+
|
|
60
|
+
print(f"Sessions: {len(sessions)}")
|
|
61
|
+
print(f"Tool calls: {total}")
|
|
62
|
+
print(f"Tool failures: {len(failures)} ({fail_rate:.1f}%)")
|
|
63
|
+
|
|
64
|
+
per_session = sorted(len(v) for v in sessions.values())
|
|
65
|
+
mid = per_session[len(per_session) // 2]
|
|
66
|
+
print(f"Calls/session (median): {mid} (max {per_session[-1]})")
|
|
67
|
+
|
|
68
|
+
print("\n─ Top tools by usage ─")
|
|
69
|
+
for tool, n in Counter(e.get("tool", "?") for e in rows).most_common(8):
|
|
70
|
+
f = sum(1 for e in failures if e.get("tool") == tool)
|
|
71
|
+
mark = f" (failures {f})" if f else ""
|
|
72
|
+
print(f" {tool:<28}{n:>5}{mark}")
|
|
73
|
+
|
|
74
|
+
if failures:
|
|
75
|
+
print("\n─ Top repeated failure signatures (ratchet candidates) ─")
|
|
76
|
+
sigs = Counter((e.get("tool", "?"), (e.get("error") or "")[:70]) for e in failures)
|
|
77
|
+
for (tool, err), n in sigs.most_common(5):
|
|
78
|
+
if n >= 2:
|
|
79
|
+
print(f" [{n}x] {tool}: {err}")
|
|
80
|
+
print("\n -> For failures repeated 2+ times, consider /ratchet to turn them into rules.")
|
|
81
|
+
|
|
82
|
+
print("\n─ How to read this ─")
|
|
83
|
+
print(" · Is the failure rate trending down week over week?")
|
|
84
|
+
print(" · The same failure signature next week means the ratchet isn't working")
|
|
85
|
+
print(" · The growth rate of CLAUDE.md rules should be declining (a maturity signal)")
|
|
86
|
+
return 0
|
|
87
|
+
|
|
88
|
+
|
|
89
|
+
if __name__ == "__main__":
|
|
90
|
+
sys.exit(main())
|
|
@@ -0,0 +1,129 @@
|
|
|
1
|
+
#!/usr/bin/env bash
|
|
2
|
+
# ════════════════════════════════════════════════════════════════
|
|
3
|
+
# Harness hook self-tests
|
|
4
|
+
# Run: bash harness/tests/test_hooks.sh
|
|
5
|
+
# Feeds sample JSON into each hook and asserts the expected behavior
|
|
6
|
+
# (block / inject / silence). "Pass silently when unconfigured" is
|
|
7
|
+
# part of the hook contract, so that path is tested too.
|
|
8
|
+
# ════════════════════════════════════════════════════════════════
|
|
9
|
+
set -u
|
|
10
|
+
|
|
11
|
+
ROOT="$(cd "$(dirname "$0")/../.." && pwd)"
|
|
12
|
+
HOOKS="$ROOT/.claude/hooks"
|
|
13
|
+
PASS=0
|
|
14
|
+
FAIL=0
|
|
15
|
+
|
|
16
|
+
# check <name> <expected substring|EMPTY> <actual output>
|
|
17
|
+
check() {
|
|
18
|
+
local name="$1" expect="$2" actual="$3"
|
|
19
|
+
if [ "$expect" = "EMPTY" ]; then
|
|
20
|
+
if [ -z "$actual" ]; then PASS=$((PASS+1)); echo " ok: $name"
|
|
21
|
+
else FAIL=$((FAIL+1)); echo " FAIL: $name — expected no output, got: ${actual:0:120}"; fi
|
|
22
|
+
else
|
|
23
|
+
if printf '%s' "$actual" | grep -qF "$expect"; then PASS=$((PASS+1)); echo " ok: $name"
|
|
24
|
+
else FAIL=$((FAIL+1)); echo " FAIL: $name — missing '$expect': ${actual:0:160}"; fi
|
|
25
|
+
fi
|
|
26
|
+
}
|
|
27
|
+
|
|
28
|
+
fresh_project() {
|
|
29
|
+
TMP=$(mktemp -d)
|
|
30
|
+
export CLAUDE_PROJECT_DIR="$TMP"
|
|
31
|
+
mkdir -p "$TMP/.harness/logs"
|
|
32
|
+
}
|
|
33
|
+
|
|
34
|
+
# ── guard-pre-bash.sh ────────────────────────────────────────────
|
|
35
|
+
echo "guard-pre-bash.sh"
|
|
36
|
+
fresh_project
|
|
37
|
+
OUT=$(echo '{"tool_name":"Bash","tool_input":{"command":"rm -rf /"}}' | bash "$HOOKS/guard-pre-bash.sh")
|
|
38
|
+
check "blocks recursive root delete" '"permissionDecision": "deny"' "$OUT"
|
|
39
|
+
OUT=$(echo '{"tool_name":"Bash","tool_input":{"command":"git commit -m x --no-verify"}}' | bash "$HOOKS/guard-pre-bash.sh")
|
|
40
|
+
check "blocks --no-verify" "no-verify" "$OUT"
|
|
41
|
+
OUT=$(echo '{"tool_name":"Bash","tool_input":{"command":"git push --force origin main"}}' | bash "$HOOKS/guard-pre-bash.sh")
|
|
42
|
+
check "blocks force push" '"permissionDecision": "deny"' "$OUT"
|
|
43
|
+
OUT=$(echo '{"tool_name":"Bash","tool_input":{"command":"git push --force-with-lease origin main"}}' | bash "$HOOKS/guard-pre-bash.sh")
|
|
44
|
+
check "allows force-with-lease" "EMPTY" "$OUT"
|
|
45
|
+
OUT=$(echo '{"tool_name":"Bash","tool_input":{"command":"git status"}}' | bash "$HOOKS/guard-pre-bash.sh")
|
|
46
|
+
check "passes safe commands" "EMPTY" "$OUT"
|
|
47
|
+
|
|
48
|
+
# ── observe-log.sh ───────────────────────────────────────────────
|
|
49
|
+
echo "observe-log.sh"
|
|
50
|
+
fresh_project
|
|
51
|
+
echo '{"hook_event_name":"PostToolUse","session_id":"s1","tool_name":"Bash","tool_input":{"command":"ls"}}' | bash "$HOOKS/observe-log.sh" > /dev/null
|
|
52
|
+
LINES=$(wc -l < "$TMP/.harness/logs/tool_calls.jsonl" | tr -d ' ')
|
|
53
|
+
check "logs one call" "1" "$LINES"
|
|
54
|
+
echo '{"hook_event_name":"PostToolUse","session_id":"s1","tool_name":"Edit","tool_input":{"file_path":"a.py"},"agent_id":"x1","agent_type":"code-reviewer"}' | bash "$HOOKS/observe-log.sh" > /dev/null
|
|
55
|
+
check "records subagent fields" "code-reviewer" "$(tail -1 "$TMP/.harness/logs/tool_calls.jsonl")"
|
|
56
|
+
OUT=""
|
|
57
|
+
for _ in 1 2 3; do
|
|
58
|
+
OUT=$(echo '{"hook_event_name":"PostToolUseFailure","session_id":"s1","tool_name":"Bash","tool_input":{"command":"pytest"},"tool_response":"E: same error"}' | bash "$HOOKS/observe-log.sh")
|
|
59
|
+
done
|
|
60
|
+
check "tripwire on 3 identical failures" "The same tool failed" "$OUT"
|
|
61
|
+
|
|
62
|
+
# ── sensor-post-edit.sh ──────────────────────────────────────────
|
|
63
|
+
echo "sensor-post-edit.sh"
|
|
64
|
+
fresh_project
|
|
65
|
+
OUT=$(echo '{"session_id":"s1","tool_input":{"file_path":"a.py"}}' | bash "$HOOKS/sensor-post-edit.sh")
|
|
66
|
+
check "silent without commands.env" "EMPTY" "$OUT"
|
|
67
|
+
printf 'LINT_CMD="false"\nTEST_CMD=""\n' > "$TMP/.harness/commands.env"
|
|
68
|
+
OUT=$(echo '{"session_id":"s1","tool_input":{"file_path":"README.md"}}' | bash "$HOOKS/sensor-post-edit.sh")
|
|
69
|
+
check "passes doc files" "EMPTY" "$OUT"
|
|
70
|
+
OUT=$(echo '{"session_id":"s1","tool_input":{"file_path":"a.py"}}' | bash "$HOOKS/sensor-post-edit.sh")
|
|
71
|
+
check "feeds back lint failure" "LINT FAILED" "$OUT"
|
|
72
|
+
check "records pending_test marker" "a.py" "$(cat "$TMP/.harness/logs/pending_test.s1" 2>/dev/null)"
|
|
73
|
+
printf 'LINT_CMD="true"\nTEST_CMD=""\n' > "$TMP/.harness/commands.env"
|
|
74
|
+
OUT=$(echo '{"session_id":"s1","tool_input":{"file_path":"a.py"}}' | bash "$HOOKS/sensor-post-edit.sh")
|
|
75
|
+
check "silent when lint passes" "EMPTY" "$OUT"
|
|
76
|
+
|
|
77
|
+
# ── stop-gate.sh ─────────────────────────────────────────────────
|
|
78
|
+
echo "stop-gate.sh"
|
|
79
|
+
fresh_project
|
|
80
|
+
printf 'LINT_CMD=""\nTEST_CMD="false"\n' > "$TMP/.harness/commands.env"
|
|
81
|
+
OUT=$(echo '{"session_id":"s1"}' | bash "$HOOKS/stop-gate.sh")
|
|
82
|
+
check "passes sessions with no edits" "EMPTY" "$OUT"
|
|
83
|
+
echo "a.py" > "$TMP/.harness/logs/pending_test.s1"
|
|
84
|
+
OUT=$(echo '{"session_id":"s1"}' | bash "$HOOKS/stop-gate.sh")
|
|
85
|
+
check "blocks stop on failing tests" '"decision": "block"' "$OUT"
|
|
86
|
+
check "shows block count" "1/3" "$OUT"
|
|
87
|
+
OUT=$(echo '{"session_id":"s1"}' | bash "$HOOKS/stop-gate.sh")
|
|
88
|
+
check "second block" "2/3" "$OUT"
|
|
89
|
+
OUT=$(echo '{"session_id":"s1"}' | bash "$HOOKS/stop-gate.sh")
|
|
90
|
+
check "third block demands escalation" "escalation packet" "$OUT"
|
|
91
|
+
OUT=$(echo '{"session_id":"s1"}' | bash "$HOOKS/stop-gate.sh")
|
|
92
|
+
check "passes after cap clears marker" "EMPTY" "$OUT"
|
|
93
|
+
echo "a.py" > "$TMP/.harness/logs/pending_test.s1"
|
|
94
|
+
printf 'LINT_CMD=""\nTEST_CMD="true"\n' > "$TMP/.harness/commands.env"
|
|
95
|
+
OUT=$(echo '{"session_id":"s1"}' | bash "$HOOKS/stop-gate.sh")
|
|
96
|
+
check "silent when tests pass" "EMPTY" "$OUT"
|
|
97
|
+
[ -f "$TMP/.harness/logs/pending_test.s1" ] && MARKER_LEFT="yes" || MARKER_LEFT=""
|
|
98
|
+
check "clears marker on pass" "EMPTY" "$MARKER_LEFT"
|
|
99
|
+
|
|
100
|
+
# ── session-start.sh ─────────────────────────────────────────────
|
|
101
|
+
echo "session-start.sh"
|
|
102
|
+
fresh_project
|
|
103
|
+
OUT=$(bash "$HOOKS/session-start.sh")
|
|
104
|
+
check "silent without checkpoint" "EMPTY" "$OUT"
|
|
105
|
+
echo '{"task_id":"t1","status":"in_progress","next_step":"do X"}' > "$TMP/progress.json"
|
|
106
|
+
OUT=$(bash "$HOOKS/session-start.sh")
|
|
107
|
+
check "injects in-progress checkpoint" "progress.json" "$OUT"
|
|
108
|
+
echo '{"task_id":"t1","status":"completed"}' > "$TMP/progress.json"
|
|
109
|
+
OUT=$(bash "$HOOKS/session-start.sh")
|
|
110
|
+
check "skips completed checkpoint" "EMPTY" "$OUT"
|
|
111
|
+
rm "$TMP/progress.json"
|
|
112
|
+
NOW=$(python3 -c 'import datetime; print(datetime.datetime.now().isoformat(timespec="seconds"))')
|
|
113
|
+
for _ in 1 2 3; do
|
|
114
|
+
echo "{\"ts\":\"$NOW\",\"session_id\":\"old\",\"event\":\"PostToolUseFailure\",\"tool\":\"Bash\",\"error\":\"x\"}" >> "$TMP/.harness/logs/tool_calls.jsonl"
|
|
115
|
+
done
|
|
116
|
+
OUT=$(bash "$HOOKS/session-start.sh")
|
|
117
|
+
check "nudges /ratchet on accumulated failures" "/ratchet" "$OUT"
|
|
118
|
+
fresh_project
|
|
119
|
+
printf 'LINT_CMD=""\nTEST_CMD=""\n' > "$TMP/.harness/commands.env"
|
|
120
|
+
OUT=$(bash "$HOOKS/session-start.sh")
|
|
121
|
+
check "offers /harness-init when unconfigured" "/harness-init" "$OUT"
|
|
122
|
+
printf 'LINT_CMD="true"\nTEST_CMD=""\n' > "$TMP/.harness/commands.env"
|
|
123
|
+
OUT=$(bash "$HOOKS/session-start.sh")
|
|
124
|
+
check "no setup nudge once configured" "EMPTY" "$OUT"
|
|
125
|
+
|
|
126
|
+
# ── result ───────────────────────────────────────────────────────
|
|
127
|
+
echo
|
|
128
|
+
echo "passed: $PASS, failed: $FAIL"
|
|
129
|
+
[ "$FAIL" -eq 0 ] || exit 1
|