uni-harness 0.2.2 → 0.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.claude/hooks/guard-pre-bash.sh +39 -4
- package/.claude/hooks/guard-pre-edit.sh +122 -0
- package/.claude/hooks/observe-log.sh +21 -10
- package/.claude/hooks/pre-compact.sh +53 -0
- package/.claude/hooks/session-end.sh +103 -0
- package/.claude/hooks/session-start.sh +27 -2
- package/.claude/hooks/stop-gate.sh +84 -0
- package/.claude/settings.json +33 -0
- package/.claude/skills/distill/SKILL.md +49 -0
- package/.claude/skills/guide-audit/SKILL.md +14 -0
- package/.claude/skills/harness-init/SKILL.md +5 -1
- package/.claude/skills/ratchet/SKILL.md +31 -0
- package/AGENTS.md +26 -0
- package/README.ja.md +13 -5
- package/README.ko.md +13 -5
- package/README.md +14 -5
- package/README.zh-CN.md +13 -5
- package/bin/cli.js +5 -2
- package/harness/harness_report.py +16 -1
- package/harness/tests/test_hooks.sh +112 -14
- package/harness/tests/test_installer.sh +17 -0
- package/package.json +2 -1
|
@@ -1,17 +1,22 @@
|
|
|
1
1
|
#!/usr/bin/env bash
|
|
2
2
|
# ════════════════════════════════════════════════════════════════
|
|
3
3
|
# PreToolUse(Bash) guard — blocks destructive/irreversible commands
|
|
4
|
-
# before they run
|
|
5
|
-
#
|
|
4
|
+
# before they run, and asks before a git commit that would land edits
|
|
5
|
+
# no test run has seen (state-aware gate: the pending_test marker is
|
|
6
|
+
# set by sensor-post-edit.sh and cleared only when stop-gate.sh sees
|
|
7
|
+
# the tests pass).
|
|
8
|
+
# exit 0 + JSON(deny/ask) = intercepted / exit 0 (no output) = normal flow
|
|
6
9
|
# ════════════════════════════════════════════════════════════════
|
|
7
10
|
set -euo pipefail
|
|
8
11
|
|
|
9
12
|
INPUT=$(cat)
|
|
13
|
+
PROJECT_DIR="${CLAUDE_PROJECT_DIR:-$(pwd)}"
|
|
10
14
|
|
|
11
|
-
python3 - "$INPUT" <<'PYEOF'
|
|
12
|
-
import json, re, sys
|
|
15
|
+
python3 - "$INPUT" "$PROJECT_DIR" <<'PYEOF'
|
|
16
|
+
import json, os, re, sys
|
|
13
17
|
|
|
14
18
|
data = json.loads(sys.argv[1])
|
|
19
|
+
proj = sys.argv[2]
|
|
15
20
|
cmd = (data.get("tool_input") or {}).get("command", "") or ""
|
|
16
21
|
|
|
17
22
|
# DENY: irreversible/destructive patterns (extend per project as needed)
|
|
@@ -41,6 +46,36 @@ for pattern, reason in DENY_PATTERNS:
|
|
|
41
46
|
}))
|
|
42
47
|
sys.exit(0)
|
|
43
48
|
|
|
49
|
+
# State-aware commit gate: committing while the pending_test marker
|
|
50
|
+
# exists means tests have not passed since the last code edit. "ask",
|
|
51
|
+
# not "deny" — WIP commits and doc-only commits are legitimate, so the
|
|
52
|
+
# user decides. Fires only when TEST_CMD is configured (otherwise the
|
|
53
|
+
# marker never clears and this would nag on every commit).
|
|
54
|
+
try:
|
|
55
|
+
sid = data.get("session_id", "")
|
|
56
|
+
marker = os.path.join(proj, ".harness/logs", f"pending_test.{sid}")
|
|
57
|
+
env_path = os.path.join(proj, ".harness/commands.env")
|
|
58
|
+
configured = False
|
|
59
|
+
if os.path.exists(env_path):
|
|
60
|
+
with open(env_path) as f:
|
|
61
|
+
configured = bool(re.search(r'^TEST_CMD=".+"', f.read(), re.M))
|
|
62
|
+
if (configured and sid and os.path.exists(marker)
|
|
63
|
+
and re.search(r"\bgit\s+([a-z-]+\s+)*commit\b", cmd)):
|
|
64
|
+
print(json.dumps({
|
|
65
|
+
"hookSpecificOutput": {
|
|
66
|
+
"hookEventName": "PreToolUse",
|
|
67
|
+
"permissionDecision": "ask",
|
|
68
|
+
"permissionDecisionReason":
|
|
69
|
+
"[harness guard] Code was edited this session but the "
|
|
70
|
+
"test suite has not passed since (pending_test marker "
|
|
71
|
+
"present). Run the TEST command first, or confirm this "
|
|
72
|
+
"commit is intentionally untested (WIP/docs)."
|
|
73
|
+
}
|
|
74
|
+
}))
|
|
75
|
+
sys.exit(0)
|
|
76
|
+
except Exception:
|
|
77
|
+
pass
|
|
78
|
+
|
|
44
79
|
# No match -> exit silently (normal permission flow)
|
|
45
80
|
sys.exit(0)
|
|
46
81
|
PYEOF
|
|
@@ -0,0 +1,122 @@
|
|
|
1
|
+
#!/usr/bin/env bash
|
|
2
|
+
# ════════════════════════════════════════════════════════════════
|
|
3
|
+
# PreToolUse(Edit|Write) guard — two checks, both "ask" not "deny":
|
|
4
|
+
# (1) protected paths: .claude/**, .harness/** (except state/),
|
|
5
|
+
# .github/**, .env*, *.config.* — the CLAUDE.md rule "never modify
|
|
6
|
+
# config/harness files without asking", promoted to a gate.
|
|
7
|
+
# (2) test-weakening: edits to test files that add skip markers,
|
|
8
|
+
# remove test functions, or gut assertions. The #1 reward hack is
|
|
9
|
+
# making tests pass by making tests weaker — surface it to the
|
|
10
|
+
# user instead of letting it slide through.
|
|
11
|
+
# Precision-first: fire only on unambiguous signals; fail silent on
|
|
12
|
+
# any parse error.
|
|
13
|
+
# ════════════════════════════════════════════════════════════════
|
|
14
|
+
set -uo pipefail
|
|
15
|
+
|
|
16
|
+
INPUT=$(cat)
|
|
17
|
+
PROJECT_DIR="${CLAUDE_PROJECT_DIR:-$(pwd)}"
|
|
18
|
+
|
|
19
|
+
python3 - "$INPUT" "$PROJECT_DIR" <<'PYEOF'
|
|
20
|
+
import json, sys, os, re
|
|
21
|
+
|
|
22
|
+
try:
|
|
23
|
+
data = json.loads(sys.argv[1])
|
|
24
|
+
except Exception:
|
|
25
|
+
sys.exit(0)
|
|
26
|
+
proj = os.path.realpath(sys.argv[2])
|
|
27
|
+
|
|
28
|
+
ti = data.get("tool_input") or {}
|
|
29
|
+
path = str(ti.get("file_path") or "")
|
|
30
|
+
if not path:
|
|
31
|
+
sys.exit(0)
|
|
32
|
+
|
|
33
|
+
# Only police files inside the project
|
|
34
|
+
rp = os.path.realpath(path if os.path.isabs(path) else os.path.join(proj, path))
|
|
35
|
+
if not (rp + os.sep).startswith(proj + os.sep):
|
|
36
|
+
sys.exit(0)
|
|
37
|
+
rel = os.path.relpath(rp, proj).replace(os.sep, "/")
|
|
38
|
+
base = os.path.basename(rel)
|
|
39
|
+
|
|
40
|
+
|
|
41
|
+
def ask(reason):
|
|
42
|
+
print(json.dumps({"hookSpecificOutput": {
|
|
43
|
+
"hookEventName": "PreToolUse",
|
|
44
|
+
"permissionDecision": "ask",
|
|
45
|
+
"permissionDecisionReason": "[harness guard] " + reason,
|
|
46
|
+
}}, ensure_ascii=False))
|
|
47
|
+
sys.exit(0)
|
|
48
|
+
|
|
49
|
+
|
|
50
|
+
# ── (1) protected paths ──────────────────────────────────────────
|
|
51
|
+
# .harness/state/ is the checkpoint area Claude legitimately writes.
|
|
52
|
+
if not rel.startswith(".harness/state/"):
|
|
53
|
+
protected = None
|
|
54
|
+
if rel.startswith(".claude/"):
|
|
55
|
+
protected = "the harness machinery (.claude/)"
|
|
56
|
+
elif rel.startswith(".harness/"):
|
|
57
|
+
protected = "the harness configuration (.harness/)"
|
|
58
|
+
elif rel.startswith(".github/"):
|
|
59
|
+
protected = "CI configuration (.github/)"
|
|
60
|
+
elif base == ".env" or base.startswith(".env."):
|
|
61
|
+
protected = "an environment file"
|
|
62
|
+
elif ".config." in base:
|
|
63
|
+
protected = "a tool configuration file"
|
|
64
|
+
if protected:
|
|
65
|
+
ask(f"'{rel}' is {protected}. Per the project rules, changes here "
|
|
66
|
+
"need explicit user approval — especially anything that would "
|
|
67
|
+
"weaken tests, lint, or guards. State what you want to change "
|
|
68
|
+
"and why.")
|
|
69
|
+
|
|
70
|
+
# ── (2) test-weakening detection ─────────────────────────────────
|
|
71
|
+
segs = rel.lower().split("/")
|
|
72
|
+
is_test = (
|
|
73
|
+
any(s in ("test", "tests", "__tests__", "spec", "specs") for s in segs[:-1])
|
|
74
|
+
or re.search(r"(^|[._-])(test|spec)s?[._-]", base.lower())
|
|
75
|
+
)
|
|
76
|
+
if not is_test:
|
|
77
|
+
sys.exit(0)
|
|
78
|
+
|
|
79
|
+
tool = data.get("tool_name", "")
|
|
80
|
+
if tool == "Write":
|
|
81
|
+
try:
|
|
82
|
+
with open(rp) as f:
|
|
83
|
+
old = f.read()
|
|
84
|
+
except Exception:
|
|
85
|
+
sys.exit(0) # brand-new test file: nothing to weaken yet
|
|
86
|
+
new = str(ti.get("content") or "")
|
|
87
|
+
elif tool in ("Edit", "MultiEdit"):
|
|
88
|
+
edits = ti.get("edits") or [ti]
|
|
89
|
+
old = "\n".join(str(e.get("old_string") or "") for e in edits)
|
|
90
|
+
new = "\n".join(str(e.get("new_string") or "") for e in edits)
|
|
91
|
+
else:
|
|
92
|
+
sys.exit(0)
|
|
93
|
+
|
|
94
|
+
SKIP_MARKERS = [
|
|
95
|
+
r"@pytest\.mark\.skip", r"@pytest\.mark\.xfail", r"pytest\.skip\(",
|
|
96
|
+
r"@unittest\.skip", r"\.only\(", r"\.skip\(", r"\bxit\(", r"\bxdescribe\(",
|
|
97
|
+
r"\bxtest\(", r"it\.skip", r"describe\.skip", r"test\.skip",
|
|
98
|
+
r"@Disabled", r"t\.Skip\(", r"#\[ignore\]",
|
|
99
|
+
]
|
|
100
|
+
GUT_MARKERS = [r"assert\s+True\b", r"expect\(true\)\.toBe\(true\)", r"assert\s+1\s*==\s*1\b"]
|
|
101
|
+
TEST_FN = re.compile(r"^\s*(def test_|func Test|it\(|test\(|Deno\.test\()", re.M)
|
|
102
|
+
|
|
103
|
+
findings = []
|
|
104
|
+
for pat in SKIP_MARKERS:
|
|
105
|
+
if len(re.findall(pat, new)) > len(re.findall(pat, old)):
|
|
106
|
+
findings.append(f"adds a skip/only marker ({pat.replace(chr(92), '')})")
|
|
107
|
+
break
|
|
108
|
+
for pat in GUT_MARKERS:
|
|
109
|
+
if len(re.findall(pat, new)) > len(re.findall(pat, old)):
|
|
110
|
+
findings.append("adds a trivially-true assertion")
|
|
111
|
+
break
|
|
112
|
+
n_old, n_new = len(TEST_FN.findall(old)), len(TEST_FN.findall(new))
|
|
113
|
+
if n_new < n_old:
|
|
114
|
+
findings.append(f"removes {n_old - n_new} test function(s)")
|
|
115
|
+
|
|
116
|
+
if findings:
|
|
117
|
+
ask(f"this change to test file '{rel}' " + " and ".join(findings) +
|
|
118
|
+
". Weakening tests to make them pass is never an acceptable fix. "
|
|
119
|
+
"If the test itself is genuinely wrong or obsolete, explain that to "
|
|
120
|
+
"the user and get approval first.")
|
|
121
|
+
PYEOF
|
|
122
|
+
exit 0
|
|
@@ -3,7 +3,8 @@
|
|
|
3
3
|
# Observability log — records every tool call as JSONL + tripwires
|
|
4
4
|
# Log: .harness/logs/tool_calls.jsonl
|
|
5
5
|
# Tripwires: (1) warn when per-session tool calls exceed a ceiling
|
|
6
|
-
# (2) same
|
|
6
|
+
# (2) same normalized error 3x in a session -> demand an
|
|
7
|
+
# escalation packet instead of another retry
|
|
7
8
|
# ════════════════════════════════════════════════════════════════
|
|
8
9
|
set -uo pipefail
|
|
9
10
|
|
|
@@ -67,15 +68,25 @@ if n > 0 and n % 100 == 0 and n >= 300:
|
|
|
67
68
|
warnings.append(f"This session has exceeded {n} tool calls. "
|
|
68
69
|
"Re-examine the plan — this may be a runaway loop.")
|
|
69
70
|
|
|
70
|
-
# (2) same
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
|
|
71
|
+
# (2) same error 3x this session (normalized, not necessarily consecutive)
|
|
72
|
+
if event == "PostToolUseFailure":
|
|
73
|
+
import re
|
|
74
|
+
def norm(e):
|
|
75
|
+
s = (e.get("error") or "")[:300]
|
|
76
|
+
s = re.sub(r"0x[0-9a-fA-F]+", "N", s) # addresses/ids
|
|
77
|
+
s = re.sub(r"\d+", "N", s) # line numbers, counts
|
|
78
|
+
s = re.sub(r"(/[^\s:'\"]+)+", "PATH", s) # file paths
|
|
79
|
+
return (e.get("tool", ""), s[:120])
|
|
80
|
+
sig = norm(entry)
|
|
81
|
+
count = sum(1 for e in lines
|
|
82
|
+
if e.get("event") == "PostToolUseFailure" and norm(e) == sig)
|
|
83
|
+
if count >= 3 and count % 3 == 0:
|
|
84
|
+
warnings.append(f"[tripwire] The same error has now occurred {count} "
|
|
85
|
+
f"times this session (tool: {entry.get('tool', '')}). "
|
|
86
|
+
"Stop repeating this approach. Write an escalation "
|
|
87
|
+
"packet per the Work Loop section of CLAUDE.md "
|
|
88
|
+
"(decision needed / alternatives tried / cost of "
|
|
89
|
+
"waiting / safest default) and report to the user.")
|
|
79
90
|
|
|
80
91
|
if warnings:
|
|
81
92
|
print(json.dumps({
|
|
@@ -0,0 +1,53 @@
|
|
|
1
|
+
#!/usr/bin/env bash
|
|
2
|
+
# ════════════════════════════════════════════════════════════════
|
|
3
|
+
# PreCompact — runs just before context compaction (auto or /compact).
|
|
4
|
+
# Two jobs:
|
|
5
|
+
# (1) snapshot .harness/state/ so whatever checkpoint exists at this
|
|
6
|
+
# moment survives even if the post-compaction session corrupts it
|
|
7
|
+
# (2) record the PreCompact event in the log.
|
|
8
|
+
# Context re-injection after compaction is session-start.sh's job
|
|
9
|
+
# (matcher "compact") — PreCompact hook output supports no
|
|
10
|
+
# hookSpecificOutput/additionalContext channel, only snapshot work.
|
|
11
|
+
# Fail silent: an unconfigured or half-installed harness must never
|
|
12
|
+
# break compaction.
|
|
13
|
+
# ════════════════════════════════════════════════════════════════
|
|
14
|
+
set -uo pipefail
|
|
15
|
+
|
|
16
|
+
INPUT=$(cat)
|
|
17
|
+
PROJECT_DIR="${CLAUDE_PROJECT_DIR:-$(pwd)}"
|
|
18
|
+
STATE_DIR="$PROJECT_DIR/.harness/state"
|
|
19
|
+
|
|
20
|
+
# (1) snapshot current checkpoints (best effort, keep only the latest)
|
|
21
|
+
if [ -d "$STATE_DIR" ]; then
|
|
22
|
+
mkdir -p "$STATE_DIR/.pre-compact-backup" 2>/dev/null || true
|
|
23
|
+
for f in plan.md progress.json decisions.jsonl; do
|
|
24
|
+
[ -f "$STATE_DIR/$f" ] && cp "$STATE_DIR/$f" "$STATE_DIR/.pre-compact-backup/$f" 2>/dev/null
|
|
25
|
+
done
|
|
26
|
+
fi
|
|
27
|
+
|
|
28
|
+
# (2) note the event in the log (no JSON output — the PreCompact hook
|
|
29
|
+
# schema accepts no context injection; emitting one fails validation)
|
|
30
|
+
python3 - "$INPUT" "$PROJECT_DIR" <<'PYEOF'
|
|
31
|
+
import json, sys, os, datetime
|
|
32
|
+
|
|
33
|
+
try:
|
|
34
|
+
data = json.loads(sys.argv[1])
|
|
35
|
+
except Exception:
|
|
36
|
+
data = {}
|
|
37
|
+
proj = sys.argv[2]
|
|
38
|
+
|
|
39
|
+
try:
|
|
40
|
+
log_dir = os.path.join(proj, ".harness/logs")
|
|
41
|
+
os.makedirs(log_dir, exist_ok=True)
|
|
42
|
+
with open(os.path.join(log_dir, "tool_calls.jsonl"), "a") as f:
|
|
43
|
+
f.write(json.dumps({
|
|
44
|
+
"ts": datetime.datetime.now().isoformat(timespec="seconds"),
|
|
45
|
+
"session_id": data.get("session_id", ""),
|
|
46
|
+
"event": "PreCompact",
|
|
47
|
+
"tool": "",
|
|
48
|
+
"summary": data.get("trigger", ""),
|
|
49
|
+
}, ensure_ascii=False) + "\n")
|
|
50
|
+
except Exception:
|
|
51
|
+
pass
|
|
52
|
+
PYEOF
|
|
53
|
+
exit 0
|
|
@@ -0,0 +1,103 @@
|
|
|
1
|
+
#!/usr/bin/env bash
|
|
2
|
+
# ════════════════════════════════════════════════════════════════
|
|
3
|
+
# SessionEnd ledger — one JSONL row per session.
|
|
4
|
+
# Aggregates this session's entries from tool_calls.jsonl into
|
|
5
|
+
# .harness/logs/sessions.jsonl: duration, call/failure/edit counts,
|
|
6
|
+
# stop blocks, and whether edits ended with tests in a clean state.
|
|
7
|
+
# This is the data /guide-audit and harness_report.py read instead of
|
|
8
|
+
# guessing — per-session rollups beat re-scanning the raw log.
|
|
9
|
+
# Also clears this session's marker files (pending_test/stop_blocks)
|
|
10
|
+
# so stale markers never leak into the next session.
|
|
11
|
+
# Fail silent: no logs, no harness, half-installed — always exit 0.
|
|
12
|
+
# ════════════════════════════════════════════════════════════════
|
|
13
|
+
set -uo pipefail
|
|
14
|
+
|
|
15
|
+
INPUT=$(cat)
|
|
16
|
+
PROJECT_DIR="${CLAUDE_PROJECT_DIR:-$(pwd)}"
|
|
17
|
+
|
|
18
|
+
python3 - "$INPUT" "$PROJECT_DIR" <<'PYEOF'
|
|
19
|
+
import json, sys, os, datetime
|
|
20
|
+
|
|
21
|
+
try:
|
|
22
|
+
data = json.loads(sys.argv[1])
|
|
23
|
+
except Exception:
|
|
24
|
+
data = {}
|
|
25
|
+
proj = sys.argv[2]
|
|
26
|
+
sid = data.get("session_id", "")
|
|
27
|
+
if not sid:
|
|
28
|
+
raise SystemExit(0)
|
|
29
|
+
|
|
30
|
+
log_dir = os.path.join(proj, ".harness/logs")
|
|
31
|
+
log_path = os.path.join(log_dir, "tool_calls.jsonl")
|
|
32
|
+
|
|
33
|
+
calls = failures = edits = 0
|
|
34
|
+
first_ts = last_ts = None
|
|
35
|
+
try:
|
|
36
|
+
with open(log_path) as f:
|
|
37
|
+
for line in f:
|
|
38
|
+
try:
|
|
39
|
+
e = json.loads(line)
|
|
40
|
+
except Exception:
|
|
41
|
+
continue
|
|
42
|
+
if e.get("session_id") != sid:
|
|
43
|
+
continue
|
|
44
|
+
ts = e.get("ts", "")
|
|
45
|
+
first_ts = first_ts or ts
|
|
46
|
+
last_ts = ts or last_ts
|
|
47
|
+
ev = e.get("event", "")
|
|
48
|
+
if ev == "PostToolUse":
|
|
49
|
+
calls += 1
|
|
50
|
+
if e.get("tool") in ("Edit", "Write", "MultiEdit", "NotebookEdit"):
|
|
51
|
+
edits += 1
|
|
52
|
+
elif ev == "PostToolUseFailure":
|
|
53
|
+
calls += 1
|
|
54
|
+
failures += 1
|
|
55
|
+
except Exception:
|
|
56
|
+
raise SystemExit(0)
|
|
57
|
+
|
|
58
|
+
if calls == 0:
|
|
59
|
+
raise SystemExit(0) # nothing happened; no row
|
|
60
|
+
|
|
61
|
+
duration_s = None
|
|
62
|
+
try:
|
|
63
|
+
duration_s = int((datetime.datetime.fromisoformat(last_ts)
|
|
64
|
+
- datetime.datetime.fromisoformat(first_ts)).total_seconds())
|
|
65
|
+
except Exception:
|
|
66
|
+
pass
|
|
67
|
+
|
|
68
|
+
pending = os.path.join(log_dir, f"pending_test.{sid}")
|
|
69
|
+
blocks_f = os.path.join(log_dir, f"stop_blocks.{sid}")
|
|
70
|
+
stop_blocks = 0
|
|
71
|
+
try:
|
|
72
|
+
with open(blocks_f) as f:
|
|
73
|
+
stop_blocks = int(f.read().strip() or 0)
|
|
74
|
+
except Exception:
|
|
75
|
+
pass
|
|
76
|
+
|
|
77
|
+
row = {
|
|
78
|
+
"ts": datetime.datetime.now().isoformat(timespec="seconds"),
|
|
79
|
+
"session_id": sid,
|
|
80
|
+
"event": "SessionEnd",
|
|
81
|
+
"reason": data.get("reason", ""),
|
|
82
|
+
"duration_s": duration_s,
|
|
83
|
+
"calls": calls,
|
|
84
|
+
"failures": failures,
|
|
85
|
+
"edits": edits,
|
|
86
|
+
"stop_blocks": stop_blocks,
|
|
87
|
+
# edits happened but the stop-gate never saw them pass -> not clean
|
|
88
|
+
"tests_clean": not os.path.exists(pending),
|
|
89
|
+
}
|
|
90
|
+
try:
|
|
91
|
+
os.makedirs(log_dir, exist_ok=True)
|
|
92
|
+
with open(os.path.join(log_dir, "sessions.jsonl"), "a") as f:
|
|
93
|
+
f.write(json.dumps(row, ensure_ascii=False) + "\n")
|
|
94
|
+
except Exception:
|
|
95
|
+
pass
|
|
96
|
+
|
|
97
|
+
for p in (pending, blocks_f):
|
|
98
|
+
try:
|
|
99
|
+
os.remove(p)
|
|
100
|
+
except Exception:
|
|
101
|
+
pass
|
|
102
|
+
PYEOF
|
|
103
|
+
exit 0
|
|
@@ -9,17 +9,42 @@
|
|
|
9
9
|
# suggest running /ratchet.
|
|
10
10
|
# (3) If the harness is installed but unconfigured (commands.env has
|
|
11
11
|
# no commands), have Claude offer to run /harness-init.
|
|
12
|
+
# (5) Right after a compaction (source=compact), instruct a checkpoint
|
|
13
|
+
# refresh before resuming — pre-compact.sh snapshotted the state
|
|
14
|
+
# but cannot inject context itself (schema limitation).
|
|
12
15
|
# ════════════════════════════════════════════════════════════════
|
|
13
16
|
set -uo pipefail
|
|
14
17
|
|
|
18
|
+
INPUT=$(cat)
|
|
15
19
|
PROJECT_DIR="${CLAUDE_PROJECT_DIR:-$(pwd)}"
|
|
16
20
|
|
|
17
|
-
python3 - "$PROJECT_DIR" <<'PYEOF'
|
|
21
|
+
python3 - "$INPUT" "$PROJECT_DIR" <<'PYEOF'
|
|
18
22
|
import json, sys, os, datetime
|
|
19
23
|
|
|
20
|
-
|
|
24
|
+
try:
|
|
25
|
+
hook_in = json.loads(sys.argv[1])
|
|
26
|
+
except Exception:
|
|
27
|
+
hook_in = {}
|
|
28
|
+
proj = sys.argv[2]
|
|
21
29
|
parts = []
|
|
22
30
|
|
|
31
|
+
# (5) resuming right after compaction -> refresh checkpoint first
|
|
32
|
+
try:
|
|
33
|
+
if hook_in.get("source") == "compact":
|
|
34
|
+
backup = os.path.join(proj, ".harness/state/.pre-compact-backup")
|
|
35
|
+
note = ("[harness] Context was just compacted. Before resuming work, "
|
|
36
|
+
"verify the summary against .harness/state/ (plan.md / "
|
|
37
|
+
"progress.json / decisions.jsonl) and update progress.json "
|
|
38
|
+
"(/checkpoint) so the next step is recorded outside the "
|
|
39
|
+
"conversation.")
|
|
40
|
+
if os.path.isdir(backup):
|
|
41
|
+
note += (" A snapshot of the pre-compaction checkpoint was saved "
|
|
42
|
+
"to .harness/state/.pre-compact-backup/ — consult it if "
|
|
43
|
+
"anything seems lost.")
|
|
44
|
+
parts.append(note)
|
|
45
|
+
except Exception:
|
|
46
|
+
pass
|
|
47
|
+
|
|
23
48
|
# (3) harness installed but not configured -> offer /harness-init
|
|
24
49
|
# (checked first so a fresh install greets the user with setup)
|
|
25
50
|
try:
|
|
@@ -6,6 +6,16 @@
|
|
|
6
6
|
# stop and feed the output back. Blocking is capped at 3 per
|
|
7
7
|
# session — after that, instruct an escalation packet and let the
|
|
8
8
|
# stop through (prevents infinite loops).
|
|
9
|
+
#
|
|
10
|
+
# Trace rules v1 (.harness/trace.rules, optional): lines of
|
|
11
|
+
# require-before-stop <regex> <message>
|
|
12
|
+
# <regex> must contain no whitespace (use . or \s inside it) — the
|
|
13
|
+
# line is whitespace-split into keyword / regex / message.
|
|
14
|
+
# In a session that edited code, each rule requires at least one
|
|
15
|
+
# logged tool call matching <regex> before the turn may end.
|
|
16
|
+
# An unsatisfied rule blocks the stop ONCE per session (a reminder,
|
|
17
|
+
# not a cage — the marker in trace_notified.<sid> prevents loops).
|
|
18
|
+
# File absent -> feature entirely off (fail silent).
|
|
9
19
|
# ════════════════════════════════════════════════════════════════
|
|
10
20
|
set -uo pipefail
|
|
11
21
|
|
|
@@ -22,6 +32,80 @@ print(json.loads(sys.argv[1]).get("session_id",""))' "$INPUT" 2>/dev/null || ech
|
|
|
22
32
|
MARKER="$LOG_DIR/pending_test.$SESSION_ID"
|
|
23
33
|
COUNTER="$LOG_DIR/stop_blocks.$SESSION_ID"
|
|
24
34
|
|
|
35
|
+
# ── trace rules: require-before-stop ─────────────────────────────
|
|
36
|
+
TRACE_FILE="$PROJECT_DIR/.harness/trace.rules"
|
|
37
|
+
if [ -f "$TRACE_FILE" ] && [ -s "$MARKER" ]; then
|
|
38
|
+
TRACE_OUT=$(python3 - "$SESSION_ID" "$PROJECT_DIR" <<'PYEOF'
|
|
39
|
+
import json, os, re, sys
|
|
40
|
+
|
|
41
|
+
sid, proj = sys.argv[1], sys.argv[2]
|
|
42
|
+
log_dir = os.path.join(proj, ".harness/logs")
|
|
43
|
+
notified_path = os.path.join(log_dir, f"trace_notified.{sid}")
|
|
44
|
+
|
|
45
|
+
try:
|
|
46
|
+
with open(os.path.join(proj, ".harness/trace.rules")) as f:
|
|
47
|
+
raw_rules = f.read().splitlines()
|
|
48
|
+
except Exception:
|
|
49
|
+
raise SystemExit(0)
|
|
50
|
+
|
|
51
|
+
rules = []
|
|
52
|
+
for line in raw_rules:
|
|
53
|
+
line = line.strip()
|
|
54
|
+
if not line or line.startswith("#"):
|
|
55
|
+
continue
|
|
56
|
+
parts = line.split(None, 2)
|
|
57
|
+
if len(parts) == 3 and parts[0] == "require-before-stop":
|
|
58
|
+
rules.append((parts[1], parts[2]))
|
|
59
|
+
if not rules:
|
|
60
|
+
raise SystemExit(0)
|
|
61
|
+
|
|
62
|
+
session_lines = []
|
|
63
|
+
try:
|
|
64
|
+
with open(os.path.join(log_dir, "tool_calls.jsonl")) as f:
|
|
65
|
+
for line in f:
|
|
66
|
+
try:
|
|
67
|
+
if json.loads(line).get("session_id") == sid:
|
|
68
|
+
session_lines.append(line)
|
|
69
|
+
except Exception:
|
|
70
|
+
continue
|
|
71
|
+
except Exception:
|
|
72
|
+
raise SystemExit(0)
|
|
73
|
+
|
|
74
|
+
notified = set()
|
|
75
|
+
try:
|
|
76
|
+
with open(notified_path) as f:
|
|
77
|
+
notified = set(f.read().splitlines())
|
|
78
|
+
except Exception:
|
|
79
|
+
pass
|
|
80
|
+
|
|
81
|
+
for regex, message in rules:
|
|
82
|
+
if regex in notified:
|
|
83
|
+
continue
|
|
84
|
+
try:
|
|
85
|
+
pat = re.compile(regex)
|
|
86
|
+
except Exception:
|
|
87
|
+
continue # a broken regex must never block work
|
|
88
|
+
if any(pat.search(line) for line in session_lines):
|
|
89
|
+
continue
|
|
90
|
+
try:
|
|
91
|
+
with open(notified_path, "a") as f:
|
|
92
|
+
f.write(regex + "\n")
|
|
93
|
+
except Exception:
|
|
94
|
+
pass
|
|
95
|
+
print(json.dumps({"decision": "block", "reason":
|
|
96
|
+
f"[harness trace rule] Required step not seen this session "
|
|
97
|
+
f"(no logged tool call matched /{regex}/): {message} "
|
|
98
|
+
f"Do it now, or state explicitly why it does not apply, then stop. "
|
|
99
|
+
f"(This rule blocks only once per session.)"}, ensure_ascii=False))
|
|
100
|
+
break
|
|
101
|
+
PYEOF
|
|
102
|
+
)
|
|
103
|
+
if [ -n "$TRACE_OUT" ]; then
|
|
104
|
+
printf '%s\n' "$TRACE_OUT"
|
|
105
|
+
exit 0
|
|
106
|
+
fi
|
|
107
|
+
fi
|
|
108
|
+
|
|
25
109
|
# No code modified this session -> pass
|
|
26
110
|
[ -s "$MARKER" ] || exit 0
|
|
27
111
|
|
package/.claude/settings.json
CHANGED
|
@@ -11,6 +11,28 @@
|
|
|
11
11
|
"statusMessage": "harness: command safety check"
|
|
12
12
|
}
|
|
13
13
|
]
|
|
14
|
+
},
|
|
15
|
+
{
|
|
16
|
+
"matcher": "Edit|Write|MultiEdit",
|
|
17
|
+
"hooks": [
|
|
18
|
+
{
|
|
19
|
+
"type": "command",
|
|
20
|
+
"command": "${CLAUDE_PROJECT_DIR}/.claude/hooks/guard-pre-edit.sh",
|
|
21
|
+
"timeout": 15,
|
|
22
|
+
"statusMessage": "harness: edit safety check"
|
|
23
|
+
}
|
|
24
|
+
]
|
|
25
|
+
}
|
|
26
|
+
],
|
|
27
|
+
"PreCompact": [
|
|
28
|
+
{
|
|
29
|
+
"hooks": [
|
|
30
|
+
{
|
|
31
|
+
"type": "command",
|
|
32
|
+
"command": "${CLAUDE_PROJECT_DIR}/.claude/hooks/pre-compact.sh",
|
|
33
|
+
"timeout": 15
|
|
34
|
+
}
|
|
35
|
+
]
|
|
14
36
|
}
|
|
15
37
|
],
|
|
16
38
|
"PostToolUse": [
|
|
@@ -71,6 +93,17 @@
|
|
|
71
93
|
}
|
|
72
94
|
]
|
|
73
95
|
}
|
|
96
|
+
],
|
|
97
|
+
"SessionEnd": [
|
|
98
|
+
{
|
|
99
|
+
"hooks": [
|
|
100
|
+
{
|
|
101
|
+
"type": "command",
|
|
102
|
+
"command": "${CLAUDE_PROJECT_DIR}/.claude/hooks/session-end.sh",
|
|
103
|
+
"timeout": 15
|
|
104
|
+
}
|
|
105
|
+
]
|
|
106
|
+
}
|
|
74
107
|
]
|
|
75
108
|
}
|
|
76
109
|
}
|
|
@@ -0,0 +1,49 @@
|
|
|
1
|
+
---
|
|
2
|
+
name: distill
|
|
3
|
+
description: Mine a successfully completed task from the harness logs and distill the reusable procedure into a project skill (success-side ratchet). Use after a nontrivial task succeeded and is likely to recur (/distill [what succeeded]). Proposes only — creating the skill requires user approval.
|
|
4
|
+
---
|
|
5
|
+
|
|
6
|
+
# Distill — Turning Successes into Structure
|
|
7
|
+
|
|
8
|
+
`/ratchet` makes failures permanent structure; `/distill` does the same
|
|
9
|
+
for **successes** (the workflow-memory idea, arXiv:2409.07429). A
|
|
10
|
+
multi-step procedure that worked once and will be needed again should
|
|
11
|
+
become a project skill — not something to rediscover from scratch.
|
|
12
|
+
|
|
13
|
+
## When to Use
|
|
14
|
+
|
|
15
|
+
- The task took multiple nonobvious steps that a future session would
|
|
16
|
+
have to rediscover (env quirks, ordering constraints, project-specific
|
|
17
|
+
commands).
|
|
18
|
+
- The task will plausibly recur (release flow, data migration, fixture
|
|
19
|
+
regeneration, deploy checklist).
|
|
20
|
+
|
|
21
|
+
Do NOT distill one-off work, trivial sequences the code already
|
|
22
|
+
documents, or a procedure that only worked once and remains unverified.
|
|
23
|
+
|
|
24
|
+
## Procedure
|
|
25
|
+
|
|
26
|
+
1. **Locate the trajectory** — from `.harness/logs/tool_calls.jsonl`,
|
|
27
|
+
collect this session's (or the named task's) entries: the commands
|
|
28
|
+
run, files touched, and order. Cross-check `.harness/state/plan.md`
|
|
29
|
+
and `decisions.jsonl` for the decisions that shaped it.
|
|
30
|
+
2. **Separate the reusable from the incidental** — keep steps that will
|
|
31
|
+
repeat (commands, ordering, verification points); drop exploration,
|
|
32
|
+
dead ends, and one-time values. Parameterize what varies (version
|
|
33
|
+
numbers, file names) as `[placeholders]`.
|
|
34
|
+
3. **Attach verification** — every distilled skill must state how to
|
|
35
|
+
check the procedure worked (which command proves success). A recipe
|
|
36
|
+
without a check is how quality drift starts.
|
|
37
|
+
4. **Draft the skill** — propose
|
|
38
|
+
`.claude/skills/<name>/SKILL.md` (YAML frontmatter: name,
|
|
39
|
+
description with clear trigger conditions; body: numbered steps,
|
|
40
|
+
verification, known failure modes from this run).
|
|
41
|
+
5. **Get approval** — show the full draft. Create the file only after
|
|
42
|
+
the user approves. Record the decision in `decisions.jsonl`.
|
|
43
|
+
|
|
44
|
+
## Forbidden
|
|
45
|
+
|
|
46
|
+
- No auto-creating skills without approval.
|
|
47
|
+
- No distilling procedures whose success was never verified.
|
|
48
|
+
- No secrets, tokens, or user-specific paths inside the skill body.
|
|
49
|
+
- One skill per distinct procedure — do not merge unrelated recipes.
|
|
@@ -26,6 +26,20 @@ For each entry in RULES and ANTI-PATTERNS, run five checks:
|
|
|
26
26
|
5. **Freshness** — when was it last seen actually working (blocking or
|
|
27
27
|
catching a violation)? Cite `.harness/logs/tool_calls.jsonl` if evidence
|
|
28
28
|
exists.
|
|
29
|
+
6. **Contract verification** — if `.harness/state/predictions.jsonl`
|
|
30
|
+
exists, check every prediction whose window has elapsed
|
|
31
|
+
(`ts + window_days` in the past) deterministically:
|
|
32
|
+
count `predict_absent` matches among `PostToolUseFailure` entries in
|
|
33
|
+
`.harness/logs/tool_calls.jsonl` with a timestamp after `ts`.
|
|
34
|
+
- **0 matches** → the rule demonstrably worked: verdict *keep*, cite
|
|
35
|
+
the count.
|
|
36
|
+
- **matches at or near `baseline_count`** → the rule failed its
|
|
37
|
+
contract: verdict *delete candidate* or *convert-to-sensor candidate*
|
|
38
|
+
(a rule that didn't stop the failure needs a stronger layer, not
|
|
39
|
+
rewording).
|
|
40
|
+
- Report verified predictions in the output table and propose removing
|
|
41
|
+
their lines from predictions.jsonl (the contract is settled either
|
|
42
|
+
way). This check is grep-and-count, not judgment — show the numbers.
|
|
29
43
|
|
|
30
44
|
## Output Format
|
|
31
45
|
|
|
@@ -95,7 +95,11 @@ before and tell them to re-run /harness-init once code exists.
|
|
|
95
95
|
- Show everything as a diff and get user approval:
|
|
96
96
|
1. The PROJECT section of CLAUDE.md (PROJECT/LANGUAGE/BUILD/TEST/LINT)
|
|
97
97
|
2. LINT_CMD / TEST_CMD in `.harness/commands.env` (identical to PROJECT)
|
|
98
|
-
3.
|
|
98
|
+
3. The Commands section of AGENTS.md (mirror of PROJECT — this is what
|
|
99
|
+
non-Claude tools like Codex/Cursor/Aider read; if the project already
|
|
100
|
+
had its own AGENTS.md, propose appending the harness quick-contract
|
|
101
|
+
instead of replacing anything)
|
|
102
|
+
4. Initial RULES / ANTI-PATTERNS entries (if any)
|
|
99
103
|
- Write only what was approved. Afterwards, run TEST_CMD once more to
|
|
100
104
|
confirm the sensor configuration points at a command that really runs.
|
|
101
105
|
|
|
@@ -44,6 +44,13 @@ Handle one mistake end-to-end:
|
|
|
44
44
|
| Quality drift | declared done without verification | strengthen sensor (TEST_CMD scope) |
|
|
45
45
|
| Lost state | repeated already-completed steps | strengthen checkpoint protocol |
|
|
46
46
|
| Runaway/cost blowup | tool-call surge, same-error loop | adjust tripwire thresholds |
|
|
47
|
+
| Step repetition | same command/edit repeated with no progress between | tripwire threshold or trace rule |
|
|
48
|
+
| Spec deviation | did work the task never asked for (scope creep, unrequested refactor) | CLAUDE.md guide entry |
|
|
49
|
+
| Premature completion | declared done while a required step never ran | trace rule (`require-before-stop`) |
|
|
50
|
+
|
|
51
|
+
(The last three classes come from the MAST failure taxonomy,
|
|
52
|
+
arXiv:2503.13657 — they recur across agent systems, so classify against
|
|
53
|
+
them before inventing a new class.)
|
|
47
54
|
|
|
48
55
|
## Proposal Format
|
|
49
56
|
|
|
@@ -51,11 +58,35 @@ Handle one mistake end-to-end:
|
|
|
51
58
|
- Guard pattern → the regex to add to DENY_PATTERNS in guard-pre-bash.sh
|
|
52
59
|
- Permission → the allow/ask/deny change for settings.json permissions
|
|
53
60
|
- Sensor → commands.env change or file-extension additions in sensor-post-edit.sh
|
|
61
|
+
- Trace rule → a `require-before-stop` line for `.harness/trace.rules`
|
|
62
|
+
(format: `require-before-stop <marker-regex> <message>` — stop-gate.sh
|
|
63
|
+
blocks the stop if no logged tool call this session matched the regex)
|
|
54
64
|
|
|
55
65
|
With each proposal, include: (1) the failure class it prevents,
|
|
56
66
|
(2) possible side effects (over-blocking etc.), (3) overlap/conflict with
|
|
57
67
|
existing rules.
|
|
58
68
|
|
|
69
|
+
## Falsifiable Contract (predictions.jsonl)
|
|
70
|
+
|
|
71
|
+
Every proposal ships with a **machine-checkable prediction** — the claim
|
|
72
|
+
that makes the rule falsifiable instead of decorative. When the user
|
|
73
|
+
approves and the fix is applied, append one line to
|
|
74
|
+
`.harness/state/predictions.jsonl`:
|
|
75
|
+
|
|
76
|
+
```json
|
|
77
|
+
{"ts": "<today ISO date>", "rule": "<short rule summary>",
|
|
78
|
+
"class": "<failure class>", "predict_absent": "<substring/regex that
|
|
79
|
+
matched the original failure in tool_calls.jsonl>",
|
|
80
|
+
"baseline_count": <how many times it fired in the last 30 days>,
|
|
81
|
+
"window_days": 30}
|
|
82
|
+
```
|
|
83
|
+
|
|
84
|
+
`/guide-audit` later counts `predict_absent` matches in the logs after
|
|
85
|
+
`ts`. Zero matches inside the window = the rule demonstrably worked;
|
|
86
|
+
matches at or near baseline = the rule failed its contract and becomes a
|
|
87
|
+
delete/rewrite candidate. A proposal you cannot attach a prediction to is
|
|
88
|
+
a smell — say so explicitly rather than inventing an untestable one.
|
|
89
|
+
|
|
59
90
|
## Approval and Cleanup
|
|
60
91
|
|
|
61
92
|
- Apply only what the user approves.
|
package/AGENTS.md
ADDED
|
@@ -0,0 +1,26 @@
|
|
|
1
|
+
# Agent Instructions
|
|
2
|
+
|
|
3
|
+
This project's canonical agent guide is **CLAUDE.md**, maintained by the
|
|
4
|
+
uni-harness kit — commands, rules, anti-patterns, work-loop and checkpoint
|
|
5
|
+
protocols all live there. Read CLAUDE.md and follow it.
|
|
6
|
+
|
|
7
|
+
## Commands
|
|
8
|
+
|
|
9
|
+
(filled by /harness-init — mirrors the PROJECT section of CLAUDE.md)
|
|
10
|
+
|
|
11
|
+
## Quick contract for any coding agent working here
|
|
12
|
+
|
|
13
|
+
- Run the TEST command after modifying code. Never report work as done
|
|
14
|
+
with failing tests, and never weaken a test (skip/only/deleted cases,
|
|
15
|
+
trivially-true assertions) to make it pass.
|
|
16
|
+
- Do not modify `.claude/**`, `.harness/**`, `.github/**`, `.env*`, or
|
|
17
|
+
`*.config.*` files without explicit user approval — especially changes
|
|
18
|
+
that would weaken tests, lint, or guards.
|
|
19
|
+
- After 3 failed attempts at the same goal, stop and escalate: what
|
|
20
|
+
decision is needed, what was tried, the cost of waiting, and the safest
|
|
21
|
+
default action.
|
|
22
|
+
- Keep durable working state in `.harness/state/` (plan.md,
|
|
23
|
+
progress.json, decisions.jsonl), not in scattered notes.
|
|
24
|
+
|
|
25
|
+
Keep this file in sync with CLAUDE.md (in Claude Code, /harness-init
|
|
26
|
+
maintains both).
|
package/README.ja.md
CHANGED
|
@@ -54,11 +54,15 @@ npx uni-harness uninstall --yes # キットの機構を削除、あなたのフ
|
|
|
54
54
|
| ファイル | 役割 |
|
|
55
55
|
|---|---|
|
|
56
56
|
| `CLAUDE.md` | プロジェクトのコマンド、ルール、アンチパターン、ワークループとチェックポイントのプロトコル |
|
|
57
|
+
| `AGENTS.md` | ルールのクロスツール・ミラー — Codex、Cursor、Aider、Gemini CLI など20以上のツールが読み取る |
|
|
57
58
|
| `.claude/settings.json` | フック登録 |
|
|
58
59
|
| `.claude/hooks/sensor-post-edit.sh` | コード編集のたびに即座にリントを実行し、失敗をフィードバック |
|
|
59
|
-
| `.claude/hooks/stop-gate.sh` | ターン終了時にテストを一括実行。失敗状態での終了をブロック(セッションあたり3
|
|
60
|
-
| `.claude/hooks/guard-pre-bash.sh` | 破壊的コマンドと検証バイパス(`--no-verify
|
|
60
|
+
| `.claude/hooks/stop-gate.sh` | ターン終了時にテストを一括実行。失敗状態での終了をブロック(セッションあたり3回上限、超過でエスカレーション要求)。任意のトレースルール(`.harness/trace.rules`: 「ターン終了前に必ず X が起きていること」)も強制 |
|
|
61
|
+
| `.claude/hooks/guard-pre-bash.sh` | 破壊的コマンドと検証バイパス(`--no-verify`)を実行前にブロック。テストが一度も通っていない編集をコミットしようとするとユーザーに確認 |
|
|
62
|
+
| `.claude/hooks/guard-pre-edit.sh` | アンチゲーミング・ゲート: テストを弱める編集(skip/only マーカー、テスト削除、アサーションの骨抜き)と設定/ハーネスファイルの変更にユーザー承認を要求 |
|
|
63
|
+
| `.claude/hooks/pre-compact.sh` | コンテキスト圧縮の直前にチェックポイントをスナップショット。再開時のチェックポイント更新指示は `session-start.sh` が担当 |
|
|
61
64
|
| `.claude/hooks/session-start.sh` | セッション開始/再開/圧縮時に進行中チェックポイントを再注入。失敗が蓄積すると `/ratchet` を提案 |
|
|
65
|
+
| `.claude/hooks/session-end.sh` | セッションごとに台帳1行を記録(所要時間、呼び出し/失敗/編集数、終了ブロック回数、テスト状態)→ `sessions.jsonl` |
|
|
62
66
|
| `.claude/hooks/observe-log.sh` | 全ツール呼び出しを JSONL で記録 + トリップワイヤー(同一失敗3回、呼び出し急増) |
|
|
63
67
|
| `harness/harness_report.py` | ログからのヘルススコアカード |
|
|
64
68
|
| `harness/tests/` | フックとインストーラーのセルフテスト |
|
|
@@ -70,8 +74,9 @@ npx uni-harness uninstall --yes # キットの機構を削除、あなたのフ
|
|
|
70
74
|
|---|---|
|
|
71
75
|
| `/harness-init` | リポジトリをスキャン → PROJECT セクションと commands.env を記入(インストール時に1回) |
|
|
72
76
|
| `/checkpoint` | `.harness/state/` に状態を保存(plan.md / decisions.jsonl / progress.json) |
|
|
73
|
-
| `/ratchet [ミス]` | 再現 → 分類 → ルール/センサー/権限を提案 → 検証(引数なし:
|
|
74
|
-
| `/guide-audit` | CLAUDE.md ルールの監査 — 維持 / 削除 /
|
|
77
|
+
| `/ratchet [ミス]` | 再現 → 分類 → ルール/センサー/権限を提案 → 検証(引数なし: ログ診断)。全提案に反証可能な予測(`predictions.jsonl`)を添付 |
|
|
78
|
+
| `/guide-audit` | CLAUDE.md ルールの監査 — 維持 / 削除 / センサー化(月次)。過去の予測をログと照合検証し、効かなかったルールは証拠に基づいて削除 |
|
|
79
|
+
| `/distill [成功]` | 成功側のラチェット: 完了したタスクをログから発掘し、再利用可能なプロジェクトスキルとして提案 |
|
|
75
80
|
|
|
76
81
|
## 推奨パーミッション(任意)
|
|
77
82
|
|
|
@@ -105,8 +110,11 @@ npx uni-harness uninstall --yes # キットの機構を削除、あなたのフ
|
|
|
105
110
|
ルールへ。同じルール違反3回 → ガード/権限へ昇格。
|
|
106
111
|
- **毎週:** `python3 harness/harness_report.py` でスコアカードを
|
|
107
112
|
確認。繰り返しの失敗が見えたら `/ratchet` を実行。
|
|
113
|
+
- **意味のある成功のあと:** 繰り返す手順なら `/distill` で
|
|
114
|
+
プロジェクトスキルにしてください。
|
|
108
115
|
- **毎月:** `/guide-audit` で CLAUDE.md を監査 — センサーが既に
|
|
109
|
-
|
|
116
|
+
強制しているルールは削除し、矛盾はマージ。各ルールの予測をログと
|
|
117
|
+
照合(同じ失敗が続いたルールは、感覚でなく数字で削除/昇格)。
|
|
110
118
|
|
|
111
119
|
## カスタマイズ
|
|
112
120
|
|
package/README.ko.md
CHANGED
|
@@ -52,11 +52,15 @@ npx uni-harness uninstall --yes # 킷 기계 부품 제거, 사용자 파일은
|
|
|
52
52
|
| 파일 | 역할 |
|
|
53
53
|
|---|---|
|
|
54
54
|
| `CLAUDE.md` | 프로젝트 명령, 규칙, 안티패턴, 작업 루프·체크포인트 프로토콜 |
|
|
55
|
+
| `AGENTS.md` | 규칙의 크로스툴 미러 — Codex, Cursor, Aider, Gemini CLI 등 20+ 도구가 읽음 |
|
|
55
56
|
| `.claude/settings.json` | 훅 등록 |
|
|
56
57
|
| `.claude/hooks/sensor-post-edit.sh` | 코드 수정 즉시 lint 실행, 실패를 피드백 |
|
|
57
|
-
| `.claude/hooks/stop-gate.sh` | 턴 종료 시 테스트 일괄 실행; 실패 상태의 종료를 차단 (세션당 3회 상한, 초과 시 에스컬레이션 강제) |
|
|
58
|
-
| `.claude/hooks/guard-pre-bash.sh` | 파괴적 명령과 검증 우회(`--no-verify`)를 실행 전
|
|
58
|
+
| `.claude/hooks/stop-gate.sh` | 턴 종료 시 테스트 일괄 실행; 실패 상태의 종료를 차단 (세션당 3회 상한, 초과 시 에스컬레이션 강제); 선택적 트레이스 규칙(`.harness/trace.rules`: "턴이 끝나기 전 반드시 X가 있었어야 함") 집행 |
|
|
59
|
+
| `.claude/hooks/guard-pre-bash.sh` | 파괴적 명령과 검증 우회(`--no-verify`)를 실행 전 차단; 테스트가 한 번도 통과하지 않은 편집을 커밋하려 하면 사용자에게 확인 |
|
|
60
|
+
| `.claude/hooks/guard-pre-edit.sh` | 안티게이밍 게이트: 테스트 약화 편집(skip/only 마커, 테스트 삭제, 단언 무력화)과 설정/하네스 파일 변경에 사용자 승인 요구 |
|
|
61
|
+
| `.claude/hooks/pre-compact.sh` | 컨텍스트 압축 직전 체크포인트 스냅샷; 재개 시 체크포인트 갱신 지시는 `session-start.sh`가 담당 |
|
|
59
62
|
| `.claude/hooks/session-start.sh` | 세션 시작/재개/압축 시 진행 중 체크포인트 재주입; 실패 누적 시 `/ratchet` 제안 |
|
|
63
|
+
| `.claude/hooks/session-end.sh` | 세션당 원장(ledger) 한 줄 기록 (소요 시간, 호출/실패/편집 수, 종료 차단 횟수, 테스트 상태) → `sessions.jsonl` |
|
|
60
64
|
| `.claude/hooks/observe-log.sh` | 모든 도구 호출을 JSONL로 기록 + 트립와이어 (동일 실패 3회, 호출 급증) |
|
|
61
65
|
| `harness/harness_report.py` | 로그 기반 상태 스코어카드 |
|
|
62
66
|
| `harness/tests/` | 훅·인스톨러 자체 테스트 |
|
|
@@ -68,8 +72,9 @@ npx uni-harness uninstall --yes # 킷 기계 부품 제거, 사용자 파일은
|
|
|
68
72
|
|---|---|
|
|
69
73
|
| `/harness-init` | 저장소 스캔 → PROJECT 섹션 & commands.env 채우기 (설치 시 1회) |
|
|
70
74
|
| `/checkpoint` | `.harness/state/`에 상태 저장 (plan.md / decisions.jsonl / progress.json) |
|
|
71
|
-
| `/ratchet [실수]` | 재현 → 분류 → 규칙/센서/권한 제안 → 검증 (인자 없이: 로그 진단) |
|
|
72
|
-
| `/guide-audit` | CLAUDE.md 규칙 감사 — 유지 / 삭제 / 센서 전환 (월 1회) |
|
|
75
|
+
| `/ratchet [실수]` | 재현 → 분류 → 규칙/센서/권한 제안 → 검증 (인자 없이: 로그 진단). 모든 제안에 반증 가능한 예측(`predictions.jsonl`)을 첨부 |
|
|
76
|
+
| `/guide-audit` | CLAUDE.md 규칙 감사 — 유지 / 삭제 / 센서 전환 (월 1회); 과거 예측을 로그와 대조 검증해 안 통한 규칙은 증거로 삭제 |
|
|
77
|
+
| `/distill [성공]` | 성공 쪽 래칫: 완료된 작업을 로그에서 발굴해 재사용 가능한 프로젝트 스킬로 제안 |
|
|
73
78
|
|
|
74
79
|
## 권장 권한 설정 (선택)
|
|
75
80
|
|
|
@@ -102,8 +107,11 @@ npx uni-harness uninstall --yes # 킷 기계 부품 제거, 사용자 파일은
|
|
|
102
107
|
규칙 위반 3회 → 가드/권한으로 승격.
|
|
103
108
|
- **매주:** `python3 harness/harness_report.py`로 스코어카드 확인;
|
|
104
109
|
반복 실패가 보이면 `/ratchet` 실행.
|
|
110
|
+
- **의미 있는 성공 후:** 반복될 절차라면 `/distill`로 프로젝트 스킬로
|
|
111
|
+
만드세요.
|
|
105
112
|
- **매월:** `/guide-audit`으로 CLAUDE.md 감사 — 센서가 이미 강제하는
|
|
106
|
-
규칙은 삭제, 모순은
|
|
113
|
+
규칙은 삭제, 모순은 병합, 그리고 각 규칙의 예측을 로그와 대조
|
|
114
|
+
(같은 실패가 계속된 규칙은 감이 아니라 숫자로 삭제/승격).
|
|
107
115
|
|
|
108
116
|
## 커스터마이징
|
|
109
117
|
|
package/README.md
CHANGED
|
@@ -52,11 +52,15 @@ anything you've customized is skipped (listed, with `--force` to override).
|
|
|
52
52
|
| File | Role |
|
|
53
53
|
|---|---|
|
|
54
54
|
| `CLAUDE.md` | Project commands, rules, anti-patterns, work-loop and checkpoint protocols |
|
|
55
|
+
| `AGENTS.md` | Cross-tool mirror of the rules — read by Codex, Cursor, Aider, Gemini CLI and 20+ other tools |
|
|
55
56
|
| `.claude/settings.json` | Hook registration |
|
|
56
57
|
| `.claude/hooks/sensor-post-edit.sh` | Runs lint immediately on code edits, feeds failures back |
|
|
57
|
-
| `.claude/hooks/stop-gate.sh` | Runs tests in batch at turn end; blocks stopping on failure (3-per-session cap, then demands escalation) |
|
|
58
|
-
| `.claude/hooks/guard-pre-bash.sh` | Blocks destructive commands and verification bypasses (`--no-verify`) before execution |
|
|
58
|
+
| `.claude/hooks/stop-gate.sh` | Runs tests in batch at turn end; blocks stopping on failure (3-per-session cap, then demands escalation); enforces optional trace rules (`.harness/trace.rules`: "step X must have happened before the turn ends") |
|
|
59
|
+
| `.claude/hooks/guard-pre-bash.sh` | Blocks destructive commands and verification bypasses (`--no-verify`) before execution; asks before a `git commit` that would land edits no test run has seen |
|
|
60
|
+
| `.claude/hooks/guard-pre-edit.sh` | Anti-gaming gate: test-weakening edits (skip/only markers, deleted tests, gutted assertions) and config/harness file changes require user approval |
|
|
61
|
+
| `.claude/hooks/pre-compact.sh` | Snapshots checkpoints just before context compaction; `session-start.sh` then instructs a checkpoint refresh when the session resumes |
|
|
59
62
|
| `.claude/hooks/session-start.sh` | Re-injects in-progress checkpoints at session start/resume/compaction; nudges `/ratchet` when failures pile up |
|
|
63
|
+
| `.claude/hooks/session-end.sh` | Writes one ledger row per session (duration, calls, failures, edits, stop blocks, test state) to `sessions.jsonl` |
|
|
60
64
|
| `.claude/hooks/observe-log.sh` | Logs every tool call as JSONL + tripwires (same failure 3x, call surge) |
|
|
61
65
|
| `harness/harness_report.py` | Health scorecard from the logs |
|
|
62
66
|
| `harness/tests/` | Self-tests for the hooks and the installer |
|
|
@@ -68,8 +72,9 @@ Skills:
|
|
|
68
72
|
|---|---|
|
|
69
73
|
| `/harness-init` | Scan the repo → fill PROJECT section & commands.env (once, at install) |
|
|
70
74
|
| `/checkpoint` | Save state to `.harness/state/` (plan.md / decisions.jsonl / progress.json) |
|
|
71
|
-
| `/ratchet [mistake]` | Reproduce → classify → propose rule/sensor/permission → verify (no args: diagnose the logs) |
|
|
72
|
-
| `/guide-audit` | Audit CLAUDE.md rules — keep / delete / convert-to-sensor (monthly) |
|
|
75
|
+
| `/ratchet [mistake]` | Reproduce → classify → propose rule/sensor/permission → verify (no args: diagnose the logs). Every proposal ships a falsifiable prediction (`predictions.jsonl`) |
|
|
76
|
+
| `/guide-audit` | Audit CLAUDE.md rules — keep / delete / convert-to-sensor (monthly); verifies past predictions against the logs, so rules that didn't work get deleted on evidence |
|
|
77
|
+
| `/distill [success]` | The success-side ratchet: mine a completed task from the logs and propose a reusable project skill |
|
|
73
78
|
|
|
74
79
|
## Recommended Permissions (optional)
|
|
75
80
|
|
|
@@ -103,8 +108,12 @@ from editing its own harness:
|
|
|
103
108
|
3x → rule; same rule violated 3x → promote to a guard/permission.
|
|
104
109
|
- **Weekly:** check the scorecard with `python3 harness/harness_report.py`;
|
|
105
110
|
if repeated failures show up, run `/ratchet`.
|
|
111
|
+
- **After a nontrivial success:** if the procedure will recur, run
|
|
112
|
+
`/distill` to turn it into a project skill.
|
|
106
113
|
- **Monthly:** audit CLAUDE.md with `/guide-audit` — delete rules the
|
|
107
|
-
sensors now enforce, merge contradictions
|
|
114
|
+
sensors now enforce, merge contradictions, and check each rule's
|
|
115
|
+
prediction against the logs (a rule whose failure kept happening is
|
|
116
|
+
deleted or promoted on numbers, not vibes).
|
|
108
117
|
|
|
109
118
|
## Customizing
|
|
110
119
|
|
package/README.zh-CN.md
CHANGED
|
@@ -51,11 +51,15 @@ node ≥16。
|
|
|
51
51
|
| 文件 | 作用 |
|
|
52
52
|
|---|---|
|
|
53
53
|
| `CLAUDE.md` | 项目命令、规则、反模式、工作循环与检查点协议 |
|
|
54
|
+
| `AGENTS.md` | 规则的跨工具镜像 — Codex、Cursor、Aider、Gemini CLI 等 20+ 工具都会读取 |
|
|
54
55
|
| `.claude/settings.json` | 钩子注册 |
|
|
55
56
|
| `.claude/hooks/sensor-post-edit.sh` | 每次代码编辑立即运行 lint,将失败反馈回去 |
|
|
56
|
-
| `.claude/hooks/stop-gate.sh` | 回合结束时批量运行测试;失败状态下阻止收工(每会话 3
|
|
57
|
-
| `.claude/hooks/guard-pre-bash.sh` | 在执行前拦截破坏性命令与验证绕过(`--no-verify
|
|
57
|
+
| `.claude/hooks/stop-gate.sh` | 回合结束时批量运行测试;失败状态下阻止收工(每会话 3 次上限,超过则强制升级上报);执行可选的轨迹规则(`.harness/trace.rules`:"回合结束前必须发生过 X") |
|
|
58
|
+
| `.claude/hooks/guard-pre-bash.sh` | 在执行前拦截破坏性命令与验证绕过(`--no-verify`);当要提交从未通过测试的编辑时先向用户确认 |
|
|
59
|
+
| `.claude/hooks/guard-pre-edit.sh` | 反作弊关卡:削弱测试的编辑(skip/only 标记、删除测试、掏空断言)以及配置/挽具文件的更改都需要用户批准 |
|
|
60
|
+
| `.claude/hooks/pre-compact.sh` | 在上下文压缩前对检查点做快照;恢复时的检查点刷新指令由 `session-start.sh` 负责 |
|
|
58
61
|
| `.claude/hooks/session-start.sh` | 会话启动/恢复/压缩时重新注入进行中的检查点;失败堆积时提示 `/ratchet` |
|
|
62
|
+
| `.claude/hooks/session-end.sh` | 每个会话写入一行台账(时长、调用/失败/编辑数、收工拦截次数、测试状态)→ `sessions.jsonl` |
|
|
59
63
|
| `.claude/hooks/observe-log.sh` | 以 JSONL 记录每次工具调用 + 绊线(同一失败 3 次、调用激增) |
|
|
60
64
|
| `harness/harness_report.py` | 基于日志的健康记分卡 |
|
|
61
65
|
| `harness/tests/` | 钩子与安装器的自测 |
|
|
@@ -67,8 +71,9 @@ node ≥16。
|
|
|
67
71
|
|---|---|
|
|
68
72
|
| `/harness-init` | 扫描仓库 → 填写 PROJECT 章节 & commands.env(安装时一次) |
|
|
69
73
|
| `/checkpoint` | 将状态保存到 `.harness/state/`(plan.md / decisions.jsonl / progress.json) |
|
|
70
|
-
| `/ratchet [失误]` | 复现 → 归类 → 提议规则/传感器/权限 →
|
|
71
|
-
| `/guide-audit` | 审计 CLAUDE.md 规则 — 保留 / 删除 /
|
|
74
|
+
| `/ratchet [失误]` | 复现 → 归类 → 提议规则/传感器/权限 → 验证(无参数:诊断日志)。每个提议附带可证伪的预测(`predictions.jsonl`) |
|
|
75
|
+
| `/guide-audit` | 审计 CLAUDE.md 规则 — 保留 / 删除 / 转为传感器(每月);将过往预测与日志比对验证,没起作用的规则凭证据删除 |
|
|
76
|
+
| `/distill [成功]` | 成功侧的棘轮:从日志中挖掘已完成的任务,提议为可复用的项目技能 |
|
|
72
77
|
|
|
73
78
|
## 推荐权限配置(可选)
|
|
74
79
|
|
|
@@ -101,8 +106,11 @@ node ≥16。
|
|
|
101
106
|
3 次 → 升级为守卫/权限。
|
|
102
107
|
- **每周:** 用 `python3 harness/harness_report.py` 查看记分卡;
|
|
103
108
|
出现重复失败就运行 `/ratchet`。
|
|
109
|
+
- **一次有意义的成功之后:** 若流程还会重复,用 `/distill` 把它
|
|
110
|
+
变成项目技能。
|
|
104
111
|
- **每月:** 用 `/guide-audit` 审计 CLAUDE.md — 删除传感器已自动
|
|
105
|
-
|
|
112
|
+
强制执行的规则,合并互相矛盾的规则,并将每条规则的预测与日志
|
|
113
|
+
比对(同一失败仍在发生的规则,凭数字而非感觉删除或升级)。
|
|
106
114
|
|
|
107
115
|
## 自定义
|
|
108
116
|
|
package/bin/cli.js
CHANGED
|
@@ -229,7 +229,7 @@ function init(target, force) {
|
|
|
229
229
|
}
|
|
230
230
|
|
|
231
231
|
let claudeMdExisted = false;
|
|
232
|
-
for (const rel of ['CLAUDE.md', '.harness/commands.env']) {
|
|
232
|
+
for (const rel of ['CLAUDE.md', 'AGENTS.md', '.harness/commands.env']) {
|
|
233
233
|
const dst = path.join(target, rel);
|
|
234
234
|
if (fs.existsSync(dst)) {
|
|
235
235
|
log(' = ' + rel + ' (existing file kept)');
|
|
@@ -304,6 +304,9 @@ function update(target, force) {
|
|
|
304
304
|
writeManifest(target, files);
|
|
305
305
|
ensureRuntimeDirs(target);
|
|
306
306
|
migrateLegacyState(target);
|
|
307
|
+
// hooks added in newer kit versions need registering too — mergeSettings
|
|
308
|
+
// is append-only and idempotent, so this never disturbs user settings
|
|
309
|
+
mergeSettings(target);
|
|
307
310
|
log(`\n${changed} file(s) updated (v${manifest.kitVersion} -> v${KIT_VERSION}). ` +
|
|
308
311
|
'CLAUDE.md / commands.env / settings.json were not touched.');
|
|
309
312
|
|
|
@@ -360,7 +363,7 @@ function doctor(target) {
|
|
|
360
363
|
const body = fs.readFileSync(sp, 'utf8');
|
|
361
364
|
JSON.parse(body);
|
|
362
365
|
check(true, '.claude/settings.json parses');
|
|
363
|
-
for (const h of ['guard-pre-bash.sh', 'sensor-post-edit.sh', 'stop-gate.sh', 'observe-log.sh', 'session-start.sh']) {
|
|
366
|
+
for (const h of ['guard-pre-bash.sh', 'guard-pre-edit.sh', 'sensor-post-edit.sh', 'stop-gate.sh', 'observe-log.sh', 'session-start.sh', 'pre-compact.sh', 'session-end.sh']) {
|
|
364
367
|
if (!body.includes(h)) check(false, `hook registered: ${h}`, 'not in settings.json');
|
|
365
368
|
}
|
|
366
369
|
} catch { check(false, '.claude/settings.json', 'missing or failed to parse'); }
|
|
@@ -1,7 +1,9 @@
|
|
|
1
1
|
#!/usr/bin/env python3
|
|
2
2
|
"""Harness health scorecard.
|
|
3
3
|
|
|
4
|
-
Reads .harness/logs/tool_calls.jsonl and
|
|
4
|
+
Reads .harness/logs/tool_calls.jsonl (raw tool calls) and, if present,
|
|
5
|
+
.harness/logs/sessions.jsonl (the per-session ledger written by
|
|
6
|
+
session-end.sh) and prints health metrics.
|
|
5
7
|
|
|
6
8
|
Usage:
|
|
7
9
|
python3 harness/harness_report.py [--days 7]
|
|
@@ -79,6 +81,19 @@ def main() -> int:
|
|
|
79
81
|
print(f" [{n}x] {tool}: {err}")
|
|
80
82
|
print("\n -> For failures repeated 2+ times, consider /ratchet to turn them into rules.")
|
|
81
83
|
|
|
84
|
+
ledger = load(root / ".harness" / "logs" / "sessions.jsonl", since)
|
|
85
|
+
if ledger:
|
|
86
|
+
clean = sum(1 for e in ledger if e.get("tests_clean"))
|
|
87
|
+
blocked = sum(e.get("stop_blocks") or 0 for e in ledger)
|
|
88
|
+
durations = sorted(e.get("duration_s") or 0 for e in ledger)
|
|
89
|
+
print("\n─ Session ledger ─")
|
|
90
|
+
print(f" Ended sessions: {len(ledger)}")
|
|
91
|
+
print(f" Tests clean at end: {clean}/{len(ledger)}")
|
|
92
|
+
print(f" Stop-gate blocks: {blocked}")
|
|
93
|
+
print(f" Duration (median): {durations[len(durations) // 2]}s")
|
|
94
|
+
if clean < len(ledger):
|
|
95
|
+
print(" -> Sessions ending with untested edits are quality-drift risk; check why.")
|
|
96
|
+
|
|
82
97
|
print("\n─ How to read this ─")
|
|
83
98
|
print(" · Is the failure rate trending down week over week?")
|
|
84
99
|
print(" · The same failure signature next week means the ratchet isn't working")
|
|
@@ -44,6 +44,55 @@ OUT=$(echo '{"tool_name":"Bash","tool_input":{"command":"git push --force-with-l
|
|
|
44
44
|
check "allows force-with-lease" "EMPTY" "$OUT"
|
|
45
45
|
OUT=$(echo '{"tool_name":"Bash","tool_input":{"command":"git status"}}' | bash "$HOOKS/guard-pre-bash.sh")
|
|
46
46
|
check "passes safe commands" "EMPTY" "$OUT"
|
|
47
|
+
# state-aware commit gate: pending_test marker + configured TEST_CMD -> ask
|
|
48
|
+
printf 'LINT_CMD=""\nTEST_CMD="true"\n' > "$TMP/.harness/commands.env"
|
|
49
|
+
echo "a.py" > "$TMP/.harness/logs/pending_test.s1"
|
|
50
|
+
OUT=$(echo '{"tool_name":"Bash","session_id":"s1","tool_input":{"command":"git commit -m wip"}}' | bash "$HOOKS/guard-pre-bash.sh")
|
|
51
|
+
check "asks on commit with untested edits" '"permissionDecision": "ask"' "$OUT"
|
|
52
|
+
OUT=$(echo '{"tool_name":"Bash","session_id":"s2","tool_input":{"command":"git commit -m ok"}}' | bash "$HOOKS/guard-pre-bash.sh")
|
|
53
|
+
check "allows commit in a session without pending edits" "EMPTY" "$OUT"
|
|
54
|
+
printf 'LINT_CMD=""\nTEST_CMD=""\n' > "$TMP/.harness/commands.env"
|
|
55
|
+
OUT=$(echo '{"tool_name":"Bash","session_id":"s1","tool_input":{"command":"git commit -m x"}}' | bash "$HOOKS/guard-pre-bash.sh")
|
|
56
|
+
check "commit gate silent when TEST_CMD unconfigured" "EMPTY" "$OUT"
|
|
57
|
+
rm -f "$TMP/.harness/logs/pending_test.s1"
|
|
58
|
+
|
|
59
|
+
# ── guard-pre-edit.sh ────────────────────────────────────────────
|
|
60
|
+
echo "guard-pre-edit.sh"
|
|
61
|
+
fresh_project
|
|
62
|
+
OUT=$(echo '{"tool_name":"Edit","tool_input":{"file_path":"'"$TMP"'/.claude/settings.json","old_string":"a","new_string":"b"}}' | bash "$HOOKS/guard-pre-edit.sh")
|
|
63
|
+
check "asks on harness machinery edit" '"permissionDecision": "ask"' "$OUT"
|
|
64
|
+
OUT=$(echo '{"tool_name":"Write","tool_input":{"file_path":"'"$TMP"'/.harness/state/progress.json","content":"{}"}}' | bash "$HOOKS/guard-pre-edit.sh")
|
|
65
|
+
check "allows checkpoint writes" "EMPTY" "$OUT"
|
|
66
|
+
OUT=$(echo '{"tool_name":"Edit","tool_input":{"file_path":"'"$TMP"'/src/app.py","old_string":"a","new_string":"b"}}' | bash "$HOOKS/guard-pre-edit.sh")
|
|
67
|
+
check "allows normal source edits" "EMPTY" "$OUT"
|
|
68
|
+
OUT=$(echo '{"tool_name":"Edit","tool_input":{"file_path":"'"$TMP"'/.env","old_string":"a","new_string":"b"}}' | bash "$HOOKS/guard-pre-edit.sh")
|
|
69
|
+
check "asks on env file edit" '"permissionDecision": "ask"' "$OUT"
|
|
70
|
+
OUT=$(echo '{"tool_name":"Edit","tool_input":{"file_path":"'"$TMP"'/vite.config.ts","old_string":"a","new_string":"b"}}' | bash "$HOOKS/guard-pre-edit.sh")
|
|
71
|
+
check "asks on tool config edit" '"permissionDecision": "ask"' "$OUT"
|
|
72
|
+
OUT=$(echo '{"tool_name":"Edit","tool_input":{"file_path":"'"$TMP"'/tests/test_app.py","old_string":"def test_a():","new_string":"@pytest.mark.skip\ndef test_a():"}}' | bash "$HOOKS/guard-pre-edit.sh")
|
|
73
|
+
check "asks on added skip marker" "skip" "$OUT"
|
|
74
|
+
OUT=$(echo '{"tool_name":"Edit","tool_input":{"file_path":"'"$TMP"'/src/api.test.ts","old_string":"it(\"works\", () => {","new_string":"it.skip(\"works\", () => {"}}' | bash "$HOOKS/guard-pre-edit.sh")
|
|
75
|
+
check "asks on it.skip in js tests" '"permissionDecision": "ask"' "$OUT"
|
|
76
|
+
mkdir -p "$TMP/tests"
|
|
77
|
+
printf 'def test_a():\n pass\ndef test_b():\n pass\n' > "$TMP/tests/test_app.py"
|
|
78
|
+
OUT=$(echo '{"tool_name":"Write","tool_input":{"file_path":"'"$TMP"'/tests/test_app.py","content":"def test_a():\n pass\n"}}' | bash "$HOOKS/guard-pre-edit.sh")
|
|
79
|
+
check "asks on removed test function" "removes 1 test function" "$OUT"
|
|
80
|
+
OUT=$(echo '{"tool_name":"Edit","tool_input":{"file_path":"'"$TMP"'/tests/test_app.py","old_string":"def test_a():","new_string":"def test_a():\ndef test_c():"}}' | bash "$HOOKS/guard-pre-edit.sh")
|
|
81
|
+
check "allows adding tests" "EMPTY" "$OUT"
|
|
82
|
+
OUT=$(echo '{"tool_name":"Write","tool_input":{"file_path":"'"$TMP"'/tests/test_new.py","content":"def test_x():\n pass\n"}}' | bash "$HOOKS/guard-pre-edit.sh")
|
|
83
|
+
check "allows brand-new test file" "EMPTY" "$OUT"
|
|
84
|
+
OUT=$(echo '{"tool_name":"Edit","tool_input":{"file_path":"/somewhere/else/test_x.py","old_string":"def test_a():","new_string":""}}' | bash "$HOOKS/guard-pre-edit.sh")
|
|
85
|
+
check "ignores files outside the project" "EMPTY" "$OUT"
|
|
86
|
+
|
|
87
|
+
# ── pre-compact.sh ───────────────────────────────────────────────
|
|
88
|
+
echo "pre-compact.sh"
|
|
89
|
+
fresh_project
|
|
90
|
+
mkdir -p "$TMP/.harness/state"
|
|
91
|
+
echo '{"task_id":"t1"}' > "$TMP/.harness/state/progress.json"
|
|
92
|
+
OUT=$(echo '{"hook_event_name":"PreCompact","session_id":"s1","trigger":"auto"}' | bash "$HOOKS/pre-compact.sh")
|
|
93
|
+
check "emits no JSON (PreCompact schema has no context channel)" "EMPTY" "$OUT"
|
|
94
|
+
check "snapshots checkpoint" "yes" "$([ -f "$TMP/.harness/state/.pre-compact-backup/progress.json" ] && echo yes)"
|
|
95
|
+
check "logs the compaction event" "PreCompact" "$(cat "$TMP/.harness/logs/tool_calls.jsonl")"
|
|
47
96
|
|
|
48
97
|
# ── observe-log.sh ───────────────────────────────────────────────
|
|
49
98
|
echo "observe-log.sh"
|
|
@@ -53,11 +102,16 @@ LINES=$(wc -l < "$TMP/.harness/logs/tool_calls.jsonl" | tr -d ' ')
|
|
|
53
102
|
check "logs one call" "1" "$LINES"
|
|
54
103
|
echo '{"hook_event_name":"PostToolUse","session_id":"s1","tool_name":"Edit","tool_input":{"file_path":"a.py"},"agent_id":"x1","agent_type":"code-reviewer"}' | bash "$HOOKS/observe-log.sh" > /dev/null
|
|
55
104
|
check "records subagent fields" "code-reviewer" "$(tail -1 "$TMP/.harness/logs/tool_calls.jsonl")"
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
|
|
105
|
+
# same error 3x — line numbers differ (normalization) and a success is
|
|
106
|
+
# interleaved (no longer requires "in a row")
|
|
107
|
+
OUT=$(echo '{"hook_event_name":"PostToolUseFailure","session_id":"s1","tool_name":"Bash","tool_input":{"command":"pytest"},"tool_response":"E: boom at line 12"}' | bash "$HOOKS/observe-log.sh")
|
|
108
|
+
check "no tripwire on first failure" "EMPTY" "$OUT"
|
|
109
|
+
echo '{"hook_event_name":"PostToolUse","session_id":"s1","tool_name":"Read","tool_input":{"file_path":"a.py"}}' | bash "$HOOKS/observe-log.sh" > /dev/null
|
|
110
|
+
OUT=$(echo '{"hook_event_name":"PostToolUseFailure","session_id":"s1","tool_name":"Bash","tool_input":{"command":"pytest"},"tool_response":"E: boom at line 34"}' | bash "$HOOKS/observe-log.sh")
|
|
111
|
+
OUT=$(echo '{"hook_event_name":"PostToolUseFailure","session_id":"s1","tool_name":"Bash","tool_input":{"command":"pytest"},"tool_response":"E: boom at line 56"}' | bash "$HOOKS/observe-log.sh")
|
|
112
|
+
check "tripwire on 3 normalized repeats" "escalation packet" "$OUT"
|
|
113
|
+
OUT=$(echo '{"hook_event_name":"PostToolUseFailure","session_id":"s1","tool_name":"Bash","tool_input":{"command":"npm test"},"tool_response":"E: different failure"}' | bash "$HOOKS/observe-log.sh")
|
|
114
|
+
check "no tripwire on a different error" "EMPTY" "$OUT"
|
|
61
115
|
|
|
62
116
|
# ── sensor-post-edit.sh ──────────────────────────────────────────
|
|
63
117
|
echo "sensor-post-edit.sh"
|
|
@@ -96,46 +150,90 @@ OUT=$(echo '{"session_id":"s1"}' | bash "$HOOKS/stop-gate.sh")
|
|
|
96
150
|
check "silent when tests pass" "EMPTY" "$OUT"
|
|
97
151
|
[ -f "$TMP/.harness/logs/pending_test.s1" ] && MARKER_LEFT="yes" || MARKER_LEFT=""
|
|
98
152
|
check "clears marker on pass" "EMPTY" "$MARKER_LEFT"
|
|
153
|
+
# trace rules: require-before-stop blocks once, then lets the stop pass
|
|
154
|
+
fresh_project
|
|
155
|
+
printf 'LINT_CMD=""\nTEST_CMD="true"\n' > "$TMP/.harness/commands.env"
|
|
156
|
+
printf '# comment line\nrequire-before-stop npm.run.build Run the build before ending the turn.\n' > "$TMP/.harness/trace.rules"
|
|
157
|
+
echo "a.py" > "$TMP/.harness/logs/pending_test.s1"
|
|
158
|
+
echo '{"ts":"2026-01-01T00:00:00","session_id":"s1","event":"PostToolUse","tool":"Edit","summary":"a.py"}' > "$TMP/.harness/logs/tool_calls.jsonl"
|
|
159
|
+
OUT=$(echo '{"session_id":"s1"}' | bash "$HOOKS/stop-gate.sh")
|
|
160
|
+
check "trace rule blocks on missing required step" "trace rule" "$OUT"
|
|
161
|
+
OUT=$(echo '{"session_id":"s1"}' | bash "$HOOKS/stop-gate.sh")
|
|
162
|
+
check "trace rule blocks only once per session" "EMPTY" "$OUT"
|
|
163
|
+
echo "a.py" > "$TMP/.harness/logs/pending_test.s2"
|
|
164
|
+
echo '{"ts":"2026-01-01T00:01:00","session_id":"s2","event":"PostToolUse","tool":"Bash","summary":"npm run build"}' >> "$TMP/.harness/logs/tool_calls.jsonl"
|
|
165
|
+
OUT=$(echo '{"session_id":"s2"}' | bash "$HOOKS/stop-gate.sh")
|
|
166
|
+
check "trace rule satisfied by matching log entry" "EMPTY" "$OUT"
|
|
167
|
+
rm -f "$TMP/.harness/trace.rules"
|
|
168
|
+
|
|
169
|
+
# ── session-end.sh ───────────────────────────────────────────────
|
|
170
|
+
echo "session-end.sh"
|
|
171
|
+
fresh_project
|
|
172
|
+
OUT=$(echo '{"hook_event_name":"SessionEnd","session_id":"s9","reason":"exit"}' | bash "$HOOKS/session-end.sh")
|
|
173
|
+
check "silent with no session activity" "EMPTY" "$([ -f "$TMP/.harness/logs/sessions.jsonl" ] && echo present)"
|
|
174
|
+
cat > "$TMP/.harness/logs/tool_calls.jsonl" <<'EOF'
|
|
175
|
+
{"ts":"2026-01-01T10:00:00","session_id":"s9","event":"PostToolUse","tool":"Bash","summary":"ls"}
|
|
176
|
+
{"ts":"2026-01-01T10:05:00","session_id":"s9","event":"PostToolUse","tool":"Edit","summary":"a.py"}
|
|
177
|
+
{"ts":"2026-01-01T10:06:00","session_id":"s9","event":"PostToolUseFailure","tool":"Bash","error":"boom"}
|
|
178
|
+
{"ts":"2026-01-01T10:07:00","session_id":"other","event":"PostToolUse","tool":"Bash","summary":"ls"}
|
|
179
|
+
EOF
|
|
180
|
+
echo "a.py" > "$TMP/.harness/logs/pending_test.s9"
|
|
181
|
+
echo "2" > "$TMP/.harness/logs/stop_blocks.s9"
|
|
182
|
+
OUT=$(echo '{"hook_event_name":"SessionEnd","session_id":"s9","reason":"exit"}' | bash "$HOOKS/session-end.sh")
|
|
183
|
+
ROW=$(cat "$TMP/.harness/logs/sessions.jsonl" 2>/dev/null)
|
|
184
|
+
check "writes one ledger row" '"session_id": "s9"' "$ROW"
|
|
185
|
+
check "counts calls for this session only" '"calls": 3' "$ROW"
|
|
186
|
+
check "counts failures and edits" '"failures": 1' "$ROW"
|
|
187
|
+
check "records duration" '"duration_s": 360' "$ROW"
|
|
188
|
+
check "records unclean test state" '"tests_clean": false' "$ROW"
|
|
189
|
+
check "records stop blocks" '"stop_blocks": 2' "$ROW"
|
|
190
|
+
[ -f "$TMP/.harness/logs/pending_test.s9" ] && LEFT="yes" || LEFT=""
|
|
191
|
+
check "cleans session marker files" "EMPTY" "$LEFT"
|
|
99
192
|
|
|
100
193
|
# ── session-start.sh ─────────────────────────────────────────────
|
|
101
194
|
echo "session-start.sh"
|
|
102
195
|
fresh_project
|
|
103
|
-
OUT=$(bash "$HOOKS/session-start.sh")
|
|
196
|
+
OUT=$(echo '{"hook_event_name":"SessionStart","source":"startup"}' | bash "$HOOKS/session-start.sh")
|
|
104
197
|
check "silent without checkpoint" "EMPTY" "$OUT"
|
|
105
198
|
mkdir -p "$TMP/.harness/state"
|
|
106
199
|
echo '{"task_id":"t1","status":"in_progress","next_step":"do X"}' > "$TMP/.harness/state/progress.json"
|
|
107
|
-
OUT=$(bash "$HOOKS/session-start.sh")
|
|
200
|
+
OUT=$(echo '{"hook_event_name":"SessionStart","source":"startup"}' | bash "$HOOKS/session-start.sh")
|
|
108
201
|
check "injects in-progress checkpoint" ".harness/state" "$OUT"
|
|
109
202
|
echo '{"task_id":"t1","status":"completed"}' > "$TMP/.harness/state/progress.json"
|
|
110
|
-
OUT=$(bash "$HOOKS/session-start.sh")
|
|
203
|
+
OUT=$(echo '{"hook_event_name":"SessionStart","source":"startup"}' | bash "$HOOKS/session-start.sh")
|
|
111
204
|
check "skips completed checkpoint" "EMPTY" "$OUT"
|
|
112
205
|
rm "$TMP/.harness/state/progress.json"
|
|
113
206
|
# legacy layout (pre-v0.2): checkpoint at the project root still read
|
|
114
207
|
echo '{"task_id":"t1","status":"in_progress","next_step":"do X"}' > "$TMP/progress.json"
|
|
115
|
-
OUT=$(bash "$HOOKS/session-start.sh")
|
|
208
|
+
OUT=$(echo '{"hook_event_name":"SessionStart","source":"startup"}' | bash "$HOOKS/session-start.sh")
|
|
116
209
|
check "reads legacy root checkpoint" "progress.json" "$OUT"
|
|
117
210
|
rm "$TMP/progress.json"
|
|
118
211
|
# CLAUDE.md still pointing checkpoints at the root -> propose-a-diff nudge
|
|
119
212
|
printf '# P\n- write the plan to plan.md\n' > "$TMP/CLAUDE.md"
|
|
120
|
-
OUT=$(bash "$HOOKS/session-start.sh")
|
|
213
|
+
OUT=$(echo '{"hook_event_name":"SessionStart","source":"startup"}' | bash "$HOOKS/session-start.sh")
|
|
121
214
|
check "nudges CLAUDE.md path migration" "user's approval" "$OUT"
|
|
122
215
|
printf '# P\n- write the plan to .harness/state/plan.md\n' > "$TMP/CLAUDE.md"
|
|
123
|
-
OUT=$(bash "$HOOKS/session-start.sh")
|
|
216
|
+
OUT=$(echo '{"hook_event_name":"SessionStart","source":"startup"}' | bash "$HOOKS/session-start.sh")
|
|
124
217
|
check "silent when CLAUDE.md paths are current" "EMPTY" "$OUT"
|
|
125
218
|
rm "$TMP/CLAUDE.md"
|
|
126
219
|
NOW=$(python3 -c 'import datetime; print(datetime.datetime.now().isoformat(timespec="seconds"))')
|
|
127
220
|
for _ in 1 2 3; do
|
|
128
221
|
echo "{\"ts\":\"$NOW\",\"session_id\":\"old\",\"event\":\"PostToolUseFailure\",\"tool\":\"Bash\",\"error\":\"x\"}" >> "$TMP/.harness/logs/tool_calls.jsonl"
|
|
129
222
|
done
|
|
130
|
-
OUT=$(bash "$HOOKS/session-start.sh")
|
|
223
|
+
OUT=$(echo '{"hook_event_name":"SessionStart","source":"startup"}' | bash "$HOOKS/session-start.sh")
|
|
131
224
|
check "nudges /ratchet on accumulated failures" "/ratchet" "$OUT"
|
|
132
225
|
fresh_project
|
|
133
226
|
printf 'LINT_CMD=""\nTEST_CMD=""\n' > "$TMP/.harness/commands.env"
|
|
134
|
-
OUT=$(bash "$HOOKS/session-start.sh")
|
|
227
|
+
OUT=$(echo '{"hook_event_name":"SessionStart","source":"startup"}' | bash "$HOOKS/session-start.sh")
|
|
135
228
|
check "offers /harness-init when unconfigured" "/harness-init" "$OUT"
|
|
136
229
|
printf 'LINT_CMD="true"\nTEST_CMD=""\n' > "$TMP/.harness/commands.env"
|
|
137
|
-
OUT=$(bash "$HOOKS/session-start.sh")
|
|
230
|
+
OUT=$(echo '{"hook_event_name":"SessionStart","source":"startup"}' | bash "$HOOKS/session-start.sh")
|
|
138
231
|
check "no setup nudge once configured" "EMPTY" "$OUT"
|
|
232
|
+
# resuming right after a compaction -> checkpoint-refresh instruction
|
|
233
|
+
# (pre-compact.sh cannot inject context; this is the supported channel)
|
|
234
|
+
mkdir -p "$TMP/.harness/state/.pre-compact-backup"
|
|
235
|
+
OUT=$(echo '{"hook_event_name":"SessionStart","source":"compact"}' | bash "$HOOKS/session-start.sh")
|
|
236
|
+
check "instructs checkpoint refresh after compaction" "pre-compact-backup" "$OUT"
|
|
139
237
|
|
|
140
238
|
# ── result ───────────────────────────────────────────────────────
|
|
141
239
|
echo
|
|
@@ -35,6 +35,7 @@ check "installs hooks" "yes" "$([ -f "$TGT/.claude/hooks/stop-gate.sh" ] && echo
|
|
|
35
35
|
check "hooks are executable" "yes" "$([ -x "$TGT/.claude/hooks/stop-gate.sh" ] && echo yes)"
|
|
36
36
|
check "installs skills" "yes" "$([ -f "$TGT/.claude/skills/ratchet/SKILL.md" ] && echo yes)"
|
|
37
37
|
check "installs CLAUDE.md" "yes" "$([ -f "$TGT/CLAUDE.md" ] && echo yes)"
|
|
38
|
+
check "installs AGENTS.md" "yes" "$([ -f "$TGT/AGENTS.md" ] && echo yes)"
|
|
38
39
|
check "installs settings.json" "yes" "$([ -f "$TGT/.claude/settings.json" ] && echo yes)"
|
|
39
40
|
check "writes manifest" "yes" "$([ -f "$TGT/.harness/kit-manifest.json" ] && echo yes)"
|
|
40
41
|
check "gitignores logs" ".harness/logs/" "$(cat "$TGT/.gitignore")"
|
|
@@ -50,6 +51,7 @@ check "does not ship test_installer.sh" "EMPTY" "$([ -f "$TGT/harness/tests/test
|
|
|
50
51
|
echo "init (brownfield)"
|
|
51
52
|
BF=$(mktemp -d)
|
|
52
53
|
echo "MY PROJECT RULES" > "$BF/CLAUDE.md"
|
|
54
|
+
echo "MY OWN AGENTS FILE" > "$BF/AGENTS.md"
|
|
53
55
|
echo "node_modules/" > "$BF/.gitignore"
|
|
54
56
|
mkdir -p "$BF/.claude"
|
|
55
57
|
cat > "$BF/.claude/settings.json" <<'EOF'
|
|
@@ -64,6 +66,7 @@ cat > "$BF/.claude/settings.json" <<'EOF'
|
|
|
64
66
|
EOF
|
|
65
67
|
OUT=$(node "$CLI" init "$BF" 2>&1)
|
|
66
68
|
check "keeps existing CLAUDE.md" "MY PROJECT RULES" "$(cat "$BF/CLAUDE.md")"
|
|
69
|
+
check "keeps existing AGENTS.md" "MY OWN AGENTS FILE" "$(cat "$BF/AGENTS.md")"
|
|
67
70
|
check "notes CLAUDE.md was kept" "existing CLAUDE.md was kept" "$OUT"
|
|
68
71
|
check "keeps user permissions" '"Read"' "$(cat "$BF/.claude/settings.json")"
|
|
69
72
|
check "keeps user's own hook" "my-own-hook.sh" "$(cat "$BF/.claude/settings.json")"
|
|
@@ -108,6 +111,20 @@ sed -i.bak '/.harness\/state/d' "$TGT/.gitignore" && rm -f "$TGT/.gitignore.bak"
|
|
|
108
111
|
OUT=$(node "$CLI" update "$TGT" 2>&1)
|
|
109
112
|
check "update re-adds state gitignore entry" ".harness/state/" "$(cat "$TGT/.gitignore")"
|
|
110
113
|
|
|
114
|
+
# update on an install from before a hook existed must register the new
|
|
115
|
+
# hook in settings.json (mergeSettings runs on update too, append-only)
|
|
116
|
+
python3 - "$TGT" <<'EOF'
|
|
117
|
+
import json, sys, os
|
|
118
|
+
sp = os.path.join(sys.argv[1], '.claude/settings.json')
|
|
119
|
+
s = json.load(open(sp))
|
|
120
|
+
s['hooks'].pop('SessionEnd', None)
|
|
121
|
+
s['user_marker'] = 'kept'
|
|
122
|
+
json.dump(s, open(sp, 'w'), indent=2)
|
|
123
|
+
EOF
|
|
124
|
+
OUT=$(node "$CLI" update "$TGT" 2>&1)
|
|
125
|
+
check "update registers newly shipped hooks" "session-end.sh" "$(cat "$TGT/.claude/settings.json")"
|
|
126
|
+
check "update merge keeps user settings" '"user_marker"' "$(cat "$TGT/.claude/settings.json")"
|
|
127
|
+
|
|
111
128
|
# pre-v0.2 CLAUDE.md pointing checkpoints at the project root -> migration
|
|
112
129
|
# note (told, never touched); the current template must NOT trigger it
|
|
113
130
|
printf '# old project\n- read progress.json first\n' > "$TGT/CLAUDE.md"
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "uni-harness",
|
|
3
|
-
"version": "0.
|
|
3
|
+
"version": "0.4.0",
|
|
4
4
|
"description": "Agent harness kit for Claude Code — installs verification sensors, destructive-command guards, checkpoint recovery, observability logs, and ratchet skills into your project",
|
|
5
5
|
"bin": {
|
|
6
6
|
"uni-harness": "bin/cli.js"
|
|
@@ -8,6 +8,7 @@
|
|
|
8
8
|
"files": [
|
|
9
9
|
"bin",
|
|
10
10
|
"CLAUDE.md",
|
|
11
|
+
"AGENTS.md",
|
|
11
12
|
".claude",
|
|
12
13
|
".harness/commands.env",
|
|
13
14
|
"harness",
|