rungate 0.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.claude/agents/aditi.md +47 -0
- package/.claude/agents/marcus.md +50 -0
- package/.claude/agents/quinn.md +58 -0
- package/.claude/agents/rook.md +47 -0
- package/.claude/agents/serena.md +46 -0
- package/.claude/rungate.json +11 -0
- package/.fallowrc.json +28 -0
- package/.github/copilot-instructions.md +20 -0
- package/.rungate/conformity-findings.json +648 -0
- package/AGENTS.md +106 -0
- package/CODE-MAP.md +50 -0
- package/HARNESS.md +88 -0
- package/bun.lock +26 -0
- package/bunfig.toml +2 -0
- package/config/brief-policies.json +28 -0
- package/config/ceremony-profiles.json +212 -0
- package/config/consumer-cascade-map.json +24 -0
- package/config/doc-cascade-map.json +24 -0
- package/config/eval.json +84 -0
- package/config/profiles/account-plan.yaml +36 -0
- package/config/profiles/campaign.yaml +94 -0
- package/config/prove-evidence.schema.json +65 -0
- package/config/spec-policies.json +36 -0
- package/config/workflow-schema.json +281 -0
- package/gates/SCHEMA-GUIDE.md +121 -0
- package/gates/ac-quality.test.ts +106 -0
- package/gates/adversarial.test.ts +274 -0
- package/gates/brief-assembler.test.ts +429 -0
- package/gates/brief-assembler.ts +334 -0
- package/gates/bunfig.toml +2 -0
- package/gates/chain.test.ts +91 -0
- package/gates/e2e-smoke.test.ts +234 -0
- package/gates/error-classifier.test.ts +249 -0
- package/gates/error-classifier.ts +89 -0
- package/gates/orchestrator.test.ts +950 -0
- package/gates/orchestrator.ts +569 -0
- package/gates/preload.ts +77 -0
- package/gates/prompts/ac-adversary.md +20 -0
- package/gates/prompts/evidence-validator.md +22 -0
- package/gates/prompts/prove-reproducer.md +53 -0
- package/gates/prove.test.ts +164 -0
- package/gates/run-gate.ts +936 -0
- package/gates/schema-parity.test.ts +197 -0
- package/gates/schema.ts +362 -0
- package/gates/self-heal.test.ts +511 -0
- package/gates/self-heal.ts +121 -0
- package/gates/ship-orchestrator.test.ts +379 -0
- package/gates/ship-orchestrator.ts +363 -0
- package/gates/test-fixtures/broken-no-environments.json +60 -0
- package/gates/test-fixtures/valid-standard.json +54 -0
- package/gates/test-utils.ts +43 -0
- package/gates/witness.ts +220 -0
- package/gates/workflow.test.ts +1053 -0
- package/hooks/AgentBriefGuard.hook.ts +586 -0
- package/hooks/AgentVerdictCapture.hook.ts +134 -0
- package/hooks/AutoVerifyGate.hook.ts +76 -0
- package/hooks/GateEnforcement.hook.ts +273 -0
- package/hooks/IssueCloseGuard.hook.ts +116 -0
- package/hooks/MergeGuard.hook.ts +55 -0
- package/hooks/StaleTTLCleanup.hook.ts +133 -0
- package/hooks/VerifyPhaseLock.hook.ts +81 -0
- package/hooks/WorkflowStateGuard.hook.ts +37 -0
- package/hooks/lib/agentDetection.ts +70 -0
- package/hooks/lib/findWorkflow.ts +101 -0
- package/hooks/lib/parseStdin.ts +37 -0
- package/hooks/lib/paths.ts +111 -0
- package/hooks/lib/utils.ts +81 -0
- package/lib/conformity.ts +959 -0
- package/lib/paths.ts +23 -0
- package/lib/rungate-schema.ts +60 -0
- package/lib/signal-phrases.ts +7 -0
- package/package.json +20 -0
- package/prompts/ac-adversary.md +20 -0
- package/prompts/ac-format.md +34 -0
- package/prompts/aditi.md +29 -0
- package/prompts/blast-radius.md +29 -0
- package/prompts/coding-principles.md +34 -0
- package/prompts/container-rebuild.md +29 -0
- package/prompts/container-verify.md +28 -0
- package/prompts/discovery.md +27 -0
- package/prompts/environment.md +28 -0
- package/prompts/escalation-decision-tree.md +38 -0
- package/prompts/evidence-hierarchy.md +31 -0
- package/prompts/evidence-validator.md +22 -0
- package/prompts/marcus.md +29 -0
- package/prompts/prevention.md +26 -0
- package/prompts/prove-reproducer.md +43 -0
- package/prompts/quinn-decision-tree.md +31 -0
- package/prompts/quinn-ui-brief.md +106 -0
- package/prompts/quinn.md +29 -0
- package/prompts/rca.md +28 -0
- package/prompts/read-before-write.md +30 -0
- package/prompts/regression.md +26 -0
- package/prompts/rook.md +29 -0
- package/prompts/serena.md +28 -0
- package/prompts/testing-strategy.md +34 -0
- package/reference/migration-manifest.json +554 -0
- package/reference/prompts/agent-aditi.md +38 -0
- package/reference/prompts/agent-marcus.md +38 -0
- package/reference/prompts/agent-quinn.md +38 -0
- package/reference/prompts/agent-rook.md +37 -0
- package/reference/prompts/agent-serena.md +37 -0
- package/reference/prompts/council-skill.md +170 -0
- package/reference/prompts/goal-skill.md +235 -0
- package/reference/prompts/prove-skill.md +283 -0
- package/reference/prompts/quinn-standard.md +246 -0
- package/reference/prompts/ship-convergence.md +65 -0
- package/reference/prompts/ship-escalation.md +43 -0
- package/reference/prompts/ship-iteration.md +382 -0
- package/reference/prompts/ship-pending-patches.md +246 -0
- package/reference/prompts/ship-reference.md +250 -0
- package/reference/prompts/ship-skill.md +347 -0
- package/reference/prompts/ship-templates.md +215 -0
- package/reference/scripts/batch-status.sh +68 -0
- package/reference/scripts/chain-compat-check.sh +67 -0
- package/reference/scripts/context-audit.sh +314 -0
- package/reference/scripts/dead-export-check.sh +61 -0
- package/reference/scripts/generate-qc-tables.sh +92 -0
- package/reference/scripts/harness-init.sh +111 -0
- package/reference/scripts/pai-054-baseline-check.sh +40 -0
- package/reference/scripts/prove-smoke-test.sh +112 -0
- package/reference/scripts/setup-keychain-salt.sh +12 -0
- package/reference/scripts/skill-optimizer-friday.sh +88 -0
- package/reference/scripts/skill-qc-test.sh +351 -0
- package/reference/scripts/skill-synthesize-tonight.sh +66 -0
- package/reference/specs/HARNESS-EXTRACTION-SPEC.md +263 -0
- package/reference/specs/harness-v3-bash-deletion-plan.md +65 -0
- package/reference/specs/harness-v3-migration-plan.md +188 -0
- package/scripts/decision-reconcile.sh +74 -0
- package/scripts/decision-reconcile.ts +152 -0
- package/scripts/extract-constraints.ts +260 -0
- package/scripts/generate-code-map.ts +339 -0
- package/scripts/git-hooks/pre-commit +76 -0
- package/scripts/git-hooks/pre-push +271 -0
- package/scripts/harness-skill-check.sh +52 -0
- package/scripts/lib/gap-emit.sh +65 -0
- package/scripts/lib/jq-update.sh +30 -0
- package/scripts/lib/resolve-slug.sh +30 -0
- package/scripts/lib/verify-spec-compliance.sh +102 -0
- package/scripts/prove-backfill.sh +87 -0
- package/scripts/scaffold-project.ts +1488 -0
- package/scripts/scaffold-rungate-config.ts +91 -0
- package/scripts/self-containment-check.sh +109 -0
- package/scripts/skill-qc-validator.sh +402 -0
- package/scripts/slug-resolver.sh +14 -0
- package/scripts/sync-spec-tests.ts +236 -0
- package/specs/BOOTSTRAP-DATA-FLOW-SPEC.md +2493 -0
- package/specs/BOOTSTRAP-TEST-PLAN.md +571 -0
- package/specs/HARNESS-GATES.md +256 -0
- package/specs/HARNESS-SKILL-CHAIN.md +286 -0
- package/specs/HARNESS-SKILL-CONTRACT.md +93 -0
- package/specs/HARNESS-STANDARD.md +488 -0
- package/specs/SPEC-TEMPLATE.md +53 -0
- package/specs/bootstrap-data-flow.png +0 -0
- package/specs/harness-automation-matrix.md +314 -0
- package/specs/harness-automation-matrix.rules.json +64 -0
- package/test/anti.test.ts +80 -0
- package/test/contract-negative.test.ts +86 -0
- package/test/contract.test.ts +158 -0
- package/test/external-deps.test.ts +48 -0
- package/test/fixtures/golden-project/.env.example +3 -0
- package/test/fixtures/golden-project/Makefile +7 -0
- package/test/fixtures/golden-project/docs/adr/ADR-001-framework.md +23 -0
- package/test/fixtures/golden-project/docs/setup-guide.md +10 -0
- package/test/fixtures/golden-project/package.json +10 -0
- package/test/fixtures/golden-project/specs/api-spec.md +20 -0
- package/test/fixtures/golden-project/src/auth.ts +8 -0
- package/test/fixtures/golden-project/src/db.ts +11 -0
- package/test/fixtures/golden-project/src/index.ts +19 -0
- package/test/fixtures/golden-project/src/utils/format.ts +3 -0
- package/test/fixtures/golden-project/test/auth.test.ts +15 -0
- package/test/fixtures/golden-project/tsconfig.json +12 -0
- package/test/meta-sc-coverage.test.ts +159 -0
- package/test/phase-0.test.ts +846 -0
- package/test/phase-1.test.ts +617 -0
- package/test/phase-2.test.ts +620 -0
- package/test/phase-3.test.ts +99 -0
- package/test/scaffold-conformity.test.ts +8 -0
- package/test/schema-canary.test.ts +51 -0
- package/test/spec-compliance-auto.test.ts +64 -0
- package/test/spec-compliance.test.ts +295 -0
- package/test/spec-discovery.test.ts +68 -0
- package/test/spec-drift.test.ts +5 -0
- package/test/structure.test.ts +165 -0
- package/test/unit/agent-brief-guard-ship.test.ts +293 -0
- package/test/unit/extract-constraints.test.ts +145 -0
- package/test/unit/issue-close-guard.test.ts +204 -0
- package/test/unit/scaffold-project.test.ts +417 -0
- package/test/unit/skill-cooccurrence.test.ts +177 -0
- package/test/unit/skill-sequence-logger.test.ts +244 -0
- package/test/unit/workflow-state-guard.test.ts +95 -0
- package/workflows/batch-ship.js +295 -0
- package/workflows/council.js +481 -0
- package/workflows/prove.js +560 -0
- package/workflows/ship.js +789 -0
- package/workflows/verify.js +236 -0
|
@@ -0,0 +1,256 @@
|
|
|
1
|
+
---
|
|
2
|
+
doc-type: reference
|
|
3
|
+
status: active
|
|
4
|
+
owner: jason
|
|
5
|
+
updated: 2026-09-17
|
|
6
|
+
testable: true
|
|
7
|
+
---
|
|
8
|
+
|
|
9
|
+
# Harness Gates
|
|
10
|
+
# Source of truth for all gate definitions, enforcement config, and learning state.
|
|
11
|
+
# Read by: gate scripts, GateEnforcement.hook.ts, HarnessFailureSurfacing.hook.ts, LearningReview.ts
|
|
12
|
+
# Written by: LearningReview.ts (nightly — tighten/loosen/prune), DA (manual edits)
|
|
13
|
+
|
|
14
|
+
---
|
|
15
|
+
|
|
16
|
+
## goal-gate
|
|
17
|
+
- **Step:** GOAL (optional — only for L-sized work via harness)
|
|
18
|
+
- **Fires after:** Skill("goal") completes
|
|
19
|
+
- **Script:** `skills/ship/goal-gate.sh --issue NUM`
|
|
20
|
+
- **Checks:**
|
|
21
|
+
- [ ] Issue has `## Goal` section
|
|
22
|
+
- [ ] Issue has `## Success Criteria` with SC- or AC- prefixed items
|
|
23
|
+
- [ ] SCs/ACs contain no vague words: exists, works, properly, correct, appropriate, should (without accompanying threshold number)
|
|
24
|
+
- [ ] Issue has `## Scope Boundary` with In + Out items
|
|
25
|
+
- [ ] Issue has `## Circuit Breakers` with iteration + cost limits
|
|
26
|
+
- [ ] Checkpoint written to `$RUNGATE_WORK_DIR/{slug}/`
|
|
27
|
+
- **Strike threshold:** 3
|
|
28
|
+
- **Escalated:** false
|
|
29
|
+
- **Skip history (30d):** 0 skips
|
|
30
|
+
|
|
31
|
+
---
|
|
32
|
+
|
|
33
|
+
## scope-gate
|
|
34
|
+
- **Step:** SCOPE (after sizing + ACs, before BUILD)
|
|
35
|
+
- **Fires after:** Ship SCOPE completes
|
|
36
|
+
- **Script:** `skills/ship/scope-gate.sh --issue NUM --slug SLUG`
|
|
37
|
+
- **Checks:**
|
|
38
|
+
- [ ] Issue has structured DISCOVERY comment with file:line citations (citation count > 0)
|
|
39
|
+
- [ ] PRINCIPLES.md referenced in DISCOVERY comment (if project has one)
|
|
40
|
+
- [ ] ARCHITECTURE.md referenced in DISCOVERY comment (if project has one)
|
|
41
|
+
- [ ] Research decision explicit: findings posted OR "no research needed" with evidence
|
|
42
|
+
- [ ] ACs posted to issue with AC-N IDs (count > 0)
|
|
43
|
+
- [ ] No vague words in AC statements without threshold numbers
|
|
44
|
+
- [ ] Sizing declaration posted: size + reasoning + expected files/time/boundaries
|
|
45
|
+
- [ ] Each AC tagged [CODE] or [OUTCOME]
|
|
46
|
+
- [ ] [OUTCOME] ACs have all 10 fields: Type, Statement, Metric, Threshold, Baseline, Evidence Method, Pass Criteria, Source, User, Conversation, Action
|
|
47
|
+
- [ ] [CODE] ACs have all 7 fields: Type, Statement, Metric, Threshold, Baseline, Evidence Method, Pass Criteria, Source
|
|
48
|
+
- [ ] Brief posted with ## Marcus Brief or ## Implementation Brief marker
|
|
49
|
+
- [ ] Brief contains template markers: ## Context, ## Task, ## Verify, ## Report back
|
|
50
|
+
- [ ] Adversarial goal test: ## Anti-Gaming table posted with defenses for every shortcut (M/L: required, XS/S: recommended)
|
|
51
|
+
- [ ] **QC validator (STANDARD+):** `skill-qc-validator.sh --skill goal` — runs G1-G8 checks from `schemas/skill-qc-checks.json`. Currently WARN; promotes to FAIL after calibration.
|
|
52
|
+
- **Strike threshold:** 3
|
|
53
|
+
- **Escalated:** false
|
|
54
|
+
- **Skip history (30d):** 0 skips
|
|
55
|
+
|
|
56
|
+
---
|
|
57
|
+
|
|
58
|
+
## verify-gate
|
|
59
|
+
- **Step:** VERIFY (after evidence collected, before DURABILITY)
|
|
60
|
+
- **Fires after:** Ship VERIFY completes
|
|
61
|
+
- **Script:** `skills/ship/verify-gate.sh --issue NUM --slug SLUG`
|
|
62
|
+
- **Checks:**
|
|
63
|
+
- [ ] Every AC-N in issue has matching evidence in completion report
|
|
64
|
+
- [ ] [OUTCOME] ACs have live evidence: screenshot URL, curl output block, or browser verification — not code review alone
|
|
65
|
+
- [ ] [CODE] ACs have file:line citation in evidence
|
|
66
|
+
- [ ] Evidence references threshold number from AC (number appears in evidence text)
|
|
67
|
+
- [ ] curl commands use correct test port (read from project CLAUDE.md)
|
|
68
|
+
- [ ] Consumer chain: if consumer files changed (from git diff + PRINCIPLES.md consumer list), 4 artifacts present: curl output, screenshot, rendered output, goal statement reference
|
|
69
|
+
- [ ] Report uses completion template markers from REFERENCE.md
|
|
70
|
+
- [ ] **QC validator (STANDARD+):** `skill-qc-validator.sh --skill ship` — runs S1-S8 checks from `schemas/skill-qc-checks.json`. Currently WARN; promotes to FAIL after calibration.
|
|
71
|
+
- **Strike threshold (ceremony checks):** 3
|
|
72
|
+
- **Strike threshold (OUTCOME AC failures):** 0 (immediate block, no skip)
|
|
73
|
+
- **AC failures:** Must fix — no skip allowed. Iterate until pass or circuit breaker.
|
|
74
|
+
- **Escalated:** false
|
|
75
|
+
- **Skip history (30d):** 0 skips
|
|
76
|
+
|
|
77
|
+
---
|
|
78
|
+
|
|
79
|
+
## durability-gate
|
|
80
|
+
- **Step:** DURABILITY (after VERIFY, before CLOSE)
|
|
81
|
+
- **Fires after:** verify-gate passes
|
|
82
|
+
- **Script:** `skills/ship/durability-gate.sh --issue NUM --slug SLUG`
|
|
83
|
+
- **Checks:**
|
|
84
|
+
- [ ] Doc cascade: git diff matched against doc-cascade-map.json
|
|
85
|
+
- [ ] Structured findings: FAIL ACs have FINDING: comments posted
|
|
86
|
+
- [ ] Quinn gate: .tsx changed → Quinn verification on issue
|
|
87
|
+
- [ ] Checkpoint freshness: updated within 2 hours
|
|
88
|
+
- [ ] Hedge-grep: zero hedged claims in issue comments (phrases from patterns.json `unverified_claim_phrases`)
|
|
89
|
+
- [ ] Rook gate: M/L size → Rook scan on issue
|
|
90
|
+
- [ ] Boundary crossing: XS/S + 2+ dirs → WARN
|
|
91
|
+
- [ ] ADR status: referenced ADRs not "proposed"
|
|
92
|
+
- [ ] Issue/brief current: if FINDING posted after ACs, ACs updated
|
|
93
|
+
- [ ] Follow-ups logged: TODO/FOLLOWUP markers have corresponding issues
|
|
94
|
+
- [ ] Failure pattern acknowledged: top pattern from HarnessFailureSurfacing referenced
|
|
95
|
+
- **Strike threshold:** 3
|
|
96
|
+
- **Escalated:** false
|
|
97
|
+
- **Skip history (30d):** 0 skips
|
|
98
|
+
|
|
99
|
+
---
|
|
100
|
+
|
|
101
|
+
## close-gate
|
|
102
|
+
- **Step:** CLOSE (after DURABILITY, before next issue)
|
|
103
|
+
- **Fires after:** Ship CLOSE actions
|
|
104
|
+
- **Script:** `skills/ship/close-gate.sh --issue NUM --slug SLUG`
|
|
105
|
+
- **Checks:**
|
|
106
|
+
- [ ] Issue has `## Completion Report` comment
|
|
107
|
+
- [ ] doc-hygiene invoked this session (from signals.jsonl)
|
|
108
|
+
- [ ] Follow-up tracking: Marcus unrelated findings, Quinn/Rook FINDINGs, council scope_change — all have corresponding issues. Parent has `has-follow-ups` label if children exist. Children have `follow-up` + `needs-triage` labels
|
|
109
|
+
- [ ] Sizing outcome posted: `## Sizing Outcome` with actual files/time/dirs vs declared
|
|
110
|
+
- [ ] Checkpoint marked complete
|
|
111
|
+
- [ ] Routing outcomes logged: if ATTEMPT comments exist, routing_outcome entries in signals.jsonl with required fields
|
|
112
|
+
- [ ] Issue labeled `shipped`
|
|
113
|
+
- **Strike threshold:** 3
|
|
114
|
+
- **Escalated:** false
|
|
115
|
+
- **Skip history (30d):** 0 skips
|
|
116
|
+
|
|
117
|
+
---
|
|
118
|
+
|
|
119
|
+
## iteration-gate
|
|
120
|
+
- **Step:** ITERATION (after VERIFY fails, before looping back)
|
|
121
|
+
- **Fires at:** Every iteration boundary
|
|
122
|
+
- **Script:** `skills/ship/iteration-gate.sh --issue NUM --slug SLUG`
|
|
123
|
+
- **Checks:**
|
|
124
|
+
- [ ] ATTEMPT comment posted for this iteration (count > previous)
|
|
125
|
+
- [ ] ATTEMPT has required fields: Approach, Files Changed, Result, Evidence, Why It Failed
|
|
126
|
+
- [ ] Iteration count tracked in stuck-detection.json
|
|
127
|
+
- [ ] Same error hash not repeated 3x (hash "Why it failed" sections)
|
|
128
|
+
- [ ] Iteration count < circuit breaker limit from issue
|
|
129
|
+
- [ ] Wall clock < 90 minutes (or custom limit from issue)
|
|
130
|
+
- [ ] Progress check: ≥1 new AC passed since last iteration, or new information acquired
|
|
131
|
+
- [ ] Goal Audit: no baseline drift >20%, no entity non-existence, no research contradiction
|
|
132
|
+
- [ ] Checkpoint has compaction fields: Current State, Next Action, Files Changed, Gate Status
|
|
133
|
+
- [ ] Squashed commits: ≤1 new commit since last iteration on feature branch
|
|
134
|
+
- **Tier 1 threshold:** 3 iterations → auto-escalate via routing-decision.sh
|
|
135
|
+
- **Tier 2 threshold:** 6 iterations (3 pre + 3 post-escalation) → hard stop
|
|
136
|
+
- **Meta-circuit-breaker:** Max 2 goal amendments per issue
|
|
137
|
+
- **Escalated:** false
|
|
138
|
+
- **Skip history (30d):** 0 skips
|
|
139
|
+
|
|
140
|
+
---
|
|
141
|
+
|
|
142
|
+
## Gate Enforcement Model
|
|
143
|
+
|
|
144
|
+
**Strike escalation (ceremony gates):**
|
|
145
|
+
- Strike 1-2: system-reminder injected on every tool call
|
|
146
|
+
- Strike 3: hard block on Skill calls (Bash/Read/Agent still work for fixing)
|
|
147
|
+
- Escape: fix the failure OR post skip-reason to issue → block clears → skip logged
|
|
148
|
+
|
|
149
|
+
**AC failures (non-skippable):**
|
|
150
|
+
- [CODE] AC fail: iterate, no skip. Circuit breaker is only escape.
|
|
151
|
+
- [OUTCOME] AC fail: immediate block, iterate, no skip. Circuit breaker is only escape.
|
|
152
|
+
|
|
153
|
+
**Self-heal:** DA can use Bash/Read/Agent to resolve gate failures. Only Skill calls are blocked.
|
|
154
|
+
|
|
155
|
+
**Skip logging:** Every skip gets logged to signals.jsonl with type: "gate_skip". LearningReview reads these nightly.
|
|
156
|
+
|
|
157
|
+
**De-escalation:** Gates with 0 skips in 60 days → strike threshold resets to default (3). LearningReview manages this.
|
|
158
|
+
|
|
159
|
+
**Pruning:** Checks with 0 catches in 90 days → retired from gate definition. Skip history older than 30 days → pruned.
|
|
160
|
+
|
|
161
|
+
---
|
|
162
|
+
|
|
163
|
+
## Agent Enforcement (PreToolUse on Agent)
|
|
164
|
+
|
|
165
|
+
**Applies to:** All named agent spawns (Marcus, Quinn, Rook, Serena, Aditi)
|
|
166
|
+
|
|
167
|
+
- Brief must contain template markers (## Context, ## Task, ## Verify, ## Report back)
|
|
168
|
+
- `mode: "bypassPermissions"` must be set
|
|
169
|
+
- `max_turns` set by sizing: XS=20, S=30, M=50, L=75
|
|
170
|
+
- Single-attempt constraint injected: "STOP on persistent error, report what happened"
|
|
171
|
+
- Isolation routing: `.claude/` repo → worktree, external → direct
|
|
172
|
+
- Post-report: hedge-grep + spiral detection before accepting
|
|
173
|
+
- Brief block: if template markers missing → `{"decision": "block"}`
|
|
174
|
+
- bypassPermissions block: if missing → `{"decision": "block"}`
|
|
175
|
+
|
|
176
|
+
---
|
|
177
|
+
|
|
178
|
+
## Routing Table
|
|
179
|
+
|
|
180
|
+
**Decision tree (deterministic — from 109 real failure resolutions):**
|
|
181
|
+
|
|
182
|
+
1. Did agent read relevant files? (grep ATTEMPT for file reads) → NO: back to agent with file list
|
|
183
|
+
2. Did agent run it and check output? (grep for curl/screenshot/test) → NO: back to agent with evidence method
|
|
184
|
+
3. Framework/library error? (grep for framework names) → YES: Context7 first, then Research
|
|
185
|
+
4. Scope/requirements mismatch? (grep for "doesn't match", "expected") → YES: re-read issue + PRINCIPLES.md, then council
|
|
186
|
+
5. 3+ failed attempts? (count ATTEMPTs) → YES: council — approach is wrong
|
|
187
|
+
6. DEFAULT: Research — missing information
|
|
188
|
+
|
|
189
|
+
**Accuracy tracking:** Every routing decision logged to signals.jsonl with route + resolved outcome. LearningReview reorders steps by resolution rate, adds new patterns from successes, removes patterns with 0 matches in 90 days, demotes routes with <30% accuracy.
|
|
190
|
+
|
|
191
|
+
---
|
|
192
|
+
|
|
193
|
+
## Scoring
|
|
194
|
+
|
|
195
|
+
**Per-issue scorecard** (score-issue.sh):
|
|
196
|
+
- Gate pass per step (first-pass rate)
|
|
197
|
+
- Iteration count
|
|
198
|
+
- Sizing accuracy (declared vs actual)
|
|
199
|
+
- Time within budget
|
|
200
|
+
- Follow-ups created
|
|
201
|
+
- Routing used
|
|
202
|
+
- Letter grade: A (≥90% first-pass, ≤1 iter), B (≥75%, ≤2), C (≥60%, ≤3), D (<60% or Tier 2), F (hard stop)
|
|
203
|
+
|
|
204
|
+
**System dashboard** (nightly via LearningReview):
|
|
205
|
+
- Gate health: pass rates + top failure + trend per gate
|
|
206
|
+
- Issue metrics: avg grade, first-pass, iterations, sizing accuracy
|
|
207
|
+
- Routing performance: accuracy per route
|
|
208
|
+
- Gate escalation status
|
|
209
|
+
|
|
210
|
+
---
|
|
211
|
+
|
|
212
|
+
## Gate Infrastructure (from adversarial council 2026-09-17)
|
|
213
|
+
|
|
214
|
+
Source: 4-member adversarial council, 3 rounds, unanimous on all findings. Root cause analysis of #1436 6-gate-run failure.
|
|
215
|
+
|
|
216
|
+
### Error Surfacing
|
|
217
|
+
|
|
218
|
+
Gate failure detail must include actual Zod validation error paths — not hardcoded "failed" strings. The Zod errors (path + message) are computed at `workflow.test.ts:59` but discarded at `run-gate.ts:291` where `detail` is hardcoded to `"failed"`. The heal agent at `ship.js:152-159` receives failures with no specifics, making single-pass repair impossible.
|
|
219
|
+
|
|
220
|
+
**Rule:** After matching a `(fail)` test line, `run-gate.ts` must capture subsequent non-result lines as the detail string. Detail must include the Zod error path (e.g., `acs.0.threshold.op: Invalid enum value. Expected '==' | '>=' | ..., received 'equals'`).
|
|
221
|
+
|
|
222
|
+
### Schema Validation at Write Time
|
|
223
|
+
|
|
224
|
+
All writes to `workflow-state.json` must go through `writeWorkflowState()` at `orchestrator.ts:414-430` which validates via Zod with enum-specific error formatting. Per ADR-009:417-418: "writeWorkflowState() validates via Zod" — this is a design decision, not optional.
|
|
225
|
+
|
|
226
|
+
**Rule:** Agent heal prompts (ship.js) must not instruct agents to use the Write tool for workflow-state.json edits. All writes must route through `writeWorkflowState()` (via `bun -e` or equivalent) so agents get immediate Zod error feedback at write time, not opaque gate failures.
|
|
227
|
+
|
|
228
|
+
### DISCOVERY_SCHEMA Enum Constraints
|
|
229
|
+
|
|
230
|
+
The DISCOVERY_SCHEMA (JSON Schema for Claude structured output) at `ship.js:44` must constrain `threshold.op` and `evidenceMethod.type` to the same enum values as the Zod schema at `schema.ts:16` and `schema.ts:22-25`. Invalid values must be rejected at LLM structured output time, not deferred to gate time.
|
|
231
|
+
|
|
232
|
+
**Rule:** Enum values must be exported as const arrays from `schema.ts` and referenced in both the Zod schema and DISCOVERY_SCHEMA. A schema-parity test must assert they match.
|
|
233
|
+
|
|
234
|
+
### Schema Strictness Consistency
|
|
235
|
+
|
|
236
|
+
`writeWorkflowState()` at `orchestrator.ts:416` uses `.passthrough()` (allows extra keys) while `workflow.test.ts:57` uses strict `.safeParse()`. State that passes write-time validation can fail gate-time validation if it contains extra keys.
|
|
237
|
+
|
|
238
|
+
**Rule:** Both validation paths must use the same strictness level. Recommendation: `.passthrough()` in both, since extra keys are harmless and the schema evolves frequently.
|
|
239
|
+
|
|
240
|
+
### Workflow State Lifecycle
|
|
241
|
+
|
|
242
|
+
The `~/.rungate/` directory accumulates workflow state slugs with no archival mechanism. As of 2026-09-17: 1,065 slugs, 2 archived, 14 stuck in non-DONE phases, 1,001 with no workflow-state.json.
|
|
243
|
+
|
|
244
|
+
**Rule:** No background cleanup crons or daemons (unanimously rejected — race conditions with active workflows). Cleanup is lazy: `initWorkflow()` at `orchestrator.ts:467` logs a warning when overwriting non-DONE state. One-shot manual cleanup via Makefile target for accumulated slugs.
|
|
245
|
+
|
|
246
|
+
### Success Criteria
|
|
247
|
+
|
|
248
|
+
- [ ] GI-1: Gate failure detail includes actual Zod error path + message (not hardcoded "failed")
|
|
249
|
+
- [ ] GI-2: DISCOVERY_SCHEMA threshold.op has enum constraint matching schema.ts values
|
|
250
|
+
- [ ] GI-3: DISCOVERY_SCHEMA evidenceMethod.type has enum constraint matching schema.ts values
|
|
251
|
+
- [ ] GI-4: Enum values exported as const arrays from schema.ts (single source of truth)
|
|
252
|
+
- [ ] GI-5: schema-parity.test.ts asserts DISCOVERY_SCHEMA enums match Zod enums
|
|
253
|
+
- [ ] GI-6: Heal agent write prompts in ship.js route through writeWorkflowState() (not Write tool)
|
|
254
|
+
- [ ] GI-7: SCHEMA-GUIDE.md references validated write path (not Write tool)
|
|
255
|
+
- [ ] GI-8: writeWorkflowState() and workflow.test.ts use same parse strictness
|
|
256
|
+
- [ ] GI-9: initWorkflow() warns when overwriting non-DONE workflow state
|
|
@@ -0,0 +1,286 @@
|
|
|
1
|
+
---
|
|
2
|
+
doc-type: spec
|
|
3
|
+
status: draft
|
|
4
|
+
owner: jason
|
|
5
|
+
created: 2026-09-03
|
|
6
|
+
updated: 2026-09-03
|
|
7
|
+
testable: true
|
|
8
|
+
---
|
|
9
|
+
|
|
10
|
+
# Harness Skill Chain Spec
|
|
11
|
+
# Version 0.1 — Draft from audit data
|
|
12
|
+
|
|
13
|
+
## Problem Statement
|
|
14
|
+
|
|
15
|
+
The harness is good at committing code and running tests. It is bad at proving the right problem was solved. Audit of 30+ ship cycles shows:
|
|
16
|
+
|
|
17
|
+
- Reproduce-first: almost never done (#82 closed 3x without reproducing)
|
|
18
|
+
- Before/after evidence: never captured structurally
|
|
19
|
+
- Quinn skipped with wrong logic (file-type based, not UX-change based)
|
|
20
|
+
- No bug-specific E2E tests
|
|
21
|
+
- No artifact linking before→commit→after→release tag
|
|
22
|
+
|
|
23
|
+
The harness enforces "code is correct" but not "user's problem is solved."
|
|
24
|
+
|
|
25
|
+
## Design Principle
|
|
26
|
+
|
|
27
|
+
**Skills as a chain, not a monolith.** Each skill has one job, one gate, one artifact. The harness validates handoffs between skills.
|
|
28
|
+
|
|
29
|
+
**Every step has a scorecard.** Telemetry tracks success/failure per skill, per step. No more "unknown" categories — if a step runs, it's measured.
|
|
30
|
+
|
|
31
|
+
## The Chain
|
|
32
|
+
|
|
33
|
+
```
|
|
34
|
+
INTAKE → DEFINE → SHIP → PROVE → RELEASE
|
|
35
|
+
↑ |
|
|
36
|
+
└──── council review + iterate ────┘
|
|
37
|
+
```
|
|
38
|
+
|
|
39
|
+
### Step 1: INTAKE (`/goal`)
|
|
40
|
+
|
|
41
|
+
**Trigger:** Jason brings observation, screenshot, email, link, verbal description.
|
|
42
|
+
|
|
43
|
+
**What happens:**
|
|
44
|
+
1. Research the observation — reproduce it, trace the data source
|
|
45
|
+
2. Capture before-state (screenshot, API response, or data snapshot)
|
|
46
|
+
3. Create GitHub issue with measurable ACs
|
|
47
|
+
4. Garbage-test every AC
|
|
48
|
+
|
|
49
|
+
**Gate:** Issue exists with ACs. Before-state captured.
|
|
50
|
+
|
|
51
|
+
**Artifact:** GitHub issue + before-state evidence file in workflow-state.json
|
|
52
|
+
|
|
53
|
+
**Scorecard metrics:**
|
|
54
|
+
- Did we reproduce before defining? (reproduce_attempted: true/false)
|
|
55
|
+
- Did we capture before-state? (beforeState.captured: true/false)
|
|
56
|
+
- Did ACs survive garbage test? (acsGarbageTested: true/false)
|
|
57
|
+
|
|
58
|
+
### Step 2: DEFINE (optional — for M/L work)
|
|
59
|
+
|
|
60
|
+
**Trigger:** Goal is complex enough to need spec, ADR, or council.
|
|
61
|
+
|
|
62
|
+
**What happens:**
|
|
63
|
+
1. `/council` on the findings — is this the right problem? Right fix?
|
|
64
|
+
2. Write spec or ADR if architectural
|
|
65
|
+
3. Break into sub-issues if L-sized
|
|
66
|
+
|
|
67
|
+
**Gate:** Council ran (if uncertain). Spec exists (if M/L).
|
|
68
|
+
|
|
69
|
+
**Artifact:** Council synthesis + spec/ADR link on issue
|
|
70
|
+
|
|
71
|
+
**Scorecard metrics:**
|
|
72
|
+
- Council ran when uncertain? (councilRan: true/false)
|
|
73
|
+
- Spec written for M/L? (specWritten: true/false)
|
|
74
|
+
|
|
75
|
+
### Step 3: SHIP (`/ship`)
|
|
76
|
+
|
|
77
|
+
**Trigger:** Issue exists with ACs. Scope gate passes.
|
|
78
|
+
|
|
79
|
+
**What happens (3-layer verification before PR):**
|
|
80
|
+
|
|
81
|
+
**Layer 1 — Local Dev (fast feedback)**
|
|
82
|
+
1. Marcus codes on local dev (make dev-all, :7778 API / :5173 UI)
|
|
83
|
+
2. Unit tests pass (`bun test`), types check (`tsc --noEmit`)
|
|
84
|
+
3. Quinn validates on local dev (:5173 UI, :7778 API)
|
|
85
|
+
- Tests as brand-new user, captures screenshots for UI bugs
|
|
86
|
+
- If Quinn fails → back to Marcus → fix → Quinn retests (fast cycle)
|
|
87
|
+
4. **GATE: Quinn local PASS before container build**
|
|
88
|
+
|
|
89
|
+
**Layer 2 — Container Verification (deployment correctness)**
|
|
90
|
+
5. `make test-rebuild` → rebuilds container image from current code, starts on :7776 with seed data
|
|
91
|
+
6. Quinn validates on container (:7776)
|
|
92
|
+
- Same tests as local dev, against containerized app
|
|
93
|
+
- Verifies deployment artifact matches dev behavior
|
|
94
|
+
- container-has-fix gate validates container has the commit SHA
|
|
95
|
+
7. Rook security scan on changed files (STANDARD+ tiers)
|
|
96
|
+
8. If Quinn/Rook fail → back to Marcus → fix → rebuild → retest
|
|
97
|
+
9. **GATE: Quinn container PASS before PR is opened**
|
|
98
|
+
|
|
99
|
+
**Layer 3 — PR + Ship Gate**
|
|
100
|
+
10. Commit → push → PR created via `gh pr create`
|
|
101
|
+
11. CI runs on Mac Mini (typecheck, build, container E2E, harness gates)
|
|
102
|
+
12. Ship gate runs (verify-gate + ship-gate checks)
|
|
103
|
+
13. **GATE: Ship gate PASS. PR ready for review.**
|
|
104
|
+
|
|
105
|
+
**Gate:** verify-gate PASS + ship-gate PASS. Quinn passed on both local and container.
|
|
106
|
+
|
|
107
|
+
**Artifact:** PR with verified code, ready for Jason to review and merge.
|
|
108
|
+
|
|
109
|
+
**Scorecard metrics:**
|
|
110
|
+
- Iterations to pass verify-gate (iterationCount)
|
|
111
|
+
- Quinn local verdict (quinn.local.verdict)
|
|
112
|
+
- Quinn container verdict (quinn.container.verdict)
|
|
113
|
+
- Gate telemetry (gate-telemetry.jsonl)
|
|
114
|
+
- Test/tsc results
|
|
115
|
+
|
|
116
|
+
### Step 4: PROVE (post-merge — prod data verification)
|
|
117
|
+
|
|
118
|
+
**Trigger:** Jason reviewed and merged the PR. Code is on main.
|
|
119
|
+
|
|
120
|
+
**What happens:**
|
|
121
|
+
1. Verify fix commit is on main (`git branch --contains {sha} main`)
|
|
122
|
+
2. **Rebuild image + spin up isolated prove container** (`make prove-up`, port 7776)
|
|
123
|
+
- Rebuilds the container image (includes `build` dependency — ensures fix code is in the image)
|
|
124
|
+
- Rsyncs PROD data to data-test/, starts test container with ALLOW_RESET=true
|
|
125
|
+
- Container uses real prod data but is isolated from live container
|
|
126
|
+
- P11 (container-has-fix) FAILs if image was built before fix commit
|
|
127
|
+
3. **Quinn validates on prove container** (port 7776, prod data)
|
|
128
|
+
- Capture after-state (prod). The prove container with rsynced prod data IS prod evidence.
|
|
129
|
+
- For UI bugs: navigate to affected page, verify fix with real customer data
|
|
130
|
+
4. **Before/after evidence artifact:**
|
|
131
|
+
```json
|
|
132
|
+
{
|
|
133
|
+
"beforeState": {
|
|
134
|
+
"type": "screenshot|api-response|data-snapshot",
|
|
135
|
+
"capturedAt": "...",
|
|
136
|
+
"path": "~/.rungate/{slug}/evidence/before.png"
|
|
137
|
+
},
|
|
138
|
+
"afterLocal": {
|
|
139
|
+
"type": "screenshot|api-response",
|
|
140
|
+
"capturedAt": "...",
|
|
141
|
+
"path": "~/.rungate/{slug}/evidence/after-local.png"
|
|
142
|
+
},
|
|
143
|
+
"afterProd": {
|
|
144
|
+
"type": "screenshot|api-response",
|
|
145
|
+
"capturedAt": "...",
|
|
146
|
+
"path": "~/.rungate/{slug}/evidence/after-prod.png"
|
|
147
|
+
}
|
|
148
|
+
}
|
|
149
|
+
```
|
|
150
|
+
|
|
151
|
+
**Key difference from SHIP container test:** SHIP uses `make test-rebuild` (seed data — catches deployment bugs). PROVE uses `make prove-up` (prod data — catches data-shape bugs). Not redundant — different data, different failure classes.
|
|
152
|
+
|
|
153
|
+
**Gate:** Before/after evidence exists. Quinn passed on prove container with prod data. **Mechanically enforced (#294):** prove gate FAILs if STANDARD+ tier and `afterEvidence.environment !== "prod"`. LIGHT tier exempted (local-only sufficient). Prove container lifecycle: make prove-up (start) / make prove-down (auto on prove gate PASS).
|
|
154
|
+
|
|
155
|
+
**Artifact:** prove-evidence.json with before/after comparison + Quinn verdict.
|
|
156
|
+
|
|
157
|
+
**Scorecard metrics:**
|
|
158
|
+
- Before-state captured? (beforeState.captured)
|
|
159
|
+
- After-state captured prod? (afterProd.captured)
|
|
160
|
+
- Quinn passed prod? (quinn.prod.verdict)
|
|
161
|
+
- Verdict: PROVEN/UNPROVEN/INCONCLUSIVE
|
|
162
|
+
|
|
163
|
+
### Step 5: RELEASE (`/release` — new)
|
|
164
|
+
|
|
165
|
+
**Trigger:** PROVE passed. Code is verified on prod container.
|
|
166
|
+
|
|
167
|
+
**What happens:**
|
|
168
|
+
1. `make release-dry-run` (optional confidence check)
|
|
169
|
+
2. `make release-patch` → version bump → release-test → tag push
|
|
170
|
+
3. Gate 4 runs on GitHub Actions (E2E on seeded server)
|
|
171
|
+
4. Draft release → upload assets → verify → publish
|
|
172
|
+
5. Post release proof to issue:
|
|
173
|
+
```
|
|
174
|
+
## Release Proof
|
|
175
|
+
- Before: [screenshot] — bug visible in v1.7.6.0
|
|
176
|
+
- Fix: commit abc123
|
|
177
|
+
- After (local): [screenshot] — bug fixed on dev
|
|
178
|
+
- After (prod): [screenshot] — bug fixed on prod
|
|
179
|
+
- Release: v1.7.6.1 — Gate 4 PASS
|
|
180
|
+
- Download: setup.sh verified 200 OK
|
|
181
|
+
```
|
|
182
|
+
|
|
183
|
+
**Gate:** Release tag exists. Gate 4 passed. Asset URLs return 200.
|
|
184
|
+
|
|
185
|
+
**Artifact:** Published GitHub release with proof comment on issue.
|
|
186
|
+
|
|
187
|
+
**Scorecard metrics:**
|
|
188
|
+
- Release-dry-run passed? (releaseDryRun.result)
|
|
189
|
+
- Gate 4 passed? (gate4.result)
|
|
190
|
+
- Asset verification passed? (assetVerification.result)
|
|
191
|
+
- Proof posted to issue? (releaseProof.posted)
|
|
192
|
+
|
|
193
|
+
## Convergence Loop
|
|
194
|
+
|
|
195
|
+
After PROVE or RELEASE:
|
|
196
|
+
|
|
197
|
+
```
|
|
198
|
+
Post-ship council → findings? → file issues → SHIP them → council again → clean? → DONE
|
|
199
|
+
```
|
|
200
|
+
|
|
201
|
+
**Council-recommended signal (advisory, not auto-triggered):** After STANDARD+ tier ships, the harness emits a `council-recommended` signal. DA reads the signal and decides whether to run council. Council is NOT auto-triggered — auto-triggering costs 200-400K tokens without human judgment. If council finds issues, they become new issues that enter the chain at DEFINE. *(Updated 2026-09-04: council decision overrode original "auto-trigger" to advisory-only.)*
|
|
202
|
+
|
|
203
|
+
**Stall in ship:** Same verify failures 2x → DA decides whether to run council on the failures → ship mechanical recommendations → re-verify.
|
|
204
|
+
|
|
205
|
+
## QC Validation (automatic at gates)
|
|
206
|
+
|
|
207
|
+
`skill-qc-validator.sh` runs automatically at each gate, checking the skill's output against `schemas/skill-qc-checks.json`:
|
|
208
|
+
- **scope gate** → goal QC (G1-G8) at STANDARD+ tiers
|
|
209
|
+
- **verify gate** → ship QC (S1-S8) at STANDARD+ tiers
|
|
210
|
+
- **prove gate** → prove QC (P1-P9) at all tiers
|
|
211
|
+
|
|
212
|
+
Currently WARN severity (2-week calibration). Manual: `bash scripts/skill-qc-validator.sh --skill NAME --issue N --slug SLUG`
|
|
213
|
+
|
|
214
|
+
## Validation Layers (reconciled 2026-09-15, aligned with CI-ENFORCEMENT-SPEC)
|
|
215
|
+
|
|
216
|
+
```
|
|
217
|
+
Layer 1: Local Dev (SHIP step 1-4) — fast feedback
|
|
218
|
+
make dev-all → :7778 (API) + :5173 (UI)
|
|
219
|
+
Marcus codes + tests. Quinn validates on dev server.
|
|
220
|
+
GATE: Quinn local PASS before container build.
|
|
221
|
+
|
|
222
|
+
Layer 2: Test Container (SHIP step 5-9) — deployment correctness
|
|
223
|
+
make test-rebuild → :7776 (seed data, isolated)
|
|
224
|
+
container-has-fix gate (P11 FAIL if stale). Quinn re-validates.
|
|
225
|
+
GATE: Quinn container PASS before PR is opened.
|
|
226
|
+
|
|
227
|
+
Layer 3: Prove Container (PROVE step 2-3) — prod data correctness
|
|
228
|
+
make prove-up → :7776 (rsynced PROD data, isolated from live)
|
|
229
|
+
Quinn validates with real customer data shapes.
|
|
230
|
+
Post-merge only. make prove-down auto-runs on PASS.
|
|
231
|
+
|
|
232
|
+
Layer 4: Release Pipeline (RELEASE step)
|
|
233
|
+
make release-dry-run → make release-patch → Gate 4
|
|
234
|
+
E2E on seeded server. Asset verification.
|
|
235
|
+
```
|
|
236
|
+
|
|
237
|
+
**Industry alignment:** Google presubmit/postsubmit, Dagger "same pipeline everywhere", Vercel preview deployments. Layer 2 is pre-merge (shift left). Layer 3 is post-merge (prod data verification). Not redundant — different data, different failure classes.
|
|
238
|
+
|
|
239
|
+
## Per-Skill Scorecard
|
|
240
|
+
|
|
241
|
+
Every skill appends to `~/.claude/state/skill-telemetry.jsonl`:
|
|
242
|
+
|
|
243
|
+
```json
|
|
244
|
+
{
|
|
245
|
+
"ts": "2026-09-03T...",
|
|
246
|
+
"skill": "goal|research|council|ship|prove|release",
|
|
247
|
+
"issue": 1229,
|
|
248
|
+
"metrics": {
|
|
249
|
+
"reproduced": true,
|
|
250
|
+
"beforeCaptured": true,
|
|
251
|
+
"councilRan": false,
|
|
252
|
+
"iterations": 2,
|
|
253
|
+
"quinnLocal": "PASS",
|
|
254
|
+
"quinnProd": "PASS",
|
|
255
|
+
"gate4": "PASS"
|
|
256
|
+
}
|
|
257
|
+
}
|
|
258
|
+
```
|
|
259
|
+
|
|
260
|
+
Monthly audit: query telemetry, find skills with low success rates, council on improvements.
|
|
261
|
+
|
|
262
|
+
## What This Replaces
|
|
263
|
+
|
|
264
|
+
- Ship skill still handles SCOPE → BUILD → VERIFY (Step 3)
|
|
265
|
+
- PROVE replaces the manual "did you test it?" question
|
|
266
|
+
- RELEASE connects to CI-RELEASE-PIPELINE.md (DDB docs/)
|
|
267
|
+
- Council convergence replaces manual "should we run council?"
|
|
268
|
+
- Skill telemetry replaces "unknown" category from gate audit
|
|
269
|
+
|
|
270
|
+
## Open Questions
|
|
271
|
+
|
|
272
|
+
1. ~~Should PROVE be a separate skill or integrated into ship's VERIFY phase?~~ **RESOLVED: Separate skill.** /prove is standalone, produces prove-evidence.json. (ADR-007)
|
|
273
|
+
2. ~~Should /release be a skill or a Makefile-only step?~~ **RESOLVED: Separate skill.** /release validates, generates changelog, monitors Gate 4. (ADR-007)
|
|
274
|
+
3. How does Quinn capture screenshots programmatically? (Playwright? Browser agent?) **OPEN**
|
|
275
|
+
4. Where do before/after evidence files live long-term? (workflow-state.json gets TTL'd) **OPEN** — GoalRecords are TTL-exempt per ADR-007; evidence files need similar treatment
|
|
276
|
+
5. Should the convergence loop have a budget cap? **OPEN**
|
|
277
|
+
|
|
278
|
+
## Related Specs
|
|
279
|
+
|
|
280
|
+
- `~/.claude/PAI/HARNESS-STANDARD.md` — current harness loop
|
|
281
|
+
- `~/.claude/PAI/HARNESS-GATES.md` — gate documentation
|
|
282
|
+
- `~/Projects/DailyBriefDashboard/docs/CI-RELEASE-PIPELINE.md` — 3-layer validation + release flow
|
|
283
|
+
- `~/.claude/skills/ship/SKILL.md` — ship skill v2
|
|
284
|
+
- `~/.claude/skills/goal/SKILL.md` — goal skill
|
|
285
|
+
- `~/.claude/skills/testing-and-qa-validation/SKILL.md` — QA workflow (15 steps)
|
|
286
|
+
- `~/.claude/workflows/council.js` — council workflow with enforcement classification
|
|
@@ -0,0 +1,93 @@
|
|
|
1
|
+
---
|
|
2
|
+
doc-type: spec
|
|
3
|
+
status: active
|
|
4
|
+
owner: jason
|
|
5
|
+
created: 2026-09-03
|
|
6
|
+
updated: 2026-09-03
|
|
7
|
+
testable: true
|
|
8
|
+
---
|
|
9
|
+
|
|
10
|
+
# Harness Skill Contract
|
|
11
|
+
|
|
12
|
+
Every harness-compatible skill must implement this contract in its SKILL.md YAML frontmatter.
|
|
13
|
+
|
|
14
|
+
## Required Fields
|
|
15
|
+
|
|
16
|
+
### 1. input
|
|
17
|
+
What the skill reads. Issue number, workflow-state.json, observation text, etc.
|
|
18
|
+
|
|
19
|
+
### 2. gateIn
|
|
20
|
+
Preconditions — what must be true before this skill can run. Expressed as verifiable checks.
|
|
21
|
+
|
|
22
|
+
### 3. gateOut
|
|
23
|
+
Postconditions — what must be true when this skill completes. The next skill in the chain checks these.
|
|
24
|
+
|
|
25
|
+
### 4. artifact
|
|
26
|
+
What this skill produces. File path, GitHub comment, state update. Must be a file that survives session boundaries.
|
|
27
|
+
|
|
28
|
+
### 5. telemetry
|
|
29
|
+
Which telemetry file this skill writes to and what fields it logs. All skills write to harness-telemetry.jsonl with a skill field.
|
|
30
|
+
|
|
31
|
+
### 6. errorRecovery
|
|
32
|
+
How the skill handles failures:
|
|
33
|
+
- timeout_seconds: max execution time
|
|
34
|
+
- failure_mode: BLOCK (stop chain) or WARN (continue with note)
|
|
35
|
+
- cleanup_action: what to clean up on failure
|
|
36
|
+
- resumable: can the skill be re-run to pick up where it left off?
|
|
37
|
+
|
|
38
|
+
## YAML Frontmatter Format
|
|
39
|
+
|
|
40
|
+
```yaml
|
|
41
|
+
---
|
|
42
|
+
name: SkillName
|
|
43
|
+
description: One-line description
|
|
44
|
+
version: 1.0.0
|
|
45
|
+
contract:
|
|
46
|
+
input: "what it reads"
|
|
47
|
+
gateIn: "preconditions"
|
|
48
|
+
gateOut: "postconditions"
|
|
49
|
+
artifact: "what it produces"
|
|
50
|
+
telemetry: "harness-telemetry.jsonl with skill=name"
|
|
51
|
+
errorRecovery:
|
|
52
|
+
timeout_seconds: 600
|
|
53
|
+
failure_mode: BLOCK
|
|
54
|
+
cleanup_action: "cleanup command"
|
|
55
|
+
resumable: true
|
|
56
|
+
---
|
|
57
|
+
```
|
|
58
|
+
|
|
59
|
+
## Chain Handoffs
|
|
60
|
+
|
|
61
|
+
Each skill's gateOut must satisfy the next skill's gateIn:
|
|
62
|
+
|
|
63
|
+
```
|
|
64
|
+
/goal gateOut: "GitHub issue exists with ACs + beforeState"
|
|
65
|
+
-> /ship gateIn: "Issue exists with ACs"
|
|
66
|
+
|
|
67
|
+
/ship gateOut: "verify-gate PASS, code on main, mergeCommitSha set"
|
|
68
|
+
-> /prove gateIn: "Code merged to main (mergeCommitSha exists)"
|
|
69
|
+
|
|
70
|
+
/prove gateOut: "prove-evidence.json with verdict"
|
|
71
|
+
-> /release gateIn: "main is clean"
|
|
72
|
+
```
|
|
73
|
+
|
|
74
|
+
## Validation
|
|
75
|
+
|
|
76
|
+
Run `bash scripts/harness-skill-check.sh skills/{name}/SKILL.md` to validate any skill.
|
|
77
|
+
|
|
78
|
+
## Adding a New Skill
|
|
79
|
+
|
|
80
|
+
1. Create `skills/{name}/SKILL.md` with YAML frontmatter including all 6 contract fields
|
|
81
|
+
2. Run `bash scripts/harness-skill-check.sh skills/{name}/SKILL.md` — must PASS
|
|
82
|
+
3. Add to chain documentation if it participates in the harness chain
|
|
83
|
+
4. Call `/doc-hygiene` to register in project index
|
|
84
|
+
|
|
85
|
+
## Current Skills
|
|
86
|
+
|
|
87
|
+
| Skill | Contract? | Chain position |
|
|
88
|
+
|-------|-----------|---------------|
|
|
89
|
+
| /goal | Yes | Start |
|
|
90
|
+
| /ship | Yes | After /goal |
|
|
91
|
+
| /prove | Yes | After /ship |
|
|
92
|
+
| /release | Yes | After /prove (end) |
|
|
93
|
+
| /council | Yes | Lateral (any point) |
|