rungate 0.2.0 → 0.2.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (164) hide show
  1. package/package.json +14 -3
  2. package/.claude/agents/aditi.md +0 -47
  3. package/.claude/agents/marcus.md +0 -50
  4. package/.claude/agents/quinn.md +0 -58
  5. package/.claude/agents/rook.md +0 -47
  6. package/.claude/agents/serena.md +0 -46
  7. package/.claude/rungate.json +0 -11
  8. package/.fallowrc.json +0 -28
  9. package/.github/copilot-instructions.md +0 -20
  10. package/.rungate/conformity-findings.json +0 -648
  11. package/AGENTS.md +0 -106
  12. package/CODE-MAP.md +0 -50
  13. package/HARNESS.md +0 -88
  14. package/bun.lock +0 -26
  15. package/bunfig.toml +0 -2
  16. package/config/brief-policies.json +0 -28
  17. package/config/ceremony-profiles.json +0 -212
  18. package/config/consumer-cascade-map.json +0 -24
  19. package/config/doc-cascade-map.json +0 -24
  20. package/config/eval.json +0 -84
  21. package/config/profiles/account-plan.yaml +0 -36
  22. package/config/profiles/campaign.yaml +0 -94
  23. package/config/prove-evidence.schema.json +0 -65
  24. package/config/spec-policies.json +0 -36
  25. package/config/workflow-schema.json +0 -281
  26. package/gates/SCHEMA-GUIDE.md +0 -121
  27. package/gates/ac-quality.test.ts +0 -106
  28. package/gates/adversarial.test.ts +0 -274
  29. package/gates/brief-assembler.test.ts +0 -429
  30. package/gates/brief-assembler.ts +0 -334
  31. package/gates/bunfig.toml +0 -2
  32. package/gates/chain.test.ts +0 -91
  33. package/gates/e2e-smoke.test.ts +0 -234
  34. package/gates/error-classifier.test.ts +0 -249
  35. package/gates/error-classifier.ts +0 -89
  36. package/gates/orchestrator.test.ts +0 -950
  37. package/gates/orchestrator.ts +0 -569
  38. package/gates/preload.ts +0 -77
  39. package/gates/prompts/ac-adversary.md +0 -20
  40. package/gates/prompts/evidence-validator.md +0 -22
  41. package/gates/prompts/prove-reproducer.md +0 -53
  42. package/gates/prove.test.ts +0 -164
  43. package/gates/run-gate.ts +0 -936
  44. package/gates/schema-parity.test.ts +0 -197
  45. package/gates/schema.ts +0 -362
  46. package/gates/self-heal.test.ts +0 -511
  47. package/gates/self-heal.ts +0 -121
  48. package/gates/ship-orchestrator.test.ts +0 -379
  49. package/gates/ship-orchestrator.ts +0 -363
  50. package/gates/test-fixtures/broken-no-environments.json +0 -60
  51. package/gates/test-fixtures/valid-standard.json +0 -54
  52. package/gates/test-utils.ts +0 -43
  53. package/gates/witness.ts +0 -220
  54. package/gates/workflow.test.ts +0 -1053
  55. package/hooks/AgentBriefGuard.hook.ts +0 -586
  56. package/hooks/AgentVerdictCapture.hook.ts +0 -134
  57. package/hooks/AutoVerifyGate.hook.ts +0 -76
  58. package/hooks/GateEnforcement.hook.ts +0 -273
  59. package/hooks/IssueCloseGuard.hook.ts +0 -116
  60. package/hooks/MergeGuard.hook.ts +0 -55
  61. package/hooks/StaleTTLCleanup.hook.ts +0 -133
  62. package/hooks/VerifyPhaseLock.hook.ts +0 -81
  63. package/hooks/WorkflowStateGuard.hook.ts +0 -37
  64. package/hooks/lib/agentDetection.ts +0 -70
  65. package/hooks/lib/findWorkflow.ts +0 -101
  66. package/hooks/lib/parseStdin.ts +0 -37
  67. package/hooks/lib/paths.ts +0 -111
  68. package/hooks/lib/utils.ts +0 -81
  69. package/reference/migration-manifest.json +0 -554
  70. package/reference/prompts/agent-aditi.md +0 -38
  71. package/reference/prompts/agent-marcus.md +0 -38
  72. package/reference/prompts/agent-quinn.md +0 -38
  73. package/reference/prompts/agent-rook.md +0 -37
  74. package/reference/prompts/agent-serena.md +0 -37
  75. package/reference/prompts/council-skill.md +0 -170
  76. package/reference/prompts/goal-skill.md +0 -235
  77. package/reference/prompts/prove-skill.md +0 -283
  78. package/reference/prompts/quinn-standard.md +0 -246
  79. package/reference/prompts/ship-convergence.md +0 -65
  80. package/reference/prompts/ship-escalation.md +0 -43
  81. package/reference/prompts/ship-iteration.md +0 -382
  82. package/reference/prompts/ship-pending-patches.md +0 -246
  83. package/reference/prompts/ship-reference.md +0 -250
  84. package/reference/prompts/ship-skill.md +0 -347
  85. package/reference/prompts/ship-templates.md +0 -215
  86. package/reference/scripts/batch-status.sh +0 -68
  87. package/reference/scripts/chain-compat-check.sh +0 -67
  88. package/reference/scripts/context-audit.sh +0 -314
  89. package/reference/scripts/dead-export-check.sh +0 -61
  90. package/reference/scripts/generate-qc-tables.sh +0 -92
  91. package/reference/scripts/harness-init.sh +0 -111
  92. package/reference/scripts/pai-054-baseline-check.sh +0 -40
  93. package/reference/scripts/prove-smoke-test.sh +0 -112
  94. package/reference/scripts/setup-keychain-salt.sh +0 -12
  95. package/reference/scripts/skill-optimizer-friday.sh +0 -88
  96. package/reference/scripts/skill-qc-test.sh +0 -351
  97. package/reference/scripts/skill-synthesize-tonight.sh +0 -66
  98. package/reference/specs/HARNESS-EXTRACTION-SPEC.md +0 -263
  99. package/reference/specs/harness-v3-bash-deletion-plan.md +0 -65
  100. package/reference/specs/harness-v3-migration-plan.md +0 -188
  101. package/scripts/decision-reconcile.sh +0 -74
  102. package/scripts/decision-reconcile.ts +0 -152
  103. package/scripts/git-hooks/pre-commit +0 -76
  104. package/scripts/git-hooks/pre-push +0 -271
  105. package/scripts/harness-skill-check.sh +0 -52
  106. package/scripts/lib/gap-emit.sh +0 -65
  107. package/scripts/lib/jq-update.sh +0 -30
  108. package/scripts/lib/resolve-slug.sh +0 -30
  109. package/scripts/lib/verify-spec-compliance.sh +0 -102
  110. package/scripts/prove-backfill.sh +0 -87
  111. package/scripts/scaffold-rungate-config.ts +0 -91
  112. package/scripts/self-containment-check.sh +0 -109
  113. package/scripts/skill-qc-validator.sh +0 -402
  114. package/scripts/slug-resolver.sh +0 -14
  115. package/specs/BOOTSTRAP-DATA-FLOW-SPEC.md +0 -2493
  116. package/specs/BOOTSTRAP-TEST-PLAN.md +0 -571
  117. package/specs/HARNESS-GATES.md +0 -256
  118. package/specs/HARNESS-SKILL-CHAIN.md +0 -286
  119. package/specs/HARNESS-SKILL-CONTRACT.md +0 -93
  120. package/specs/HARNESS-STANDARD.md +0 -488
  121. package/specs/SPEC-TEMPLATE.md +0 -53
  122. package/specs/bootstrap-data-flow.png +0 -0
  123. package/specs/harness-automation-matrix.md +0 -314
  124. package/specs/harness-automation-matrix.rules.json +0 -64
  125. package/test/anti.test.ts +0 -80
  126. package/test/contract-negative.test.ts +0 -86
  127. package/test/contract.test.ts +0 -158
  128. package/test/external-deps.test.ts +0 -48
  129. package/test/fixtures/golden-project/.env.example +0 -3
  130. package/test/fixtures/golden-project/Makefile +0 -7
  131. package/test/fixtures/golden-project/docs/adr/ADR-001-framework.md +0 -23
  132. package/test/fixtures/golden-project/docs/setup-guide.md +0 -10
  133. package/test/fixtures/golden-project/package.json +0 -10
  134. package/test/fixtures/golden-project/specs/api-spec.md +0 -20
  135. package/test/fixtures/golden-project/src/auth.ts +0 -8
  136. package/test/fixtures/golden-project/src/db.ts +0 -11
  137. package/test/fixtures/golden-project/src/index.ts +0 -19
  138. package/test/fixtures/golden-project/src/utils/format.ts +0 -3
  139. package/test/fixtures/golden-project/test/auth.test.ts +0 -15
  140. package/test/fixtures/golden-project/tsconfig.json +0 -12
  141. package/test/meta-sc-coverage.test.ts +0 -159
  142. package/test/phase-0.test.ts +0 -846
  143. package/test/phase-1.test.ts +0 -617
  144. package/test/phase-2.test.ts +0 -620
  145. package/test/phase-3.test.ts +0 -99
  146. package/test/scaffold-conformity.test.ts +0 -8
  147. package/test/schema-canary.test.ts +0 -51
  148. package/test/spec-compliance-auto.test.ts +0 -64
  149. package/test/spec-compliance.test.ts +0 -295
  150. package/test/spec-discovery.test.ts +0 -68
  151. package/test/spec-drift.test.ts +0 -5
  152. package/test/structure.test.ts +0 -165
  153. package/test/unit/agent-brief-guard-ship.test.ts +0 -293
  154. package/test/unit/extract-constraints.test.ts +0 -145
  155. package/test/unit/issue-close-guard.test.ts +0 -204
  156. package/test/unit/scaffold-project.test.ts +0 -417
  157. package/test/unit/skill-cooccurrence.test.ts +0 -177
  158. package/test/unit/skill-sequence-logger.test.ts +0 -244
  159. package/test/unit/workflow-state-guard.test.ts +0 -95
  160. package/workflows/batch-ship.js +0 -295
  161. package/workflows/council.js +0 -481
  162. package/workflows/prove.js +0 -560
  163. package/workflows/ship.js +0 -789
  164. package/workflows/verify.js +0 -236
@@ -1,256 +0,0 @@
1
- ---
2
- doc-type: reference
3
- status: active
4
- owner: jason
5
- updated: 2026-09-17
6
- testable: true
7
- ---
8
-
9
- # Harness Gates
10
- # Source of truth for all gate definitions, enforcement config, and learning state.
11
- # Read by: gate scripts, GateEnforcement.hook.ts, HarnessFailureSurfacing.hook.ts, LearningReview.ts
12
- # Written by: LearningReview.ts (nightly — tighten/loosen/prune), DA (manual edits)
13
-
14
- ---
15
-
16
- ## goal-gate
17
- - **Step:** GOAL (optional — only for L-sized work via harness)
18
- - **Fires after:** Skill("goal") completes
19
- - **Script:** `skills/ship/goal-gate.sh --issue NUM`
20
- - **Checks:**
21
- - [ ] Issue has `## Goal` section
22
- - [ ] Issue has `## Success Criteria` with SC- or AC- prefixed items
23
- - [ ] SCs/ACs contain no vague words: exists, works, properly, correct, appropriate, should (without accompanying threshold number)
24
- - [ ] Issue has `## Scope Boundary` with In + Out items
25
- - [ ] Issue has `## Circuit Breakers` with iteration + cost limits
26
- - [ ] Checkpoint written to `$RUNGATE_WORK_DIR/{slug}/`
27
- - **Strike threshold:** 3
28
- - **Escalated:** false
29
- - **Skip history (30d):** 0 skips
30
-
31
- ---
32
-
33
- ## scope-gate
34
- - **Step:** SCOPE (after sizing + ACs, before BUILD)
35
- - **Fires after:** Ship SCOPE completes
36
- - **Script:** `skills/ship/scope-gate.sh --issue NUM --slug SLUG`
37
- - **Checks:**
38
- - [ ] Issue has structured DISCOVERY comment with file:line citations (citation count > 0)
39
- - [ ] PRINCIPLES.md referenced in DISCOVERY comment (if project has one)
40
- - [ ] ARCHITECTURE.md referenced in DISCOVERY comment (if project has one)
41
- - [ ] Research decision explicit: findings posted OR "no research needed" with evidence
42
- - [ ] ACs posted to issue with AC-N IDs (count > 0)
43
- - [ ] No vague words in AC statements without threshold numbers
44
- - [ ] Sizing declaration posted: size + reasoning + expected files/time/boundaries
45
- - [ ] Each AC tagged [CODE] or [OUTCOME]
46
- - [ ] [OUTCOME] ACs have all 10 fields: Type, Statement, Metric, Threshold, Baseline, Evidence Method, Pass Criteria, Source, User, Conversation, Action
47
- - [ ] [CODE] ACs have all 7 fields: Type, Statement, Metric, Threshold, Baseline, Evidence Method, Pass Criteria, Source
48
- - [ ] Brief posted with ## Marcus Brief or ## Implementation Brief marker
49
- - [ ] Brief contains template markers: ## Context, ## Task, ## Verify, ## Report back
50
- - [ ] Adversarial goal test: ## Anti-Gaming table posted with defenses for every shortcut (M/L: required, XS/S: recommended)
51
- - [ ] **QC validator (STANDARD+):** `skill-qc-validator.sh --skill goal` — runs G1-G8 checks from `schemas/skill-qc-checks.json`. Currently WARN; promotes to FAIL after calibration.
52
- - **Strike threshold:** 3
53
- - **Escalated:** false
54
- - **Skip history (30d):** 0 skips
55
-
56
- ---
57
-
58
- ## verify-gate
59
- - **Step:** VERIFY (after evidence collected, before DURABILITY)
60
- - **Fires after:** Ship VERIFY completes
61
- - **Script:** `skills/ship/verify-gate.sh --issue NUM --slug SLUG`
62
- - **Checks:**
63
- - [ ] Every AC-N in issue has matching evidence in completion report
64
- - [ ] [OUTCOME] ACs have live evidence: screenshot URL, curl output block, or browser verification — not code review alone
65
- - [ ] [CODE] ACs have file:line citation in evidence
66
- - [ ] Evidence references threshold number from AC (number appears in evidence text)
67
- - [ ] curl commands use correct test port (read from project CLAUDE.md)
68
- - [ ] Consumer chain: if consumer files changed (from git diff + PRINCIPLES.md consumer list), 4 artifacts present: curl output, screenshot, rendered output, goal statement reference
69
- - [ ] Report uses completion template markers from REFERENCE.md
70
- - [ ] **QC validator (STANDARD+):** `skill-qc-validator.sh --skill ship` — runs S1-S8 checks from `schemas/skill-qc-checks.json`. Currently WARN; promotes to FAIL after calibration.
71
- - **Strike threshold (ceremony checks):** 3
72
- - **Strike threshold (OUTCOME AC failures):** 0 (immediate block, no skip)
73
- - **AC failures:** Must fix — no skip allowed. Iterate until pass or circuit breaker.
74
- - **Escalated:** false
75
- - **Skip history (30d):** 0 skips
76
-
77
- ---
78
-
79
- ## durability-gate
80
- - **Step:** DURABILITY (after VERIFY, before CLOSE)
81
- - **Fires after:** verify-gate passes
82
- - **Script:** `skills/ship/durability-gate.sh --issue NUM --slug SLUG`
83
- - **Checks:**
84
- - [ ] Doc cascade: git diff matched against doc-cascade-map.json
85
- - [ ] Structured findings: FAIL ACs have FINDING: comments posted
86
- - [ ] Quinn gate: .tsx changed → Quinn verification on issue
87
- - [ ] Checkpoint freshness: updated within 2 hours
88
- - [ ] Hedge-grep: zero hedged claims in issue comments (phrases from patterns.json `unverified_claim_phrases`)
89
- - [ ] Rook gate: M/L size → Rook scan on issue
90
- - [ ] Boundary crossing: XS/S + 2+ dirs → WARN
91
- - [ ] ADR status: referenced ADRs not "proposed"
92
- - [ ] Issue/brief current: if FINDING posted after ACs, ACs updated
93
- - [ ] Follow-ups logged: TODO/FOLLOWUP markers have corresponding issues
94
- - [ ] Failure pattern acknowledged: top pattern from HarnessFailureSurfacing referenced
95
- - **Strike threshold:** 3
96
- - **Escalated:** false
97
- - **Skip history (30d):** 0 skips
98
-
99
- ---
100
-
101
- ## close-gate
102
- - **Step:** CLOSE (after DURABILITY, before next issue)
103
- - **Fires after:** Ship CLOSE actions
104
- - **Script:** `skills/ship/close-gate.sh --issue NUM --slug SLUG`
105
- - **Checks:**
106
- - [ ] Issue has `## Completion Report` comment
107
- - [ ] doc-hygiene invoked this session (from signals.jsonl)
108
- - [ ] Follow-up tracking: Marcus unrelated findings, Quinn/Rook FINDINGs, council scope_change — all have corresponding issues. Parent has `has-follow-ups` label if children exist. Children have `follow-up` + `needs-triage` labels
109
- - [ ] Sizing outcome posted: `## Sizing Outcome` with actual files/time/dirs vs declared
110
- - [ ] Checkpoint marked complete
111
- - [ ] Routing outcomes logged: if ATTEMPT comments exist, routing_outcome entries in signals.jsonl with required fields
112
- - [ ] Issue labeled `shipped`
113
- - **Strike threshold:** 3
114
- - **Escalated:** false
115
- - **Skip history (30d):** 0 skips
116
-
117
- ---
118
-
119
- ## iteration-gate
120
- - **Step:** ITERATION (after VERIFY fails, before looping back)
121
- - **Fires at:** Every iteration boundary
122
- - **Script:** `skills/ship/iteration-gate.sh --issue NUM --slug SLUG`
123
- - **Checks:**
124
- - [ ] ATTEMPT comment posted for this iteration (count > previous)
125
- - [ ] ATTEMPT has required fields: Approach, Files Changed, Result, Evidence, Why It Failed
126
- - [ ] Iteration count tracked in stuck-detection.json
127
- - [ ] Same error hash not repeated 3x (hash "Why it failed" sections)
128
- - [ ] Iteration count < circuit breaker limit from issue
129
- - [ ] Wall clock < 90 minutes (or custom limit from issue)
130
- - [ ] Progress check: ≥1 new AC passed since last iteration, or new information acquired
131
- - [ ] Goal Audit: no baseline drift >20%, no entity non-existence, no research contradiction
132
- - [ ] Checkpoint has compaction fields: Current State, Next Action, Files Changed, Gate Status
133
- - [ ] Squashed commits: ≤1 new commit since last iteration on feature branch
134
- - **Tier 1 threshold:** 3 iterations → auto-escalate via routing-decision.sh
135
- - **Tier 2 threshold:** 6 iterations (3 pre + 3 post-escalation) → hard stop
136
- - **Meta-circuit-breaker:** Max 2 goal amendments per issue
137
- - **Escalated:** false
138
- - **Skip history (30d):** 0 skips
139
-
140
- ---
141
-
142
- ## Gate Enforcement Model
143
-
144
- **Strike escalation (ceremony gates):**
145
- - Strike 1-2: system-reminder injected on every tool call
146
- - Strike 3: hard block on Skill calls (Bash/Read/Agent still work for fixing)
147
- - Escape: fix the failure OR post skip-reason to issue → block clears → skip logged
148
-
149
- **AC failures (non-skippable):**
150
- - [CODE] AC fail: iterate, no skip. Circuit breaker is only escape.
151
- - [OUTCOME] AC fail: immediate block, iterate, no skip. Circuit breaker is only escape.
152
-
153
- **Self-heal:** DA can use Bash/Read/Agent to resolve gate failures. Only Skill calls are blocked.
154
-
155
- **Skip logging:** Every skip gets logged to signals.jsonl with type: "gate_skip". LearningReview reads these nightly.
156
-
157
- **De-escalation:** Gates with 0 skips in 60 days → strike threshold resets to default (3). LearningReview manages this.
158
-
159
- **Pruning:** Checks with 0 catches in 90 days → retired from gate definition. Skip history older than 30 days → pruned.
160
-
161
- ---
162
-
163
- ## Agent Enforcement (PreToolUse on Agent)
164
-
165
- **Applies to:** All named agent spawns (Marcus, Quinn, Rook, Serena, Aditi)
166
-
167
- - Brief must contain template markers (## Context, ## Task, ## Verify, ## Report back)
168
- - `mode: "bypassPermissions"` must be set
169
- - `max_turns` set by sizing: XS=20, S=30, M=50, L=75
170
- - Single-attempt constraint injected: "STOP on persistent error, report what happened"
171
- - Isolation routing: `.claude/` repo → worktree, external → direct
172
- - Post-report: hedge-grep + spiral detection before accepting
173
- - Brief block: if template markers missing → `{"decision": "block"}`
174
- - bypassPermissions block: if missing → `{"decision": "block"}`
175
-
176
- ---
177
-
178
- ## Routing Table
179
-
180
- **Decision tree (deterministic — from 109 real failure resolutions):**
181
-
182
- 1. Did agent read relevant files? (grep ATTEMPT for file reads) → NO: back to agent with file list
183
- 2. Did agent run it and check output? (grep for curl/screenshot/test) → NO: back to agent with evidence method
184
- 3. Framework/library error? (grep for framework names) → YES: Context7 first, then Research
185
- 4. Scope/requirements mismatch? (grep for "doesn't match", "expected") → YES: re-read issue + PRINCIPLES.md, then council
186
- 5. 3+ failed attempts? (count ATTEMPTs) → YES: council — approach is wrong
187
- 6. DEFAULT: Research — missing information
188
-
189
- **Accuracy tracking:** Every routing decision logged to signals.jsonl with route + resolved outcome. LearningReview reorders steps by resolution rate, adds new patterns from successes, removes patterns with 0 matches in 90 days, demotes routes with <30% accuracy.
190
-
191
- ---
192
-
193
- ## Scoring
194
-
195
- **Per-issue scorecard** (score-issue.sh):
196
- - Gate pass per step (first-pass rate)
197
- - Iteration count
198
- - Sizing accuracy (declared vs actual)
199
- - Time within budget
200
- - Follow-ups created
201
- - Routing used
202
- - Letter grade: A (≥90% first-pass, ≤1 iter), B (≥75%, ≤2), C (≥60%, ≤3), D (<60% or Tier 2), F (hard stop)
203
-
204
- **System dashboard** (nightly via LearningReview):
205
- - Gate health: pass rates + top failure + trend per gate
206
- - Issue metrics: avg grade, first-pass, iterations, sizing accuracy
207
- - Routing performance: accuracy per route
208
- - Gate escalation status
209
-
210
- ---
211
-
212
- ## Gate Infrastructure (from adversarial council 2026-09-17)
213
-
214
- Source: 4-member adversarial council, 3 rounds, unanimous on all findings. Root cause analysis of #1436 6-gate-run failure.
215
-
216
- ### Error Surfacing
217
-
218
- Gate failure detail must include actual Zod validation error paths — not hardcoded "failed" strings. The Zod errors (path + message) are computed at `workflow.test.ts:59` but discarded at `run-gate.ts:291` where `detail` is hardcoded to `"failed"`. The heal agent at `ship.js:152-159` receives failures with no specifics, making single-pass repair impossible.
219
-
220
- **Rule:** After matching a `(fail)` test line, `run-gate.ts` must capture subsequent non-result lines as the detail string. Detail must include the Zod error path (e.g., `acs.0.threshold.op: Invalid enum value. Expected '==' | '>=' | ..., received 'equals'`).
221
-
222
- ### Schema Validation at Write Time
223
-
224
- All writes to `workflow-state.json` must go through `writeWorkflowState()` at `orchestrator.ts:414-430` which validates via Zod with enum-specific error formatting. Per ADR-009:417-418: "writeWorkflowState() validates via Zod" — this is a design decision, not optional.
225
-
226
- **Rule:** Agent heal prompts (ship.js) must not instruct agents to use the Write tool for workflow-state.json edits. All writes must route through `writeWorkflowState()` (via `bun -e` or equivalent) so agents get immediate Zod error feedback at write time, not opaque gate failures.
227
-
228
- ### DISCOVERY_SCHEMA Enum Constraints
229
-
230
- The DISCOVERY_SCHEMA (JSON Schema for Claude structured output) at `ship.js:44` must constrain `threshold.op` and `evidenceMethod.type` to the same enum values as the Zod schema at `schema.ts:16` and `schema.ts:22-25`. Invalid values must be rejected at LLM structured output time, not deferred to gate time.
231
-
232
- **Rule:** Enum values must be exported as const arrays from `schema.ts` and referenced in both the Zod schema and DISCOVERY_SCHEMA. A schema-parity test must assert they match.
233
-
234
- ### Schema Strictness Consistency
235
-
236
- `writeWorkflowState()` at `orchestrator.ts:416` uses `.passthrough()` (allows extra keys) while `workflow.test.ts:57` uses strict `.safeParse()`. State that passes write-time validation can fail gate-time validation if it contains extra keys.
237
-
238
- **Rule:** Both validation paths must use the same strictness level. Recommendation: `.passthrough()` in both, since extra keys are harmless and the schema evolves frequently.
239
-
240
- ### Workflow State Lifecycle
241
-
242
- The `~/.rungate/` directory accumulates workflow state slugs with no archival mechanism. As of 2026-09-17: 1,065 slugs, 2 archived, 14 stuck in non-DONE phases, 1,001 with no workflow-state.json.
243
-
244
- **Rule:** No background cleanup crons or daemons (unanimously rejected — race conditions with active workflows). Cleanup is lazy: `initWorkflow()` at `orchestrator.ts:467` logs a warning when overwriting non-DONE state. One-shot manual cleanup via Makefile target for accumulated slugs.
245
-
246
- ### Success Criteria
247
-
248
- - [ ] GI-1: Gate failure detail includes actual Zod error path + message (not hardcoded "failed")
249
- - [ ] GI-2: DISCOVERY_SCHEMA threshold.op has enum constraint matching schema.ts values
250
- - [ ] GI-3: DISCOVERY_SCHEMA evidenceMethod.type has enum constraint matching schema.ts values
251
- - [ ] GI-4: Enum values exported as const arrays from schema.ts (single source of truth)
252
- - [ ] GI-5: schema-parity.test.ts asserts DISCOVERY_SCHEMA enums match Zod enums
253
- - [ ] GI-6: Heal agent write prompts in ship.js route through writeWorkflowState() (not Write tool)
254
- - [ ] GI-7: SCHEMA-GUIDE.md references validated write path (not Write tool)
255
- - [ ] GI-8: writeWorkflowState() and workflow.test.ts use same parse strictness
256
- - [ ] GI-9: initWorkflow() warns when overwriting non-DONE workflow state
@@ -1,286 +0,0 @@
1
- ---
2
- doc-type: spec
3
- status: draft
4
- owner: jason
5
- created: 2026-09-03
6
- updated: 2026-09-03
7
- testable: true
8
- ---
9
-
10
- # Harness Skill Chain Spec
11
- # Version 0.1 — Draft from audit data
12
-
13
- ## Problem Statement
14
-
15
- The harness is good at committing code and running tests. It is bad at proving the right problem was solved. Audit of 30+ ship cycles shows:
16
-
17
- - Reproduce-first: almost never done (#82 closed 3x without reproducing)
18
- - Before/after evidence: never captured structurally
19
- - Quinn skipped with wrong logic (file-type based, not UX-change based)
20
- - No bug-specific E2E tests
21
- - No artifact linking before→commit→after→release tag
22
-
23
- The harness enforces "code is correct" but not "user's problem is solved."
24
-
25
- ## Design Principle
26
-
27
- **Skills as a chain, not a monolith.** Each skill has one job, one gate, one artifact. The harness validates handoffs between skills.
28
-
29
- **Every step has a scorecard.** Telemetry tracks success/failure per skill, per step. No more "unknown" categories — if a step runs, it's measured.
30
-
31
- ## The Chain
32
-
33
- ```
34
- INTAKE → DEFINE → SHIP → PROVE → RELEASE
35
- ↑ |
36
- └──── council review + iterate ────┘
37
- ```
38
-
39
- ### Step 1: INTAKE (`/goal`)
40
-
41
- **Trigger:** Jason brings observation, screenshot, email, link, verbal description.
42
-
43
- **What happens:**
44
- 1. Research the observation — reproduce it, trace the data source
45
- 2. Capture before-state (screenshot, API response, or data snapshot)
46
- 3. Create GitHub issue with measurable ACs
47
- 4. Garbage-test every AC
48
-
49
- **Gate:** Issue exists with ACs. Before-state captured.
50
-
51
- **Artifact:** GitHub issue + before-state evidence file in workflow-state.json
52
-
53
- **Scorecard metrics:**
54
- - Did we reproduce before defining? (reproduce_attempted: true/false)
55
- - Did we capture before-state? (beforeState.captured: true/false)
56
- - Did ACs survive garbage test? (acsGarbageTested: true/false)
57
-
58
- ### Step 2: DEFINE (optional — for M/L work)
59
-
60
- **Trigger:** Goal is complex enough to need spec, ADR, or council.
61
-
62
- **What happens:**
63
- 1. `/council` on the findings — is this the right problem? Right fix?
64
- 2. Write spec or ADR if architectural
65
- 3. Break into sub-issues if L-sized
66
-
67
- **Gate:** Council ran (if uncertain). Spec exists (if M/L).
68
-
69
- **Artifact:** Council synthesis + spec/ADR link on issue
70
-
71
- **Scorecard metrics:**
72
- - Council ran when uncertain? (councilRan: true/false)
73
- - Spec written for M/L? (specWritten: true/false)
74
-
75
- ### Step 3: SHIP (`/ship`)
76
-
77
- **Trigger:** Issue exists with ACs. Scope gate passes.
78
-
79
- **What happens (3-layer verification before PR):**
80
-
81
- **Layer 1 — Local Dev (fast feedback)**
82
- 1. Marcus codes on local dev (make dev-all, :7778 API / :5173 UI)
83
- 2. Unit tests pass (`bun test`), types check (`tsc --noEmit`)
84
- 3. Quinn validates on local dev (:5173 UI, :7778 API)
85
- - Tests as brand-new user, captures screenshots for UI bugs
86
- - If Quinn fails → back to Marcus → fix → Quinn retests (fast cycle)
87
- 4. **GATE: Quinn local PASS before container build**
88
-
89
- **Layer 2 — Container Verification (deployment correctness)**
90
- 5. `make test-rebuild` → rebuilds container image from current code, starts on :7776 with seed data
91
- 6. Quinn validates on container (:7776)
92
- - Same tests as local dev, against containerized app
93
- - Verifies deployment artifact matches dev behavior
94
- - container-has-fix gate validates container has the commit SHA
95
- 7. Rook security scan on changed files (STANDARD+ tiers)
96
- 8. If Quinn/Rook fail → back to Marcus → fix → rebuild → retest
97
- 9. **GATE: Quinn container PASS before PR is opened**
98
-
99
- **Layer 3 — PR + Ship Gate**
100
- 10. Commit → push → PR created via `gh pr create`
101
- 11. CI runs on Mac Mini (typecheck, build, container E2E, harness gates)
102
- 12. Ship gate runs (verify-gate + ship-gate checks)
103
- 13. **GATE: Ship gate PASS. PR ready for review.**
104
-
105
- **Gate:** verify-gate PASS + ship-gate PASS. Quinn passed on both local and container.
106
-
107
- **Artifact:** PR with verified code, ready for Jason to review and merge.
108
-
109
- **Scorecard metrics:**
110
- - Iterations to pass verify-gate (iterationCount)
111
- - Quinn local verdict (quinn.local.verdict)
112
- - Quinn container verdict (quinn.container.verdict)
113
- - Gate telemetry (gate-telemetry.jsonl)
114
- - Test/tsc results
115
-
116
- ### Step 4: PROVE (post-merge — prod data verification)
117
-
118
- **Trigger:** Jason reviewed and merged the PR. Code is on main.
119
-
120
- **What happens:**
121
- 1. Verify fix commit is on main (`git branch --contains {sha} main`)
122
- 2. **Rebuild image + spin up isolated prove container** (`make prove-up`, port 7776)
123
- - Rebuilds the container image (includes `build` dependency — ensures fix code is in the image)
124
- - Rsyncs PROD data to data-test/, starts test container with ALLOW_RESET=true
125
- - Container uses real prod data but is isolated from live container
126
- - P11 (container-has-fix) FAILs if image was built before fix commit
127
- 3. **Quinn validates on prove container** (port 7776, prod data)
128
- - Capture after-state (prod). The prove container with rsynced prod data IS prod evidence.
129
- - For UI bugs: navigate to affected page, verify fix with real customer data
130
- 4. **Before/after evidence artifact:**
131
- ```json
132
- {
133
- "beforeState": {
134
- "type": "screenshot|api-response|data-snapshot",
135
- "capturedAt": "...",
136
- "path": "~/.rungate/{slug}/evidence/before.png"
137
- },
138
- "afterLocal": {
139
- "type": "screenshot|api-response",
140
- "capturedAt": "...",
141
- "path": "~/.rungate/{slug}/evidence/after-local.png"
142
- },
143
- "afterProd": {
144
- "type": "screenshot|api-response",
145
- "capturedAt": "...",
146
- "path": "~/.rungate/{slug}/evidence/after-prod.png"
147
- }
148
- }
149
- ```
150
-
151
- **Key difference from SHIP container test:** SHIP uses `make test-rebuild` (seed data — catches deployment bugs). PROVE uses `make prove-up` (prod data — catches data-shape bugs). Not redundant — different data, different failure classes.
152
-
153
- **Gate:** Before/after evidence exists. Quinn passed on prove container with prod data. **Mechanically enforced (#294):** prove gate FAILs if STANDARD+ tier and `afterEvidence.environment !== "prod"`. LIGHT tier exempted (local-only sufficient). Prove container lifecycle: make prove-up (start) / make prove-down (auto on prove gate PASS).
154
-
155
- **Artifact:** prove-evidence.json with before/after comparison + Quinn verdict.
156
-
157
- **Scorecard metrics:**
158
- - Before-state captured? (beforeState.captured)
159
- - After-state captured prod? (afterProd.captured)
160
- - Quinn passed prod? (quinn.prod.verdict)
161
- - Verdict: PROVEN/UNPROVEN/INCONCLUSIVE
162
-
163
- ### Step 5: RELEASE (`/release` — new)
164
-
165
- **Trigger:** PROVE passed. Code is verified on prod container.
166
-
167
- **What happens:**
168
- 1. `make release-dry-run` (optional confidence check)
169
- 2. `make release-patch` → version bump → release-test → tag push
170
- 3. Gate 4 runs on GitHub Actions (E2E on seeded server)
171
- 4. Draft release → upload assets → verify → publish
172
- 5. Post release proof to issue:
173
- ```
174
- ## Release Proof
175
- - Before: [screenshot] — bug visible in v1.7.6.0
176
- - Fix: commit abc123
177
- - After (local): [screenshot] — bug fixed on dev
178
- - After (prod): [screenshot] — bug fixed on prod
179
- - Release: v1.7.6.1 — Gate 4 PASS
180
- - Download: setup.sh verified 200 OK
181
- ```
182
-
183
- **Gate:** Release tag exists. Gate 4 passed. Asset URLs return 200.
184
-
185
- **Artifact:** Published GitHub release with proof comment on issue.
186
-
187
- **Scorecard metrics:**
188
- - Release-dry-run passed? (releaseDryRun.result)
189
- - Gate 4 passed? (gate4.result)
190
- - Asset verification passed? (assetVerification.result)
191
- - Proof posted to issue? (releaseProof.posted)
192
-
193
- ## Convergence Loop
194
-
195
- After PROVE or RELEASE:
196
-
197
- ```
198
- Post-ship council → findings? → file issues → SHIP them → council again → clean? → DONE
199
- ```
200
-
201
- **Council-recommended signal (advisory, not auto-triggered):** After STANDARD+ tier ships, the harness emits a `council-recommended` signal. DA reads the signal and decides whether to run council. Council is NOT auto-triggered — auto-triggering costs 200-400K tokens without human judgment. If council finds issues, they become new issues that enter the chain at DEFINE. *(Updated 2026-09-04: council decision overrode original "auto-trigger" to advisory-only.)*
202
-
203
- **Stall in ship:** Same verify failures 2x → DA decides whether to run council on the failures → ship mechanical recommendations → re-verify.
204
-
205
- ## QC Validation (automatic at gates)
206
-
207
- `skill-qc-validator.sh` runs automatically at each gate, checking the skill's output against `schemas/skill-qc-checks.json`:
208
- - **scope gate** → goal QC (G1-G8) at STANDARD+ tiers
209
- - **verify gate** → ship QC (S1-S8) at STANDARD+ tiers
210
- - **prove gate** → prove QC (P1-P9) at all tiers
211
-
212
- Currently WARN severity (2-week calibration). Manual: `bash scripts/skill-qc-validator.sh --skill NAME --issue N --slug SLUG`
213
-
214
- ## Validation Layers (reconciled 2026-09-15, aligned with CI-ENFORCEMENT-SPEC)
215
-
216
- ```
217
- Layer 1: Local Dev (SHIP step 1-4) — fast feedback
218
- make dev-all → :7778 (API) + :5173 (UI)
219
- Marcus codes + tests. Quinn validates on dev server.
220
- GATE: Quinn local PASS before container build.
221
-
222
- Layer 2: Test Container (SHIP step 5-9) — deployment correctness
223
- make test-rebuild → :7776 (seed data, isolated)
224
- container-has-fix gate (P11 FAIL if stale). Quinn re-validates.
225
- GATE: Quinn container PASS before PR is opened.
226
-
227
- Layer 3: Prove Container (PROVE step 2-3) — prod data correctness
228
- make prove-up → :7776 (rsynced PROD data, isolated from live)
229
- Quinn validates with real customer data shapes.
230
- Post-merge only. make prove-down auto-runs on PASS.
231
-
232
- Layer 4: Release Pipeline (RELEASE step)
233
- make release-dry-run → make release-patch → Gate 4
234
- E2E on seeded server. Asset verification.
235
- ```
236
-
237
- **Industry alignment:** Google presubmit/postsubmit, Dagger "same pipeline everywhere", Vercel preview deployments. Layer 2 is pre-merge (shift left). Layer 3 is post-merge (prod data verification). Not redundant — different data, different failure classes.
238
-
239
- ## Per-Skill Scorecard
240
-
241
- Every skill appends to `~/.claude/state/skill-telemetry.jsonl`:
242
-
243
- ```json
244
- {
245
- "ts": "2026-09-03T...",
246
- "skill": "goal|research|council|ship|prove|release",
247
- "issue": 1229,
248
- "metrics": {
249
- "reproduced": true,
250
- "beforeCaptured": true,
251
- "councilRan": false,
252
- "iterations": 2,
253
- "quinnLocal": "PASS",
254
- "quinnProd": "PASS",
255
- "gate4": "PASS"
256
- }
257
- }
258
- ```
259
-
260
- Monthly audit: query telemetry, find skills with low success rates, council on improvements.
261
-
262
- ## What This Replaces
263
-
264
- - Ship skill still handles SCOPE → BUILD → VERIFY (Step 3)
265
- - PROVE replaces the manual "did you test it?" question
266
- - RELEASE connects to CI-RELEASE-PIPELINE.md (DDB docs/)
267
- - Council convergence replaces manual "should we run council?"
268
- - Skill telemetry replaces "unknown" category from gate audit
269
-
270
- ## Open Questions
271
-
272
- 1. ~~Should PROVE be a separate skill or integrated into ship's VERIFY phase?~~ **RESOLVED: Separate skill.** /prove is standalone, produces prove-evidence.json. (ADR-007)
273
- 2. ~~Should /release be a skill or a Makefile-only step?~~ **RESOLVED: Separate skill.** /release validates, generates changelog, monitors Gate 4. (ADR-007)
274
- 3. How does Quinn capture screenshots programmatically? (Playwright? Browser agent?) **OPEN**
275
- 4. Where do before/after evidence files live long-term? (workflow-state.json gets TTL'd) **OPEN** — GoalRecords are TTL-exempt per ADR-007; evidence files need similar treatment
276
- 5. Should the convergence loop have a budget cap? **OPEN**
277
-
278
- ## Related Specs
279
-
280
- - `~/.claude/PAI/HARNESS-STANDARD.md` — current harness loop
281
- - `~/.claude/PAI/HARNESS-GATES.md` — gate documentation
282
- - `~/Projects/DailyBriefDashboard/docs/CI-RELEASE-PIPELINE.md` — 3-layer validation + release flow
283
- - `~/.claude/skills/ship/SKILL.md` — ship skill v2
284
- - `~/.claude/skills/goal/SKILL.md` — goal skill
285
- - `~/.claude/skills/testing-and-qa-validation/SKILL.md` — QA workflow (15 steps)
286
- - `~/.claude/workflows/council.js` — council workflow with enforcement classification
@@ -1,93 +0,0 @@
1
- ---
2
- doc-type: spec
3
- status: active
4
- owner: jason
5
- created: 2026-09-03
6
- updated: 2026-09-03
7
- testable: true
8
- ---
9
-
10
- # Harness Skill Contract
11
-
12
- Every harness-compatible skill must implement this contract in its SKILL.md YAML frontmatter.
13
-
14
- ## Required Fields
15
-
16
- ### 1. input
17
- What the skill reads. Issue number, workflow-state.json, observation text, etc.
18
-
19
- ### 2. gateIn
20
- Preconditions — what must be true before this skill can run. Expressed as verifiable checks.
21
-
22
- ### 3. gateOut
23
- Postconditions — what must be true when this skill completes. The next skill in the chain checks these.
24
-
25
- ### 4. artifact
26
- What this skill produces. File path, GitHub comment, state update. Must be a file that survives session boundaries.
27
-
28
- ### 5. telemetry
29
- Which telemetry file this skill writes to and what fields it logs. All skills write to harness-telemetry.jsonl with a skill field.
30
-
31
- ### 6. errorRecovery
32
- How the skill handles failures:
33
- - timeout_seconds: max execution time
34
- - failure_mode: BLOCK (stop chain) or WARN (continue with note)
35
- - cleanup_action: what to clean up on failure
36
- - resumable: can the skill be re-run to pick up where it left off?
37
-
38
- ## YAML Frontmatter Format
39
-
40
- ```yaml
41
- ---
42
- name: SkillName
43
- description: One-line description
44
- version: 1.0.0
45
- contract:
46
- input: "what it reads"
47
- gateIn: "preconditions"
48
- gateOut: "postconditions"
49
- artifact: "what it produces"
50
- telemetry: "harness-telemetry.jsonl with skill=name"
51
- errorRecovery:
52
- timeout_seconds: 600
53
- failure_mode: BLOCK
54
- cleanup_action: "cleanup command"
55
- resumable: true
56
- ---
57
- ```
58
-
59
- ## Chain Handoffs
60
-
61
- Each skill's gateOut must satisfy the next skill's gateIn:
62
-
63
- ```
64
- /goal gateOut: "GitHub issue exists with ACs + beforeState"
65
- -> /ship gateIn: "Issue exists with ACs"
66
-
67
- /ship gateOut: "verify-gate PASS, code on main, mergeCommitSha set"
68
- -> /prove gateIn: "Code merged to main (mergeCommitSha exists)"
69
-
70
- /prove gateOut: "prove-evidence.json with verdict"
71
- -> /release gateIn: "main is clean"
72
- ```
73
-
74
- ## Validation
75
-
76
- Run `bash scripts/harness-skill-check.sh skills/{name}/SKILL.md` to validate any skill.
77
-
78
- ## Adding a New Skill
79
-
80
- 1. Create `skills/{name}/SKILL.md` with YAML frontmatter including all 6 contract fields
81
- 2. Run `bash scripts/harness-skill-check.sh skills/{name}/SKILL.md` — must PASS
82
- 3. Add to chain documentation if it participates in the harness chain
83
- 4. Call `/doc-hygiene` to register in project index
84
-
85
- ## Current Skills
86
-
87
- | Skill | Contract? | Chain position |
88
- |-------|-----------|---------------|
89
- | /goal | Yes | Start |
90
- | /ship | Yes | After /goal |
91
- | /prove | Yes | After /ship |
92
- | /release | Yes | After /prove (end) |
93
- | /council | Yes | Lateral (any point) |