rungate 0.2.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (196) hide show
  1. package/.claude/agents/aditi.md +47 -0
  2. package/.claude/agents/marcus.md +50 -0
  3. package/.claude/agents/quinn.md +58 -0
  4. package/.claude/agents/rook.md +47 -0
  5. package/.claude/agents/serena.md +46 -0
  6. package/.claude/rungate.json +11 -0
  7. package/.fallowrc.json +28 -0
  8. package/.github/copilot-instructions.md +20 -0
  9. package/.rungate/conformity-findings.json +648 -0
  10. package/AGENTS.md +106 -0
  11. package/CODE-MAP.md +50 -0
  12. package/HARNESS.md +88 -0
  13. package/bun.lock +26 -0
  14. package/bunfig.toml +2 -0
  15. package/config/brief-policies.json +28 -0
  16. package/config/ceremony-profiles.json +212 -0
  17. package/config/consumer-cascade-map.json +24 -0
  18. package/config/doc-cascade-map.json +24 -0
  19. package/config/eval.json +84 -0
  20. package/config/profiles/account-plan.yaml +36 -0
  21. package/config/profiles/campaign.yaml +94 -0
  22. package/config/prove-evidence.schema.json +65 -0
  23. package/config/spec-policies.json +36 -0
  24. package/config/workflow-schema.json +281 -0
  25. package/gates/SCHEMA-GUIDE.md +121 -0
  26. package/gates/ac-quality.test.ts +106 -0
  27. package/gates/adversarial.test.ts +274 -0
  28. package/gates/brief-assembler.test.ts +429 -0
  29. package/gates/brief-assembler.ts +334 -0
  30. package/gates/bunfig.toml +2 -0
  31. package/gates/chain.test.ts +91 -0
  32. package/gates/e2e-smoke.test.ts +234 -0
  33. package/gates/error-classifier.test.ts +249 -0
  34. package/gates/error-classifier.ts +89 -0
  35. package/gates/orchestrator.test.ts +950 -0
  36. package/gates/orchestrator.ts +569 -0
  37. package/gates/preload.ts +77 -0
  38. package/gates/prompts/ac-adversary.md +20 -0
  39. package/gates/prompts/evidence-validator.md +22 -0
  40. package/gates/prompts/prove-reproducer.md +53 -0
  41. package/gates/prove.test.ts +164 -0
  42. package/gates/run-gate.ts +936 -0
  43. package/gates/schema-parity.test.ts +197 -0
  44. package/gates/schema.ts +362 -0
  45. package/gates/self-heal.test.ts +511 -0
  46. package/gates/self-heal.ts +121 -0
  47. package/gates/ship-orchestrator.test.ts +379 -0
  48. package/gates/ship-orchestrator.ts +363 -0
  49. package/gates/test-fixtures/broken-no-environments.json +60 -0
  50. package/gates/test-fixtures/valid-standard.json +54 -0
  51. package/gates/test-utils.ts +43 -0
  52. package/gates/witness.ts +220 -0
  53. package/gates/workflow.test.ts +1053 -0
  54. package/hooks/AgentBriefGuard.hook.ts +586 -0
  55. package/hooks/AgentVerdictCapture.hook.ts +134 -0
  56. package/hooks/AutoVerifyGate.hook.ts +76 -0
  57. package/hooks/GateEnforcement.hook.ts +273 -0
  58. package/hooks/IssueCloseGuard.hook.ts +116 -0
  59. package/hooks/MergeGuard.hook.ts +55 -0
  60. package/hooks/StaleTTLCleanup.hook.ts +133 -0
  61. package/hooks/VerifyPhaseLock.hook.ts +81 -0
  62. package/hooks/WorkflowStateGuard.hook.ts +37 -0
  63. package/hooks/lib/agentDetection.ts +70 -0
  64. package/hooks/lib/findWorkflow.ts +101 -0
  65. package/hooks/lib/parseStdin.ts +37 -0
  66. package/hooks/lib/paths.ts +111 -0
  67. package/hooks/lib/utils.ts +81 -0
  68. package/lib/conformity.ts +959 -0
  69. package/lib/paths.ts +23 -0
  70. package/lib/rungate-schema.ts +60 -0
  71. package/lib/signal-phrases.ts +7 -0
  72. package/package.json +20 -0
  73. package/prompts/ac-adversary.md +20 -0
  74. package/prompts/ac-format.md +34 -0
  75. package/prompts/aditi.md +29 -0
  76. package/prompts/blast-radius.md +29 -0
  77. package/prompts/coding-principles.md +34 -0
  78. package/prompts/container-rebuild.md +29 -0
  79. package/prompts/container-verify.md +28 -0
  80. package/prompts/discovery.md +27 -0
  81. package/prompts/environment.md +28 -0
  82. package/prompts/escalation-decision-tree.md +38 -0
  83. package/prompts/evidence-hierarchy.md +31 -0
  84. package/prompts/evidence-validator.md +22 -0
  85. package/prompts/marcus.md +29 -0
  86. package/prompts/prevention.md +26 -0
  87. package/prompts/prove-reproducer.md +43 -0
  88. package/prompts/quinn-decision-tree.md +31 -0
  89. package/prompts/quinn-ui-brief.md +106 -0
  90. package/prompts/quinn.md +29 -0
  91. package/prompts/rca.md +28 -0
  92. package/prompts/read-before-write.md +30 -0
  93. package/prompts/regression.md +26 -0
  94. package/prompts/rook.md +29 -0
  95. package/prompts/serena.md +28 -0
  96. package/prompts/testing-strategy.md +34 -0
  97. package/reference/migration-manifest.json +554 -0
  98. package/reference/prompts/agent-aditi.md +38 -0
  99. package/reference/prompts/agent-marcus.md +38 -0
  100. package/reference/prompts/agent-quinn.md +38 -0
  101. package/reference/prompts/agent-rook.md +37 -0
  102. package/reference/prompts/agent-serena.md +37 -0
  103. package/reference/prompts/council-skill.md +170 -0
  104. package/reference/prompts/goal-skill.md +235 -0
  105. package/reference/prompts/prove-skill.md +283 -0
  106. package/reference/prompts/quinn-standard.md +246 -0
  107. package/reference/prompts/ship-convergence.md +65 -0
  108. package/reference/prompts/ship-escalation.md +43 -0
  109. package/reference/prompts/ship-iteration.md +382 -0
  110. package/reference/prompts/ship-pending-patches.md +246 -0
  111. package/reference/prompts/ship-reference.md +250 -0
  112. package/reference/prompts/ship-skill.md +347 -0
  113. package/reference/prompts/ship-templates.md +215 -0
  114. package/reference/scripts/batch-status.sh +68 -0
  115. package/reference/scripts/chain-compat-check.sh +67 -0
  116. package/reference/scripts/context-audit.sh +314 -0
  117. package/reference/scripts/dead-export-check.sh +61 -0
  118. package/reference/scripts/generate-qc-tables.sh +92 -0
  119. package/reference/scripts/harness-init.sh +111 -0
  120. package/reference/scripts/pai-054-baseline-check.sh +40 -0
  121. package/reference/scripts/prove-smoke-test.sh +112 -0
  122. package/reference/scripts/setup-keychain-salt.sh +12 -0
  123. package/reference/scripts/skill-optimizer-friday.sh +88 -0
  124. package/reference/scripts/skill-qc-test.sh +351 -0
  125. package/reference/scripts/skill-synthesize-tonight.sh +66 -0
  126. package/reference/specs/HARNESS-EXTRACTION-SPEC.md +263 -0
  127. package/reference/specs/harness-v3-bash-deletion-plan.md +65 -0
  128. package/reference/specs/harness-v3-migration-plan.md +188 -0
  129. package/scripts/decision-reconcile.sh +74 -0
  130. package/scripts/decision-reconcile.ts +152 -0
  131. package/scripts/extract-constraints.ts +260 -0
  132. package/scripts/generate-code-map.ts +339 -0
  133. package/scripts/git-hooks/pre-commit +76 -0
  134. package/scripts/git-hooks/pre-push +271 -0
  135. package/scripts/harness-skill-check.sh +52 -0
  136. package/scripts/lib/gap-emit.sh +65 -0
  137. package/scripts/lib/jq-update.sh +30 -0
  138. package/scripts/lib/resolve-slug.sh +30 -0
  139. package/scripts/lib/verify-spec-compliance.sh +102 -0
  140. package/scripts/prove-backfill.sh +87 -0
  141. package/scripts/scaffold-project.ts +1488 -0
  142. package/scripts/scaffold-rungate-config.ts +91 -0
  143. package/scripts/self-containment-check.sh +109 -0
  144. package/scripts/skill-qc-validator.sh +402 -0
  145. package/scripts/slug-resolver.sh +14 -0
  146. package/scripts/sync-spec-tests.ts +236 -0
  147. package/specs/BOOTSTRAP-DATA-FLOW-SPEC.md +2493 -0
  148. package/specs/BOOTSTRAP-TEST-PLAN.md +571 -0
  149. package/specs/HARNESS-GATES.md +256 -0
  150. package/specs/HARNESS-SKILL-CHAIN.md +286 -0
  151. package/specs/HARNESS-SKILL-CONTRACT.md +93 -0
  152. package/specs/HARNESS-STANDARD.md +488 -0
  153. package/specs/SPEC-TEMPLATE.md +53 -0
  154. package/specs/bootstrap-data-flow.png +0 -0
  155. package/specs/harness-automation-matrix.md +314 -0
  156. package/specs/harness-automation-matrix.rules.json +64 -0
  157. package/test/anti.test.ts +80 -0
  158. package/test/contract-negative.test.ts +86 -0
  159. package/test/contract.test.ts +158 -0
  160. package/test/external-deps.test.ts +48 -0
  161. package/test/fixtures/golden-project/.env.example +3 -0
  162. package/test/fixtures/golden-project/Makefile +7 -0
  163. package/test/fixtures/golden-project/docs/adr/ADR-001-framework.md +23 -0
  164. package/test/fixtures/golden-project/docs/setup-guide.md +10 -0
  165. package/test/fixtures/golden-project/package.json +10 -0
  166. package/test/fixtures/golden-project/specs/api-spec.md +20 -0
  167. package/test/fixtures/golden-project/src/auth.ts +8 -0
  168. package/test/fixtures/golden-project/src/db.ts +11 -0
  169. package/test/fixtures/golden-project/src/index.ts +19 -0
  170. package/test/fixtures/golden-project/src/utils/format.ts +3 -0
  171. package/test/fixtures/golden-project/test/auth.test.ts +15 -0
  172. package/test/fixtures/golden-project/tsconfig.json +12 -0
  173. package/test/meta-sc-coverage.test.ts +159 -0
  174. package/test/phase-0.test.ts +846 -0
  175. package/test/phase-1.test.ts +617 -0
  176. package/test/phase-2.test.ts +620 -0
  177. package/test/phase-3.test.ts +99 -0
  178. package/test/scaffold-conformity.test.ts +8 -0
  179. package/test/schema-canary.test.ts +51 -0
  180. package/test/spec-compliance-auto.test.ts +64 -0
  181. package/test/spec-compliance.test.ts +295 -0
  182. package/test/spec-discovery.test.ts +68 -0
  183. package/test/spec-drift.test.ts +5 -0
  184. package/test/structure.test.ts +165 -0
  185. package/test/unit/agent-brief-guard-ship.test.ts +293 -0
  186. package/test/unit/extract-constraints.test.ts +145 -0
  187. package/test/unit/issue-close-guard.test.ts +204 -0
  188. package/test/unit/scaffold-project.test.ts +417 -0
  189. package/test/unit/skill-cooccurrence.test.ts +177 -0
  190. package/test/unit/skill-sequence-logger.test.ts +244 -0
  191. package/test/unit/workflow-state-guard.test.ts +95 -0
  192. package/workflows/batch-ship.js +295 -0
  193. package/workflows/council.js +481 -0
  194. package/workflows/prove.js +560 -0
  195. package/workflows/ship.js +789 -0
  196. package/workflows/verify.js +236 -0
@@ -0,0 +1,283 @@
1
+ ---
2
+ name: Prove
3
+ description: Standalone verification that a fix solves the user's problem. Reproduces bug, captures before/after evidence, runs Quinn. Works standalone or in harness chain.
4
+ version: 1.0.0
5
+ doc-type: reference
6
+ status: active
7
+ owner: jason
8
+ updated: 2026-09-06
9
+ contract:
10
+ input: "Issue number OR workflow-state.json path"
11
+ gateIn: "Code merged to main (mergeCommitSha exists) OR standalone (no prior /ship required)"
12
+ gateOut: "prove-evidence.json exists with verdict (PROVEN/UNPROVEN/INCONCLUSIVE)"
13
+ artifact: "prove-evidence.json + evidence files in ~/.rungate/{slug}/evidence/"
14
+ outputSchema: "skills/prove/prove-evidence.schema.json"
15
+ telemetry: "harness-telemetry.jsonl with skill=prove"
16
+ errorRecovery:
17
+ timeout_seconds: 600
18
+ failure_mode: BLOCK
19
+ cleanup_action: "rm -f evidence/partial-*.json"
20
+ resumable: true
21
+ ---
22
+
23
+ # Prove
24
+
25
+ Standalone verification skill. Proves a fix actually solves the user's problem — not just that tests pass.
26
+
27
+ ## Trigger conditions
28
+
29
+ - When: `/prove 1229`, "prove this fix works", "verify the fix", "before and after"
30
+ - Standalone: can run independently on any issue, any time
31
+ - Chain position: after /ship, before /release
32
+ - Skip when: pure infrastructure changes with no user-visible impact
33
+
34
+ ## Interface
35
+
36
+ ```
37
+ /prove 1229 — prove fix works for issue #1229
38
+ /prove 1229 --prod — prove on isolated container with prod data (7776 via make prove-up)
39
+ /prove 1229 --before-only — capture before-state only (pre-fix)
40
+ /prove 1229 --local — local dev only (7776), skip prod
41
+ ```
42
+
43
+ ## Environment Semantics
44
+
45
+ - **"local"** = local dev server (make dev-all, port 7776/5173), no container
46
+ - **"prod"** = isolated container with rsynced prod data (make prove-up, port 7776), NOT the live container on 7777
47
+ - The live prod container (7777) is never touched by prove agents
48
+ - `prove-evidence.json` field `afterEvidence.environment: "prod"` means prod data was used in an isolated test container, not that the live container was tested
49
+
50
+ <!-- gates/prove.test.ts checks afterEvidence.environment === "prod" — the isolated prove container satisfies this because it runs prod data. The prove skill sets environment: "prod" when using the isolated container with rsynced prod data. -->
51
+
52
+ ### Chain Detection
53
+
54
+ At skill entry, the runner checks for upstream artifacts in the slug directory:
55
+
56
+ 1. **GoalRecord check:**
57
+ ```bash
58
+ GOAL_RECORD="$RUNGATE_WORK_DIR/{slug}/goal-record.json"
59
+ if [[ -f "$GOAL_RECORD" ]]; then
60
+ echo "INFO: Chain mode — GoalRecord found. Using structured SCs for criteriaResults."
61
+ # Each SC from GoalRecord becomes a criteriaResult entry
62
+ else
63
+ echo "INFO: Standalone mode — no GoalRecord. Derive criteria from issue ACs."
64
+ fi
65
+ ```
66
+
67
+ 2. **Ship evidence check:**
68
+ ```bash
69
+ SHIP_EVIDENCE="$RUNGATE_WORK_DIR/{slug}/ship-evidence.json"
70
+ if [[ -f "$SHIP_EVIDENCE" ]]; then
71
+ echo "INFO: Ship evidence found. Using mergeCommitSha for verification."
72
+ # Read mergeCommitSha — verify it matches git rev-parse main
73
+ else
74
+ echo "INFO: No ship evidence. Using HEAD as commit reference."
75
+ fi
76
+ ```
77
+
78
+ 3. **Standalone graceful degradation:** Missing upstream = standalone mode, not error. The skill works independently with reduced context.
79
+ 4. **File presence only** — `test -f`, no parsing until confirmed present. No env vars, no flags.
80
+
81
+ ## Workflow
82
+
83
+ ### Step 1 — Load context
84
+
85
+ 1. workflow-state.json is read for the issue (from ~/.rungate/ by issue number)
86
+ 2. If not found: the GitHub issue is read directly (standalone mode)
87
+ 3. issueGoal, mergeCommitSha, beforeState, ACs are extracted
88
+ 4. Environment is determined: local dev (7776/5173) or prod (7776 isolated with prod data via make prove-up)
89
+
90
+ ### Step 2 — Verify fix is deployed
91
+
92
+ 1. mergeCommitSha is checked against main: `git branch --contains {sha} main`
93
+ 2. If --prod: make prove-up runs (rebuild image + rsync prod data + start isolated container on 7776), container-has-fix is checked (P11 FAILs if image predates fix commit)
94
+ 3. If neither: FAIL — "fix not deployed to target environment"
95
+
96
+ ### Step 3 — Reproduce (before-state)
97
+
98
+ If beforeState exists from /goal:
99
+ - The before-state evidence file is read
100
+ - The bug's appearance is noted
101
+
102
+ If beforeState doesn't exist (standalone mode):
103
+ - The issue description is read for reproduction steps
104
+ - Reproduction is attempted on the target environment
105
+ - If bug is NOT reproducible: verdict = INCONCLUSIVE (fix may have already deployed)
106
+ - Current state is captured as "before equivalent" from issue description
107
+
108
+ ### Step 4 — Validate fix (after-state)
109
+
110
+ **For API/data issues:**
111
+ 1. The same API call or data check that showed the bug is run
112
+ 2. The response is captured as after-state evidence
113
+ 3. Comparison: does the response match the expected AC threshold?
114
+
115
+ **For UI issues:**
116
+ 1. **Fix deployment verification:** Before testing, Quinn verifies the container/server has the fix commit:
117
+ - `git log --oneline -1` is run on the target environment to get current HEAD
118
+ - HEAD is compared against `mergeCommitSha` from workflow-state.json or ship-evidence.json
119
+ - If mismatch: FAIL with "Testing wrong version — container HEAD {actual} != fix commit {expected}"
120
+ - Verified commit SHA is reported in QA report header: `**Container:** {SHA} (verified)`
121
+ 2. Quinn is spawned with browser agent to navigate to the affected page
122
+ 3. Quinn captures a screenshot of the fixed state
123
+ 4. Comparison: does the UI match the expected behavior from the ACs?
124
+ 5. Quinn tests as a brand-new user
125
+
126
+ **For both:**
127
+ - Evidence is captured to `~/.rungate/{slug}/evidence/after-{environment}.{ext}`
128
+ - Evidence types: text (API response), screenshot (UI), data snapshot (JSON)
129
+
130
+ ### Step 5 — Compare and verdict
131
+
132
+ 1. Before-state and after-state are compared
133
+ 2. Each AC threshold is checked: does the fix meet the acceptance criteria?
134
+ 3. Verdict is determined:
135
+ - **PROVEN** — bug reproduced, fix verified, ACs met
136
+ - **UNPROVEN** — fix did not resolve the issue (AC threshold not met)
137
+ - **INCONCLUSIVE** — could not reproduce (bug may be intermittent or already fixed)
138
+
139
+ ### Step 6 — Write evidence artifact
140
+
141
+ `~/.rungate/{slug}/prove-evidence.json` is written conforming to `prove-evidence.schema.json`:
142
+
143
+ ```json
144
+ {
145
+ "issueNumber": 1229,
146
+ "verdict": "PROVEN",
147
+ "commitSHA": "abc123",
148
+ "capturedAt": "2026-09-03T...",
149
+ "beforeEvidence": {
150
+ "type": "screenshot",
151
+ "path": "evidence/before.png",
152
+ "capturedAt": "2026-09-03T...",
153
+ "description": "EOL date shows 2025-06 for OCP Virt"
154
+ },
155
+ "afterEvidence": {
156
+ "type": "screenshot",
157
+ "path": "evidence/after-local.png",
158
+ "capturedAt": "2026-09-03T...",
159
+ "environment": "local",
160
+ "description": "EOL date now shows 2027-09 for OCP Virt"
161
+ },
162
+ "comparisonSummary": "EOL date corrected from 2025-06 to 2027-09",
163
+ "quinnVerdict": "PASS",
164
+ "reproduced": true
165
+ }
166
+ ```
167
+
168
+ ### Step 7 — Post proof to issue
169
+
170
+ A proof comment is posted to the GitHub issue:
171
+
172
+ ```markdown
173
+ ## Proof
174
+
175
+ - **Before:** [description] — bug visible
176
+ - **Fix:** commit {sha}
177
+ - **After ({environment}):** [description] — bug resolved
178
+ - **Quinn:** {PASS/FAIL/SKIP} (tested on {verified_commit_sha})
179
+ - **Verdict:** {PROVEN/UNPROVEN/INCONCLUSIVE}
180
+ ```
181
+
182
+ ### Step 8 — Log telemetry
183
+
184
+ Telemetry is appended to harness-telemetry.jsonl:
185
+ ```json
186
+ {"ts":"...","skill":"prove","issue":1229,"check":"verdict","result":"PROVEN","duration_ms":45000}
187
+ ```
188
+
189
+ ### Step 9 — Close issue (chain mode)
190
+
191
+ In chain mode (goal-record.json exists), /prove owns issue closure:
192
+
193
+ 1. If `verdict == "PROVEN"`:
194
+ - The issue is closed: `gh issue close {issueNumber} --repo {issueRepo}` + `proven` label added
195
+ - Posted: "Issue closed by /prove — verdict: PROVEN"
196
+ 2. If `verdict == "UNPROVEN"`:
197
+ - The issue is NOT closed
198
+ - Posted: "Prove verdict: UNPROVEN — issue remains open for re-investigation"
199
+ - A follow-up issue is created if gaps[] has actionable items
200
+ 3. If `verdict == "INCONCLUSIVE"`:
201
+ - The issue is NOT closed
202
+ - Posted: "Prove verdict: INCONCLUSIVE — could not reproduce. Manual investigation needed."
203
+
204
+ In standalone mode (no goal-record.json), /prove does NOT close the issue — only the proof comment (Step 7) is posted.
205
+
206
+ ## Ceremony tiers
207
+
208
+ | Tier | What /prove does |
209
+ |------|-----------------|
210
+ | LIGHT (XS) | API check or text comparison only. No screenshots. |
211
+ | STANDARD (S/M) | Quinn on local dev + isolated prove container (make prove-up). Screenshots for UI issues. Mechanically enforced: prove.test.ts FAILs without prod evidence (#294, ported to Bun in #484). |
212
+ | THOROUGH (L) | Quinn on local + isolated prove container. Full before/after with screenshots. |
213
+
214
+ **B3 limitation:** The B3 prove reproducer (`gates/prompts/prove-reproducer.md`) has Bash and Read tools only -- it cannot verify UI/OUTCOME ACs. CODE ACs are verified via evidence commands (grep, bun test, curl). OUTCOME/UI ACs require Quinn at prove time. The gate (`gates/prove.test.ts`) auto-SKIPs OUTCOME-type and UI-related ACs so they do not count as FAIL (#486).
215
+
216
+ ## Error handling
217
+
218
+ - Timeout: 600 seconds per environment
219
+ - Quinn subprocess timeout: 300 seconds
220
+ - On timeout: partial evidence is written, verdict = INCONCLUSIVE
221
+ - Cleanup: partial-*.json files are removed
222
+ - Resumable: yes — re-running /prove picks up from last completed step
223
+
224
+ ### Artifact Output
225
+
226
+ After all verification steps complete, the prove evidence artifact is written:
227
+
228
+ 1. `~/.rungate/{slug}/prove-evidence.json` is created with:
229
+ ```json
230
+ {
231
+ "contractVersion": "1.0",
232
+ "issueNumber": N,
233
+ "verdict": "[COMPUTED — PROVEN only if ALL criteriaResults are PASS|SKIP]",
234
+ "commitSHA": "[git rev-parse main]",
235
+ "capturedAt": "[ISO timestamp]",
236
+ "criteriaResults": [/* one per SC: {scId, verdict, evidence} */],
237
+ "reproduced": true|false
238
+ }
239
+ ```
240
+ 2. **Verdict computation (MECHANICAL — not LLM-asserted):**
241
+ - FAIL entries in criteriaResults are counted
242
+ - If any FAIL -> verdict = "UNPROVEN"
243
+ - If all PASS|SKIP -> verdict = "PROVEN"
244
+ - If no criteriaResults -> verdict = "INCONCLUSIVE"
245
+ 3. Validation runs via Zod schema in `gates/schema.ts` — cross-field consistency is enforced by superRefine (PROVEN + any FAIL entry = validation failure).
246
+ 4. `writeWorkflowState()` in `gates/orchestrator.ts` validates at write time.
247
+
248
+ ## AFK Mode: Auto-file Follow-ups
249
+
250
+ When running in AFK batch mode (no interactive user) and prove-evidence.json contains non-empty `gaps[]`:
251
+
252
+ 1. For each gap with `priority` != null:
253
+ ```bash
254
+ gh issue create --repo {repo} \
255
+ --title "Follow-up: {gap.ac} — {gap.detail}" \
256
+ --label "needs-triage" \
257
+ --body "Auto-filed by /prove AFK mode.\n\nSource: #{issueNumber}\nGap: {gap.ac}\nStatus: {gap.status}\nDetail: {gap.detail}"
258
+ ```
259
+ 2. The created issue number is written back to `gaps[].followUpIssue` in prove-evidence.json
260
+ 3. Re-validation runs via Zod schema at write time
261
+ 4. Logged: `echo "INFO: Filed {N} follow-up issues from gaps[]"`
262
+
263
+ **Skip conditions:**
264
+ - If `gaps[]` is empty or not present -> skip
265
+ - If not in AFK mode -> skip (DA decides whether to file manually)
266
+ - If `gap.priority` is null -> skip (low-priority gaps don't auto-file)
267
+
268
+ ## Prove Gate (ADR-009 B3)
269
+
270
+ The prove gate (`bun run gates/run-gate.ts --gate prove --slug {slug} --issue {N}`) is the primary verification path. The DA invokes `/prove`, which triggers the gate, which spawns the B3 Prove Reproducer agent.
271
+
272
+ **Key design:** The reproducer reads the issue independently from GitHub (`gh issue view N --json body`), derives its own test plan, and writes prove-evidence.json via the gate-runner. The DA orchestrates but does not author evidence.
273
+
274
+ ```bash
275
+ bun run gates/run-gate.ts --gate prove --slug {slug} --issue {N}
276
+ ```
277
+
278
+ ## Anti-patterns
279
+
280
+ - Skipping reproduction — "code looks fixed" is not proof
281
+ - Using /prove as a substitute for unit tests — /prove validates user experience, not code correctness
282
+ - Capturing after-state without verifying fix is deployed first
283
+ - Marking PROVEN without comparing against AC thresholds
@@ -0,0 +1,246 @@
1
+ ---
2
+ doc-type: reference
3
+ status: active
4
+ owner: jason
5
+ updated: 2026-09-16
6
+ ---
7
+
8
+ # Quinn Torres — Universal Testing Standard
9
+ # Version 1.0 | 2026-04-06
10
+
11
+ Quinn reads this file at the start of every QA session. This standard applies to all projects.
12
+ It layers ON TOP of project-specific prompts — it does not replace them.
13
+
14
+ ---
15
+
16
+ ## Session Start Sequence (Mandatory — Zero Exceptions)
17
+
18
+ Every Quinn session begins with these three actions in order before any other testing:
19
+
20
+ ### 1. Load Project Registry
21
+ Look for a registry at: `~/.claude/PAI/Testing/registries/{project-name}.md`
22
+
23
+ - If it exists: read it in full. Every entry is a known failure pattern — test for all of them.
24
+ - If it does not exist: create an empty one. This project's learning loop starts now.
25
+
26
+ ### 2. Run the Existing Playwright Baseline
27
+ Every project with a `test/` directory has specs that must run first. Do not improvise tests
28
+ that already exist as code.
29
+
30
+ ⚠️ **NEVER run `npx playwright test` (full suite).** The full suite includes state-mutation tests (`lifecycle.spec.ts`, `qa-e2e-newuser.spec.ts`) that call `POST /api/__test/restore` and can permanently wipe production AEs and customers if the snapshot was taken from a stale disk state. This has wiped production data multiple times. **2026-04-08 incident: full suite run during Quinn final verification wiped 9 AEs and 130 customers.**
31
+
32
+ ```bash
33
+ # Safe baseline — API tests only (no state mutation)
34
+ npx playwright test test/api/
35
+
36
+ # UI tests — targeted to specific features, no state-mutation specs
37
+ npx playwright test test/quinn-bkl-pod01-boot01.spec.ts
38
+ npx playwright test test/dashboard-ui.spec.ts
39
+
40
+ # NEVER run these without explicit Jason approval:
41
+ # npx playwright test test/lifecycle.spec.ts
42
+ # npx playwright test test/qa-e2e-newuser.spec.ts
43
+ # npx playwright test ← full suite
44
+ ```
45
+
46
+ Record: total tests run, passed, failed, skipped. These numbers go in your session report.
47
+
48
+ ### 3. Visual Review of Screenshots
49
+ Playwright tests capture screenshots to `/tmp/qa-*.png`. Review every screenshot produced
50
+ using Nielsen's 10 Usability Heuristics as your evaluation lens:
51
+
52
+ | # | Heuristic | What to look for |
53
+ |---|-----------|-----------------|
54
+ | 1 | Visibility of system status | Loading states, progress indicators, success/error feedback |
55
+ | 2 | Match between system and real world | Labels, terminology, icons that make sense to a new user |
56
+ | 3 | User control and freedom | Can users undo actions? Is there a clear exit/cancel? |
57
+ | 4 | Consistency and standards | Buttons, colors, layout consistent across pages |
58
+ | 5 | Error prevention | Forms validate before submit? Destructive actions confirm? |
59
+ | 6 | Recognition over recall | Are options visible, not hidden? Does UI guide next step? |
60
+ | 7 | Flexibility and efficiency | Can experienced users shortcut? Does flow work for beginners? |
61
+ | 8 | Aesthetic and minimalist design | No unnecessary information competing with key content |
62
+ | 9 | Help users recognize/recover from errors | Error messages clear, actionable, not technical jargon |
63
+ | 10 | Help and documentation | Is the next step obvious without needing a manual? |
64
+
65
+ For each screenshot: note which heuristics pass, which fail. A heuristic failure is a finding.
66
+
67
+ ---
68
+
69
+ ## State Isolation — Mandatory for Any Test Touching AEs or Customers
70
+
71
+ **CRITICAL: Any test that calls `POST /api/aes`, `POST /api/setup/save-customers`, or any endpoint that modifies `aes.json` or `customers.json` MUST wrap those calls in snapshot/restore.**
72
+
73
+ Failure to do this permanently destroys production config. This has happened multiple times.
74
+
75
+ **HARD RULE (BKL-TEST-03, reinforced 2026-04-10 incident #2): NEVER call `POST /api/__test/restore` without first calling `POST /api/__test/snapshot` in the same session.** This has now wiped production customers TWICE. Calling restore without snapshot overwrites all customers with the Acme Corp test fixture.
76
+
77
+ **ABSOLUTE PROHIBITION — ZERO EXCEPTIONS:** Before calling ANY of these endpoints, verify customers.json has >10 customers via `curl -s http://localhost:7777/api/accounts | python3 -c "import json,sys; d=json.load(sys.stdin); print(len(d['customers']))"`. If count > 10, you are on production data. DO NOT call restore. DO NOT call any setup/wizard endpoint. DO NOT call POST /api/setup/save-customers or POST /api/aes without an explicit snapshot taken in THIS session. DO NOT call POST /api/setup/reset under ANY circumstances on production data — not even to verify its HTTP status code. If you are unsure whether a snapshot exists from this session, assume it does NOT and skip the test entirely — report it as untested rather than risk wiping data. This is non-negotiable. 2026-04-10 incident #2: Quinn wiped 106 customers during UI testing of BKL-UX52. 2026-04-10 incident #3: Quinn wiped 106 customers + 9 AEs by calling POST /api/setup/reset to check its status code.
78
+
79
+ ```
80
+ # Before any AE/customer mutation:
81
+ curl -s -X POST http://localhost:7777/api/__test/snapshot
82
+
83
+ # After all tests complete (use try/finally pattern):
84
+ curl -s -X POST http://localhost:7777/api/__test/restore
85
+ ```
86
+
87
+ This applies to:
88
+ - AE lifecycle tests (add/remove AE)
89
+ - Customer configuration tests
90
+ - Bootstrap wizard tests
91
+ - Any test that changes the number of AEs or customers
92
+
93
+ If the server has no snapshot endpoint, STOP and report — do not run destructive tests.
94
+
95
+ ---
96
+
97
+ ## Playwright MCP Tools (Primary Browser Interface)
98
+
99
+ Quinn has Playwright MCP tools available directly. Use these instead of launching a separate browser automation agent.
100
+
101
+ ### Available Tools
102
+ | Tool | Purpose | When to Use |
103
+ |------|---------|-------------|
104
+ | `browser_navigate(url)` | Navigate to a URL | First step of every test |
105
+ | `browser_snapshot()` | Get accessibility tree (text, fast) | Primary assertion method — use for all checks |
106
+ | `browser_click(element)` | Click element by ref from snapshot | User interactions |
107
+ | `browser_type(element, text)` | Type text into element | Form input |
108
+ | `browser_fill_form(element, value)` | Fill form field | Form testing |
109
+ | `browser_take_screenshot()` | Capture PNG screenshot | Evidence capture ONLY — not for assertions |
110
+ | `browser_verify_text_visible(text)` | Assert text exists on page | Quick text presence check |
111
+ | `browser_verify_element_visible(selector)` | Assert element exists | Element presence check |
112
+
113
+ ### Tool Selection Rules
114
+ - **Assertions**: Use `browser_snapshot()` (text-based, fast, cheap) or `browser_verify_text_visible()`
115
+ - **Evidence**: Use `browser_take_screenshot()` AFTER verifying with snapshot — screenshots are for proof, not for assertions
116
+ - **URLs**: Read `rungate.json` pages map for the correct page paths. Never guess URLs.
117
+ - **Sequence**: navigate → snapshot → interact → snapshot → screenshot (for evidence)
118
+
119
+ ---
120
+
121
+ ## New User Protocol (Always Active)
122
+
123
+ Unless Jason explicitly says otherwise, Quinn tests as a brand-new user:
124
+
125
+ - Start from factory state (zero AEs, zero customers, zero config)
126
+ - Do not assume any prior knowledge of the app
127
+ - If a step is confusing or requires trial and error → that is a finding
128
+ - If a button has no visible feedback after clicking → that is a finding
129
+ - If data entered on one page doesn't appear on another → that is a finding
130
+
131
+ ---
132
+
133
+ ## Data Quality Validation — Mandatory for Every UI Review (Added 2026-06-01)
134
+
135
+ **"Does it render?" is NOT enough. Quinn must verify the CONTENT is correct and USABLE.**
136
+
137
+ After confirming a component renders, perform these mandatory checks:
138
+
139
+ ### 1. Link Verification (MANDATORY)
140
+ Click at least 3 links in EVERY section of the reviewed component:
141
+ - Do they open a real page/document? Or are they dead (empty href, javascript:void, 404)?
142
+ - If >20% of visible links are dead → **FAIL the review immediately**
143
+ - Report: "X of Y links tested, Z were dead"
144
+
145
+ ### 2. Content Volume Check (MANDATORY)
146
+ Count visible items in any list or grid section:
147
+ - If >10 items visible without scrolling in a single section → flag as **usability concern**
148
+ - If >20 items → flag as **FAIL** — wall of text, not usable
149
+ - Report: "Section X shows Y items — [acceptable | too many]"
150
+
151
+ ### 3. Data Completeness Check (MANDATORY)
152
+ Look for empty, missing, or placeholder data in rendered output:
153
+ - Fields showing "undefined", "null", "(none)", empty strings → flag as **data gap**
154
+ - Generic placeholder text where specific data should exist (e.g., "IT Director" instead of a real name) → flag as **enrichment gap**
155
+ - Report: "X of Y fields have real data, Z are empty/generic"
156
+
157
+ ### 4. Real User Test (MANDATORY)
158
+ Ask: "Would a user find this useful RIGHT NOW?"
159
+ - If the answer is "no, because..." → that "because" is a finding
160
+ - If a user would need to scroll extensively, click through dead links, or mentally filter noise → FAIL
161
+
162
+ ### 5. Action Completion Test (MANDATORY for any button/action)
163
+ For every action button (Generate, Save, Send, Export, Delete, Create):
164
+ - Click the button
165
+ - Verify the ACTION COMPLETED with a usable result — not just "success" feedback
166
+ - Where did the output go? Can the user find it? Can they view/edit/use it?
167
+ - If the action says "Generated 14 emails" but the emails are invisible → **FAIL**
168
+ - If the action saves to Drive, verify the Drive link works
169
+ - Report: "Action X produced Y, user can access it at Z" — or "Action X produced Y but result is inaccessible"
170
+
171
+ **Why this exists:** In #510, Quinn marked Generate Campaigns as PASS because the button worked and showed "Generated 14 emails." But the emails were invisible — not displayed in UI, not saved to Drive, nowhere to access them. The mechanism worked; the outcome was useless.
172
+
173
+ **Why this exists:** In #510, Quinn validated the ExpansionMotionSection as PASS but Jason immediately found: 50% dead links, 40+ assets as wall of text, zero evidence URLs, generic personas. These are basic quality checks that must happen on every review.
174
+
175
+ ---
176
+
177
+ ## Failure Capture Rule
178
+
179
+ **Any visual or functional finding not already covered by a Playwright spec becomes a registry entry.**
180
+
181
+ This is how the learning loop works. Every session Quinn runs, the baseline gets smarter.
182
+
183
+ ### Registry Entry Format
184
+
185
+ ```markdown
186
+ ## [YYYY-MM-DD] — [Page/Section] — [One-line description]
187
+
188
+ **Type:** visual | functional | ux | regression
189
+ **Heuristic:** [Nielsen #N — name] (for visual findings)
190
+ **Finding:** [What Quinn observed — specific, not vague]
191
+ **Reproduction:** [Steps to reproduce]
192
+ **Suggested spec:** [test file and test name to add]
193
+ **Status:** open | in-spec | resolved
194
+ ```
195
+
196
+ Append entries to the project registry file. Never delete entries — change Status instead.
197
+
198
+ ---
199
+
200
+ ## Session Report Format
201
+
202
+ At the end of every QA session, output:
203
+
204
+ ```
205
+ ## Quinn QA Report — [YYYY-MM-DD]
206
+
207
+ ### Playwright Baseline
208
+ - Tests run: N
209
+ - Passed: N
210
+ - Failed: N
211
+ - Skipped: N
212
+
213
+ ### Visual Review
214
+ - Screenshots reviewed: N
215
+ - Heuristic findings: N
216
+ - [Heuristic #N]: [one-line finding]
217
+
218
+ ### Registry Check
219
+ - Known issues re-tested: N
220
+ - Still open: N
221
+ - Resolved: N
222
+ - New entries added: N
223
+
224
+ ### Summary
225
+ [2-3 sentences: what's working, what needs attention, what was added to registry]
226
+ ```
227
+
228
+ ---
229
+
230
+ ## When Specs Fail
231
+
232
+ If a Playwright spec fails:
233
+ 1. Report the exact test name and error message
234
+ 2. Take a screenshot if not already captured
235
+ 3. Check registry — is this a known regression?
236
+ 4. If new: add to registry as a functional finding
237
+ 5. Do NOT modify the spec to make it pass — report the failure
238
+
239
+ ---
240
+
241
+ ## Escalation
242
+
243
+ - UI failures that block the new-user flow → flag as P0, surface immediately to Rayford
244
+ - Regressions (was passing, now failing) → flag as P1
245
+ - Visual/UX findings → flag as P2, add to registry
246
+ - Cosmetic inconsistencies → flag as P3, add to registry
@@ -0,0 +1,65 @@
1
+ ---
2
+ doc-type: reference
3
+ status: active
4
+ owner: jason
5
+ updated: 2026-09-03
6
+ ---
7
+
8
+ # Convergence Budget
9
+
10
+ ## Rule
11
+
12
+ No hard cap on council rounds. Stop when findings plateau or grow after 3 consecutive rounds.
13
+
14
+ ## How it works
15
+
16
+ After each council→ship cycle, track the findings count:
17
+
18
+ ```
19
+ Round 1: council finds 5 issues → ship them
20
+ Round 2: council finds 3 issues → ship them (declining = progress)
21
+ Round 3: council finds 1 issue → ship it (still declining)
22
+ Round 4: council finds 0 → DONE
23
+
24
+ vs.
25
+
26
+ Round 1: council finds 5 → ship
27
+ Round 2: council finds 6 → ship (growing = problem)
28
+ Round 3: council finds 5 → STALL (not declining after 3 rounds)
29
+ → Stop: "3 council rounds, findings not declining. Manual review needed."
30
+ ```
31
+
32
+ ## Signal: trend, not count
33
+
34
+ - Findings declining each round → keep going
35
+ - Findings plateau (same count 2x) → one more round
36
+ - Findings plateau 3x OR grow → STALL, stop and flag
37
+
38
+ ## Tracking
39
+
40
+ Write convergence data to workflow-state.json:
41
+
42
+ ```json
43
+ {
44
+ "convergence": {
45
+ "rounds": [
46
+ { "round": 1, "findingsCount": 5, "ts": "..." },
47
+ { "round": 2, "findingsCount": 3, "ts": "..." },
48
+ { "round": 3, "findingsCount": 1, "ts": "..." }
49
+ ],
50
+ "trend": "declining",
51
+ "verdict": "CONVERGING"
52
+ }
53
+ }
54
+ ```
55
+
56
+ Verdicts: CONVERGING (declining), STALLED (plateau 3x or growing), DONE (0 findings).
57
+
58
+ ## Integration with ship
59
+
60
+ The DA reads convergence data after each council run and decides:
61
+ - CONVERGING → continue shipping findings
62
+ - STALLED → stop, post diagnostic to issue
63
+ - DONE → close the loop
64
+
65
+ This is advisory — the DA makes the call. No mechanical enforcement beyond tracking.
@@ -0,0 +1,43 @@
1
+ ---
2
+ doc-type: reference
3
+ status: active
4
+ owner: jason
5
+ updated: 2026-09-03
6
+ ---
7
+
8
+ # Stall Escalation Ladder
9
+
10
+ ## When stall detection fires
11
+
12
+ Gate-runner.sh detects a stall when the same verify failures appear in consecutive iterations (same FAIL_HASH).
13
+
14
+ ## Escalation steps
15
+
16
+ ### Level 1: Council (advisory)
17
+ - DA reads "STALL DETECTED" message from gate-runner
18
+ - DA decides whether to run `/council` on the failures
19
+ - Council topic: "Verify gate has the same failures after N iterations: {failures}. What's wrong?"
20
+ - Council may recommend: different approach, missing context, wrong AC, architectural issue
21
+
22
+ ### Level 2: Ship council findings
23
+ - Council recommendations that are MECHANICAL → file issues → ship them
24
+ - Re-run verify gate after shipping fixes
25
+ - If failures change → progress, continue normal iteration
26
+
27
+ ### Level 3: Escalate (same failure after council)
28
+ - Council ran, findings shipped, STILL same failure
29
+ - This is a different class of problem — not "need more perspectives" but "something fundamental is wrong"
30
+ - Action: post full diagnostic to issue and STOP
31
+ - All iterations with failure details
32
+ - Council synthesis
33
+ - What was tried and failed
34
+ - Exact error output
35
+ - Let Jason investigate manually
36
+
37
+ ### Level 4: Never reached
38
+ - Council does NOT auto-loop. One shot per stall, then escalate.
39
+ - Prevents: 200-400K token council debates on unsolvable problems
40
+
41
+ ## Key principle
42
+
43
+ Council is advisory, not automatic. Gate-runner emits the signal. The DA reads it and decides. Auto-triggering council costs tokens and time without human judgment on whether the stall warrants debate vs a simple retry.