rungate 0.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.claude/agents/aditi.md +47 -0
- package/.claude/agents/marcus.md +50 -0
- package/.claude/agents/quinn.md +58 -0
- package/.claude/agents/rook.md +47 -0
- package/.claude/agents/serena.md +46 -0
- package/.claude/rungate.json +11 -0
- package/.fallowrc.json +28 -0
- package/.github/copilot-instructions.md +20 -0
- package/.rungate/conformity-findings.json +648 -0
- package/AGENTS.md +106 -0
- package/CODE-MAP.md +50 -0
- package/HARNESS.md +88 -0
- package/bun.lock +26 -0
- package/bunfig.toml +2 -0
- package/config/brief-policies.json +28 -0
- package/config/ceremony-profiles.json +212 -0
- package/config/consumer-cascade-map.json +24 -0
- package/config/doc-cascade-map.json +24 -0
- package/config/eval.json +84 -0
- package/config/profiles/account-plan.yaml +36 -0
- package/config/profiles/campaign.yaml +94 -0
- package/config/prove-evidence.schema.json +65 -0
- package/config/spec-policies.json +36 -0
- package/config/workflow-schema.json +281 -0
- package/gates/SCHEMA-GUIDE.md +121 -0
- package/gates/ac-quality.test.ts +106 -0
- package/gates/adversarial.test.ts +274 -0
- package/gates/brief-assembler.test.ts +429 -0
- package/gates/brief-assembler.ts +334 -0
- package/gates/bunfig.toml +2 -0
- package/gates/chain.test.ts +91 -0
- package/gates/e2e-smoke.test.ts +234 -0
- package/gates/error-classifier.test.ts +249 -0
- package/gates/error-classifier.ts +89 -0
- package/gates/orchestrator.test.ts +950 -0
- package/gates/orchestrator.ts +569 -0
- package/gates/preload.ts +77 -0
- package/gates/prompts/ac-adversary.md +20 -0
- package/gates/prompts/evidence-validator.md +22 -0
- package/gates/prompts/prove-reproducer.md +53 -0
- package/gates/prove.test.ts +164 -0
- package/gates/run-gate.ts +936 -0
- package/gates/schema-parity.test.ts +197 -0
- package/gates/schema.ts +362 -0
- package/gates/self-heal.test.ts +511 -0
- package/gates/self-heal.ts +121 -0
- package/gates/ship-orchestrator.test.ts +379 -0
- package/gates/ship-orchestrator.ts +363 -0
- package/gates/test-fixtures/broken-no-environments.json +60 -0
- package/gates/test-fixtures/valid-standard.json +54 -0
- package/gates/test-utils.ts +43 -0
- package/gates/witness.ts +220 -0
- package/gates/workflow.test.ts +1053 -0
- package/hooks/AgentBriefGuard.hook.ts +586 -0
- package/hooks/AgentVerdictCapture.hook.ts +134 -0
- package/hooks/AutoVerifyGate.hook.ts +76 -0
- package/hooks/GateEnforcement.hook.ts +273 -0
- package/hooks/IssueCloseGuard.hook.ts +116 -0
- package/hooks/MergeGuard.hook.ts +55 -0
- package/hooks/StaleTTLCleanup.hook.ts +133 -0
- package/hooks/VerifyPhaseLock.hook.ts +81 -0
- package/hooks/WorkflowStateGuard.hook.ts +37 -0
- package/hooks/lib/agentDetection.ts +70 -0
- package/hooks/lib/findWorkflow.ts +101 -0
- package/hooks/lib/parseStdin.ts +37 -0
- package/hooks/lib/paths.ts +111 -0
- package/hooks/lib/utils.ts +81 -0
- package/lib/conformity.ts +959 -0
- package/lib/paths.ts +23 -0
- package/lib/rungate-schema.ts +60 -0
- package/lib/signal-phrases.ts +7 -0
- package/package.json +20 -0
- package/prompts/ac-adversary.md +20 -0
- package/prompts/ac-format.md +34 -0
- package/prompts/aditi.md +29 -0
- package/prompts/blast-radius.md +29 -0
- package/prompts/coding-principles.md +34 -0
- package/prompts/container-rebuild.md +29 -0
- package/prompts/container-verify.md +28 -0
- package/prompts/discovery.md +27 -0
- package/prompts/environment.md +28 -0
- package/prompts/escalation-decision-tree.md +38 -0
- package/prompts/evidence-hierarchy.md +31 -0
- package/prompts/evidence-validator.md +22 -0
- package/prompts/marcus.md +29 -0
- package/prompts/prevention.md +26 -0
- package/prompts/prove-reproducer.md +43 -0
- package/prompts/quinn-decision-tree.md +31 -0
- package/prompts/quinn-ui-brief.md +106 -0
- package/prompts/quinn.md +29 -0
- package/prompts/rca.md +28 -0
- package/prompts/read-before-write.md +30 -0
- package/prompts/regression.md +26 -0
- package/prompts/rook.md +29 -0
- package/prompts/serena.md +28 -0
- package/prompts/testing-strategy.md +34 -0
- package/reference/migration-manifest.json +554 -0
- package/reference/prompts/agent-aditi.md +38 -0
- package/reference/prompts/agent-marcus.md +38 -0
- package/reference/prompts/agent-quinn.md +38 -0
- package/reference/prompts/agent-rook.md +37 -0
- package/reference/prompts/agent-serena.md +37 -0
- package/reference/prompts/council-skill.md +170 -0
- package/reference/prompts/goal-skill.md +235 -0
- package/reference/prompts/prove-skill.md +283 -0
- package/reference/prompts/quinn-standard.md +246 -0
- package/reference/prompts/ship-convergence.md +65 -0
- package/reference/prompts/ship-escalation.md +43 -0
- package/reference/prompts/ship-iteration.md +382 -0
- package/reference/prompts/ship-pending-patches.md +246 -0
- package/reference/prompts/ship-reference.md +250 -0
- package/reference/prompts/ship-skill.md +347 -0
- package/reference/prompts/ship-templates.md +215 -0
- package/reference/scripts/batch-status.sh +68 -0
- package/reference/scripts/chain-compat-check.sh +67 -0
- package/reference/scripts/context-audit.sh +314 -0
- package/reference/scripts/dead-export-check.sh +61 -0
- package/reference/scripts/generate-qc-tables.sh +92 -0
- package/reference/scripts/harness-init.sh +111 -0
- package/reference/scripts/pai-054-baseline-check.sh +40 -0
- package/reference/scripts/prove-smoke-test.sh +112 -0
- package/reference/scripts/setup-keychain-salt.sh +12 -0
- package/reference/scripts/skill-optimizer-friday.sh +88 -0
- package/reference/scripts/skill-qc-test.sh +351 -0
- package/reference/scripts/skill-synthesize-tonight.sh +66 -0
- package/reference/specs/HARNESS-EXTRACTION-SPEC.md +263 -0
- package/reference/specs/harness-v3-bash-deletion-plan.md +65 -0
- package/reference/specs/harness-v3-migration-plan.md +188 -0
- package/scripts/decision-reconcile.sh +74 -0
- package/scripts/decision-reconcile.ts +152 -0
- package/scripts/extract-constraints.ts +260 -0
- package/scripts/generate-code-map.ts +339 -0
- package/scripts/git-hooks/pre-commit +76 -0
- package/scripts/git-hooks/pre-push +271 -0
- package/scripts/harness-skill-check.sh +52 -0
- package/scripts/lib/gap-emit.sh +65 -0
- package/scripts/lib/jq-update.sh +30 -0
- package/scripts/lib/resolve-slug.sh +30 -0
- package/scripts/lib/verify-spec-compliance.sh +102 -0
- package/scripts/prove-backfill.sh +87 -0
- package/scripts/scaffold-project.ts +1488 -0
- package/scripts/scaffold-rungate-config.ts +91 -0
- package/scripts/self-containment-check.sh +109 -0
- package/scripts/skill-qc-validator.sh +402 -0
- package/scripts/slug-resolver.sh +14 -0
- package/scripts/sync-spec-tests.ts +236 -0
- package/specs/BOOTSTRAP-DATA-FLOW-SPEC.md +2493 -0
- package/specs/BOOTSTRAP-TEST-PLAN.md +571 -0
- package/specs/HARNESS-GATES.md +256 -0
- package/specs/HARNESS-SKILL-CHAIN.md +286 -0
- package/specs/HARNESS-SKILL-CONTRACT.md +93 -0
- package/specs/HARNESS-STANDARD.md +488 -0
- package/specs/SPEC-TEMPLATE.md +53 -0
- package/specs/bootstrap-data-flow.png +0 -0
- package/specs/harness-automation-matrix.md +314 -0
- package/specs/harness-automation-matrix.rules.json +64 -0
- package/test/anti.test.ts +80 -0
- package/test/contract-negative.test.ts +86 -0
- package/test/contract.test.ts +158 -0
- package/test/external-deps.test.ts +48 -0
- package/test/fixtures/golden-project/.env.example +3 -0
- package/test/fixtures/golden-project/Makefile +7 -0
- package/test/fixtures/golden-project/docs/adr/ADR-001-framework.md +23 -0
- package/test/fixtures/golden-project/docs/setup-guide.md +10 -0
- package/test/fixtures/golden-project/package.json +10 -0
- package/test/fixtures/golden-project/specs/api-spec.md +20 -0
- package/test/fixtures/golden-project/src/auth.ts +8 -0
- package/test/fixtures/golden-project/src/db.ts +11 -0
- package/test/fixtures/golden-project/src/index.ts +19 -0
- package/test/fixtures/golden-project/src/utils/format.ts +3 -0
- package/test/fixtures/golden-project/test/auth.test.ts +15 -0
- package/test/fixtures/golden-project/tsconfig.json +12 -0
- package/test/meta-sc-coverage.test.ts +159 -0
- package/test/phase-0.test.ts +846 -0
- package/test/phase-1.test.ts +617 -0
- package/test/phase-2.test.ts +620 -0
- package/test/phase-3.test.ts +99 -0
- package/test/scaffold-conformity.test.ts +8 -0
- package/test/schema-canary.test.ts +51 -0
- package/test/spec-compliance-auto.test.ts +64 -0
- package/test/spec-compliance.test.ts +295 -0
- package/test/spec-discovery.test.ts +68 -0
- package/test/spec-drift.test.ts +5 -0
- package/test/structure.test.ts +165 -0
- package/test/unit/agent-brief-guard-ship.test.ts +293 -0
- package/test/unit/extract-constraints.test.ts +145 -0
- package/test/unit/issue-close-guard.test.ts +204 -0
- package/test/unit/scaffold-project.test.ts +417 -0
- package/test/unit/skill-cooccurrence.test.ts +177 -0
- package/test/unit/skill-sequence-logger.test.ts +244 -0
- package/test/unit/workflow-state-guard.test.ts +95 -0
- package/workflows/batch-ship.js +295 -0
- package/workflows/council.js +481 -0
- package/workflows/prove.js +560 -0
- package/workflows/ship.js +789 -0
- package/workflows/verify.js +236 -0
package/AGENTS.md
ADDED
|
@@ -0,0 +1,106 @@
|
|
|
1
|
+
---
|
|
2
|
+
doc-type: reference
|
|
3
|
+
status: active
|
|
4
|
+
owner: jason
|
|
5
|
+
updated: 2026-09-17
|
|
6
|
+
---
|
|
7
|
+
|
|
8
|
+
# PAI Harness
|
|
9
|
+
|
|
10
|
+
## Project Identity
|
|
11
|
+
|
|
12
|
+
Implementation quality framework for PAI (Personal AI Infrastructure). Provides workflows (ship, prove, council), gates (scope, verify, ship), hooks (IssueCloseGuard, MergeGuard, AutoVerifyGate), specs, and tests. Built with Bun/TypeScript.
|
|
13
|
+
|
|
14
|
+
- **Issues:** github.com/hornjason/pai-config (not this repo)
|
|
15
|
+
- **Code:** github.com/hornjason/rungate
|
|
16
|
+
|
|
17
|
+
## Key Files
|
|
18
|
+
|
|
19
|
+
| File | What | When to Read |
|
|
20
|
+
|------|------|--------------|
|
|
21
|
+
| [.claude/rungate.json](.claude/rungate.json) | Project config | When shipping through harness |
|
|
22
|
+
| [gates/orchestrator.ts](gates/orchestrator.ts) | Gate runner + workflow state | When debugging gates |
|
|
23
|
+
| [gates/schema.ts](gates/schema.ts) | Zod schemas for workflow-state.json | When adding fields |
|
|
24
|
+
| [gates/brief-assembler.ts](gates/brief-assembler.ts) | Generates Marcus briefs from state | When changing brief format |
|
|
25
|
+
| [workflows/ship.js](workflows/ship.js) | Ship lifecycle workflow | When modifying ship process |
|
|
26
|
+
| [workflows/prove.js](workflows/prove.js) | Prove lifecycle workflow | When modifying prove process |
|
|
27
|
+
| [workflows/council.js](workflows/council.js) | Council debate workflow | When modifying council |
|
|
28
|
+
| [hooks/IssueCloseGuard.hook.ts](hooks/IssueCloseGuard.hook.ts) | Blocks premature issue close | When changing close rules |
|
|
29
|
+
| [hooks/AutoVerifyGate.hook.ts](hooks/AutoVerifyGate.hook.ts) | Auto-triggers verify after Marcus | When changing verify flow |
|
|
30
|
+
| [hooks/MergeGuard.hook.ts](hooks/MergeGuard.hook.ts) | Blocks merge without verify gate | When changing merge rules |
|
|
31
|
+
| [config/ceremony-profiles.json](config/ceremony-profiles.json) | LIGHT/STANDARD/THOROUGH tiers | When changing ceremony levels |
|
|
32
|
+
| [scripts/scaffold-project.ts](scripts/scaffold-project.ts) | Bootstrap any project to conformity | When onboarding a new project |
|
|
33
|
+
| [lib/conformity.ts](lib/conformity.ts) | Exportable conformity + fallow integration | When projects import tests |
|
|
34
|
+
| [.fallowrc.json](.fallowrc.json) | Fallow static analysis config | When adding entry points or ignore patterns |
|
|
35
|
+
| [gates/self-heal.ts](gates/self-heal.ts) | Prove self-healing loop | When debugging prove iterations |
|
|
36
|
+
|
|
37
|
+
## Specs
|
|
38
|
+
|
|
39
|
+
All specs live in `specs/` with YAML frontmatter declaring `testable: true/false`.
|
|
40
|
+
|
|
41
|
+
| Spec | Testable | Governs |
|
|
42
|
+
|------|----------|---------|
|
|
43
|
+
| HARNESS-STANDARD.md | true | Agentic harness lifecycle and AFK rules |
|
|
44
|
+
| HARNESS-SKILL-CHAIN.md | true | Phase ordering and skill chain contract |
|
|
45
|
+
| HARNESS-SKILL-CONTRACT.md | true | Skill I/O contract definitions |
|
|
46
|
+
| HARNESS-GATES.md | true | Gate definitions and enforcement config |
|
|
47
|
+
| harness-automation-matrix.md | true | Gate automation coverage matrix |
|
|
48
|
+
| SPEC-TEMPLATE.md | false | Template for creating new specs |
|
|
49
|
+
|
|
50
|
+
New specs: copy `specs/SPEC-TEMPLATE.md`, follow the SC patterns documented in it. Tests auto-generate from `- [ ] SC-N:` lines.
|
|
51
|
+
|
|
52
|
+
## Tests
|
|
53
|
+
|
|
54
|
+
Run all tests:
|
|
55
|
+
|
|
56
|
+
```bash
|
|
57
|
+
bun test
|
|
58
|
+
```
|
|
59
|
+
|
|
60
|
+
| Category | File | What |
|
|
61
|
+
|----------|------|------|
|
|
62
|
+
| Structure | structure.test.ts | ST-1..ST-7 migration guards |
|
|
63
|
+
| Spec discovery | spec-discovery.test.ts | Frontmatter validation |
|
|
64
|
+
| Spec compliance | spec-compliance.test.ts | Manual compliance checks |
|
|
65
|
+
| Spec compliance (auto) | spec-compliance-auto.test.ts | Auto-generated compliance |
|
|
66
|
+
| Schema canary | schema-canary.test.ts | Zod schema drift detection |
|
|
67
|
+
| Scaffold conformity | scaffold-conformity.test.ts | REPO-SCAFFOLD-SPEC + doc hygiene + fallow |
|
|
68
|
+
| External deps | external-deps.test.ts | Cross-repo CLAUDE.md drift |
|
|
69
|
+
| Contract | contract.test.ts | Skill contract validation |
|
|
70
|
+
|
|
71
|
+
## Workflow
|
|
72
|
+
|
|
73
|
+
Issues live on **hornjason/pai-config**; code lives here. The harness enforces a deterministic lifecycle:
|
|
74
|
+
|
|
75
|
+
```
|
|
76
|
+
GOAL → DISCOVERY → SCOPE → BUILD → VERIFY → SHIP → PROVE
|
|
77
|
+
↑ |
|
|
78
|
+
└── self-heal ───────┘
|
|
79
|
+
```
|
|
80
|
+
|
|
81
|
+
- **Gates** enforce quality mechanically at each transition
|
|
82
|
+
- **Self-healing**: gates fail → classify error → fix → re-run (max 3 attempts per gate)
|
|
83
|
+
- **Prove iteration**: UNPROVEN → spawn Marcus to fix → re-prove (circuit breaker at 3)
|
|
84
|
+
- **Witness-verdict cross-validation**: AC verdicts must have matching witness chain entries
|
|
85
|
+
- **Hooks** prevent premature closure (IssueCloseGuard) and unverified merges (MergeGuard)
|
|
86
|
+
- **Ceremony profiles** (LIGHT/STANDARD/THOROUGH) control how much ceremony each gate demands
|
|
87
|
+
- **workflow-state.json** is the spine — every phase reads/writes it
|
|
88
|
+
|
|
89
|
+
Workflow invocation always uses `scriptPath`, never `name`:
|
|
90
|
+
|
|
91
|
+
```js
|
|
92
|
+
Workflow({ scriptPath: "~/Projects/rungate/workflows/ship.js" })
|
|
93
|
+
```
|
|
94
|
+
|
|
95
|
+
## Quick Reference
|
|
96
|
+
|
|
97
|
+
1. Always use `scriptPath` for workflow invocation — `name` resolves to cached snapshots
|
|
98
|
+
2. `rungate.json` is the thin interface each project provides to the harness
|
|
99
|
+
3. Gates are the enforcement layer — behavioral rules alone don't work
|
|
100
|
+
4. Specs self-describe via YAML frontmatter (`testable`, `status`, `doc-type`)
|
|
101
|
+
|
|
102
|
+
## Repo Boundary
|
|
103
|
+
|
|
104
|
+
- **Owns:** execution machinery (workflows, gates, hooks, specs, config, tests)
|
|
105
|
+
- **Does NOT own:** behavioral rules (CLAUDE.md), algorithm mode, project configs, PAI routing
|
|
106
|
+
- **Reference:** See [HARNESS.md](HARNESS.md) for workflow schema details and external dependency list
|
package/CODE-MAP.md
ADDED
|
@@ -0,0 +1,50 @@
|
|
|
1
|
+
---
|
|
2
|
+
doc-type: code-map
|
|
3
|
+
status: generated
|
|
4
|
+
updated: 2026-09-17
|
|
5
|
+
generator: rungate/scripts/generate-code-map.ts
|
|
6
|
+
---
|
|
7
|
+
|
|
8
|
+
# Code Map — rungate
|
|
9
|
+
|
|
10
|
+
Auto-generated architecture snapshot. Re-run `bun generate-code-map.ts /Users/jhorn/Projects/rungate` to refresh.
|
|
11
|
+
Stale after 14 days or 50+ commits since last scan.
|
|
12
|
+
|
|
13
|
+
## Summary
|
|
14
|
+
|
|
15
|
+
| Metric | Count |
|
|
16
|
+
|--------|-------|
|
|
17
|
+
| Source directories | 10 |
|
|
18
|
+
| Dependencies | 1 |
|
|
19
|
+
| Dev dependencies | 1 |
|
|
20
|
+
| API routes | 0 |
|
|
21
|
+
| React components | 0 |
|
|
22
|
+
| Entry points (fallow) | 45 |
|
|
23
|
+
| Unused files | N/A |
|
|
24
|
+
| Unused exports | N/A |
|
|
25
|
+
| Circular dependencies | N/A |
|
|
26
|
+
| Module import mappings | 0 |
|
|
27
|
+
| Page-component mappings | 0 |
|
|
28
|
+
|
|
29
|
+
## Directory Structure
|
|
30
|
+
|
|
31
|
+
| Directory | Files | Types |
|
|
32
|
+
|-----------|-------|-------|
|
|
33
|
+
| gates/ | 28 | ts, gate-salt, toml, md, test-fixtures |
|
|
34
|
+
| scripts/ | 19 | ts, sh, git-hooks, lib, git-hooks/pre-push |
|
|
35
|
+
| test/ | 15 | unit, ts |
|
|
36
|
+
| config/ | 11 | json, profiles, yaml |
|
|
37
|
+
| .claude/ | 8 | agents, json, worktrees, md |
|
|
38
|
+
| specs/ | 7 | md, json |
|
|
39
|
+
| workflows/ | 5 | js |
|
|
40
|
+
| hooks/ | 5 | ts, lib |
|
|
41
|
+
| prompts/ | 4 | md |
|
|
42
|
+
| lib/ | 3 | ts |
|
|
43
|
+
|
|
44
|
+
## Code Health (fallow)
|
|
45
|
+
|
|
46
|
+
## Package Scripts
|
|
47
|
+
|
|
48
|
+
- `test`
|
|
49
|
+
- `test:structure`
|
|
50
|
+
- `test:external-deps`
|
package/HARNESS.md
ADDED
|
@@ -0,0 +1,88 @@
|
|
|
1
|
+
---
|
|
2
|
+
doc-type: reference
|
|
3
|
+
status: active
|
|
4
|
+
owner: jason
|
|
5
|
+
updated: 2026-09-18
|
|
6
|
+
---
|
|
7
|
+
|
|
8
|
+
# PAI Harness
|
|
9
|
+
|
|
10
|
+
Agent entry point for the PAI ship/prove/council execution engine. This repo owns all execution machinery — phases, gates, specs, agent prompts, and guard tests.
|
|
11
|
+
|
|
12
|
+
## Workflow Entry Points
|
|
13
|
+
|
|
14
|
+
### `workflows/ship.js`
|
|
15
|
+
- **Input:** GitHub issue number, rungate.json config
|
|
16
|
+
- **Output:** SHIP_PASSED or SHIP_FAILED with gate evidence chain
|
|
17
|
+
- **Phases:** SCOPE → BUILD → VERIFY → CLOSE
|
|
18
|
+
- **Invocation:** `Workflow({scriptPath: "<HARNESS_ROOT>/workflows/ship.js"})`
|
|
19
|
+
|
|
20
|
+
### `workflows/prove.js`
|
|
21
|
+
- **Input:** GitHub issue number, ship evidence chain
|
|
22
|
+
- **Output:** PROVE_PROVEN or PROVE_FAILED with reproduction evidence
|
|
23
|
+
- **Phases:** REPRODUCE → VERIFY → ATTEST
|
|
24
|
+
- **Invocation:** `Workflow({scriptPath: "<HARNESS_ROOT>/workflows/prove.js"})`
|
|
25
|
+
|
|
26
|
+
### `workflows/council.js`
|
|
27
|
+
- **Input:** Decision prompt, council mode (collaborative/adversarial/research)
|
|
28
|
+
- **Output:** Council transcript with convergence points and verdicts
|
|
29
|
+
- **Invocation:** `Workflow({scriptPath: "<HARNESS_ROOT>/workflows/council.js"})`
|
|
30
|
+
|
|
31
|
+
## rungate.json Schema
|
|
32
|
+
|
|
33
|
+
Each project provides a `.claude/rungate.json` validated by Zod at load time.
|
|
34
|
+
|
|
35
|
+
**Required fields:**
|
|
36
|
+
- `project` — project slug (e.g., "daily-brief-dashboard")
|
|
37
|
+
- `repo` — code repo (e.g., "hornjason/asaCommandCenter")
|
|
38
|
+
- `issueRepo` — issue tracker repo
|
|
39
|
+
- `dev.start` — dev server start command
|
|
40
|
+
- `dev.apiBase` — dev API base URL
|
|
41
|
+
- `dev.uiBase` — dev UI base URL
|
|
42
|
+
|
|
43
|
+
**Optional fields:**
|
|
44
|
+
- `dev.preStart`, `dev.testCmd`, `dev.typeCheck`
|
|
45
|
+
- `test.start`, `test.apiBase`
|
|
46
|
+
- `prod.rebuild`, `prod.apiBase`, `prod.uiBase`, `prod.smokeTest`
|
|
47
|
+
- `pages`, `codeCommittedPaths`, `consumers`, `contextDocs`
|
|
48
|
+
|
|
49
|
+
**Dropped:** `harnessRoot` (use `HARNESS_ROOT` env var)
|
|
50
|
+
|
|
51
|
+
## Running Tests
|
|
52
|
+
|
|
53
|
+
```bash
|
|
54
|
+
bun test # all tests
|
|
55
|
+
bun test test/structure.test.ts # migration guard tests
|
|
56
|
+
bun test test/external-deps.test.ts # cross-repo drift detector
|
|
57
|
+
```
|
|
58
|
+
|
|
59
|
+
## Spec Index
|
|
60
|
+
|
|
61
|
+
Specs in `specs/` govern harness behavior. Each has `testable: true/false` frontmatter.
|
|
62
|
+
|
|
63
|
+
| Spec | Testable | Purpose |
|
|
64
|
+
|------|----------|---------|
|
|
65
|
+
| HARNESS-SKILL-CHAIN.md | true | Phase ordering and skill chain contract |
|
|
66
|
+
| HARNESS-EXTRACTION-SPEC.md | true | Migration success criteria and guard tests |
|
|
67
|
+
| harness-automation-matrix.md | true | Gate automation coverage matrix |
|
|
68
|
+
| HARNESS-SKILL-CONTRACT.md | true | Skill I/O contract definitions |
|
|
69
|
+
| harness-v3-migration-plan.md | false | Historical migration plan |
|
|
70
|
+
| harness-v3-bash-deletion-plan.md | false | Historical bash deletion plan |
|
|
71
|
+
|
|
72
|
+
## External Dependencies
|
|
73
|
+
|
|
74
|
+
These CLAUDE.md behavioral rules govern harness execution. The harness does not own them — they live in `~/.claude/CLAUDE.md`. The `test/external-deps.test.ts` drift detector verifies they still exist.
|
|
75
|
+
|
|
76
|
+
- `OUTCOME ACs require live verification` — SKIP not valid for OUTCOME ACs at ship gate
|
|
77
|
+
- `mechanical-enforcement-over-rules` — hard rules don't prevent step-skipping; use code gates
|
|
78
|
+
- `AFK means no stops between issues` — execute sequentially without stopping between issues
|
|
79
|
+
- `Auto-read template on format failure` — read TEMPLATES.md before fixing format errors
|
|
80
|
+
- `Always use scriptPath for workflow invocation` — never use name param; scriptPath reads actual file
|
|
81
|
+
- `Batch-diagnose gate failures before fixing` — enumerate ALL failures, batch-fix, then re-run once
|
|
82
|
+
- `Check git log for prior commits in DISCOVERY` — ship only the gaps, not the whole issue
|
|
83
|
+
- `Verify cwd after worktree merge` — gate runner asserts cwd matches project root
|
|
84
|
+
|
|
85
|
+
## Repo Boundary
|
|
86
|
+
|
|
87
|
+
**Owns:** execution machinery (phases, gates, specs, agent prompts, config)
|
|
88
|
+
**Does NOT own:** behavioral rules (CLAUDE.md), orchestration mode (Algorithm), project config (rungate.json), PAI-wide routing (DOCS.md)
|
package/bun.lock
ADDED
|
@@ -0,0 +1,26 @@
|
|
|
1
|
+
{
|
|
2
|
+
"lockfileVersion": 1,
|
|
3
|
+
"configVersion": 1,
|
|
4
|
+
"workspaces": {
|
|
5
|
+
"": {
|
|
6
|
+
"name": "pai-harness",
|
|
7
|
+
"dependencies": {
|
|
8
|
+
"zod": "^3.23.0",
|
|
9
|
+
},
|
|
10
|
+
"devDependencies": {
|
|
11
|
+
"@types/bun": "latest",
|
|
12
|
+
},
|
|
13
|
+
},
|
|
14
|
+
},
|
|
15
|
+
"packages": {
|
|
16
|
+
"@types/bun": ["@types/bun@1.4.2", "", { "dependencies": { "bun-types": "1.4.2" } }, "sha512-GimotNn7+ZV0uVArItBbriZsR1oNf0+WTzPkdcFrzShI7k2norL0uzEaJT8T33dWr7O/c9ZDuAFQrctKCi72oQ=="],
|
|
17
|
+
|
|
18
|
+
"@types/node": ["@types/node@22.20.3", "", { "dependencies": { "undici-types": "~6.21.0" } }, "sha512-DZmzkmwHzXrLPAXPyKNDzlIwMMUZCVacoD25ywdy5YTKGbOx/2ld+Q38Im2zJ0vBuZP5Prd3VZutKZyXwkOS8A=="],
|
|
19
|
+
|
|
20
|
+
"bun-types": ["bun-types@1.4.2", "", { "dependencies": { "@types/node": "*" } }, "sha512-bxV1FgK7yBIzjRe5zBozIM4Bem11ZJcCXSrjWRG3YWLt8yFDePu4cLjpebO8OvPeIE9trbyPF4fuj3Cia4Fj3w=="],
|
|
21
|
+
|
|
22
|
+
"undici-types": ["undici-types@6.21.0", "", {}, "sha512-iwDZqg0QAGrg9Rav5H4n0M64c3mkR59cJ6wQp+7C4nI0gsmExaedaYLNO44eT4AtBBwjbTiGPMlt2Md0T9H9JQ=="],
|
|
23
|
+
|
|
24
|
+
"zod": ["zod@3.25.76", "", {}, "sha512-gzUt/qt81nXsFGKIFcC3YnfEAx5NkunCfnDlvuBSSFS02bcXu4Lmea0AFIUwbLWxWPx3d9p8S5QoaujKcNQxcQ=="],
|
|
25
|
+
}
|
|
26
|
+
}
|
package/bunfig.toml
ADDED
|
@@ -0,0 +1,28 @@
|
|
|
1
|
+
{
|
|
2
|
+
"description": "Content validation policies per agent type. AgentBriefGuard reads this at runtime.",
|
|
3
|
+
"policies": {
|
|
4
|
+
"marcus": {
|
|
5
|
+
"requiredPatterns": [
|
|
6
|
+
{"name": "Goal", "pattern": "(?i)(goal|objective|what to implement)", "description": "Issue goal statement"},
|
|
7
|
+
{"name": "ACs", "pattern": "(?i)(acceptance criter|\\bAC-|\\bSC-|success criter)", "description": "Acceptance or success criteria"},
|
|
8
|
+
{"name": "Files", "pattern": "(?i)(files to|files changed|file.*read|file.*modify)", "description": "Specific files to read/modify"},
|
|
9
|
+
{"name": "Scope", "pattern": "(?i)(scope.*not|do not touch|out of scope|scope bound)", "description": "Scope boundaries"},
|
|
10
|
+
{"name": "Verify", "pattern": "(?i)(verify|verification|verify command|test command)", "description": "Verification commands"}
|
|
11
|
+
],
|
|
12
|
+
"enforcement": "WARN"
|
|
13
|
+
},
|
|
14
|
+
"quinn": {
|
|
15
|
+
"requiredPatterns": [
|
|
16
|
+
{"name": "Environment", "pattern": "(?i)(port|localhost|url|environment|7776|5173)", "description": "Test environment details"},
|
|
17
|
+
{"name": "What to test", "pattern": "(?i)(test|verify|check|validate)", "description": "Test scope"}
|
|
18
|
+
],
|
|
19
|
+
"enforcement": "WARN"
|
|
20
|
+
},
|
|
21
|
+
"rook": {
|
|
22
|
+
"requiredPatterns": [
|
|
23
|
+
{"name": "Files", "pattern": "(?i)(files|changed|scan|review)", "description": "Files to scan"}
|
|
24
|
+
],
|
|
25
|
+
"enforcement": "WARN"
|
|
26
|
+
}
|
|
27
|
+
}
|
|
28
|
+
}
|
|
@@ -0,0 +1,212 @@
|
|
|
1
|
+
{
|
|
2
|
+
"$schema": "./ceremony-profiles-schema.json",
|
|
3
|
+
"description": "Ceremony tiers — evaluator depth determines which checks run. Gate-runner reads this file.",
|
|
4
|
+
"tierSelection": {
|
|
5
|
+
"description": "Mechanical tier selection — harness selects, not DA",
|
|
6
|
+
"rules": [
|
|
7
|
+
{
|
|
8
|
+
"tier": "LIGHT",
|
|
9
|
+
"when": "size in [XS] AND unknowns == 0 AND files <= 3 AND no cascade-glob match"
|
|
10
|
+
},
|
|
11
|
+
{
|
|
12
|
+
"tier": "STANDARD",
|
|
13
|
+
"when": "size in [S, M] OR (size == XS AND any rule above fails)"
|
|
14
|
+
},
|
|
15
|
+
{
|
|
16
|
+
"tier": "THOROUGH",
|
|
17
|
+
"when": "size in [L] OR unfamiliar domain OR cross-repo"
|
|
18
|
+
}
|
|
19
|
+
]
|
|
20
|
+
},
|
|
21
|
+
"tiers": {
|
|
22
|
+
"LIGHT": {
|
|
23
|
+
"description": "Self-verify. DA checks own work. Computational sensors only.",
|
|
24
|
+
"evaluator": "self",
|
|
25
|
+
"gates": [
|
|
26
|
+
"ship"
|
|
27
|
+
],
|
|
28
|
+
"skipScopeGate": true,
|
|
29
|
+
"agents": {
|
|
30
|
+
"marcus": "optional",
|
|
31
|
+
"quinn": false,
|
|
32
|
+
"rook": false
|
|
33
|
+
},
|
|
34
|
+
"githubComments": 2,
|
|
35
|
+
"stopMarkers": 0,
|
|
36
|
+
"maxIterations": 3,
|
|
37
|
+
"checks": {
|
|
38
|
+
"scope": [
|
|
39
|
+
"acs-exist",
|
|
40
|
+
"acs-have-type",
|
|
41
|
+
"acs-have-threshold",
|
|
42
|
+
"source-spec-cited"
|
|
43
|
+
],
|
|
44
|
+
"verify": [
|
|
45
|
+
"all-acs-have-evidence",
|
|
46
|
+
"all-acs-pass",
|
|
47
|
+
"tests-pass",
|
|
48
|
+
"tsc-pass",
|
|
49
|
+
"code-committed"
|
|
50
|
+
],
|
|
51
|
+
"ship": [
|
|
52
|
+
"all-gates-pass",
|
|
53
|
+
"branch-merged",
|
|
54
|
+
"code-pushed"
|
|
55
|
+
]
|
|
56
|
+
}
|
|
57
|
+
},
|
|
58
|
+
"STANDARD": {
|
|
59
|
+
"description": "Separated verifier. Quinn validates independently.",
|
|
60
|
+
"evaluator": "separated",
|
|
61
|
+
"gates": [
|
|
62
|
+
"scope",
|
|
63
|
+
"verify",
|
|
64
|
+
"ship"
|
|
65
|
+
],
|
|
66
|
+
"skipScopeGate": false,
|
|
67
|
+
"agents": {
|
|
68
|
+
"marcus": true,
|
|
69
|
+
"quinn": true,
|
|
70
|
+
"rook": false
|
|
71
|
+
},
|
|
72
|
+
"githubComments": 3,
|
|
73
|
+
"stopMarkers": 1,
|
|
74
|
+
"maxIterations": 5,
|
|
75
|
+
"checks": {
|
|
76
|
+
"scope": [
|
|
77
|
+
"acs-exist",
|
|
78
|
+
"acs-have-type",
|
|
79
|
+
"acs-have-threshold",
|
|
80
|
+
"acs-have-evidence-method",
|
|
81
|
+
"source-spec-cited",
|
|
82
|
+
"spec-elements-in-acs",
|
|
83
|
+
"sizing-declared",
|
|
84
|
+
"issue-goal-captured",
|
|
85
|
+
"skill-qc-goal",
|
|
86
|
+
"acs-no-skillmd-enforcement",
|
|
87
|
+
"acs-no-behavioral-language",
|
|
88
|
+
"ac-issue-alignment"
|
|
89
|
+
],
|
|
90
|
+
"verify": [
|
|
91
|
+
"all-acs-have-evidence",
|
|
92
|
+
"all-acs-pass",
|
|
93
|
+
"evidence-type-valid",
|
|
94
|
+
"evidence-type-ratio",
|
|
95
|
+
"tests-pass",
|
|
96
|
+
"tsc-pass",
|
|
97
|
+
"code-committed",
|
|
98
|
+
"quinn-on-ui-change",
|
|
99
|
+
"local-api-validated",
|
|
100
|
+
"local-ui-validated",
|
|
101
|
+
"local-before-prod",
|
|
102
|
+
"container-has-fix",
|
|
103
|
+
"spec-drift",
|
|
104
|
+
"decision-id-valid",
|
|
105
|
+
"spec-decision-compliance",
|
|
106
|
+
"evidence-target-valid",
|
|
107
|
+
"skill-qc-ship",
|
|
108
|
+
"brief-ac-alignment"
|
|
109
|
+
],
|
|
110
|
+
"ship": [
|
|
111
|
+
"all-gates-pass",
|
|
112
|
+
"branch-merged",
|
|
113
|
+
"code-pushed",
|
|
114
|
+
"prod-rebuild-pass",
|
|
115
|
+
"prod-smoke-pass",
|
|
116
|
+
"prod-quinn-pass",
|
|
117
|
+
"prod-quinn-spot"
|
|
118
|
+
]
|
|
119
|
+
},
|
|
120
|
+
"batchMode": {
|
|
121
|
+
"description": "When batchMode is active, use reduced check set. Activated by ship skill for 3+ dependent issues.",
|
|
122
|
+
"skipScopeGate": true,
|
|
123
|
+
"checks": {
|
|
124
|
+
"scope": [
|
|
125
|
+
"acs-measurable",
|
|
126
|
+
"acs-no-skillmd-enforcement",
|
|
127
|
+
"acs-no-behavioral-language"
|
|
128
|
+
],
|
|
129
|
+
"verify": [
|
|
130
|
+
"code-committed",
|
|
131
|
+
"tests-pass",
|
|
132
|
+
"tsc-pass",
|
|
133
|
+
"local-before-prod",
|
|
134
|
+
"container-has-fix"
|
|
135
|
+
],
|
|
136
|
+
"ship": [
|
|
137
|
+
"branch-merged",
|
|
138
|
+
"code-pushed",
|
|
139
|
+
"prod-smoke-pass"
|
|
140
|
+
]
|
|
141
|
+
}
|
|
142
|
+
}
|
|
143
|
+
},
|
|
144
|
+
"THOROUGH": {
|
|
145
|
+
"description": "Adversarial multi-agent. Quinn + Rook + challenge agent.",
|
|
146
|
+
"evaluator": "adversarial",
|
|
147
|
+
"gates": [
|
|
148
|
+
"scope",
|
|
149
|
+
"verify",
|
|
150
|
+
"ship"
|
|
151
|
+
],
|
|
152
|
+
"skipScopeGate": false,
|
|
153
|
+
"agents": {
|
|
154
|
+
"marcus": true,
|
|
155
|
+
"quinn": true,
|
|
156
|
+
"rook": true
|
|
157
|
+
},
|
|
158
|
+
"githubComments": 5,
|
|
159
|
+
"stopMarkers": 3,
|
|
160
|
+
"maxIterations": 8,
|
|
161
|
+
"checks": {
|
|
162
|
+
"scope": [
|
|
163
|
+
"acs-exist",
|
|
164
|
+
"acs-have-type",
|
|
165
|
+
"acs-have-threshold",
|
|
166
|
+
"acs-have-evidence-method",
|
|
167
|
+
"source-spec-cited",
|
|
168
|
+
"spec-elements-in-acs",
|
|
169
|
+
"sizing-declared",
|
|
170
|
+
"issue-goal-captured",
|
|
171
|
+
"skill-qc-goal",
|
|
172
|
+
"acs-no-skillmd-enforcement",
|
|
173
|
+
"acs-no-behavioral-language",
|
|
174
|
+
"ac-issue-alignment"
|
|
175
|
+
],
|
|
176
|
+
"verify": [
|
|
177
|
+
"all-acs-have-evidence",
|
|
178
|
+
"all-acs-pass",
|
|
179
|
+
"evidence-type-valid",
|
|
180
|
+
"evidence-type-ratio",
|
|
181
|
+
"evidence-method-executed",
|
|
182
|
+
"threshold-crosscheck",
|
|
183
|
+
"tests-pass",
|
|
184
|
+
"tsc-pass",
|
|
185
|
+
"code-committed",
|
|
186
|
+
"quinn-on-ui-change",
|
|
187
|
+
"local-api-validated",
|
|
188
|
+
"local-ui-validated",
|
|
189
|
+
"local-before-prod",
|
|
190
|
+
"container-has-fix",
|
|
191
|
+
"consumer-chain-verified",
|
|
192
|
+
"dead-exports",
|
|
193
|
+
"spec-drift",
|
|
194
|
+
"decision-id-valid",
|
|
195
|
+
"spec-decision-compliance",
|
|
196
|
+
"evidence-target-valid",
|
|
197
|
+
"skill-qc-ship",
|
|
198
|
+
"brief-ac-alignment"
|
|
199
|
+
],
|
|
200
|
+
"ship": [
|
|
201
|
+
"all-gates-pass",
|
|
202
|
+
"branch-merged",
|
|
203
|
+
"code-pushed",
|
|
204
|
+
"prod-rebuild-pass",
|
|
205
|
+
"prod-smoke-pass",
|
|
206
|
+
"prod-quinn-pass",
|
|
207
|
+
"prod-quinn-spot"
|
|
208
|
+
]
|
|
209
|
+
}
|
|
210
|
+
}
|
|
211
|
+
}
|
|
212
|
+
}
|
|
@@ -0,0 +1,24 @@
|
|
|
1
|
+
{
|
|
2
|
+
"consumers": [
|
|
3
|
+
{ "glob": "src/dashboard-service.ts", "consumer": "Morning Summary", "verify": "curl /api/morning-summary" },
|
|
4
|
+
{ "glob": "src/value-positioning.ts", "consumer": "Value Positioning", "verify": "curl /api/customer/:name/value-positioning" },
|
|
5
|
+
{ "glob": "src/account-plan.ts", "consumer": "Account Plan", "verify": "curl /api/customer/:name/account-plan" },
|
|
6
|
+
{ "glob": "src/customer.ts", "consumer": "Customer Detail", "verify": "curl /api/customer/:name/brief" },
|
|
7
|
+
{ "glob": "src/brief-pipeline.ts", "consumer": "Brief Pipeline", "verify": "curl /api/customer/:name/brief" },
|
|
8
|
+
{ "glob": "src/campaign-service.ts", "consumer": "Campaign", "verify": "curl /api/customer/:name/campaigns" },
|
|
9
|
+
{ "glob": "src/meeting-prep-service.ts", "consumer": "Meeting Prep", "verify": "curl /api/customer/:name/meeting-prep" },
|
|
10
|
+
{ "glob": "src/playbook-generator.ts", "consumer": "Playbook", "verify": "curl /api/customer/:name/playbook" }
|
|
11
|
+
],
|
|
12
|
+
"feeders": [
|
|
13
|
+
{ "glob": "src/lib/motion-builder.ts", "affects": ["Campaign", "Meeting Prep", "Playbook"] },
|
|
14
|
+
{ "glob": "src/lib/tactic-scorer.ts", "affects": ["Campaign", "Meeting Prep", "Playbook"] },
|
|
15
|
+
{ "glob": "src/lib/customer-product-context.ts", "affects": ["All consumers"] },
|
|
16
|
+
{ "glob": "src/lib/product-vocabulary.ts", "affects": ["All consumers"] },
|
|
17
|
+
{ "glob": "src/lib/templates/", "affects": ["All consumers"] },
|
|
18
|
+
{ "glob": "src/lib/signal-", "affects": ["Morning Summary", "Campaign", "Meeting Prep"] },
|
|
19
|
+
{ "glob": "src/quality-validators/", "affects": ["Campaign", "Meeting Prep", "Account Plan"] },
|
|
20
|
+
{ "glob": "src/modules/", "affects": ["All consumers"] },
|
|
21
|
+
{ "glob": "src/refresh-engine.ts", "affects": ["All consumers"] }
|
|
22
|
+
],
|
|
23
|
+
"comment": "Maps changed files to consumer output verification requirements. Used by durability-gate.sh. Compliance-checked by architecture-compliance.test.ts against PRINCIPLES.md consumer table."
|
|
24
|
+
}
|
|
@@ -0,0 +1,24 @@
|
|
|
1
|
+
{
|
|
2
|
+
"patterns": [
|
|
3
|
+
{ "glob": "docs/adr/*.md", "requires": ["PRINCIPLES.md", "ARCHITECTURE.md"] },
|
|
4
|
+
{ "glob": "src/modules/*.ts", "requires": ["ARCHITECTURE.md", "PROJECT-STATE.md"] },
|
|
5
|
+
{ "glob": "src/*-routes.ts", "requires": ["PROJECT-STATE.md"] },
|
|
6
|
+
{ "glob": "dashboard/src/pages/*.tsx", "requires": ["PROJECT-STATE.md"] },
|
|
7
|
+
{ "glob": "dashboard/src/components/*.tsx", "requires": ["PROJECT-STATE.md"] },
|
|
8
|
+
{ "glob": "src/lib/*.ts", "requires": ["ARCHITECTURE.md"] },
|
|
9
|
+
{ "glob": "src/*.ts", "requires": ["ARCHITECTURE.md"] },
|
|
10
|
+
{ "glob": "skills/**/*.md", "requires": ["skills/{skill}/SKILL.md"] },
|
|
11
|
+
{ "glob": "PAI/**/*.md", "requires": ["PAI/BRIEF-TEMPLATES.md"] },
|
|
12
|
+
{ "glob": "config-templates/**", "requires": ["PROJECT-STATE.md"] },
|
|
13
|
+
{ "glob": "scripts/*.ts", "requires": ["ARCHITECTURE.md"] },
|
|
14
|
+
{ "glob": "Makefile", "requires": ["PROJECT-STATE.md"] },
|
|
15
|
+
{ "glob": "Dockerfile*", "requires": ["PROJECT-STATE.md", "ARCHITECTURE.md"] },
|
|
16
|
+
{ "glob": "src/campaign-*.ts", "requires": ["docs/specs/consumers/campaign.md"] },
|
|
17
|
+
{ "glob": "src/meeting-prep*.ts", "requires": ["docs/specs/consumers/meeting-prep.md"] },
|
|
18
|
+
{ "glob": "src/account-plan*.ts", "requires": ["docs/specs/consumers/account-plan.md"] },
|
|
19
|
+
{ "glob": "src/playbook*.ts", "requires": ["docs/specs/consumers/playbook.md"] },
|
|
20
|
+
{ "glob": "src/value-positioning*.ts", "requires": ["docs/specs/consumers/value-positioning.md"] }
|
|
21
|
+
],
|
|
22
|
+
"always_check": ["PROJECT-STATE.md", "MODEL.md"],
|
|
23
|
+
"comment": "Machine-readable version of Ship SKILL.md doc cascade matrix. Used by durability-gate.sh. Consumer-spec cascade rules added 2026-08-16 per ADR-045 audit."
|
|
24
|
+
}
|
package/config/eval.json
ADDED
|
@@ -0,0 +1,84 @@
|
|
|
1
|
+
{
|
|
2
|
+
"version": "1.3",
|
|
3
|
+
"skill": "Ship",
|
|
4
|
+
"evaluator": "fast",
|
|
5
|
+
"criteria": [
|
|
6
|
+
{
|
|
7
|
+
"name": "template_first_no_retroactive",
|
|
8
|
+
"weight": 0.14,
|
|
9
|
+
"description": "TEMPLATES.md and BRIEF-TEMPLATES.md read BEFORE any artifact is written. GitHub issue comments, ACs, agent briefs, and completion reports use the template structure on FIRST write — not edited retroactively after a gate rejection. Zero gh issue edit / gh issue comment --edit calls to fix gate failures. If a gate script rejects content, the criterion fails — the content should have used the template from the start. This is the #1 time-wasting anti-pattern: writing freeform, failing the gate, then reverse-engineering markers into the content."
|
|
10
|
+
},
|
|
11
|
+
{
|
|
12
|
+
"name": "task_fully_complete",
|
|
13
|
+
"weight": 0.12,
|
|
14
|
+
"description": "All acceptance criteria satisfied with concrete evidence before claiming done. No partial delivery presented as complete. Every AC has a file:line reference, test result, curl output, or screenshot — not a prose assertion. If any AC cannot be evidenced, it is explicitly marked incomplete rather than glossed over. Langfuse data shows 93% of mid-range sessions are weak on task completion."
|
|
15
|
+
},
|
|
16
|
+
{
|
|
17
|
+
"name": "rule_adherence_autonomy",
|
|
18
|
+
"weight": 0.12,
|
|
19
|
+
"description": "After plan approval, executes autonomously with zero AskUserQuestion interruptions. No 'should I proceed?', 'want me to continue?', or option-presenting mid-execution. No pausing at phase boundaries. The only acceptable interruptions are circuit breakers, destructive actions on shared state, and the final completion report. Langfuse data shows 100% of mid-range sessions are weak on rule adherence — autonomy violations are the most common form."
|
|
20
|
+
},
|
|
21
|
+
{
|
|
22
|
+
"name": "all_five_steps_walked",
|
|
23
|
+
"weight": 0.10,
|
|
24
|
+
"description": "All five ship steps explicitly walked: SCOPE, BUILD, VERIFY, DURABILITY, CLOSE. No steps skipped or collapsed. Each step header mentioned by name with its actions listed. Skipping DURABILITY or CLOSE is the most common failure."
|
|
25
|
+
},
|
|
26
|
+
{
|
|
27
|
+
"name": "output_conciseness",
|
|
28
|
+
"weight": 0.08,
|
|
29
|
+
"description": "Responses use bullet fragments over full sentences. No restating the question. No trailing summaries after tool calls. No preamble. Data and decisions only. Status updates are one sentence. Completion reports use the template — no multi-paragraph narrative. Langfuse data shows 73% of mid-range sessions are weak on conciseness."
|
|
30
|
+
},
|
|
31
|
+
{
|
|
32
|
+
"name": "gate_passes_first_attempt",
|
|
33
|
+
"weight": 0.08,
|
|
34
|
+
"description": "Gate scripts (scope-gate.sh, verify-gate.sh, durability-gate.sh, close-gate.sh) pass on first invocation. No cycle of: run gate → fail → edit content → re-run gate. If a gate fails, the criterion fails regardless of whether it eventually passes. Measures whether the template was followed correctly the first time, not whether the agent can iterate until gates pass."
|
|
35
|
+
},
|
|
36
|
+
{
|
|
37
|
+
"name": "posted_to_github_issue",
|
|
38
|
+
"weight": 0.08,
|
|
39
|
+
"description": "ACs posted to GitHub issue via gh issue comment BEFORE BUILD. Sizing declaration posted. DISCOVERY comment posted. Findings posted. ATTEMPT comments posted on iteration. Completion Report posted at CLOSE. The GitHub issue is the system of record."
|
|
40
|
+
},
|
|
41
|
+
{
|
|
42
|
+
"name": "scope_acs_structured",
|
|
43
|
+
"weight": 0.06,
|
|
44
|
+
"description": "Acceptance criteria use the structured template with all required fields filled. CODE ACs have 7 fields, OUTCOME ACs have 10. No placeholder text. Template sourced from TEMPLATES.md, not written from memory."
|
|
45
|
+
},
|
|
46
|
+
{
|
|
47
|
+
"name": "agent_brief_from_template",
|
|
48
|
+
"weight": 0.06,
|
|
49
|
+
"description": "Agent brief written using BRIEF-TEMPLATES.md template — with Context, Task, Git protocol, Files to read, Scope do-not-touch, Skills available, Verify, Report sections. mode:bypassPermissions set. Never a freeform prompt."
|
|
50
|
+
},
|
|
51
|
+
{
|
|
52
|
+
"name": "tdd_red_green",
|
|
53
|
+
"weight": 0.04,
|
|
54
|
+
"description": "TDD invoked for code changes — regression test written first, observed failing (red), fix applied, test passes (green). Exempt only for config-only, pure doc, JSON data, UI copy changes."
|
|
55
|
+
},
|
|
56
|
+
{
|
|
57
|
+
"name": "batch_diagnostic_for_bugs",
|
|
58
|
+
"weight": 0.04,
|
|
59
|
+
"description": "For bugs: environment confirmed (port 7776 vs 7777), all logs + errors + source + state read in one pass. Single hypothesis formed. No read-hypothesis-read loops."
|
|
60
|
+
},
|
|
61
|
+
{
|
|
62
|
+
"name": "verify_evidence_per_ac",
|
|
63
|
+
"weight": 0.04,
|
|
64
|
+
"description": "Every AC has concrete evidence at VERIFY — screenshot, curl output, test result, file:line reference. Completion Report uses template from TEMPLATES.md. No freeform done claims."
|
|
65
|
+
},
|
|
66
|
+
{
|
|
67
|
+
"name": "gate_scripts_run",
|
|
68
|
+
"weight": 0.04,
|
|
69
|
+
"description": "Gate scripts explicitly invoked: scope-gate.sh before BUILD, verify-gate.sh before DURABILITY, durability-gate.sh before CLOSE, close-gate.sh before next issue."
|
|
70
|
+
}
|
|
71
|
+
],
|
|
72
|
+
"benchmark_prompts": [
|
|
73
|
+
"Fix a bug where the morning brief email shows stale customer data from yesterday — the cron ran but the brief template reads from cache not fresh data",
|
|
74
|
+
"The accounts table CSV export silently drops accounts with special characters in the name — diagnose and fix",
|
|
75
|
+
"Fix the admin sidebar showing wrong pod count — it counts archived accounts that should be excluded",
|
|
76
|
+
"The campaign generator Playwright tests are flaky — Gemini response sometimes exceeds the 30s timeout",
|
|
77
|
+
"Customer health score widget shows wrong totals for multi-AE accounts — the aggregation query double-counts shared accounts",
|
|
78
|
+
"Signal scoring engine returns zero for customers with no tech stack data — should fall back to industry baseline instead of zero",
|
|
79
|
+
"Implement a new expansion motions section in the customer intelligence dashboard showing cross-sell opportunities",
|
|
80
|
+
"Add webhook notifications that fire when a customer renewal date is within 30 days — new API endpoint and UI toggle",
|
|
81
|
+
"Refactor signal scoring to use the new weighted formula from ADR-032 — touches scoring.ts, brief-generator.ts, and 3 consumers",
|
|
82
|
+
"The setup wizard connection test fails silently when Red Hat portal cookies expire — need visible error and re-auth prompt"
|
|
83
|
+
]
|
|
84
|
+
}
|