rungate 0.8.0 → 0.8.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/gates/.gate-salt +1 -0
- package/gates/SCHEMA-GUIDE.md +485 -0
- package/gates/ac-quality.test.ts +106 -0
- package/gates/adversarial.test.ts +283 -0
- package/gates/brief-assembler.test.ts +487 -0
- package/gates/brief-assembler.ts +448 -0
- package/gates/bunfig.toml +2 -0
- package/gates/chain.test.ts +91 -0
- package/gates/e2e-smoke.test.ts +234 -0
- package/gates/error-classifier.test.ts +250 -0
- package/gates/error-classifier.ts +99 -0
- package/gates/gate-executor.ts +3071 -0
- package/gates/orchestrator.test.ts +968 -0
- package/gates/orchestrator.ts +827 -0
- package/gates/preload.ts +89 -0
- package/gates/prompts/ac-adversary.md +20 -0
- package/gates/prompts/evidence-validator.md +22 -0
- package/gates/prompts/prove-reproducer.md +53 -0
- package/gates/prompts.sha256 +3 -0
- package/gates/prove.test.ts +164 -0
- package/gates/rook-state.test.ts +108 -0
- package/gates/run-attribution.ts +290 -0
- package/gates/run-gate.ts +163 -0
- package/gates/schema-parity.test.ts +378 -0
- package/gates/schema.ts +1209 -0
- package/gates/self-heal.test.ts +511 -0
- package/gates/self-heal.ts +125 -0
- package/gates/ship-orchestrator.test.ts +399 -0
- package/gates/ship-orchestrator.ts +384 -0
- package/gates/spec-compliance-probe.ts +159 -0
- package/gates/test-fixtures/broken-no-environments.json +110 -0
- package/gates/test-fixtures/consumer-directory-layout.json +90 -0
- package/gates/test-fixtures/valid-standard.json +91 -0
- package/gates/test-utils.ts +43 -0
- package/gates/typecheck-gate.ts +111 -0
- package/gates/witness.ts +349 -0
- package/gates/workflow.test.ts +2018 -0
- package/hooks/BashWriteGuard.hook.ts +19 -0
- package/hooks/GateResultProtection.hook.ts +27 -1
- package/hooks/VerifyPhaseLock.hook.ts +51 -11
- package/hooks/WorkflowStateGuard.hook.ts +24 -0
- package/lib/ai-review-rounds.ts +462 -0
- package/lib/bash-file-read.ts +17 -3
- package/lib/bash-write-guard.ts +1285 -50
- package/lib/distribution-standing.ts +75 -4
- package/lib/failure-intake.ts +12 -4
- package/lib/file-extensions.ts +81 -0
- package/lib/finding-arbiter.ts +1015 -0
- package/lib/finding-jury.ts +537 -0
- package/lib/gate-enforcement.ts +171 -43
- package/lib/gate-result-protection.ts +12 -2
- package/lib/generators/code-map.ts +55 -4
- package/lib/generators/types.ts +27 -1
- package/lib/github.ts +55 -0
- package/lib/interpreters.ts +64 -0
- package/lib/issue-create-override.ts +69 -8
- package/lib/issue-filing-gate.ts +605 -0
- package/lib/js-top-level-code.ts +20 -2
- package/lib/junit-checks.ts +239 -1
- package/lib/package-closure.ts +114 -0
- package/lib/publish-window.ts +287 -0
- package/lib/scaffold/ai-review-findings.ts +142 -3
- package/lib/scaffold/ai-review-step.ts +1012 -18
- package/lib/scaffold/managed-workflow.ts +163 -4
- package/lib/scaffold/steps.ts +498 -22
- package/lib/scanner.ts +13 -4
- package/lib/ticked-sc-symbols.ts +361 -0
- package/lib/transcript-checker.ts +6 -0
- package/lib/workflow-actions.ts +18 -0
- package/package.json +3 -1
- package/scripts/arbitrate-finding.ts +224 -0
- package/scripts/generate-code-map.ts +18 -10
- package/scripts/sync-sc-status.ts +82 -8
package/gates/.gate-salt
ADDED
|
@@ -0,0 +1 @@
|
|
|
1
|
+
4bf447cd9f1aa5df2cbb1c260d54255a2d976fb28149b342a2b0360d7c4adbe4
|
|
@@ -0,0 +1,485 @@
|
|
|
1
|
+
---
|
|
2
|
+
doc-type: reference
|
|
3
|
+
status: active
|
|
4
|
+
owner: jason
|
|
5
|
+
updated: 2026-09-14
|
|
6
|
+
---
|
|
7
|
+
|
|
8
|
+
# workflow-state.json Schema Guide
|
|
9
|
+
|
|
10
|
+
Read this BEFORE writing ACs or workflow-state.json. The scope gate validates every field.
|
|
11
|
+
|
|
12
|
+
## AC Required Fields
|
|
13
|
+
|
|
14
|
+
Every AC must have all of these:
|
|
15
|
+
|
|
16
|
+
| Field | Type | Constraints |
|
|
17
|
+
|---|---|---|
|
|
18
|
+
| `id` | string | `AC-1`, `AC-2`, `AC-A1` (A-prefix for anti-criteria) |
|
|
19
|
+
| `type` | enum | `CODE` or `OUTCOME` only |
|
|
20
|
+
| `statement` | string | Min 5 words. No garbage (see below) |
|
|
21
|
+
| `threshold` | object | `{ op, value, unit }` — see Threshold Rules |
|
|
22
|
+
| `evidenceMethod` | object | `{ type, command }` — see Evidence Types |
|
|
23
|
+
|
|
24
|
+
Optional: `specElement` (string), `contextFiles` (array), `verdict`, `evidence`.
|
|
25
|
+
|
|
26
|
+
## Threshold Rules
|
|
27
|
+
|
|
28
|
+
```
|
|
29
|
+
value: string | number ← NEVER boolean (true/false will fail Zod validation)
|
|
30
|
+
op: "==" | ">=" | "<=" | ">" | "<" | "!=" | "contains" | "exists"
|
|
31
|
+
unit: string (optional)
|
|
32
|
+
```
|
|
33
|
+
|
|
34
|
+
**Common mistake:** `{ "op": "exists", "value": true }` — FAILS. Use `"value": "true"` (string).
|
|
35
|
+
|
|
36
|
+
**Weak threshold:** `{ "op": ">=", "value": 1 }` — FAILS. Threshold `>= 1` effectively always passes.
|
|
37
|
+
|
|
38
|
+
## Evidence Method Types
|
|
39
|
+
|
|
40
|
+
Valid `type` values (case-sensitive): `GREP_CHECK`, `FILE_EXISTS`, `CURL_CHECK`, `BUN_TEST`, `SCREENSHOT`, `PLAYWRIGHT`, `COMMAND`, `MANUAL`, `grep`, `command`, `api`, `screenshot`, `manual`
|
|
41
|
+
|
|
42
|
+
### Rewrite provenance — `authoredCommand` and `rewrites` (#614)
|
|
43
|
+
|
|
44
|
+
`command` is what will RUN. Three places legitimately change it before it
|
|
45
|
+
does, and each one records what it changed.
|
|
46
|
+
|
|
47
|
+
| Field | Type | Meaning |
|
|
48
|
+
|---|---|---|
|
|
49
|
+
| `authoredCommand` | string | The command as the discovery agent wrote it. Written once by the FIRST rewrite and never overwritten |
|
|
50
|
+
| `rewrites` | `{by, from, to}[]` | Every change to `command`, oldest first. `by` names the site: `discovery-path-normalize`, `evidence-type-ratio`, `normalize-git-refs` |
|
|
51
|
+
|
|
52
|
+
**A `rewrites` list with no `authoredCommand` beside it is rejected at write
|
|
53
|
+
time.** It is the same pairing as `evidencePrevalidation`'s
|
|
54
|
+
`autoFixed`/`originalCommand`/`fixedCommand`, and for the same reason: a
|
|
55
|
+
rewrite that reports only its replacement cannot be reviewed, and a repaired
|
|
56
|
+
command is then indistinguishable from one that was always written that way.
|
|
57
|
+
|
|
58
|
+
Both are absent on an AC nothing has rewritten. Absence means "as authored",
|
|
59
|
+
not "unknown".
|
|
60
|
+
|
|
61
|
+
```json
|
|
62
|
+
{ "evidenceMethod": {
|
|
63
|
+
"type": "COMMAND",
|
|
64
|
+
"command": "git log origin/main..HEAD --oneline | wc -l | tr -d ' '",
|
|
65
|
+
"authoredCommand": "git log main..HEAD --oneline | wc -l | tr -d ' '",
|
|
66
|
+
"rewrites": [{ "by": "normalize-git-refs",
|
|
67
|
+
"from": "git log main..HEAD --oneline | wc -l | tr -d ' '",
|
|
68
|
+
"to": "git log origin/main..HEAD --oneline | wc -l | tr -d ' '" }]
|
|
69
|
+
} }
|
|
70
|
+
```
|
|
71
|
+
|
|
72
|
+
Write it with `recordEvidenceRewrite` in `lib/evidence-command-transport.ts`,
|
|
73
|
+
never by assigning `command` directly — that function is the single site that
|
|
74
|
+
changes an evidence command, and a direct assignment is what made run
|
|
75
|
+
`wf_c92dd0a7-fe6`'s corruption unattributable.
|
|
76
|
+
|
|
77
|
+
## Statement Constraints
|
|
78
|
+
|
|
79
|
+
**Garbage patterns (auto-rejected):**
|
|
80
|
+
- "file exists", "it works", "code is correct/good/done"
|
|
81
|
+
- "tests pass", "changes are made/applied/done", "no errors"
|
|
82
|
+
|
|
83
|
+
**Behavioral language (auto-rejected):**
|
|
84
|
+
- "DA should", "remember to", "make sure", "don't forget"
|
|
85
|
+
|
|
86
|
+
## Ceremony Tier Requirements
|
|
87
|
+
|
|
88
|
+
| Requirement | LIGHT (XS) | STANDARD (S/M) | THOROUGH (L) |
|
|
89
|
+
|---|---|---|---|
|
|
90
|
+
| `sourceSpecs[]` | not required | **REQUIRED** | **REQUIRED** |
|
|
91
|
+
| `governingSpec` | not required | **REQUIRED** | **REQUIRED** |
|
|
92
|
+
| `specElement` on ACs | not required | at least 1 AC per specElement | at least 1 AC per specElement |
|
|
93
|
+
| spec-prohibited checks | skipped | **ACTIVE** | **ACTIVE** |
|
|
94
|
+
| hardcoded value checks | skipped | **ACTIVE** | **ACTIVE** |
|
|
95
|
+
|
|
96
|
+
### sourceSpecs format
|
|
97
|
+
```json
|
|
98
|
+
{ "path": ".github/workflows/issue-triage.md", "citedInDiscovery": true,
|
|
99
|
+
"specElements": ["frontmatter schema pattern", "add-comment config"] }
|
|
100
|
+
```
|
|
101
|
+
Paths must be relative to projectRoot (not `~/`). The gate resolves: `join(projectRoot, path)`.
|
|
102
|
+
|
|
103
|
+
### governingSpec format
|
|
104
|
+
```json
|
|
105
|
+
{ "path": "/Users/jhorn/.claude/PAI/specs/CI-ENFORCEMENT-SPEC.md",
|
|
106
|
+
"detectedFrom": "ship-workflow-discovery" }
|
|
107
|
+
```
|
|
108
|
+
Path must be absolute (starts with `/`) or relative to projectRoot. `~` is NOT expanded.
|
|
109
|
+
|
|
110
|
+
## agents.rook
|
|
111
|
+
|
|
112
|
+
The security reviewer's result lives at `agents.rook`. Six fields carry it:
|
|
113
|
+
|
|
114
|
+
| Field | Type | Meaning |
|
|
115
|
+
|---|---|---|
|
|
116
|
+
| `spawned` | boolean | Whether rook actually ran on this ship run |
|
|
117
|
+
| `verdict` | `PASS` \| `FAIL` \| `SKIP` \| `EXHAUSTED` \| `null` | The review outcome |
|
|
118
|
+
| `failures` | string[] | One entry per blocking problem found |
|
|
119
|
+
| `testedPaths` | string[] | The files git said changed — what was actually reviewed |
|
|
120
|
+
| `refusal` | `"SECURITY_REREVIEW_EXHAUSTED"` | The named refusal. Only on an `EXHAUSTED` record |
|
|
121
|
+
| `rounds` | integer ≥ 1 | Remediate/re-review rounds spent. Required on `EXHAUSTED` |
|
|
122
|
+
|
|
123
|
+
**A `FAIL` verdict must carry a non-empty `failures` list.** A FAIL with an empty
|
|
124
|
+
or absent list is a verdict nobody can act on — it blocks the run without naming
|
|
125
|
+
what to fix. `failures` must be an array of strings; a bare string is rejected at
|
|
126
|
+
write time, so put one finding per entry rather than joining them into prose.
|
|
127
|
+
|
|
128
|
+
**`EXHAUSTED` means the remediate/re-review cycle ran out of attempts, and it is
|
|
129
|
+
not a pass.** A remediation committed after the review leaves the run holding a
|
|
130
|
+
verdict for code it no longer ships (#169), so the new tip is re-reviewed — and
|
|
131
|
+
that loop is capped (#171). Spending the cap ends the run with code nobody
|
|
132
|
+
reviewed, which is the exact thing this gate exists to stop. It is its own
|
|
133
|
+
outcome, distinguishable from both neighbours and readable as neither:
|
|
134
|
+
|
|
135
|
+
- It is **not a `PASS`**. Nothing concluded the code was clean; the attempts ran
|
|
136
|
+
out. Any reader asking `verdict === "PASS"` must get `false`, and
|
|
137
|
+
`securityReviewIsClean` in `scripts/record-security-verdict.ts` is that reader.
|
|
138
|
+
- It is **not a `FAIL`**. Rook named no blocking problem, so an `EXHAUSTED`
|
|
139
|
+
record carries **no `failures`** — a findings list is how "the review found
|
|
140
|
+
something" is written down. Zod rejects an `EXHAUSTED` record that has one.
|
|
141
|
+
|
|
142
|
+
Zod also rejects `refusal` on any verdict other than `EXHAUSTED`, and an
|
|
143
|
+
`EXHAUSTED` verdict missing `refusal` or `rounds`. Recording the refusal
|
|
144
|
+
honestly while parking the verdict at `PASS` is the fail-open the pairing
|
|
145
|
+
closes.
|
|
146
|
+
|
|
147
|
+
```json
|
|
148
|
+
{ "agents": { "rook": {
|
|
149
|
+
"spawned": true,
|
|
150
|
+
"verdict": "EXHAUSTED",
|
|
151
|
+
"refusal": "SECURITY_REREVIEW_EXHAUSTED",
|
|
152
|
+
"rounds": 2,
|
|
153
|
+
"testedSha": "3192a75...", "testedPaths": ["gates/run-gate.ts"]
|
|
154
|
+
} } }
|
|
155
|
+
```
|
|
156
|
+
|
|
157
|
+
```json
|
|
158
|
+
{ "agents": { "rook": {
|
|
159
|
+
"spawned": true,
|
|
160
|
+
"verdict": "FAIL",
|
|
161
|
+
"failures": ["secret logged at gates/run-gate.ts:120", "unvalidated path join in lib/github.ts"],
|
|
162
|
+
"testedSha": "3192a75...", "testedPaths": ["gates/run-gate.ts", "lib/github.ts"]
|
|
163
|
+
} } }
|
|
164
|
+
```
|
|
165
|
+
|
|
166
|
+
A `PASS` or `SKIP` verdict may omit `failures` entirely.
|
|
167
|
+
|
|
168
|
+
Write it with `scripts/record-security-verdict.ts`, not by hand — `workflows/ship.js`
|
|
169
|
+
calls it in the Verify fan-out (#129). The exhausted state is
|
|
170
|
+
`--verdict EXHAUSTED --rounds <n>`; the recorder derives `refusal` from the
|
|
171
|
+
verdict rather than accepting it as an argument.
|
|
172
|
+
|
|
173
|
+
## suiteMeasurement
|
|
174
|
+
|
|
175
|
+
The suite result, bound to the commit it ran against (#224). Top-level, optional,
|
|
176
|
+
and **absent means UNMEASURED** — not "clean".
|
|
177
|
+
|
|
178
|
+
| Field | Type | Meaning |
|
|
179
|
+
|---|---|---|
|
|
180
|
+
| `verdict` | `PASS` \| `FAIL` \| `PARTIAL` | Derived from the count and the declared scope, never supplied by a caller |
|
|
181
|
+
| `measuredSha` | string | The commit the suite actually ran against — 7-40 hex, never a ref name |
|
|
182
|
+
| `failures` | number | Failing tests: `0` for a PASS, at least `1` for a FAIL |
|
|
183
|
+
| `measuredAt` | string | ISO timestamp of the run (optional) |
|
|
184
|
+
| `tests` | number \| null | Tests the runner COLLECTED, or `null` when it did not say (#490) |
|
|
185
|
+
| `ran` | number | Tests that actually executed — `pass + fail`. Absent only on a record older than #673 |
|
|
186
|
+
| `skip` | number \| null | Tests the runner skipped, or `null` when it reports no skip count. Absent only on a record older than #673 |
|
|
187
|
+
| `todo` | number \| null | Tests marked todo, or `null` when the runner reports no todo count. Absent only on a record older than #673 |
|
|
188
|
+
| `files` | number \| null | Test files the runner said it discovered, or `null` when it did not say. Never `0` — a run over no files is refused, not recorded (#490) |
|
|
189
|
+
| `failingTests` | string[] | The names the runner printed, in order. May be SHORTER than `failures` — a module-scope error counts toward the total and has no test name (#323) |
|
|
190
|
+
| `errors` | number \| null | Unhandled errors between tests, or `null` when the runner reports no error count. This is *how* a failure loses its name (#323) |
|
|
191
|
+
| `expectCalls` | number \| null | Assertions the run executed, or `null` when the runner did not say. No gate reads it: the 2026-10-11 series left an unexplained 384-assertion delta between a failing run and two clean ones, and this is that number written down (#323 SC-4) |
|
|
192
|
+
|
|
193
|
+
**`tests` and `ran` are different quantities and both are recorded (#673).**
|
|
194
|
+
On run `wf_62d68d55-81c` they were `13915` and `13084`; the changelog line carried
|
|
195
|
+
one, this record carried the other, and nothing said which was which, so a reader
|
|
196
|
+
comparing either against a previous run's number could be comparing different
|
|
197
|
+
things. The difference is exactly `skip + todo`, which is why all four are here —
|
|
198
|
+
nobody should have to subtract to learn what a number means.
|
|
199
|
+
|
|
200
|
+
**`PARTIAL` means the reading was taken over less than the whole suite**, declared
|
|
201
|
+
by the caller through `--scope` because the runner's output cannot say it: `bun
|
|
202
|
+
test test/one.ts` and `bun test` print the same shape. It is writable and it is
|
|
203
|
+
not a result — `readSuiteMeasurement` resolves it to `UNMEASURED` by name, so no
|
|
204
|
+
gate can act on it, and the recorder writes no `environments.local.tests` at all
|
|
205
|
+
for one (`SKIP` would SATISFY the gate, which is the defect inverted).
|
|
206
|
+
|
|
207
|
+
**A measurement with no `measuredSha` is rejected at write time.** A count with no
|
|
208
|
+
commit attached cannot be checked against the tree being shipped, so it is not
|
|
209
|
+
evidence: run `wf_7ac5f614-d21` published `regressions: 0` over a branch whose
|
|
210
|
+
suite had one failing test, and nothing could tell. `"HEAD"` is rejected for the
|
|
211
|
+
same reason — it names whatever the checkout happened to be.
|
|
212
|
+
|
|
213
|
+
**The verdict and the count may not contradict each other.** A `PASS` with failing
|
|
214
|
+
tests, and a `FAIL` with none, are both refused by Zod.
|
|
215
|
+
|
|
216
|
+
```json
|
|
217
|
+
{ "suiteMeasurement": {
|
|
218
|
+
"verdict": "FAIL", "measuredSha": "dd242a62aea9371d788abb58f3d26922d5dd6cbc",
|
|
219
|
+
"failures": 1, "measuredAt": "2026-10-09T18:04:11Z"
|
|
220
|
+
} }
|
|
221
|
+
```
|
|
222
|
+
|
|
223
|
+
`UNMEASURED` is **not** a writable verdict. It is what `lib/suite-measurement.ts`
|
|
224
|
+
returns for the absence of a usable record — no record, no SHA, a malformed SHA,
|
|
225
|
+
no count, or a self-contradicting one — and it reads with `failures: null` rather
|
|
226
|
+
than `0`, so a consumer cannot add an unmeasured run into a clean total.
|
|
227
|
+
|
|
228
|
+
Write it with `scripts/record-suite-measurement.ts`, not by hand:
|
|
229
|
+
|
|
230
|
+
```
|
|
231
|
+
bun scripts/record-suite-measurement.ts --state <workflow-state.json> --project <project root> --output <captured suite output> --scope full|partial
|
|
232
|
+
```
|
|
233
|
+
|
|
234
|
+
The commit is read from `--project`, not supplied (#312). A SHA a caller types
|
|
235
|
+
can agree with HEAD by construction, which is the provenance check answering
|
|
236
|
+
itself — so `--sha` is refused by name, as is `--result`: the verdict comes
|
|
237
|
+
from the count and has no other source.
|
|
238
|
+
|
|
239
|
+
**The count is read from the runner's output, not supplied either (#469, #490).**
|
|
240
|
+
`--failures <n>` is refused by name for the same reason as `--sha`: it was a
|
|
241
|
+
number an agent typed after reading a summary, and the run this guards against
|
|
242
|
+
had no summary to read. A run whose output carries no pass/fail counts, or
|
|
243
|
+
which executed nothing, is **refused rather than recorded** — the absence then
|
|
244
|
+
reads UNMEASURED through `lib/suite-measurement.ts`, which is what absence
|
|
245
|
+
already means here.
|
|
246
|
+
|
|
247
|
+
## evidencePrevalidation
|
|
248
|
+
|
|
249
|
+
The scope gate's dry-run of every AC evidence command, written before the gate
|
|
250
|
+
tests read the file (#235). Top-level, optional, and **absent means UNMEASURED**
|
|
251
|
+
— the gate never got as far as measuring.
|
|
252
|
+
|
|
253
|
+
| Field | Type | Meaning |
|
|
254
|
+
|---|---|---|
|
|
255
|
+
| `verdict` | `PASS` \| `FAIL` \| `UNMEASURED` | Derived from the per-AC results, never supplied by a caller |
|
|
256
|
+
| `checkedAt` | string | ISO timestamp of the dry-run |
|
|
257
|
+
| `acs` | array | One entry per AC: `id`, `status`, and the auto-fix fields below. Empty — and only empty — when UNMEASURED |
|
|
258
|
+
| `broken` | array of AC ids | The ACs `acs` classified `broken`, in measured order. Derived, never supplied. Required on a FAIL (#274) |
|
|
259
|
+
| `alreadySatisfied` | array of AC ids | The ACs whose **pre-change** output already cleared their own threshold. Derived from the baselines under `acs`, never supplied (#669) |
|
|
260
|
+
| `reason` | string \| null | Why the reading is UNMEASURED. Null, and only null, when it is not |
|
|
261
|
+
| `takenAt` | `scope` \| `verify` \| `ship`, optional | Which gate measured it (#264, #280). Absent means a run older than the field |
|
|
262
|
+
|
|
263
|
+
**The three verdicts:**
|
|
264
|
+
|
|
265
|
+
| Verdict | Means |
|
|
266
|
+
|---|---|
|
|
267
|
+
| `PASS` | Every evidence command was dry-run and none is broken. `empty` output is a warning, not a refusal, and still appears in `acs` |
|
|
268
|
+
| `FAIL` | At least one AC's command is broken and could not be auto-fixed. A FAIL must name one — a FAIL with no `broken` entry is refused by Zod |
|
|
269
|
+
| `UNMEASURED` | The pre-validation did not produce a result: it threw, or there were no ACs. Carries a `reason` and no `acs` |
|
|
270
|
+
|
|
271
|
+
**The FAIL shape, and why `broken` is part of it (#274).** A FAIL carries two
|
|
272
|
+
views of the same fact: the per-AC results under `acs`, and the ids under
|
|
273
|
+
`broken`. The second is not redundancy. `prevalidationViolations` in
|
|
274
|
+
`gates/workflow.test.ts` — the single site that turns this reading into a
|
|
275
|
+
refusal — names the ACs to fix out of `broken`, and until #274 **no writer
|
|
276
|
+
produced that key**: `measureEvidencePrevalidation` returned per-AC results and
|
|
277
|
+
nothing else, so every genuine FAIL refused with "no broken AC ids were
|
|
278
|
+
recorded", which was true of the key and false of the reading. Fail-closed, so
|
|
279
|
+
nothing shipped that should not have — but a refusal the heal loop cannot act
|
|
280
|
+
on costs a round and teaches it nothing.
|
|
281
|
+
|
|
282
|
+
So a FAIL must satisfy all three of these, and Zod refuses it otherwise:
|
|
283
|
+
|
|
284
|
+
| On a FAIL | Rule |
|
|
285
|
+
|---|---|
|
|
286
|
+
| `acs` | At least one entry with `status: "broken"` |
|
|
287
|
+
| `broken` | Present, and exactly the ids of those entries, in the same order |
|
|
288
|
+
| `reason` | `null` — a measured verdict explains itself through its ACs |
|
|
289
|
+
|
|
290
|
+
`broken` is **derived, never supplied.** It is mapped from the same filter the
|
|
291
|
+
verdict is derived from, at the single site that derives it, and the schema
|
|
292
|
+
compares it against `acs` on every verdict — a PASS or an UNMEASURED that names
|
|
293
|
+
a broken AC is refused the same way a FAIL that names the wrong one is. A list a
|
|
294
|
+
writer could set independently of `acs` would be a second source of truth for
|
|
295
|
+
one fact, and the refusal would name whichever of the two was wrong.
|
|
296
|
+
|
|
297
|
+
It is optional only for backward compatibility, the same way `takenAt` is: a
|
|
298
|
+
reading written before the field existed still validates. A FAIL does not get
|
|
299
|
+
that amnesty, because naming what to fix is the entire job of that verdict.
|
|
300
|
+
|
|
301
|
+
Unlike `suiteMeasurement`, `UNMEASURED` **is** writable here. There, recording it
|
|
302
|
+
would give a caller a way to park an unmeasured run in the field that proves the
|
|
303
|
+
suite ran. Here the failure runs the other way: a pre-validation that threw and
|
|
304
|
+
one that found nothing wrong both used to leave the field empty, and absence
|
|
305
|
+
reads downstream exactly like "clean". Recording the throw, with its reason, is
|
|
306
|
+
what makes the two distinguishable — and absence still means nobody measured.
|
|
307
|
+
|
|
308
|
+
Each entry in `acs` carries:
|
|
309
|
+
|
|
310
|
+
| AC field | Type | Meaning |
|
|
311
|
+
|---|---|---|
|
|
312
|
+
| `id` | string | The AC the command belongs to |
|
|
313
|
+
| `status` | `ok` \| `broken` \| `empty` \| `skipped` | Dry-run outcome. `skipped` means the AC has no command |
|
|
314
|
+
| `baseline` | object | What the command printed on the **pre-change** tree, beside the threshold it will be judged against (#669). See below |
|
|
315
|
+
| `autoFixed` | boolean | The command was repaired. Requires **both** command fields below |
|
|
316
|
+
| `originalCommand` | string | The command as the AC was written |
|
|
317
|
+
| `fixedCommand` | string | The command to run instead |
|
|
318
|
+
| `needsRewrite` | boolean | Broken and not auto-fixable — a human or Marcus must rewrite it |
|
|
319
|
+
| `diagnostic` | string | Why it is broken |
|
|
320
|
+
|
|
321
|
+
**An `autoFixed` entry must carry both `originalCommand` and `fixedCommand`.** A
|
|
322
|
+
repair that reports only its replacement cannot be reviewed — there is nothing
|
|
323
|
+
to compare it to, and it is indistinguishable from a command that was always
|
|
324
|
+
written that way. This includes the ref-rewrite path (`main` → `origin/main`),
|
|
325
|
+
which returns `status: "ok"`: the command dry-runs clean and only diverges at
|
|
326
|
+
Verify, so the rewrite is the only record that anything happened.
|
|
327
|
+
|
|
328
|
+
### `baseline` — the pre-change reading (#669)
|
|
329
|
+
|
|
330
|
+
The pre-change tree stops existing the moment Marcus writes a line. Until
|
|
331
|
+
#669 the pre-validation ran every AC's command against it, kept `ok` /
|
|
332
|
+
`broken` / `empty` / `skipped`, and threw the output away — so the one tree on
|
|
333
|
+
which a threshold is known **not** to be satisfied was measured and then
|
|
334
|
+
forgotten, and nothing ever compared the output to the AC's own bar.
|
|
335
|
+
|
|
336
|
+
| Baseline field | Type | Meaning |
|
|
337
|
+
|---|---|---|
|
|
338
|
+
| `verdict` | `MEASURED` \| `UNMEASURED` | Whether the output is a value a threshold can be compared against |
|
|
339
|
+
| `output` | string | What the command printed, verbatim and trimmed. Non-empty on a MEASURED baseline |
|
|
340
|
+
| `reason` | string \| null | Why it is UNMEASURED. Null, and only null, when it is MEASURED |
|
|
341
|
+
| `threshold` | object \| null | The AC's threshold as written — `op`, `value`, `unit`. Recorded beside the output so both read without re-running anything |
|
|
342
|
+
| `satisfiesThreshold` | boolean \| null | Whether the pre-change output already clears the bar. **Null, and only null, when no comparison was made** |
|
|
343
|
+
|
|
344
|
+
**An UNMEASURED baseline is never reduced to a threshold comparison.** Two
|
|
345
|
+
outputs reach that verdict: nothing at all, and a tool version banner. The
|
|
346
|
+
second is the one that hid. Run `wf_448af2c4-6c1` carried an AC whose evidence
|
|
347
|
+
command kept only the line bun prints before it runs anything —
|
|
348
|
+
|
|
349
|
+
```
|
|
350
|
+
bun test v1.3.14 (d1632b29)
|
|
351
|
+
```
|
|
352
|
+
|
|
353
|
+
— which is 27 characters from a command that exited zero, so it classified
|
|
354
|
+
`ok` and every downstream reader saw an AC with evidence. There is no
|
|
355
|
+
measurement in it. Worse, a `contains "bun test"` threshold is **true** of
|
|
356
|
+
that string, so the comparison it was about to be fed would have reported the
|
|
357
|
+
AC satisfied by the tool's own name. The two unmeasured states carry different
|
|
358
|
+
reasons, because "it printed nothing" and "it printed only a banner" are fixed
|
|
359
|
+
by different edits.
|
|
360
|
+
|
|
361
|
+
`satisfiesThreshold` is `null` rather than `false` whenever no comparison
|
|
362
|
+
happened. `false` claims the bar was tested and missed — a statement about the
|
|
363
|
+
tree — while null says nobody could test it. Collapsing the two is how an AC
|
|
364
|
+
that cannot measure anything reads as one that simply has not landed yet.
|
|
365
|
+
|
|
366
|
+
The schema refuses a baseline that contradicts itself, the same way it already
|
|
367
|
+
refuses an `UNMEASURED` reading with no reason:
|
|
368
|
+
|
|
369
|
+
| Contradiction | Why |
|
|
370
|
+
|---|---|
|
|
371
|
+
| `UNMEASURED` with no `reason` | An absence nobody can explain is not actionable |
|
|
372
|
+
| `UNMEASURED` with a non-null `satisfiesThreshold` | An output nobody could read was reduced to a comparison |
|
|
373
|
+
| `MEASURED` with a `reason` | A measured output explains itself |
|
|
374
|
+
| `MEASURED` with empty `output` | A command that printed nothing measured nothing |
|
|
375
|
+
| `satisfiesThreshold` with `threshold: null` | There was no bar to clear |
|
|
376
|
+
| `alreadySatisfied` disagreeing with the baselines under `acs` | The refusal would name the wrong AC |
|
|
377
|
+
| `alreadySatisfied` absent while a baseline is satisfied | A vacuous AC nothing records is one nothing can refuse |
|
|
378
|
+
|
|
379
|
+
**`alreadySatisfied` is a refusal at scope and nowhere else.**
|
|
380
|
+
`prevalidationViolations` names those ids only when `takenAt` is `scope`,
|
|
381
|
+
which is the one phase whose tree is the pre-change one. At verify and at ship
|
|
382
|
+
the work has landed, so an AC that satisfies its threshold there is an AC that
|
|
383
|
+
**passed** — refusing those would turn every successful run red. A reading
|
|
384
|
+
with no `takenAt` predates the field and cannot say which tree it describes,
|
|
385
|
+
so it is not refused either.
|
|
386
|
+
|
|
387
|
+
```json
|
|
388
|
+
{ "evidencePrevalidation": {
|
|
389
|
+
"verdict": "FAIL", "checkedAt": "2026-10-09T18:04:11.000Z", "reason": null,
|
|
390
|
+
"broken": ["AC-2"], "alreadySatisfied": [], "takenAt": "scope",
|
|
391
|
+
"acs": [
|
|
392
|
+
{ "id": "AC-1", "status": "ok", "autoFixed": true,
|
|
393
|
+
"originalCommand": "git log main..HEAD --oneline",
|
|
394
|
+
"fixedCommand": "git log origin/main..HEAD --oneline",
|
|
395
|
+
"baseline": { "verdict": "MEASURED", "output": "0", "reason": null,
|
|
396
|
+
"threshold": { "op": ">=", "value": 1 }, "satisfiesThreshold": false } },
|
|
397
|
+
{ "id": "AC-2", "status": "broken", "needsRewrite": true,
|
|
398
|
+
"diagnostic": "Command failed with exit code 127",
|
|
399
|
+
"baseline": { "verdict": "UNMEASURED", "output": "",
|
|
400
|
+
"reason": "the command ran and printed nothing, so there is no value to compare against the threshold",
|
|
401
|
+
"threshold": { "op": "contains", "value": "ok" }, "satisfiesThreshold": null } }
|
|
402
|
+
]
|
|
403
|
+
} }
|
|
404
|
+
```
|
|
405
|
+
|
|
406
|
+
Three gates write it — `gates/gate-executor.ts`, `runScopePreflightsAndPersist`
|
|
407
|
+
at scope and `ensurePrevalidationPersisted` at verify and at ship — after the
|
|
408
|
+
dry-run resolves and before the gate tests run. Not by hand.
|
|
409
|
+
|
|
410
|
+
**`takenAt` decides whether the gate now running needs its own reading, and
|
|
411
|
+
the policy lives in one predicate: `shouldRetakePrevalidation`.**
|
|
412
|
+
|
|
413
|
+
| Gate | What survives | Why |
|
|
414
|
+
|---|---|---|
|
|
415
|
+
| verify | a `scope` reading | It measured the run's starting conditions. Re-measuring those after Marcus has run answers a different question than the one the scope gate refuses on (#264) |
|
|
416
|
+
| verify | nothing else | The premise of another verify attempt is that the branch moved. Freezing the reading means no heal round can clear a refusal. Run `wf_c71f950f-c88` made four verify attempts with all six ACs PASS, every one judged against a single frozen reading, and reported SHIP_FAILED |
|
|
417
|
+
| ship | **nothing** | Between verify and ship the run commits, pushes, and may move the branch. Defensible one phase later, indefensible three (#280) |
|
|
418
|
+
|
|
419
|
+
Before #280 there was no ship branch at all, so the ship gate consumed
|
|
420
|
+
whatever block verify had left behind. On run `wf_ea3e428f-64f` that was a
|
|
421
|
+
reading an hour old, stamped `takenAt: "verify"`, that disagreed with the tree
|
|
422
|
+
by four of six ACs — the ship gate recorded a description of a tree it was not
|
|
423
|
+
gating. It still refused, and for the right reason; what it got wrong was
|
|
424
|
+
filing a stale artefact as current, which is precisely what `takenAt` was
|
|
425
|
+
added to make impossible to misread.
|
|
426
|
+
|
|
427
|
+
**The consumer enforces it too.** `prevalidationViolations` in
|
|
428
|
+
`gates/workflow.test.ts` refuses a reading at ship whose `takenAt` is anything
|
|
429
|
+
but `ship`, so a producer that stops re-measuring turns the gate RED rather
|
|
430
|
+
than quietly going stale again. There is no general "an earlier phase is
|
|
431
|
+
stale" rule — a `scope` reading surviving to verify is correct — so the check
|
|
432
|
+
is scoped to ship on purpose.
|
|
433
|
+
|
|
434
|
+
**The dry-run uses `EVIDENCE_CWD` when it is set**, which is the directory the
|
|
435
|
+
evidence collector uses, and which `workflows/ship.js` sets under worktree
|
|
436
|
+
isolation. A pre-validator looking somewhere other than where the evidence will
|
|
437
|
+
be collected calls the branch's own new files missing.
|
|
438
|
+
|
|
439
|
+
## Changelog Actor Values
|
|
440
|
+
|
|
441
|
+
Valid `actor` enum (lowercase only): `da`, `marcus`, `quinn`, `rook`, `gate-runner`
|
|
442
|
+
|
|
443
|
+
## Common Scope Gate Failures
|
|
444
|
+
|
|
445
|
+
| Check name | What it validates | Fix |
|
|
446
|
+
|---|---|---|
|
|
447
|
+
| `workflow-state.json validates` | Zod schema parse | Fix threshold value types, actor casing, enum values |
|
|
448
|
+
| `governing spec exists and is readable` | sourceSpecs[0].path resolves to a file | Use absolute path or path relative to projectRoot |
|
|
449
|
+
| `sourceSpecs-required` | STANDARD+ has sourceSpecs[] | Add sourceSpecs with referenced pattern files |
|
|
450
|
+
| `discovery-evidence` | sourceSpecs have `citedInDiscovery: true` | Set the flag on each sourceSpec |
|
|
451
|
+
| `spec-elements-in-acs` | ACs reference declared specElements | Add `specElement` field to ACs matching sourceSpec elements |
|
|
452
|
+
| `no 'DA should'` | No behavioral delegation in AC statements | Rewrite as measurable assertions |
|
|
453
|
+
| `no hardcoded paths` | No `/usr/`, `/opt/`, `/home/`, `/tmp/` in statements | Use relative descriptions, not absolute paths |
|
|
454
|
+
|
|
455
|
+
## Correct vs Incorrect AC
|
|
456
|
+
|
|
457
|
+
```json
|
|
458
|
+
// CORRECT
|
|
459
|
+
{
|
|
460
|
+
"id": "AC-1", "type": "CODE",
|
|
461
|
+
"statement": "sc-verification.md exists with pull_request trigger in frontmatter",
|
|
462
|
+
"threshold": { "op": "contains", "value": "pull_request", "unit": "trigger" },
|
|
463
|
+
"evidenceMethod": { "type": "grep", "command": "grep 'pull_request' .github/workflows/sc-verification.md" },
|
|
464
|
+
"specElement": "gh-aw workflow frontmatter schema pattern"
|
|
465
|
+
}
|
|
466
|
+
|
|
467
|
+
// INCORRECT — 4 problems
|
|
468
|
+
{
|
|
469
|
+
"id": "AC-1", "type": "CODE",
|
|
470
|
+
"statement": "file exists", // garbage pattern, < 5 words
|
|
471
|
+
"threshold": { "op": "exists", "value": true }, // boolean value, needs "true" string
|
|
472
|
+
"evidenceMethod": { "type": "check" }, // invalid type, use "command"
|
|
473
|
+
"specElement": "something" // doesn't match any sourceSpec specElement
|
|
474
|
+
}
|
|
475
|
+
```
|
|
476
|
+
|
|
477
|
+
## Writing workflow-state.json
|
|
478
|
+
|
|
479
|
+
Use **writeWorkflowState()** via `bun -e`, not the Write tool or Bash `cat >` / `echo >`:
|
|
480
|
+
|
|
481
|
+
```
|
|
482
|
+
bun -e "import {writeWorkflowState} from './gates/orchestrator.ts'; import {readFileSync} from 'fs'; const s = JSON.parse(readFileSync('WORK_DIR/workflow-state.json','utf8')); /* apply fix */; writeWorkflowState('WORK_DIR/workflow-state.json', s);"
|
|
483
|
+
```
|
|
484
|
+
|
|
485
|
+
This validates via Zod at write time with enum-specific error formatting. Invalid values produce immediate feedback (e.g., `acs.0.threshold.op: expected one of [==, >=, ...], got "equals"`) instead of opaque gate failures later.
|
|
@@ -0,0 +1,106 @@
|
|
|
1
|
+
import { test, expect, describe } from "bun:test";
|
|
2
|
+
import { ACSchema } from "./schema";
|
|
3
|
+
|
|
4
|
+
function makeAC(overrides: Record<string, any> = {}) {
|
|
5
|
+
return {
|
|
6
|
+
id: "AC-1",
|
|
7
|
+
statement: "Email body assembled from 7 template blocks with personalization tokens",
|
|
8
|
+
threshold: { op: ">=", value: 7 },
|
|
9
|
+
evidence: null,
|
|
10
|
+
...overrides,
|
|
11
|
+
};
|
|
12
|
+
}
|
|
13
|
+
|
|
14
|
+
function validate(ac: Record<string, any>) {
|
|
15
|
+
return ACSchema.safeParse(ac);
|
|
16
|
+
}
|
|
17
|
+
|
|
18
|
+
function issueMessages(result: ReturnType<typeof validate>): string[] {
|
|
19
|
+
if (result.success) return [];
|
|
20
|
+
return result.error.issues.map((i) => i.message);
|
|
21
|
+
}
|
|
22
|
+
|
|
23
|
+
describe("AC heuristic quality checks", () => {
|
|
24
|
+
test("garbage statement 'file exists' fails validation", () => {
|
|
25
|
+
const result = validate(makeAC({ statement: "file exists" }));
|
|
26
|
+
const msgs = issueMessages(result);
|
|
27
|
+
expect(msgs.some((m) => m.includes("tautological/garbage"))).toBe(true);
|
|
28
|
+
});
|
|
29
|
+
|
|
30
|
+
test("garbage statement 'it works' fails validation", () => {
|
|
31
|
+
const result = validate(makeAC({ statement: "it works" }));
|
|
32
|
+
const msgs = issueMessages(result);
|
|
33
|
+
expect(msgs.some((m) => m.includes("tautological/garbage"))).toBe(true);
|
|
34
|
+
});
|
|
35
|
+
|
|
36
|
+
test("garbage statement 'tests pass' fails validation", () => {
|
|
37
|
+
const result = validate(makeAC({ statement: "tests pass" }));
|
|
38
|
+
const msgs = issueMessages(result);
|
|
39
|
+
expect(msgs.some((m) => m.includes("tautological/garbage"))).toBe(true);
|
|
40
|
+
});
|
|
41
|
+
|
|
42
|
+
test("garbage statement 'code is correct' fails validation", () => {
|
|
43
|
+
const result = validate(makeAC({ statement: "code is correct" }));
|
|
44
|
+
const msgs = issueMessages(result);
|
|
45
|
+
expect(msgs.some((m) => m.includes("tautological/garbage"))).toBe(true);
|
|
46
|
+
});
|
|
47
|
+
|
|
48
|
+
test("garbage statement 'no errors' fails validation", () => {
|
|
49
|
+
const result = validate(makeAC({ statement: "no errors" }));
|
|
50
|
+
const msgs = issueMessages(result);
|
|
51
|
+
expect(msgs.some((m) => m.includes("tautological/garbage"))).toBe(true);
|
|
52
|
+
});
|
|
53
|
+
|
|
54
|
+
test("garbage statement 'changes are made' fails validation", () => {
|
|
55
|
+
const result = validate(makeAC({ statement: "changes are made" }));
|
|
56
|
+
const msgs = issueMessages(result);
|
|
57
|
+
expect(msgs.some((m) => m.includes("tautological/garbage"))).toBe(true);
|
|
58
|
+
});
|
|
59
|
+
|
|
60
|
+
test("weak threshold >= 1 fails validation", () => {
|
|
61
|
+
const result = validate(makeAC({ threshold: { op: ">=", value: 1 } }));
|
|
62
|
+
const msgs = issueMessages(result);
|
|
63
|
+
expect(msgs.some((m) => m.includes("threshold too weak"))).toBe(true);
|
|
64
|
+
});
|
|
65
|
+
|
|
66
|
+
test("weak threshold >= 0 fails validation", () => {
|
|
67
|
+
const result = validate(makeAC({ threshold: { op: ">=", value: 0 } }));
|
|
68
|
+
const msgs = issueMessages(result);
|
|
69
|
+
expect(msgs.some((m) => m.includes("threshold too weak"))).toBe(true);
|
|
70
|
+
});
|
|
71
|
+
|
|
72
|
+
test("valid statement with threshold >= 7 passes", () => {
|
|
73
|
+
const result = validate(makeAC({
|
|
74
|
+
statement: "Email body assembled from 7 blocks",
|
|
75
|
+
threshold: { op: ">=", value: 7 },
|
|
76
|
+
}));
|
|
77
|
+
const msgs = issueMessages(result);
|
|
78
|
+
expect(msgs.some((m) => m.includes("tautological/garbage"))).toBe(false);
|
|
79
|
+
expect(msgs.some((m) => m.includes("threshold too weak"))).toBe(false);
|
|
80
|
+
expect(msgs.some((m) => m.includes("statement too short"))).toBe(false);
|
|
81
|
+
});
|
|
82
|
+
|
|
83
|
+
test("short statement 'fix bug' fails validation (< 5 words)", () => {
|
|
84
|
+
const result = validate(makeAC({ statement: "fix bug" }));
|
|
85
|
+
const msgs = issueMessages(result);
|
|
86
|
+
expect(msgs.some((m) => m.includes("statement too short"))).toBe(true);
|
|
87
|
+
});
|
|
88
|
+
|
|
89
|
+
test("short statement 'add test' fails validation (< 5 words)", () => {
|
|
90
|
+
const result = validate(makeAC({ statement: "add test" }));
|
|
91
|
+
const msgs = issueMessages(result);
|
|
92
|
+
expect(msgs.some((m) => m.includes("statement too short"))).toBe(true);
|
|
93
|
+
});
|
|
94
|
+
|
|
95
|
+
test("threshold >= 5 does NOT trigger weak threshold", () => {
|
|
96
|
+
const result = validate(makeAC({ threshold: { op: ">=", value: 5 } }));
|
|
97
|
+
const msgs = issueMessages(result);
|
|
98
|
+
expect(msgs.some((m) => m.includes("threshold too weak"))).toBe(false);
|
|
99
|
+
});
|
|
100
|
+
|
|
101
|
+
test("threshold == 1 does NOT trigger weak threshold (only >= is checked)", () => {
|
|
102
|
+
const result = validate(makeAC({ threshold: { op: "==", value: 1 } }));
|
|
103
|
+
const msgs = issueMessages(result);
|
|
104
|
+
expect(msgs.some((m) => m.includes("threshold too weak"))).toBe(false);
|
|
105
|
+
});
|
|
106
|
+
});
|