rungate 0.6.0 → 0.7.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/lib/affected-tests.ts +398 -0
- package/lib/agent-audit.ts +2 -3
- package/lib/agent-worktree.ts +287 -0
- package/lib/agents-md-sections.ts +81 -16
- package/lib/behavioral-cache.ts +150 -2
- package/lib/branch-protection.ts +790 -0
- package/lib/brief-ac-alignment.ts +104 -0
- package/lib/brief-references.ts +205 -0
- package/lib/bun-version.ts +93 -14
- package/lib/check-remedy.ts +140 -0
- package/lib/ci-checkout.ts +17 -0
- package/lib/ci-history-depth.ts +159 -0
- package/lib/ci-setup-step.ts +96 -0
- package/lib/claude-code-action.ts +85 -0
- package/lib/claude-tool-vocabulary.ts +188 -0
- package/lib/cli-commands.ts +15 -0
- package/lib/commit-sha.ts +42 -0
- package/lib/commit-subject.ts +90 -0
- package/lib/compliance-report.ts +377 -45
- package/lib/compliance.ts +191 -17
- package/lib/config-loader.ts +39 -1
- package/lib/config-trackable.ts +391 -0
- package/lib/conformity.ts +1087 -100
- package/lib/containment.ts +238 -0
- package/lib/content-hash.ts +24 -0
- package/lib/create-sc.ts +249 -16
- package/lib/dev-liveness.ts +180 -0
- package/lib/discovery-branch-points.ts +432 -0
- package/lib/discovery-stability.ts +481 -0
- package/lib/distribution-currency.ts +186 -0
- package/lib/doc-routing.ts +157 -0
- package/lib/evidence-prevalidator.ts +224 -2
- package/lib/evidence-satisfiability.ts +534 -12
- package/lib/failure-intake.ts +317 -0
- package/lib/fallow-availability.ts +157 -0
- package/lib/fs-write-capabilities.ts +232 -0
- package/lib/gate-run-split.ts +147 -0
- package/lib/generators/agents-md.ts +140 -32
- package/lib/git-isolation.ts +140 -0
- package/lib/github.ts +15 -0
- package/lib/google-auth-action.ts +93 -0
- package/lib/harness-project.ts +48 -0
- package/lib/harness-provenance.ts +255 -0
- package/lib/hook-evidence.ts +235 -0
- package/lib/hook-scopes.ts +386 -0
- package/lib/hook-target-tree.ts +398 -0
- package/lib/hook-trees.ts +447 -0
- package/lib/js-top-level-code.ts +78 -10
- package/lib/junit-checks.ts +52 -0
- package/lib/launch-base.ts +253 -0
- package/lib/line-citation.ts +372 -0
- package/lib/orphan-reaper.ts +468 -0
- package/lib/parity-binding.ts +205 -0
- package/lib/per-test-timeouts.ts +219 -0
- package/lib/plan-coverage.ts +151 -0
- package/lib/prior-branch.ts +201 -8
- package/lib/process-table.ts +146 -0
- package/lib/project-state-derivation.ts +219 -0
- package/lib/project-state-root.ts +258 -0
- package/lib/published-artifact.ts +525 -0
- package/lib/readdir-sorted.ts +45 -0
- package/lib/review-execution-reading.ts +206 -0
- package/lib/review-tool-agreement.ts +433 -0
- package/lib/rule-promotion.ts +234 -0
- package/lib/rule-retirement.ts +525 -0
- package/lib/run-branch.ts +612 -0
- package/lib/rungate-schema.ts +19 -0
- package/lib/runner-home.ts +274 -0
- package/lib/sc-id-cross-tree.ts +19 -7
- package/lib/sc-ids.ts +121 -13
- package/lib/sc-reservations.ts +289 -0
- package/lib/scaffold/ai-review-findings.ts +403 -0
- package/lib/scaffold/ai-review-step.ts +4661 -64
- package/lib/scaffold/managed-workflow.ts +587 -192
- package/lib/scaffold/owned-files.ts +152 -45
- package/lib/scaffold/pre-push-hook.ts +49 -5
- package/lib/scaffold/steps.ts +1413 -244
- package/lib/scaffold/workflow-census.ts +217 -0
- package/lib/scaffold/workflow-drift.ts +217 -0
- package/lib/scaffold/write-steps.ts +4 -0
- package/lib/scan-budget.ts +247 -0
- package/lib/scanner.ts +43 -51
- package/lib/scratch-roots.ts +50 -5
- package/lib/screenshot-discovery.ts +121 -0
- package/lib/screenshot-upload.ts +41 -1
- package/lib/secret-patterns.ts +511 -8
- package/lib/ship-mode.ts +219 -0
- package/lib/source-slice.ts +294 -0
- package/lib/stale-cleanup.ts +37 -3
- package/lib/suite-flake.ts +278 -0
- package/lib/suite-measurement.ts +23 -4
- package/lib/suite-run-reading.ts +241 -1
- package/lib/suite-timeout-margin.ts +193 -0
- package/lib/test-case-bodies.ts +883 -0
- package/lib/test-case-deletion.ts +653 -0
- package/lib/test-census.ts +97 -29
- package/lib/test-command-positions.ts +274 -0
- package/lib/test-suite-lock.ts +159 -16
- package/lib/tool-probe.ts +31 -9
- package/lib/top-level-cd.ts +393 -0
- package/lib/tree-entries.ts +89 -0
- package/lib/ts-ast.ts +200 -0
- package/lib/walk-test-files.ts +97 -0
- package/lib/workflow-actions.ts +263 -0
- package/lib/worktree-collect.ts +177 -5
- package/lib/worktree-repo.ts +36 -1
- package/package.json +2 -1
- package/scripts/ai-review-findings.ts +45 -0
- package/scripts/create-sc.ts +21 -1
- package/scripts/extract-constraints.ts +107 -7
- package/scripts/generate-code-map.ts +23 -30
- package/scripts/scaffold-project.ts +8 -1
- package/scripts/sync-sc-status.ts +77 -16
|
@@ -0,0 +1,481 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* What "the same answer twice" means for DISCOVERY's model output (#523).
|
|
3
|
+
*
|
|
4
|
+
* #61 bundled two questions and only answered one. `lib/workflow-test-harness.ts`
|
|
5
|
+
* proves the ORCHESTRATION is deterministic — same call graph, k=5, in
|
|
6
|
+
* milliseconds, every suite run. This module is the other half: given the same
|
|
7
|
+
* issue, does the model produce the same *decisions*?
|
|
8
|
+
*
|
|
9
|
+
* ## The equality this asserts, and the one it refuses to
|
|
10
|
+
*
|
|
11
|
+
* **Byte-identical model output is not the property.** Asserting it produces a
|
|
12
|
+
* check that reds on wording and is disabled inside a week, which is strictly
|
|
13
|
+
* worse than no check — see `.claude/rules/checks-must-be-able-to-fail.md`. So
|
|
14
|
+
* this compares a NORMALISED PROJECTION, and the projection is chosen by one
|
|
15
|
+
* criterion rather than by a hand-written list:
|
|
16
|
+
*
|
|
17
|
+
* **A discovery field is required to be stable exactly when something
|
|
18
|
+
* downstream BRANCHES on it.**
|
|
19
|
+
*
|
|
20
|
+
* Variance in a field nothing reads is prose. Variance in a field a gate reads
|
|
21
|
+
* means two runs of the same issue took different code paths — the harness
|
|
22
|
+
* making a different decision about one input, which is the thing worth
|
|
23
|
+
* measuring whatever it is called this month.
|
|
24
|
+
*
|
|
25
|
+
* Every entry in `STABLE_DISCOVERY_FIELDS` therefore carries `branchedOnBy`:
|
|
26
|
+
* the consumer that makes it load-bearing. An entry with no branch consumer
|
|
27
|
+
* does not belong here, and `NOT_ASSERTED` records what is deliberately
|
|
28
|
+
* excluded so the silence is legible rather than accidental.
|
|
29
|
+
*
|
|
30
|
+
* ## What this is NOT
|
|
31
|
+
*
|
|
32
|
+
* Not a gate. The issue is explicit that a rerun next week measures a
|
|
33
|
+
* different model-day, so a red row cannot be distinguished from drift; the
|
|
34
|
+
* output is a dated MEASUREMENT. Nothing here is wired into a CI gate and
|
|
35
|
+
* nothing should be. The CI-runnable part is the comparator's own behaviour,
|
|
36
|
+
* tested against synthetic states — no model call, no flake.
|
|
37
|
+
*
|
|
38
|
+
* ## Why the refusals are the load-bearing part
|
|
39
|
+
*
|
|
40
|
+
* A comparator over k run states has three ways to report perfect stability
|
|
41
|
+
* while having measured nothing, and all three have happened to instruments in
|
|
42
|
+
* this repository:
|
|
43
|
+
*
|
|
44
|
+
* 1. **k < 2.** One run cannot vary. Reported as REFUSED, not as stable.
|
|
45
|
+
* 2. **The runs are not the same input.** Measured on 2026-10-21 against the
|
|
46
|
+
* 44 recorded run states in `~/.rungate`: exactly one issue (agentgrit
|
|
47
|
+
* #258) appears more than once, and all three of its runs carry a
|
|
48
|
+
* DIFFERENT `issueGoal` — they were re-dispatched with new instructions
|
|
49
|
+
* after a failure. Comparing them reports wild model instability when what
|
|
50
|
+
* actually varied was the prompt. `seriesInputIdentity` refuses that
|
|
51
|
+
* series rather than reporting its variance.
|
|
52
|
+
* 3. **The projection read nothing.** A path that resolves to `undefined` in
|
|
53
|
+
* every run is identical in every run. `unobserved` names those, and a
|
|
54
|
+
* series where every declared field is unobserved is REFUSED.
|
|
55
|
+
*
|
|
56
|
+
* The first two are the reason this file exists at all rather than a `diff`.
|
|
57
|
+
*/
|
|
58
|
+
|
|
59
|
+
/** How two readings of one field are compared. */
|
|
60
|
+
export type Equality =
|
|
61
|
+
/** Same string. For enums and paths a gate switches on. */
|
|
62
|
+
| "exact"
|
|
63
|
+
/** Same multiset of strings, order ignored. Discovery does not promise an order. */
|
|
64
|
+
| "set"
|
|
65
|
+
/** Same number of elements. The values are not compared. */
|
|
66
|
+
| "count";
|
|
67
|
+
|
|
68
|
+
export interface StableField {
|
|
69
|
+
/**
|
|
70
|
+
* Dotted path into the workflow state. `[]` marks an array the remaining
|
|
71
|
+
* segments are read from on every element — `acs[].id` reads `id` off each
|
|
72
|
+
* entry of `acs`.
|
|
73
|
+
*/
|
|
74
|
+
path: string;
|
|
75
|
+
equality: Equality;
|
|
76
|
+
/**
|
|
77
|
+
* The consumer that BRANCHES on this field. This is the justification for
|
|
78
|
+
* the entry, not a note about it: an entry whose consumer does not branch is
|
|
79
|
+
* asserting stability on prose.
|
|
80
|
+
*/
|
|
81
|
+
branchedOnBy: string;
|
|
82
|
+
/**
|
|
83
|
+
* The properties of the DISCOVERY object in `workflows/ship.js` that this
|
|
84
|
+
* state path is written from — `sizing.ceremonyTier` in the state file is
|
|
85
|
+
* `discovery.ceremonyTier` in the workflow, because ship.js flattens one
|
|
86
|
+
* into the other when it writes the state.
|
|
87
|
+
*
|
|
88
|
+
* This is not documentation. `test/discovery-branch-coverage.test.ts` parses
|
|
89
|
+
* the workflow, collects every discovery property a CONDITION reads, and
|
|
90
|
+
* reddens on any name this mapping does not cover — which is how a branch
|
|
91
|
+
* added tomorrow on a field nobody declared stable gets caught, instead of
|
|
92
|
+
* the measurement quietly going on reporting "0 varied" over the wrong set.
|
|
93
|
+
* An entry with no `readAs` covers no branch.
|
|
94
|
+
*/
|
|
95
|
+
readAs?: readonly string[];
|
|
96
|
+
}
|
|
97
|
+
|
|
98
|
+
/** An exclusion, with the reason it is one. */
|
|
99
|
+
export interface NotAssertedField {
|
|
100
|
+
path: string;
|
|
101
|
+
reason: string;
|
|
102
|
+
/** As `StableField.readAs` — the workflow properties this excuse covers. */
|
|
103
|
+
readAs?: readonly string[];
|
|
104
|
+
}
|
|
105
|
+
|
|
106
|
+
/**
|
|
107
|
+
* The fields a second run of the same issue must agree with the first on.
|
|
108
|
+
*
|
|
109
|
+
* Derived from the branch-point criterion above, not enumerated by taste. Each
|
|
110
|
+
* path is checked against the real `WorkflowStateSchema` by
|
|
111
|
+
* `test/discovery-stability.test.ts` — against the zod object that EMITS the
|
|
112
|
+
* state, never against this list — so a renamed or deleted field reddens here
|
|
113
|
+
* instead of silently becoming an unobservable field that is stable because it
|
|
114
|
+
* is absent.
|
|
115
|
+
*/
|
|
116
|
+
export const STABLE_DISCOVERY_FIELDS: StableField[] = [
|
|
117
|
+
{
|
|
118
|
+
path: "sizing.ceremonyTier",
|
|
119
|
+
equality: "exact",
|
|
120
|
+
branchedOnBy:
|
|
121
|
+
"the scope gate, Quinn local, container verify and prove each run or do not run on it (#215, #274). Variance here is two different pipelines.",
|
|
122
|
+
readAs: ["ceremonyTier"],
|
|
123
|
+
},
|
|
124
|
+
{
|
|
125
|
+
path: "sizing.predicted",
|
|
126
|
+
equality: "exact",
|
|
127
|
+
branchedOnBy:
|
|
128
|
+
"feeds the ceremony tier floor (#215). The value is already a bucket label (XS/S/M/L), so exact equality on the label IS bucket equality — there is no finer reading to collapse.",
|
|
129
|
+
readAs: ["sizing"],
|
|
130
|
+
},
|
|
131
|
+
{
|
|
132
|
+
path: "acs[]",
|
|
133
|
+
equality: "count",
|
|
134
|
+
branchedOnBy:
|
|
135
|
+
"prove verifies `criteriaResults` count against the SC count (#384). A varying count is a varying definition of done.",
|
|
136
|
+
readAs: ["acs"],
|
|
137
|
+
},
|
|
138
|
+
{
|
|
139
|
+
path: "acs[].id",
|
|
140
|
+
equality: "set",
|
|
141
|
+
branchedOnBy:
|
|
142
|
+
"every per-AC result is keyed by id (#384). Compared as a set because discovery promises no ordering — the three recorded #258 runs emit AC-1..AC-6 in three different orders.",
|
|
143
|
+
readAs: ["acs"],
|
|
144
|
+
},
|
|
145
|
+
{
|
|
146
|
+
path: "acs[].evidenceMethod.type",
|
|
147
|
+
equality: "set",
|
|
148
|
+
branchedOnBy:
|
|
149
|
+
"selects how the evidence is obtained, and the evidence-type ratio check reads it (#235).",
|
|
150
|
+
readAs: ["acs"],
|
|
151
|
+
},
|
|
152
|
+
{
|
|
153
|
+
path: "acs[].evidenceMethod.authoredCommand??command",
|
|
154
|
+
equality: "set",
|
|
155
|
+
branchedOnBy:
|
|
156
|
+
"the scope gate dry-runs every one of them (#235). A command that varies is a gate that varies. `authoredCommand` FIRST and `command` only as the fallback: #614 records that three sites rewrite `command` after the model emits it (discovery's path normaliser, its evidence-type-ratio upgrade, and the git-ref qualification at the execution boundary), and those rewrites are deterministic harness behaviour. Reading `command` would attribute the normaliser's output to the model.",
|
|
157
|
+
readAs: ["acs"],
|
|
158
|
+
},
|
|
159
|
+
{
|
|
160
|
+
path: "governingSpec.path",
|
|
161
|
+
equality: "exact",
|
|
162
|
+
branchedOnBy:
|
|
163
|
+
"selects which rules govern the run. Two runs reading different specs are not two runs of the same thing.",
|
|
164
|
+
readAs: ["governingSpec"],
|
|
165
|
+
},
|
|
166
|
+
{
|
|
167
|
+
path: "sourceSpecs[].path",
|
|
168
|
+
equality: "set",
|
|
169
|
+
branchedOnBy: "same as `governingSpec.path` — the set of rules in force.",
|
|
170
|
+
readAs: ["sourceSpecs"],
|
|
171
|
+
},
|
|
172
|
+
];
|
|
173
|
+
|
|
174
|
+
/**
|
|
175
|
+
* Fields deliberately NOT asserted, with the reason.
|
|
176
|
+
*
|
|
177
|
+
* Written down because an unstated exclusion is indistinguishable from an
|
|
178
|
+
* oversight, and the next reader otherwise has to re-derive why the obvious
|
|
179
|
+
* field is missing.
|
|
180
|
+
*/
|
|
181
|
+
export const NOT_ASSERTED: NotAssertedField[] = [
|
|
182
|
+
{
|
|
183
|
+
path: "acs[].statement",
|
|
184
|
+
reason:
|
|
185
|
+
"prose. Wording is what a language model is for, and asserting it is the flaky-test failure mode this module exists to avoid.",
|
|
186
|
+
},
|
|
187
|
+
{
|
|
188
|
+
path: "issueGoal",
|
|
189
|
+
reason:
|
|
190
|
+
"NOT A MODEL OUTPUT in the normal path. The ship skill pre-computes it from the issue body via `gh issue view` and hands it in as `goalData`, which makes the workflow skip the extraction agent entirely. A series that leaves `goalData` populated is measuring parsing. It is used here as the series' INPUT IDENTITY instead — see `seriesInputIdentity`.",
|
|
191
|
+
},
|
|
192
|
+
{
|
|
193
|
+
path: "successCriteria",
|
|
194
|
+
reason: "same as `issueGoal` — pre-computed from the issue body, not emitted by the model.",
|
|
195
|
+
},
|
|
196
|
+
{
|
|
197
|
+
path: "acs[].threshold.op",
|
|
198
|
+
reason:
|
|
199
|
+
"a real candidate, deliberately deferred. The evidence pre-validation gate does compare evidence against the operator, so it plausibly meets the branch criterion — but it is not added on plausibility. Adding it needs the branch point read and cited the way every entry above is.",
|
|
200
|
+
},
|
|
201
|
+
{
|
|
202
|
+
path: "priorWork",
|
|
203
|
+
readAs: ["priorWork"],
|
|
204
|
+
reason:
|
|
205
|
+
"BRANCHED ON — `workflows/ship.js:3797` short-circuits the whole run on it, and its `acStatus` then selects which ACs survive into BUILD — and excluded anyway, which is why the reason has to be good. It is not a reading of the ISSUE, it is a reading of the REPOSITORY: whether a branch or PR for this issue already exists. Two runs of the same issue are therefore SUPPOSED to disagree here, because the first run creates the branch the second one finds. Asserting stability would red on the harness working correctly, which is the exact failure mode `.claude/rules/checks-must-be-able-to-fail.md` calls a check that cannot fail for the reason it claims to — inverted.",
|
|
206
|
+
},
|
|
207
|
+
{
|
|
208
|
+
path: "acs[].verdict, acs[].evidence, gates.*, agents.*",
|
|
209
|
+
reason: "not discovery output at all — written later in the run by the phases that produce them.",
|
|
210
|
+
},
|
|
211
|
+
];
|
|
212
|
+
|
|
213
|
+
/** One run's recorded state, labelled so a report can name which run varied. */
|
|
214
|
+
export interface RunState {
|
|
215
|
+
label: string;
|
|
216
|
+
state: unknown;
|
|
217
|
+
}
|
|
218
|
+
|
|
219
|
+
export interface FieldReading {
|
|
220
|
+
path: string;
|
|
221
|
+
equality: Equality;
|
|
222
|
+
/** One normalised reading per run, in the order the runs were given. */
|
|
223
|
+
readings: { label: string; value: string }[];
|
|
224
|
+
/** True when every run read the same thing under this field's equality. */
|
|
225
|
+
stable: boolean;
|
|
226
|
+
/** True when NO run produced a reading — identical, and meaningless. */
|
|
227
|
+
unobserved: boolean;
|
|
228
|
+
}
|
|
229
|
+
|
|
230
|
+
export interface SeriesIdentity {
|
|
231
|
+
identical: boolean;
|
|
232
|
+
/** A short digest of each run's input, so a reader can see WHICH run differs. */
|
|
233
|
+
fingerprints: { label: string; issue: string; goalDigest: string }[];
|
|
234
|
+
}
|
|
235
|
+
|
|
236
|
+
export type StabilityReport =
|
|
237
|
+
| {
|
|
238
|
+
verdict: "REFUSED";
|
|
239
|
+
/** Why nothing was measured. Never a pass. */
|
|
240
|
+
reason: string;
|
|
241
|
+
identity?: SeriesIdentity;
|
|
242
|
+
}
|
|
243
|
+
| {
|
|
244
|
+
verdict: "MEASURED";
|
|
245
|
+
runs: string[];
|
|
246
|
+
identity: SeriesIdentity;
|
|
247
|
+
fields: FieldReading[];
|
|
248
|
+
/** Fields that varied. The finding. */
|
|
249
|
+
varied: FieldReading[];
|
|
250
|
+
/** Fields no run produced a value for. Not evidence of stability. */
|
|
251
|
+
unobserved: FieldReading[];
|
|
252
|
+
};
|
|
253
|
+
|
|
254
|
+
/* ─────────────────────────── reading the state ─────────────────────────── */
|
|
255
|
+
|
|
256
|
+
function isRecord(v: unknown): v is Record<string, unknown> {
|
|
257
|
+
return typeof v === "object" && v !== null && !Array.isArray(v);
|
|
258
|
+
}
|
|
259
|
+
|
|
260
|
+
/**
|
|
261
|
+
* Collapse a value to the string the comparison uses.
|
|
262
|
+
*
|
|
263
|
+
* Whitespace is collapsed and the result trimmed: an evidence command
|
|
264
|
+
* reflowed across two lines is the same command, and treating it as a
|
|
265
|
+
* difference is the noise that gets a check disabled.
|
|
266
|
+
*/
|
|
267
|
+
function normalise(v: unknown): string | null {
|
|
268
|
+
if (v === undefined || v === null) return null;
|
|
269
|
+
if (typeof v === "string") {
|
|
270
|
+
const t = v.replace(/\s+/g, " ").trim();
|
|
271
|
+
return t === "" ? null : t;
|
|
272
|
+
}
|
|
273
|
+
if (typeof v === "number" || typeof v === "boolean") return String(v);
|
|
274
|
+
return null;
|
|
275
|
+
}
|
|
276
|
+
|
|
277
|
+
/**
|
|
278
|
+
* Resolve a declared path to the raw nodes it reaches, before normalisation.
|
|
279
|
+
*
|
|
280
|
+
* Kept separate from `readPath` because `count` has to count what the path
|
|
281
|
+
* REACHED, not what survived normalisation: `acs[]` reaches two AC objects,
|
|
282
|
+
* and an object normalises to nothing, so counting after normalisation
|
|
283
|
+
* reported every run as having zero ACs — identical, and therefore stable,
|
|
284
|
+
* for the one field whose whole point is that a varying count is a varying
|
|
285
|
+
* definition of done.
|
|
286
|
+
*/
|
|
287
|
+
function resolveNodes(state: unknown, path: string): unknown[] {
|
|
288
|
+
const segments = path.split(".");
|
|
289
|
+
let frontier: unknown[] = [state];
|
|
290
|
+
|
|
291
|
+
for (const rawSeg of segments) {
|
|
292
|
+
const isArray = rawSeg.endsWith("[]");
|
|
293
|
+
const seg = isArray ? rawSeg.slice(0, -2) : rawSeg;
|
|
294
|
+
const alternatives = seg.split("??");
|
|
295
|
+
const next: unknown[] = [];
|
|
296
|
+
|
|
297
|
+
for (const node of frontier) {
|
|
298
|
+
if (!isRecord(node)) continue;
|
|
299
|
+
let picked: unknown = undefined;
|
|
300
|
+
for (const name of alternatives) {
|
|
301
|
+
if (node[name] !== undefined && node[name] !== null) {
|
|
302
|
+
picked = node[name];
|
|
303
|
+
break;
|
|
304
|
+
}
|
|
305
|
+
}
|
|
306
|
+
if (picked === undefined) continue;
|
|
307
|
+
if (isArray) {
|
|
308
|
+
if (Array.isArray(picked)) next.push(...picked);
|
|
309
|
+
} else {
|
|
310
|
+
next.push(picked);
|
|
311
|
+
}
|
|
312
|
+
}
|
|
313
|
+
frontier = next;
|
|
314
|
+
}
|
|
315
|
+
|
|
316
|
+
return frontier;
|
|
317
|
+
}
|
|
318
|
+
|
|
319
|
+
/**
|
|
320
|
+
* The normalised readings one declared path produced — one entry for a scalar
|
|
321
|
+
* path, one per array element for a `[]` path, and `[]` when the path resolves
|
|
322
|
+
* to nothing. `a??b` as a segment reads `a` and falls back to `b`.
|
|
323
|
+
*/
|
|
324
|
+
export function readPath(state: unknown, path: string): string[] {
|
|
325
|
+
return resolveNodes(state, path)
|
|
326
|
+
.map(normalise)
|
|
327
|
+
.filter((s): s is string => s !== null);
|
|
328
|
+
}
|
|
329
|
+
|
|
330
|
+
/**
|
|
331
|
+
* The reading one field contributes for one run, already reduced by its
|
|
332
|
+
* equality kind to the single string the comparison tests.
|
|
333
|
+
*/
|
|
334
|
+
function reduce(nodes: unknown[], equality: Equality): string | null {
|
|
335
|
+
if (equality === "count") return String(nodes.length);
|
|
336
|
+
const readings = nodes.map(normalise).filter((s): s is string => s !== null);
|
|
337
|
+
if (readings.length === 0) return null;
|
|
338
|
+
if (equality === "set") return [...readings].sort().join(" | ");
|
|
339
|
+
// "exact": a scalar path. More than one reading means the path was written
|
|
340
|
+
// for an array and declared as a scalar — report it rather than pick one.
|
|
341
|
+
return readings.length === 1 ? readings[0] : `AMBIGUOUS(${readings.length}): ${readings.join(" | ")}`;
|
|
342
|
+
}
|
|
343
|
+
|
|
344
|
+
/* ───────────────────────────── input identity ──────────────────────────── */
|
|
345
|
+
|
|
346
|
+
/** A short, stable digest. Not cryptographic — it only has to separate inputs. */
|
|
347
|
+
function digest(s: string): string {
|
|
348
|
+
let h1 = 0x811c9dc5;
|
|
349
|
+
let h2 = 0x01000193;
|
|
350
|
+
for (let i = 0; i < s.length; i++) {
|
|
351
|
+
h1 = Math.imul(h1 ^ s.charCodeAt(i), 0x01000193) >>> 0;
|
|
352
|
+
h2 = Math.imul(h2 + s.charCodeAt(i), 0x85ebca6b) >>> 0;
|
|
353
|
+
}
|
|
354
|
+
return (h1 >>> 0).toString(16).padStart(8, "0") + (h2 >>> 0).toString(16).padStart(8, "0");
|
|
355
|
+
}
|
|
356
|
+
|
|
357
|
+
/**
|
|
358
|
+
* Whether these runs are runs of THE SAME THING.
|
|
359
|
+
*
|
|
360
|
+
* This is the control the measurement needs and the one a plain diff has no
|
|
361
|
+
* way to express. The three recorded runs of agentgrit #258 in `~/.rungate`
|
|
362
|
+
* disagree on `sizing.predicted` three ways (M, XS, S) — which reads as severe
|
|
363
|
+
* model instability until you notice their `issueGoal` values are three
|
|
364
|
+
* different sets of instructions. They are one run each of three inputs, not
|
|
365
|
+
* three runs of one input.
|
|
366
|
+
*/
|
|
367
|
+
export function seriesInputIdentity(runs: RunState[]): SeriesIdentity {
|
|
368
|
+
const fingerprints = runs.map(r => {
|
|
369
|
+
const s = isRecord(r.state) ? r.state : {};
|
|
370
|
+
return {
|
|
371
|
+
label: r.label,
|
|
372
|
+
issue: String(s.issue ?? "?"),
|
|
373
|
+
goalDigest: digest(normalise(s.issueGoal) ?? ""),
|
|
374
|
+
};
|
|
375
|
+
});
|
|
376
|
+
const key = (f: { issue: string; goalDigest: string }) => `${f.issue}/${f.goalDigest}`;
|
|
377
|
+
const identical = fingerprints.length > 0 && fingerprints.every(f => key(f) === key(fingerprints[0]));
|
|
378
|
+
return { identical, fingerprints };
|
|
379
|
+
}
|
|
380
|
+
|
|
381
|
+
/* ──────────────────────────── the comparison ───────────────────────────── */
|
|
382
|
+
|
|
383
|
+
export function compareDiscoveryRuns(
|
|
384
|
+
runs: RunState[],
|
|
385
|
+
fields: StableField[] = STABLE_DISCOVERY_FIELDS,
|
|
386
|
+
): StabilityReport {
|
|
387
|
+
if (runs.length < 2) {
|
|
388
|
+
return {
|
|
389
|
+
verdict: "REFUSED",
|
|
390
|
+
reason: `a series of ${runs.length} cannot vary — k must be at least 2. One run reporting "stable" is the vacuous pass this comparator exists to refuse.`,
|
|
391
|
+
};
|
|
392
|
+
}
|
|
393
|
+
|
|
394
|
+
const identity = seriesInputIdentity(runs);
|
|
395
|
+
if (!identity.identical) {
|
|
396
|
+
const shown = identity.fingerprints
|
|
397
|
+
.map(f => `${f.label}: issue=${f.issue} goal=${f.goalDigest}`)
|
|
398
|
+
.join("; ");
|
|
399
|
+
return {
|
|
400
|
+
verdict: "REFUSED",
|
|
401
|
+
reason:
|
|
402
|
+
`these runs do not share an input, so their differences measure the PROMPT, not the model. ` +
|
|
403
|
+
`Re-dispatch the series with one fixed issue body and no re-instruction between runs. ` +
|
|
404
|
+
`Fingerprints — ${shown}`,
|
|
405
|
+
identity,
|
|
406
|
+
};
|
|
407
|
+
}
|
|
408
|
+
|
|
409
|
+
if (fields.length === 0) {
|
|
410
|
+
return {
|
|
411
|
+
verdict: "REFUSED",
|
|
412
|
+
reason: "no fields declared stable — a comparison over zero fields is stable over anything.",
|
|
413
|
+
identity,
|
|
414
|
+
};
|
|
415
|
+
}
|
|
416
|
+
|
|
417
|
+
const readings: FieldReading[] = fields.map(f => {
|
|
418
|
+
const perRun = runs.map(r => ({
|
|
419
|
+
label: r.label,
|
|
420
|
+
value: reduce(resolveNodes(r.state, f.path), f.equality) ?? "«absent»",
|
|
421
|
+
}));
|
|
422
|
+
// Nothing was compared when no run produced a value — and for a `count`
|
|
423
|
+
// that means a count of zero everywhere, which is "no ACs in any run",
|
|
424
|
+
// not "every run agreed on the number of ACs".
|
|
425
|
+
const nothing = f.equality === "count" ? "0" : "«absent»";
|
|
426
|
+
const unobserved = perRun.every(p => p.value === nothing);
|
|
427
|
+
const stable = perRun.every(p => p.value === perRun[0].value);
|
|
428
|
+
return { path: f.path, equality: f.equality, readings: perRun, stable, unobserved };
|
|
429
|
+
});
|
|
430
|
+
|
|
431
|
+
const observable = readings.filter(r => !r.unobserved);
|
|
432
|
+
if (observable.length === 0) {
|
|
433
|
+
return {
|
|
434
|
+
verdict: "REFUSED",
|
|
435
|
+
reason:
|
|
436
|
+
`not one of the ${fields.length} declared fields produced a value in any run. ` +
|
|
437
|
+
`A projection that reads nothing reports perfect stability, which is why this is a refusal and not a pass.`,
|
|
438
|
+
identity,
|
|
439
|
+
};
|
|
440
|
+
}
|
|
441
|
+
|
|
442
|
+
return {
|
|
443
|
+
verdict: "MEASURED",
|
|
444
|
+
runs: runs.map(r => r.label),
|
|
445
|
+
identity,
|
|
446
|
+
fields: readings,
|
|
447
|
+
varied: readings.filter(r => !r.stable && !r.unobserved),
|
|
448
|
+
unobserved: readings.filter(r => r.unobserved),
|
|
449
|
+
};
|
|
450
|
+
}
|
|
451
|
+
|
|
452
|
+
/** The dated measurement, as text. Never a verdict a gate consumes. */
|
|
453
|
+
export function formatStabilityReport(report: StabilityReport, measuredOn: string): string {
|
|
454
|
+
const head = `Discovery-output stability (#523) — measured ${measuredOn}`;
|
|
455
|
+
if (report.verdict === "REFUSED") {
|
|
456
|
+
return `${head}\n\nUNMEASURED — ${report.reason}`;
|
|
457
|
+
}
|
|
458
|
+
const lines = [
|
|
459
|
+
head,
|
|
460
|
+
``,
|
|
461
|
+
`k = ${report.runs.length}: ${report.runs.join(", ")}`,
|
|
462
|
+
`input identity: shared (issue ${report.identity.fingerprints[0].issue}, goal ${report.identity.fingerprints[0].goalDigest})`,
|
|
463
|
+
``,
|
|
464
|
+
`${report.fields.length - report.unobserved.length} field(s) observed, ${report.varied.length} varied, ${report.unobserved.length} unobserved.`,
|
|
465
|
+
``,
|
|
466
|
+
];
|
|
467
|
+
for (const f of report.fields) {
|
|
468
|
+
const mark = f.unobserved ? "UNOBSERVED" : f.stable ? "stable " : "VARIED ";
|
|
469
|
+
lines.push(`${mark} ${f.path} [${f.equality}]`);
|
|
470
|
+
if (!f.stable || f.unobserved) {
|
|
471
|
+
for (const r of f.readings) lines.push(` ${r.label}: ${r.value}`);
|
|
472
|
+
}
|
|
473
|
+
}
|
|
474
|
+
if (report.unobserved.length > 0) {
|
|
475
|
+
lines.push(
|
|
476
|
+
``,
|
|
477
|
+
`An UNOBSERVED field is not a stable one — no run produced a value, so nothing was compared.`,
|
|
478
|
+
);
|
|
479
|
+
}
|
|
480
|
+
return lines.join("\n");
|
|
481
|
+
}
|
|
@@ -0,0 +1,186 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Whether what a consumer can install IS what this tree is (#875).
|
|
3
|
+
*
|
|
4
|
+
* ## The question this asks, and why it is not the one next door
|
|
5
|
+
*
|
|
6
|
+
* `lib/published-artifact.ts` compares the registry tarball against the tree
|
|
7
|
+
* by content and classifies the result. Its refusal — `artifactViolations` —
|
|
8
|
+
* is about a LABEL: `DIVERGED` means one version string is being used for two
|
|
9
|
+
* different trees, and `AHEAD` is waved through because a repository past its
|
|
10
|
+
* last release is what a repository between releases is.
|
|
11
|
+
*
|
|
12
|
+
* That is the right refusal for the question it asks and it is the wrong one
|
|
13
|
+
* for the question #875 was filed about. The harm in the issue is not a label:
|
|
14
|
+
* it is that `bunx rungate` hands a consumer the **pre-#863 tier-2 secret
|
|
15
|
+
* scan**, which reported five property references as credential assignments
|
|
16
|
+
* and turned `gates` red on `hornjason/agentgrit`. That consumer is no better
|
|
17
|
+
* off if `package.json` says `0.7.0`.
|
|
18
|
+
*
|
|
19
|
+
* So this module asks the consumer's question: **is the published artifact the
|
|
20
|
+
* content this tree would publish?** No version string is an input to it.
|
|
21
|
+
*
|
|
22
|
+
* ## Which is also what makes it survive the cheapest wrong fix
|
|
23
|
+
*
|
|
24
|
+
* The 5pm repair for a red distribution check is to bump `package.json`. It
|
|
25
|
+
* works on the label — `artifactStanding` reads `AHEAD` and
|
|
26
|
+
* `artifactViolations` returns `[]` — and it moves nothing a consumer can
|
|
27
|
+
* observe. Measured on this tree 2026-10-21, the reading either side of that
|
|
28
|
+
* bump:
|
|
29
|
+
*
|
|
30
|
+
* | | tree `0.6.0` | tree `0.7.0` |
|
|
31
|
+
* |---|---|---|
|
|
32
|
+
* | `artifactStanding` | `DIVERGED` | `AHEAD` |
|
|
33
|
+
* | `artifactViolations` | 1 violation | **none** |
|
|
34
|
+
* | files differing in bytes | 45 | **45** |
|
|
35
|
+
* | files the artifact lacks | 58 | **58** |
|
|
36
|
+
* | commits since the published tag | 183 | **183** |
|
|
37
|
+
*
|
|
38
|
+
* `currencyRefusal` reads the bottom three rows, so the bump moves it not at
|
|
39
|
+
* all. That is the fifth shape in `.claude/rules/checks-must-be-able-to-fail.md`
|
|
40
|
+
* — *a case that goes green under the WRONG fix* — answered by choosing inputs
|
|
41
|
+
* the wrong fix does not touch, rather than by hoping nobody reaches for it.
|
|
42
|
+
*
|
|
43
|
+
* ## Two refusals, not one decider wearing two hats
|
|
44
|
+
*
|
|
45
|
+
* `currencyRefusal` and `artifactViolations` are NOT the fourth shape. They
|
|
46
|
+
* are distinguishable, and the distinguishing input is named rather than
|
|
47
|
+
* asserted: a tree at `0.7.0` against a published `0.6.0` whose content
|
|
48
|
+
* differs. `artifactViolations` returns `[]` — no label is lying — and
|
|
49
|
+
* `currencyRefusal` refuses, because a consumer still cannot obtain this tree.
|
|
50
|
+
* A mutation of either is visible on that input alone.
|
|
51
|
+
*
|
|
52
|
+
* ## This is a script's refusal, deliberately, and not the suite's
|
|
53
|
+
*
|
|
54
|
+
* `STALE` is the NORMAL state of this repository between releases, because
|
|
55
|
+
* `preflightRefusal` in `lib/release-preflight.ts` requires the tree's version
|
|
56
|
+
* to be absent from the registry at release time — so the bump happens AT the
|
|
57
|
+
* release and never before it. A gate wired into `bun test` would therefore be
|
|
58
|
+
* red on `main` every day of the repo's life, and `lib/release-preflight.ts`
|
|
59
|
+
* already records what that costs: *a red job over a green release is how a
|
|
60
|
+
* gate teaches people to stop reading it.*
|
|
61
|
+
*
|
|
62
|
+
* So the reading lives behind `scripts/check-distribution-currency.ts`, which
|
|
63
|
+
* exits non-zero. It is a thing you run before cutting a release and a thing
|
|
64
|
+
* that answers "how far behind is npm" in one command — not a thing that
|
|
65
|
+
* blocks an unrelated PR on a condition its author cannot clear.
|
|
66
|
+
*/
|
|
67
|
+
import {
|
|
68
|
+
contentAgrees,
|
|
69
|
+
type ArtifactStanding,
|
|
70
|
+
type ContentDifference,
|
|
71
|
+
} from "./published-artifact";
|
|
72
|
+
|
|
73
|
+
/**
|
|
74
|
+
* The one non-zero exit in the distribution-currency path.
|
|
75
|
+
*
|
|
76
|
+
* Declared once and exported so a test can build a mutant with it set to `0`
|
|
77
|
+
* and observe the refusal being removed — `.claude/rules/checks-must-be-able-
|
|
78
|
+
* to-fail.md`, *For a script whose job is to exit non-zero*. A later branch
|
|
79
|
+
* that inlines `process.exit(1)` bypasses the mutation and the case goes green
|
|
80
|
+
* having witnessed nothing, so `test/distribution-currency.test.ts` also
|
|
81
|
+
* asserts the constant is the script's only exit code.
|
|
82
|
+
*/
|
|
83
|
+
export const DISTRIBUTION_STALE_EXIT = 1;
|
|
84
|
+
|
|
85
|
+
/**
|
|
86
|
+
* `CURRENT` — a consumer installing from npm gets this tree.
|
|
87
|
+
* `STALE` — they do not, whatever the version strings say.
|
|
88
|
+
* `UNMEASURED` — nobody looked. Never a pass.
|
|
89
|
+
*/
|
|
90
|
+
export type CurrencyReading = "CURRENT" | "STALE" | "UNMEASURED";
|
|
91
|
+
|
|
92
|
+
export interface DistributionCurrency {
|
|
93
|
+
reading: CurrencyReading;
|
|
94
|
+
/** Published files whose bytes differ from the tree's. */
|
|
95
|
+
staleFiles: string[];
|
|
96
|
+
/** Files the tree would publish that the artifact does not have at all. */
|
|
97
|
+
missingFiles: string[];
|
|
98
|
+
/** Files in the artifact that the tree no longer publishes. */
|
|
99
|
+
removedFiles: string[];
|
|
100
|
+
/** Commits between the published tag and `HEAD`. `null` is a reading nobody took. */
|
|
101
|
+
commitsAhead: number | null;
|
|
102
|
+
/** The version a consumer can install, for the report. Never an input to the verdict. */
|
|
103
|
+
publishedVersion: string | null;
|
|
104
|
+
/** Non-empty exactly when `reading` is `UNMEASURED`. */
|
|
105
|
+
unmeasuredReasons: string[];
|
|
106
|
+
}
|
|
107
|
+
|
|
108
|
+
/**
|
|
109
|
+
* Project an `ArtifactStanding` onto the consumer's question.
|
|
110
|
+
*
|
|
111
|
+
* A projection and not a second measurement: every field here is read off the
|
|
112
|
+
* standing, so this cannot disagree with `lib/published-artifact.ts` about
|
|
113
|
+
* what was observed — only about what the observation means.
|
|
114
|
+
*
|
|
115
|
+
* `reading` is derived from `contentAgrees`, which is the same predicate
|
|
116
|
+
* `artifactStanding` uses to separate `MATCHED` from `DIVERGED`. The version
|
|
117
|
+
* ordering that sends an equal-content tree to `AHEAD` is simply not consulted.
|
|
118
|
+
*/
|
|
119
|
+
export function distributionCurrency(s: ArtifactStanding): DistributionCurrency {
|
|
120
|
+
const d: ContentDifference | null = s.difference;
|
|
121
|
+
const base = {
|
|
122
|
+
staleFiles: d?.differing ?? [],
|
|
123
|
+
missingFiles: d?.onlyLocal ?? [],
|
|
124
|
+
removedFiles: d?.onlyPublished ?? [],
|
|
125
|
+
commitsAhead: s.commitsAhead,
|
|
126
|
+
publishedVersion: s.publishedVersion,
|
|
127
|
+
};
|
|
128
|
+
if (s.reading === "UNMEASURED" || d === null) {
|
|
129
|
+
return {
|
|
130
|
+
...base,
|
|
131
|
+
reading: "UNMEASURED",
|
|
132
|
+
unmeasuredReasons:
|
|
133
|
+
s.unmeasuredReasons.length > 0
|
|
134
|
+
? s.unmeasuredReasons
|
|
135
|
+
: ["the content comparison was not performed"],
|
|
136
|
+
};
|
|
137
|
+
}
|
|
138
|
+
return {
|
|
139
|
+
...base,
|
|
140
|
+
reading: contentAgrees(d) ? "CURRENT" : "STALE",
|
|
141
|
+
unmeasuredReasons: [],
|
|
142
|
+
};
|
|
143
|
+
}
|
|
144
|
+
|
|
145
|
+
/**
|
|
146
|
+
* The single refusal. `null` allows.
|
|
147
|
+
*
|
|
148
|
+
* One exported binding, so there is one thing to mutate. A second predicate
|
|
149
|
+
* over the same fields would always agree with this one and would survive any
|
|
150
|
+
* mutation of it.
|
|
151
|
+
*/
|
|
152
|
+
export function currencyRefusal(c: DistributionCurrency): string | null {
|
|
153
|
+
if (c.reading === "UNMEASURED") {
|
|
154
|
+
return (
|
|
155
|
+
`UNMEASURED: nothing was compared — ${c.unmeasuredReasons.join("; ")}. ` +
|
|
156
|
+
`A reading that could not be taken is not one that passed, so this exits ` +
|
|
157
|
+
`non-zero rather than reporting that the distribution is fine.`
|
|
158
|
+
);
|
|
159
|
+
}
|
|
160
|
+
if (c.reading === "STALE") {
|
|
161
|
+
const n = c.staleFiles.length + c.missingFiles.length + c.removedFiles.length;
|
|
162
|
+
return (
|
|
163
|
+
`STALE: a consumer running \`bunx rungate\` today does NOT get this tree. ` +
|
|
164
|
+
`rungate@${c.publishedVersion} on the registry differs from what this tree ` +
|
|
165
|
+
`would publish in ${n} files — ${c.staleFiles.length} whose bytes differ, ` +
|
|
166
|
+
`${c.missingFiles.length} it does not have at all, ${c.removedFiles.length} ` +
|
|
167
|
+
`it still has` +
|
|
168
|
+
(c.commitsAhead === null ? "" : `, across ${c.commitsAhead} commits since the published tag`) +
|
|
169
|
+
`. Bumping package.json does not change one of those numbers. The remedy ` +
|
|
170
|
+
`is a release.`
|
|
171
|
+
);
|
|
172
|
+
}
|
|
173
|
+
return null;
|
|
174
|
+
}
|
|
175
|
+
|
|
176
|
+
/** Every number this reading produced, whether or not it refuses. */
|
|
177
|
+
export function currencyReportLines(c: DistributionCurrency): string[] {
|
|
178
|
+
return [
|
|
179
|
+
`distribution currency: ${c.reading}`,
|
|
180
|
+
` published version ${c.publishedVersion ?? "UNREAD"}`,
|
|
181
|
+
` commits since its tag ${c.commitsAhead ?? "UNREAD"}`,
|
|
182
|
+
` files differing in bytes ${c.staleFiles.length}`,
|
|
183
|
+
` files npm does not have ${c.missingFiles.length}`,
|
|
184
|
+
` files npm still has ${c.removedFiles.length}`,
|
|
185
|
+
];
|
|
186
|
+
}
|