tickmarkr 2.7.7 → 2.7.9
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/cli/commands/fleet.js +206 -18
- package/dist/gates/baseline.d.ts +38 -9
- package/dist/gates/baseline.js +234 -45
- package/dist/gates/test-manifest.js +2 -2
- package/dist/run/journal.d.ts +1 -1
- package/dist/run/merge.js +9 -4
- package/package.json +2 -1
- package/skills/tickmarkr-overseer/SKILL.md +6 -3
- package/skills/tickmarkr-overseer/scripts/grade-ci.sh +159 -42
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
import { existsSync, mkdirSync, readFileSync, renameSync, unlinkSync, writeFileSync } from "node:fs";
|
|
2
|
+
import { createRequire } from "node:module";
|
|
2
3
|
import { homedir } from "node:os";
|
|
3
4
|
import { dirname, join, relative } from "node:path";
|
|
4
5
|
import { parseArgs } from "node:util";
|
|
@@ -92,6 +93,131 @@ function formatFleetSteering(cfg) {
|
|
|
92
93
|
// T10: what every review says about where its bytes land and when they take effect
|
|
93
94
|
const REPO_DESTINATION_NOTE = (path) => `destination: the repository overlay ${path} holds this repository's routing.allow/deny membership, so the edit lands there — the file is tracked: Fleet never stages or commits it; commit it to share the change`;
|
|
94
95
|
const FUTURE_LOADS_NOTE = "the save changes future config loads only — a tickmarkr run already in progress keeps the config it loaded";
|
|
96
|
+
// Queue 134: routing.mode, review.prefer, consult.prefer and judge each follow the overlay that holds
|
|
97
|
+
// them. A YAML merge holds one only when that merge supplies it — an unrelated sibling merge does not —
|
|
98
|
+
// and the review refuses a write through a supplying merge (D-987). A scalar alias key is the name it
|
|
99
|
+
// resolves to, at the alias's own line; a parent alias is refused because the writer cannot update
|
|
100
|
+
// through it without replacing its siblings. A merge sequence is shallow and first-source-wins.
|
|
101
|
+
// Membership stays on heldLine.
|
|
102
|
+
const CONTROL_PATH = {
|
|
103
|
+
"routing.mode": ["routing", "mode"],
|
|
104
|
+
"review.prefer": ["review", "prefer"],
|
|
105
|
+
"consult.prefer": ["consult", "prefer"],
|
|
106
|
+
judge: ["judge"],
|
|
107
|
+
};
|
|
108
|
+
const CONTROL_NAMES = new Set(Object.keys(CONTROL_PATH));
|
|
109
|
+
const { isAlias: yamlIsAlias, isMap: yamlIsMap, isNode: yamlIsNode, isScalar: yamlIsScalar, isSeq: yamlIsSeq, parseDocument: yamlParseDocument, } = createRequire(import.meta.url)("yaml");
|
|
110
|
+
const controlHoldMemo = new Map();
|
|
111
|
+
const controlLineOf = (bytes, offset) => bytes.slice(0, offset).split("\n").length;
|
|
112
|
+
const resolveYaml = (doc, node) => yamlIsAlias(node) ? node.resolve(doc) : node;
|
|
113
|
+
// A plain scalar, or a scalar alias (`*modeKey` resolving to mode). The merge key `<<` is a symbol
|
|
114
|
+
// and is not a name. Callers that report a line keep the key node's own range — the alias as written,
|
|
115
|
+
// never the anchor it points at.
|
|
116
|
+
const scalarKeyName = (doc, key) => {
|
|
117
|
+
const resolved = resolveYaml(doc, key);
|
|
118
|
+
if (!yamlIsScalar(resolved) || typeof resolved.value === "symbol" || resolved.value == null)
|
|
119
|
+
return undefined;
|
|
120
|
+
return String(resolved.value);
|
|
121
|
+
};
|
|
122
|
+
const namedPair = (doc, map, name) => map.items.find((item) => scalarKeyName(doc, item.key) === name);
|
|
123
|
+
const mergePair = (map) => map.items.find((item) => yamlIsScalar(item.key) && typeof item.key.value === "symbol");
|
|
124
|
+
const mergeShapeOf = (bytes, value) => {
|
|
125
|
+
const range = yamlIsNode(value) ? value.range : null;
|
|
126
|
+
const text = range ? bytes.slice(range[0], range[1]).trim().replace(/\s+/g, " ") : "…";
|
|
127
|
+
return `<<: ${text}`;
|
|
128
|
+
};
|
|
129
|
+
const mergeSources = (doc, value) => {
|
|
130
|
+
const node = resolveYaml(doc, value);
|
|
131
|
+
return yamlIsSeq(node) ? node.items.map((item) => resolveYaml(doc, item)) : [node];
|
|
132
|
+
};
|
|
133
|
+
// YAML 1.1 merge: an explicit key wins over `<<`, and a sequence is shallow — the first source that
|
|
134
|
+
// declares a key owns that key whole. A later source's prefer or mode does not survive an earlier
|
|
135
|
+
// source that already declared the parent mapping. `seen` stops a merge that aliases itself.
|
|
136
|
+
const definesKey = (doc, node, key, seen) => {
|
|
137
|
+
const here = resolveYaml(doc, node);
|
|
138
|
+
if (!yamlIsMap(here) || seen.has(here))
|
|
139
|
+
return false;
|
|
140
|
+
if (namedPair(doc, here, key))
|
|
141
|
+
return true;
|
|
142
|
+
const merge = mergePair(here);
|
|
143
|
+
if (!merge)
|
|
144
|
+
return false;
|
|
145
|
+
const next = new Set(seen);
|
|
146
|
+
next.add(here);
|
|
147
|
+
return mergeSources(doc, merge.value).some((source) => definesKey(doc, source, key, next));
|
|
148
|
+
};
|
|
149
|
+
const controlSupplied = (doc, node, rest, seen = new Set()) => {
|
|
150
|
+
const here = resolveYaml(doc, node);
|
|
151
|
+
if (!yamlIsMap(here) || seen.has(here))
|
|
152
|
+
return false;
|
|
153
|
+
if (rest.length === 0)
|
|
154
|
+
return true;
|
|
155
|
+
const [head, ...tail] = rest;
|
|
156
|
+
if (head === undefined)
|
|
157
|
+
return false;
|
|
158
|
+
const pair = namedPair(doc, here, head);
|
|
159
|
+
if (pair) {
|
|
160
|
+
const value = resolveYaml(doc, pair.value);
|
|
161
|
+
if (yamlIsScalar(value) && value.value === null)
|
|
162
|
+
return true;
|
|
163
|
+
if (tail.length === 0)
|
|
164
|
+
return true;
|
|
165
|
+
const next = new Set(seen);
|
|
166
|
+
next.add(here);
|
|
167
|
+
return controlSupplied(doc, value, tail, next);
|
|
168
|
+
}
|
|
169
|
+
const merge = mergePair(here);
|
|
170
|
+
if (!merge)
|
|
171
|
+
return false;
|
|
172
|
+
const next = new Set(seen);
|
|
173
|
+
next.add(here);
|
|
174
|
+
for (const source of mergeSources(doc, merge.value)) {
|
|
175
|
+
if (!definesKey(doc, source, head, next))
|
|
176
|
+
continue;
|
|
177
|
+
return controlSupplied(doc, source, rest, next);
|
|
178
|
+
}
|
|
179
|
+
return false;
|
|
180
|
+
};
|
|
181
|
+
/** The repository (or user) overlay holds a control at its own key, at a null mask at it or above, or at a
|
|
182
|
+
* merge key that supplies it. An explicit key wins over a sibling merge. Parse errors hold nothing. */
|
|
183
|
+
function controlHold(bytes, path) {
|
|
184
|
+
const key = `${path.join(".")}\0${bytes}`;
|
|
185
|
+
if (controlHoldMemo.has(key))
|
|
186
|
+
return controlHoldMemo.get(key);
|
|
187
|
+
if (controlHoldMemo.size > 256)
|
|
188
|
+
controlHoldMemo.clear();
|
|
189
|
+
const doc = yamlParseDocument(bytes);
|
|
190
|
+
const hold = doc.errors.length ? undefined : walkControl(doc, bytes, doc.contents, path);
|
|
191
|
+
controlHoldMemo.set(key, hold);
|
|
192
|
+
return hold;
|
|
193
|
+
}
|
|
194
|
+
function walkControl(doc, bytes, node, rest) {
|
|
195
|
+
const here = resolveYaml(doc, node);
|
|
196
|
+
if (!yamlIsMap(here) || rest.length === 0)
|
|
197
|
+
return undefined;
|
|
198
|
+
const [head, ...tail] = rest;
|
|
199
|
+
const pair = head === undefined ? undefined : namedPair(doc, here, head);
|
|
200
|
+
if (pair) {
|
|
201
|
+
const range = yamlIsNode(pair.key) ? pair.key.range : null;
|
|
202
|
+
if (!range)
|
|
203
|
+
return undefined;
|
|
204
|
+
const line = controlLineOf(bytes, range[0]);
|
|
205
|
+
const value = resolveYaml(doc, pair.value);
|
|
206
|
+
if (yamlIsScalar(value) && value.value === null) {
|
|
207
|
+
return { line, via: yamlIsAlias(pair.key) && tail.length > 0 ? "alias-parent" : "mask" };
|
|
208
|
+
}
|
|
209
|
+
if (tail.length === 0)
|
|
210
|
+
return { line, via: "declared" };
|
|
211
|
+
const nested = walkControl(doc, bytes, value, tail);
|
|
212
|
+
return nested && yamlIsAlias(pair.key) ? { line, via: "alias-parent" } : nested;
|
|
213
|
+
}
|
|
214
|
+
const merge = mergePair(here);
|
|
215
|
+
if (!merge || !yamlIsScalar(merge.key) || !merge.key.range)
|
|
216
|
+
return undefined;
|
|
217
|
+
if (!controlSupplied(doc, here, rest))
|
|
218
|
+
return undefined;
|
|
219
|
+
return { line: controlLineOf(bytes, merge.key.range[0]), via: "merge", shape: mergeShapeOf(bytes, merge.value) };
|
|
220
|
+
}
|
|
95
221
|
const MODE_GLOSS = {
|
|
96
222
|
"partner-led": "every shape frontier · explore off",
|
|
97
223
|
"risk-based": "risk-tiered default floors",
|
|
@@ -224,8 +350,10 @@ export async function fleet(argv, cwd = process.cwd(), adapters = allAdapters(),
|
|
|
224
350
|
export async function assembleFleetEditor(cwd, adapters, io, opts) {
|
|
225
351
|
const globalDir = opts.globalDir;
|
|
226
352
|
// B2: the editor's choices are the operator's MACHINE choices — a save lands in the USER overlay,
|
|
227
|
-
// which every repository on this machine inherits. T10 (D-925/D-926):
|
|
228
|
-
//
|
|
353
|
+
// which every repository on this machine inherits. T10 (D-925/D-926): a membership edit and a
|
|
354
|
+
// routing.mode, review.prefer, consult.prefer or judge edit each land in the overlay that holds
|
|
355
|
+
// them (destinationEdits below) — the repository overlay when it declares the key or masks it
|
|
356
|
+
// with null at it or above, else the user overlay.
|
|
229
357
|
const userPath = join(globalDir, "config.yaml");
|
|
230
358
|
const repoPath = repoOverlayPath(cwd);
|
|
231
359
|
const currentUserText = () => (existsSync(userPath) ? readFileSync(userPath, "utf8") : "");
|
|
@@ -552,11 +680,12 @@ export async function assembleFleetEditor(cwd, adapters, io, opts) {
|
|
|
552
680
|
const home = homedir();
|
|
553
681
|
const shown = (path) => (path.startsWith(`${home}/`) ? `~${path.slice(home.length)}` : path);
|
|
554
682
|
const holderOf = (dotted) => {
|
|
555
|
-
const
|
|
556
|
-
const
|
|
683
|
+
const control = dotted in CONTROL_PATH ? CONTROL_PATH[dotted] : undefined;
|
|
684
|
+
const lineIn = (text) => control ? controlHold(text, control)?.line : heldLine(text, dotted.split("."));
|
|
685
|
+
const repoLine = lineIn(currentRepoOverlayText(cwd));
|
|
557
686
|
if (repoLine !== undefined)
|
|
558
687
|
return `${relative(cwd, repoPath)}:${repoLine}`;
|
|
559
|
-
const userLine =
|
|
688
|
+
const userLine = lineIn(currentUserText());
|
|
560
689
|
return userLine === undefined ? undefined : `${shown(userPath)}:${userLine}`;
|
|
561
690
|
};
|
|
562
691
|
// the allow + flat-deny family is ONE coupled write (the allow form is
|
|
@@ -615,7 +744,14 @@ export async function assembleFleetEditor(cwd, adapters, io, opts) {
|
|
|
615
744
|
}
|
|
616
745
|
if (JSON.stringify(initial.efforts ?? {}) !== JSON.stringify(staged.efforts ?? {}))
|
|
617
746
|
edits.push({ name: "model efforts", destination: "user" });
|
|
618
|
-
|
|
747
|
+
// a control lands where it is held. A merge that supplies it is still the repository (the review
|
|
748
|
+
// then refuses the write); review.prefer and consult.prefer are two edits, never one
|
|
749
|
+
const controlDestination = (name) => {
|
|
750
|
+
if (!(name in CONTROL_PATH))
|
|
751
|
+
return "user";
|
|
752
|
+
return controlHold(currentRepoOverlayText(cwd), CONTROL_PATH[name]) !== undefined ? "repo" : "user";
|
|
753
|
+
};
|
|
754
|
+
return [...edits, ...controls.map((name) => ({ name, destination: controlDestination(name) }))];
|
|
619
755
|
};
|
|
620
756
|
// ONE review confirms ONE destination; a batch needing both is refused before any write (no
|
|
621
757
|
// multi-file transaction) and names what to save in separate sessions
|
|
@@ -800,8 +936,8 @@ export async function assembleFleetEditor(cwd, adapters, io, opts) {
|
|
|
800
936
|
parts.push(`${unroutable} unroutable`);
|
|
801
937
|
return ` mix: ${mix} — ${parts.join(" · ")}`;
|
|
802
938
|
};
|
|
803
|
-
//
|
|
804
|
-
//
|
|
939
|
+
// Floors come from the candidate the chosen destination would load. A routing.mode the repository
|
|
940
|
+
// holds is previewed there, so those floors move; an unheld mode is previewed in the user overlay.
|
|
805
941
|
const floorPreview = (mode, map, deny, stage) => {
|
|
806
942
|
if (mode === rm.mode.mode)
|
|
807
943
|
return [];
|
|
@@ -818,9 +954,9 @@ export async function assembleFleetEditor(cwd, adapters, io, opts) {
|
|
|
818
954
|
: [" (no floor changes)"]),
|
|
819
955
|
];
|
|
820
956
|
};
|
|
821
|
-
//
|
|
822
|
-
//
|
|
823
|
-
//
|
|
957
|
+
// `resolution` is the candidate destination's mode. A routing.mode the repository holds is written
|
|
958
|
+
// into that preview, so a changed mode pins the floors the loader will apply.
|
|
959
|
+
// `entry` is the slot the candidate's loader applies — a staged slot a repo declaration shadows
|
|
824
960
|
// is no operator pin here; the repo's winning declaration keeps its provenance
|
|
825
961
|
const whyDeclaration = (shape, resolution, entry, mapProducedValue) => {
|
|
826
962
|
const mapChanged = JSON.stringify(mapSlot(entry)) !== JSON.stringify(mapSlot(editable.map[shape]));
|
|
@@ -1154,7 +1290,9 @@ export async function assembleFleetEditor(cwd, adapters, io, opts) {
|
|
|
1154
1290
|
// the diff and asks for confirmation, but owns neither filesystem access nor a writer.
|
|
1155
1291
|
// B2: defaults/user/repo are re-read here, at review. T10: so is the destination.
|
|
1156
1292
|
const modeChanged = state.selectedMode !== rm.mode.mode;
|
|
1157
|
-
const
|
|
1293
|
+
const reviewChanged = JSON.stringify(state.steering.review) !== JSON.stringify(initialSteering.review);
|
|
1294
|
+
const consultChanged = JSON.stringify(state.steering.consult) !== JSON.stringify(initialSteering.consult);
|
|
1295
|
+
const steeringChanged = reviewChanged || consultChanged;
|
|
1158
1296
|
const judgeSeat = state.judgeSeat;
|
|
1159
1297
|
const judgeChanged = judgeSeat !== undefined
|
|
1160
1298
|
&& `${judgeSeat.adapter}:${judgeSeat.model}` !== initialJudge;
|
|
@@ -1162,11 +1300,14 @@ export async function assembleFleetEditor(cwd, adapters, io, opts) {
|
|
|
1162
1300
|
pendingWrite = null;
|
|
1163
1301
|
return { kind: "empty" };
|
|
1164
1302
|
}
|
|
1165
|
-
const
|
|
1303
|
+
const controlNames = [
|
|
1166
1304
|
...(modeChanged ? ["routing.mode"] : []),
|
|
1167
|
-
...(
|
|
1305
|
+
...(reviewChanged ? ["review.prefer"] : []),
|
|
1306
|
+
...(consultChanged ? ["consult.prefer"] : []),
|
|
1168
1307
|
...(judgeChanged ? ["judge"] : []),
|
|
1169
|
-
]
|
|
1308
|
+
];
|
|
1309
|
+
const edits = destinationEdits(staged, metadata.seeds, controlNames);
|
|
1310
|
+
const chosen = chooseDestination(edits);
|
|
1170
1311
|
if (!chosen.ok) {
|
|
1171
1312
|
pendingWrite = null;
|
|
1172
1313
|
return { kind: "refused", reason: chosen.reason };
|
|
@@ -1174,6 +1315,43 @@ export async function assembleFleetEditor(cwd, adapters, io, opts) {
|
|
|
1174
1315
|
const destination = chosen.destination;
|
|
1175
1316
|
const user = currentUserText();
|
|
1176
1317
|
const repo = currentRepoOverlayText(cwd);
|
|
1318
|
+
// A merge that supplies a control or an alias used as its parent is a holder, but the writer
|
|
1319
|
+
// cannot update through either key. Refuse before any bytes change, naming the key's own line.
|
|
1320
|
+
if (destination === "repo") {
|
|
1321
|
+
const refusals = edits.flatMap((edit) => {
|
|
1322
|
+
if (!(edit.name in CONTROL_PATH))
|
|
1323
|
+
return [];
|
|
1324
|
+
const hold = controlHold(repo, CONTROL_PATH[edit.name]);
|
|
1325
|
+
if (hold?.via === "merge") {
|
|
1326
|
+
const shape = hold.shape ?? "<<: …";
|
|
1327
|
+
return [`${edit.name} takes keys through the YAML merge key \`${shape}\` at ${holderOf(edit.name)} — Fleet does not write through merge keys; inline the merged keys there by hand, then save again`];
|
|
1328
|
+
}
|
|
1329
|
+
if (hold?.via === "alias-parent") {
|
|
1330
|
+
return [`${edit.name} takes keys through a YAML alias parent key at ${holderOf(edit.name)} — Fleet does not write through alias parent keys; replace that key with its scalar name, then save again`];
|
|
1331
|
+
}
|
|
1332
|
+
return [];
|
|
1333
|
+
});
|
|
1334
|
+
if (refusals.length) {
|
|
1335
|
+
pendingWrite = null;
|
|
1336
|
+
return { kind: "refused", reason: refusals.join("; ") };
|
|
1337
|
+
}
|
|
1338
|
+
}
|
|
1339
|
+
// clearing a repository-held prefer writes an explicit empty sequence, so a lower user list cannot
|
|
1340
|
+
// reappear. A user-held or unheld clear stays undefined and the writer deletes the key, as before.
|
|
1341
|
+
const steeringEdited = { ...state.steering };
|
|
1342
|
+
if (destination === "repo") {
|
|
1343
|
+
for (const key of ["review", "consult"]) {
|
|
1344
|
+
if (JSON.stringify(steeringEdited[key]) === JSON.stringify(initialSteering[key]))
|
|
1345
|
+
continue;
|
|
1346
|
+
const chain = steeringEdited[key];
|
|
1347
|
+
if (chain != null && chain.length > 0)
|
|
1348
|
+
continue;
|
|
1349
|
+
const name = `${key}.prefer`;
|
|
1350
|
+
const hold = controlHold(repo, CONTROL_PATH[name]);
|
|
1351
|
+
if (hold !== undefined && hold.via !== "merge")
|
|
1352
|
+
steeringEdited[key] = [];
|
|
1353
|
+
}
|
|
1354
|
+
}
|
|
1177
1355
|
const before = destination === "repo" ? repo : user;
|
|
1178
1356
|
const path = pathOf(destination);
|
|
1179
1357
|
const lower = lowerLayers(destination);
|
|
@@ -1186,7 +1364,7 @@ export async function assembleFleetEditor(cwd, adapters, io, opts) {
|
|
|
1186
1364
|
lowerOverrides: lowerOverrides[destination],
|
|
1187
1365
|
...(modeChanged ? { mode: state.selectedMode } : {}),
|
|
1188
1366
|
...(judgeChanged && judgeSeat ? { judge: judgeSeat } : {}),
|
|
1189
|
-
steering: { initial: initialSteering, edited:
|
|
1367
|
+
steering: { initial: initialSteering, edited: steeringEdited },
|
|
1190
1368
|
});
|
|
1191
1369
|
after = renderFleetOverlayWrite(before, write); // T10 (D-987): a merge-key path refuses here
|
|
1192
1370
|
}
|
|
@@ -1211,9 +1389,19 @@ export async function assembleFleetEditor(cwd, adapters, io, opts) {
|
|
|
1211
1389
|
catch {
|
|
1212
1390
|
poolNotes = [];
|
|
1213
1391
|
}
|
|
1392
|
+
const controlEdits = edits.filter((edit) => CONTROL_NAMES.has(edit.name));
|
|
1393
|
+
const controlNote = controlEdits.length === 0
|
|
1394
|
+
? undefined
|
|
1395
|
+
: `destination: ${controlEdits.map((edit) => `${edit.name} at ${holderOf(edit.name)}`).join("; ")} — the file is tracked and Fleet never commits it`;
|
|
1214
1396
|
const notes = [
|
|
1215
|
-
// T10:
|
|
1216
|
-
|
|
1397
|
+
// T10: a membership edit keeps the membership destination note. Every repository-destined
|
|
1398
|
+
// control in the same batch is named beside it, path:line from holderOf, file tracked.
|
|
1399
|
+
...(destination === "repo"
|
|
1400
|
+
? [
|
|
1401
|
+
...(edits.some((edit) => !CONTROL_NAMES.has(edit.name)) ? [REPO_DESTINATION_NOTE(repoPath)] : []),
|
|
1402
|
+
...(controlNote ? [controlNote] : []),
|
|
1403
|
+
]
|
|
1404
|
+
: repoShadowNotes(before, after)),
|
|
1217
1405
|
FUTURE_LOADS_NOTE,
|
|
1218
1406
|
...(lower.ok ? [] : [`${lower.error} — y will be refused`]),
|
|
1219
1407
|
...poolNotes,
|
package/dist/gates/baseline.d.ts
CHANGED
|
@@ -39,11 +39,11 @@ export type RedactionCounts = NonNullable<GateEvidenceReceipt["redaction"]["coun
|
|
|
39
39
|
* span counted once under its highest-priority member — token, assignment, secret environment, benign
|
|
40
40
|
* HOME/TMPDIR — so precedence never depends on which match starts first. Counts only: never a value.
|
|
41
41
|
*/
|
|
42
|
-
export declare function classifyGateOutput(text: string, env: NodeJS.ProcessEnv): {
|
|
42
|
+
export declare function classifyGateOutput(text: string, env: NodeJS.ProcessEnv, command?: string, inheritedEnv?: NodeJS.ProcessEnv): {
|
|
43
43
|
text: string;
|
|
44
44
|
counts: RedactionCounts;
|
|
45
45
|
};
|
|
46
|
-
export declare function redactGateOutput(text: string, env: NodeJS.ProcessEnv): string;
|
|
46
|
+
export declare function redactGateOutput(text: string, env: NodeJS.ProcessEnv, command?: string): string;
|
|
47
47
|
/** Material means bytes were withheld; a benign location substitution withholds nothing. */
|
|
48
48
|
export declare const redactionMaterial: (c: RedactionCounts) => boolean;
|
|
49
49
|
/** Resolve a snapshot reference against actual bytes: eviction never fabricates a live artifact. */
|
|
@@ -103,6 +103,11 @@ export interface BaselineCommand {
|
|
|
103
103
|
*/
|
|
104
104
|
exitCode?: number;
|
|
105
105
|
fingerprints: string[];
|
|
106
|
+
/**
|
|
107
|
+
* S22: `2` when captureBaseline wrote one table-T / layout-L fingerprint per occurrence. Absent on every
|
|
108
|
+
* earlier and every hand-built entry (LEGACY, compared exactly as 2.7.8 did); any other value is unreadable.
|
|
109
|
+
*/
|
|
110
|
+
fingerprintVersion?: number;
|
|
106
111
|
missingCommand?: boolean;
|
|
107
112
|
/** Why a capture returned no verdict. */
|
|
108
113
|
invalidCause?: "ceiling-kill" | "resource-exhaustion" | "infra";
|
|
@@ -186,16 +191,40 @@ export declare function classifyFailureOutput(output: string): FailureClassifica
|
|
|
186
191
|
export declare function classifiedFileDurations(entry: BaselineCommand | undefined): BaselineFileDuration[] | null | undefined;
|
|
187
192
|
export declare const UNRECOGNIZED_FAILURE = "<unrecognized failure output>";
|
|
188
193
|
export declare function fingerprint(output: string): string[];
|
|
189
|
-
/**
|
|
190
|
-
|
|
191
|
-
|
|
192
|
-
|
|
193
|
-
*/
|
|
194
|
-
export declare function freshFailures(entry: BaselineCommand | undefined, raw: string): {
|
|
194
|
+
/** Which reading of a baseline entry decided forgiveness (S22). Only `legacy` and `v2` carry a forgivable verdict. */
|
|
195
|
+
export type BaselineEntryState = "missing" | "unreadable" | "legacy" | "v2" | "no-verdict";
|
|
196
|
+
export interface FreshFailureVerdict {
|
|
197
|
+
/** Fresh failure identities, each once. */
|
|
195
198
|
failing: string[];
|
|
199
|
+
/** The output carried no recognizable failure shape at all. */
|
|
196
200
|
unreadable: boolean;
|
|
201
|
+
/** Baseline-recorded identities this output repeated, each once. */
|
|
197
202
|
forgiven: string[];
|
|
198
|
-
|
|
203
|
+
/** The ORIGINAL runner lines behind the fresh identities: classification, host-starved and diagnostics read these, never the keys. */
|
|
204
|
+
failingLines: string[];
|
|
205
|
+
state: BaselineEntryState;
|
|
206
|
+
/** Why an `unreadable` entry could not be read. */
|
|
207
|
+
why?: string;
|
|
208
|
+
/** The one shared forgiveness decision both readers extend: the entry recorded a red VERDICT. */
|
|
209
|
+
baselineRed: boolean;
|
|
210
|
+
}
|
|
211
|
+
/** The red reason a missing or unreadable entry owes in BOTH readers; undefined for every other state. */
|
|
212
|
+
export declare function baselineEntryRefusal(gate: string, verdict: Pick<FreshFailureVerdict, "state" | "why">): string | undefined;
|
|
213
|
+
/**
|
|
214
|
+
* Q121s / S22: the battery's forgiveness math, exported so tip verify applies the IDENTICAL rule, and
|
|
215
|
+
* the one place an entry's state is decided — each reader renders its reason, neither re-derives it.
|
|
216
|
+
*
|
|
217
|
+
* A LEGACY entry (no fingerprintVersion) compares exactly as 2.7.8 did: set membership over lines whose
|
|
218
|
+
* every digit run is masked. A v2 entry is a MULTISET under two closed grammars (ANL-021 S22):
|
|
219
|
+
* - table T: only whole tokens of a closed volatile shape (durations, numbers, positions, ratios, ISO
|
|
220
|
+
* times, key=number, hex addresses) lose their digits; every other token keeps them. Declared cost: a
|
|
221
|
+
* volatile token outside T stays verbatim, so its change reads fresh (a closed false red, not a defect).
|
|
222
|
+
* - layout L: a stylish error row is keyed by its stream label and the header layout L proves (blank or
|
|
223
|
+
* nothing before it, rows right after it). Declared costs: a row outside every proven block is never
|
|
224
|
+
* forgiven, and a header printed under a realpath variant of the cwd compares as a different header.
|
|
225
|
+
* f is fresh when the candidate repeats it more often than the capture recorded it.
|
|
226
|
+
*/
|
|
227
|
+
export declare function freshFailures(entry: BaselineCommand | undefined, raw: string): FreshFailureVerdict;
|
|
199
228
|
export type PackageManager = "npm" | "pnpm" | "yarn" | "bun";
|
|
200
229
|
/**
|
|
201
230
|
* Operator directive 2026-08-12 (D-OBS-10): tickmarkr is package-manager agnostic.
|
package/dist/gates/baseline.js
CHANGED
|
@@ -14,21 +14,44 @@ export const EVIDENCE_RUN_QUOTA_BYTES = DEFAULT_EVIDENCE_QUOTA_BYTES;
|
|
|
14
14
|
/** Benign environment locations substituted by name; every other redaction is material. */
|
|
15
15
|
const BENIGN_ENV_KEYS = ["HOME", "TMPDIR"];
|
|
16
16
|
const REDACTION_CATEGORIES = ["token", "assignment", "secretEnv", "benignEnv"];
|
|
17
|
+
/** Keep inherited, evidence, and inline values as separate pairs, including overridden values. */
|
|
18
|
+
function gateEvidencePairs(env, command, inheritedEnv) {
|
|
19
|
+
const pairs = [...Object.entries(inheritedEnv), ...Object.entries(env)]
|
|
20
|
+
.filter((entry) => entry[1] !== undefined);
|
|
21
|
+
for (const match of command.matchAll(/(?:^|\s)([A-Za-z_][A-Za-z_0-9]*)=(?:'([^']*)'|"([^"$]*)"|([^\s;&|]+))/g)) {
|
|
22
|
+
pairs.push([match[1], match[2] ?? match[3] ?? match[4]]);
|
|
23
|
+
}
|
|
24
|
+
return pairs;
|
|
25
|
+
}
|
|
17
26
|
/**
|
|
18
27
|
* OBS-1139: every category scans the ORIGINAL text on its own; overlapping spans then merge into one
|
|
19
28
|
* span counted once under its highest-priority member — token, assignment, secret environment, benign
|
|
20
29
|
* HOME/TMPDIR — so precedence never depends on which match starts first. Counts only: never a value.
|
|
21
30
|
*/
|
|
22
|
-
export function classifyGateOutput(text, env) {
|
|
23
|
-
|
|
24
|
-
const
|
|
25
|
-
//
|
|
26
|
-
//
|
|
27
|
-
//
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
|
|
31
|
+
export function classifyGateOutput(text, env, command = "", inheritedEnv = process.env) {
|
|
32
|
+
const benign = [];
|
|
33
|
+
const secrets = new Set();
|
|
34
|
+
// Closed pair rule, first match wins: HOME/TMPDIR is benign when longer than one; empty is nothing;
|
|
35
|
+
// length >= 8 is secret; a secret name with a flag literal (0, 1, true, false, yes, no, on, off)
|
|
36
|
+
// or all ASCII digits is nothing; another short secret-named value is secret; other short values
|
|
37
|
+
// are nothing. Cost: short all-digit secrets and flag literals are no longer withheld. Span
|
|
38
|
+
// precedence still makes a secret equal to HOME/TMPDIR win over the benign location label.
|
|
39
|
+
for (const [name, value] of gateEvidencePairs(env, command, inheritedEnv)) {
|
|
40
|
+
if (BENIGN_ENV_KEYS.includes(name)) {
|
|
41
|
+
if (value.length > 1)
|
|
42
|
+
benign.push({ value, label: `$${name}` });
|
|
43
|
+
}
|
|
44
|
+
else if (!value) {
|
|
45
|
+
continue;
|
|
46
|
+
}
|
|
47
|
+
else if (value.length >= 8) {
|
|
48
|
+
secrets.add(value);
|
|
49
|
+
}
|
|
50
|
+
else if (/KEY|TOKEN|SECRET|PASSWORD|CREDENTIAL|AUTH/i.test(name)
|
|
51
|
+
&& !/^(?:0|1|true|false|yes|no|on|off)$/i.test(value) && !/^[0-9]+$/.test(value)) {
|
|
52
|
+
secrets.add(value);
|
|
53
|
+
}
|
|
54
|
+
}
|
|
32
55
|
const literal = (v) => v.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
|
|
33
56
|
const tokens = String.raw `\b(?:sk-[A-Za-z0-9_-]{12,}|gh[pousr]_[A-Za-z0-9_]{16,}|github_pat_[A-Za-z0-9_]{16,}|AKIA[A-Z0-9]{16}|eyJ[A-Za-z0-9_-]+\.[A-Za-z0-9_-]+\.[A-Za-z0-9_-]+)\b`;
|
|
34
57
|
const assignments = String.raw `(?:authorization\s*:\s*bearer|(?:api[_-]?key|token|password|secret)\s*[:=])\s*[^\s"']+`;
|
|
@@ -37,7 +60,7 @@ export function classifyGateOutput(text, env) {
|
|
|
37
60
|
const scan = (source, rank, label) => [...text.matchAll(new RegExp(source, "gi"))].map(m => ({ start: m.index, end: m.index + m[0].length, rank, label }));
|
|
38
61
|
const spans = [
|
|
39
62
|
...scan(tokens, 0), ...scan(assignments, 1),
|
|
40
|
-
...secrets.flatMap(v => scan(literal(v), 2)),
|
|
63
|
+
...[...secrets].flatMap(v => scan(literal(v), 2)),
|
|
41
64
|
...benign.flatMap(b => scan(literal(b.value), 3, b.label)),
|
|
42
65
|
].sort((a, b) => a.start - b.start || b.end - a.end);
|
|
43
66
|
const merged = [];
|
|
@@ -64,8 +87,8 @@ export function classifyGateOutput(text, env) {
|
|
|
64
87
|
}
|
|
65
88
|
return { text: out + text.slice(at), counts };
|
|
66
89
|
}
|
|
67
|
-
export function redactGateOutput(text, env) {
|
|
68
|
-
return classifyGateOutput(text, env).text;
|
|
90
|
+
export function redactGateOutput(text, env, command = "") {
|
|
91
|
+
return classifyGateOutput(text, env, command).text;
|
|
69
92
|
}
|
|
70
93
|
/** Material means bytes were withheld; a benign location substitution withholds nothing. */
|
|
71
94
|
export const redactionMaterial = (c) => c.token + c.assignment + c.secretEnv > 0;
|
|
@@ -103,10 +126,8 @@ export function beginGateEvidence(cwd, gate, command, opts = {}, nonce) {
|
|
|
103
126
|
catch { /* unknown subject */ }
|
|
104
127
|
}
|
|
105
128
|
const root = opts.artifactDir ?? join(tmpdir(), "tickmarkr-gate-evidence", createHash("sha256").update(cwd).digest("hex"));
|
|
106
|
-
const
|
|
107
|
-
|
|
108
|
-
env[`assignment_${match.index}`] = match[1] ?? match[2] ?? match[3];
|
|
109
|
-
}
|
|
129
|
+
const inheritedEnv = { ...process.env };
|
|
130
|
+
const evidenceEnv = { ...opts.env };
|
|
110
131
|
let observed;
|
|
111
132
|
let pendingId;
|
|
112
133
|
const history = [];
|
|
@@ -131,7 +152,7 @@ export function beginGateEvidence(cwd, gate, command, opts = {}, nonce) {
|
|
|
131
152
|
exitCode: observed?.exitCode ?? null, signal: observed?.signal ?? null,
|
|
132
153
|
timedOut: observed ? observed.outcome === "timed-out" : null,
|
|
133
154
|
};
|
|
134
|
-
const classified = [classifyGateOutput(stdout,
|
|
155
|
+
const classified = [classifyGateOutput(stdout, evidenceEnv, command, inheritedEnv), classifyGateOutput(stderr, evidenceEnv, command, inheritedEnv)];
|
|
135
156
|
const clean = classified.map(c => c.text);
|
|
136
157
|
const counts = { token: 0, assignment: 0, secretEnv: 0, benignEnv: 0 };
|
|
137
158
|
for (const c of classified)
|
|
@@ -494,6 +515,102 @@ export function fingerprint(output) {
|
|
|
494
515
|
return lines.some((l) => l.trim()) ? [UNRECOGNIZED_FAILURE] : [];
|
|
495
516
|
return [...new Set(shaped.map(normalizeLine))];
|
|
496
517
|
}
|
|
518
|
+
// S22 table T (ANL-021): the ONLY whole-token shapes whose digits a v2 fingerprint canonicalizes, first
|
|
519
|
+
// match wins. Every other token keeps its digits verbatim — TestParse1, TS2304, 1ms.js, my/no-1ms-timeout.
|
|
520
|
+
const T4_PATH_POSITION_RE = /^(.*[^\d:])((?::\d+){1,2})$/;
|
|
521
|
+
const T5_TSC_POSITION_RE = /^(.*)\((\d+),(\d+)\)$/;
|
|
522
|
+
const TABLE_T = [
|
|
523
|
+
[/^(?:\d+h)?(?:\d+m(?!s))?(?:\d*\.)?\d+(?:ns|ps|us|µs|μs|ms|s|m|h)$/, () => "<dur>"], // T1 duration
|
|
524
|
+
[/^[+-]?(?:\d+(?:\.\d+)?|\.\d+)(?:[eE][+-]?\d+)?%?$/, () => "#"], // T2 number
|
|
525
|
+
[/^\d+(?:[:.]\d+)+$/, (m) => m[0].replace(/\d+/g, "#")], // T3 position, clock, version
|
|
526
|
+
[T4_PATH_POSITION_RE, (m) => m[1] + m[2].replace(/\d+/g, "#")], // T4 path with position
|
|
527
|
+
[T5_TSC_POSITION_RE, (m) => `${m[1]}(#,#)`], // T5 tsc position
|
|
528
|
+
[/^\d+\/\d+$/, () => "#/#"], // T6 ratio
|
|
529
|
+
[/^\d{4}-\d{2}-\d{2}(?:[T ]?\d{2}:\d{2}(?::\d{2}(?:\.\d+)?)?)?(?:Z|[+-]\d{2}:?\d{2})?$/, () => "<time>"], // T7 ISO date-time
|
|
530
|
+
[/^([A-Za-z_][\w.-]*)=([+-]?\d+(?:\.\d+)?)$/, (m) => `${m[1]}=#`], // T8 key=number
|
|
531
|
+
[/^0x[0-9a-fA-F]+$/, () => "0x#"], // T9 hex address
|
|
532
|
+
];
|
|
533
|
+
const canonical = (core, rows) => {
|
|
534
|
+
for (const [re, to] of rows) {
|
|
535
|
+
const m = re.exec(core);
|
|
536
|
+
if (m)
|
|
537
|
+
return to(m);
|
|
538
|
+
}
|
|
539
|
+
return undefined;
|
|
540
|
+
};
|
|
541
|
+
const canonicalToken = (token) => {
|
|
542
|
+
const trail = /[,;:.!?]+$/.exec(token)?.[0] ?? "";
|
|
543
|
+
const rest = token.slice(0, token.length - trail.length);
|
|
544
|
+
const positioned = canonical(rest, [TABLE_T[4], TABLE_T[3]]);
|
|
545
|
+
if (positioned !== undefined)
|
|
546
|
+
return positioned + trail;
|
|
547
|
+
const lead = /^[([{"']/.exec(rest)?.[0] ?? "";
|
|
548
|
+
const close = /[)\]}"']+$/.exec(rest.slice(lead.length))?.[0] ?? "";
|
|
549
|
+
const core = rest.slice(lead.length, rest.length - close.length);
|
|
550
|
+
const out = canonical(core, TABLE_T);
|
|
551
|
+
return out === undefined ? token : lead + out + close + trail;
|
|
552
|
+
};
|
|
553
|
+
/** A v2 line: every whitespace token canonicalized by table T, joined with one space. */
|
|
554
|
+
const normalizeV2 = (l) => l.split(/\s+/).filter(Boolean).map(canonicalToken).join(" ");
|
|
555
|
+
// S22 layout L: when a stylish header is PROVEN for an error row. A stream is keyed by its turbo-style
|
|
556
|
+
// label (CONTENT keeps the runner's own indentation, unlike TURBO_PREFIX_RE); unlabelled output is one stream.
|
|
557
|
+
const L_LABEL_RE = /^([\w@./-]+(?::[A-Za-z_][\w.-]*)+):(?: |$)/;
|
|
558
|
+
const L_ROW_RE = /^\s+\d+:\d+\s+(?:error|warning)\s+\S/;
|
|
559
|
+
const splitLabel = (l) => {
|
|
560
|
+
const m = L_LABEL_RE.exec(l);
|
|
561
|
+
return m ? { label: m[1], content: l.slice(m[0].length) } : { label: "", content: l };
|
|
562
|
+
};
|
|
563
|
+
/** Line index → proven header of every ROW inside a BLOCK (a header plus the maximal run of rows after it). */
|
|
564
|
+
function stylishHeaders(lines) {
|
|
565
|
+
const streams = new Map();
|
|
566
|
+
lines.forEach((line, index) => {
|
|
567
|
+
const { label, content } = splitLabel(line);
|
|
568
|
+
const stream = streams.get(label) ?? [];
|
|
569
|
+
stream.push({ index, content });
|
|
570
|
+
streams.set(label, stream);
|
|
571
|
+
});
|
|
572
|
+
const headers = new Map();
|
|
573
|
+
for (const stream of streams.values()) {
|
|
574
|
+
stream.forEach(({ content }, i) => {
|
|
575
|
+
const proven = /^\S/.test(content) && !L_ROW_RE.test(content) && !isFingerprintShaped(content)
|
|
576
|
+
&& (i === 0 || stream[i - 1].content.trim() === "") && L_ROW_RE.test(stream[i + 1]?.content ?? "");
|
|
577
|
+
for (let j = i + 1; proven && j < stream.length && L_ROW_RE.test(stream[j].content); j++)
|
|
578
|
+
headers.set(stream[j].index, content);
|
|
579
|
+
});
|
|
580
|
+
}
|
|
581
|
+
return headers;
|
|
582
|
+
}
|
|
583
|
+
/**
|
|
584
|
+
* S22: one identity per failure OCCURRENCE (never deduplicated), each kept beside the original line it came
|
|
585
|
+
* from. Same filters as `fingerprint`; a stylish error ROW is emitted once, as its layout-L tuple.
|
|
586
|
+
*/
|
|
587
|
+
function fingerprintOccurrences(output) {
|
|
588
|
+
const all = withoutVitestEchoBlocks(output).map((l) => l.replace(ANSI_RE, ""));
|
|
589
|
+
const headers = stylishHeaders(all);
|
|
590
|
+
const occurrences = [];
|
|
591
|
+
let printed = false;
|
|
592
|
+
all.forEach((l, index) => {
|
|
593
|
+
if (PASS_LINE_RE.test(l) || OPERATOR_LINE_RE.test(l))
|
|
594
|
+
return;
|
|
595
|
+
printed ||= l.trim() !== "";
|
|
596
|
+
const stripped = stripTurboPrefix(l);
|
|
597
|
+
if (stripped !== undefined && OPERATOR_LINE_RE.test(stripped))
|
|
598
|
+
return; // OBS-888: operator prose under a turbo prefix
|
|
599
|
+
const { label, content } = splitLabel(l);
|
|
600
|
+
if (L_ROW_RE.test(content) && LINTER_ERROR_RE.test(content)) {
|
|
601
|
+
const header = headers.get(index) ?? null;
|
|
602
|
+
occurrences.push({ id: JSON.stringify(["stylish", label, header, normalizeV2(content)]), line: l, ...(header === null ? { unattributed: true } : {}) });
|
|
603
|
+
return;
|
|
604
|
+
}
|
|
605
|
+
if (isFingerprintShaped(l))
|
|
606
|
+
occurrences.push({ id: normalizeV2(l), line: l });
|
|
607
|
+
if (stripped !== undefined && !PASS_LINE_RE.test(stripped) && isFingerprintShaped(stripped))
|
|
608
|
+
occurrences.push({ id: normalizeV2(stripped), line: stripped });
|
|
609
|
+
});
|
|
610
|
+
if (!occurrences.length && printed)
|
|
611
|
+
return [{ id: UNRECOGNIZED_FAILURE, line: UNRECOGNIZED_FAILURE }];
|
|
612
|
+
return occurrences;
|
|
613
|
+
}
|
|
497
614
|
// The diagnostic channel for a runner we cannot read: raw output lines, never fingerprints, never a
|
|
498
615
|
// verdict. They exist so the operator sees SOMETHING when a green command turns red unrecognizably.
|
|
499
616
|
const unrecognizedEvidence = (raw) => raw
|
|
@@ -505,23 +622,78 @@ const unrecognizedEvidence = (raw) => raw
|
|
|
505
622
|
// stored baselines may predate ANSI/pass-marker hardening — renormalize at compare time so existing
|
|
506
623
|
// on-disk baseline.json files stay comparable without recapture (compat invariant, CLAUDE.md)
|
|
507
624
|
const renormalize = (fp) => normalizeLine(fp.replace(ANSI_RE, ""));
|
|
625
|
+
const entryState = (entry) => {
|
|
626
|
+
if (entry === undefined)
|
|
627
|
+
return { state: "missing" };
|
|
628
|
+
if (entry === null || typeof entry !== "object")
|
|
629
|
+
return { state: "unreadable", why: "entry is not an object" };
|
|
630
|
+
const version = entry.fingerprintVersion;
|
|
631
|
+
if (version !== undefined && version !== 2)
|
|
632
|
+
return { state: "unreadable", why: `fingerprintVersion ${JSON.stringify(version)} is not 2` };
|
|
633
|
+
const fps = entry.fingerprints;
|
|
634
|
+
// ponytail: an absent list keeps 2.7.8's `?? []` for legacy entries; a present one must be all strings
|
|
635
|
+
if ((fps !== undefined || version === 2) && !(Array.isArray(fps) && fps.every((f) => typeof f === "string"))) {
|
|
636
|
+
return { state: "unreadable", why: "fingerprints is not an array of strings" };
|
|
637
|
+
}
|
|
638
|
+
if (entry.infra === true)
|
|
639
|
+
return { state: "no-verdict" };
|
|
640
|
+
return { state: version === 2 ? "v2" : "legacy" };
|
|
641
|
+
};
|
|
642
|
+
/** The red reason a missing or unreadable entry owes in BOTH readers; undefined for every other state. */
|
|
643
|
+
export function baselineEntryRefusal(gate, verdict) {
|
|
644
|
+
if (verdict.state === "missing")
|
|
645
|
+
return `no baseline entry for ${gate}`;
|
|
646
|
+
if (verdict.state === "unreadable")
|
|
647
|
+
return `baseline entry for ${gate} is unreadable (${verdict.why})`;
|
|
648
|
+
return undefined;
|
|
649
|
+
}
|
|
650
|
+
// OBS-42: diagnostic headings enrich fingerprints but cannot reject on their own.
|
|
651
|
+
const rejects = (f) => !FAIL_ANCHOR_RE.test(f) || f.startsWith("FAIL ");
|
|
508
652
|
/**
|
|
509
|
-
* Q121s: the battery's forgiveness math, exported so tip
|
|
510
|
-
*
|
|
511
|
-
*
|
|
653
|
+
* Q121s / S22: the battery's forgiveness math, exported so tip verify applies the IDENTICAL rule, and
|
|
654
|
+
* the one place an entry's state is decided — each reader renders its reason, neither re-derives it.
|
|
655
|
+
*
|
|
656
|
+
* A LEGACY entry (no fingerprintVersion) compares exactly as 2.7.8 did: set membership over lines whose
|
|
657
|
+
* every digit run is masked. A v2 entry is a MULTISET under two closed grammars (ANL-021 S22):
|
|
658
|
+
* - table T: only whole tokens of a closed volatile shape (durations, numbers, positions, ratios, ISO
|
|
659
|
+
* times, key=number, hex addresses) lose their digits; every other token keeps them. Declared cost: a
|
|
660
|
+
* volatile token outside T stays verbatim, so its change reads fresh (a closed false red, not a defect).
|
|
661
|
+
* - layout L: a stylish error row is keyed by its stream label and the header layout L proves (blank or
|
|
662
|
+
* nothing before it, rows right after it). Declared costs: a row outside every proven block is never
|
|
663
|
+
* forgiven, and a header printed under a realpath variant of the cwd compares as a different header.
|
|
664
|
+
* f is fresh when the candidate repeats it more often than the capture recorded it.
|
|
512
665
|
*/
|
|
513
666
|
export function freshFailures(entry, raw) {
|
|
667
|
+
const { state, why } = entryState(entry);
|
|
668
|
+
const baselineRed = (state === "legacy" || state === "v2") && (entry.exitCode ?? 1) !== 0;
|
|
669
|
+
if (state === "v2") {
|
|
670
|
+
const recorded = new Map();
|
|
671
|
+
for (const f of entry.fingerprints)
|
|
672
|
+
recorded.set(f, (recorded.get(f) ?? 0) + 1);
|
|
673
|
+
const occurrences = fingerprintOccurrences(raw);
|
|
674
|
+
const seen = new Map();
|
|
675
|
+
for (const o of occurrences)
|
|
676
|
+
seen.set(o.id, (seen.get(o.id) ?? 0) + 1);
|
|
677
|
+
const isFresh = (o) => o.id !== UNRECOGNIZED_FAILURE && rejects(o.id)
|
|
678
|
+
&& (o.unattributed === true || seen.get(o.id) > (recorded.get(o.id) ?? 0));
|
|
679
|
+
const fresh = occurrences.filter(isFresh);
|
|
680
|
+
return {
|
|
681
|
+
failing: [...new Set(fresh.map((o) => o.id))],
|
|
682
|
+
unreadable: seen.has(UNRECOGNIZED_FAILURE),
|
|
683
|
+
forgiven: [...new Set(occurrences.filter((o) => o.unattributed !== true && seen.get(o.id) <= (recorded.get(o.id) ?? 0)).map((o) => o.id))],
|
|
684
|
+
failingLines: fresh.map((o) => o.line),
|
|
685
|
+
state, baselineRed,
|
|
686
|
+
};
|
|
687
|
+
}
|
|
514
688
|
// OBS-534 (T2): an infra-recorded entry is a KILL, not a verdict — whatever fingerprints it carries
|
|
515
689
|
// came from output flushed before the kill, over a suite that never finished. Nothing there is
|
|
516
|
-
// "pre-existing", so it forgives nothing
|
|
517
|
-
|
|
518
|
-
const known = new Set(entry?.infra === true ? [] : (entry?.fingerprints ?? []).map(renormalize));
|
|
519
|
-
// OBS-42: diagnostic headings enrich fingerprints but cannot invalidate legacy baselines.
|
|
690
|
+
// "pre-existing", so it forgives nothing; neither does a missing or unreadable entry.
|
|
691
|
+
const known = new Set(state === "legacy" ? (entry.fingerprints ?? []).map(renormalize) : []);
|
|
520
692
|
const current = fingerprint(raw);
|
|
521
|
-
const fresh = current.filter((f) => !known.has(f) && (
|
|
693
|
+
const fresh = current.filter((f) => !known.has(f) && rejects(f) && f !== UNRECOGNIZED_FAILURE);
|
|
522
694
|
// OBS-1123: the baseline-recorded half, named so a reader never has to infer it from the fresh half.
|
|
523
|
-
return { failing: fresh
|
|
524
|
-
forgiven: current.filter((f) => known.has(f)) };
|
|
695
|
+
return { failing: fresh, unreadable: current.includes(UNRECOGNIZED_FAILURE),
|
|
696
|
+
forgiven: current.filter((f) => known.has(f)), failingLines: fresh, state, ...(why ? { why } : {}), baselineRed };
|
|
525
697
|
}
|
|
526
698
|
/**
|
|
527
699
|
* Operator directive 2026-08-12 (D-OBS-10): tickmarkr is package-manager agnostic.
|
|
@@ -785,8 +957,9 @@ export async function captureBaseline(cwd, commands, opts = {}) {
|
|
|
785
957
|
exitCode: runnerVerdict === "green-teardown" ? 0 : r.code,
|
|
786
958
|
...(runnerVerdict === "green-teardown" ? { teardownFingerprint: true } : {}),
|
|
787
959
|
// a command that exits 0 has no failures to fingerprint — recording any would be a lie the
|
|
788
|
-
// compare step then has to forgive
|
|
789
|
-
fingerprints: r.code === 0 || runnerVerdict === "green-teardown" ? [] :
|
|
960
|
+
// compare step then has to forgive. S22: one v2 identity per occurrence, sorted.
|
|
961
|
+
fingerprints: r.code === 0 || runnerVerdict === "green-teardown" ? [] : fingerprintOccurrences(raw).map((o) => o.id).sort(),
|
|
962
|
+
fingerprintVersion: 2,
|
|
790
963
|
missingCommand: missingConfiguredCommand(cmd, r),
|
|
791
964
|
durationMs,
|
|
792
965
|
...fileTiming(raw, durationMs),
|
|
@@ -840,15 +1013,20 @@ export async function detectVacuousOracles(cwd, tasks) {
|
|
|
840
1013
|
// drop a FAIL line that fingerprint-collides with baseline noise). No headline anchors → byte-identical
|
|
841
1014
|
// fallback to today's text (non-vitest runners lose nothing). RED-pinned by tests/gates/baseline.test.ts
|
|
842
1015
|
// "HYG-08: details headlines the failing test, not the noise".
|
|
843
|
-
function headlineDetails(raw, fresh) {
|
|
1016
|
+
function headlineDetails(raw, fresh, freshLines) {
|
|
844
1017
|
const headlines = raw
|
|
845
1018
|
.split("\n")
|
|
846
1019
|
.map((l) => l.replace(ANSI_RE, ""))
|
|
847
1020
|
.filter((l) => namesFailureEitherForm(l) || SUMMARY_FAIL_RE.test(l));
|
|
1021
|
+
// S22: the ORIGINAL runner lines behind v2 comparison keys are the diagnostic; the keys are secondary.
|
|
1022
|
+
// A legacy entry's lines ARE its keys, so its details stay byte-identical.
|
|
1023
|
+
const lines = [...new Set(freshLines)].join("\n");
|
|
1024
|
+
const keys = `new failure fingerprints vs baseline (secondary):\n${fresh.join("\n")}`;
|
|
1025
|
+
const same = lines === fresh.join("\n");
|
|
848
1026
|
if (!headlines.length)
|
|
849
|
-
return { details: `new failures vs baseline:\n${
|
|
1027
|
+
return { details: same ? `new failures vs baseline:\n${lines}` : `new failures vs baseline:\n${lines}\n\n${keys}` };
|
|
850
1028
|
return {
|
|
851
|
-
details: `failing tests:\n${headlines.join("\n")}\n\
|
|
1029
|
+
details: `failing tests:\n${headlines.join("\n")}\n\n${same ? "" : `new failure lines vs baseline:\n${lines}\n\n`}${keys}`,
|
|
852
1030
|
meta: { failingTests: headlines.filter(namesFailureEitherForm) },
|
|
853
1031
|
};
|
|
854
1032
|
}
|
|
@@ -871,6 +1049,8 @@ export async function compareToBaseline(cwd, commands, baseline, enabled, opts =
|
|
|
871
1049
|
continue;
|
|
872
1050
|
}
|
|
873
1051
|
const entry = baseline.commands[name];
|
|
1052
|
+
const entryRead = entryState(entry);
|
|
1053
|
+
const refusal = baselineEntryRefusal(name, entryRead);
|
|
874
1054
|
const ceilingMs = effectiveCeilingMs(entry);
|
|
875
1055
|
const evidence = beginGateEvidence(cwd, name, cmd, opts.evidence);
|
|
876
1056
|
let r;
|
|
@@ -902,6 +1082,12 @@ export async function compareToBaseline(cwd, commands, baseline, enabled, opts =
|
|
|
902
1082
|
let recoveryBlocked;
|
|
903
1083
|
const record = (g) => {
|
|
904
1084
|
g = { ...g, evidenceReceipt, evidenceReceipts: [...evidence.history, evidenceReceipt] };
|
|
1085
|
+
// S22: every red that read this command's output names a missing or unreadable entry, once, beside
|
|
1086
|
+
// its own cause (infra, deficit, fresh). A ceiling kill is read before the output and forgives
|
|
1087
|
+
// nothing either way; the tip's kill clause is queue row 19's, so neither reader appends it there.
|
|
1088
|
+
if (!g.pass && refusal && g.meta?.kind !== "ceiling-kill") {
|
|
1089
|
+
g = { ...g, details: g.details.includes(refusal) ? g.details : `${g.details}\n\n${refusal}`, meta: { ...g.meta, baselineEntry: entryRead.state } };
|
|
1090
|
+
}
|
|
905
1091
|
if (recoveryBlocked)
|
|
906
1092
|
g = { ...g, meta: { ...g.meta, recoveryBlocked } };
|
|
907
1093
|
const withReap = r.reapedGroup ? { ...g, meta: { ...g.meta, reapedGroup: true } } : g;
|
|
@@ -977,7 +1163,8 @@ export async function compareToBaseline(cwd, commands, baseline, enabled, opts =
|
|
|
977
1163
|
// below rather than reading as a verified green. Raise the ceiling by teaching isFailureShaped
|
|
978
1164
|
// that runner's position rule (leading verdict + identifier, or identifier + separator + trailing
|
|
979
1165
|
// verdict); loosening back to vocabulary re-opens OBS-278.
|
|
980
|
-
const
|
|
1166
|
+
const verdict = freshFailures(entry, raw);
|
|
1167
|
+
const { failing, unreadable, forgiven, failingLines, baselineRed } = verdict;
|
|
981
1168
|
// OBS-1123: which baseline-recorded fingerprints this verdict carried, and the capture that
|
|
982
1169
|
// recorded them. Observational only — nothing below reads it to decide a verdict.
|
|
983
1170
|
const provenance = baseline.provenance ? { baselineProvenance: baseline.provenance } : {};
|
|
@@ -1004,7 +1191,7 @@ export async function compareToBaseline(cwd, commands, baseline, enabled, opts =
|
|
|
1004
1191
|
// OBS-896: every fresh failure must be timeout-class, and the suite must take at least twice its
|
|
1005
1192
|
// own baseline measurement. The first read buys one calm rerun here, never a worker repair.
|
|
1006
1193
|
if (name === "test" && classification !== "infra" && failing.length && !rerunOf && !opts.infraRerun
|
|
1007
|
-
&& hostStarved(
|
|
1194
|
+
&& hostStarved(failingLines.join("\n"), r.durationMs ?? 0, entry?.durationMs)) {
|
|
1008
1195
|
const waitedMs = await waitForCalmWindow(executionSignal());
|
|
1009
1196
|
if (opts.authorizeRetry && !calmWindowReady()) {
|
|
1010
1197
|
recoveryBlocked = "calm window unavailable within the existing wait ceiling";
|
|
@@ -1021,8 +1208,8 @@ export async function compareToBaseline(cwd, commands, baseline, enabled, opts =
|
|
|
1021
1208
|
}
|
|
1022
1209
|
}
|
|
1023
1210
|
if (classification === "infra") {
|
|
1024
|
-
const evidence =
|
|
1025
|
-
?
|
|
1211
|
+
const evidence = failingLines.length
|
|
1212
|
+
? failingLines.slice(0, 10).join("\n")
|
|
1026
1213
|
: unrecognizedEvidence(raw) || raw.trim().split("\n").slice(0, 10).join("\n");
|
|
1027
1214
|
record({
|
|
1028
1215
|
gate: name,
|
|
@@ -1035,17 +1222,19 @@ export async function compareToBaseline(cwd, commands, baseline, enabled, opts =
|
|
|
1035
1222
|
// OBS-534 (T2): only a recorded VERDICT can be forgiven. A green baseline has no red to forgive,
|
|
1036
1223
|
// and neither has a capture that was killed at its ceiling — it recorded a cause instead, so it
|
|
1037
1224
|
// fails closed on the same branch rather than reading as "only pre-existing failures". Legacy
|
|
1038
|
-
// entries with no exitCode keep the `?? 1` red default both readers share (
|
|
1039
|
-
|
|
1225
|
+
// entries with no exitCode keep the `?? 1` red default both readers share (freshFailures'
|
|
1226
|
+
// baselineRed); S22: a missing or unreadable entry forgives nothing and says so.
|
|
1040
1227
|
if (!failing.length && !baselineRed) {
|
|
1041
1228
|
const recordedCause = entry?.invalidCause === "infra"
|
|
1042
1229
|
? "was invalidated by its recorded runner-infrastructure cause"
|
|
1043
1230
|
: entry?.invalidCause === "resource-exhaustion" || entry?.invalidatingLines?.length
|
|
1044
1231
|
? `was invalidated by its recorded process/resource-exhaustion cause${entry.invalidatingLines?.[0] ? ` (${entry.invalidatingLines[0]})` : ""}`
|
|
1045
1232
|
: "was killed at its ceiling";
|
|
1046
|
-
const closed =
|
|
1047
|
-
? `
|
|
1048
|
-
:
|
|
1233
|
+
const closed = refusal
|
|
1234
|
+
? `exit ${r.code}; ${refusal} — nothing is forgiven, failing closed`
|
|
1235
|
+
: entry?.infra === true
|
|
1236
|
+
? `the baseline capture for this command ${recordedCause} and recorded no verdict, so nothing here is forgivable — it now exits ${r.code} with no recognizable failure lines — failing closed`
|
|
1237
|
+
: `command was green at baseline but now exits ${r.code} with no recognizable failure lines — failing closed`;
|
|
1049
1238
|
const evidence = unrecognizedEvidence(raw);
|
|
1050
1239
|
record({
|
|
1051
1240
|
gate: name,
|
|
@@ -1056,7 +1245,7 @@ export async function compareToBaseline(cwd, commands, baseline, enabled, opts =
|
|
|
1056
1245
|
continue;
|
|
1057
1246
|
}
|
|
1058
1247
|
if (failing.length) {
|
|
1059
|
-
const headlined = headlineDetails(raw, failing);
|
|
1248
|
+
const headlined = headlineDetails(raw, failing, failingLines);
|
|
1060
1249
|
// A fresh red beside baseline-recorded ones names both halves, so no reader shows the new one as forgiven.
|
|
1061
1250
|
const meta = { ...headlined.meta, ...(classification ? { classification } : {}),
|
|
1062
1251
|
...(forgiven.length ? { freshFingerprints: failing, forgivenFingerprints: forgiven, ...provenance } : {}) };
|
|
@@ -1171,7 +1360,7 @@ export function fileCountDeficit(entry, raw, opts) {
|
|
|
1171
1360
|
export function classifyFreshRunnerOutput(entry, raw, code) {
|
|
1172
1361
|
if (classifyRunnerOutput(raw, code) === "green-teardown")
|
|
1173
1362
|
return undefined;
|
|
1174
|
-
const {
|
|
1175
|
-
const verdict = classifyRunnerOutput(
|
|
1363
|
+
const { failingLines } = freshFailures(entry, raw);
|
|
1364
|
+
const verdict = classifyRunnerOutput(failingLines.length ? failingLines.join("\n") : raw, code);
|
|
1176
1365
|
return verdict === "infra" || verdict === "regression" ? verdict : undefined;
|
|
1177
1366
|
}
|
|
@@ -719,8 +719,8 @@ export async function evaluateManifestedTest(cmd, cwd, opts) {
|
|
|
719
719
|
const evidenceRoot = opts.evidence?.artifactDir ?? dir;
|
|
720
720
|
const stdoutPath = join(evidenceRoot, evidenceReceipt.stdout.path);
|
|
721
721
|
const stderrPath = join(evidenceRoot, evidenceReceipt.stderr.path);
|
|
722
|
-
const stdoutTail = Buffer.from(redactGateOutput(invoked.stdout, env)).subarray(-16 * 1024);
|
|
723
|
-
const stderrTail = Buffer.from(redactGateOutput(invoked.stderr, env)).subarray(-16 * 1024);
|
|
722
|
+
const stdoutTail = Buffer.from(redactGateOutput(invoked.stdout, env, spawnedCommand)).subarray(-16 * 1024);
|
|
723
|
+
const stderrTail = Buffer.from(redactGateOutput(invoked.stderr, env, spawnedCommand)).subarray(-16 * 1024);
|
|
724
724
|
const report = invoked.report?.nonce === nonce ? invoked.report : undefined;
|
|
725
725
|
const neverStarted = report ? files.filter(file => !(file in report.started)).length : "unknown";
|
|
726
726
|
const errors = report?.certificate?.errors;
|
package/dist/run/journal.d.ts
CHANGED
|
@@ -455,9 +455,9 @@ export declare const TelemetryRowSchema: z.ZodObject<{
|
|
|
455
455
|
signalBasis: z.ZodOptional<z.ZodEnum<{
|
|
456
456
|
"judge-only": "judge-only";
|
|
457
457
|
skipped: "skipped";
|
|
458
|
+
legacy: "legacy";
|
|
458
459
|
proved: "proved";
|
|
459
460
|
"review-agree": "review-agree";
|
|
460
|
-
legacy: "legacy";
|
|
461
461
|
vacuous: "vacuous";
|
|
462
462
|
}>>;
|
|
463
463
|
kind: z.ZodOptional<z.ZodLiteral<"judge">>;
|
package/dist/run/merge.js
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
import { existsSync, readFileSync, writeFileSync } from "node:fs";
|
|
2
2
|
import { join } from "node:path";
|
|
3
3
|
import { shq } from "../adapters/types.js";
|
|
4
|
-
import { beginGateEvidence, ceilingKillResult, classifyFreshRunnerOutput, classifyRunnerOutput, fileCountDeficit, waitForCalmWindow, effectiveCeilingMs, classifiedFileDurations, fingerprint, freshFailures, } from "../gates/baseline.js";
|
|
4
|
+
import { baselineEntryRefusal, beginGateEvidence, ceilingKillResult, classifyFreshRunnerOutput, classifyRunnerOutput, fileCountDeficit, waitForCalmWindow, effectiveCeilingMs, classifiedFileDurations, fingerprint, freshFailures, } from "../gates/baseline.js";
|
|
5
5
|
import { evaluateManifestedTest, isVitestTestCommand } from "../gates/test-manifest.js";
|
|
6
6
|
import { computeVerificationIdentity, formatReusedDetails, getVerdictStore, getWorktreeTree, resolveStateDir, isInfraResult, } from "../gates/cache.js";
|
|
7
7
|
import { tickmarkrDir } from "../graph/graph.js";
|
|
@@ -339,7 +339,10 @@ export async function verifyIntegrationTip(intWt, commands, runDir, baseline, ev
|
|
|
339
339
|
});
|
|
340
340
|
continue;
|
|
341
341
|
}
|
|
342
|
-
const
|
|
342
|
+
const verdict = freshFailures(entry, stripped);
|
|
343
|
+
const { failing, unreadable } = verdict;
|
|
344
|
+
// S22: a missing or unreadable entry forgives nothing; the red names it rather than a bare exit code.
|
|
345
|
+
const refusal = baselineEntryRefusal(gate, verdict);
|
|
343
346
|
const deficit = fileCountDeficit(entry, stripped);
|
|
344
347
|
const cause = deficit ? "infra" : classifyFreshRunnerOutput(entry, stripped, r.code);
|
|
345
348
|
const greenTeardown = classifyRunnerOutput(stripped, r.code) === "green-teardown";
|
|
@@ -354,7 +357,7 @@ export async function verifyIntegrationTip(intWt, commands, runDir, baseline, ev
|
|
|
354
357
|
// fingerprintable yet OBS-42-exempt from rejecting, so `failing` comes back empty, `unreadable`
|
|
355
358
|
// false and the cause reads "regression" — every other guard satisfied, and a real red forgiven
|
|
356
359
|
// against a capture that never finished asking the question.
|
|
357
|
-
const baselineRed =
|
|
360
|
+
const baselineRed = verdict.baselineRed;
|
|
358
361
|
// Battery parity on the infra rule too (T9): infrastructure-only output means the runner never
|
|
359
362
|
// completed a suite — nothing was verified, so nothing is forgivable, however familiar its
|
|
360
363
|
// fingerprints. Stricter-than-battery edge kept: unreadable output never forgives.
|
|
@@ -382,7 +385,9 @@ export async function verifyIntegrationTip(intWt, commands, runDir, baseline, ev
|
|
|
382
385
|
? `exit ${r.code}; every failure is baseline-recorded, but that capture ran under `
|
|
383
386
|
+ `${describeCapacity(entry?.capacity)} and this verification ran under ${describeCapacity(r.capacity)} `
|
|
384
387
|
+ `— forgiveness across a changed capacity is not evidence`
|
|
385
|
-
: `exit ${r.code}`))
|
|
388
|
+
: `exit ${r.code}`))
|
|
389
|
+
// S22: every red here names a missing or unreadable entry beside its own cause, deficit included
|
|
390
|
+
+ (!pass && refusal ? `; ${refusal}` : ""),
|
|
386
391
|
...(forgiven && !deficit ? { forgiven: true, ...(baseline?.provenance ? { baselineProvenance: baseline.provenance } : {}) } : {}),
|
|
387
392
|
...(cause ? { cause } : {}),
|
|
388
393
|
...(pass ? {} : { artifact }),
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "tickmarkr",
|
|
3
|
-
"version": "2.7.
|
|
3
|
+
"version": "2.7.9",
|
|
4
4
|
"description": "Spec in, verified work out.",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"license": "MIT",
|
|
@@ -94,6 +94,7 @@
|
|
|
94
94
|
"skills/tickmarkr-auto/",
|
|
95
95
|
"skills/tickmarkr-loop/",
|
|
96
96
|
"skills/tickmarkr-overseer/",
|
|
97
|
+
"scripts/ci/",
|
|
97
98
|
"src/",
|
|
98
99
|
"tests/"
|
|
99
100
|
]
|
|
@@ -1601,9 +1601,12 @@ Create specialist seats using `orca terminal create --worktree path:<repo> --com
|
|
|
1601
1601
|
|
|
1602
1602
|
- **GRADE PUBLIC CI FROM JOB LOGS AT RUN-END ONLY (law 48).** A badge is not the grade, and
|
|
1603
1603
|
`gh run view --job --log` is empty while the run is in progress. At run-end execute
|
|
1604
|
-
`scripts/grade-ci.sh <run-id> <expected-count> <tag>` and require GREEN from
|
|
1605
|
-
|
|
1606
|
-
`
|
|
1604
|
+
`scripts/grade-ci.sh <run-id> <expected-count> <tag>` and require GREEN from every job of the
|
|
1605
|
+
declared inventory it prints first — serial `test` and `test-macos`, or sharded, per OS, the four
|
|
1606
|
+
`suite <os> k/4` shards, `single-fork <os>` and `aggregate <os>`; a missing, duplicated or unknown
|
|
1607
|
+
job reads UNREADABLE, never a skip. Its UNREADABLE state is a hard stop, not a green default.
|
|
1608
|
+
Derive the expected glob pathspec count with `git ls-files ':(glob)tests/**/*.test.ts'`
|
|
1609
|
+
(including the `.tsx` sibling where present), never a bare
|
|
1607
1610
|
`tests/**/*.test.ts`: bare `**` is not a git glob (law 49).
|
|
1608
1611
|
|
|
1609
1612
|
- **RE-READ THE REGISTRY BEFORE RULING (law 51).** A successful Release run and the first
|
|
@@ -1,9 +1,36 @@
|
|
|
1
1
|
#!/bin/bash
|
|
2
|
-
# grade-ci.sh <run-id> <expected-count> [tag] — grade
|
|
3
|
-
# Grade only at run-end. A missing/in-progress/empty log is UNREADABLE, never evidence of green.
|
|
2
|
+
# grade-ci.sh <run-id> <expected-count> [tag] — grade EVERY job of a public CI run's closed inventory from
|
|
3
|
+
# its JOB LOG. Grade only at run-end. A missing/in-progress/empty log is UNREADABLE, never evidence of green.
|
|
4
4
|
# Exit 0 GREEN, 1 RED, 2 UNREADABLE, 3 INFRA_KILLED: a job the GitHub runner shut down or cancelled with
|
|
5
5
|
# no failed or timed-out test (the classifier's class, D-897) — never green, never a pass, only unproven.
|
|
6
6
|
# UNREADABLE dominates a mixed result, then RED: a failure anywhere dominates a kill.
|
|
7
|
+
#
|
|
8
|
+
# Two CLOSED inventories (v2.7.8 T4; the workflow renders these names): serial {test, test-macos}; sharded,
|
|
9
|
+
# for os in ubuntu-latest and macos-latest, `suite <os> k/4` for k 1..4, `single-fork <os>`, `aggregate <os>`.
|
|
10
|
+
# A run holding any sharded name is graded against the sharded inventory, else the serial one. The first
|
|
11
|
+
# line is `INVENTORY form=<serial|sharded> declared=<n> observed=<n>`; each missing, duplicated or unknown
|
|
12
|
+
# name prints `INVENTORY_DEFECT kind=<missing|duplicate|unknown> job=<json-quoted name>` and makes the run
|
|
13
|
+
# UNREADABLE: a job the inventory does not name is never skipped.
|
|
14
|
+
#
|
|
15
|
+
# A verdict line is read only from the log lines whose gh step column equals its declared step, exactly
|
|
16
|
+
# once, else UNREADABLE — a line a test or any other step prints, before or after the real one, never decides:
|
|
17
|
+
# COUNT_ORACLE <GREEN|RED> expected=<n|UNREADABLE> actual=<n|UNREADABLE> step `Assert collected test-file count`
|
|
18
|
+
# COVERAGE_FLOORS verdict=… floors=… failing=… (T1's final floors line) step `Recheck coverage floors`
|
|
19
|
+
# SET_ORACLE verdict=… expected=… actual=… defects=… (T3's set oracle) step `Assert collected test-file set`
|
|
20
|
+
# ROLES — the shared classifier (classify-vitest-log.sh) stays the one log verdict:
|
|
21
|
+
# serial the base rule (count oracle GREEN at the expected count, the classifier given that count
|
|
22
|
+
# reading CLEAN or RPC_ONLY, passed+skipped equal to it) AND exactly one COVERAGE_FLOORS
|
|
23
|
+
# verdict=GREEN, so a dropped floors step reads UNREADABLE.
|
|
24
|
+
# shard, single-fork
|
|
25
|
+
# GREEN only on conclusion success whose whole log the classifier, given no count, reads CLEAN
|
|
26
|
+
# or RPC_ONLY: their wrapper already applied the shard contract in-job.
|
|
27
|
+
# aggregate GREEN only on conclusion success with exactly one COVERAGE_FLOORS verdict=GREEN, exactly one
|
|
28
|
+
# `SET_ORACLE verdict=GREEN expected=N actual=N defects=0`, and the lines of its step
|
|
29
|
+
# `Merge coverage (one threshold check per OS)` classified CLEAN or RPC_ONLY.
|
|
30
|
+
# Exit annotations: SERIAL keeps the one exception run-ci-vitest.sh grants (a failure explained ONLY by steps
|
|
31
|
+
# that exited exactly 1 on a log of their own the classifier reads RPC_ONLY); any other failing annotation is
|
|
32
|
+
# never GREEN. Shard, single-fork and aggregate admit none — their wrapper applied the exception in-job.
|
|
33
|
+
# INFRA_KILLED is the base rule for every role.
|
|
7
34
|
set -u
|
|
8
35
|
|
|
9
36
|
run=${1:?run id required}
|
|
@@ -14,34 +41,96 @@ repo=${TKR_GRADE_CI_REPO:-alzahrani-khalid/tickmarkr}
|
|
|
14
41
|
out_dir=${TKR_GRADE_CI_DIR:-${TKR_STATE_DIR:-.tickmarkr}/overseer/diag}
|
|
15
42
|
mkdir -p "$out_dir" || { echo "UNREADABLE: cannot create log directory $out_dir"; exit 2; }
|
|
16
43
|
|
|
44
|
+
serial_inventory=$'test\ntest-macos'
|
|
45
|
+
sharded_inventory=$(for os in ubuntu-latest macos-latest; do
|
|
46
|
+
for k in 1 2 3 4; do echo "suite $os $k/4"; done
|
|
47
|
+
echo "single-fork $os"
|
|
48
|
+
echo "aggregate $os"
|
|
49
|
+
done)
|
|
50
|
+
COUNT_STEP="Assert collected test-file count"
|
|
51
|
+
FLOORS_STEP="Recheck coverage floors"
|
|
52
|
+
SET_STEP="Assert collected test-file set"
|
|
53
|
+
MERGE_STEP="Merge coverage (one threshold check per OS)"
|
|
54
|
+
COUNT_FORMAT='COUNT_ORACLE (GREEN|RED) expected=([0-9]+|UNREADABLE) actual=([0-9]+|UNREADABLE)'
|
|
55
|
+
FLOORS_FORMAT='COVERAGE_FLOORS verdict=(GREEN|RED|UNREADABLE) floors=([0-9]+|UNREADABLE) failing=([0-9]+|UNREADABLE)'
|
|
56
|
+
SET_FORMAT='SET_ORACLE verdict=(GREEN|RED|UNREADABLE) expected=([0-9]+|UNREADABLE) actual=([0-9]+|UNREADABLE) defects=[0-9]+'
|
|
57
|
+
|
|
17
58
|
verdict=0
|
|
18
59
|
mark_unreadable() { verdict=2; }
|
|
19
60
|
mark_red() { [ "$verdict" -eq 2 ] || verdict=1; }
|
|
20
61
|
mark_killed() { [ "$verdict" -ne 0 ] || verdict=3; }
|
|
21
62
|
field() { printf '%s\n' "$classified" | sed -n "s/^VITEST_LOG.* $1=\([^ ]*\).*/\1/p"; }
|
|
63
|
+
count() { printf '%s' "$1" | grep -c ''; }
|
|
64
|
+
# json <name> — one JSON string; awk -v decodes gh's @tsv escapes (\t \n \r \\) first
|
|
65
|
+
json() {
|
|
66
|
+
LC_ALL=C awk -v s="$1" 'BEGIN {
|
|
67
|
+
for (i = 1; i < 32; i++) esc[sprintf("%c", i)] = sprintf("\\u%04x", i)
|
|
68
|
+
esc["\""] = "\\\""; esc["\\"] = "\\\\"
|
|
69
|
+
for (i = 1; i <= length(s); i++) { c = substr(s, i, 1); out = out (c in esc ? esc[c] : c) }
|
|
70
|
+
print "\"" out "\""
|
|
71
|
+
}'
|
|
72
|
+
}
|
|
73
|
+
defect() { echo "INVENTORY_DEFECT kind=$1 job=$(json "$2")"; mark_unreadable; }
|
|
74
|
+
# step_log <log> <step> — the job-log lines gh's step column attributes to <step>, as gh printed them
|
|
75
|
+
step_log() { awk -F'\t' -v s="$2" 'NF >= 3 && $2 == s' "$1"; }
|
|
76
|
+
# verdict_line <log> <step> <family> <format> — the ONE <family> line <step> printed, in <format>;
|
|
77
|
+
# else MISSING, MULTIPLE(n) or MALFORMED, each UNREADABLE
|
|
78
|
+
verdict_line() {
|
|
79
|
+
local lines
|
|
80
|
+
lines=$(step_log "$1" "$2" | awk -v esc="$(printf '\033')" '{
|
|
81
|
+
gsub(/\^\[\[[0-9;]*m/, ""); gsub(esc "\\[[0-9;]*m", "")
|
|
82
|
+
sub(/^[^\t]*\t[^\t]*\t[0-9T:.-]+Z ?/, ""); sub(/[ \t\r]+$/, ""); print
|
|
83
|
+
}' | grep -E "^$3( |\$)")
|
|
84
|
+
case $(count "$lines") in
|
|
85
|
+
0) echo MISSING ;;
|
|
86
|
+
1) if printf '%s\n' "$lines" | grep -qxE "$4"; then printf '%s\n' "$lines"; else echo MALFORMED; fi ;;
|
|
87
|
+
*) echo "MULTIPLE($(count "$lines"))" ;;
|
|
88
|
+
esac
|
|
89
|
+
}
|
|
90
|
+
readable() { case $1 in MISSING|MULTIPLE*|MALFORMED) return 1 ;; esac; }
|
|
91
|
+
floors_green() { printf '%s\n' "$1" | grep -qxE 'COVERAGE_FLOORS verdict=GREEN floors=[0-9]+ failing=0'; }
|
|
22
92
|
|
|
23
93
|
jobs=$(gh run view "$run" --repo "$repo" --json jobs \
|
|
24
94
|
--jq '.jobs[] | [.databaseId, .name, .status, (.conclusion // "")] | @tsv') \
|
|
25
95
|
|| { echo "UNREADABLE: job list"; exit 2; }
|
|
26
|
-
|
|
96
|
+
# an empty job list is an inventory like any other: serial, observed=0, every declared job missing
|
|
97
|
+
|
|
98
|
+
names=$(printf '%s\n' "$jobs" | cut -f2)
|
|
99
|
+
if printf '%s\n' "$names" | grep -qxF -f <(printf '%s\n' "$sharded_inventory"); then
|
|
100
|
+
form=sharded; declared=$sharded_inventory
|
|
101
|
+
else
|
|
102
|
+
form=serial; declared=$serial_inventory
|
|
103
|
+
fi
|
|
104
|
+
echo "INVENTORY form=$form declared=$(count "$declared") observed=$(count "$names")"
|
|
105
|
+
while IFS= read -r name; do
|
|
106
|
+
case $(printf '%s\n' "$names" | grep -cxF -- "$name") in
|
|
107
|
+
0) defect missing "$name" ;;
|
|
108
|
+
1) ;;
|
|
109
|
+
*) defect duplicate "$name" ;;
|
|
110
|
+
esac
|
|
111
|
+
done <<< "$declared"
|
|
112
|
+
while IFS= read -r name; do
|
|
113
|
+
printf '%s\n' "$declared" | grep -qxF -- "$name" || defect unknown "$name"
|
|
114
|
+
done < <([ -z "$jobs" ] || printf '%s\n' "$names" | awk '!seen[$0]++')
|
|
27
115
|
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
116
|
+
[ -z "$jobs" ] || while IFS=$'\t' read -r id name status conclusion; do
|
|
117
|
+
# an unknown job has no role to grade: its INVENTORY_DEFECT already made the run UNREADABLE
|
|
118
|
+
printf '%s\n' "$declared" | grep -qxF -- "$name" || continue
|
|
31
119
|
case "$name" in
|
|
32
|
-
test)
|
|
33
|
-
|
|
34
|
-
*)
|
|
120
|
+
test|test-macos) role=serial ;;
|
|
121
|
+
suite\ *) role=shard ;;
|
|
122
|
+
single-fork\ *) role=single-fork ;;
|
|
123
|
+
*) role=aggregate ;;
|
|
35
124
|
esac
|
|
36
125
|
|
|
37
|
-
echo "$name: status=$status conclusion=${conclusion:-none} job=$id"
|
|
126
|
+
echo "$name: role=$role status=$status conclusion=${conclusion:-none} job=$id"
|
|
38
127
|
if [ "$status" != "completed" ]; then
|
|
39
128
|
echo "$name: UNREADABLE (job has not reached run-end)"
|
|
40
129
|
mark_unreadable
|
|
41
130
|
continue
|
|
42
131
|
fi
|
|
43
132
|
|
|
44
|
-
log="$out_dir/CI-$tag-$name.log"
|
|
133
|
+
log="$out_dir/CI-$tag-$(printf '%s' "$name" | tr -c 'A-Za-z0-9._-' '-').log"
|
|
45
134
|
gh run view --repo "$repo" --job "$id" --log > "$log" 2>/dev/null
|
|
46
135
|
if [ ! -s "$log" ]; then
|
|
47
136
|
echo "$name: UNREADABLE (empty log)"
|
|
@@ -49,24 +138,28 @@ while IFS=$'\t' read -r id name status conclusion; do
|
|
|
49
138
|
continue
|
|
50
139
|
fi
|
|
51
140
|
|
|
52
|
-
oracle=$(grep -oE 'COUNT_ORACLE [A-Z]+ expected=[0-9A-Z]+ actual=[0-9A-Z]+' "$log" | tail -1)
|
|
53
141
|
files=$(grep -oE 'Test Files .*' "$log" | sed 's/[[:space:]]*$//')
|
|
54
142
|
# OBS-1184: the log verdict comes from the ONE classifier the public CI wrapper also uses
|
|
55
143
|
# (scripts/run-ci-vitest.sh), so the badge and this grade cannot disagree. Its rules — complete
|
|
56
144
|
# summaries, failed/timed-out tests, coverage-threshold misses, and the RPC-timeout-only exception
|
|
57
145
|
# to "any unhandled error is RED" (OBS-1058) — live in classify-vitest-log.sh, stated once there.
|
|
58
|
-
#
|
|
59
|
-
|
|
146
|
+
# A serial job's expected count is its declared file contract: both steps' summaries must collect it all.
|
|
147
|
+
# A shard or single-fork job carries no count: its wrapper held it to its plan in-job.
|
|
148
|
+
if [ "$role" = serial ]; then
|
|
149
|
+
classified=$(bash "$here/classify-vitest-log.sh" "$log" "$expected" 2>&1)
|
|
150
|
+
else
|
|
151
|
+
classified=$(bash "$here/classify-vitest-log.sh" "$log" 2>&1)
|
|
152
|
+
fi
|
|
60
153
|
log_verdict=$(field verdict)
|
|
61
154
|
passed=$(field passed); skipped=$(field skipped); failed=$(field failed); timedout=$(field timedout)
|
|
62
155
|
unhandled=$(field unhandled); rpc=$(field runner_rpc_timeouts); coverage=$(field coverage_misses)
|
|
63
156
|
kills=$(field runner_kills)
|
|
64
157
|
errors=$(grep -oE '##\[error\].*' "$log" | sort | uniq -c | sed 's/^ *//' | tr '\n' ';')
|
|
65
158
|
# The commands' own outcomes stand, as in the wrapper: GREEN needs a success conclusion with no step
|
|
66
|
-
# exit annotation, or a failure explained ONLY by steps that exited exactly 1 on a log
|
|
67
|
-
# (gh's step column) that the classifier calls RPC_ONLY
|
|
68
|
-
#
|
|
69
|
-
#
|
|
159
|
+
# exit annotation, or — SERIAL only — a failure explained ONLY by steps that exited exactly 1 on a log
|
|
160
|
+
# of their own (gh's step column) that the classifier calls RPC_ONLY. Any other code (2, a signal's
|
|
161
|
+
# 137), any annotation on another role, an unexplained failure, or a cancelled, timed-out or missing
|
|
162
|
+
# conclusion is RED.
|
|
70
163
|
exits=$(grep -E '##\[error\]Process completed with exit code [0-9]+' "$log")
|
|
71
164
|
case "$conclusion" in
|
|
72
165
|
success) outcome=ok ;;
|
|
@@ -76,9 +169,9 @@ while IFS=$'\t' read -r id name status conclusion; do
|
|
|
76
169
|
if [ -n "$exits" ]; then
|
|
77
170
|
while IFS= read -r exit_line; do
|
|
78
171
|
code=$(printf '%s\n' "$exit_line" | sed -E 's/.*Process completed with exit code ([0-9]+).*/\1/')
|
|
79
|
-
if [ "$code" != 1 ]; then outcome="exit-$code"; break; fi
|
|
172
|
+
if [ "$code" != 1 ] || [ "$role" != serial ]; then outcome="exit-$code"; break; fi
|
|
80
173
|
step=$(printf '%s\n' "$exit_line" | awk -F'\t' 'NF >= 3 { print $2 }')
|
|
81
|
-
|
|
174
|
+
step_log "$log" "$step" > "$log.step"
|
|
82
175
|
case $(bash "$here/classify-vitest-log.sh" "$log.step" 2>&1) in
|
|
83
176
|
"VITEST_LOG verdict=RPC_ONLY "*) ;;
|
|
84
177
|
*) outcome="exit-1-not-rpc-only"; break ;;
|
|
@@ -86,36 +179,60 @@ while IFS=$'\t' read -r id name status conclusion; do
|
|
|
86
179
|
done <<< "$exits"
|
|
87
180
|
fi
|
|
88
181
|
|
|
89
|
-
|
|
90
|
-
|
|
182
|
+
oracles=
|
|
183
|
+
case $role in
|
|
184
|
+
serial)
|
|
185
|
+
oracle=$(verdict_line "$log" "$COUNT_STEP" COUNT_ORACLE "$COUNT_FORMAT")
|
|
186
|
+
floors=$(verdict_line "$log" "$FLOORS_STEP" COVERAGE_FLOORS "$FLOORS_FORMAT")
|
|
187
|
+
oracles="oracle=[$oracle] floors=[$floors] " ;;
|
|
188
|
+
aggregate)
|
|
189
|
+
floors=$(verdict_line "$log" "$FLOORS_STEP" COVERAGE_FLOORS "$FLOORS_FORMAT")
|
|
190
|
+
set_oracle=$(verdict_line "$log" "$SET_STEP" SET_ORACLE "$SET_FORMAT")
|
|
191
|
+
step_log "$log" "$MERGE_STEP" > "$log.merge"
|
|
192
|
+
merge=$(bash "$here/classify-vitest-log.sh" "$log.merge" 2>&1 | sed -n 's/^VITEST_LOG verdict=\([^ ]*\) reason=\([^ ]*\) .*/\1 \2/p')
|
|
193
|
+
oracles="floors=[$floors] set=[$set_oracle] merge=[$merge] " ;;
|
|
194
|
+
esac
|
|
195
|
+
|
|
196
|
+
echo "$name: ${oracles}files=[$(printf '%s' "$files" | tr '\n' '|')] passed=$passed skipped=$skipped failed=$failed timedout=$timedout unhandled=$unhandled runner_rpc_timeouts=$rpc coverage_misses=$coverage runner_kills=${kills:-?} log=[$(field reason)] outcome=[$outcome] errors=[$errors]"
|
|
197
|
+
# A killed runner skips the oracle steps, so a missing verdict line is the kill's, not an unreadable log:
|
|
91
198
|
# INFRA_KILLED unless a step's own failing exit or a failed/timed-out test dominates it (then RED).
|
|
92
199
|
if [ "$log_verdict" = INFRA_KILLED ] && [ "${outcome#exit-}" = "$outcome" ]; then
|
|
93
200
|
echo "$name: INFRA_KILLED"
|
|
94
201
|
mark_killed
|
|
202
|
+
continue
|
|
95
203
|
elif [ "${kills:-0}" -gt 0 ] 2>/dev/null; then
|
|
96
|
-
|
|
97
|
-
|
|
98
|
-
|
|
99
|
-
echo "$name: UNREADABLE"
|
|
100
|
-
mark_unreadable
|
|
101
|
-
elif [ "$oracle" = "COUNT_ORACLE GREEN expected=$expected actual=$expected" ] \
|
|
102
|
-
&& { [ "$log_verdict" = CLEAN ] || [ "$log_verdict" = RPC_ONLY ]; } && [ "$outcome" = ok ] \
|
|
103
|
-
&& [ $((passed + skipped)) -eq "$expected" ]; then
|
|
104
|
-
echo "$name: GREEN"
|
|
204
|
+
grade=RED
|
|
205
|
+
elif [ -z "$log_verdict" ]; then
|
|
206
|
+
grade=UNREADABLE
|
|
105
207
|
else
|
|
106
|
-
|
|
107
|
-
|
|
208
|
+
case $log_verdict in CLEAN|RPC_ONLY) log_ok=1 ;; *) log_ok= ;; esac
|
|
209
|
+
grade=RED
|
|
210
|
+
case $role in
|
|
211
|
+
serial)
|
|
212
|
+
if ! readable "$oracle" || ! readable "$floors" || [ -z "$files" ]; then
|
|
213
|
+
grade=UNREADABLE
|
|
214
|
+
elif [ "$oracle" = "COUNT_ORACLE GREEN expected=$expected actual=$expected" ] && floors_green "$floors" \
|
|
215
|
+
&& [ -n "$log_ok" ] && [ "$outcome" = ok ] && [ $((passed + skipped)) -eq "$expected" ]; then
|
|
216
|
+
grade=GREEN
|
|
217
|
+
fi ;;
|
|
218
|
+
shard|single-fork)
|
|
219
|
+
if [ "$outcome" = ok ] && [ "$conclusion" = success ] && [ -n "$log_ok" ]; then grade=GREEN; fi ;;
|
|
220
|
+
aggregate)
|
|
221
|
+
if ! readable "$floors" || ! readable "$set_oracle"; then
|
|
222
|
+
grade=UNREADABLE
|
|
223
|
+
elif [ "$outcome" = ok ] && [ "$conclusion" = success ] && floors_green "$floors" \
|
|
224
|
+
&& [ "$set_oracle" = "SET_ORACLE verdict=GREEN expected=$expected actual=$expected defects=0" ] \
|
|
225
|
+
&& { [ "${merge%% *}" = CLEAN ] || [ "${merge%% *}" = RPC_ONLY ]; }; then
|
|
226
|
+
grade=GREEN
|
|
227
|
+
fi ;;
|
|
228
|
+
esac
|
|
108
229
|
fi
|
|
230
|
+
echo "$name: $grade"
|
|
231
|
+
case $grade in
|
|
232
|
+
RED) mark_red ;;
|
|
233
|
+
UNREADABLE) mark_unreadable ;;
|
|
234
|
+
esac
|
|
109
235
|
done <<< "$jobs"
|
|
110
236
|
|
|
111
|
-
if [ "$seen_test" -ne 1 ]; then
|
|
112
|
-
echo "test: UNREADABLE (job missing from run)"
|
|
113
|
-
mark_unreadable
|
|
114
|
-
fi
|
|
115
|
-
if [ "$seen_macos" -ne 1 ]; then
|
|
116
|
-
echo "test-macos: UNREADABLE (job missing from run)"
|
|
117
|
-
mark_unreadable
|
|
118
|
-
fi
|
|
119
|
-
|
|
120
237
|
echo "VERDICT rc=$verdict (0=all GREEN 1=RED 2=UNREADABLE 3=INFRA_KILLED)"
|
|
121
238
|
exit "$verdict"
|