staysfixed 0.9.1 → 0.11.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +182 -0
- package/README.md +17 -5
- package/docs/getting-started.md +10 -0
- package/docs/how-v2-works.md +5 -2
- package/package.json +2 -2
- package/src/guard/api.js +107 -3
- package/src/guard/run.js +154 -20
- package/src/report/console.js +235 -17
- package/src/report/html.js +75 -19
- package/src/types.js +5 -0
- package/src/v2/adapters/android-driver.js +62 -12
- package/src/v2/adapters/contract.js +18 -4
- package/src/v2/adapters/electron.js +96 -14
- package/src/v2/adapters/http.js +264 -23
- package/src/v2/adapters/ios-driver.js +22 -4
- package/src/v2/adapters/ios.js +5 -2
- package/src/v2/adapters/isolate.js +78 -5
- package/src/v2/adapters/process.js +350 -92
- package/src/v2/adapters/web-driver.js +23 -1
- package/src/v2/adapters/web.js +42 -3
- package/src/v2/adapters/windows.js +32 -15
- package/src/v2/check.js +526 -19
- package/src/v2/cli.js +345 -3
- package/src/v2/cluster.js +112 -4
- package/src/v2/coverage.js +293 -8
- package/src/v2/detect.js +182 -9
- package/src/v2/doctor.js +253 -30
- package/src/v2/init.js +102 -10
- package/src/v2/mcp/server.js +4 -1
- package/src/v2/mcp/tools.js +291 -24
- package/src/v2/normalise.js +11 -0
- package/src/v2/observation.js +57 -5
- package/src/v2/reference.js +133 -14
- package/src/v2/refusal.js +389 -0
- package/src/v2/remote.js +24 -3
- package/src/v2/run.js +306 -16
- package/src/v2/sealed.js +14 -2
- package/src/v2/ship.js +286 -22
- package/src/v2/store.js +101 -2
- package/src/v2/types.js +5 -0
- package/src/v2/waiver.js +9 -2
- package/src/watch/panel.js +12 -1
package/src/guard/run.js
CHANGED
|
@@ -16,7 +16,7 @@
|
|
|
16
16
|
* this one is not" is most of the value of running it at all.
|
|
17
17
|
*/
|
|
18
18
|
|
|
19
|
-
import { makeGuardApi, ExpectationFailed } from './api.js';
|
|
19
|
+
import { makeGuardApi, ExpectationFailed, GuardAbandoned } from './api.js';
|
|
20
20
|
import { resetWindow } from '../drive/launch.js';
|
|
21
21
|
import { emitEvent } from '../core/events.js';
|
|
22
22
|
|
|
@@ -29,6 +29,7 @@ const FRESH_KEY = 'fresh';
|
|
|
29
29
|
* @typedef {import('../types.js').GuardResult & {
|
|
30
30
|
* retriedToPass?: boolean,
|
|
31
31
|
* assertedNothing?: boolean,
|
|
32
|
+
* timedOut?: boolean,
|
|
32
33
|
* checks?: import('../types.js').CheckStep[],
|
|
33
34
|
* }} GuardRunResult
|
|
34
35
|
*/
|
|
@@ -38,6 +39,8 @@ const FRESH_KEY = 'fresh';
|
|
|
38
39
|
* @property {boolean} ok
|
|
39
40
|
* @property {string} [message]
|
|
40
41
|
* @property {string} [failedAt]
|
|
42
|
+
* @property {boolean} [timedOut] The clock ran out before the guard answered. Not the same
|
|
43
|
+
* as the answer being no — see the note on the result below.
|
|
41
44
|
*/
|
|
42
45
|
|
|
43
46
|
/** @typedef {(step: import('../types.js').CheckStep) => void} StepSink */
|
|
@@ -162,10 +165,31 @@ export async function runGuards(project, app, guards, opts = {}) {
|
|
|
162
165
|
|
|
163
166
|
if (assertedNothing) {
|
|
164
167
|
result.assertedNothing = true;
|
|
168
|
+
// The story of the bug is not repeated in here. It travels on `because`, and the console,
|
|
169
|
+
// the HTML report, the MCP answer and the live panel each print it themselves — so
|
|
170
|
+
// putting it in the message too printed it twice on one screen.
|
|
165
171
|
result.message =
|
|
166
172
|
`This guard checked nothing. Its \`run()\` finished without asking a single question, so it cannot fail ` +
|
|
167
173
|
`and it is not protecting anything — it would report "still holds" every day for ever. ` +
|
|
168
|
-
`Give it at least one \`expect(...)
|
|
174
|
+
`Give it at least one \`expect(...)\`.`;
|
|
175
|
+
} else if (outcome.timedOut) {
|
|
176
|
+
// A third thing wearing the same status, for the same reason as the second one.
|
|
177
|
+
// Measured on 2026-08-31 against a healthy shop with three guards — one genuinely broken,
|
|
178
|
+
// one whose `run()` slept six seconds against its own limit of one and a half: the run
|
|
179
|
+
// announced "2 guards failed — bugs that were already fixed are back." One bug was back.
|
|
180
|
+
// The other guard was never answered, and somebody reading that line goes hunting a
|
|
181
|
+
// regression in checkout that never happened. Running out of time is the guard failing to
|
|
182
|
+
// report, not the product failing — the same distance as "nothing changed" from "nothing
|
|
183
|
+
// was compared".
|
|
184
|
+
//
|
|
185
|
+
// The status stays 'failed' on purpose. A question nobody got an answer to must never
|
|
186
|
+
// count as a pass, or a stuck guard becomes the quietest way there is to go green. The
|
|
187
|
+
// flag is what lets the words be right without letting the verdict go soft.
|
|
188
|
+
result.timedOut = true;
|
|
189
|
+
result.message = outcome.message ?? 'This guard ran out of time before it answered.';
|
|
190
|
+
// The story of the original bug is deliberately left off. It is printed beside every
|
|
191
|
+
// failure to say whether the failure matters — and here nothing yet says the bug is
|
|
192
|
+
// back, so leading with its story is the exact wrong impression to give.
|
|
169
193
|
} else if (outcome.ok) {
|
|
170
194
|
// Passing only on the second go is not passing. The flake register picks
|
|
171
195
|
// this up and condemns the guard, because a guard nobody trusts is worse
|
|
@@ -173,7 +197,11 @@ export async function runGuards(project, app, guards, opts = {}) {
|
|
|
173
197
|
if (attempts > 1) result.retriedToPass = true;
|
|
174
198
|
} else {
|
|
175
199
|
if (outcome.failedAt) result.failedAt = outcome.failedAt;
|
|
176
|
-
|
|
200
|
+
// Said once. The story of the original bug used to be appended here as well, and every
|
|
201
|
+
// renderer prints `because` on its own line underneath — so a failing guard showed the
|
|
202
|
+
// same sentence twice, with a blank line in the middle of the message that broke the
|
|
203
|
+
// one-line-per-guard layout and the results table under it. Measured 2026-08-31.
|
|
204
|
+
result.message = outcome.message ?? 'This guard failed.';
|
|
177
205
|
}
|
|
178
206
|
|
|
179
207
|
results.push(result);
|
|
@@ -221,6 +249,11 @@ function emitGuardDone(events, result) {
|
|
|
221
249
|
message: result.message,
|
|
222
250
|
failedAt: result.failedAt,
|
|
223
251
|
because: result.because,
|
|
252
|
+
// Three different things wear the status 'failed', and a watcher that only sees the
|
|
253
|
+
// status calls all three "broken again". The result knows which it was; until this went
|
|
254
|
+
// out with it, only the terminal did.
|
|
255
|
+
timedOut: result.timedOut,
|
|
256
|
+
assertedNothing: result.assertedNothing,
|
|
224
257
|
// Everything this guard actually asserted, in its own words and its own
|
|
225
258
|
// order — so a listener that arrived late, or one that only keeps the
|
|
226
259
|
// verdicts, still has the working.
|
|
@@ -243,12 +276,30 @@ async function attemptGuard(project, app, guard, baseUrl, timeoutMs, onStep) {
|
|
|
243
276
|
/** @type {ReturnType<typeof setTimeout>|undefined} */
|
|
244
277
|
let timer;
|
|
245
278
|
|
|
279
|
+
/**
|
|
280
|
+
* Losing the race below does not stop the loser, so this is what does.
|
|
281
|
+
*
|
|
282
|
+
* Measured on 2026-08-31: a guard with `timeoutMs: 200` whose body ticked every 25
|
|
283
|
+
* milliseconds had written 7 lines to a file by the time the run reported it, and 26 half
|
|
284
|
+
* a second later. It was still clicking, still reading, still holding the page — behind a
|
|
285
|
+
* run that had already printed its verdict and started the guard after it. That is how one
|
|
286
|
+
* slow guard makes the next three wobble, and the wobble is blamed on the product.
|
|
287
|
+
*
|
|
288
|
+
* A promise cannot be taken back in JavaScript. What can be done is shut every door the
|
|
289
|
+
* guard reaches the app through: from here on `app.open`, `app.click`, `app.expect`,
|
|
290
|
+
* `app.run`, `app.read` and everything on `app.page` refuse, so the body unwinds at its
|
|
291
|
+
* very next step, and a shell command it started is killed rather than left running.
|
|
292
|
+
*/
|
|
293
|
+
const givenUp = new AbortController();
|
|
294
|
+
|
|
246
295
|
/**
|
|
247
296
|
* @param {import('../types.js').CheckStep} step
|
|
248
297
|
* @returns {void}
|
|
249
298
|
*/
|
|
250
299
|
const tell = (step) => {
|
|
251
|
-
|
|
300
|
+
// Nothing is said on behalf of a guard the run has already reported. A step arriving
|
|
301
|
+
// after the verdict reads, to anything watching, as a guard that is still going.
|
|
302
|
+
if (!onStep || givenUp.signal.aborted) return;
|
|
252
303
|
try {
|
|
253
304
|
onStep(step);
|
|
254
305
|
} catch {
|
|
@@ -291,10 +342,20 @@ async function attemptGuard(project, app, guard, baseUrl, timeoutMs, onStep) {
|
|
|
291
342
|
});
|
|
292
343
|
|
|
293
344
|
clearConsole(app);
|
|
294
|
-
|
|
345
|
+
// The clock can run out during the clean start, on a page that will not load. Starting
|
|
346
|
+
// a body the run has already given up on would put a second guard on the same page.
|
|
347
|
+
if (givenUp.signal.aborted) {
|
|
348
|
+
throw new GuardAbandoned('The run gave up on this guard before its body ever started.');
|
|
349
|
+
}
|
|
350
|
+
await guard.run(
|
|
351
|
+
makeGuardApi(app.page, project, onStep ? { onStep, signal: givenUp.signal } : { signal: givenUp.signal }),
|
|
352
|
+
);
|
|
295
353
|
})(),
|
|
296
354
|
new Promise((_resolve, reject) => {
|
|
297
355
|
timer = setTimeout(() => {
|
|
356
|
+
// Given up on FIRST, then reported. The other way round leaves a gap in which the
|
|
357
|
+
// run has moved on and the guard is still driving the app.
|
|
358
|
+
givenUp.abort();
|
|
298
359
|
reject(new TookTooLong(`'${guard.name}' did not finish within ${humanSeconds(timeoutMs)}.`));
|
|
299
360
|
}, timeoutMs);
|
|
300
361
|
}),
|
|
@@ -307,12 +368,35 @@ async function attemptGuard(project, app, guard, baseUrl, timeoutMs, onStep) {
|
|
|
307
368
|
message: `This should still be true, and it is not: "${error.claim}".${consoleNote(app)}`,
|
|
308
369
|
};
|
|
309
370
|
}
|
|
371
|
+
// Out of time is its own answer, and it is not "no". The guard was still going when the
|
|
372
|
+
// clock stopped, so all anyone knows is that nobody asked it anything it managed to
|
|
373
|
+
// finish. Said in those words rather than as a returned bug.
|
|
374
|
+
if (error instanceof TookTooLong || error instanceof GuardAbandoned) {
|
|
375
|
+
return {
|
|
376
|
+
ok: false,
|
|
377
|
+
timedOut: true,
|
|
378
|
+
message:
|
|
379
|
+
`This guard ran out of time after ${humanSeconds(timeoutMs)} and never gave an answer, so nothing ` +
|
|
380
|
+
`here says the bug is back — only that the guard could not be asked. Either it genuinely needs ` +
|
|
381
|
+
`longer than its own \`timeoutMs\`, or it is waiting for something that never arrives. ` +
|
|
382
|
+
// Precisely what was done about it, and no more. The doors are shut, which is the
|
|
383
|
+
// only stopping there is; a sleep or a request the guard is sitting inside still has
|
|
384
|
+
// to finish by itself, and claiming otherwise would be its own small lie.
|
|
385
|
+
`Its \`run()\` has been cut off — every step it takes through \`app\` from here refuses — so it ` +
|
|
386
|
+
`cannot go on driving the app behind the rest of the run.${consoleNote(app)}`,
|
|
387
|
+
};
|
|
388
|
+
}
|
|
310
389
|
const raw = error instanceof Error ? error.message : String(error);
|
|
311
|
-
return { ok: false, message: `${explainApiSlip(raw, app.page, project)}${consoleNote(app)}` };
|
|
390
|
+
return { ok: false, message: `${explainApiSlip(plainly(raw), app.page, project)}${consoleNote(app)}` };
|
|
312
391
|
} finally {
|
|
313
392
|
// The losing side of the race keeps running otherwise, and a stray timer
|
|
314
393
|
// holds the process open long after the run is reported.
|
|
315
394
|
if (timer) clearTimeout(timer);
|
|
395
|
+
// One attempt owns the app only while it is being awaited. Whatever is still in flight
|
|
396
|
+
// when this returns — a timed-out body, a step a guard started and never waited for — is
|
|
397
|
+
// given up on here, so it cannot touch the app that the next guard, or the next attempt
|
|
398
|
+
// at this one, is about to use.
|
|
399
|
+
givenUp.abort();
|
|
316
400
|
}
|
|
317
401
|
|
|
318
402
|
return { ok: true };
|
|
@@ -366,6 +450,65 @@ export function explainApiSlip(raw, page, project) {
|
|
|
366
450
|
);
|
|
367
451
|
}
|
|
368
452
|
|
|
453
|
+
/**
|
|
454
|
+
* Fragments that only ever come out of the debugging protocol talking to itself.
|
|
455
|
+
*
|
|
456
|
+
* Chrome's parameter reader answers in its own vocabulary — the byte it got stuck on, the
|
|
457
|
+
* name of an internal binding, a JSON-RPC error number. Measured on 2026-08-31: a guard that
|
|
458
|
+
* passed the wrong sort of value to `app.page.send` failed with `Invalid parameters (Failed to
|
|
459
|
+
* deserialize params.expression - BINDINGS: string value expected at position 19)` printed at
|
|
460
|
+
* the person as the whole explanation. Nothing in the bracket is about their app, their guard
|
|
461
|
+
* or their bug.
|
|
462
|
+
*/
|
|
463
|
+
const PROTOCOL_CHATTER = /Failed to deserialize|BINDINGS:|at position \d|-32\d{3}|"method"\s*:|"params"\s*:/;
|
|
464
|
+
|
|
465
|
+
/** Past this a failure message stops being something anyone reads and starts being a wall. */
|
|
466
|
+
const MAX_MESSAGE = 400;
|
|
467
|
+
|
|
468
|
+
/**
|
|
469
|
+
* Say a failure in words, with the machinery's own muttering taken out.
|
|
470
|
+
*
|
|
471
|
+
* Three things had to go, all of them seen printed at somebody:
|
|
472
|
+
*
|
|
473
|
+
* - **More than one line.** A stack trace, a browser library's "Call log" block or a dump of
|
|
474
|
+
* the debugging protocol all arrive as one long message with newlines in it, and the
|
|
475
|
+
* results table this sits inside is built of rows — the second line lands in the next
|
|
476
|
+
* column and the table comes apart. The same lesson `explainApiSlip` above learned by
|
|
477
|
+
* printing thirty method names inline.
|
|
478
|
+
* - **The protocol's own diagnostics**, which Chrome hangs off the end in brackets.
|
|
479
|
+
* - **`[object Object]`**, which is what a guard that throws something other than an error
|
|
480
|
+
* turns into. It was, on its own, the entire reason given for a failing guard.
|
|
481
|
+
*
|
|
482
|
+
* @param {string} raw
|
|
483
|
+
* @returns {string}
|
|
484
|
+
*/
|
|
485
|
+
export function plainly(raw) {
|
|
486
|
+
const text = String(raw ?? '').trim();
|
|
487
|
+
|
|
488
|
+
if (text === '' || text === '[object Object]' || text === '[object Undefined]') {
|
|
489
|
+
return (
|
|
490
|
+
'This guard threw something that is not an error, so there is no message to read. ' +
|
|
491
|
+
"Throw `new Error('what went wrong')`, or better, use `app.expect(...)` — it fails with the " +
|
|
492
|
+
'plain sentence you wrote, which is the sentence that gets printed.'
|
|
493
|
+
);
|
|
494
|
+
}
|
|
495
|
+
|
|
496
|
+
// One line. Everything after it is the machinery describing itself.
|
|
497
|
+
let line = text.split('\n')[0].trim();
|
|
498
|
+
|
|
499
|
+
// A whole protocol frame, pasted in as if it were a sentence.
|
|
500
|
+
if (/^[[{]/.test(line) && PROTOCOL_CHATTER.test(line)) {
|
|
501
|
+
return 'The app answered this guard with a raw debugging-protocol message rather than a result, so the guard could not finish.';
|
|
502
|
+
}
|
|
503
|
+
|
|
504
|
+
// Chrome's own diagnostics, bracketed on the end. Only dropped when the bracket really is
|
|
505
|
+
// protocol talk — plenty of failures put something useful in brackets.
|
|
506
|
+
line = line.replace(/\s*\([^()]*\)$/, (blob) => (PROTOCOL_CHATTER.test(blob) ? '' : blob)).trim();
|
|
507
|
+
|
|
508
|
+
if (line.length > MAX_MESSAGE) return `${line.slice(0, MAX_MESSAGE).trimEnd()}… (cut short)`;
|
|
509
|
+
return line;
|
|
510
|
+
}
|
|
511
|
+
|
|
369
512
|
/** A timeout, kept apart from a real error so the wording stays ours. */
|
|
370
513
|
class TookTooLong extends Error {
|
|
371
514
|
/** @param {string} message */
|
|
@@ -404,25 +547,16 @@ function consoleNote(app) {
|
|
|
404
547
|
return `\nThe page also logged an error while this guard ran: ${first}${rest}`;
|
|
405
548
|
}
|
|
406
549
|
|
|
407
|
-
/**
|
|
408
|
-
* The story of the original bug is the single most useful thing to print when a
|
|
409
|
-
* guard fails — it says whether the failure matters.
|
|
410
|
-
*
|
|
411
|
-
* @param {string} message
|
|
412
|
-
* @param {string|undefined} because
|
|
413
|
-
* @returns {string}
|
|
414
|
-
*/
|
|
415
|
-
function withStory(message, because) {
|
|
416
|
-
if (typeof because !== 'string' || because.trim() === '') return message;
|
|
417
|
-
return `${message}\n\nWhy this guard exists: ${because.trim()}`;
|
|
418
|
-
}
|
|
419
|
-
|
|
420
550
|
/**
|
|
421
551
|
* @param {number} ms
|
|
422
552
|
* @returns {string}
|
|
423
553
|
*/
|
|
424
554
|
function humanSeconds(ms) {
|
|
425
555
|
if (ms < 1000) return `${Math.round(ms)} milliseconds`;
|
|
426
|
-
|
|
556
|
+
// Rounded to whole seconds, `timeoutMs: 1500` came back as "did not finish within 2
|
|
557
|
+
// seconds" — a number the person cannot find anywhere in their own guard, which reads
|
|
558
|
+
// as the tool having waited longer than it did. One decimal place, so it matches what
|
|
559
|
+
// they typed: 1.5, 8, 30.
|
|
560
|
+
const seconds = Math.round(ms / 100) / 10;
|
|
427
561
|
return seconds === 1 ? '1 second' : `${seconds} seconds`;
|
|
428
562
|
}
|
package/src/report/console.js
CHANGED
|
@@ -43,6 +43,12 @@ const MAX_COMMANDS = 10;
|
|
|
43
43
|
/** How many touched files a trace prints before summarising the rest. */
|
|
44
44
|
const MAX_TRACE_FILES = 12;
|
|
45
45
|
|
|
46
|
+
/**
|
|
47
|
+
* How many of a check's recorded answers are printed. The register keeps twelve; six is
|
|
48
|
+
* enough to see a check flipping, and the whole row is one `staysfixed flake --json` away.
|
|
49
|
+
*/
|
|
50
|
+
const MAX_RECENT_STATUSES = 6;
|
|
51
|
+
|
|
46
52
|
/**
|
|
47
53
|
* Thousands separators, always. "1024 pixels" reads as noise; "1,024" reads as a number.
|
|
48
54
|
* @param {number} n
|
|
@@ -108,6 +114,63 @@ function shorten(s, max) {
|
|
|
108
114
|
return text.length > max ? text.slice(0, max - 1) + '…' : text;
|
|
109
115
|
}
|
|
110
116
|
|
|
117
|
+
/**
|
|
118
|
+
* What a guard's result actually says, in one word.
|
|
119
|
+
*
|
|
120
|
+
* Four different things wear the status 'failed' or 'skipped', and every renderer used to
|
|
121
|
+
* decide for itself which was which — the terminal line, the verdict sentence, the results
|
|
122
|
+
* table and the HTML report, four times, differently. That is how "1 guard failed — a bug
|
|
123
|
+
* that was already fixed is back" ended up printed on 2026-08-31 over a healthy tree, for a
|
|
124
|
+
* guard whose clock had simply run out. It is decided once here now, and everything reads it.
|
|
125
|
+
*
|
|
126
|
+
* - `back` — the guard asked its question and the answer was no. The bug is back.
|
|
127
|
+
* - `unanswered` — nobody got an answer: it ran out of time, or it asserted nothing at all.
|
|
128
|
+
* Not a pass, and not a returned bug either.
|
|
129
|
+
* - `left out` — marked skip. Never ran.
|
|
130
|
+
* - `held` — asked, and the answer was yes.
|
|
131
|
+
*
|
|
132
|
+
* Anything unrecognised counts as `unanswered`, never as `held`: the one thing that must
|
|
133
|
+
* never happen here is a result nobody understood being read as a clean bill of health.
|
|
134
|
+
*
|
|
135
|
+
* @param {import('../types.js').GuardResult} guard
|
|
136
|
+
* @returns {'held'|'back'|'unanswered'|'left out'}
|
|
137
|
+
*/
|
|
138
|
+
export function guardVerdict(guard) {
|
|
139
|
+
const g = /** @type {any} */ (guard ?? {});
|
|
140
|
+
if (g.status === 'passed') return 'held';
|
|
141
|
+
if (g.status === 'skipped') return 'left out';
|
|
142
|
+
if (g.timedOut === true || g.assertedNothing === true) return 'unanswered';
|
|
143
|
+
if (g.status === 'failed') return 'back';
|
|
144
|
+
return 'unanswered';
|
|
145
|
+
}
|
|
146
|
+
|
|
147
|
+
/**
|
|
148
|
+
* The one short phrase a guard gets in the results table.
|
|
149
|
+
*
|
|
150
|
+
* The table is built of rows, and a message with a newline in it or four sentences of advice
|
|
151
|
+
* lands in the next column and takes the table apart — the same lesson `plainly()` in
|
|
152
|
+
* src/guard/run.js learned the hard way. The full message is printed on the guard's own line
|
|
153
|
+
* above; this is the scannable version.
|
|
154
|
+
*
|
|
155
|
+
* @param {import('../types.js').GuardResult} g
|
|
156
|
+
* @returns {string}
|
|
157
|
+
*/
|
|
158
|
+
function guardOutcome(g) {
|
|
159
|
+
const any = /** @type {any} */ (g);
|
|
160
|
+
switch (guardVerdict(g)) {
|
|
161
|
+
case 'held':
|
|
162
|
+
return 'still holds';
|
|
163
|
+
case 'left out':
|
|
164
|
+
return 'left out on purpose';
|
|
165
|
+
case 'unanswered':
|
|
166
|
+
if (any.timedOut === true) return 'ran out of time — nothing was proved either way';
|
|
167
|
+
if (any.assertedNothing === true) return 'checks nothing, so it is protecting nothing';
|
|
168
|
+
return shorten(g.message || 'gave no answer', 90);
|
|
169
|
+
default:
|
|
170
|
+
return g.failedAt ? `expected: ${g.failedAt}` : shorten(g.message || 'this one is broken again', 90);
|
|
171
|
+
}
|
|
172
|
+
}
|
|
173
|
+
|
|
111
174
|
/**
|
|
112
175
|
* The counts every verdict is built from. Taken from the results themselves
|
|
113
176
|
* rather than `totals`, so the sentence can never disagree with the list above it.
|
|
@@ -124,11 +187,18 @@ function tally(run) {
|
|
|
124
187
|
missing: pictures.filter((p) => p.status === 'missing').length,
|
|
125
188
|
broken: pictures.filter((p) => p.status === 'failed').length,
|
|
126
189
|
wobbled: pictures.filter((p) => p.status === 'flaky').length,
|
|
127
|
-
//
|
|
190
|
+
// Three different things wear the same status, and calling all of them "a bug is back"
|
|
128
191
|
// sends somebody hunting a regression that never happened. A guard that asked no
|
|
129
|
-
// question at all has not caught anything;
|
|
130
|
-
guardsFailed: guards.filter((g) => g
|
|
192
|
+
// question at all has not caught anything; nor has one whose clock ran out.
|
|
193
|
+
guardsFailed: guards.filter((g) => guardVerdict(g) === 'back').length,
|
|
131
194
|
guardsEmpty: guards.filter((g) => /** @type {any} */ (g).assertedNothing === true).length,
|
|
195
|
+
guardsTimedOut: guards.filter((g) => guardVerdict(g) === 'unanswered' && /** @type {any} */ (g).timedOut === true).length,
|
|
196
|
+
// How much this run actually looked at. A run that skipped everything it had, or had
|
|
197
|
+
// nothing to begin with, proves nothing — and the sentence for "nothing was wrong" and
|
|
198
|
+
// the sentence for "nothing was checked" must never be the same sentence.
|
|
199
|
+
looked:
|
|
200
|
+
pictures.filter((p) => p.status !== 'skipped').length +
|
|
201
|
+
guards.filter((g) => g.status !== 'skipped').length,
|
|
132
202
|
};
|
|
133
203
|
}
|
|
134
204
|
|
|
@@ -157,6 +227,10 @@ export function verdictFor(run) {
|
|
|
157
227
|
}
|
|
158
228
|
if (t.guardsEmpty === 1) parts.push({ n: 1, text: '1 guard checks nothing, so it is not protecting anything.' });
|
|
159
229
|
else if (t.guardsEmpty > 1) parts.push({ n: t.guardsEmpty, text: `${countText(t.guardsEmpty)} guards check nothing, so they are not protecting anything.` });
|
|
230
|
+
// Out of time is not an answer of no. Said in its own words, or a healthy tree gets told a
|
|
231
|
+
// bug is back — measured 2026-08-31 on a shop where one guard simply slept past its limit.
|
|
232
|
+
if (t.guardsTimedOut === 1) parts.push({ n: 1, text: '1 guard ran out of time, so nobody knows whether that bug is back.' });
|
|
233
|
+
else if (t.guardsTimedOut > 1) parts.push({ n: t.guardsTimedOut, text: `${countText(t.guardsTimedOut)} guards ran out of time, so nobody knows whether those bugs are back.` });
|
|
160
234
|
if (t.changed === 1) parts.push({ n: 1, text: '1 thing changed. Look at it before you ship.' });
|
|
161
235
|
else if (t.changed > 1) parts.push({ n: t.changed, text: `${countText(t.changed)} things changed. Look at them before you ship.` });
|
|
162
236
|
if (t.fresh === 1) parts.push({ n: 1, text: '1 new screen is waiting for a person to approve it.' });
|
|
@@ -168,12 +242,22 @@ export function verdictFor(run) {
|
|
|
168
242
|
if (parts.length === 0 && t.wobbled > 0) {
|
|
169
243
|
parts.push({ n: t.wobbled, text: `${countText(t.wobbled)} ${plural(t.wobbled, 'check', 'checks')} could not make up ${plural(t.wobbled, 'its', 'their')} mind.` });
|
|
170
244
|
}
|
|
245
|
+
// "Everything that worked still works" over a run that looked at nothing is the same
|
|
246
|
+
// false all-clear as any other, and the friendliest-sounding one this tool can print.
|
|
247
|
+
if (parts.length === 0 && t.looked === 0) {
|
|
248
|
+
return t.pictures + t.guards === 0
|
|
249
|
+
? 'Nothing was checked, so nothing is proved — this project has no screens and no guards yet.'
|
|
250
|
+
: 'Nothing was checked, so nothing is proved — every screen and guard here was left out.';
|
|
251
|
+
}
|
|
171
252
|
if (parts.length === 0) return 'Everything that worked still works.';
|
|
172
253
|
// Two sentences is as much as anyone reads standing up; the table underneath
|
|
173
254
|
// still names every single one, so nothing is hidden by shortening this.
|
|
174
255
|
if (parts.length <= 2) return parts.map((p) => p.text).join(' ');
|
|
175
256
|
const rest = parts.slice(2).reduce((sum, p) => sum + p.n, 0);
|
|
176
|
-
|
|
257
|
+
// "And 1 other screen needs a look" — said on 2026-08-31 about a run of five guards and no
|
|
258
|
+
// screens at all. Whatever gets collapsed in here can be either, so it is called what this
|
|
259
|
+
// tool calls both of them everywhere else. The table underneath still names every one.
|
|
260
|
+
return `${parts[0].text} ${parts[1].text} And ${countText(rest)} other ${plural(rest, 'check needs', 'checks need')} a look.`;
|
|
177
261
|
}
|
|
178
262
|
|
|
179
263
|
/**
|
|
@@ -182,10 +266,14 @@ export function verdictFor(run) {
|
|
|
182
266
|
*/
|
|
183
267
|
export function allClear(run) {
|
|
184
268
|
const t = tally(run);
|
|
185
|
-
//
|
|
186
|
-
|
|
187
|
-
//
|
|
188
|
-
|
|
269
|
+
// A run that looked at nothing is not a clean run. It is a run.
|
|
270
|
+
if (t.looked === 0) return false;
|
|
271
|
+
// `guardsEmpty` and `guardsTimedOut` count too. Splitting them out of `guardsFailed` was so
|
|
272
|
+
// the SENTENCE could tell a returned bug from a question nobody answered — not so that one
|
|
273
|
+
// of them could quietly become a pass. A stuck guard is the quietest way there is to go green.
|
|
274
|
+
return (
|
|
275
|
+
t.changed + t.fresh + t.missing + t.broken + t.wobbled + t.guardsFailed + t.guardsEmpty + t.guardsTimedOut === 0
|
|
276
|
+
);
|
|
189
277
|
}
|
|
190
278
|
|
|
191
279
|
/**
|
|
@@ -277,24 +365,39 @@ export function printPictureResult(r) {
|
|
|
277
365
|
}
|
|
278
366
|
|
|
279
367
|
/**
|
|
280
|
-
* One line per guard; a
|
|
368
|
+
* One line per guard; a returned bug also tells the story of the bug it watches.
|
|
369
|
+
*
|
|
370
|
+
* The story is deliberately NOT printed under a guard that ran out of time. It is printed to
|
|
371
|
+
* say whether a failure matters — and a timeout says nothing about the bug at all, so
|
|
372
|
+
* "why this guard exists: checking out twice used to leave the old basket behind" under a red
|
|
373
|
+
* line is read as that bug being back. Which is the thing this whole file is trying to stop.
|
|
374
|
+
*
|
|
281
375
|
* @param {import('../types.js').GuardResult} r
|
|
282
376
|
* @returns {void}
|
|
283
377
|
*/
|
|
284
378
|
export function printGuardResult(r) {
|
|
285
379
|
const time = paint.grey(duration(r.durationMs ?? 0));
|
|
286
380
|
const name = r.name.padEnd(NAME_WIDTH);
|
|
287
|
-
|
|
381
|
+
const verdict = guardVerdict(r);
|
|
382
|
+
if (verdict === 'held') {
|
|
288
383
|
say(`${paint.green(sym(mark.pass))} ${name} ${paint.grey('still holds')} ${time}`);
|
|
289
384
|
return;
|
|
290
385
|
}
|
|
291
|
-
if (
|
|
386
|
+
if (verdict === 'left out') {
|
|
292
387
|
say(`${paint.grey(sym(mark.info))} ${paint.grey(`${name} left out on purpose`)}`);
|
|
293
388
|
return;
|
|
294
389
|
}
|
|
295
|
-
|
|
390
|
+
// A question nobody answered is not painted like a bug coming back. It still keeps the run
|
|
391
|
+
// out of the green — `allClear` counts it — but the colour a person scans for should not
|
|
392
|
+
// say "regression" about something the run has no opinion on.
|
|
393
|
+
const shout = verdict === 'back' ? paint.red : paint.yellow;
|
|
394
|
+
const symbol = verdict === 'back' ? mark.fail : mark.warn;
|
|
395
|
+
say(`${shout(sym(symbol))} ${shout(`${name} ${r.message || 'this one is broken again'}`)} ${time}`);
|
|
296
396
|
if (r.failedAt) say(paint.red(` expected: ${r.failedAt}`));
|
|
297
|
-
if (r.because) say(paint.grey(` why this guard exists: ${r.because}`));
|
|
397
|
+
if (r.because && verdict !== 'unanswered') say(paint.grey(` why this guard exists: ${r.because}`));
|
|
398
|
+
else if (r.because && /** @type {any} */ (r).assertedNothing === true) {
|
|
399
|
+
say(paint.grey(` what it was meant to protect: ${r.because}`));
|
|
400
|
+
}
|
|
298
401
|
if (r.file) detail(` ${shortPath(r.file)}`);
|
|
299
402
|
}
|
|
300
403
|
|
|
@@ -336,8 +439,7 @@ export function printRunSummary(run, project, opts = {}) {
|
|
|
336
439
|
}
|
|
337
440
|
for (const g of guards) {
|
|
338
441
|
if (g.status === 'passed' || g.status === 'skipped') continue;
|
|
339
|
-
|
|
340
|
-
rows.push([g.name, what, paint.grey(g.file ? shortPath(g.file) : '')]);
|
|
442
|
+
rows.push([g.name, guardOutcome(g), paint.grey(g.file ? shortPath(g.file) : '')]);
|
|
341
443
|
}
|
|
342
444
|
if (rows.length) {
|
|
343
445
|
heading('What is not right');
|
|
@@ -532,6 +634,65 @@ export function printTrace(report) {
|
|
|
532
634
|
blank();
|
|
533
635
|
}
|
|
534
636
|
|
|
637
|
+
/**
|
|
638
|
+
* A status that means the check actually reached a verdict.
|
|
639
|
+
*
|
|
640
|
+
* The same three as `isDecided` in src/core/history.js, which does not export it. Said
|
|
641
|
+
* again rather than guessed at, because the three it leaves out matter: 'skipped', 'new'
|
|
642
|
+
* and 'missing' are checks that never ran, and reading the move from "never ran" to
|
|
643
|
+
* "passed" as a change of mind would accuse every check somebody switched back on.
|
|
644
|
+
*
|
|
645
|
+
* @param {import('../types.js').CheckStatus} status
|
|
646
|
+
* @returns {boolean}
|
|
647
|
+
*/
|
|
648
|
+
function decided(status) {
|
|
649
|
+
return status === 'passed' || status === 'changed' || status === 'failed';
|
|
650
|
+
}
|
|
651
|
+
|
|
652
|
+
/**
|
|
653
|
+
* Checks whose own recorded answers disagree, and which nothing has counted as wobbling.
|
|
654
|
+
*
|
|
655
|
+
* This is the list the register used to have no way to mention, and its absence produced
|
|
656
|
+
* the worst sentence this tool can print. Measured 2026-08-31 on a real product: a guard
|
|
657
|
+
* recorded "passed" and then "failed", `staysfixed flake --json` printed both statuses,
|
|
658
|
+
* and `staysfixed flake` answered "No check here has ever changed its mind. That is exactly
|
|
659
|
+
* how it should be." — a clean bill of health contradicted by the very file it had just
|
|
660
|
+
* read out.
|
|
661
|
+
*
|
|
662
|
+
* The register was not lying about `flakes`. It counts a wobble only when the code can be
|
|
663
|
+
* PROVED to have stood still, which needs a commit and a clean tree, and there was no such
|
|
664
|
+
* proof. But "not counted" and "did not happen" are two different facts, and the second one
|
|
665
|
+
* was being printed for the first.
|
|
666
|
+
*
|
|
667
|
+
* These are not accused of anything here. A check that passed at one commit and failed at
|
|
668
|
+
* the next is a real break, not a wobble, and calling it flaky would be as corrosive as a
|
|
669
|
+
* false failure — which is exactly why `foldRun` refuses to count it. All this does is show
|
|
670
|
+
* what is on record and say plainly that nobody can tell which it was.
|
|
671
|
+
*
|
|
672
|
+
* @param {import('../types.js').History} history
|
|
673
|
+
* @returns {import('../types.js').HistoryEntry[]}
|
|
674
|
+
*/
|
|
675
|
+
function disagreedUncounted(history) {
|
|
676
|
+
return Object.values(history)
|
|
677
|
+
.filter((e) => (e.flakes ?? 0) === 0)
|
|
678
|
+
.filter((e) => new Set((e.recent ?? []).filter(decided)).size > 1);
|
|
679
|
+
}
|
|
680
|
+
|
|
681
|
+
/**
|
|
682
|
+
* The commit the last run pinned this check's answer to, if it could pin one.
|
|
683
|
+
*
|
|
684
|
+
* `foldRun` writes it, and writes `null` when there was nothing to pin to — no commit, or a
|
|
685
|
+
* working tree with uncommitted changes in it. It is not in the `HistoryEntry` type, which
|
|
686
|
+
* is why it is read through a cast rather than off the shape.
|
|
687
|
+
*
|
|
688
|
+
* @param {import('../types.js').HistoryEntry} entry
|
|
689
|
+
* @returns {string|null}
|
|
690
|
+
*/
|
|
691
|
+
function pinnedTo(entry) {
|
|
692
|
+
const sha = /** @type {{lastSha?: string|null}} */ (entry).lastSha;
|
|
693
|
+
return typeof sha === 'string' ? sha : null;
|
|
694
|
+
}
|
|
695
|
+
|
|
535
696
|
/**
|
|
536
697
|
* The flake register, for `staysfixed flake`.
|
|
537
698
|
* @param {import('../types.js').History} history
|
|
@@ -539,14 +700,32 @@ export function printTrace(report) {
|
|
|
539
700
|
* @returns {void}
|
|
540
701
|
*/
|
|
541
702
|
export function printFlakes(history, flakeLimit = 2) {
|
|
542
|
-
const
|
|
703
|
+
const register = history ?? {};
|
|
704
|
+
const entries = wobbly(register);
|
|
705
|
+
const uncounted = disagreedUncounted(register);
|
|
543
706
|
blank();
|
|
544
|
-
|
|
707
|
+
// The all-clear is now earned rather than assumed: it needs an empty register AND a
|
|
708
|
+
// record with nothing in it that disagrees with itself.
|
|
709
|
+
if (entries.length === 0 && uncounted.length === 0) {
|
|
545
710
|
ok('No check here has ever changed its mind. That is exactly how it should be.');
|
|
546
711
|
blank();
|
|
547
712
|
return;
|
|
548
713
|
}
|
|
549
714
|
|
|
715
|
+
if (entries.length > 0) printCountedFlakes(entries, flakeLimit);
|
|
716
|
+
if (uncounted.length > 0) printUncountedDisagreements(uncounted);
|
|
717
|
+
blank();
|
|
718
|
+
}
|
|
719
|
+
|
|
720
|
+
/**
|
|
721
|
+
* The register proper: checks that changed their mind while the code demonstrably stood
|
|
722
|
+
* still.
|
|
723
|
+
*
|
|
724
|
+
* @param {import('../types.js').HistoryEntry[]} entries
|
|
725
|
+
* @param {number} flakeLimit
|
|
726
|
+
* @returns {void}
|
|
727
|
+
*/
|
|
728
|
+
function printCountedFlakes(entries, flakeLimit) {
|
|
550
729
|
heading('Checks that have changed their mind');
|
|
551
730
|
/** @type {string[][]} */
|
|
552
731
|
const rows = [[paint.grey('check'), paint.grey('wobbled'), paint.grey('last time'), paint.grey('verdict')]];
|
|
@@ -568,7 +747,46 @@ export function printFlakes(history, flakeLimit = 2) {
|
|
|
568
747
|
say(paint.red('A condemned check is not a warning to live with. Fix it, or delete it.'));
|
|
569
748
|
say(paint.grey('Once it is genuinely fixed, forgive it with: ') + paint.cyan('staysfixed flake --clear <name>'));
|
|
570
749
|
}
|
|
750
|
+
}
|
|
751
|
+
|
|
752
|
+
/**
|
|
753
|
+
* What is on record and cannot be judged.
|
|
754
|
+
*
|
|
755
|
+
* The sequence is printed rather than summarised, because the sequence is the fact — it is
|
|
756
|
+
* what `staysfixed flake --json` shows, and a person who has read that file and then reads
|
|
757
|
+
* this needs the two to agree.
|
|
758
|
+
*
|
|
759
|
+
* @param {import('../types.js').HistoryEntry[]} entries
|
|
760
|
+
* @returns {void}
|
|
761
|
+
*/
|
|
762
|
+
function printUncountedDisagreements(entries) {
|
|
763
|
+
heading('Different answers on record, and nothing can say why');
|
|
764
|
+
say(' These gave one answer on one run and a different one on the next. Nothing on record');
|
|
765
|
+
say(' proves the code stood still in between, so none of it is counted as wobbling — and');
|
|
766
|
+
say(' none of it is a clean bill of health either. It may be flakiness; it may be a real');
|
|
767
|
+
say(' break somebody then fixed. Nobody can tell from here.');
|
|
571
768
|
blank();
|
|
769
|
+
for (const entry of entries) {
|
|
770
|
+
const recent = (entry.recent ?? []).filter(decided).slice(-MAX_RECENT_STATUSES);
|
|
771
|
+
say(` ${entry.name}`);
|
|
772
|
+
say(paint.grey(` ${recent.join(' then ')} (${countText(entry.runs)} ${plural(entry.runs, 'run', 'runs')} on record)`));
|
|
773
|
+
}
|
|
774
|
+
|
|
775
|
+
// Why the register could not judge them, and it is almost always the same reason.
|
|
776
|
+
//
|
|
777
|
+
// `staysfixed flake` used to say "this folder has no commit to pin results to", which was
|
|
778
|
+
// untrue on the product this was measured against 2026-08-31: that folder had a commit.
|
|
779
|
+
// What it had as well was an uncommitted `.staysfixed/history.json` — written by the
|
|
780
|
+
// first run — which makes the tree dirty for the second one and every one after it. So a
|
|
781
|
+
// project that does not ignore this tool's own folder is blind from run two onwards, and
|
|
782
|
+
// was being told the opposite of the reason.
|
|
783
|
+
if (entries.some((entry) => pinnedTo(entry) === null)) {
|
|
784
|
+
blank();
|
|
785
|
+
say(paint.grey(' The last run could not pin these to a commit, so it had nothing to compare them at.'));
|
|
786
|
+
say(paint.grey(' Either this folder has no commit yet, or the working tree had uncommitted changes'));
|
|
787
|
+
say(paint.grey(' when the check finished — and Stays Fixed writes .staysfixed itself, so a project'));
|
|
788
|
+
say(paint.grey(' that does not ignore that folder is dirty on every run after the first.'));
|
|
789
|
+
}
|
|
572
790
|
}
|
|
573
791
|
|
|
574
792
|
/**
|