staysfixed 0.9.1 → 0.11.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +182 -0
- package/README.md +17 -5
- package/docs/getting-started.md +10 -0
- package/docs/how-v2-works.md +5 -2
- package/package.json +2 -2
- package/src/guard/api.js +107 -3
- package/src/guard/run.js +154 -20
- package/src/report/console.js +235 -17
- package/src/report/html.js +75 -19
- package/src/types.js +5 -0
- package/src/v2/adapters/android-driver.js +62 -12
- package/src/v2/adapters/contract.js +18 -4
- package/src/v2/adapters/electron.js +96 -14
- package/src/v2/adapters/http.js +264 -23
- package/src/v2/adapters/ios-driver.js +22 -4
- package/src/v2/adapters/ios.js +5 -2
- package/src/v2/adapters/isolate.js +78 -5
- package/src/v2/adapters/process.js +350 -92
- package/src/v2/adapters/web-driver.js +23 -1
- package/src/v2/adapters/web.js +42 -3
- package/src/v2/adapters/windows.js +32 -15
- package/src/v2/check.js +526 -19
- package/src/v2/cli.js +345 -3
- package/src/v2/cluster.js +112 -4
- package/src/v2/coverage.js +293 -8
- package/src/v2/detect.js +182 -9
- package/src/v2/doctor.js +253 -30
- package/src/v2/init.js +102 -10
- package/src/v2/mcp/server.js +4 -1
- package/src/v2/mcp/tools.js +291 -24
- package/src/v2/normalise.js +11 -0
- package/src/v2/observation.js +57 -5
- package/src/v2/reference.js +133 -14
- package/src/v2/refusal.js +389 -0
- package/src/v2/remote.js +24 -3
- package/src/v2/run.js +306 -16
- package/src/v2/sealed.js +14 -2
- package/src/v2/ship.js +286 -22
- package/src/v2/store.js +101 -2
- package/src/v2/types.js +5 -0
- package/src/v2/waiver.js +9 -2
- package/src/watch/panel.js +12 -1
package/src/v2/remote.js
CHANGED
|
@@ -39,6 +39,9 @@
|
|
|
39
39
|
import { spawn } from 'node:child_process';
|
|
40
40
|
import { StaysFixedError } from '../core/errors.js';
|
|
41
41
|
import { howLongItTook, joinPath, notCovered, observation, sizeBucket, timeBucket, trimForStorage } from './adapters/contract.js';
|
|
42
|
+
// Every wait here has a limit and every limit says what it was waiting for. One copy of that,
|
|
43
|
+
// shared with the adapters, rather than one per file.
|
|
44
|
+
import { boundedMs, letGoOf } from './adapters/process.js';
|
|
42
45
|
|
|
43
46
|
/** @typedef {import('./types.js').Observation} Observation */
|
|
44
47
|
/** @typedef {import('./types.js').Journey} Journey */
|
|
@@ -563,7 +566,17 @@ export function remoteRunner(opts) {
|
|
|
563
566
|
// The program itself, on the first line of standard input, where no command-line limit
|
|
564
567
|
// applies. See powerShellBootstrap for why this is not on the command line.
|
|
565
568
|
proc.stdin.write(`${Buffer.from(agentSource, 'utf8').toString('base64')}\n`);
|
|
566
|
-
|
|
569
|
+
try {
|
|
570
|
+
await hello;
|
|
571
|
+
} catch (e) {
|
|
572
|
+
// The handshake has a clock on it, but the ssh it was waiting for does not stop on its
|
|
573
|
+
// own — and while it lives, this process is reading its pipes, which keeps the event
|
|
574
|
+
// loop awake and stops the tool exiting long after it has given up and said so.
|
|
575
|
+
try { proc.kill('SIGKILL'); } catch { /* already gone */ }
|
|
576
|
+
letGoOf(proc);
|
|
577
|
+
child = null;
|
|
578
|
+
throw e;
|
|
579
|
+
}
|
|
567
580
|
say(`${host} answered: ${describeFacts(facts)}`);
|
|
568
581
|
return facts;
|
|
569
582
|
},
|
|
@@ -584,7 +597,10 @@ export function remoteRunner(opts) {
|
|
|
584
597
|
if (dead) throw dead;
|
|
585
598
|
if (!child) throw new StaysFixedError(`Cannot talk to ${host} before opening the connection.`);
|
|
586
599
|
const id = `r${++counter}`;
|
|
587
|
-
|
|
600
|
+
// Guarded, so a limit that came out of a settings file as text cannot become NaN — which
|
|
601
|
+
// `setTimeout` reads as one millisecond and which would report every call on a perfectly
|
|
602
|
+
// healthy machine as "it did not answer".
|
|
603
|
+
const timeoutMs = boundedMs(callOpts.timeoutMs ?? opts.callTimeoutMs, DEFAULT_CALL_MS);
|
|
588
604
|
const promise = new Promise((resolve, reject) => {
|
|
589
605
|
const timer = setTimeout(() => {
|
|
590
606
|
waiting.delete(id);
|
|
@@ -699,9 +715,14 @@ export function remoteRunner(opts) {
|
|
|
699
715
|
try { proc.stdin.end(); } catch { /* nothing to end */ }
|
|
700
716
|
await new Promise((resolve) => {
|
|
701
717
|
const timer = setTimeout(() => { try { proc.kill(); } catch { /* gone */ } resolve(undefined); }, 3000);
|
|
702
|
-
|
|
718
|
+
// `exit` and not `close`: `close` means nobody anywhere is holding the pipes any more,
|
|
719
|
+
// which anything ssh left behind can refuse for ever. `exit` means ssh ended, which is
|
|
720
|
+
// the thing actually being waited for.
|
|
721
|
+
proc.on('exit', () => { clearTimeout(timer); resolve(undefined); });
|
|
703
722
|
if (proc.exitCode !== null) { clearTimeout(timer); resolve(undefined); }
|
|
704
723
|
});
|
|
724
|
+
// And let go of the pipes rather than trusting them to close on their own.
|
|
725
|
+
letGoOf(proc);
|
|
705
726
|
child = null;
|
|
706
727
|
},
|
|
707
728
|
};
|
package/src/v2/run.js
CHANGED
|
@@ -23,7 +23,6 @@ import { createRequire } from 'node:module';
|
|
|
23
23
|
import { makeEvents } from '../core/events.js';
|
|
24
24
|
import { StaysFixedError, messageOf } from '../core/errors.js';
|
|
25
25
|
import {
|
|
26
|
-
diffCaptures,
|
|
27
26
|
findDuplicatePaths,
|
|
28
27
|
measureWobble,
|
|
29
28
|
mergeWobble,
|
|
@@ -33,8 +32,15 @@ import {
|
|
|
33
32
|
indexByPath,
|
|
34
33
|
wobbleStorm,
|
|
35
34
|
} from './observation.js';
|
|
36
|
-
|
|
35
|
+
// `diffCaptures` is no longer called from here directly. Everything goes through
|
|
36
|
+
// `compareAnswers`, which is that same comparison with one rule around it: an address where
|
|
37
|
+
// either side holds a refusal is not compared at all. Reaching past it would put the bug of
|
|
38
|
+
// 2026-08-31 straight back — two refusals compared equal and a product that could not start
|
|
39
|
+
// came back "Nothing that worked has changed".
|
|
40
|
+
import { compareAnswers, answeredAnything, isAnswer, refusalsIn, whyNoAnswer } from './refusal.js';
|
|
41
|
+
import { ensureStore, saveBuild, saveCapture, latestCapture, loadCapture, referenceFor, listBuilds } from './store.js';
|
|
37
42
|
import { describeRuleChange } from './normalise.js';
|
|
43
|
+
import { currentReference } from './reference.js';
|
|
38
44
|
import { clusterDifferences } from './cluster.js';
|
|
39
45
|
import { rankFindings } from './rank.js';
|
|
40
46
|
|
|
@@ -211,6 +217,17 @@ export async function runCheck(opts) {
|
|
|
211
217
|
|
|
212
218
|
// 1 — what counts as working.
|
|
213
219
|
const reference = await resolveReference(opts.store, opts.product, opts.against);
|
|
220
|
+
|
|
221
|
+
// WHICH captures of the reference build are the record.
|
|
222
|
+
//
|
|
223
|
+
// The store keeps every capture a build ever produced, and the reader took the NEWEST of
|
|
224
|
+
// them. So the moment somebody checked out the old commit and ran a check, the record the
|
|
225
|
+
// whole comparison rests on quietly moved to whatever that run happened to see. Only `ship`
|
|
226
|
+
// may decide what "working" means, and a record that drifts on its own is that rule leaking.
|
|
227
|
+
// The two captures blessed at ship time are already written down beside the cut; they are
|
|
228
|
+
// used when they are still there, and the newest is the fallback for a reference cut before
|
|
229
|
+
// this was recorded. Found by the identical-runs lane, 2026-08-31.
|
|
230
|
+
const blessedRuns = await blessedCapturePairs(opts.store, opts.product, reference);
|
|
214
231
|
say({
|
|
215
232
|
type: 'reference',
|
|
216
233
|
at: events.elapsed(),
|
|
@@ -364,7 +381,7 @@ export async function runCheck(opts) {
|
|
|
364
381
|
surface: journey.surface,
|
|
365
382
|
});
|
|
366
383
|
}
|
|
367
|
-
const stored = await storedReference(opts.store, reference.id, journey.name);
|
|
384
|
+
const stored = await storedReference(opts.store, reference.id, journey.name, blessedRuns.get(journey.name));
|
|
368
385
|
for (const problem of stored.problems) {
|
|
369
386
|
gaps.push({
|
|
370
387
|
what: `Part of the old build's record of "${journey.describe || journey.name}" could not be read.`,
|
|
@@ -562,17 +579,115 @@ export async function runCheck(opts) {
|
|
|
562
579
|
// every address the new build produced and read like a full comparison.
|
|
563
580
|
/** @type {Set<string>} */
|
|
564
581
|
const comparedAddresses = new Set();
|
|
582
|
+
// Addresses that could not be put side by side because one side of them is a refusal.
|
|
583
|
+
// Until 2026-08-31 there was no such list: a refusal was a value like any other, so two
|
|
584
|
+
// of them compared EQUAL and vanished into the silence that this tool reads as "nothing
|
|
585
|
+
// changed", while a refusal opposite a real value came back as a difference nobody
|
|
586
|
+
// caused. Both are now counted here and neither is a finding.
|
|
587
|
+
/** @type {import('./refusal.js').Uncompared[]} */
|
|
588
|
+
const uncompared = [];
|
|
589
|
+
// The same thing one level up: a whole walk that never got the product to say anything.
|
|
590
|
+
/** @type {string[]} */
|
|
591
|
+
const standardHasNoRecordOf = [];
|
|
592
|
+
/** @type {{journey: string, why: string}[]} */
|
|
593
|
+
const thisBuildWouldNotAnswer = [];
|
|
565
594
|
for (const journey of journeys) {
|
|
566
595
|
const was = before.get(journey.name);
|
|
567
596
|
const is = walked.get(journey.name);
|
|
568
597
|
if (!was || !is) continue;
|
|
569
|
-
|
|
570
|
-
|
|
571
|
-
|
|
598
|
+
|
|
599
|
+
// A WALK THAT ONLY EVER MET A REFUSAL IS NOT A SIDE OF A COMPARISON.
|
|
600
|
+
//
|
|
601
|
+
// Handled here as a whole rather than address by address because that is the shape it
|
|
602
|
+
// has. When the old build's record of a journey is nothing but refusals, EVERY address
|
|
603
|
+
// this build now answers at has nothing opposite it, so every one of them reports as
|
|
604
|
+
// having appeared out of nowhere: four findings on a two-journey fixture, thirteen on a
|
|
605
|
+
// three-route server, and the ones whose names say money or signing in land in a class
|
|
606
|
+
// no agent may wave through — so a phantom goes to a person and stays there. And the
|
|
607
|
+
// other way round is worse, not better: when THIS build is the one that would not
|
|
608
|
+
// answer, every address the old build had reports as vanished, and the real news — the
|
|
609
|
+
// product does not start — is nowhere in a list of forty findings. One sentence each,
|
|
610
|
+
// in the coverage list, is the honest form of both.
|
|
611
|
+
const standardAnswered = answeredAnything(was);
|
|
612
|
+
const thisBuildAnswered = answeredAnything(is.a);
|
|
613
|
+
if (!standardAnswered || !thisBuildAnswered) {
|
|
614
|
+
const name = journey.describe || journey.name;
|
|
615
|
+
if (!standardAnswered) standardHasNoRecordOf.push(name);
|
|
616
|
+
if (!thisBuildAnswered) {
|
|
617
|
+
const first = refusalsIn(is.a)[0];
|
|
618
|
+
thisBuildWouldNotAnswer.push({
|
|
619
|
+
journey: name,
|
|
620
|
+
why: first ? whyNoAnswer(first.value, first) : 'nothing it was asked answered.',
|
|
621
|
+
});
|
|
622
|
+
}
|
|
623
|
+
continue;
|
|
624
|
+
}
|
|
625
|
+
|
|
626
|
+
const compared = compareAnswers(was, is.a);
|
|
627
|
+
// Counted over answers only. The number goes into the closing sentence as "N addresses
|
|
628
|
+
// checked", and an address holding a refusal was never checked at anything.
|
|
629
|
+
for (const o of was.observations) if (isAnswer(o.value)) comparedAddresses.add(`${journey.name} ${o.path}`);
|
|
630
|
+
for (const o of is.a.observations) if (isAnswer(o.value)) comparedAddresses.add(`${journey.name} ${o.path}`);
|
|
631
|
+
raw.push(...compared.differences);
|
|
632
|
+
uncompared.push(...compared.uncompared);
|
|
633
|
+
}
|
|
634
|
+
|
|
635
|
+
// What the refusals cost, said in the coverage list where nothing is allowed to be
|
|
636
|
+
// skimmed past. Grouped by journey and by which side was missing, because one line per
|
|
637
|
+
// address on a product with a dead surface is a wall nobody reads.
|
|
638
|
+
const lost = uncompared.filter((u) => u.kind === 'lost');
|
|
639
|
+
const recovered = uncompared.filter((u) => u.kind === 'recovered');
|
|
640
|
+
const neverAnswered = uncompared.filter((u) => u.kind === 'never-answered');
|
|
641
|
+
for (const [kind, list] of /** @type {const} */ ([['lost', lost], ['recovered', recovered], ['never-answered', neverAnswered]])) {
|
|
642
|
+
if (list.length === 0) continue;
|
|
643
|
+
const names = unique(list.map((u) => u.journey ?? '')).filter(Boolean);
|
|
644
|
+
const where = names.length > 0 ? ` in ${names.slice(0, 4).join(', ')}${names.length > 4 ? ', and more' : ''}` : '';
|
|
645
|
+
const some = list.slice(0, 4).map((u) => u.path).join(', ');
|
|
646
|
+
const andMore = list.length > 4 ? `, and ${list.length - 4} more` : '';
|
|
647
|
+
if (kind === 'lost') {
|
|
648
|
+
gaps.push({
|
|
649
|
+
what: `${list.length} ${plural(list.length, 'address', 'addresses')} the old build answers at could not be answered by this build${where}, so ${plural(list.length, 'it was', 'they were')} not compared: ${some}${andMore}.`,
|
|
650
|
+
why: `${list[0].why} An address that used to be checked and cannot be now is coverage this build has taken away. It is not reported as a difference, because there is no answer here to differ from — but it is not a pass either.`,
|
|
651
|
+
unlockedBy: 'Get the product answering there again and run the check. Until then nothing about those addresses is being watched.',
|
|
652
|
+
});
|
|
653
|
+
} else if (kind === 'recovered') {
|
|
654
|
+
gaps.push({
|
|
655
|
+
what: `${list.length} ${plural(list.length, 'address', 'addresses')} this build answers at ${plural(list.length, 'has', 'have')} no answer in the standard${where}, so ${plural(list.length, 'it was', 'they were')} not compared: ${some}${andMore}.`,
|
|
656
|
+
why: `The build on record as working never answered here — ${list[0].why} There is nothing to hold today's answer against, so this is new coverage rather than a change. It used to arrive as a difference nobody caused.`,
|
|
657
|
+
unlockedBy: 'Ship once from a run that saw these, and from then on they are part of what "working" means and are compared like everything else.',
|
|
658
|
+
});
|
|
659
|
+
} else {
|
|
660
|
+
gaps.push({
|
|
661
|
+
what: `${list.length} ${plural(list.length, 'address', 'addresses')} answered on neither build${where}, so nothing was compared there: ${some}${andMore}.`,
|
|
662
|
+
why: `${list[0].why} Two refusals used to compare equal, which read exactly like two matching answers and counted towards "nothing has changed". They are counted here instead.`,
|
|
663
|
+
unlockedBy: 'Make the product answerable there — supply what the adapter said was missing — and these start being watched.',
|
|
664
|
+
});
|
|
665
|
+
}
|
|
572
666
|
}
|
|
667
|
+
if (standardHasNoRecordOf.length > 0) {
|
|
668
|
+
gaps.push({
|
|
669
|
+
what: `${standardHasNoRecordOf.length} ${plural(standardHasNoRecordOf.length, 'journey', 'journeys')} could not be compared, because the build on record as working never got the product to do anything there: ${standardHasNoRecordOf.join(', ')}.`,
|
|
670
|
+
why: 'Everything this build did on those journeys is new coverage, not a change: there is nothing on the old build\'s side of it. Reported as findings until 2026-08-31, which is how a product that started working came back as a pile of regressions nobody had caused.',
|
|
671
|
+
unlockedBy: 'Ship once from a run in which these journeys actually ran, and they become part of what "working" means.',
|
|
672
|
+
});
|
|
673
|
+
}
|
|
674
|
+
for (const dead of thisBuildWouldNotAnswer) {
|
|
675
|
+
gaps.push({
|
|
676
|
+
what: `"${dead.journey}" was not compared: this build never got the product to do anything there.`,
|
|
677
|
+
why: `${dead.why} The old build has a record of what this journey does and this build has none, so there is nothing to compare — which is not the same as nothing having changed, and is not a pass.`,
|
|
678
|
+
unlockedBy: 'Run that one journey on its own and see what stops it. Nothing behind it is being watched until it runs.',
|
|
679
|
+
});
|
|
680
|
+
}
|
|
681
|
+
// The two build ids, because whether they are the SAME build decides whether any of this
|
|
682
|
+
// can be a change at all. When nothing has been edited they match, the stored record of
|
|
683
|
+
// "the old build" is the previous check's own run out of that build's folder, and every
|
|
684
|
+
// flicker between the two runs used to be reported as a change nobody asked for. See the
|
|
685
|
+
// long note on subtractWobble: it files them as this build arguing with itself instead.
|
|
573
686
|
const subtraction = subtractWobble(raw, wobble, {
|
|
574
687
|
referenceWobble: referenceWobbles.length > 0 ? mergeWobble(referenceWobbles) : undefined,
|
|
575
688
|
steadyInReference: referenceWobbleMeasured && steadyInReference.length > 0 ? steadyInReference : undefined,
|
|
689
|
+
referenceBuildId: reference.id,
|
|
690
|
+
candidateBuildId: opts.candidate.id,
|
|
576
691
|
});
|
|
577
692
|
say({
|
|
578
693
|
type: 'suspicion',
|
|
@@ -665,8 +780,31 @@ export async function runCheck(opts) {
|
|
|
665
780
|
const warning = modeWarning(mode, provedLive, reference);
|
|
666
781
|
if (warning) gaps.push(...warningGaps(mode, provedLive));
|
|
667
782
|
|
|
783
|
+
// COVERAGE THIS BUILD TOOK AWAY IS NOT A PASS. An address the standard answers at and
|
|
784
|
+
// this build cannot is not a difference — there is no answer here to differ from — so it
|
|
785
|
+
// produces no finding, and before this it produced nothing at all: the run came back
|
|
786
|
+
// `ok: true` with the hole three paragraphs down in the coverage list. `recovered` and
|
|
787
|
+
// `never-answered` deliberately do NOT come in here. One is good news and the other was
|
|
788
|
+
// already true of the build on record, and neither is something this change caused.
|
|
789
|
+
const answersLost = lost.length + thisBuildWouldNotAnswer.length;
|
|
790
|
+
// BY NAME, not by adding two lists. A journey where NEITHER side reached the product is
|
|
791
|
+
// in both lists, and subtracting both counts took it off the compared total twice — one
|
|
792
|
+
// journey compared out of two came out as nought of two, which is a different and worse
|
|
793
|
+
// claim than the true one.
|
|
794
|
+
const notReallyCompared = new Set([...standardHasNoRecordOf, ...thisBuildWouldNotAnswer.map((d) => d.journey)]);
|
|
668
795
|
return finish(opts, {
|
|
669
|
-
ok:
|
|
796
|
+
ok:
|
|
797
|
+
ranked.findings.length === 0 &&
|
|
798
|
+
subtraction.newlyUnstable.length === 0 &&
|
|
799
|
+
subtraction.couldNotTell !== true &&
|
|
800
|
+
answersLost === 0 &&
|
|
801
|
+
// AND SOMETHING HAS TO HAVE BEEN COMPARED. There is already a branch above for the
|
|
802
|
+
// case where no journey had an old-build side at all; this is the same law one notch
|
|
803
|
+
// finer, for the run where every journey HAD a record and every address in it holds
|
|
804
|
+
// a refusal on one side or the other. Measured 2026-08-31: a product fixed after a
|
|
805
|
+
// reference had been cut from a crash came back with nought findings, nought
|
|
806
|
+
// addresses compared, and `ok: true`.
|
|
807
|
+
comparedAddresses.size > 0,
|
|
670
808
|
mode,
|
|
671
809
|
modeWarning: warning,
|
|
672
810
|
reference,
|
|
@@ -678,9 +816,15 @@ export async function runCheck(opts) {
|
|
|
678
816
|
summary:
|
|
679
817
|
(subtraction.couldNotTell === true ? `NO ANSWER FROM THIS RUN. ${subtraction.couldNotTellWhy} ` : '') +
|
|
680
818
|
summarise(ranked.findings, subtraction, warning, [...runNotes, ...ranked.notes], reference, provedLive, dropped, {
|
|
681
|
-
compared: comparedJourneys.length,
|
|
819
|
+
compared: comparedJourneys.length - notReallyCompared.size,
|
|
682
820
|
asked: journeys.length,
|
|
683
821
|
addresses: comparedAddresses.size,
|
|
822
|
+
// The headline has to carry this. "Nothing that worked has changed" beside a
|
|
823
|
+
// hundred addresses that could not be answered is a sentence somebody stops
|
|
824
|
+
// reading after, and the whole reason two refusals comparing equal went unnoticed
|
|
825
|
+
// for as long as it did is that the silence looked exactly like agreement.
|
|
826
|
+
unanswered: uncompared.length,
|
|
827
|
+
lost: answersLost,
|
|
684
828
|
}),
|
|
685
829
|
startedAt,
|
|
686
830
|
started,
|
|
@@ -754,6 +898,55 @@ function namesBuild(build, wanted) {
|
|
|
754
898
|
return false;
|
|
755
899
|
}
|
|
756
900
|
|
|
901
|
+
/**
|
|
902
|
+
* The two captures `ship` blessed, per journey.
|
|
903
|
+
*
|
|
904
|
+
* Empty when there is no cut record — a reference set before this was written down, or one
|
|
905
|
+
* pointed at by hand — and the caller falls back to the newest capture, which is what every
|
|
906
|
+
* run did before.
|
|
907
|
+
*
|
|
908
|
+
* @param {Store} store
|
|
909
|
+
* @param {string} product
|
|
910
|
+
* @param {BuildFingerprint|null} reference
|
|
911
|
+
* @returns {Promise<Map<string, [string, string]>>}
|
|
912
|
+
*/
|
|
913
|
+
async function blessedCapturePairs(store, product, reference) {
|
|
914
|
+
/** @type {Map<string, [string, string]>} */
|
|
915
|
+
const out = new Map();
|
|
916
|
+
if (!reference?.id) return out;
|
|
917
|
+
try {
|
|
918
|
+
const current = await currentReference(store, product);
|
|
919
|
+
if (!current?.cut || current.cut.buildId !== reference.id) return out;
|
|
920
|
+
for (const journey of current.cut.stability?.byJourney ?? []) {
|
|
921
|
+
if (journey.runs && journey.runs.length === 2) out.set(journey.journey, journey.runs);
|
|
922
|
+
}
|
|
923
|
+
} catch {
|
|
924
|
+
// A reference log that cannot be read is somebody else's problem to report. Falling back
|
|
925
|
+
// to the newest capture keeps the check running, which is the honest degradation.
|
|
926
|
+
}
|
|
927
|
+
return out;
|
|
928
|
+
}
|
|
929
|
+
|
|
930
|
+
/**
|
|
931
|
+
* One capture by its id, or null when it is no longer there.
|
|
932
|
+
*
|
|
933
|
+
* @param {Store} store
|
|
934
|
+
* @param {string} buildId
|
|
935
|
+
* @param {string} journey
|
|
936
|
+
* @param {string} id
|
|
937
|
+
* @param {(m: string) => void} onProblem
|
|
938
|
+
* @returns {Promise<Capture|null>}
|
|
939
|
+
*/
|
|
940
|
+
async function captureById(store, buildId, journey, id, onProblem) {
|
|
941
|
+
if (!id) return null;
|
|
942
|
+
try {
|
|
943
|
+
return await loadCapture(store, { buildId, journey, captureId: id });
|
|
944
|
+
} catch (e) {
|
|
945
|
+
onProblem(`The capture ${id}, which is part of what this product calls working, could not be read. ${messageOf(e)}`);
|
|
946
|
+
return null;
|
|
947
|
+
}
|
|
948
|
+
}
|
|
949
|
+
|
|
757
950
|
/**
|
|
758
951
|
* The reference build's stored record for one journey, and how steady it was.
|
|
759
952
|
*
|
|
@@ -764,13 +957,28 @@ function namesBuild(build, wanted) {
|
|
|
764
957
|
* @param {Store} store
|
|
765
958
|
* @param {string} buildId
|
|
766
959
|
* @param {string} journey
|
|
960
|
+
* @param {[string, string]} [blessed] The two captures `ship` blessed, when it wrote them down.
|
|
767
961
|
* @returns {Promise<{capture: Capture|null, wobble: Wobble|null, problems: string[]}>}
|
|
768
962
|
*/
|
|
769
|
-
async function storedReference(store, buildId, journey) {
|
|
963
|
+
export async function storedReference(store, buildId, journey, blessed) {
|
|
770
964
|
/** @type {string[]} */
|
|
771
965
|
const problems = [];
|
|
772
966
|
/** @param {string} m */
|
|
773
967
|
const onProblem = (m) => problems.push(m);
|
|
968
|
+
// The blessed pair first. These are the captures that were the product's definition of
|
|
969
|
+
// working at the moment somebody shipped, and nothing since may replace them.
|
|
970
|
+
if (blessed) {
|
|
971
|
+
const pinnedA = await captureById(store, buildId, journey, blessed[0], onProblem);
|
|
972
|
+
const pinnedB = await captureById(store, buildId, journey, blessed[1], onProblem);
|
|
973
|
+
if (pinnedA) {
|
|
974
|
+
if (!pinnedB || pinnedB.id === pinnedA.id) return { capture: pinnedA, wobble: null, problems };
|
|
975
|
+
try {
|
|
976
|
+
return { capture: pinnedA, wobble: measureWobble(pinnedA, pinnedB), problems };
|
|
977
|
+
} catch {
|
|
978
|
+
return { capture: pinnedA, wobble: null, problems };
|
|
979
|
+
}
|
|
980
|
+
}
|
|
981
|
+
}
|
|
774
982
|
const a =
|
|
775
983
|
(await latestCapture(store, { buildId, journey, run: 'a', onProblem })) ??
|
|
776
984
|
(await latestCapture(store, { buildId, journey, onProblem }));
|
|
@@ -898,13 +1106,23 @@ export function proveAgainstLive(suspicions, live, now) {
|
|
|
898
1106
|
// `observation()` in adapters/contract.js turns `covered: false` into `meta.refused`.
|
|
899
1107
|
// Filtering on `o.covered` therefore matched everything and did nothing at all — the
|
|
900
1108
|
// fix above was written correctly and then read the wrong field.
|
|
901
|
-
|
|
1109
|
+
//
|
|
1110
|
+
// The test is now the VALUE rather than `meta.refused`, and the two are not the same
|
|
1111
|
+
// thing. `meta.refused` is also set on an observation holding a real value that was only
|
|
1112
|
+
// partly read — a stdout too big to keep whole is still an answer, and filtering it out
|
|
1113
|
+
// here threw away a comparison that works perfectly well. What has to go is an
|
|
1114
|
+
// observation with no answer in it at all.
|
|
1115
|
+
const walked = capture.observations.filter((o) => isAnswer(o.value));
|
|
902
1116
|
if (walked.length === 0) continue;
|
|
903
1117
|
liveIndex.set(name, indexByPath(walked));
|
|
904
1118
|
}
|
|
905
1119
|
/** @type {Map<string, Map<string, Observation>>} */
|
|
906
1120
|
const nowIndex = new Map();
|
|
907
|
-
|
|
1121
|
+
// Answers only on this side too. Without it, an address the new build now refuses at came
|
|
1122
|
+
// back from the expensive proof stamped `proven: true` with the words "not checked" in it
|
|
1123
|
+
// as its candidate value — a refusal dressed as a re-verified regression, which is the
|
|
1124
|
+
// strongest claim this tool can make about anything.
|
|
1125
|
+
for (const [name, pair] of now) nowIndex.set(name, indexByPath(pair.a.observations.filter((o) => isAnswer(o.value))));
|
|
908
1126
|
|
|
909
1127
|
/** @type {Difference[]} */
|
|
910
1128
|
const kept = [];
|
|
@@ -922,6 +1140,32 @@ export function proveAgainstLive(suspicions, live, now) {
|
|
|
922
1140
|
if (!was && !is) continue;
|
|
923
1141
|
if (was && is && sameValue(was.value, is.value)) continue;
|
|
924
1142
|
if (!was && is) {
|
|
1143
|
+
// AN ADDRESS THE RECORD HOLDS A VALUE FOR IS NEVER "was not there before".
|
|
1144
|
+
//
|
|
1145
|
+
// A live walk of the old build answering DIFFERENTLY from its own record is drift, and
|
|
1146
|
+
// subtracting it is the whole reason this function exists. A live walk that does not
|
|
1147
|
+
// answer at that address AT ALL is not drift — it is the booted build failing to
|
|
1148
|
+
// reproduce its own record, and the two are not the same news.
|
|
1149
|
+
//
|
|
1150
|
+
// Measured 2026-08-31 on a Node API. `ship` cut the reference from a working tree with
|
|
1151
|
+
// uncommitted changes and said so; the record was filed under the build fingerprint of
|
|
1152
|
+
// the tree that was actually walked, `work-76ac0155c8b9`, and it holds
|
|
1153
|
+
// `api.GET /api/session.shape` with the value `{"token":"string"}` on disk. `bootReference`
|
|
1154
|
+
// then fetched "the old build" by a DIFFERENT key off the same reference object — its
|
|
1155
|
+
// `gitSha` — and `git archive` of that commit has no `/api/session` route in it at all,
|
|
1156
|
+
// so nothing was observed at that address. This branch then threw the recorded value
|
|
1157
|
+
// away and the run printed: `"GET /api/session / shape" is there now and was not before.`
|
|
1158
|
+
// The record was sitting in the repository saying the opposite, and the difference was
|
|
1159
|
+
// stamped `proven: true` on top, which the summary reads out as re-checked against the
|
|
1160
|
+
// old build. A confident sentence, contradicted by this tool's own evidence.
|
|
1161
|
+
//
|
|
1162
|
+
// So the record wins where the live walk is silent: the difference keeps the values it
|
|
1163
|
+
// came in with and goes back unproven, which is what the tool already says for every
|
|
1164
|
+
// journey the old build could not walk.
|
|
1165
|
+
if (d.reference !== undefined) {
|
|
1166
|
+
kept.push({ ...d, proven: false });
|
|
1167
|
+
continue;
|
|
1168
|
+
}
|
|
925
1169
|
kept.push({ ...d, kind: 'appeared', reference: undefined, candidate: is.value, proven: true });
|
|
926
1170
|
continue;
|
|
927
1171
|
}
|
|
@@ -1060,9 +1304,10 @@ function warningGaps(mode, provedLive) {
|
|
|
1060
1304
|
* @param {BuildFingerprint} reference
|
|
1061
1305
|
* @param {boolean} provedLive
|
|
1062
1306
|
* @param {number} dropped Suspicions the old build turned out to have as well.
|
|
1063
|
-
* @param {{compared: number, asked: number, addresses: number
|
|
1064
|
-
* this sentence covers: journeys that
|
|
1065
|
-
* addresses really
|
|
1307
|
+
* @param {{compared: number, asked: number, addresses: number, unanswered?: number, lost?: number}} how
|
|
1308
|
+
* How much of the run this sentence covers: journeys that were really put beside the old
|
|
1309
|
+
* build, journeys asked for, the addresses really compared, and the addresses that could
|
|
1310
|
+
* not be compared because one side of them was a refusal rather than an answer.
|
|
1066
1311
|
* @returns {string}
|
|
1067
1312
|
*/
|
|
1068
1313
|
function summarise(findings, subtraction, warning, notes, reference, provedLive, dropped, how) {
|
|
@@ -1076,7 +1321,17 @@ function summarise(findings, subtraction, warning, notes, reference, provedLive,
|
|
|
1076
1321
|
missed > 0
|
|
1077
1322
|
? ` ${how.compared} of ${how.asked} journeys had anything on the old build's side to be compared against; the other ${missed} ${plural(missed, 'was', 'were')} not compared at all, and ${plural(missed, 'is', 'are')} named in the coverage list.`
|
|
1078
1323
|
: '';
|
|
1079
|
-
if (
|
|
1324
|
+
if (subtraction.sameBuild === true) {
|
|
1325
|
+
// The first sentence is the only one some readers get, so it may not be "Nothing that
|
|
1326
|
+
// worked has changed. N addresses checked against the stored record of 1.0.0" when the
|
|
1327
|
+
// record and the run are the same build. Nothing was held against anything: the tool ran
|
|
1328
|
+
// the shipped build again and compared it with itself. Measured 2026-08-31 on an untouched
|
|
1329
|
+
// Next.js app, where that sentence sat on top of a comparison that had no other side.
|
|
1330
|
+
parts.push(
|
|
1331
|
+
`This is the build that is already on record as working, run again and compared with itself, so nothing here could be a change. ` +
|
|
1332
|
+
`${how.addresses} ${plural(how.addresses, 'address was', 'addresses were')} watched.${reach}`,
|
|
1333
|
+
);
|
|
1334
|
+
} else if (findings.length === 0 && subtraction.newlyUnstable.length > 0) {
|
|
1080
1335
|
// Findings and newly unpredictable addresses are two different lists, and only the first
|
|
1081
1336
|
// one was ever in the headline. A run with no findings and four addresses that have
|
|
1082
1337
|
// stopped sitting still opened with "Nothing that worked has changed", which is the
|
|
@@ -1088,6 +1343,22 @@ function summarise(findings, subtraction, warning, notes, reference, provedLive,
|
|
|
1088
1343
|
`Nothing behaves differently, but ${n} ${plural(n, 'address', 'addresses')} that used to give the same answer every time ${plural(n, 'does', 'do')} not any more. ` +
|
|
1089
1344
|
`That is a change too: something is now unpredictable that was not. ${how.addresses} ${plural(how.addresses, 'address was', 'addresses were')} compared against ${against}.${reach}`,
|
|
1090
1345
|
);
|
|
1346
|
+
} else if (findings.length === 0 && how.addresses === 0) {
|
|
1347
|
+
// Nought compared is never a pass. "Nothing that worked has changed. 0 addresses
|
|
1348
|
+
// checked" is the exact sentence measured on 2026-08-31 over a product that threw on its
|
|
1349
|
+
// first line, and it is arithmetically true and completely false as an answer.
|
|
1350
|
+
parts.push(
|
|
1351
|
+
`NO ANSWER FROM THIS RUN. Not one address could be put beside ${against}: every one of them holds a refusal on one side or the other, so nothing at all was compared. This is not a pass and not a failure.${reach}`,
|
|
1352
|
+
);
|
|
1353
|
+
} else if (findings.length === 0 && (how.lost ?? 0) > 0) {
|
|
1354
|
+
// The all-clear may not be said over coverage this build took away. It is not a finding
|
|
1355
|
+
// — there is no answer here to differ from — and until 2026-08-31 that made it nothing
|
|
1356
|
+
// at all: the headline read "Nothing that worked has changed" and the hole sat three
|
|
1357
|
+
// paragraphs down in a list.
|
|
1358
|
+
const n = how.lost ?? 0;
|
|
1359
|
+
parts.push(
|
|
1360
|
+
`Nothing that COULD be compared has changed — but ${n} ${plural(n, 'address', 'addresses')} the old build answers at could not be answered by this build at all, so ${plural(n, 'it was', 'they were')} not compared. That is coverage this build has taken away, and it is not a pass. ${how.addresses} ${plural(how.addresses, 'address was', 'addresses were')} really put beside ${against}.${reach}`,
|
|
1361
|
+
);
|
|
1091
1362
|
} else if (findings.length === 0) {
|
|
1092
1363
|
parts.push(`Nothing that worked has changed. ${how.addresses} ${plural(how.addresses, 'address', 'addresses')} checked against ${against}.${reach}`);
|
|
1093
1364
|
} else {
|
|
@@ -1098,6 +1369,20 @@ function summarise(findings, subtraction, warning, notes, reference, provedLive,
|
|
|
1098
1369
|
reach,
|
|
1099
1370
|
);
|
|
1100
1371
|
}
|
|
1372
|
+
// Said in the same breath as the headline, never further down. A refusal is not an answer,
|
|
1373
|
+
// so an address holding one was not checked at all — and until 2026-08-31 two of them
|
|
1374
|
+
// compared equal, which is silence, which reads exactly like agreement. This sentence is
|
|
1375
|
+
// what makes the difference visible to somebody who reads one line.
|
|
1376
|
+
const unanswered = how.unanswered ?? 0;
|
|
1377
|
+
if (unanswered > 0) {
|
|
1378
|
+
const lostCount = how.lost ?? 0;
|
|
1379
|
+
parts.push(
|
|
1380
|
+
`${unanswered} ${plural(unanswered, 'address', 'addresses')} could not be compared at all, because one side of ${plural(unanswered, 'it', 'them')} is a refusal rather than an answer` +
|
|
1381
|
+
(lostCount > 0
|
|
1382
|
+
? `, and ${lostCount} of ${plural(unanswered, 'those is', 'those are')} coverage this build has taken away — the old build answers there and this one does not.`
|
|
1383
|
+
: '. Nothing about them is being watched, and none of them is a finding.'),
|
|
1384
|
+
);
|
|
1385
|
+
}
|
|
1101
1386
|
parts.push(subtraction.note);
|
|
1102
1387
|
if (dropped > 0) {
|
|
1103
1388
|
parts.push(
|
|
@@ -1298,7 +1583,12 @@ function touchMap(walked) {
|
|
|
1298
1583
|
*/
|
|
1299
1584
|
function steadyPaths(capture, wobble) {
|
|
1300
1585
|
const unstable = new Set(wobble.unstable);
|
|
1301
|
-
|
|
1586
|
+
// Answers only. An address that held a refusal on both runs is not an address the build
|
|
1587
|
+
// answered the same way twice; it is an address the build was never able to answer, and
|
|
1588
|
+
// counting it as steady is the same mistake in miniature that let `ship` print "all 7
|
|
1589
|
+
// addresses it was watched at answered the same way twice" about a product that threw on
|
|
1590
|
+
// its first line (measured 2026-08-31).
|
|
1591
|
+
return capture.observations.filter((o) => isAnswer(o.value)).map((o) => o.path).filter((p) => !unstable.has(p));
|
|
1302
1592
|
}
|
|
1303
1593
|
|
|
1304
1594
|
/**
|
package/src/v2/sealed.js
CHANGED
|
@@ -299,11 +299,21 @@ export function classify(finding, opts = {}) {
|
|
|
299
299
|
* The refusal, written out for whoever reads it — an agent that has just been told no, or a
|
|
300
300
|
* person reading the closing summary.
|
|
301
301
|
*
|
|
302
|
+
* The verdict is identical either way; only the last line moves. It used to read "No agent
|
|
303
|
+
* can wave this through … put it in front of a person", which is exactly right for an agent
|
|
304
|
+
* and says nothing to the person who has just typed `staysfixed waive` themselves — they ARE
|
|
305
|
+
* the person it points at, and the sentence reads as a rule about somebody else with an
|
|
306
|
+
* obvious loophole. The rule has no loophole: a sealed class is refused to everybody, and
|
|
307
|
+
* from 2026-08-31, when `waive` became a command, it has had to say so to both readers.
|
|
308
|
+
*
|
|
302
309
|
* @param {SealedVerdict} verdict
|
|
303
310
|
* @param {Finding} [finding]
|
|
311
|
+
* @param {'agent'|'person'} [audience] Defaults to an agent: this file is called from the
|
|
312
|
+
* gate, and the gate is reached over MCP unless the
|
|
313
|
+
* command line says otherwise.
|
|
304
314
|
* @returns {string}
|
|
305
315
|
*/
|
|
306
|
-
export function sayRefusal(verdict, finding) {
|
|
316
|
+
export function sayRefusal(verdict, finding, audience) {
|
|
307
317
|
const lines = [`Refused. ${verdict.why}`];
|
|
308
318
|
if (finding?.title) lines.push(` ${trim(finding.title, 200)}`);
|
|
309
319
|
if (verdict.matched.length > 0) {
|
|
@@ -311,7 +321,9 @@ export function sayRefusal(verdict, finding) {
|
|
|
311
321
|
}
|
|
312
322
|
lines.push(
|
|
313
323
|
'',
|
|
314
|
-
|
|
324
|
+
audience === 'person'
|
|
325
|
+
? 'Nothing can wave this through — not you here, and not an agent — and asking again in different words will get the same answer. Fix it, or take it to whoever owns this and say plainly what changed.'
|
|
326
|
+
: 'No agent can wave this through, whatever the reason, and asking again in different words will get the same answer. Fix it, or put it in front of a person and say plainly what changed.'
|
|
315
327
|
);
|
|
316
328
|
return lines.join('\n');
|
|
317
329
|
}
|