staysfixed 0.4.0 → 0.6.0

package/src/v2/observation.js

@@ -78,6 +78,17 @@ const MAX_PATH_LENGTH = 512;
 /** Deepest value we will store. Past this something is recursing, not observing. */
 const MAX_VALUE_DEPTH = 64;
 
+/**
+ * The share of its own addresses a build may disagree with itself about before the run stops
+ * counting as a measurement at all. Half is not a tuned number and nothing depends on its
+ * exact value: it is the point past which more of the comparison has been thrown away than
+ * kept, and no answer computed from what is left deserves to be called clean.
+ */
+const STORM_SHARE = 0.5;
+
+/** Below this many addresses the share means nothing — three out of four is not a storm. */
+const STORM_FLOOR = 12;
+
 /** Control characters and newlines, which would break the store and every log line. */
 const CONTROL_CHARS = /[\u0000-\u001f\u007f]/;
 
@@ -787,6 +798,42 @@ export function mergeWobble(wobbles) {
   };
 }
 
+/**
+ * When a wobble measurement stops being a measurement.
+ *
+ * Subtraction is set subtraction: a difference at an address the build cannot answer the same
+ * way twice is dropped. That is right, and it has one failure shape, which is the worst shape
+ * this tool has. If the second run of the new build FALLS OVER — the app crashed half way, a
+ * port was taken, a device went to sleep, a first run wrote a cache the second one read — then
+ * most of the addresses the first run answered are missing from the second, every one of them
+ * is filed as unsteady, every real difference at them is subtracted, and the run ends
+ * "nothing that already worked has changed". Confident, clean, and about nothing.
+ *
+ * So the share is looked at. A product that disagrees with itself about a handful of addresses
+ * is normal — a timestamp, an id, a port. A product that disagrees with itself about MOST of
+ * them did not wobble; something went wrong with the run. This is not a tolerance: no number
+ * here decides whether any difference is real. It decides one thing only — whether this run is
+ * entitled to say the word "clean".
+ *
+ * @param {Wobble} wobble
+ * @returns {{stormy: boolean, share: number, looked: number, vanished: number, why: string}}
+ */
+export function wobbleStorm(wobble) {
+  const unstable = wobble.unstable.length;
+  const looked = unstable + wobble.steady;
+  const vanished = wobble.entries.filter((e) => e.kind === 'vanished').length;
+  const share = looked === 0 ? 0 : unstable / looked;
+  if (!wobble.measured || looked < STORM_FLOOR || share <= STORM_SHARE) {
+    return { stormy: false, share, looked, vanished, why: '' };
+  }
+  const percent = Math.round(share * 100);
+  const why =
+    `The new build was run twice and gave a different answer at ${unstable} of the ${looked} addresses it was asked about — ${percent}% of them` +
+    (vanished > 0 ? `, and ${vanished} address${vanished === 1 ? '' : 'es'} the first run answered were missing from the second altogether` : '') +
+    '. That is not a product wobbling; that is a run that went wrong. Everything it disagreed with itself about is dropped before anything is compared, so on this run the comparison covered almost nothing. This is not a pass and not a failure — there is no answer here. Run it again on a quiet machine, and if it happens twice, something in the product or its setup does not survive being started a second time.';
+  return { stormy: true, share, looked, vanished, why };
+}
+
 /**
  * Subtract the measured noise from the differences.
  *
@@ -807,6 +854,12 @@ export function mergeWobble(wobbles) {
  */
 export function subtractWobble(differences, wobble, opts = {}) {
   const unstableNow = new Set(wobble.unstable);
+  // NOT symmetric, and that is deliberate. Subtracting the OLD build's wobble as well was
+  // tried on 2026-08-30 and taken straight back out: a path the old build answered randomly
+  // and the new build now answers the same way every time is a REAL change — somebody made
+  // something deterministic, or hard-coded what used to vary — and subtracting the old
+  // build's wobble is exactly what would hide it. Where both builds wobble at a path, the
+  // candidate's own wobble already covers it, so nothing is lost by leaving this alone.
 
   /** @type {Difference[]} */
   const real = [];
@@ -838,13 +891,21 @@ export function subtractWobble(differences, wobble, opts = {}) {
     });
   }
 
-  return {
+  const storm = wobbleStorm(wobble);
+  /** @type {WobbleSubtraction} */
+  const out = {
     real,
     noise,
     newlyUnstable,
     couldTellNewlyUnstable: couldTell,
     note: subtractionNote(wobble, couldTell, real.length, noise.length, newlyUnstable.length),
   };
+  if (storm.stormy) {
+    out.couldNotTell = true;
+    out.couldNotTellWhy = storm.why;
+    out.note = `${storm.why} ${out.note}`;
+  }
+  return out;
 }
 
 /**

package/src/v2/remote.js

@@ -38,7 +38,7 @@
 
 import { spawn } from 'node:child_process';
 import { StaysFixedError } from '../core/errors.js';
-import { joinPath, notCovered, observation, sizeBucket, timeBucket, trimForStorage } from './adapters/contract.js';
+import { howLongItTook, joinPath, notCovered, observation, sizeBucket, timeBucket, trimForStorage } from './adapters/contract.js';
 
 /** @typedef {import('./types.js').Observation} Observation */
 /** @typedef {import('./types.js').Journey} Journey */
@@ -629,13 +629,12 @@ export function remoteRunner(opts) {
             journey: journey.name,
             surface,
           }));
-          seen.push(observation({
+          seen.push(howLongItTook({
             channel: 'counters',
             path: joinPath('remote', host, journey.name, String(index), 'took'),
-            value: timeBucket(result.ms),
-            says: `On ${host}, "${label}" took ${timeBucket(result.ms)}.`,
+            ms: result.ms,
+            what: `On ${host}, "${label}"`,
             journey: journey.name,
-            surface,
           }));
         } catch (error) {
           // The link went. Everything from here on is unchecked, and it says so.

package/src/v2/run.js

@@ -24,6 +24,7 @@ import { makeEvents } from '../core/events.js';
 import { StaysFixedError, messageOf } from '../core/errors.js';
 import {
   diffCaptures,
+  findDuplicatePaths,
   measureWobble,
   mergeWobble,
   unmeasuredWobble,
@@ -104,6 +105,9 @@ const VERSION = /** @type {{version?: string}} */ (require('../../package.json')
  * @property {string} product              One repo can build five products. This names one.
  * @property {BuildFingerprint} candidate  The build you just made.
  * @property {Journey[]} journeys
+ * @property {CoverageGap[]} [gaps]      Holes found before any journey ran — an adapter that
+ *                                       fell over while listing what it would walk, a name
+ *                                       that matched nothing. They belong in the coverage.
  * @property {Walker} walk
  * @property {string} cwd                  Project root — where the working diff is read.
  * @property {(candidate: BuildFingerprint, ctx: {events?: CheckEvents, signal?: AbortSignal}) => Promise<LiveBuild|null>} [bootReference]
@@ -178,7 +182,7 @@ export async function runCheck(opts) {
 
   const journeys = (opts.journeys ?? []).filter((j) => !j.skip);
   /** @type {CoverageGap[]} */
-  const gaps = [];
+  const gaps = [...(opts.gaps ?? [])];
   for (const skipped of (opts.journeys ?? []).filter((j) => j.skip)) {
     // A switched-off journey is missing coverage, never a pass. Anything else
     // lets a product go quiet by having its checks turned off one at a time.
@@ -227,6 +231,11 @@ export async function runCheck(opts) {
     const walked = new Map();
     /** @type {Map<string, Capture>} */
     const before = new Map();
+    // Counted rather than inferred. Both the live build and the stored record carry the SAME
+    // build id — they are two ways of looking at one build — so nothing about a capture in
+    // `before` says which of the two it came from, and the mode has to be recorded as it
+    // happens or not at all.
+    let liveWalks = 0;
     /** @type {Wobble[]} */
     const wobbles = [];
     /** @type {Wobble[]} */
@@ -240,6 +249,7 @@ export async function runCheck(opts) {
       say({ type: 'journey:start', at: events.elapsed(), journey: journey.name, message: `Walking ${journey.describe || journey.name}.` });
 
       const a = await walkOnce(opts, journey, opts.candidate, 'a', 'candidate', undefined, events);
+      gaps.push(...duplicateGaps(a.observations, journey));
       const b = await walkOnce(opts, journey, opts.candidate, 'b', 'candidate', undefined, events);
       const wobble = measureWobble(a, b);
       walked.set(journey.name, { a, b, wobble });
@@ -264,9 +274,34 @@ export async function runCheck(opts) {
         // switched off, and nothing in the output would say so.
         const wasA = await walkOnce(opts, journey, live.build, 'a', 'reference', live, events);
         const wasB = await walkOnce(opts, journey, live.build, 'b', 'reference', live, events);
-        before.set(journey.name, wasA);
-        referenceWobbles.push(measureWobble(wasA, wasB));
-        continue;
+        // The old build being ON this machine is not the same as the old build having been
+        // WALKED. When every observation it came back with is a hole, it was not walked, and
+        // treating that as the reference makes the whole product look newly invented: every
+        // address in the new build has nothing opposite it, so every one of them 'appeared'.
+        //
+        // Measured on Terminal Deck's Android app on 2026-08-30. `--paired` exports the old
+        // commit with `git archive`; an APK is a build output and is gitignored, so the export
+        // has no app in it; the adapter honestly reported one hole per journey; and the run
+        // came back with seventeen sealed escalations claiming the sign-in screen and every
+        // permission had appeared out of nowhere, with nothing anywhere saying the old build
+        // had never run. Falling back to the stored record here is weaker and says so, which
+        // is the whole difference between a weaker answer and a wrong one.
+        if (wasA.observations.some((o) => o.meta?.refused !== true)) {
+          before.set(journey.name, wasA);
+          referenceWobbles.push(measureWobble(wasA, wasB));
+          liveWalks += 1;
+          continue;
+        }
+        gaps.push({
+          what: `The old build could not be walked for "${journey.describe || journey.name}", so this was not a paired comparison after all.`,
+          why:
+            `${nameOf(reference)} was put back on this machine, and then there was nothing there to run: ${
+              wasA.observations[0]?.meta?.describe ?? 'the adapter could not open it'
+            }. This usually means the product is BUILT rather than committed — an APK, a .app, a packaged desktop app — and a checkout of the old commit does not contain one.`,
+          unlockedBy:
+            'Build the old commit before the run, or point the settings at a kept copy of the old build\'s artifact. Until then this journey falls back to the record the old build left last time.',
+          surface: journey.surface,
+        });
       }
       const stored = await storedReference(opts.store, reference.id, journey.name);
       if (!stored.capture) {
@@ -279,6 +314,21 @@ export async function runCheck(opts) {
         continue;
       }
       before.set(journey.name, stored.capture);
+      // The rules stamp exists so a run can notice this, and until 2026-08-30 nothing ever
+      // read it. A stored capture normalised under one set of rules compared against a fresh
+      // one normalised under another produces differences that are about the RULES — either a
+      // wall of noise that reads like a regression, or, when the change was to add a rule,
+      // quiet where there should not be any. Either way the reader has to be told.
+      if (stored.capture.rules && a.rules && stored.capture.rules !== a.rules) {
+        gaps.push({
+          what: `"${journey.describe || journey.name}" is being compared across a change to the normalisation rules.`,
+          why:
+            `The stored record of the old build was tidied up by rule set ${stored.capture.rules} and this run used ${a.rules}. ` +
+            'Some of what you see may be the rules changing rather than the product, and a rule that was added since could be covering something up.',
+          unlockedBy: 'Run a paired check, which walks the old build live under today\'s rules, or ship again to cut a fresh reference.',
+          surface: journey.surface,
+        });
+      }
       if (stored.wobble) steadyInReference.push(...steadyPaths(stored.capture, stored.wobble));
       else referenceWobbleMeasured = false;
     }
@@ -337,14 +387,30 @@ export async function runCheck(opts) {
       count: subtraction.real.length,
       message: subtraction.note,
     });
+    // A wobble big enough to swallow the comparison is not a result. It is recorded as a hole
+    // here and it takes the verdict down at the bottom of this function, because the one thing
+    // that must never come out of it is a clean sentence resting on a subtraction that removed
+    // most of what was looked at.
+    if (subtraction.couldNotTell === true) {
+      gaps.push({
+        what: 'This run could not tell you anything, because the new build did not answer the same way twice.',
+        why: subtraction.couldNotTellWhy ?? 'Most of the addresses it was asked about were unsteady, so almost everything was dropped before it was compared.',
+        unlockedBy: 'Run it again when the machine is quiet. If it happens twice, look at what the product does differently on a second start.',
+      });
+    }
     stop();
 
     // 5 — expensive proof, only where it is owed. Everything the live old build
     // does too is dropped silently and counted. That silence is the point: it is
     // what keeps this list short enough to read every word of.
     let survivors = subtraction.real;
-    const mode = /** @type {'paired'|'stored-record'} */ (live ? 'paired' : 'stored-record');
-    let provedLive = Boolean(live);
+    // Booting the old build is not the same as having walked it. When every live walk came
+    // back holes-only — a built artifact that no checkout of the old commit contains — the
+    // run fell back to the stored record above, and calling that a paired run would be the
+    // report's single most misleading sentence. See the gap pushed in the walk loop.
+    const walkedLive = liveWalks > 0;
+    const mode = /** @type {'paired'|'stored-record'} */ (walkedLive ? 'paired' : 'stored-record');
+    let provedLive = walkedLive;
     // How many suspicions the old build turned out to have as well. Naming this
     // number is what makes the short list believable: it says how much work the
     // expensive half did rather than leaving the reader to assume it did none.
@@ -403,7 +469,7 @@ export async function runCheck(opts) {
     if (warning) gaps.push(...warningGaps(mode, provedLive));
 
     return finish(opts, {
-      ok: ranked.findings.length === 0 && subtraction.newlyUnstable.length === 0,
+      ok: ranked.findings.length === 0 && subtraction.newlyUnstable.length === 0 && subtraction.couldNotTell !== true,
       mode,
       modeWarning: warning,
       reference,
@@ -412,7 +478,9 @@ export async function runCheck(opts) {
       noise: subtraction.noise.length,
       newlyUnstable: subtraction.newlyUnstable,
       coverage: foldCoverage(walked, journeys, gaps),
-      summary: summarise(ranked.findings, subtraction, wobble, warning, ranked.notes, reference, provedLive, dropped),
+      summary:
+        (subtraction.couldNotTell === true ? `NO ANSWER FROM THIS RUN. ${subtraction.couldNotTellWhy} ` : '') +
+        summarise(ranked.findings, subtraction, wobble, warning, ranked.notes, reference, provedLive, dropped),
       startedAt,
       started,
       events,
@@ -506,6 +574,33 @@ async function storedReference(store, buildId, journey) {
   }
 }
 
+/**
+ * Two facts written down at one address, with two different answers.
+ *
+ * Every index in this engine keeps the FIRST observation at a path and ignores the rest, so
+ * the second fact has no address of its own: it is never compared with anything, and a door
+ * that broke behind it is invisible while the run still says "nothing that already worked has
+ * changed". The detector for this existed from the first day of v2 and until 2026-08-30
+ * nothing ever called it, which is why it is a named hole now rather than a comment.
+ *
+ * Identical repeats are not reported. Two log lines that tidy down to the same address AND the
+ * same value hide nothing, and reporting those would bury the ones that do.
+ *
+ * @param {Observation[]} observations
+ * @param {Journey} journey
+ * @returns {CoverageGap[]}
+ */
+export function duplicateGaps(observations, journey) {
+  return findDuplicatePaths(observations).map((clash) => ({
+    what: `Two different answers were written down at the same address, ${clash.path}, while walking "${journey.describe || journey.name}".`,
+    why:
+      `Only the first is kept, so ${clash.values.slice(1).map((v) => JSON.stringify(v)).join(' and ')} ` +
+      `${clash.values.length > 2 ? 'were' : 'was'} never compared against anything at all. Whatever produced that address is giving one name to more than one thing.`,
+    unlockedBy: 'The adapter that made that address has to give those two things two different names.',
+    surface: journey.surface,
+  }));
+}
+
 /**
  * @param {CheckRun} opts
  * @param {BuildFingerprint} reference
@@ -569,7 +664,28 @@ async function walkOnce(opts, journey, build, run, which, live, events) {
 export function proveAgainstLive(suspicions, live, now) {
   /** @type {Map<string, Map<string, Observation>>} */
   const liveIndex = new Map();
-  for (const [name, capture] of live) liveIndex.set(name, indexByPath(capture.observations));
+  for (const [name, capture] of live) {
+    // A capture that came back with nothing it could actually observe is NOT a walk of the
+    // old build, and treating it as one is the worst mistake this function can make: every
+    // stored before-value is thrown away, every difference is relabelled 'appeared' with no
+    // before-value at all, and the whole lot is stamped proven, which the report then reads
+    // out as "re-checked against the old build booted live, so none of it is drift".
+    //
+    // Measured on Terminal Deck's Android app on 2026-08-30. The old build is exported with
+    // `git archive`, an APK is a build output and is gitignored, so the exported checkout has
+    // no APK in it, prepare gives up, and the adapter correctly returns one uncovered
+    // observation saying so. A control that went from greyed out to usable — the exact
+    // regression the run was meant to catch — was reported as a control that had appeared out
+    // of nowhere, with `false` never mentioned. Any platform whose artifact is built rather
+    // than committed hits this, not only phones.
+    // `covered` is the ADAPTER's word for this and it does not survive onto an Observation:
+    // `observation()` in adapters/contract.js turns `covered: false` into `meta.refused`.
+    // Filtering on `o.covered` therefore matched everything and did nothing at all — the
+    // fix above was written correctly and then read the wrong field.
+    const walked = capture.observations.filter((o) => o.meta?.refused !== true);
+    if (walked.length === 0) continue;
+    liveIndex.set(name, indexByPath(walked));
+  }
   /** @type {Map<string, Map<string, Observation>>} */
   const nowIndex = new Map();
   for (const [name, pair] of now) nowIndex.set(name, indexByPath(pair.a.observations));

package/src/v2/sealed.js

@@ -205,8 +205,14 @@ const SHORTEST_GUARD_NAME = 6;
 /** How much of one value is read. A whole HTTP body would drown the match in noise. */
 const VALUE_CHARS = 400;
 
-/** How many differences of one finding are read. Clusters can hold hundreds. */
-const MAX_DIFFERENCES_READ = 80;
+// EVERY difference of a finding is read, and there is deliberately no ceiling on that.
+// Until 2026-08-30 this file read the first eighty and stopped, which meant a cluster of
+// three hundred addresses could hold the word `refund` at address two hundred and be
+// classified ordinary — waivable by an agent, never seen by a person. A cap here is not a
+// performance decision, it is a hole in the one gate that cannot have one. The cost is a
+// substring search over text already in memory: the values are trimmed to VALUE_CHARS
+// before they are searched, so reading them all costs a constant multiple of the finding
+// itself, which the caller is already holding.
 
 // ---------------------------------------------------------------------------
 // The answer
@@ -340,7 +346,6 @@ export function sayRefusal(verdict, finding) {
  *
  * @typedef {object} ReadFinding
  * @property {{text: string, where: string}[]} pieces
- * @property {string} all                 Every piece joined, for one fast test.
  * @property {Set<Channel>} channels
  * @property {Difference[]} differences
  */
@@ -370,7 +375,7 @@ function readFinding(finding) {
   for (const file of finding.nearFiles ?? []) add(file, 'a source file this points at');
   for (const p of finding.paths ?? []) add(p, p);
 
-  const differences = (finding.differences ?? []).slice(0, MAX_DIFFERENCES_READ);
+  const differences = finding.differences ?? [];
   for (const d of differences) {
     add(d.path, d.path);
     add(d.describe, d.path);
@@ -386,8 +391,7 @@ function readFinding(finding) {
 
   return {
     pieces,
-    all: pieces.map((p) => p.text).join('\n'),
-    channels: new Set((finding.differences ?? []).map((d) => d.channel)),
+    channels: new Set(differences.map((d) => d.channel)),
     differences,
   };
 }