staysfixed 0.7.2 → 0.8.0

package/src/v2/remote.js

@@ -57,9 +57,30 @@ import { howLongItTook, joinPath, notCovered, observation, sizeBucket, timeBucke
  */
 export const SENTINEL = '#SF#';
 
-/** The kinds of far side this file knows how to start. */
+/**
+ * The kinds of far side this file knows how to start.
+ *
+ * Checked at runtime rather than only in the types, because the types are not there when it
+ * matters. `farSideCommand` branches on 'windows' and treats everything else as posix, so a
+ * kind spelled 'win' used to be handed the Node bootstrap and sent to a Windows box, where it
+ * failed several seconds later as "node: not found" — a message about the far machine for a
+ * mistake made on this one.
+ */
 export const RUNNER_KINDS = /** @type {const} */ (['posix', 'windows']);
 
+/**
+ * @param {unknown} kind
+ * @returns {RunnerKind}
+ */
+function checkKind(kind) {
+  if (!(/** @type {readonly unknown[]} */ (RUNNER_KINDS).includes(kind))) {
+    throw new StaysFixedError(`There is no far side called "${String(kind)}".`, {
+      hint: `The kinds this file can start are: ${RUNNER_KINDS.join(', ')}.`,
+    });
+  }
+  return /** @type {RunnerKind} */ (kind);
+}
+
 /** @typedef {'posix'|'windows'} RunnerKind */
 
 /**
@@ -191,7 +212,7 @@ export function nodeBootstrap() {
  * @returns {string}
  */
 export function farSideCommand(kind, opts = {}) {
-  if (kind === 'windows') {
+  if (checkKind(kind) === 'windows') {
     const encoded = encodePowerShell(powerShellBootstrap());
     if (opts.psPath) return `exec "${opts.psPath}" -NoProfile -NonInteractive -EncodedCommand ${encoded}`;
     const candidates = POWERSHELL_PATHS.map((p) => `"${p}"`).join(' ');
@@ -302,6 +323,12 @@ export function makeFrames() {
  * Written as text rather than shipped as a file because it must never be installed. It exists
  * in the memory of one `node -e` for the length of one run.
  *
+ * `read` says how long the file really was and whether it cut it. It used to hand back the
+ * first 64K with nothing to say it had stopped there, and two builds of a product whose file
+ * differs only after byte 65536 would then be compared, byte for byte, on two identical
+ * halves — and reported as unchanged. A cap that cannot be seen from the outside is not a cap,
+ * it is a wrong answer.
+ *
  * @returns {string}
  */
 export function posixAgentScript() {
@@ -331,7 +358,11 @@ const ops = {
     done({ ok: true, found });
   },
   read: (req, done) => {
-    try { done({ ok: true, text: fs.readFileSync(req.file, 'utf8').slice(0, req.limit || 65536) }); }
+    try {
+      const all = fs.readFileSync(req.file, 'utf8');
+      const cap = req.limit || 65536;
+      done({ ok: true, text: all.slice(0, cap), length: all.length, truncated: all.length > cap });
+    }
     catch (e) { done({ ok: false, error: String(e.message) }); }
   },
   sh: (req, done) => {
@@ -422,7 +453,7 @@ process.stdin.on('end', () => process.exit(0));
  * @param {RemoteRunnerOptions} opts
  */
 export function remoteRunner(opts) {
-  const kind = opts.kind ?? 'posix';
+  const kind = checkKind(opts.kind ?? 'posix');
   const host = opts.host;
   const say = opts.log ?? (() => {});
   const surface = opts.surface;
@@ -740,7 +771,13 @@ export function describeFacts(f) {
 /**
  * @typedef {object} RemoteDescription
  * @property {string} host
- * @property {boolean} reachable
+ * @property {boolean} reachable          A shell on that machine answers. NOT the same
+ *   question as `runnerStarted`, and folding the two together was a real bug: a machine with
+ *   ssh working and no Node on it came back as "could not be reached", and the fix offered
+ *   was to go and check the ssh config that already works.
+ * @property {boolean} runnerStarted      The small program this tool pushes down the
+ *   connection actually ran there. False on a reachable machine means Node is missing or too
+ *   old, and everything below is unknown for that reason rather than because nothing answered.
  * @property {string} how                 Plain English: what answered, or why nothing did.
  * @property {string|null} os
  * @property {boolean} windows            A real Windows desktop sits behind this host.
@@ -767,20 +804,35 @@ const TOOLS_WORTH_ASKING_ABOUT = ['node', 'git', 'adb', 'emulator', 'java', 'pyt
  *
  * It never throws. Somebody running doctor is already stuck.
  *
+ * WHAT THE CALLER MAY ALREADY KNOW. Starting the far side needs Node on that machine, so a
+ * machine with a perfectly good shell and no Node used to come back from here as "it could
+ * not be reached" — and `missingOn` then told somebody to go and fix the ssh config that
+ * already works. A caller that has proved the shell answers with something cheaper (doctor
+ * dials every host with a plain `echo` before it gets here) says so with `answered`, and
+ * hands over whatever its own probe found, so a failure to start the runner is reported as
+ * the missing Node it actually is.
+ *
  * @param {string} host
- * @param {{timeoutMs?: number, log?: (m: string) => void}} [opts]
+ * @param {object} [opts]
+ * @param {number} [opts.timeoutMs]         How long one request may take. Default 20s.
+ * @param {number} [opts.windowsTimeoutMs]  The Windows probe on its own, which pays for
+ *   PowerShell's start-up and is therefore slower than everything else. Default 45s.
+ * @param {(m: string) => void} [opts.log]
+ * @param {boolean} [opts.answered]         The caller has already proved a shell answers here.
+ * @param {string|null} [opts.powershell]   A powershell.exe path the caller's own probe found.
  * @returns {Promise<RemoteDescription>}
  */
 export async function describeRemote(host, opts = {}) {
   /** @type {RemoteDescription} */
   const out = {
     host,
-    reachable: false,
-    how: 'it did not answer',
+    reachable: opts.answered === true,
+    runnerStarted: false,
+    how: opts.answered === true ? 'a shell on it answered when the caller dialled it' : 'it did not answer',
     os: null,
-    windows: false,
+    windows: typeof opts.powershell === 'string' && opts.powershell !== '',
     windowsVersion: null,
-    powershell: null,
+    powershell: opts.powershell ?? null,
     desktopLoggedIn: null,
     desktopLocked: null,
     tools: {},
@@ -792,6 +844,7 @@ export async function describeRemote(host, opts = {}) {
   try {
     const facts = await runner.open();
     out.reachable = true;
+    out.runnerStarted = true;
     out.how = 'it answered over ssh with the key already in the config';
     out.os = [facts.platform, facts.release].filter(Boolean).join(' ') || null;
 
@@ -799,10 +852,21 @@ export async function describeRemote(host, opts = {}) {
     out.tools = /** @type {Record<string, string|null>} */ (found.found ?? {});
 
     // The Windows question, asked of the filesystem rather than of $PATH. See POWERSHELL_PATHS.
-    const test = await runner.shell(
-      POWERSHELL_PATHS.map((p) => `if [ -x "${p}" ]; then echo "${p}"; fi`).join('; ')
-    );
-    const psPath = test.stdout.split('\n').map((l) => l.trim()).find((l) => l !== '') ?? null;
+    // Skipped entirely when the caller's own probe already found the path: it is the same
+    // question and asking it twice only creates a second chance for the two to disagree.
+    let psPath = out.powershell;
+    if (!psPath) {
+      const test = await runner.shell(
+        POWERSHELL_PATHS.map((p) => `if [ -x "${p}" ]; then echo "${p}"; fi`).join('; ')
+      );
+      psPath = test.stdout.split('\n').map((l) => l.trim()).find((l) => l !== '') ?? null;
+      // A probe that was killed comes back with an empty answer, which is the same empty
+      // answer a machine with no Windows on it gives. Reported as "no Windows here", that
+      // loses somebody the Windows runner they already have, and nothing anywhere says why.
+      if (!psPath && test.killed) {
+        out.notes.push('Whether there is a Windows desktop behind this machine is unknown: the question timed out rather than came back "no".');
+      }
+    }
     if (psPath) {
       out.powershell = psPath;
       out.windows = true;
@@ -819,7 +883,10 @@ export async function describeRemote(host, opts = {}) {
       ].join('; ');
       const probe = await runner.shell(
         `"${psPath}" -NoProfile -NonInteractive -EncodedCommand ${encodePowerShell(script)}`,
-        { timeoutMs: 45_000 }
+        // Its own knob, because this one call costs PowerShell's start-up and everything else
+        // here costs a round trip. Folding it into `timeoutMs` would mean a caller that wants
+        // the rest to be quick has to allow forty-five seconds for it too.
+        { timeoutMs: opts.windowsTimeoutMs ?? 45_000 }
       );
       const line = probe.stdout.split('\n').map((l) => l.trim()).filter(Boolean).pop() ?? '';
       const [caption, version, explorers, logonui] = line.split('|');
@@ -828,12 +895,26 @@ export async function describeRemote(host, opts = {}) {
         out.desktopLoggedIn = Number(explorers) > 0;
         out.desktopLocked = Number(logonui) > 0;
       } else {
-        out.notes.push('PowerShell is there but did not answer a question about the desktop, so how much of Windows is usable is unknown.');
+        // Which of the two it was matters to whoever reads it: a probe that ran out of time
+        // says nothing about the machine, and a probe that answered something unreadable says
+        // something is wrong with PowerShell there. Both used to arrive as one sentence.
+        out.notes.push(
+          probe.killed
+            ? `PowerShell is there and the question about the desktop was still running after ${Math.round((opts.windowsTimeoutMs ?? 45_000) / 1000)} seconds, so how much of Windows is usable is unknown — not "not much".`
+            : 'PowerShell is there but did not answer a question about the desktop, so how much of Windows is usable is unknown.',
+        );
       }
     }
     await runner.close();
   } catch (error) {
-    out.how = error instanceof RemoteLinkLost ? error.message : `it could not be reached (${String(error)})`;
+    const why = error instanceof RemoteLinkLost ? error.message : String(error);
+    // Two different failures, and they were being reported as one. A machine that never
+    // answered is an ssh problem. A machine that answers and could not start the runner is a
+    // Node problem, and saying "it could not be reached" about it sends somebody to fix a
+    // connection that is working — while the one thing that would help goes unmentioned.
+    out.how = out.reachable
+      ? `a shell on it answers, but the small program this tool sends down the connection would not start there (${why})`
+      : error instanceof RemoteLinkLost ? why : `it could not be reached (${why})`;
     try { await runner.close(); } catch { /* nothing to close */ }
   }
 
@@ -868,8 +949,16 @@ export function missingOn(d) {
   if (!d.tools.node) {
     missing.push({
       what: 'Node on that machine',
+      // Two ways to get here and they deserve different words. Either the runner started and
+      // `which node` came back empty, or the runner never started at all — in which case the
+      // tools list is empty because nothing could ask, and a Node that is present but too old
+      // looks exactly the same from here. Saying "install Node" at somebody who has Node 12
+      // and is not told the version matters is how a person spends an afternoon on the wrong
+      // thing.
       unlocks: 'the general remote runner, which is how any platform there is walked',
-      howToGet: `ssh ${d.host} 'sudo apt-get install -y nodejs' — or whatever that machine installs packages with.`,
+      howToGet: d.reachable && !d.runnerStarted
+        ? `The program this tool sends down would not start there, which is Node missing or too old. Check it with: ssh ${d.host} 'node --version' — it has to be 22 or newer. Install or upgrade it with whatever that machine installs packages with, for example: ssh ${d.host} 'sudo apt-get install -y nodejs'`
+        : `ssh ${d.host} 'sudo apt-get install -y nodejs' — or whatever that machine installs packages with.`,
       blocking: true,
     });
   }
@@ -901,6 +990,12 @@ export function notesOn(d) {
   const notes = [];
   if (!d.reachable) return notes;
   notes.push('Nothing is installed on that machine. The program that does the watching is sent down the connection each run and dies with it.');
+  // Said out loud, because everything else in this description is empty for one reason and an
+  // empty list that does not say why reads as "there is nothing there". There may be plenty
+  // there; nothing could ask.
+  if (!d.runnerStarted) {
+    notes.push(`A shell on ${d.host} answers, but the program this tool sends down the connection did not start, so nothing below was actually asked of that machine — what is installed on it is unknown rather than absent.`);
+  }
   if (d.windows) {
     notes.push(`Windows is reached through ${d.powershell}, called from the Linux side. That absolute path is used deliberately: powershell.exe is not on the path of a non-interactive ssh session even when the machine is configured to add it.`);
     notes.push('Windows shows one desktop, so two builds can never run there at the same time. Runs are one after the other, and that is a real weakening of the same-machine guarantee, not a detail.');

package/src/v2/run.js

@@ -34,6 +34,7 @@ import {
   wobbleStorm,
 } from './observation.js';
 import { ensureStore, saveBuild, saveCapture, latestCapture, referenceFor, listBuilds } from './store.js';
+import { describeRuleChange } from './normalise.js';
 import { clusterDifferences } from './cluster.js';
 import { rankFindings } from './rank.js';
 
@@ -135,6 +136,10 @@ const VERSION = /** @type {{version?: string}} */ (require('../../package.json')
  * @property {string} [journey]
  * @property {string} [run]
  * @property {number} [count]
+ * @property {number} [steady]    Only on 'wobble'. Addresses this build answered the same
+ *                                way twice, counted rather than inferred by subtraction.
+ * @property {boolean} [measured] Only on 'wobble'. False when the wobble was never taken,
+ *                                which is not the same as a wobble of nothing.
  * @property {number} [durationMs]
  * @property {Verdict} [verdict]
  */
@@ -251,6 +256,11 @@ export async function runCheck(opts) {
     /** @type {string[]} */
     const steadyInReference = [];
     let referenceWobbleMeasured = true;
+    // One fact about one pair of rule sets, gathered here and said once at the end.
+    /** @type {Set<string>} */
+    const rulesMoved = new Set();
+    /** @type {string[]} */
+    const rulesMovedOn = [];
 
     for (const journey of journeys) {
       stop();
@@ -294,12 +304,22 @@ export async function runCheck(opts) {
           surface: journey.surface,
         });
       }
+      // ADDRESSES, not rows. This used to send `a.observations.length`, which counts every
+      // observation the adapter wrote down — and two observations at the SAME address are one
+      // address written down twice, not two addresses. Every other number on the page is
+      // counted by address: `foldCoverage` builds a Set, and the wobble arithmetic indexes by
+      // path. So a walk with two duplicate addresses put "18 addresses watched" in the header
+      // beside "14 addresses watched" in the coverage ledger and "18 answered the same way
+      // twice" for a build that only ever had 14 addresses to answer at. Three numbers about
+      // one run, disagreeing, on a page whose entire job is being believed. The duplicates
+      // themselves are not swallowed — `duplicateGaps` above reports each one.
+      const addresses = new Set(a.observations.map((o) => o.path)).size;
       say({
         type: 'journey:done',
         at: events.elapsed(),
         journey: journey.name,
-        count: a.observations.length,
-        message: `${a.observations.length} ${plural(a.observations.length, 'thing', 'things')} looked at, ${wobble.unstable.length} of which this build cannot answer the same way twice.`,
+        count: addresses,
+        message: `${addresses} ${plural(addresses, 'thing', 'things')} looked at, ${wobble.unstable.length} of which this build cannot answer the same way twice.`,
       });
 
       if (!reference) continue;
@@ -337,9 +357,8 @@ export async function runCheck(opts) {
         gaps.push({
           what: `The old build could not be walked for "${journey.describe || journey.name}", so this was not a paired comparison after all.`,
           why:
-            `${nameOf(reference)} was put back on this machine, and then there was nothing there to run: ${
-              wasA.observations[0]?.meta?.describe ?? 'the adapter could not open it'
-            }. This usually means the product is BUILT rather than committed — an APK, a .app, a packaged desktop app — and a checkout of the old commit does not contain one.`,
+            `${nameOf(reference)} was put back on this machine, and then there was nothing there to run: ${whyNothingRan(wasA)} ` +
+            'This usually means the product is BUILT rather than committed — an APK, a .app, a packaged desktop app — and a checkout of the old commit does not contain one.',
           unlockedBy:
             'Build the old commit before the run, or point the settings at a kept copy of the old build\'s artifact. Until then this journey falls back to the record the old build left last time.',
           surface: journey.surface,
@@ -383,20 +402,43 @@ export async function runCheck(opts) {
       // one normalised under another produces differences that are about the RULES — either a
       // wall of noise that reads like a regression, or, when the change was to add a rule,
       // quiet where there should not be any. Either way the reader has to be told.
-      if (stored.capture.rules && a.rules && stored.capture.rules !== a.rules) {
-        gaps.push({
-          what: `"${journey.describe || journey.name}" is being compared across a change to the normalisation rules.`,
-          why:
-            `The stored record of the old build was tidied up by rule set ${stored.capture.rules} and this run used ${a.rules}. ` +
-            'Some of what you see may be the rules changing rather than the product, and a rule that was added since could be covering something up.',
-          unlockedBy: 'Run a paired check, which walks the old build live under today\'s rules, or ship again to cut a fresh reference.',
-          surface: journey.surface,
-        });
+      //
+      // WHAT changed, not two hashes. This used to print "v1-fcf4b8000217 versus
+      // v1-29141a9ec069" and nothing else, which tells an agent nothing it can act on, so it
+      // acts on nothing. `describeRuleChange` owns the judgement — a rule that rewrites
+      // something differently makes the whole comparison suspect; a rule that merely reaches
+      // one more address makes only that address suspect and names it; and a record older
+      // than the scope stamp says so rather than reporting every glob as new.
+      //
+      // ONCE PER RUN, not once per journey. It is one fact about one pair of rule sets, and
+      // it was being written out again for every journey — four identical paragraphs in the
+      // coverage list on a four-journey project, on every run, saying the same thing. The
+      // coverage list is the one section that must never be skimmed, and nothing teaches a
+      // reader to skim it faster than a block that is always there and always the same.
+      const ruleChange = describeRuleChange(
+        { fingerprint: stored.capture.rules, scope: stored.capture.rulesScope },
+        { fingerprint: a.rules, scope: a.rulesScope },
+      );
+      if (!ruleChange.same) {
+        rulesMoved.add(ruleChange.say);
+        rulesMovedOn.push(journey.describe || journey.name);
       }
       if (stored.wobble) steadyInReference.push(...steadyPaths(stored.capture, stored.wobble));
       else referenceWobbleMeasured = false;
     }
 
+    if (rulesMovedOn.length > 0) {
+      const which =
+        rulesMovedOn.length === 1
+          ? `"${rulesMovedOn[0]}" is`
+          : `${rulesMovedOn.length} journeys are`;
+      gaps.push({
+        what: `${which} being compared across a change to the normalisation rules.`,
+        why: [...rulesMoved].join(' '),
+        unlockedBy: 'Run a paired check, which walks the old build live under today\'s rules, or ship again to cut a fresh reference.',
+      });
+    }
+
     stop();
     // Booting the old build is not the same as having walked it. When every live walk came
     // back holes-only — a built artifact that no checkout of the old commit contains — the
@@ -405,10 +447,19 @@ export async function runCheck(opts) {
     const walkedLive = liveWalks > 0;
     const mode = /** @type {'paired'|'stored-record'} */ (walkedLive ? 'paired' : 'stored-record');
     const wobble = wobbles.length > 0 ? mergeWobble(wobbles) : unmeasuredWobble(opts.candidate.id, '*');
+    // `steady` and `measured` are sent because they were MEASURED here, and nothing
+    // downstream can work them out. Anything drawing this event had to guess steady as
+    // "everything watched, minus the unstable ones" — a subtraction across two different
+    // populations, so it reported addresses as having answered the same way twice when the
+    // build had never been asked at them. And `measured: false` is the difference between a
+    // build that wobbled about nothing and a build whose wobble was never taken; two noughts
+    // cannot tell those apart, and only the first one is good news.
     say({
       type: 'wobble',
       at: events.elapsed(),
       count: wobble.unstable.length,
+      steady: wobble.steady,
+      measured: wobble.measured,
       message:
         wobble.unstable.length === 0
           ? 'This build gives the same answer twice, everywhere.'
@@ -599,6 +650,16 @@ export async function runCheck(opts) {
       cwd: opts.cwd,
       guards: opts.guards ?? [],
       touches: touchMap(walked),
+      // Where "your change" starts.
+      //
+      // Without this, "the change" means the working tree and nothing else — so the moment
+      // an agent commits its work, which is exactly what an agent does at the end of a task,
+      // the distance measure goes blind, the ranking loses its ordering, and every finding
+      // carries the sentence "nothing in the working tree has changed, so there is no edit
+      // to measure this against" over a change that is perfectly well known. The reference
+      // is a shipped build, which is a commit; the diff from there to here is the change,
+      // whether it has been committed or not.
+      since: reference?.gitSha ?? undefined,
     });
 
     const warning = modeWarning(mode, provedLive, reference);
@@ -920,6 +981,34 @@ const PROVEN_LIVE_WARNING =
 const NO_REFERENCE_WARNING =
   'Until you ship once with the reference hook in place there is nothing to compare against, so this run proves nothing about what still works.';
 
+/**
+ * Why the old build came back with nothing that could be walked.
+ *
+ * The adapter always says why. It says it in the observation's own sentence, which
+ * `observation()` in adapters/contract.js files under `meta.describe` — "there is no APK in
+ * the exported checkout", "the runtime this needs is not on this machine". Reading only
+ * `observations[0]` threw that away in the two cases that matter most: a capture that came
+ * back completely EMPTY has no observation nought to read, so the reader got the generic
+ * "the adapter could not open it" while the real reason sat in the capture's note; and a
+ * capture whose first observation happens to be one the adapter did cover names the wrong
+ * hole. So the first REFUSED observation is the one that answers the question, its reason
+ * category stands in when it has no sentence of its own, and the capture's note is read
+ * before anything generic is said.
+ *
+ * The sentence is also finished properly. The adapter's `says` is already a whole sentence,
+ * and the old template appended a full stop of its own — the same fault already fixed once
+ * in check.js's explain reply, which was giving agents "...real money..".
+ *
+ * @param {Capture} was   The walk of the old build that produced nothing usable.
+ * @returns {string}      One finished sentence.
+ */
+function whyNothingRan(was) {
+  const refused = was?.observations?.find((o) => o.meta?.refused === true);
+  const said = refused?.meta?.describe || refused?.meta?.refusedWhy || was?.note || 'the adapter could not open it';
+  const text = String(said).trim();
+  return /[.!?]$/.test(text) ? text : `${text}.`;
+}
+
 /**
  * @param {'paired'|'stored-record'} mode
  * @param {boolean} provedLive

package/src/v2/sealed.js

@@ -295,26 +295,6 @@ export function classify(finding, opts = {}) {
   };
 }
 
-/**
- * The same answer as a plain class name, for callers that already work in `FindingClass`.
- *
- * @param {Finding} finding
- * @param {ClassifyOptions} [opts]
- * @returns {FindingClass}
- */
-export function sealedClassOf(finding, opts = {}) {
-  return classify(finding, opts)?.class ?? 'ordinary';
-}
-
-/**
- * @param {Finding} finding
- * @param {ClassifyOptions} [opts]
- * @returns {boolean}
- */
-export function isSealed(finding, opts = {}) {
-  return classify(finding, opts) !== null;
-}
-
 /**
  * The refusal, written out for whoever reads it — an agent that has just been told no, or a
  * person reading the closing summary.