staysfixed 0.7.2 → 0.8.0

package/src/v2/adapters/web.js

@@ -36,6 +36,7 @@
  * Every refusal is reported as a hole in the check. None of them is ever reported as a pass.
  */
 
+import crypto from 'node:crypto';
 import fs from 'node:fs';
 import fsp from 'node:fs/promises';
 import path from 'node:path';
@@ -70,6 +71,26 @@ const VIEWPORT = { width: 1280, height: 800, deviceScaleFactor: 1 };
 /** Folders that never hold a page worth walking. */
 const SKIP_DIRS = new Set(['node_modules', '.git', 'dist', 'build', 'out', '.next', 'coverage', '.staysfixed']);
 
+/**
+ * Why a folder would not open, in words somebody can act on.
+ *
+ * The code alone is no use to the person who has to fix it: `EACCES` on its own has sent
+ * more than one person looking for a bug in the tool.
+ *
+ * @param {unknown} error
+ * @returns {string}
+ */
+function whyNotOpened(error) {
+  const code = String(/** @type {any} */ (error)?.code ?? '');
+  if (code === 'EACCES' || code === 'EPERM') return 'this account does not have permission to open it';
+  if (code === 'ENOENT') return 'it was there when the walk started and is not there now';
+  if (code === 'ENOTDIR') return 'something in the way is a file, not a folder';
+  if (code === 'ELOOP') return 'the links in it point round in a circle';
+  if (code === 'EMFILE' || code === 'ENFILE') return 'this machine ran out of open files while reading it';
+  const said = String(/** @type {any} */ (error)?.message ?? error ?? '').trim();
+  return said === '' ? 'the reason was not given' : said;
+}
+
 /**
  * The pages a project has, read out of its folder names.
  *
@@ -83,12 +104,29 @@ const SKIP_DIRS = new Set(['node_modules', '.git', 'dist', 'build', 'out', '.nex
  * reported as needing a sample value rather than guessed at, because "we did not check
  * this" and "this is fine" must never be allowed to look alike.
  *
+ * A FOLDER THIS CANNOT OPEN IS NAMED, not skipped. It used to `continue` on the read that
+ * failed, which threw away that folder and every page underneath it without one word — and
+ * a page that was never listed is never walked, never counted, and never missed. The run
+ * then reported that nothing had changed about a section of the site it had not looked at.
+ * This is the same shape of bug as three others that were found and closed elsewhere in the
+ * tool; this was the fourth place it was living.
+ *
  * @param {string} root
+ * @param {{unreadable?: {folder: string, why: string}[]}} [collect]
+ *   Hand in an object and every folder that could not be opened is pushed onto
+ *   `collect.unreadable`, with the reason in plain English. Callers that pass nothing get
+ *   the old shape back and lose the holes, so the two callers inside this file both pass
+ *   one; `detect` turns them into something a person can act on and `journeys` turns them
+ *   into missing coverage, which is what stops a folder nobody could read reading as a
+ *   folder with nothing in it.
  * @returns {Promise<{url: string, file: string, needs: string[]}[]>}
  */
-export async function readPageRoutes(root) {
+export async function readPageRoutes(root, collect = {}) {
   /** @type {Map<string, {url: string, file: string, needs: string[]}>} */
   const found = new Map();
+  const unreadable = collect.unreadable;
+  /** @type {Set<string>} */
+  const alreadySaid = new Set();
 
   /**
    * @param {string} base
@@ -104,7 +142,12 @@ export async function readPageRoutes(root) {
       let entries;
       try {
         entries = fs.readdirSync(dir, { withFileTypes: true });
-      } catch {
+      } catch (error) {
+        const folder = path.relative(root, dir) || '.';
+        if (unreadable && !alreadySaid.has(folder)) {
+          alreadySaid.add(folder);
+          unreadable.push({ folder, why: whyNotOpened(error) });
+        }
         continue;
       }
       for (const entry of entries) {
@@ -163,9 +206,20 @@ export async function readPageRoutes(root) {
  * already has them), a journeys file if one was named, and the pages read out of the folder
  * names for everything neither of those covers. Never invented, never crawled.
  *
+ * TWO SCREENS WITH ONE NAME ARE TWO SCREENS. They used to be one: the list is built in a
+ * Map keyed by name, so the second `set` overwrote the first, and the screen that lost was
+ * never walked, never compared and never counted as a door nobody opened — missing from the
+ * very ledger that exists to catch exactly this. Now the second one is told apart by a
+ * number and both are walked. Two entries that are the same in every respect really are one
+ * screen written down twice, and only those are folded together.
+ *
  * @param {object} input
  * @param {Record<string, any>} input.config
  * @param {{url: string, file: string, needs: string[]}[]} input.pages
+ * @param {{folder: string, why: string}[]} [input.unreadable]
+ *   Folders `readPageRoutes` could not open. Each becomes a journey that says it was not
+ *   walked and why, which the engine records as missing coverage. A page behind one of them
+ *   was never listed, so nothing else in the run would ever mention it.
  * @returns {Journey[]}
  */
 export function journeysFrom(input) {
@@ -173,19 +227,50 @@ export function journeysFrom(input) {
   const samples = config.samples ?? {};
   /** @type {Map<string, Journey>} */
   const journeys = new Map();
+  /** @type {Map<string, string>} name -> what that journey is, so a true duplicate is spotted */
+  const shapes = new Map();
+
+  /**
+   * A name nothing else is using. Numbering rather than overwriting: a screen that is
+   * dropped here is dropped everywhere, silently, for the rest of the run.
+   *
+   * @param {string} wanted
+   * @returns {string}
+   */
+  const freeName = (wanted) => {
+    if (!journeys.has(wanted)) return wanted;
+    for (let n = 2; ; n += 1) {
+      const tried = `${wanted} (${n})`;
+      if (!journeys.has(tried)) return tried;
+    }
+  };
 
   // v1 called these "screens" and this reads them unchanged, on purpose: nobody should have
   // to write their steps twice to get a second kind of check out of them.
   for (const screen of [...(config.screens ?? []), ...(config.journeys ?? [])]) {
     if (!screen || typeof screen !== 'object') continue;
-    const name = String(screen.name ?? screen.url ?? 'a screen');
+    const wanted = String(screen.name ?? screen.url ?? 'a screen');
     /** @type {Record<string, any>[]} */
     const steps = [];
     if (screen.url !== undefined) steps.push({ act: 'open', goto: String(screen.url), note: `open ${screen.url}` });
     for (const step of screen.steps ?? []) steps.push({ act: actOf(step), ...step });
+
+    // The same name AND the same steps is one screen listed twice — the settings and the
+    // journeys file both naming it, most often — and folding those together loses nothing.
+    // A different set of steps under the same name is a different screen.
+    const shape = JSON.stringify(steps);
+    if (shapes.get(wanted) === shape) continue;
+    const name = freeName(wanted);
+    shapes.set(name, shape);
     journeys.set(name, {
       name,
-      describe: String(screen.describe ?? screen.why ?? `walk ${name}`),
+      describe: String(
+        screen.describe ??
+        screen.why ??
+        (name === wanted
+          ? `walk ${name}`
+          : `walk ${name} — the settings name two different screens "${wanted}", so this is the second of them`)
+      ),
       source: 'code',
       surface: 'web',
       from: 'the project settings',
@@ -205,8 +290,7 @@ export function journeysFrom(input) {
       if (sample === undefined) unfilled.push(need);
       else url = url.replace(new RegExp(`\\[\\.{0,3}${need}\\]|:${need}`), encodeURIComponent(String(sample)));
     }
-    const name = `page ${page.url}`;
-    if (journeys.has(name)) continue;
+    const name = freeName(`page ${page.url}`);
     journeys.set(name, {
       name,
       describe: `open ${page.url} and read what the screen says`,
@@ -230,6 +314,28 @@ export function journeysFrom(input) {
     });
   }
 
+  // Last, and after the front-page fallback so a project made entirely of holes still gets
+  // one real journey. A folder nobody could open is a journey that says out loud it was not
+  // walked: the engine turns a journey carrying `skip` into missing coverage, which is the
+  // only route from here to the ledger. Without it, a page behind that folder was never
+  // listed, so nothing anywhere in the run would ever have mentioned it — and a page nobody
+  // listed reads exactly like a page that is fine.
+  for (const hole of input.unreadable ?? []) {
+    const name = freeName(`the pages under ${hole.folder}`);
+    journeys.set(name, {
+      name,
+      describe: `the pages under ${hole.folder}`,
+      source: 'code',
+      surface: 'web',
+      from: hole.folder,
+      channels: [],
+      steps: /** @type {any} */ ([]),
+      skip:
+        `"${hole.folder}" could not be opened while looking for this project's pages — ${hole.why} — so nothing under it was listed. ` +
+        'Any page in there was not walked and is not in the count of pages that were. This is a hole, not a pass.',
+    });
+  }
+
   return [...journeys.values()].sort((a, b) => a.name.localeCompare(b.name));
 }
 
@@ -290,7 +396,16 @@ export const webAdapter = defineAdapter({
     const framework = ['next', 'react', 'vue', 'svelte', 'astro', '@remix-run/react', 'nuxt', 'solid-js', 'preact', 'vite']
       .find((name) => name in dependencies);
 
-    const pages = await readPageRoutes(project.root);
+    /** @type {{unreadable: {folder: string, why: string}[]}} */
+    const collect = { unreadable: [] };
+    const pages = await readPageRoutes(project.root, collect);
+    if (collect.unreadable.length > 0) {
+      missing.push({
+        what: `permission to read ${collect.unreadable.length === 1 ? 'a folder' : `${collect.unreadable.length} folders`} this project's pages live in: ${collect.unreadable.map((u) => `${u.folder} (${u.why})`).join(', ')}`,
+        unlocks: 'listing the pages under them at all. Nothing under a folder that will not open is walked, counted, or missed, so the run is quiet about that part of the site',
+        howToGet: `Give this account permission to read ${collect.unreadable.length === 1 ? 'it' : 'them'} — on a Mac or Linux that is: chmod +rx ${collect.unreadable[0].folder}`,
+      });
+    }
     const address = config.url ?? config.baseUrl ?? null;
 
     if (!config.start && !address) {
@@ -338,7 +453,10 @@ export const webAdapter = defineAdapter({
 
   /** @param {import('./contract.js').AdapterProject} project */
   async journeys(project) {
-    return journeysFrom({ config: project.config ?? {}, pages: await readPageRoutes(project.root) });
+    /** @type {{unreadable: {folder: string, why: string}[]}} */
+    const collect = { unreadable: [] };
+    const pages = await readPageRoutes(project.root, collect);
+    return journeysFrom({ config: project.config ?? {}, pages, unreadable: collect.unreadable });
   },
 
   /**
@@ -525,6 +643,9 @@ export const webAdapter = defineAdapter({
     const viewport = viewportFrom(config);
     const window = await openWindow({
       chromium: held.playwright.chromium,
+      // Which browser was chosen for this machine. Without it a driver that downloads no
+      // browser of its own looks for one that is not there.
+      executable: held.playwright.executable,
       scratchDir: ctx.scratchDir,
       viewport,
       colorScheme: config.colorScheme ?? 'light',
@@ -1001,9 +1122,21 @@ export function complaintKey(message) {
 }
 
 /**
+ * A name a picture can be saved under.
+ *
+ * Cut with a fingerprint on the end, never cut alone. This is a FILE name: two checkpoints
+ * whose names agreed for eighty characters were saved over each other, so the picture
+ * offered as evidence for one finding was a photograph of a different screen — and nothing
+ * about it looked wrong. Long file names are the normal case here, because the name is the
+ * build, the journey and the checkpoint run together.
+ *
  * @param {string} name
  * @returns {string}
  */
 function fileSafe(name) {
-  return String(name).replace(/[^A-Za-z0-9._-]+/g, '-').slice(0, 80) || 'checkpoint';
+  const clean = String(name).replace(/[^A-Za-z0-9._-]+/g, '-');
+  if (clean === '') return 'checkpoint';
+  if (clean.length <= 80) return clean;
+  const mark = crypto.createHash('sha256').update(clean).digest('hex').slice(0, 8);
+  return `${clean.slice(0, 71)}-${mark}`;
 }

package/src/v2/adapters/windows.js

@@ -1201,7 +1201,24 @@ export const windowsAdapter = defineAdapter({
           // Pixels last, and only as evidence. A picture is written to the evidence folder and
           // pointed at; it is never the thing compared.
           const shot = await runner.call('shot', { hwnd: window.hwnd }, { timeoutMs: 45_000 });
-          if (shot.ok && shot.png && Number(shot.bytes) <= MAX_SHOT_BYTES) {
+          // THREE WAYS OUT OF HERE AND TWO OF THEM USED TO BE SILENT. A picture that failed,
+          // came back empty, or came back bigger than the cap simply produced no observation
+          // at all — so the pixels channel dropped out of the run without a word, and the
+          // ledger reported the same coverage as a run where every window was photographed.
+          // A cap is a decision and a decision has to be visible; a failure is a hole and a
+          // hole has to be named. Neither is a reason to lose the rest of the walk.
+          const tooBig = shot.ok === true && Boolean(shot.png) && Number(shot.bytes) > MAX_SHOT_BYTES;
+          if (!shot.ok || !shot.png || tooBig) {
+            seen.push(notCovered({
+              channel: 'pixels',
+              path: joinPath('screen', label, 'picture'),
+              reason: tooBig ? 'too big' : 'crashed',
+              says: tooBig
+                ? `The picture of "${label}" came back at ${sizeBucket(Number(shot.bytes))}, over the ${sizeBucket(MAX_SHOT_BYTES)} this keeps, so it was not stored. Every other channel still looked at that window; only the picture is missing.`
+                : `No picture of "${label}" could be taken${shot.error ? `: ${String(shot.error)}` : '.'} Every other channel still looked at that window; only the picture is missing.`,
+            }));
+          }
+          if (shot.ok && shot.png && !tooBig) {
             const file = path.join(ctx.evidenceDir, `windows-${journey.name}-${label.replace(/[^a-z0-9]+/gi, '-')}.png`);
             await fsp.writeFile(file, Buffer.from(String(shot.png), 'base64'));
             seen.push(observation({

package/src/v2/browsers.js

@@ -439,7 +439,15 @@ async function takeSurvey(opts) {
  * download and a package, nothing else — no licence, no account, no clicking.
  * That is why doctor reports it as something the agent does without asking.
  */
-export const INSTALL_COMMAND = 'npm install --save-dev playwright && npx playwright install chromium';
+// One command, and it installs nothing into anybody's project.
+//
+// It used to be `npm install --save-dev playwright && npx playwright install chromium`, which
+// puts a 150MB package into a stranger's dependencies to solve a problem this tool has already
+// solved: the driver ships with the tool, and the only thing that can be missing is a browser.
+// `npx` fetches the downloader for the length of one command and leaves nothing behind, and the
+// browser it downloads lands in a shared folder outside every project, where `surveyBrowsers`
+// looks and where it survives every reinstall.
+export const INSTALL_COMMAND = 'npx playwright install chromium';
 
 /**
  * @param {BrowserFound|null} chosen

package/src/v2/cause.js

@@ -88,6 +88,7 @@ const run = promisify(execFile);
  *   candidate: BuildFingerprint,
  *   hunk?: ChangedHunk,
  *   changed?: Changed,
+ *   since?: string,
  *   normalise?: (capture: Capture) => Capture,
  *   events?: CheckEvents,
  *   signal?: AbortSignal,
@@ -102,7 +103,11 @@ export async function proveCause(finding, opts) {
     });
   }
 
-  const changed = opts.changed ?? (await whatChanged(opts.cwd));
+  // `since` is the reference build's commit, and without it this could only ever undo a
+  // change that was still uncommitted. An agent that committed its work before asking — which
+  // is the normal end of a task — got "nothing in the working tree has changed, so there is
+  // no change to undo" about a change sitting one commit back.
+  const changed = opts.changed ?? (await whatChanged(opts.cwd, { since: opts.since }));
   if (!changed.ok) return cannot(changed.why ?? 'The working tree could not be read.', null);
   // "git could not hand over the diff" and "there is no diff" both left `hunks` empty, and
   // the sentence below was said about both. Telling somebody their tree is clean when it is
@@ -114,7 +119,12 @@ export async function proveCause(finding, opts) {
     );
   }
   if (changed.hunks.length === 0 && changed.untracked.length === 0) {
-    return cannot('Nothing in the working tree has changed, so there is no change to undo.', null);
+    return cannot(
+      changed.committed
+        ? 'Nothing has changed between the build you were happy with and this one — not in a commit and not in the working tree — so there is no change to undo.'
+        : 'Nothing in the working tree has changed, so there is no change to undo. Nothing here looked at what may already be committed: name the commit the old build is at and a committed change can be undone the same way.',
+      null,
+    );
   }
 
   /** @type {{hunk: ChangedHunk|null, candidates: ChangedHunk[]}} */
@@ -137,6 +147,14 @@ export async function proveCause(finding, opts) {
   if (journeys.length === 0) {
     return cannot('None of the journeys this finding came from are available to walk again.', hunk);
   }
+  // SOME of them, and not all, is its own answer and it used to have none.
+  //
+  // A finding gathered from three journeys where only two can be walked again was walked
+  // twice, and the third journey's addresses were then read out of a map that has nothing in
+  // it for them. `gone` answers false for those — correctly, it will not call an absent
+  // journey a pass — and the arithmetic below turned that into "still here with that change
+  // undone, so it is not what caused it". A sentence about addresses nobody looked at.
+  const missing = names.filter((name) => !opts.journeys.some((j) => j.name === name));
 
   const base = await fsp.mkdtemp(path.join(os.tmpdir(), 'staysfixed-cause-'));
   const tree = path.join(base, 'tree');
@@ -161,10 +179,14 @@ export async function proveCause(finding, opts) {
     return p;
   };
   try {
-    await gitOrThrow(['worktree', 'add', '--detach', tree, 'HEAD'], changed.root);
+    // The commit the patch was measured FROM, which is not always HEAD. A change that has
+    // been committed is in the patch and is also already in HEAD, so applying it on top of
+    // HEAD would try to add the same lines twice and fail — and before this travelled with
+    // the patch, that is exactly what would have happened.
+    await gitOrThrow(['worktree', 'add', '--detach', tree, changed.base], changed.root);
     checkedOut = true;
 
-    // Everything you changed, applied to a clean copy of the last commit.
+    // Everything you changed, applied to a clean copy of the build you were happy with.
     if (changed.patch.trim().length > 0) {
       const workingPatch = path.join(base, 'working.patch');
       await fsp.writeFile(workingPatch, endWithNewline(changed.patch), 'utf8');
@@ -238,27 +260,49 @@ export async function proveCause(finding, opts) {
     }
 
     const differences = finding.differences;
+    // An address whose journey was never walked again was not re-checked, and it is counted
+    // apart from the ones that were. Folding it in with the survivors is how "nobody looked"
+    // came out of here dressed as "it is still there".
+    const rechecked = differences.filter((d) => without.has(d.journey ?? ''));
+    const notRechecked = differences.length - rechecked.length;
     let disappeared = 0;
-    for (const d of differences) if (gone(d, without)) disappeared += 1;
-
-    const proved = differences.length > 0 && disappeared === differences.length;
+    for (const d of rechecked) if (gone(d, without)) disappeared += 1;
+
+    const unseen = notRechecked > 0
+      ? ` ${notRechecked} of the ${differences.length} ${differences.length === 1 ? 'address' : 'addresses'} in this finding ${notRechecked === 1 ? 'was' : 'were'} not re-checked at all, because ${missing.length === 1 ? `the journey "${missing[0]}" is` : `the journeys ${missing.map((n) => `"${n}"`).join(', ')} are`} not available to walk again. Nothing here says anything about ${notRechecked === 1 ? 'it' : 'them'} either way.`
+      : '';
+    const proved = rechecked.length > 0 && disappeared === rechecked.length && notRechecked === 0;
+    // Everything that could be re-walked went away, and something else could not be walked at
+    // all. That is not a proof: the part nobody saw may be the part that matters, and a
+    // stamped "caused by that change" over the top of it is the worst thing this file can
+    // produce — a machine-checked reason for an agent to stop looking.
+    const partly = rechecked.length > 0 && disappeared === rechecked.length && notRechecked > 0;
     /** @type {CauseProof} */
     const result = {
       verdict:
-        differences.length === 0 ? 'could not test' : proved ? 'caused by that change' : 'not caused by that change',
-      escalates: differences.length > 0 && !proved,
-      what:
-        differences.length === 0
-          ? 'This finding carries no differences, so there was nothing to re-check.'
+        rechecked.length === 0 || partly
+          ? 'could not test'
           : proved
-            ? `Undoing that one change in ${hunk.file} made this go away. It is yours, and it is explained.`
-            : disappeared > 0
-              ? `Undoing that change in ${hunk.file} took away ${disappeared} of the ${differences.length} addresses in this finding and left ${differences.length - disappeared} exactly as ${differences.length - disappeared === 1 ? 'it was' : 'they were'}. So that change explains part of this and not the rest, and the rest has another cause nothing has looked for yet. It is not covered by undoing that one change.`
-              : `This is still here with that change undone, so ${hunk.file} is not what caused it. Something else did, and nothing knows what yet.`,
+            ? 'caused by that change'
+            : 'not caused by that change',
+      escalates: rechecked.length > 0 && !proved && !partly,
+      what:
+        rechecked.length === 0
+          ? `Nothing in this finding could be re-checked.${unseen || ' It carries no differences, so there was nothing to re-check.'}`
+          : partly
+            ? `Undoing that one change in ${hunk.file} took away every address that could be re-walked.${unseen} So this is not proved: what was not walked may be the half that matters.`
+            : proved
+              ? `Undoing that one change in ${hunk.file} made this go away. It is yours, and it is explained.`
+              : disappeared > 0
+                ? `Undoing that change in ${hunk.file} took away ${disappeared} of the ${rechecked.length} addresses that were re-checked and left ${rechecked.length - disappeared} exactly as ${rechecked.length - disappeared === 1 ? 'it was' : 'they were'}. So that change explains part of this and not the rest, and the rest has another cause nothing has looked for yet. It is not covered by undoing that one change.${unseen}`
+                : `This is still here with that change undone, so ${hunk.file} is not what caused it. Something else did, and nothing knows what yet.${unseen}`,
       hunk: { file: hunk.file, header: hunk.header },
-      checked: differences.length,
+      // What was actually re-walked. It used to be every difference in the finding, including
+      // the ones no journey ever went near, so the number said the work had been done.
+      checked: rechecked.length,
       disappeared,
     };
+    if (notRechecked > 0) result.why = unseen.trim();
     if (opts.keep === true) result.worktree = tree;
     if (events) events.emit({ type: 'proof:done', at: events.elapsed(), message: result.what });
     return done(result);