stillhunter 0.0.1 → 0.2.1

package/dist/keep.mjs

@@ -0,0 +1,195 @@
+// keep and replay: a confirmed finding becomes a test under .stillhunter/tests/
+// that runs with no model. A kept test passes when its check holds, so it
+// fails while the bug is there and guards against its return once fixed.
+import { randomBytes } from 'node:crypto';
+import { copyFileSync, existsSync, mkdirSync, readdirSync, readFileSync, writeFileSync } from 'node:fs';
+import path from 'node:path';
+import { chooseBrowser, installedOnThisMachine } from './browser.mjs';
+import { ensureRunsIgnored } from './change.mjs';
+import { block, fail, findTargetUrl, FOLDER, flag, loadRun, PROBE_NAME, readConfig, repoRoot, runsDir, signInVariableNames, stamp, testsDir } from './common.mjs';
+import { runProbe } from './probe-host.mjs';
+import { resetApp } from './reset.mjs';
+import { outcome, probesDir, signInProbe } from './results.mjs';
+import { answers } from './target.mjs';
+export const SIGNIN = '_signin';
+const WATCH_SLOW_MO_MS = 250;
+const WATCH_KEEP_OPEN_MS = 10_000;
+// A kept test is committed, so it must not carry a credential: a probe that
+// typed a sign-in value literally instead of reading process.env is refused.
+export function credentialIn(script, names, env = process.env) {
+    for (const name of names) {
+        const value = env[name];
+        if (value && value.length >= 4 && script.includes(value))
+            return name;
+    }
+    return null;
+}
+export function keep(args) {
+    const { ctx, dir, root } = loadRun(args);
+    const probe = flag(args, '--probe');
+    const check = flag(args, '--check');
+    const as = flag(args, '--as') ?? probe;
+    if (!probe || !PROBE_NAME.test(probe))
+        fail('Give the probe of the finding with --probe.');
+    if (!check)
+        fail('Give the expectation of the finding with --check.');
+    if (!as || !PROBE_NAME.test(as))
+        fail('--as is a name of lowercase letters, digits and dashes.');
+    let confirmed = [];
+    try {
+        confirmed = JSON.parse(readFileSync(path.join(dir, 'findings.json'), 'utf8')).confirmed ?? [];
+    }
+    catch {
+        fail(`Run ${ctx.runId} has not finished, so it has no confirmed findings to keep.`);
+    }
+    const finding = confirmed.find((f) => f.probe === probe && f.check === check);
+    if (!finding) {
+        const list = confirmed.map((f) => `--probe ${JSON.stringify(f.probe)} --check ${JSON.stringify(f.check)}`).join('; ');
+        fail(`Run ${ctx.runId} confirmed no finding with probe "${probe}" and expectation "${check}". Confirmed: ${list || 'none'}.`);
+    }
+    // Prefer the snapshot finish took of the script it proved: the live probe
+    // file may have been re-run and overwritten since the finding was confirmed.
+    const proofScript = path.join(dir, 'proof', `${probe}.mjs`);
+    const scriptFile = existsSync(proofScript) ? proofScript : path.join(probesDir(dir), `${probe}.mjs`);
+    const script = readFileSync(scriptFile, 'utf8');
+    const leaked = credentialIn(script, ctx.signInVariables);
+    if (leaked)
+        fail(`The probe holds the value of ${leaked}. Kept tests never hold a credential; read it from process.env.${leaked} instead.`);
+    const dest = path.join(testsDir(root), as);
+    if (existsSync(dest))
+        fail(`A kept test named ${as} exists. Choose another name with --as.`);
+    const lines = [`stillhunter | kept | ${as}`];
+    const signIn = signInProbe(dir);
+    const signInDest = path.join(testsDir(root), `${SIGNIN}.mjs`);
+    let signInScript = null;
+    if (signIn && !existsSync(signInDest)) {
+        signInScript = readFileSync(path.join(probesDir(dir), `${signIn}.mjs`), 'utf8');
+        const leakedSignIn = credentialIn(signInScript, ctx.signInVariables);
+        if (leakedSignIn)
+            fail(`The sign-in probe "${signIn}" holds the value of ${leakedSignIn}. Kept tests never hold a credential; read it from process.env.${leakedSignIn} instead.`);
+    }
+    mkdirSync(dest, { recursive: true });
+    writeFileSync(path.join(dest, 'test.mjs'), script);
+    const meta = { check, title: finding.title, severity: finding.severity, keptFrom: { run: ctx.runId, probe } };
+    writeFileSync(path.join(dest, 'test.json'), JSON.stringify(meta, null, 2) + '\n');
+    lines.push(`${FOLDER}/tests/${as}/test.mjs: "${finding.title}" (expectation "${check}")`);
+    if (signInScript !== null) {
+        writeFileSync(signInDest, signInScript);
+        lines.push(`${FOLDER}/tests/${SIGNIN}.mjs: the sign-in probe "${signIn}", run before each replay`);
+    }
+    else if (existsSync(signInDest)) {
+        lines.push(`${FOLDER}/tests/${SIGNIN}.mjs: kept already, left as it is`);
+    }
+    lines.push('While the bug is there the test fails; once it is fixed the test passes.', `Run it with: npx stillhunter replay ${as}   (or /stillhunter:replay ${as})`);
+    process.stdout.write(block(lines));
+}
+function keptTests(root) {
+    const dir = testsDir(root);
+    if (!existsSync(dir))
+        return [];
+    const out = [];
+    for (const name of readdirSync(dir).sort()) {
+        if (!PROBE_NAME.test(name) || !existsSync(path.join(dir, name, 'test.mjs')))
+            continue;
+        try {
+            out.push({ name, meta: JSON.parse(readFileSync(path.join(dir, name, 'test.json'), 'utf8')) });
+        }
+        catch {
+            // A test without readable metadata cannot say which check it guards.
+        }
+    }
+    return out;
+}
+export async function replay(args) {
+    const root = repoRoot();
+    const watch = args.includes('--watch');
+    const names = args.filter((a) => a !== '--watch');
+    const unknownFlag = names.find((n) => n.startsWith('-'));
+    if (unknownFlag)
+        fail(`Unknown replay option ${unknownFlag}. replay takes test names and --watch.`);
+    const all = keptTests(root);
+    if (all.length === 0)
+        fail(`No kept tests in ${FOLDER}/tests/. Keep a confirmed finding with /stillhunter:keep.`);
+    const missing = names.filter((n) => !all.some((t) => t.name === n));
+    if (missing.length)
+        fail(`No kept test named ${missing.join(', ')}. Kept tests: ${all.map((t) => t.name).join(', ')}.`);
+    const selected = names.length ? all.filter((t) => names.includes(t.name)) : all;
+    if (watch && selected.length !== 1)
+        fail(`--watch shows one test in a visible browser. Name one: ${selected.map((t) => t.name).join(', ')}.`);
+    const config = readConfig(root);
+    const target = await findTargetUrl(root, config);
+    if (!target)
+        fail(`No running app found. Start your dev server, or set "target": { "url": ... } in ${FOLDER}/config.json.`);
+    if ((await answers(target)) === null)
+        fail(`Nothing answers at ${target}. Start your dev server first.`);
+    const choice = chooseBrowser(config.browser ?? null, installedOnThisMachine());
+    if (!choice.ok)
+        fail(choice.message);
+    const runId = `${stamp()}-replay-${randomBytes(3).toString('hex').slice(0, 4)}`;
+    const dir = path.join(runsDir(root), runId);
+    // A replay run folder holds a signed-in session too, so keep it out of git
+    // even when replay is the first command run in a fresh clone.
+    ensureRunsIgnored(path.join(root, FOLDER, '.gitignore'));
+    mkdirSync(probesDir(dir), { recursive: true });
+    const now = Date.now();
+    const ctx = {
+        runId, root, target, browser: choice.name, base: '', startedAt: now, deadline: now, depth: 'replay', budgetMinutes: 0,
+        minSeverity: 'low', focus: null, mode: 'replay', signInVariables: signInVariableNames(),
+    };
+    writeFileSync(path.join(dir, 'context.json'), JSON.stringify(ctx, null, 2) + '\n');
+    // The child may read only the run folder, so each test is copied into it.
+    const stage = (name, from) => {
+        const file = path.join(probesDir(dir), `${name}.mjs`);
+        copyFileSync(from, file);
+        return file;
+    };
+    const host = (name, file, watching = false) => runProbe({
+        runDir: dir, probe: name, probeFile: file, target, browser: choice.name, signInVariables: ctx.signInVariables, shotsDir: path.join(dir, 'shots'),
+        ...(watching ? { headed: true, slowMo: WATCH_SLOW_MO_MS, keepOpenMs: WATCH_KEEP_OPEN_MS } : {}),
+    }).then((r) => { writeFileSync(path.join(probesDir(dir), `${name}.result.json`), JSON.stringify(r, null, 2) + '\n'); return r; });
+    const signInSource = path.join(testsDir(root), `${SIGNIN}.mjs`);
+    const signInFile = existsSync(signInSource) ? stage(SIGNIN, signInSource) : null;
+    const signInLines = [];
+    const signIn = async () => {
+        if (!signInFile)
+            return;
+        const r = await host(SIGNIN, signInFile);
+        const problem = r.error ?? (r.savedSession ? null : 'it saved no session');
+        if (problem)
+            signInLines.push(`sign-in failed: ${problem}`);
+    };
+    const notes = [];
+    if (!config.reset)
+        await signIn();
+    const lines = [];
+    let failed = 0;
+    for (const t of selected) {
+        if (config.reset) {
+            const r = await resetApp(root, config);
+            if (r)
+                notes.push(`before ${t.name}: ${r}`);
+            await signIn();
+        }
+        const result = await host(t.name, stage(t.name, path.join(testsDir(root), t.name, 'test.mjs')), watch);
+        const o = outcome(result, t.meta.check);
+        if (o.state === 'held') {
+            lines.push(`PASS  ${t.name}: ${t.meta.title} (expectation "${t.meta.check}" holds)`);
+        }
+        else {
+            failed += 1;
+            lines.push(o.state === 'failed'
+                ? `FAIL  ${t.name}: ${t.meta.title} (expectation "${t.meta.check}" fails: ${o.detail})`
+                : `FAIL  ${t.name}: ${t.meta.title} (expectation "${t.meta.check}" not reached${result.error ? `: ${result.error}` : ''})`);
+        }
+    }
+    process.stdout.write(block([
+        `stillhunter | replay | ${selected.length} test${selected.length === 1 ? '' : 's'} | ${selected.length - failed} passed, ${failed} failed`,
+        `target: ${target}${signInFile ? '' : ` | no ${SIGNIN}.mjs kept, so tests start signed out`}`,
+        ...[...new Set(signInLines)],
+        ...notes,
+        ...lines,
+        `results: ${FOLDER}/runs/${runId}/`,
+    ]));
+    if (failed > 0)
+        process.exitCode = 1;
+}

package/dist/layout.mjs

@@ -0,0 +1,143 @@
+// Layout candidates for observe(): places where a person cannot see what the
+// page holds. It runs inside the page, so it must not use anything from the
+// module around it. Only a 1 px rounding allowance is used: no threshold is
+// tuned to a particular app.
+export function layoutCandidates(limit) {
+    const SKIP_INPUT = new Set(['hidden', 'checkbox', 'radio', 'range', 'color', 'file', 'submit', 'button', 'reset', 'image']);
+    const SKIP_TAG = new Set(['SCRIPT', 'STYLE', 'NOSCRIPT', 'TEMPLATE', 'HEAD', 'META', 'LINK', 'BR', 'OPTION']);
+    const short = (s, n = 80) => {
+        const t = s.replace(/\s+/g, ' ').trim();
+        return t.length > n ? `${t.slice(0, n)}…` : t;
+    };
+    const shown = (el) => {
+        const r = el.getBoundingClientRect();
+        if (r.width <= 0 || r.height <= 0)
+            return false;
+        const check = el.checkVisibility;
+        if (check)
+            return check.call(el, { checkOpacity: true, visibilityProperty: true });
+        const s = getComputedStyle(el);
+        return s.visibility !== 'hidden' && s.display !== 'none';
+    };
+    const selectorOf = (el) => {
+        const parts = [];
+        let e = el;
+        for (let depth = 0; e && e !== document.body && depth < 4; depth++) {
+            if (e.id) {
+                parts.unshift(`#${CSS.escape(e.id)}`);
+                break;
+            }
+            let s = e.tagName.toLowerCase();
+            const classes = [...e.classList].slice(0, 2);
+            if (classes.length)
+                s += `.${classes.map((c) => CSS.escape(c)).join('.')}`;
+            const parent = e.parentElement;
+            if (parent) {
+                const same = [...parent.children].filter((x) => x.tagName === e.tagName);
+                if (same.length > 1)
+                    s += `:nth-of-type(${same.indexOf(e) + 1})`;
+            }
+            parts.unshift(s);
+            e = parent;
+        }
+        return parts.join(' > ');
+    };
+    const fieldName = (el) => short(el.labels?.[0]?.innerText || el.getAttribute('aria-label') || el.getAttribute('name') || el.id || el.tagName.toLowerCase(), 40);
+    const canvas = document.createElement('canvas').getContext('2d');
+    const textWidth = (text, s) => {
+        if (!canvas)
+            return 0;
+        canvas.font = s.font || `${s.fontStyle} ${s.fontWeight} ${s.fontSize} ${s.fontFamily}`;
+        const spacing = Number.parseFloat(s.letterSpacing) || 0;
+        return canvas.measureText(text).width + spacing * text.length;
+    };
+    const valueClipped = [];
+    for (const el of document.querySelectorAll('input, select, textarea')) {
+        if (el instanceof HTMLInputElement && SKIP_INPUT.has(el.type))
+            continue;
+        if (!shown(el))
+            continue;
+        const s = getComputedStyle(el);
+        const content = el.clientWidth - (Number.parseFloat(s.paddingLeft) || 0) - (Number.parseFloat(s.paddingRight) || 0);
+        const text = el instanceof HTMLSelectElement ? (el.selectedOptions[0]?.text ?? '') : (el.value || el.getAttribute('placeholder') || '');
+        const scrolls = el.scrollWidth > el.clientWidth + 1;
+        // A textarea wraps its text, so only a sideways scroll means it is cut.
+        const needed = el instanceof HTMLTextAreaElement || text === '' ? 0 : textWidth(text, s);
+        if (scrolls || needed > content + 1) {
+            valueClipped.push({
+                kind: 'value-clipped',
+                selector: selectorOf(el),
+                text: `${fieldName(el)}: "${short(text, 60)}"`,
+                detail: needed > content + 1 ? `text needs ${Math.ceil(needed)} px, the box shows ${Math.floor(content)} px` : `scrollWidth ${el.scrollWidth} > clientWidth ${el.clientWidth}`,
+            });
+        }
+    }
+    const all = [...document.body.querySelectorAll('*')].filter((el) => !SKIP_TAG.has(el.tagName));
+    const textClipped = [];
+    for (const el of all) {
+        if (el instanceof HTMLInputElement || el instanceof HTMLSelectElement || el instanceof HTMLTextAreaElement)
+            continue;
+        const s = getComputedStyle(el);
+        const clipX = s.overflowX === 'hidden' || s.overflowX === 'clip' || s.textOverflow === 'ellipsis';
+        const clipY = s.overflowY === 'hidden' || s.overflowY === 'clip';
+        if (!clipX && !clipY)
+            continue;
+        const r = el.getBoundingClientRect();
+        // 1 px boxes are the usual visually-hidden pattern for screen readers.
+        if (r.width <= 1 || r.height <= 1 || !shown(el))
+            continue;
+        const text = el.innerText ?? '';
+        if (!text.trim())
+            continue;
+        const wide = clipX && el.scrollWidth > el.clientWidth + 1;
+        const tall = clipY && el.scrollHeight > el.clientHeight + 1;
+        if (!wide && !tall)
+            continue;
+        textClipped.push({
+            kind: 'text-clipped',
+            selector: selectorOf(el),
+            text: short(text),
+            detail: [wide ? `content ${el.scrollWidth} px wide in a ${el.clientWidth} px box` : '', tall ? `content ${el.scrollHeight} px tall in a ${el.clientHeight} px box` : ''].filter(Boolean).join('; '),
+        });
+    }
+    const viewport = document.documentElement.clientWidth || window.innerWidth;
+    const offScreen = [];
+    const reported = new Set();
+    const scrollsSideways = (el) => {
+        for (let a = el.parentElement; a && a !== document.documentElement; a = a.parentElement) {
+            if (a === document.body)
+                break;
+            const ox = getComputedStyle(a).overflowX;
+            if (ox === 'auto' || ox === 'scroll')
+                return true;
+        }
+        return false;
+    };
+    for (const el of all) {
+        const r = el.getBoundingClientRect();
+        // Off-screen is any visible element whose right edge is past the viewport,
+        // including one pushed entirely past it, unless it scrolls sideways on
+        // purpose. An element wholly off the left (right edge at or before 0) is
+        // left out by the first test.
+        if (r.right <= viewport + 1 || r.width <= 0 || r.height <= 0)
+            continue;
+        if (!shown(el) || scrollsSideways(el))
+            continue;
+        let inner = false;
+        for (let a = el.parentElement; a; a = a.parentElement)
+            if (reported.has(a)) {
+                inner = true;
+                break;
+            }
+        if (inner)
+            continue;
+        reported.add(el);
+        offScreen.push({
+            kind: 'off-screen',
+            selector: selectorOf(el),
+            text: short(el.innerText ?? ''),
+            detail: `right edge at ${Math.round(r.right)} px, the viewport is ${viewport} px wide`,
+        });
+    }
+    return [...valueClipped, ...textClipped, ...offScreen].slice(0, limit);
+}