rvlib-shipkit 0.35.8 → 0.36.0

package/src/smoke/run-smoke.ts

@@ -0,0 +1,403 @@
+// `shipkit smoke` — a READ ONLY browser pass over a deployed web app: log in
+// through the app's own login page, visit every declared route, count what
+// rendered, collect console errors and failed requests, screenshot each
+// route, and read the host's quota headers when it sends them.
+//
+// It never submits a business form, never clicks a destructive control and
+// never writes. It runs against whatever the base url points at, which is
+// usually PRODUCTION, so the read only rule is a hard constraint.
+//
+// Playwright is NOT a dependency of this package: a toolkit every repo
+// installs must not carry a browser. It is resolved from the CONSUMER repo at
+// run time, and a repo that never installed it gets one loud line naming the
+// two commands. Everything that does not need a browser (config validation,
+// the route verdict, the report) is pure and tested; the browser half is
+// exercised in the field, by the run itself.
+import { mkdirSync } from 'fs'
+import { join } from 'path'
+
+export type SmokeRoute = {
+    /** the path to visit, always leading `/` */
+    path: string
+    /** a selector that must match at least once once the route settled
+     *  (`tbody tr` for a listing). Absent: the route only has to render text. */
+    waitFor?: string
+    /** a link selector (`a[href^="/orders/"]`): the run follows the FIRST
+     *  match's href after this route and judges the landed page as its own
+     *  row. How a detail page with a real id gets visited without knowing one. */
+    open?: string
+}
+
+export type SmokeLogin = {
+    /** where the login form lives. Default `/login`, the house router seed. */
+    path?: string
+    /** the identity typed into the login form. One of `email` / `emailSecret`. */
+    email?: string
+    /** the secret whose ACCOUNT field is the identity (read through the repo's `credsReadCommand`) */
+    emailSecret?: string
+    /** the secret whose value is the password, a NAME the repo's `credsReadCommand` resolves. Never argv, never env. */
+    passwordSecret: string
+    /** how to find the three controls: a CSS selector (starts with `#`, `.`,
+     *  `[` or a tag) or a LABEL. A label finds the field by its `<label>` text
+     *  and the submit by its button name. */
+    form: { email: string; password: string; submit: string }
+}
+
+export type SmokeConfig = {
+    /** the url the run visits. `--base` overrides it for one run. */
+    base: string
+    /** the routes to visit, in order. A bare string is `{ path }`. */
+    routes: (string | SmokeRoute)[]
+    /** absent: the run visits the routes anonymously */
+    login?: SmokeLogin
+    /** pause between routes, ms. A route change fires a burst of requests, and
+     *  a host with a CONCURRENT cap sheds some of them without CORS headers,
+     *  which then reads as a broken app. 3000 is measured against PocketHost's
+     *  15 concurrent per IP: 0 reported 8 of 12 routes, 3000 reports 12 of 12. */
+    paceMs?: number
+    /** where the screenshots land, repo relative */
+    shots?: string
+}
+
+const SMOKE_KEYS = new Set(['base', 'routes', 'login', 'paceMs', 'shots'])
+const LOGIN_KEYS = new Set(['path', 'email', 'emailSecret', 'passwordSecret', 'form'])
+const FORM_KEYS = new Set(['email', 'password', 'submit'])
+const ROUTE_KEYS = new Set(['path', 'waitFor', 'open'])
+export const DEFAULT_LOGIN_PATH = '/login'
+
+export const DEFAULT_PACE_MS = 3000
+export const DEFAULT_SHOTS = 'tmp/smoke'
+
+/** the whole shape, refused loud on the first thing that is not so. A typo
+ *  here would otherwise land as a run that visits nothing and exits 0. */
+export function validateSmokeConfig(c: unknown): SmokeConfig {
+    if (c == null || typeof c !== 'object') throw new Error('defineRepo: smoke must be an object { base, routes, login? }')
+    const s = c as Record<string, unknown>
+    for (const k of Object.keys(s)) if (!SMOKE_KEYS.has(k)) throw new Error(`defineRepo: smoke.${k} is not a smoke field (fields: ${[...SMOKE_KEYS].join(', ')})`)
+    if (typeof s['base'] !== 'string' || !/^https?:\/\//.test(s['base'])) throw new Error(`defineRepo: smoke.base must be an http(s) url`)
+    if (!Array.isArray(s['routes']) || s['routes'].length === 0) throw new Error(`defineRepo: smoke.routes must list at least one route`)
+    for (const r of s['routes'] as unknown[]) {
+        const path = typeof r === 'string' ? r : r != null && typeof r === 'object' ? (r as Record<string, unknown>)['path'] : undefined
+        if (typeof path !== 'string' || !path.startsWith('/')) throw new Error(`defineRepo: smoke.routes entry ${JSON.stringify(r)} needs a path starting with '/'`)
+        if (typeof r === 'object' && r != null) {
+            for (const k of Object.keys(r)) if (!ROUTE_KEYS.has(k)) throw new Error(`defineRepo: smoke.routes['${path}'].${k} is not a route field (fields: ${[...ROUTE_KEYS].join(', ')})`)
+            for (const k of ['waitFor', 'open'] as const) {
+                const v = (r as Record<string, unknown>)[k]
+                if (v != null && (typeof v !== 'string' || v === '')) throw new Error(`defineRepo: smoke.routes['${path}'].${k} must be a non-empty selector`)
+            }
+        }
+    }
+    if (s['login'] != null) {
+        const l = s['login'] as Record<string, unknown>
+        if (typeof l !== 'object') throw new Error('defineRepo: smoke.login must be an object')
+        for (const k of Object.keys(l)) if (!LOGIN_KEYS.has(k)) throw new Error(`defineRepo: smoke.login.${k} is not a login field (fields: ${[...LOGIN_KEYS].join(', ')})`)
+        if (l['path'] != null && (typeof l['path'] !== 'string' || !l['path'].startsWith('/'))) throw new Error(`defineRepo: smoke.login.path must start with '/'`)
+        const hasEmail = typeof l['email'] === 'string' && l['email'] !== ''
+        const hasEmailSecret = typeof l['emailSecret'] === 'string' && l['emailSecret'] !== ''
+        if (hasEmail === hasEmailSecret) throw new Error('defineRepo: smoke.login needs exactly one of email / emailSecret')
+        if (typeof l['passwordSecret'] !== 'string' || l['passwordSecret'] === '') throw new Error('defineRepo: smoke.login.passwordSecret is required (a secret name, never the value)')
+        // a password typed into config is the leak this field exists to prevent
+        if (!/^[\w./-]+$/.test(l['passwordSecret'])) throw new Error('defineRepo: smoke.login.passwordSecret must be a secret NAME (letters, digits, / . _ -), not a value')
+        const f = l['form'] as Record<string, unknown> | undefined
+        if (f == null || typeof f !== 'object') throw new Error('defineRepo: smoke.login.form { email, password, submit } is required')
+        for (const k of Object.keys(f)) if (!FORM_KEYS.has(k)) throw new Error(`defineRepo: smoke.login.form.${k} is not a form field (fields: email, password, submit)`)
+        for (const k of FORM_KEYS) if (typeof f[k] !== 'string' || f[k] === '') throw new Error(`defineRepo: smoke.login.form.${k} is required (a selector or a label)`)
+    }
+    if (s['paceMs'] != null && (typeof s['paceMs'] !== 'number' || s['paceMs'] < 0)) throw new Error('defineRepo: smoke.paceMs must be a number of milliseconds')
+    if (s['shots'] != null && (typeof s['shots'] !== 'string' || s['shots'] === '')) throw new Error('defineRepo: smoke.shots must be a repo relative dir')
+    return c as SmokeConfig
+}
+
+export const normalizeRoutes = (routes: (string | SmokeRoute)[]): SmokeRoute[] =>
+    routes.map((r) => (typeof r === 'string' ? { path: r } : r))
+
+/** `--routes /a,/b` → the declared entries for those paths, or bare paths for
+ *  routes the config never declared (a one-off visit is still a visit) */
+export function selectRoutes(declared: SmokeRoute[], flag: string | undefined): SmokeRoute[] {
+    if (flag == null || flag.trim() === '') return declared
+    const byPath = new Map(declared.map((r) => [r.path, r]))
+    return flag.split(',').map((p) => p.trim()).filter((p) => p !== '').map((p) => byPath.get(p) ?? { path: p })
+}
+
+/** the screenshot file name of a route */
+export const routeSlug = (path: string): string => (path === '/' ? 'root' : path.replace(/^\//, '').replace(/[^\w.-]+/g, '-'))
+
+export type Problem = { route: string; kind: string; detail: string }
+
+export type RouteReport = {
+    route: string
+    ok: boolean
+    /** visible text length once the route settled */
+    chars: number
+    /** `waitFor` matches (0 when the route declares none) */
+    matches: number
+    /** console, page and network problems collected while on this route */
+    problems: number
+    note: string
+}
+
+export type Quota = { remaining: number; limit: number; resetAt: number }
+
+export type SmokeReport = {
+    base: string
+    identity: string | null
+    routes: RouteReport[]
+    problems: Problem[]
+    quota: Quota | null
+    shots: string
+}
+
+/** the verdict of one route, pure: a blank shell is a failure even when
+ *  nothing threw, an empty `waitFor` on a listing is exactly how a broken
+ *  migration looks, and a page that scrolls sideways is a layout defect only
+ *  a real browser can see (happy-dom does no layout). */
+export function judgeRoute(r: { route: string; chars: number; matches: number; problems: number; waitFor?: string; overflowPx: number }): RouteReport {
+    const empty = r.chars < 40
+    const noMatch = r.waitFor != null && r.matches === 0
+    const sideways = r.overflowPx > 1
+    const note = empty ? 'EMPTY PAGE' : noMatch ? `NO MATCH FOR ${r.waitFor}` : sideways ? `PAGE SCROLLS SIDEWAYS +${r.overflowPx}px` : r.problems > 0 ? `${r.problems} problem(s)` : ''
+    return { route: r.route, ok: !empty && !noMatch && !sideways && r.problems === 0, chars: r.chars, matches: r.matches, problems: r.problems, note }
+}
+
+/** a failure that only exists because the backend refused to answer: a host
+ *  over its quota drops the CORS headers, so the browser reports a CORS error */
+export const THROTTLE_SHAPE = /ERR_FAILED|blocked by CORS policy|Failed to load resource/
+
+// noise that is not a defect: a favicon 404, a devtools hint, a Vite HMR ping
+const IGNORE = [/favicon/i, /Download the React DevTools/i, /\[vite\]/i, /React Router Future Flag/i]
+export const isNoise = (s: string): boolean => IGNORE.some((re) => re.test(s))
+
+/** the wire lines of a finished run */
+export function renderReport(rep: SmokeReport, opts: { paceMs: number }): string[] {
+    const out: string[] = []
+    out.push('route'.padEnd(42) + 'chars   matches  status')
+    for (const r of rep.routes) out.push(r.route.padEnd(42) + String(r.chars).padEnd(8) + String(r.matches).padEnd(9) + (r.ok ? 'ok' : `FAIL ${r.note}`))
+    const seen = new Set<string>()
+    const unique = rep.problems.filter((p) => {
+        const key = `${p.route}|${p.kind}|${p.detail}`
+        if (seen.has(key)) return false
+        seen.add(key)
+        return true
+    })
+    if (unique.length > 0) {
+        out.push('')
+        out.push(`${unique.length} console/network problem(s):`)
+        for (const p of unique) out.push(`  [${p.route}] ${p.kind}: ${p.detail}`)
+    }
+    if (rep.quota != null) {
+        const mins = rep.quota.resetAt > 0 ? Math.max(0, Math.round((rep.quota.resetAt * 1000 - Date.now()) / 60_000)) : 0
+        out.push('')
+        out.push(`quota: ${rep.quota.remaining}/${rep.quota.limit} requests left for this IP this hour, resets in ~${mins} min`)
+    }
+    const throttled = rep.problems.filter((p) => THROTTLE_SHAPE.test(p.detail)).length
+    if (throttled > 0) {
+        // two different caps, and naming the wrong one sends the reader to the
+        // wrong fix: the hourly quota needs waiting out, the concurrent cap
+        // needs pacing. The remaining count tells them apart.
+        const hourlyExhausted = rep.quota != null && rep.quota.remaining < 50
+        out.push('')
+        out.push(`WARN throttled: ${throttled} of the problems above are the backend refusing to answer, not the app (a shed request carries no CORS headers)`)
+        out.push(hourlyExhausted ? '     the HOURLY quota is spent: wait for the reset above, then re-run' : `     the hourly quota is healthy, so this is the CONCURRENT cap: raise smoke.paceMs (currently ${opts.paceMs}) and re-run before believing any of them`)
+    }
+    const failed = rep.routes.filter((r) => !r.ok).length
+    out.push('')
+    out.push(`${failed === 0 ? 'ok:' : 'FAIL:'} ${rep.routes.length - failed}/${rep.routes.length} routes ok${rep.identity != null ? ` as ${rep.identity}` : ''}. Screenshots in ${rep.shots}/`)
+    return out
+}
+
+/** the process exit code: the number of failed routes, held inside the byte a
+ *  shell can read (a count above it would wrap to a false green) */
+export const exitCodeFor = (rep: SmokeReport): number => Math.min(125, rep.routes.filter((r) => !r.ok).length)
+
+// ---------------------------------------------------------------------------
+// the browser half: everything under here needs playwright in the consumer
+
+/** the argv of a `credsReadCommand` template for one secret: split on
+ *  whitespace, placeholders substituted, never a shell */
+export function secretArgv(template: string, name: string, field: 'get' | 'account'): string[] {
+    return template.trim().split(/\s+/).map((w) => w.replaceAll('{name}', name).replaceAll('{field}', field))
+}
+
+/** a secret read through the repo's declared command, never argv and never
+ *  env. The ACCOUNT field of a secret carries the identity, the value the
+ *  password. */
+export function readSecret(template: string, name: string, field: 'get' | 'account' = 'get'): string {
+    const argv = secretArgv(template, name, field)
+    const bin = argv[0] ?? ''
+    if (bin === '' || Bun.which(bin) == null) throw new Error(`FAIL credsReadCommand names '${bin}', which is not on PATH`)
+    const r = Bun.spawnSync(argv)
+    const v = r.stdout.toString().trim()
+    if (r.exitCode !== 0 || v === '') throw new Error(`FAIL \`${argv.join(' ')}\` answered nothing — store the secret under that name first`)
+    return v
+}
+
+/** the smallest slice of playwright this module touches, so the package
+ *  types nothing it does not ship */
+type Locator = { fill(v: string): Promise<void>; click(): Promise<void>; count(): Promise<number>; first(): Locator; innerText(): Promise<string>; getAttribute(name: string): Promise<string | null> }
+type PwPage = {
+    on(event: string, handler: (arg: never) => void): void
+    goto(url: string, opts?: { waitUntil?: string }): Promise<unknown>
+    waitForLoadState(state: string, opts?: { timeout?: number }): Promise<void>
+    waitForSelector(sel: string, opts?: { timeout?: number }): Promise<unknown>
+    waitForURL(pred: (u: URL) => boolean, opts?: { timeout?: number }): Promise<void>
+    waitForTimeout(ms: number): Promise<void>
+    locator(sel: string): Locator
+    getByLabel(text: string): Locator
+    getByRole(role: string, opts: { name: string | RegExp }): Locator
+    screenshot(opts: { path: string; fullPage?: boolean }): Promise<unknown>
+    evaluate<T>(fn: () => T): Promise<T>
+}
+type PwBrowser = { newContext(opts: { viewport: { width: number; height: number } }): Promise<{ newPage(): Promise<PwPage> }>; close(): Promise<void> }
+type Playwright = { chromium: { launch(opts: { headless: boolean }): Promise<PwBrowser> } }
+
+export const PLAYWRIGHT_HINT = 'FAIL playwright is not installed in this repo — run: bun add -d playwright && npx playwright install chromium'
+
+/** resolved from the REPO, never from this package: the browser is the
+ *  consumer's dependency, and the message names the two commands that fix it */
+export async function loadPlaywright(root: string): Promise<Playwright> {
+    let resolved: string
+    try {
+        resolved = Bun.resolveSync('playwright', root)
+    } catch {
+        throw new Error(PLAYWRIGHT_HINT)
+    }
+    return (await import(resolved)) as Playwright
+}
+
+const isSelector = (spec: string): boolean => /^[#.[]/.test(spec) || /^[a-z][\w-]*(\[|:|$)/.test(spec) || spec.includes(' > ')
+
+const control = (page: PwPage, spec: string, role: 'field' | 'submit'): Locator =>
+    isSelector(spec) ? page.locator(spec).first() : role === 'field' ? page.getByLabel(spec) : page.getByRole('button', { name: spec })
+
+/** wait for the app to SETTLE, not merely to load: a MobX page fires its
+ *  queries after mount, so `networkidle` samples a shell with a nav and no
+ *  content. The body text is sampled until it stops growing, and a `waitFor`
+ *  selector is awaited first, because a spinner holds the text length
+ *  constant and "stopped growing" alone reports a loading page as settled. */
+async function settle(page: PwPage, waitFor?: string): Promise<void> {
+    await page.waitForLoadState('networkidle', { timeout: 20_000 }).catch(() => {})
+    if (waitFor != null) await page.waitForSelector(waitFor, { timeout: 40_000 }).catch(() => {})
+    let previous = -1
+    for (let i = 0; i < 30; i++) {
+        const size = (await page.locator('body').innerText().catch(() => '')).trim().length
+        if (size === previous && size > 0) return
+        previous = size
+        await page.waitForTimeout(400)
+    }
+}
+
+export type SmokeIo = { print: (l: string) => void }
+
+export type SmokeRunOptions = { base?: string; routes?: string; credsReadCommand?: string }
+
+/** the run. Returns the report; the caller renders it and picks the exit code. */
+export async function runSmoke(root: string, config: SmokeConfig, opts: SmokeRunOptions, io: SmokeIo): Promise<SmokeReport> {
+    const base = (opts.base ?? config.base).replace(/\/$/, '')
+    const routes = selectRoutes(normalizeRoutes(config.routes), opts.routes)
+    const paceMs = config.paceMs ?? DEFAULT_PACE_MS
+    const shotsRel = config.shots ?? DEFAULT_SHOTS
+    const shots = join(root, shotsRel)
+    mkdirSync(shots, { recursive: true })
+
+    // secrets are read BEFORE the browser starts, so a missing one costs no launch
+    const login = config.login
+    const reader = opts.credsReadCommand ?? ''
+    if (login != null && reader === '') throw new Error('FAIL smoke.login needs credsReadCommand in repo.config.ts (defineRepo refuses the pair, so this config bypassed it)')
+    const identity = login == null ? null : (login.email ?? readSecret(reader, login.emailSecret!, 'account'))
+    const password = login == null ? null : readSecret(reader, login.passwordSecret, 'get')
+
+    const pw = await loadPlaywright(root)
+    const browser = await pw.chromium.launch({ headless: true })
+    const ctx = await browser.newContext({ viewport: { width: 1440, height: 900 } })
+    const page = await ctx.newPage()
+    const problems: Problem[] = []
+    const quota: Quota = { remaining: Number.NaN, limit: Number.NaN, resetAt: 0 }
+    const loginPath = login?.path ?? DEFAULT_LOGIN_PATH
+    let current = login != null ? loginPath : routes[0]?.path ?? '/'
+
+    page.on('console', (m: { type(): string; text(): string }) => {
+        if (m.type() !== 'error' && m.type() !== 'warning') return
+        const text = m.text()
+        if (isNoise(text)) return
+        problems.push({ route: current, kind: `console.${m.type()}`, detail: text.slice(0, 300) })
+    })
+    page.on('pageerror', (e: Error) => problems.push({ route: current, kind: 'pageerror', detail: String(e.message).slice(0, 300) }))
+    page.on('requestfailed', (r: { url(): string; failure(): { errorText: string } | null }) => {
+        const url = r.url()
+        if (isNoise(url)) return
+        problems.push({ route: current, kind: 'requestfailed', detail: `${url} ${r.failure()?.errorText ?? ''}`.slice(0, 300) })
+    })
+    page.on('response', (r: { headers(): Record<string, string>; status(): number; url(): string }) => {
+        const h = r.headers()
+        // PocketHost's budget, read off every response that carries it
+        const remaining = h['x-pockethost-ratelimit-ip-hourly-remaining']
+        if (remaining != null) {
+            quota.remaining = Number(remaining)
+            quota.limit = Number(h['x-pockethost-ratelimit-ip-hourly-limit'] ?? Number.NaN)
+            quota.resetAt = Number(h['x-pockethost-ratelimit-ip-hourly-reset'] ?? 0)
+        }
+        if (r.status() < 400 || isNoise(r.url())) return
+        problems.push({ route: current, kind: `http.${r.status()}`, detail: r.url().slice(0, 300) })
+    })
+
+    io.print(`smoke: ${base}${identity != null ? ` as ${identity}` : ''}`)
+
+    try {
+        const reports: RouteReport[] = []
+        if (login != null) {
+            await page.goto(`${base}${loginPath}`, { waitUntil: 'domcontentloaded' })
+            await settle(page)
+            await control(page, login.form.email, 'field').fill(identity!)
+            await control(page, login.form.password, 'field').fill(password!)
+            await control(page, login.form.submit, 'submit').click()
+            const landed = await page
+                .waitForURL((u) => !u.pathname.startsWith(loginPath), { timeout: 25_000 })
+                .then(() => true)
+                .catch(() => false)
+            await settle(page)
+            if (!landed) {
+                const visible = await page.locator('body').innerText().catch(() => '')
+                await page.screenshot({ path: join(shots, 'login-failed.png'), fullPage: true })
+                throw new Error(`FAIL login as ${identity} never left ${loginPath}. The page said:\n${visible.slice(0, 600)}`)
+            }
+            io.print('ok: login')
+            // a problem during the login is a failure of its own row, or a 500
+            // right after the login would exit 0
+            if (problems.length > 0) reports.push(judgeRoute({ route: loginPath, chars: 40, matches: 0, problems: problems.length, overflowPx: 0 }))
+        }
+
+        /** visit one url, settle it, judge it, screenshot it */
+        const visit = async (label: string, url: string, waitFor?: string): Promise<RouteReport> => {
+            // let the previous page's requests drain out of the concurrency
+            // window BEFORE the label moves, or its late failures land on this row
+            if (paceMs > 0) await page.waitForTimeout(paceMs)
+            current = label
+            const before = problems.length
+            await page.goto(`${base}${url}`, { waitUntil: 'domcontentloaded' })
+            await settle(page, waitFor)
+            const chars = (await page.locator('body').innerText().catch(() => '')).trim().length
+            const matches = waitFor == null ? 0 : await page.locator(waitFor).count().catch(() => 0)
+            const overflowPx = await page.evaluate(() => {
+                const de = document.documentElement
+                return Math.max(de.scrollWidth - de.clientWidth, document.body.scrollWidth - document.body.clientWidth)
+            }).catch(() => 0)
+            await page.screenshot({ path: join(shots, `${routeSlug(label)}.png`), fullPage: true })
+            return judgeRoute({ route: label, chars, matches, problems: problems.length - before, waitFor, overflowPx })
+        }
+
+        for (const route of routes) {
+            reports.push(await visit(route.path, route.path, route.waitFor))
+            if (route.open == null) continue
+            // a detail page with a real id: the first link the selector matches
+            const href = await page.locator(route.open).first().getAttribute('href').catch(() => null)
+            if (href == null || !href.startsWith('/')) {
+                reports.push({ route: `${route.path} → ${route.open}`, ok: false, chars: 0, matches: 0, problems: 0, note: 'NO LINK TO OPEN' })
+                continue
+            }
+            reports.push(await visit(`${route.path} → ${href}`, href))
+        }
+        return { base, identity, routes: reports, problems, quota: Number.isNaN(quota.remaining) ? null : quota, shots: shotsRel }
+    } finally {
+        await browser.close()
+    }
+}