tewip-ai 0.46.0 → 0.47.0

package/dist/cli/help.js

@@ -118,10 +118,11 @@ Usage:
                                     is what was asked for covered. Answers ready, not ready, or cannot
                                     say - and cannot say is never rounded up to ready. --json writes a
                                     sealed verdict that can be checked later. No model, no cost.
-  tewip perf <k6-summary.json | lighthouse.json> [--record] [--label checkout] [--strict] [--out perf.md]
+  tewip perf <k6.json | lighthouse.json | run.jtl> [--against before.jtl] [--record] [--strict]
                                     Is this change slower, and does it matter? Reads a k6 run (what
                                     the server does under load) or a Lighthouse report (what one
-                                    page load costs in a browser), and answers two questions apart:
+                                    page load costs in a browser) or a JMeter .jtl, and answers two
+                                    questions apart:
                                     is anything in a bad state now - from the thresholds you wrote,
                                     or from Lighthouse's own scoring, never a budget Tewip invented
                                     - and is this slower than it was. The second only from repeated
@@ -129,7 +130,10 @@ Usage:
                                     run against one cannot tell a regression from a busy machine.
                                     --record adds a run to the baseline, and is deliberately not
                                     automatic: a regression recorded by accident becomes next week's
-                                    normal. It never predicts a duration or a capacity. No cost.
+                                    normal. A .jtl is different and better: it carries every request,
+                                    so --against before.jtl compares the two distributions with a
+                                    real test rather than a range check - and refuses a difference
+                                    that is certain but too small to act on. No cost.
   tewip access [<file or directory>] [--report axe.json|lighthouse.json] [--strict]
                                     Can everybody use this? Reads your markup and names the barriers
                                     that reading alone can prove - a picture with no alt text, a field

package/dist/cli/verbs-prove.js

@@ -35,9 +35,11 @@ export async function proveVerb(verb, args) {
         }
         case 'perf': {
             const { runPerf } = await import('../engine/perf/perf-command.js');
-            const paths = args.filter((a, i) => !a.startsWith('--') && !['--out', '--label'].includes(args[i - 1] ?? ''));
+            const paths = args.filter((a, i) => !a.startsWith('--') && !['--out', '--label', '--against'].includes(args[i - 1] ?? ''));
             if (paths.length === 0) {
-                console.error('tewip perf needs the JSON from a k6 run: tewip perf summary.json [--record] [--label checkout]');
+                console.error('tewip perf needs a k6 summary, a Lighthouse report or a JMeter .jtl.');
+                console.error('  tewip perf summary.json [--record] [--label checkout]');
+                console.error('  tewip perf now.jtl --against before.jtl');
                 process.exitCode = 1;
                 return true;
             }
@@ -45,6 +47,7 @@ export async function proveVerb(verb, args) {
                 root: process.cwd(),
                 summary: paths[0],
                 label: flag(args, 'label'),
+                against: flag(args, 'against'),
                 record: args.includes('--record'),
                 out: flag(args, 'out'),
                 gate: args.includes('--strict'),

package/dist/engine/import/junit.js

@@ -13,7 +13,7 @@ import { readdirSync, readFileSync, statSync } from 'node:fs';
 import { join } from 'node:path';
 import { SCHEMA_VERSION } from '../../reporters/types.js';
 import { descendants, parseXml } from './xml.js';
-import { jmeterCases, k6Cases, looksLikeK6, robotCases, robotTime } from './other-formats.js';
+import { jmeterCases, jmeterCsvCases, k6Cases, looksLikeCsvJtl, looksLikeK6, robotCases, robotTime } from './other-formats.js';
 import { nunitCases, trxCases } from './dotnet.js';
 import { PASSED, failure } from './case.js';
 const FAILURE = new Set(['failure', 'error']);
@@ -182,6 +182,13 @@ export function importJunit(path, meta) {
             addCases(k6Cases(parsed));
             continue;
         }
+        // JMeter's default output is CSV, not XML, so a real .jtl would otherwise be rejected as "not
+        // an XML document" - which it is not, and never was.
+        if (looksLikeCsvJtl(raw)) {
+            parsedFiles++;
+            addCases(jmeterCsvCases(raw));
+            continue;
+        }
         const root = parseXml(raw);
         let cases;
         if (root.name === 'TestRun') {

package/dist/engine/import/other-formats.js

@@ -1,4 +1,6 @@
 import { descendants } from './xml.js';
+import { looksLikeCsvJtl, readCsv } from '../perf/jtl.js';
+export { looksLikeCsvJtl };
 const text = (element) => (element ? element.text.trim() : '');
 /**
  * Robot Framework writes output.xml by default, and only writes JUnit if asked. A suite holds
@@ -125,3 +127,56 @@ export function looksLikeK6(json) {
         return false;
     return ['http_req_duration', 'iterations', 'vus', 'checks', 'http_reqs'].some((m) => m in report.metrics);
 }
+/**
+ * JMeter's CSV results file, which is the one it writes by default.
+ *
+ * The XML reader above has been the only one since this was written, and `.jtl` means CSV in every
+ * real repository you look in - so `importJunit` answered "not an XML document: no root element" on
+ * genuine JMeter output while the help text advertised JMeter support. A format nobody tested
+ * against a real file.
+ *
+ * The parsing is shared with `tewip perf`, which needs the same rows for its statistics, and it is a
+ * real CSV reader rather than a line split because JMeter quotes a failureMessage containing commas
+ * and newlines - one real record found here spans seven physical lines.
+ */
+export function jmeterCsvCases(text) {
+    const rows = readCsv(text);
+    const header = (rows[0] ?? []).map((h) => h.trim());
+    const at = (name) => header.indexOf(name);
+    const elapsed = at('elapsed');
+    const label = at('label');
+    if (elapsed === -1 || label === -1)
+        return [];
+    const success = at('success');
+    const code = at('responseCode');
+    const message = at('responseMessage');
+    const failure = at('failureMessage');
+    const byLabel = new Map();
+    for (const row of rows.slice(1)) {
+        if (row.length <= Math.max(elapsed, label))
+            continue;
+        const name = (row[label] ?? '').trim() || 'sample';
+        const found = byLabel.get(name) ?? { failed: 0, total: 0, time: 0, codes: new Map(), messages: new Set() };
+        found.total += 1;
+        const ms = Number(row[elapsed]);
+        if (Number.isFinite(ms))
+            found.time += ms;
+        if (success !== -1 && (row[success] ?? '').trim().toLowerCase() === 'false') {
+            found.failed += 1;
+            const seen = `${code === -1 ? '' : (row[code] ?? '')} ${message === -1 ? '' : (row[message] ?? '')}`.trim();
+            if (seen)
+                found.codes.set(seen, (found.codes.get(seen) ?? 0) + 1);
+            const why = failure === -1 ? '' : (row[failure] ?? '').trim();
+            if (why)
+                found.messages.add(why.replace(/\s+/g, ' ').slice(0, 200));
+        }
+        byLabel.set(name, found);
+    }
+    return [...byLabel].map(([name, s]) => {
+        const worst = [...s.codes].sort((a, b) => b[1] - a[1])[0];
+        // A 2xx that still counts as failed is an assertion failing, not the server.
+        const why = worst ? (/^2\d\d/.test(worst[0]) ? `, an assertion failed (${worst[1]} times, response ${worst[0]})` : `, mostly ${worst[0]} (${worst[1]} times)`) : '';
+        const text = s.failed === 0 ? '' : `${s.failed} of ${s.total} requests failed${why}${s.messages.size > 0 ? `: ${[...s.messages][0]}` : ''}`;
+        return { testId: `JMeter > ${name}`, file: '', duration: s.time, attempts: [{ failed: s.failed > 0, message: text, stack: [...s.messages].join('\n').slice(0, 2000) }] };
+    });
+}

package/dist/engine/perf/jtl-report.js

@@ -0,0 +1,128 @@
+// Phase 39: what Tewip says about a JMeter run.
+//
+// Kept apart from perf-command.ts because a .jtl is answered differently from a k6 summary or a
+// Lighthouse report, and for a good reason rather than a tidy one: those two hand over aggregates,
+// so the only question available is "is this number outside the baseline range". A .jtl hands over
+// every request, so the question becomes "are these two distributions different", which is a
+// better question with a real answer - per label, not just overall.
+import { byLabel, overall, readJtlSamples } from './jtl.js';
+import { compareSamples, ENOUGH_SAMPLES, median } from './samples.js';
+const ms = (n) => `${Math.round(n * 10) / 10}ms`;
+/** A run on its own: no baseline to compare against, so nothing is claimed about a change. */
+export function describeRun(groups, total) {
+    const lines = ['# Tewip performance check', ''];
+    lines.push(`Read a JMeter run of **${total} requests** across ${groups.length} ${groups.length === 1 ? 'label' : 'labels'}.`, '');
+    lines.push('**No baseline given, so nothing is said about whether this is slower.** Pass the previous run ' +
+        'with `--against baseline.jtl` and Tewip will compare the two request by request.', '');
+    lines.push('## What this run did', '');
+    lines.push('| Label | Requests | Failed | Median | p95 |', '| --- | --- | --- | --- | --- |');
+    for (const g of groups) {
+        lines.push(`| ${g.label || '(unnamed)'} | ${g.total} | ${g.failed} | ${ms(median(g.elapsed))} | ${ms(percentile(g.elapsed, 95))} |`);
+    }
+    lines.push('');
+    const failing = groups.filter((g) => g.failed > 0);
+    if (failing.length > 0) {
+        lines.push('## Requests that failed', '');
+        for (const g of failing) {
+            const codes = g.codes.map((c) => `${c.code} x${c.times}`).join(', ');
+            // A 2xx that still counts as failed is an assertion failing, not the server.
+            const assertion = g.codes.some((c) => /^2\d\d/.test(c.code)) ? ' - a 2xx counted as failed is an assertion, not the server' : '';
+            lines.push(`- **${g.label || '(unnamed)'}**: ${g.failed} of ${g.total}${codes ? ` (${codes})` : ''}${assertion}`);
+        }
+        lines.push('');
+    }
+    lines.push(...notKnown());
+    return lines.join('\n');
+}
+export function percentile(values, p) {
+    if (values.length === 0)
+        return 0;
+    const sorted = [...values].sort((a, b) => a - b);
+    const at = Math.min(sorted.length - 1, Math.ceil((p / 100) * sorted.length) - 1);
+    return sorted[Math.max(0, at)];
+}
+function notKnown() {
+    return [
+        '## What this does not tell you',
+        '',
+        '- **Nothing about production.** This is the hardware the run happened on, with that data volume and ' +
+            'that concurrency. Tewip cannot see your real machines, your real data or your real traffic.',
+        '- **No duration is predicted**, and no capacity. It says whether the two runs differ, not how many users you could hold.',
+        '- **Only what the plan exercised.** An endpoint the test never called is not covered, however green this looks.',
+        '- **A statistical difference is not a cause.** Tewip says the distributions moved; why they moved is in your change.',
+        '',
+    ];
+}
+/** Two runs, compared request by request. */
+export function compareRuns(nowPath, beforePath) {
+    const nowSamples = readJtlSamples(nowPath);
+    const beforeSamples = readJtlSamples(beforePath);
+    const nowGroups = byLabel(nowSamples);
+    const beforeByLabel = new Map(byLabel(beforeSamples).map((g) => [g.label, g]));
+    const comparisons = [];
+    // The whole run first: it is the headline, and a change that moves everything is the common case.
+    const everything = compareSamples(overall(nowSamples).elapsed, overall(beforeSamples).elapsed);
+    comparisons.push({
+        label: 'all requests',
+        verdict: everything,
+        failedNow: overall(nowSamples).failed,
+        totalNow: nowSamples.length,
+        codes: [],
+    });
+    for (const group of nowGroups) {
+        const before = beforeByLabel.get(group.label);
+        const verdict = before
+            ? compareSamples(group.elapsed, before.elapsed)
+            : {
+                direction: 'cannot-say',
+                now: median(group.elapsed),
+                before: 0,
+                slowerThanBefore: 0.5,
+                p: null,
+                sampleSize: { now: group.total, before: 0 },
+                why: 'this label is not in the baseline run, so there is nothing to compare it with',
+            };
+        comparisons.push({ label: group.label, verdict, failedNow: group.failed, totalNow: group.total, codes: group.codes });
+    }
+    const slower = comparisons.filter((c) => c.verdict.direction === 'slower');
+    const failing = comparisons.filter((c) => c.failedNow > 0);
+    return { comparisons, slower, failing, requests: nowSamples.length, report: render(comparisons, slower, failing, nowSamples.length, beforeSamples.length) };
+}
+function render(all, slower, failing, now, before) {
+    const lines = ['# Tewip performance check', ''];
+    lines.push(`Compared a JMeter run of **${now} requests** against a baseline of **${before}**, request by request.`, '');
+    if (slower.length > 0)
+        lines.push(`**${slower.length} ${slower.length === 1 ? 'label is' : 'labels are'} slower than the baseline, by more than the runs explain.**`, '');
+    const faster = all.filter((c) => c.verdict.direction === 'faster');
+    if (faster.length > 0)
+        lines.push(`${faster.length} ${faster.length === 1 ? 'is' : 'are'} faster.`, '');
+    if (slower.length === 0 && faster.length === 0)
+        lines.push('**No difference these two runs can establish.**', '');
+    if (failing.length > 0)
+        lines.push(`**${failing.reduce((n, c) => n + c.failedNow, 0)} requests failed**, which is a separate matter from speed and is below.`, '');
+    lines.push('## Is it slower?', '');
+    for (const c of all) {
+        const mark = c.verdict.direction === 'slower' ? '**slower**' : c.verdict.direction === 'faster' ? 'faster' : 'cannot say';
+        lines.push(`- \`${c.label || '(unnamed)'}\` — ${mark}`);
+        lines.push(`  - median ${ms(c.verdict.before)} → ${ms(c.verdict.now)} over ${c.verdict.sampleSize.before} and ${c.verdict.sampleSize.now} requests`);
+        lines.push(`  - ${c.verdict.why}`);
+    }
+    lines.push('');
+    if (failing.length > 0) {
+        lines.push('## Requests that failed', '');
+        for (const c of failing) {
+            const codes = c.codes.map((x) => `${x.code} x${x.times}`).join(', ');
+            const assertion = c.codes.some((x) => /^2\d\d/.test(x.code)) ? ' - a 2xx counted as failed is an assertion failing, not the server' : '';
+            lines.push(`- **${c.label || '(unnamed)'}**: ${c.failedNow} of ${c.totalNow}${codes ? ` (${codes})` : ''}${assertion}`);
+        }
+        lines.push('');
+        lines.push('A run with failures in it is a run whose timings describe something that did not work.', '');
+    }
+    const thin = all.filter((c) => c.verdict.p === null && c.verdict.sampleSize.before > 0);
+    if (thin.length > 0) {
+        lines.push(`${thin.length} ${thin.length === 1 ? 'label has' : 'labels have'} fewer than ${ENOUGH_SAMPLES} requests on one ` +
+            'side, so no test was run for them. That is said above rather than guessed at.', '');
+    }
+    lines.push(...notKnown());
+    return lines.join('\n');
+}

package/dist/engine/perf/jtl.js

@@ -0,0 +1,159 @@
+// Phase 39: reading a JMeter results file.
+//
+// Two things made this worth doing properly.
+//
+// **Tewip could not read one at all.** The importer handled `.jtl` only as XML, and CSV is JMeter's
+// default and what every real repository has - so `importJunit` answered "not an XML document: no
+// root element" on genuine JMeter output while the help text advertised JMeter support.
+//
+// **And a .jtl is the best performance data Tewip has ever been handed.** A k6 summary gives six
+// aggregates per metric and no samples, which is why Phase 30's comparison had to be a crude range
+// check. A .jtl has one row per request: real samples, per label, which support an actual
+// statistical test. The better answer was always sitting in a format that would not open.
+//
+// The parser is a real CSV reader rather than a line split, and that is not fastidiousness. In the
+// first real file read here - NikeshMan16/JmeterPerformance_testing_Project - the quoted
+// failureMessage of a 400 response contains a comma *and a newline*, so one logical record spans
+// seven physical lines. Splitting on newlines would have produced six corrupt rows out of one.
+import { readFileSync } from 'node:fs';
+/**
+ * RFC 4180 with JMeter's habits: quoted fields may contain commas, newlines and doubled quotes.
+ *
+ * Returned as rows of raw strings; nothing is interpreted here, because deciding what a column
+ * means is the caller's job and JMeter's column set is configurable.
+ */
+export function readCsv(text) {
+    const rows = [];
+    let row = [];
+    let field = '';
+    let quoted = false;
+    let i = 0;
+    // A byte order mark would otherwise become part of the first column's name.
+    if (text.charCodeAt(0) === 0xfeff)
+        i = 1;
+    for (; i < text.length; i += 1) {
+        const ch = text[i];
+        if (quoted) {
+            if (ch === '"') {
+                // A doubled quote inside a quoted field is one literal quote.
+                if (text[i + 1] === '"') {
+                    field += '"';
+                    i += 1;
+                }
+                else
+                    quoted = false;
+            }
+            else
+                field += ch;
+            continue;
+        }
+        if (ch === '"') {
+            quoted = true;
+            continue;
+        }
+        if (ch === ',') {
+            row.push(field);
+            field = '';
+            continue;
+        }
+        if (ch === '\r')
+            continue;
+        if (ch === '\n') {
+            row.push(field);
+            field = '';
+            // A blank line is not a record.
+            if (row.length > 1 || row[0] !== '')
+                rows.push(row);
+            row = [];
+            continue;
+        }
+        field += ch;
+    }
+    if (field !== '' || row.length > 0) {
+        row.push(field);
+        rows.push(row);
+    }
+    return rows;
+}
+/** JMeter's column names, which are configurable, so each is looked up rather than counted. */
+function columns(header) {
+    const at = {};
+    header.forEach((name, i) => {
+        at[name.trim()] = i;
+    });
+    return at;
+}
+const CSV_HEADER = /(^|,)\s*timeStamp\s*(,|$)/;
+/** True of a CSV results file, as opposed to the XML form or something else entirely. */
+export function looksLikeCsvJtl(text) {
+    const firstLine = text.slice(0, 2000).split(/\r?\n/)[0] ?? '';
+    return CSV_HEADER.test(firstLine) && firstLine.includes('elapsed');
+}
+export function readJtlSamples(path) {
+    const text = readFileSync(path, 'utf8');
+    if (!looksLikeCsvJtl(text)) {
+        throw new Error(`${path} is not a CSV JMeter results file: Tewip looked for a header row naming timeStamp and elapsed. ` +
+            'JMeter writes one by default; if yours is XML, it is read by `tewip explain` instead.');
+    }
+    const rows = readCsv(text);
+    const at = columns(rows[0] ?? []);
+    const elapsedAt = at.elapsed;
+    const labelAt = at.label;
+    if (elapsedAt === undefined || labelAt === undefined) {
+        throw new Error(`${path} has no elapsed or label column, so there is nothing to measure.`);
+    }
+    const out = [];
+    for (const row of rows.slice(1)) {
+        // A short row is a truncated write - a run killed mid-flush - and inventing values for it would
+        // put a zero into a latency distribution, which is the one place a zero does real damage.
+        if (row.length <= Math.max(elapsedAt, labelAt))
+            continue;
+        const elapsed = Number(row[elapsedAt]);
+        if (!Number.isFinite(elapsed) || elapsed < 0)
+            continue;
+        const success = at.success === undefined ? true : row[at.success]?.trim().toLowerCase() === 'true';
+        out.push({
+            label: (row[labelAt] ?? '').trim(),
+            elapsed,
+            success,
+            responseCode: (at.responseCode === undefined ? '' : (row[at.responseCode] ?? '')).trim(),
+        });
+    }
+    return out;
+}
+/** The samples grouped by what was being requested, busiest first. */
+export function byLabel(samples) {
+    const groups = new Map();
+    const codes = new Map();
+    for (const s of samples) {
+        let group = groups.get(s.label);
+        if (!group) {
+            group = { label: s.label, elapsed: [], failed: 0, total: 0, codes: [] };
+            groups.set(s.label, group);
+            codes.set(s.label, new Map());
+        }
+        group.elapsed.push(s.elapsed);
+        group.total += 1;
+        if (!s.success) {
+            group.failed += 1;
+            const seen = codes.get(s.label);
+            const code = s.responseCode === '' ? 'no response code' : s.responseCode;
+            seen.set(code, (seen.get(code) ?? 0) + 1);
+        }
+    }
+    for (const [label, seen] of codes) {
+        groups.get(label).codes = [...seen].map(([code, times]) => ({ code, times })).sort((a, b) => b.times - a.times);
+    }
+    return [...groups.values()].sort((a, b) => b.total - a.total);
+}
+/** The whole run as one label, for the headline number. */
+export const ALL_REQUESTS = 'all requests';
+export function overall(samples) {
+    return {
+        label: ALL_REQUESTS,
+        elapsed: samples.map((s) => s.elapsed),
+        failed: samples.filter((s) => !s.success).length,
+        total: samples.length,
+        codes: [],
+    };
+}

package/dist/engine/perf/perf-command.js

@@ -21,6 +21,8 @@ import { isLighthouseFile, readLighthouse } from './lighthouse.js';
 import { fromK6, metricsInOrder, show } from './run.js';
 import { compareMetric, ENOUGH_BASELINES } from './compare.js';
 import { baselineFor, fromSameTool, knownLabels, recordRun, storedRuns } from './history.js';
+import { byLabel, readJtlSamples } from './jtl.js';
+import { compareRuns, describeRun } from './jtl-report.js';
 /** Enough requests for a percentile to mean anything. Below this, p95 is three data points. */
 export const ENOUGH_REQUESTS = 100;
 export function readPerfFile(path) {
@@ -120,8 +122,47 @@ export function renderPerf(run, comparisons, label, baselineCount, recorded, man
         lines.push('This run was **not** recorded. Add `--record` on your main branch to make it part of the baseline; a regression recorded by accident becomes next week\'s normal.', '');
     return lines.join('\n');
 }
+/**
+ * A JMeter run. With `--against` the two are compared request by request; without it the run is
+ * described and nothing is claimed about a change, because one run is not a comparison.
+ */
+function runJtl(root, cmd) {
+    const now = resolve(root, cmd.summary);
+    if (!cmd.against) {
+        const samples = readJtlSamples(now);
+        const report = describeRun(byLabel(samples), samples.length);
+        console.log(report);
+        if (cmd.out)
+            writeOut(root, cmd.out, report);
+        return undefined;
+    }
+    const result = compareRuns(now, resolve(root, cmd.against));
+    console.log(result.report);
+    if (cmd.out)
+        writeOut(root, cmd.out, result.report);
+    // Failures are a separate matter from speed and gate on their own: a run whose requests failed
+    // is a run whose timings describe something that did not work.
+    const failed = result.failing.reduce((n, c) => n + c.failedNow, 0);
+    if (cmd.gate && (result.slower.length > 0 || failed > 0)) {
+        const why = [result.slower.length > 0 ? `${result.slower.length} label(s) slower than the runs explain` : '', failed > 0 ? `${failed} request(s) failed` : '']
+            .filter(Boolean)
+            .join(', and ');
+        console.error(`${String.fromCharCode(10)}Performance gate failed: ${why}.`);
+        process.exitCode = 1;
+    }
+    return undefined;
+}
+function writeOut(root, out, report) {
+    const path = resolve(root, out);
+    mkdirSync(dirname(path), { recursive: true });
+    writeFileSync(path, report, 'utf8');
+}
 export function runPerf(cmd) {
     const root = resolve(cmd.root);
+    // A JMeter results file is answered differently, and better: it carries every request rather
+    // than six aggregates, so two runs are compared with a real test instead of a range check.
+    if (/\.jtl$/i.test(cmd.summary))
+        return runJtl(root, cmd);
     const label = cmd.label ?? 'default';
     const { run, manual } = readPerfFile(resolve(root, cmd.summary));
     const all = storedRuns(root, label);

package/dist/engine/perf/samples.js

@@ -0,0 +1,145 @@
+// Phase 39: comparing two sets of response times properly, because a .jtl gives real samples.
+//
+// Phase 30 had to be crude. A k6 summary carries six aggregates per metric and no samples, so the
+// only honest test available was "is this value outside everything the baselines produced, by more
+// than the width of their own spread". That refuses a lot of real regressions, which was the right
+// trade when nothing better was possible.
+//
+// A JMeter .jtl carries one row per request, so something better *is* possible: a Mann-Whitney U
+// test, which asks whether one distribution is shifted from another without assuming either is
+// normal - and response times are never normal. They are a lump near the median with a long tail,
+// which is exactly the shape a t-test handles badly.
+//
+// The decision that matters most here is **not** the p-value. With ten thousand samples a one
+// millisecond shift is overwhelmingly significant and completely irrelevant, and a tool that
+// reports it as a regression has taught its team that regressions are noise. So a change has to be
+// both statistically real and large enough to care about, and the report says which of the two it
+// failed when it fails.
+/** Below this per side the normal approximation is unreliable, so no test is claimed. */
+export const ENOUGH_SAMPLES = 20;
+/** A shift smaller than this is not worth a person's attention, however certain it is. */
+const SMALLEST_SHIFT = 0.05;
+const SMALLEST_SHIFT_MS = 2;
+export function median(values) {
+    if (values.length === 0)
+        return 0;
+    const sorted = [...values].sort((a, b) => a - b);
+    const mid = Math.floor(sorted.length / 2);
+    return sorted.length % 2 === 0 ? (sorted[mid - 1] + sorted[mid]) / 2 : sorted[mid];
+}
+/**
+ * The Mann-Whitney U statistic, with the tie correction.
+ *
+ * Ties matter here rather than being a formality: response times are milliseconds rounded to
+ * integers, so a fast endpoint produces hundreds of identical values and the uncorrected variance
+ * would be wrong in the direction that invents significance.
+ */
+export function mannWhitney(a, b) {
+    const n1 = a.length;
+    const n2 = b.length;
+    if (n1 === 0 || n2 === 0)
+        return null;
+    const all = [...a.map((v) => ({ v, from: 0 })), ...b.map((v) => ({ v, from: 1 }))].sort((x, y) => x.v - y.v);
+    const ranks = new Array(all.length);
+    const tieGroups = [];
+    for (let i = 0; i < all.length;) {
+        let j = i;
+        while (j + 1 < all.length && all[j + 1].v === all[i].v)
+            j += 1;
+        // Everything in a tie shares the average of the ranks they span.
+        const shared = (i + j + 2) / 2;
+        for (let k = i; k <= j; k += 1)
+            ranks[k] = shared;
+        if (j > i)
+            tieGroups.push(j - i + 1);
+        i = j + 1;
+    }
+    let rankSumA = 0;
+    for (const [i, entry] of all.entries())
+        if (entry.from === 0)
+            rankSumA += ranks[i];
+    const u1 = rankSumA - (n1 * (n1 + 1)) / 2;
+    const n = n1 + n2;
+    const mean = (n1 * n2) / 2;
+    const tieTerm = tieGroups.reduce((sum, t) => sum + (t * t * t - t), 0);
+    const variance = ((n1 * n2) / 12) * (n + 1 - tieTerm / (n * (n - 1)));
+    if (variance <= 0)
+        return { u: u1, z: 0, p: 1 };
+    // Continuity-corrected, two-sided.
+    const z = (u1 - mean - Math.sign(u1 - mean) * 0.5) / Math.sqrt(variance);
+    return { u: u1, z, p: 2 * (1 - normalCdf(Math.abs(z))) };
+}
+/** The standard normal CDF, by Abramowitz and Stegun 26.2.17 - accurate to about 1e-7. */
+function normalCdf(x) {
+    const t = 1 / (1 + 0.2316419 * x);
+    const d = 0.3989422804014327 * Math.exp(-(x * x) / 2);
+    const poly = t * (0.319381530 + t * (-0.356563782 + t * (1.781477937 + t * (-1.821255978 + t * 1.330274429))));
+    return 1 - d * poly;
+}
+/**
+ * Whether this run is slower than the baseline, from the samples themselves.
+ *
+ * Both conditions have to hold. A difference that is not statistically real might be the machine;
+ * a difference that is real but tiny is not worth anybody's afternoon, and reporting it is how a
+ * team learns to ignore the report. The verdict says which one was missing.
+ */
+export function compareSamples(now, before) {
+    const base = {
+        now: Math.round(median(now) * 10) / 10,
+        before: Math.round(median(before) * 10) / 10,
+        sampleSize: { now: now.length, before: before.length },
+    };
+    if (now.length < ENOUGH_SAMPLES || before.length < ENOUGH_SAMPLES) {
+        return {
+            ...base,
+            direction: 'cannot-say',
+            slowerThanBefore: 0.5,
+            p: null,
+            why: `${now.length} requests now and ${before.length} before. Below ${ENOUGH_SAMPLES} on either side the test ` +
+                'is not reliable, so Tewip will not run it. Lengthen the run rather than reading the numbers.',
+        };
+    }
+    const test = mannWhitney(now, before);
+    if (test === null)
+        return { ...base, direction: 'cannot-say', slowerThanBefore: 0.5, p: null, why: 'there was nothing to compare' };
+    // U is the count of (now, before) pairs where now ranks higher, so U / (n1*n2) is directly the
+    // probability that a random request now is slower than a random one before. It is the most
+    // readable effect size there is, and it needs no units.
+    const slowerThanBefore = test.u / (now.length * before.length);
+    const shift = base.before > 0 ? (base.now - base.before) / base.before : 0;
+    const bigEnough = Math.abs(shift) >= SMALLEST_SHIFT && Math.abs(base.now - base.before) >= SMALLEST_SHIFT_MS;
+    const real = test.p < 0.01;
+    const percent = Math.round(shift * 100);
+    const chance = Math.round(slowerThanBefore * 100);
+    if (real && bigEnough) {
+        const faster = base.now < base.before;
+        return {
+            ...base,
+            direction: faster ? 'faster' : 'slower',
+            slowerThanBefore,
+            p: test.p,
+            why: `the median moved from ${base.before}ms to ${base.now}ms (${percent > 0 ? '+' : ''}${percent}%), and a random ` +
+                `request now is ${chance}% likely to be ${faster ? 'faster' : 'slower'} than a random one before. ` +
+                `Over ${now.length} and ${before.length} requests that is not the machine (p < 0.01).`,
+        };
+    }
+    if (real) {
+        return {
+            ...base,
+            direction: 'cannot-say',
+            slowerThanBefore,
+            p: test.p,
+            why: `the difference is real but too small to act on: ${base.before}ms to ${base.now}ms, ` +
+                `${percent > 0 ? '+' : ''}${percent}%. With this many samples even a millisecond is statistically certain, ` +
+                'which is why certainty alone is not the test.',
+        };
+    }
+    return {
+        ...base,
+        direction: 'cannot-say',
+        slowerThanBefore,
+        p: test.p,
+        why: `${base.before}ms to ${base.now}ms is within what these two runs could produce by themselves ` +
+            `(p = ${test.p.toFixed(2)}). A difference the runs make on their own is not evidence about this change.`,
+    };
+}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "tewip-ai",
-  "version": "0.46.0",
+  "version": "0.47.0",
   "mcpName": "io.github.hmamut39/tewip",
   "description": "Tewip finds why each failing test failed - real bug, flaky test, environment, test data or changed selector - for any stack, from the command line, CI, or your editor.",
   "keywords": [