sitespeed.io 30.3.0 → 30.4.1

package/.github/workflows/linux.yml

@@ -24,6 +24,11 @@ jobs:
         sudo apt-get update
         sudo apt-get --only-upgrade install google-chrome-stable
         google-chrome --version
+    - name: Install dependencies
+      run: |
+          python -m pip install --upgrade --user pip
+          python -m pip install --user scipy
+          python -m pip show scipy
     - name: Install Firefox
       uses: browser-actions/setup-firefox@latest
       #with:
@@ -69,4 +74,6 @@ jobs:
     - name: Run test with Influx 2.6.1
       run: bin/sitespeed.js http://127.0.0.1:3001/simple/ -n 1 --influxdb.host 127.0.0.1 --influxdb.port 8087 --influxdb.version 2 --influxdb.organisation sitespeed --influxdb.token sitespeed --xvfb
     - name: Run Chrome test with config
-      run: node bin/sitespeed.js --config test/exampleConfig.json http://127.0.0.1:3001/simple/ --xvfb
+      run: node bin/sitespeed.js --config test/exampleConfig.json http://127.0.0.1:3001/simple/ --xvfb
+    - name: Run Chrome test using compare plugin
+      run: node bin/sitespeed.js --compare.id compare --compare.saveBaseline --compare.baselinePath test/ http://127.0.0.1:3001/simple/ --xvfb

package/CHANGELOG.md

@@ -1,10 +1,22 @@
 # CHANGELOG - sitespeed.io  (we use [semantic versioning](https://semver.org))
 
+## 30.4.1 - 2023-11-28
+### Fixed
+* Fix for Firefox when generating the result HTML. It was broken since we where missing CPU data.
+
+## 30.4.0 - 2023-11-27
+### Fixed
+* Upgrade to Browsretime 19.1.0 with a fix for Geckodriver so that the correct ARM version is installed on Mac Arm machines.
+### Added
+* The new compare plugin [PR 4009](https://github.com/sitespeedio/sitespeed.io/pull/4009) makes it easy to use Mann Whitney U/Wilcox for support to find performance egressions. Read all about the plugin in the [documentation](https://www.sitespeed.io/documentation/sitespeed.io/compare/).
+* Firefox 120 in the Docker container [#4010](https://github.com/sitespeedio/sitespeed.io/pull/4010).
+* Button to download the console logs, thank you [bairov pavel](https://github.com/Amerousful) for PR [#4007](https://github.com/sitespeedio/sitespeed.io/pull/4007).
+
 ## 30.3.0 - 2023-11-09
 
 ### Added
 * Upgrade to Browsertime 18.0.0.
-* Added suupport to run user journeys with WebPageReplay [#4005](https://github.com/sitespeedio/sitespeed.io/pull/4005).
+* Added support to run user journeys with WebPageReplay [#4005](https://github.com/sitespeedio/sitespeed.io/pull/4005).
 
 ### Fixed
 * Downgrade puppeteer in the +1 container for Lighthouse, thank you [bairov pavel](https://github.com/Amerousful) for PR [#123](https://github.com/sitespeedio/plugin-lighthouse/pull/123).

package/CONTRIBUTORS.md

@@ -45,3 +45,4 @@ Many many many thanks to:
 * [Devrim Tufan](https://github.com/tufandevrim)
 * [Keith Cirkel](https://github.com/keithamus)
 * [Jonathan Lee](https://github.com/beenanner)
+* [Pavel Bairov](https://github.com/Amerousful)

package/Dockerfile

@@ -1,4 +1,4 @@
-FROM sitespeedio/webbrowsers:chrome-119.0-firefox-119.0-edge-119.0
+FROM sitespeedio/webbrowsers:chrome-119.0-firefox-120.0-edge-119.0
 
 ARG TARGETPLATFORM=linux/amd64
 
@@ -44,4 +44,6 @@ RUN echo 'ALL ALL=NOPASSWD: /usr/sbin/tc, /usr/sbin/route, /usr/sbin/ip' > /etc/
 
 ENTRYPOINT ["/start.sh"]
 VOLUME /sitespeed.io
+VOLUME /baseline
+
 WORKDIR /sitespeed.io

package/lib/cli/cli.js

@@ -1870,6 +1870,68 @@ export async function parseCommandLine() {
       group: 'API'
     });
 
+  parsed
+    .option('compare.id', {
+      type: 'string',
+      describe:
+        'The id of the test. Will be used to find the baseline test, that is using the id as a part of the name.',
+      group: 'compare'
+    })
+    .option('compare.baselinePath', {
+      type: 'string',
+      describe:
+        'Specifies the path to the baseline data file. This file is used as a reference for comparison against the current test data.',
+      group: 'compare'
+    })
+    .option('compare.saveBaseline', {
+      type: 'boolean',
+      default: false,
+      describe:
+        'Determines whether to save the current test data as the new baseline. Set to true to save the current data as baseline for future comparisons.',
+      group: 'compare'
+    })
+    .option('compare.testType', {
+      describe:
+        'Selects the statistical test type to be used for comparison. Options are mannwhitneyu for the Mann-Whitney U test and wilcoxon for the Wilcoxon signed-rank test.',
+      choices: ['mannwhitneyu', ' wilcoxon'],
+      default: 'mannwhitneyu',
+      group: 'compare'
+    })
+    .option('compare.alternative', {
+      choices: ['less', ' greater', 'two-sided'],
+      default: 'less',
+      describe:
+        'Specifies the alternative hypothesis to be tested. Options are less for one-sided test where the first group is expected to be less than the second, greater for one-sided test with the first group expected to be greater, or two-sided for a two-sided test.',
+      group: 'compare'
+    })
+    .option('compare.wilcoxon.correction', {
+      type: 'boolean',
+      describe:
+        'Enables or disables the continuity correction in the Wilcoxon signed-rank test. Set to true to enable the correction.',
+      default: false,
+      group: 'compare'
+    })
+    .option('compare.wilcoxon.zeroMethod', {
+      choices: ['wilcox', ' pratt', 'zsplit'],
+      describe:
+        'Specifies the method for handling zero differences in the Wilcoxon test. wilcox discards all zero-difference pairs, pratt includes all, and zsplit splits them evenly among positive and negative ranks.',
+      default: 'zsplit',
+      group: 'compare'
+    })
+    .option('compare.mannwhitneyu.useContinuity', {
+      type: 'boolean',
+      default: false,
+      describe:
+        'Determines whether to use continuity correction in the Mann-Whitney U test. Set to true to apply the correction.',
+      group: 'compare'
+    })
+    .option('compare.mannwhitneyu.method', {
+      choices: ['auto', ' exact', 'symptotic'],
+      escribe:
+        'Selects the method for calculating the Mann-Whitney U test. auto automatically selects between exact and asymptotic based on sample size, exact uses the exact distribution of U, and symptotic uses a normal approximation.',
+      default: 'auto',
+      group: 'compare'
+    });
   parsed
     .option('mobile', {
       describe:

package/lib/plugins/compare/baseline.js

@@ -0,0 +1,35 @@
+import fs from 'node:fs/promises';
+import { join, resolve } from 'node:path';
+
+export async function getBaseline(id, compareOptions) {
+  try {
+    return JSON.parse(
+      await fs.readFile(
+        resolve(
+          join(compareOptions.baselinePath || process.cwd(), `${id}.json`)
+        )
+      )
+    );
+  } catch {
+    return;
+  }
+}
+/*
+async function getBaselineFromInternet(url) {
+  try {
+    const response = await fetch(url);
+    return response.json();
+  } catch (error) {
+    log.error('Could not fetch', error);
+  }
+}
+
+async function getBaselineFromFile(path) {}
+
+/*
+export async function saveBaseline(json, options) {
+
+}*/
+export async function saveBaseline(json, name) {
+  return fs.writeFile(resolve(name), JSON.stringify(json));
+}

package/lib/plugins/compare/helper.js

@@ -0,0 +1,347 @@
+import { fileURLToPath } from 'node:url';
+import { join } from 'node:path';
+import path from 'node:path';
+
+import { execa } from 'execa';
+import { Stats } from 'fast-stats';
+import intel from 'intel';
+import { decimals } from '../../support/helpers/index.js';
+const log = intel.getLogger('sitespeedio.plugin.compare');
+
+const __dirname = path.dirname(fileURLToPath(import.meta.url));
+
+class Metric {
+  constructor(name, values) {
+    this.name = name;
+    this.stats = new Stats().push(values);
+  }
+
+  getName() {
+    return this.name;
+  }
+  getValues() {
+    return this.stats.data;
+  }
+  getStats() {
+    return this.stats;
+  }
+}
+
+export async function runStatisticalTests(data) {
+  let extras = '';
+  try {
+    const { stdout } = await execa(
+      process.env.PYTHON || 'python',
+      [join(__dirname, 'statistical.py')],
+      {
+        input: JSON.stringify(data)
+      }
+    );
+    extras = stdout;
+    const results = JSON.parse(stdout);
+    log.verbose('Result from the python script %j'.results);
+    return results;
+  } catch (error) {
+    log.error(error);
+    log.error(extras);
+  }
+}
+
+export function getStatistics(arrayOfValues) {
+  return new Stats().push(arrayOfValues);
+}
+
+function getExtras(data) {
+  const metrics = {};
+  const results = {};
+
+  for (const run of data.extras) {
+    for (const name of Object.keys(run)) {
+      if (!metrics[name]) {
+        metrics[name] = [];
+      }
+      metrics[name].push(run[name]);
+    }
+  }
+
+  for (const [metricName, values] of Object.entries(metrics)) {
+    results[metricName] = new Metric(metricName, values);
+  }
+
+  return results;
+}
+
+function getTimings(data) {
+  const timingMetrics = {
+    ttfb: [],
+    loadEventEnd: [],
+    firstContentfulPaint: [],
+    fullyLoaded: []
+  };
+
+  for (const run of data.browserScripts) {
+    timingMetrics['ttfb'].push(run.timings.ttfb);
+    timingMetrics['loadEventEnd'].push(run.timings.loadEventEnd);
+    timingMetrics['firstContentfulPaint'].push(
+      run.timings.paintTiming['first-contentful-paint']
+    );
+  }
+
+  for (const run of data.fullyLoaded) {
+    timingMetrics['fullyLoaded'].push(run);
+  }
+
+  const results = {};
+  for (const [metricName, values] of Object.entries(timingMetrics)) {
+    if (!results.timings) {
+      results.timings = {};
+    }
+    results.timings[metricName] = new Metric(`${metricName}`, values);
+  }
+  return results;
+}
+
+function getUserTimings(data) {
+  const userTimingMetrics = {};
+  for (const run of data.browserScripts) {
+    if (run.timings.userTimings) {
+      const { marks, measures } = run.timings.userTimings;
+
+      for (const mark of marks) {
+        if (!userTimingMetrics[mark.name]) {
+          userTimingMetrics[mark.name] = [];
+        }
+        userTimingMetrics[mark.name].push(decimals(mark.startTime));
+      }
+
+      for (const measure of measures) {
+        if (!userTimingMetrics[measure.name]) {
+          userTimingMetrics[measure.name] = [];
+        }
+        userTimingMetrics[measure.name].push(decimals(measure.startTime));
+      }
+    }
+  }
+
+  const results = {};
+  for (const [metricName, values] of Object.entries(userTimingMetrics)) {
+    if (!results.userTimings) {
+      results.userTimings = {};
+    }
+    results.userTimings[metricName] = new Metric(`${metricName}`, values);
+  }
+  return results;
+}
+
+function getElementTimings(data) {
+  const elementTimingMetrics = {};
+  for (const run of data.browserScripts) {
+    if (run.timings.elementTimings) {
+      for (const [name, timing] of Object.entries(run.timings.elementTimings)) {
+        if (!elementTimingMetrics[name]) {
+          elementTimingMetrics[name] = [];
+        }
+        elementTimingMetrics[name].push(timing.renderTime);
+      }
+    }
+  }
+
+  const results = {};
+  for (const [metricName, values] of Object.entries(elementTimingMetrics)) {
+    if (!results.elementTimings) {
+      results.elementTimings = {};
+    }
+    results.elementTimings[metricName] = new Metric(`${metricName}`, values);
+  }
+  return results;
+}
+
+function getGoogleWebVitals(data) {
+  const googleWebVitalsMetrics = {};
+  for (const run of data.googleWebVitals) {
+    for (const [name, value] of Object.entries(run)) {
+      if (!googleWebVitalsMetrics[name]) {
+        googleWebVitalsMetrics[name] = [];
+      }
+      googleWebVitalsMetrics[name].push(value);
+    }
+  }
+
+  const results = {};
+  for (const [metricName, values] of Object.entries(googleWebVitalsMetrics)) {
+    if (!results.googleWebVitals) {
+      results.googleWebVitals = {};
+    }
+    results.googleWebVitals[metricName] = new Metric(`${metricName}`, values);
+  }
+  return results;
+}
+
+function getVisualMetrics(data) {
+  const DO_NOT_USE = new Set([
+    'VisualProgress',
+    'videoRecordingStart',
+    'VisualComplete85',
+    'VisualComplete95',
+    'VisualComplete99'
+  ]);
+
+  const visualMetrics = {};
+  for (const run of data.visualMetrics) {
+    for (const [name, value] of Object.entries(run)) {
+      if (!DO_NOT_USE.has(name)) {
+        if (!visualMetrics[name]) {
+          visualMetrics[name] = [];
+        }
+        visualMetrics[name].push(value);
+      }
+    }
+  }
+
+  const results = {};
+  for (const [metricName, values] of Object.entries(visualMetrics)) {
+    if (!results.visualMetrics) {
+      results.visualMetrics = {};
+    }
+    results.visualMetrics[metricName] = new Metric(`${metricName}`, values);
+  }
+  return results;
+}
+
+function getCDPPerformance(data) {
+  const metricsToKeep = new Set([
+    'JSEventListeners',
+    'LayoutCount',
+    'RecalcStyleCount',
+    'LayoutDuration',
+    'RecalcStyleDuration',
+    'ScriptDuration',
+    'V8CompileDuration',
+    'TaskDuration',
+    'TaskOtherDuration',
+    'JSHeapUsedSize'
+  ]);
+  const cdpPerformance = {};
+  for (const run of data.cdp.performance) {
+    for (const name of Object.keys(run)) {
+      if (metricsToKeep.has(name)) {
+        if (!cdpPerformance[name]) {
+          cdpPerformance[name] = [];
+        }
+        cdpPerformance[name].push(decimals(run[name]));
+      }
+    }
+  }
+
+  // Convert to Metric objects
+  const results = {};
+  for (const [metricName, values] of Object.entries(cdpPerformance)) {
+    if (!results.cdp) {
+      results.cdp = {};
+    }
+    results.cdp[metricName] = new Metric(`${metricName}`, values);
+  }
+  return results;
+}
+
+function getCPU(data) {
+  const cpuMetrics = {
+    tasks: [],
+    totalDuration: [],
+    lastLongTask: [],
+    beforeFirstContentfulPaint: [],
+    beforeLargestContentfulPaint: []
+  };
+
+  for (const run of data.cpu) {
+    const longTasks = run.longTasks;
+    cpuMetrics['tasks'].push(longTasks['tasks']);
+    cpuMetrics['totalDuration'].push(longTasks['totalDuration']);
+    cpuMetrics['lastLongTask'].push(longTasks['lastLongTask']);
+    cpuMetrics['beforeFirstContentfulPaint'].push(
+      longTasks['beforeFirstContentfulPaint'].totalDuration
+    );
+    cpuMetrics['beforeLargestContentfulPaint'].push(
+      longTasks['beforeLargestContentfulPaint'].totalDuration
+    );
+  }
+
+  const isEmpty = Object.values(cpuMetrics).every(arr => arr.length === 0);
+  if (isEmpty) {
+    return {}; // Return an empty object if no data
+  }
+
+  const results = {};
+  for (const [metricName, values] of Object.entries(cpuMetrics)) {
+    if (!results.cpu) {
+      results.cpu = {};
+    }
+    results.cpu[metricName] = new Metric(`${metricName}`, values);
+  }
+  return results;
+}
+
+function getRenderBlocking(data) {
+  const renderBlockingMetrics = {
+    beforeFCPms: [],
+    beforeLCPms: [],
+    beforeFCPelements: [],
+    beforeLCPelements: []
+  };
+
+  for (const run of data.renderBlocking) {
+    renderBlockingMetrics['beforeFCPms'].push(
+      run.recalculateStyle.beforeFCP.durationInMillis
+    );
+    renderBlockingMetrics['beforeFCPelements'].push(
+      run.recalculateStyle.beforeFCP.elements
+    );
+    renderBlockingMetrics['beforeLCPms'].push(
+      run.recalculateStyle.beforeLCP.durationInMillis
+    );
+    renderBlockingMetrics['beforeLCPelements'].push(
+      run.recalculateStyle.beforeLCP.elements
+    );
+  }
+
+  // Check if all arrays in renderBlockingMetrics are empty
+  const isEmpty = Object.values(renderBlockingMetrics).every(
+    arr => arr.length === 0
+  );
+  if (isEmpty) {
+    return {}; // Return an empty object if no data
+  }
+
+  const results = {};
+  for (const [metricName, values] of Object.entries(renderBlockingMetrics)) {
+    if (!results.renderBlocking) {
+      results.renderBlocking = {};
+    }
+    results.renderBlocking[metricName] = new Metric(`${metricName}`, values);
+  }
+
+  return results;
+}
+
+export function getMetrics(data) {
+  const userTimings = getUserTimings(data);
+  const elementTimings = getElementTimings(data);
+  const visualMetrics = getVisualMetrics(data);
+  const rb = getRenderBlocking(data);
+  const gWV = getGoogleWebVitals(data);
+  const cdp = getCDPPerformance(data);
+  const cpu = getCPU(data);
+  const timings = getTimings(data);
+  const extras = getExtras(data);
+  return {
+    ...extras,
+    ...timings,
+    ...cpu,
+    ...cdp,
+    ...visualMetrics,
+    ...gWV,
+    ...rb,
+    ...elementTimings,
+    ...userTimings
+  };
+}