sitespeed.io 31.2.0 → 32.0.0

package/CHANGELOG.md

@@ -1,5 +1,14 @@
 # CHANGELOG - sitespeed.io  (we use [semantic versioning](https://semver.org))
 
+## 32.0.0 - 2024-01-19
+### Breaking change
+* If you use the compare plugin (`--compare.saveBaseline`) you need to remove your old baseline when you upgrade. The reason is that the original implementation was very narrow to some Browsertime metrics. The new version will be able to show more information (more metrics) between the baseline and the current test. This version allo disables comparing the CDP performance metrics, making the compare page easier to understand.
+
+
+## 31.2.1 - 2024-01-18
+### Fixed
+* Another fix for `--budget.suppressExitCode` hopefully it works better this time [#4065](https://github.com/sitespeedio/sitespeed.io/pull/4065).
+
 ## 31.2.0 - 2024-01-17
 ### Fixed
 * There was bug that caused budget.suppressExitCode to fail that I introduced and reported in [#4062](https://github.com/sitespeedio/sitespeed.io/issues/4062). It's fixed in [#4063](https://github.com/sitespeedio/sitespeed.io/pull/4063).

package/bin/sitespeed.js

@@ -163,10 +163,7 @@ async function start() {
         process.exitCode = 1;
       }
       // If you supress the exit code using budgets, we will always return 0
-      else if (
-        parsed.options.budget &&
-        parsed.options.budget.suppressExitCode
-      ) {
+      if (parsed.options.budget && parsed.options.budget.suppressExitCode) {
         process.exitCode = 0;
       }
 

package/lib/plugins/compare/helper.js

@@ -226,6 +226,7 @@ function getVisualMetrics(data) {
   return results;
 }
 
+/*
 function getCDPPerformance(data) {
   const metricsToKeep = new Set([
     'JSEventListeners',
@@ -261,6 +262,7 @@ function getCDPPerformance(data) {
   }
   return results;
 }
+*/
 
 function getCPU(data) {
   const cpuMetrics = {
@@ -351,8 +353,8 @@ export function getMetrics(data) {
     ...getElementTimings(data),
     ...getUserTimings(data),
     ...getCPU(data),
-    ...getBrowserMetrics(data),
-    ...getCDPPerformance(data)
+    ...getBrowserMetrics(data)
+    //  ...getCDPPerformance(data)
   };
 }
 

package/lib/plugins/compare/index.js

@@ -58,6 +58,8 @@ export default class ComparePlugin extends SitespeedioPlugin {
   }
 
   async open(context, options) {
+    this.pageXrays = {};
+    this.browsertimes = {};
     this.page = 0;
     this.make = context.messageMaker('compare').make;
     this.compareOptions = merge({}, defaultConfig, options.compare);
@@ -93,163 +95,199 @@ export default class ComparePlugin extends SitespeedioPlugin {
         break;
       }
       case 'browsertime.pageSummary': {
-        this.page++;
-        const id = this.options.compare.id || urlToId(message.data.info.url);
-        const baseline = await getBaseline(
-          id + '-' + this.page,
-          this.compareOptions
-        );
-        if (this.options.compare.id) {
-          log.info('Using id %s for page baseline', id);
-        } else {
-          log.info('Using auto generated id for the baseline: %s ', id);
-        }
+        this.browsertimes[message.url] = message;
+        break;
+      }
+      case 'sitespeedio.summarize': {
+        for (let url of Object.keys(this.browsertimes)) {
+          this.page++;
+          const id = this.options.compare.id || urlToId(url);
+          const baseline = await getBaseline(
+            id + '-' + this.page,
+            this.compareOptions
+          );
+          if (this.options.compare.id) {
+            log.info('Using id %s for page baseline', id);
+          } else {
+            log.info('Using auto generated id for the baseline: %s ', id);
+          }
 
-        if (baseline) {
-          if (
-            baseline &&
-            this.options.browsertime.iterations !== baseline.timestamps.length
-          )
-            log.warning(
-              'The baseline test has %s runs and you current have %s. You should make sure you test the same amount of runs',
-              baseline.timestamps.length,
-              this.options.browsertime.iterations
-            );
-          log.info('Got a baseline:' + id + '-' + this.page);
-          const newMetrics = getMetrics(message.data);
-          const baselineMetrics = getMetrics(baseline);
-          const metricsInputData = {
-            options: {
-              test_type: this.compareOptions.testType,
-              alternative: this.compareOptions.alternative
-            },
-            metrics: {}
-          };
+          if (baseline) {
+            if (
+              this.options.browsertime.iterations !==
+              baseline.browsertime.timestamps.length
+            )
+              log.warning(
+                'The baseline test has %s runs and you current have %s. You should make sure you test the same amount of runs',
+                baseline.timestamps.length,
+                this.options.browsertime.iterations
+              );
+            log.info('Got a baseline:' + id + '-' + this.page);
+            const newMetrics = getMetrics(this.browsertimes[url].data);
+            const baselineMetrics = getMetrics(baseline.browsertime);
+            const metricsInputData = {
+              options: {
+                test_type: this.compareOptions.testType,
+                alternative: this.compareOptions.alternative
+              },
+              metrics: {}
+            };
 
-          if (this.compareOptions.testType === 'mannwhitneyu') {
-            metricsInputData.options.use_continuity =
-              this.compareOptions.mannwhitneyu.useContinuity;
-            metricsInputData.options.method =
-              this.compareOptions.mannwhitneyu.method;
-            metricsInputData.options.nan_policy = 'omit';
-          } else if (this.compareOptions.testType === 'wilcoxon') {
-            metricsInputData.options.correction =
-              this.compareOptions.wilcoxon.correction;
-            metricsInputData.options.zero_method =
-              this.compareOptions.wilcoxon.zeroMethod;
-          }
+            if (this.compareOptions.testType === 'mannwhitneyu') {
+              metricsInputData.options.use_continuity =
+                this.compareOptions.mannwhitneyu.useContinuity;
+              metricsInputData.options.method =
+                this.compareOptions.mannwhitneyu.method;
+              metricsInputData.options.nan_policy = 'omit';
+            } else if (this.compareOptions.testType === 'wilcoxon') {
+              metricsInputData.options.correction =
+                this.compareOptions.wilcoxon.correction;
+              metricsInputData.options.zero_method =
+                this.compareOptions.wilcoxon.zeroMethod;
+            }
 
-          for (let group in newMetrics) {
-            if (baselineMetrics[group]) {
-              metricsInputData.metrics[group] = {};
-              for (let metricName in newMetrics[group]) {
-                // Ensure both current and baseline metrics are available
-                if (
-                  baselineMetrics[group][metricName] &&
-                  newMetrics[group][metricName]
-                ) {
-                  // Directly access the Metric instance
-                  const currentMetric = newMetrics[group][metricName];
-                  const baselineMetric = baselineMetrics[group][metricName];
+            for (let group in newMetrics) {
+              if (baselineMetrics[group]) {
+                metricsInputData.metrics[group] = {};
+                for (let metricName in newMetrics[group]) {
+                  // Ensure both current and baseline metrics are available
+                  if (
+                    baselineMetrics[group][metricName] &&
+                    newMetrics[group][metricName]
+                  ) {
+                    // Directly access the Metric instance
+                    const currentMetric = newMetrics[group][metricName];
+                    const baselineMetric = baselineMetrics[group][metricName];
 
-                  // Ensure these are indeed Metric instances
-                  const currentStats = getStatistics(currentMetric.getValues());
-                  const baselineStats = getStatistics(
-                    baselineMetric.getValues()
-                  );
-                  metricsInputData.metrics[group][metricName] = {
-                    baseline: baselineStats.data,
-                    current: currentStats.data
-                  };
-                } else {
-                  log.info(
-                    `Skipping ${group}.${metricName} as it's not present in both current and baseline metrics.`
-                  );
+                    // Ensure these are indeed Metric instances
+                    const currentStats = getStatistics(
+                      currentMetric.getValues()
+                    );
+                    const baselineStats = getStatistics(
+                      baselineMetric.getValues()
+                    );
+                    metricsInputData.metrics[group][metricName] = {
+                      baseline: baselineStats.data,
+                      current: currentStats.data
+                    };
+                  } else {
+                    log.info(
+                      `Skipping ${group}.${metricName} as it's not present in both current and baseline metrics.`
+                    );
+                  }
                 }
               }
             }
-          }
 
-          const results = await runStatisticalTests(metricsInputData);
-          const finalResult = {};
-          for (let group in results) {
-            finalResult[group] = {};
-            for (let metricName in results[group]) {
-              const result = results[group][metricName];
-              // Again, accessing the metricName within the group
-              const currentStats = getStatistics(
-                newMetrics[group][metricName].getValues()
-              );
-              const baselineStats = getStatistics(
-                baselineMetrics[group][metricName].getValues()
-              );
+            const results = await runStatisticalTests(metricsInputData);
+            const finalResult = {};
+            for (let group in results) {
+              finalResult[group] = {};
+              for (let metricName in results[group]) {
+                const result = results[group][metricName];
+                // Again, accessing the metricName within the group
+                const currentStats = getStatistics(
+                  newMetrics[group][metricName].getValues()
+                );
+                const baselineStats = getStatistics(
+                  baselineMetrics[group][metricName].getValues()
+                );
 
-              const cliffs = cliffsDelta(currentStats.data, baselineStats.data);
-              finalResult[group][metricName] = {
-                current: {
-                  stdev: currentStats.stddev(),
-                  mean: currentStats.amean(),
-                  median: currentStats.median(),
-                  values: currentStats.data
-                },
-                baseline: {
-                  stdev: baselineStats.stddev(),
-                  mean: baselineStats.amean(),
-                  median: baselineStats.median(),
-                  values: baselineStats.data
-                },
-                statisticalTestU: result['p-value'],
-                cliffsDelta: cliffs,
-                isSignificant: getIsSignificant(result['p-value'], cliffs)
-              };
+                const cliffs = cliffsDelta(
+                  currentStats.data,
+                  baselineStats.data
+                );
+                finalResult[group][metricName] = {
+                  current: {
+                    stdev: currentStats.stddev(),
+                    mean: currentStats.amean(),
+                    median: currentStats.median(),
+                    values: currentStats.data
+                  },
+                  baseline: {
+                    stdev: baselineStats.stddev(),
+                    mean: baselineStats.amean(),
+                    median: baselineStats.median(),
+                    values: baselineStats.data
+                  },
+                  statisticalTestU: result['p-value'],
+                  cliffsDelta: cliffs,
+                  isSignificant: getIsSignificant(result['p-value'], cliffs)
+                };
+              }
             }
-          }
-          const meta = {
-            baseline: {
-              timestamp: dayjs(baseline.info.timestamp).format(TIME_FORMAT),
-              url: baseline.info.url,
-              alias: baseline.info.alias
-            },
-            current: {
-              timestamp: dayjs(message.data.info.timestamp).format(TIME_FORMAT),
-              url: message.data.info.url,
-              alias: message.data.info.alias
-            },
-            testOptions: this.compareOptions,
-            iterations: this.options.browsertime.iterations
-          };
 
-          if (this.compareOptions.saveBaseline) {
-            await saveBaseline(
-              message.data,
-              join(
-                this.compareOptions.baselinePath || process.cwd(),
-                `${id}-${this.page}.json`
-              )
-            );
-          }
+            const meta = {
+              baseline: {
+                timestamp: dayjs(baseline.browsertime.info.timestamp).format(
+                  TIME_FORMAT
+                ),
+                url: baseline.browsertime.info.url,
+                alias: baseline.browsertime.info.alias
+              },
+              current: {
+                timestamp: dayjs(
+                  this.browsertimes[url].data.info.timestamp
+                ).format(TIME_FORMAT),
+                url: url,
+                alias: this.browsertimes[url].data.info.alias
+              },
+              testOptions: this.compareOptions,
+              iterations: this.options.browsertime.iterations
+            };
+
+            const raw = {
+              baseline: {
+                pagexray: baseline.pagexray,
+                browsertime: baseline.browsertime
+              },
+              current: {
+                pagexray: this.pageXrays[url].data,
+                browsertime: this.browsertimes[url].data
+              }
+            };
 
-          super.sendMessage(
-            'compare.pageSummary',
-            { metrics: finalResult, meta },
-            {
-              url: message.url,
-              group: message.group,
-              runTime: message.runTime
+            if (this.compareOptions.saveBaseline) {
+              await saveBaseline(
+                {
+                  browsertime: this.browsertimes[url].data,
+                  pagexray: this.pageXrays[url].data
+                },
+                join(
+                  this.compareOptions.baselinePath || process.cwd(),
+                  `${id}-${this.page}.json`
+                )
+              );
             }
-          );
-        } else {
-          if (this.compareOptions.saveBaseline) {
-            await saveBaseline(
-              message.data,
-              join(
-                this.compareOptions.baselinePath || process.cwd(),
-                `${id}-${this.page}.json`
-              )
+
+            super.sendMessage(
+              'compare.pageSummary',
+              { metrics: finalResult, meta, raw },
+              {
+                url: url,
+                group: this.browsertimes[url].group,
+                runTime: this.browsertimes[url].runTime
+              }
             );
+          } else {
+            if (this.compareOptions.saveBaseline) {
+              await saveBaseline(
+                {
+                  browsertime: this.browsertimes[url].data,
+                  pagexray: this.pageXrays[url].data
+                },
+                join(
+                  this.compareOptions.baselinePath || process.cwd(),
+                  `${id}-${this.page}.json`
+                )
+              );
+            }
           }
         }
-
+        break;
+      }
+      case 'pagexray.pageSummary': {
+        this.pageXrays[message.url] = message;
         break;
       }
     }

package/lib/plugins/compare/pug/index.pug

@@ -38,19 +38,85 @@ p
     | . For this test, a correction parameter (#{compare.meta.testOptions.wilcoxon.correction ? 'enabled' : 'disabled'}) was applied to adjust for small sample sizes.
 
 
-p 
-  | The baseline test 
-  if compare.meta.baseline.alias
-    a(href=compare.meta.baseline.url) #{compare.meta.baseline.alias}
-  else 
-    a(href=compare.meta.baseline.url) #{compare.meta.baseline.url}
-  |  was conducted at #{compare.meta.baseline.timestamp} and the current test 
-  if compare.meta.current.alias
-    a(href=compare.meta.current.url) #{compare.meta.current.alias}
-  else 
-    a(href=compare.meta.current.url) #{compare.meta.current.url}
-  |  was done at #{compare.meta.current.timestamp}.
+h2 Setup
+table 
+  tr
+    th 
+      b Metric
+    th 
+      b baseline 
+    th 
+      b current
+  tr 
+    td Test
+    td 
+      if compare.meta.baseline.alias
+        a(href=compare.meta.baseline.url) #{compare.meta.baseline.alias}
+      else 
+        a(href=compare.meta.baseline.url) #{compare.meta.baseline.url}
+    td
+      if compare.meta.current.alias
+        a(href=compare.meta.current.url) #{compare.meta.current.alias}
+      else 
+        a(href=compare.meta.current.url) #{compare.meta.current.url}
+  tr
+    td Run time
+    td #{compare.meta.baseline.timestamp} 
+    td #{compare.meta.current.timestamp} 
+  tr 
+    td Total
+    td #{compare.raw.baseline.pagexray.statistics.requests.median} (#{h.size.format(compare.raw.baseline.pagexray.statistics.transferSize.median)} / #{h.size.format(compare.raw.baseline.pagexray.statistics.contentSize.median)})
+    td #{compare.raw.current.pagexray.statistics.requests.median} (#{h.size.format(compare.raw.current.pagexray.statistics.transferSize.median)} / #{h.size.format(compare.raw.current.pagexray.statistics.contentSize.median)})
+  tr 
+    td HTML 
+    td #{compare.raw.baseline.pagexray.statistics.contentTypes.html.requests.median} (#{h.size.format(compare.raw.baseline.pagexray.statistics.contentTypes.html.transferSize.median)} / #{h.size.format(compare.raw.baseline.pagexray.statistics.contentTypes.html.contentSize.median)}) 
+    td #{compare.raw.current.pagexray.statistics.contentTypes.html.requests.median} (#{h.size.format(compare.raw.current.pagexray.statistics.contentTypes.html.transferSize.median)} / #{h.size.format(compare.raw.current.pagexray.statistics.contentTypes.html.contentSize.median)})
+  tr 
+    td JavaScript 
+    td #{compare.raw.baseline.pagexray.statistics.contentTypes.javascript.requests.median} (#{h.size.format(compare.raw.baseline.pagexray.statistics.contentTypes.javascript.transferSize.median)} / #{h.size.format(compare.raw.baseline.pagexray.statistics.contentTypes.javascript.contentSize.median)}) 
+    td #{compare.raw.current.pagexray.statistics.contentTypes.javascript.requests.median} (#{h.size.format(compare.raw.current.pagexray.statistics.contentTypes.javascript.transferSize.median)} / #{h.size.format(compare.raw.current.pagexray.statistics.contentTypes.javascript.contentSize.median)})
+  tr 
+    td CSS requests
+    td #{compare.raw.baseline.pagexray.statistics.contentTypes.css.requests.median} (#{h.size.format(compare.raw.baseline.pagexray.statistics.contentTypes.css.transferSize.median)} / #{h.size.format(compare.raw.baseline.pagexray.statistics.contentTypes.css.contentSize.median)})
+    td #{compare.raw.current.pagexray.statistics.contentTypes.css.requests.median} (#{h.size.format(compare.raw.current.pagexray.statistics.contentTypes.css.transferSize.median)} / #{h.size.format(compare.raw.current.pagexray.statistics.contentTypes.css.contentSize.median)})
+  tr 
+    td Image requests 
+    td #{compare.raw.baseline.pagexray.statistics.contentTypes.image.requests.median} (#{h.size.format(compare.raw.baseline.pagexray.statistics.contentTypes.image.transferSize.median)})
+    td #{compare.raw.current.pagexray.statistics.contentTypes.image.requests.median} (#{h.size.format(compare.raw.current.pagexray.statistics.contentTypes.image.transferSize.median)})
+  tr 
+    td DOM Elements
+    td #{compare.raw.baseline.browsertime.statistics.pageinfo.domElements.median}
+    td #{compare.raw.current.browsertime.statistics.pageinfo.domElements.median}
+
+  if compare.raw.current.pagexray.meta.screenshot
+    tr 
+      td Screenshot 
+      td 
+        a(href=compare.raw.baseline.pagexray.meta.screenshot)
+          img.screenshot(src=compare.raw.baseline.pagexray.meta.screenshot , width=200)
+      td 
+        a(href=compare.raw.current.pagexray.meta.screenshot)
+          img.screenshot(src=compare.raw.current.pagexray.meta.screenshot ,  width=200)
 
+  if compare.raw.current.pagexray.meta.video
+    tr 
+      td Video
+      td 
+        .videoWrapper
+          video(controls, preload='none', width=200)
+            source(src=compare.raw.baseline.pagexray.meta.video type='video/mp4')
+      td 
+        .videoWrapper
+          video(controls, preload='none', width=200)
+            source(src=compare.raw.current.pagexray.meta.video type='video/mp4')
+  
+  if compare.raw.baseline.pagexray.meta.result
+    tr 
+      td Result
+      td
+        a(href=compare.raw.baseline.pagexray.meta.result) Result
+      td 
+        a(href=compare.raw.current.pagexray.meta.result) Result
 
 h2 Comparison Data
 
@@ -74,7 +140,7 @@ table
             td
               a(href=createGraphLink(groupName, metricName)) 
                 b #{groupName + '.' + metricName}
-            if values.statisticalTestU === "N/A" || values.statisticalTestU === "Datasets are identical"
+            if values.statisticalTestU === "N/A" || values.statisticalTestU === "Datasets are identical" || values.statisticalTestU === "No variability"
               td #{values.statisticalTestU}
             else
               td #{h.decimals(values.statisticalTestU)}

package/npm-shrinkwrap.json

@@ -1,12 +1,12 @@
 {
   "name": "sitespeed.io",
-  "version": "31.2.0",
+  "version": "32.0.0",
   "lockfileVersion": 2,
   "requires": true,
   "packages": {
     "": {
       "name": "sitespeed.io",
-      "version": "31.2.0",
+      "version": "32.0.0",
       "license": "MIT",
       "dependencies": {
         "@google-cloud/storage": "6.9.5",

package/package.json

@@ -5,7 +5,7 @@
     "sitespeed.io": "./bin/sitespeed.js",
     "sitespeed.io-wpr": "./bin/browsertimeWebPageReplay.js"
   },
-  "version": "31.2.0",
+  "version": "32.0.0",
   "description": "sitespeed.io is an open-source tool for comprehensive web performance analysis, enabling you to test, monitor, and optimize your website’s speed using real browsers in various environments.",
   "keywords": [
     "performance",