sitespeed.io 30.2.1 → 30.4.0

package/.github/workflows/building-docker-autobuild.yml

@@ -9,19 +9,19 @@ jobs:
     steps:
       -
         name: Set up QEMU
-        uses: docker/setup-qemu-action@v2
+        uses: docker/setup-qemu-action@v3
       -
         name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@v2
+        uses: docker/setup-buildx-action@v3
       -
         name: Login to DockerHub
-        uses: docker/login-action@v2 
+        uses: docker/login-action@v3 
         with:
           username: ${{ secrets.DOCKERHUB_USERNAME }}
           password: ${{ secrets.DOCKERHUB_TOKEN }}
       -
         name: Build and push sitespeed.io
-        uses: docker/build-push-action@v4
+        uses: docker/build-push-action@v5
         with:
           platforms: linux/amd64
           push: true

package/.github/workflows/building-docker-release.yml

@@ -9,16 +9,16 @@ jobs:
     steps:
       -
           name: Checkout
-          uses: actions/checkout@v3
+          uses: actions/checkout@v4
       -
         name: Set up QEMU
-        uses: docker/setup-qemu-action@v1
+        uses: docker/setup-qemu-action@v3
       -
         name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@v2
+        uses: docker/setup-buildx-action@v3
       -
         name: Login to DockerHub
-        uses: docker/login-action@v2
+        uses: docker/login-action@v3
         with:
           username: ${{ secrets.DOCKERHUB_USERNAME }}
           password: ${{ secrets.DOCKERHUB_TOKEN }}
@@ -30,7 +30,7 @@ jobs:
           strip_v: true
       -
         name: Build and push sitespeed.io
-        uses: docker/build-push-action@v4
+        uses: docker/build-push-action@v5
         with:
           context: .
           platforms: linux/amd64,linux/arm64
@@ -39,7 +39,7 @@ jobs:
           tags: sitespeedio/sitespeed.io:${{steps.tag.outputs.tag}},sitespeedio/sitespeed.io:latest
       -
         name: Build and push sitespeed.io+1
-        uses: docker/build-push-action@v4
+        uses: docker/build-push-action@v5
         with:
           context: .
           platforms: linux/amd64,linux/arm64
@@ -50,7 +50,7 @@ jobs:
           tags: sitespeedio/sitespeed.io:${{steps.tag.outputs.tag}}-plus1
       -
         name: Build and push sitespeed.io+wpt
-        uses: docker/build-push-action@v4
+        uses: docker/build-push-action@v5
         with:
           context: .
           platforms: linux/amd64,linux/arm64
@@ -61,7 +61,7 @@ jobs:
           tags: sitespeedio/sitespeed.io:${{steps.tag.outputs.tag}}-webpagetest
       -
         name: Build and push sitespeed.io-slim
-        uses: docker/build-push-action@v4
+        uses: docker/build-push-action@v5
         with:
           context: .
           platforms: linux/amd64,linux/arm64

package/.github/workflows/crux-test.yml

@@ -10,9 +10,9 @@ jobs:
   build:
     runs-on: ubuntu-latest
     steps:
-    - uses: actions/checkout@v3
+    - uses: actions/checkout@v4
     - name: Use Node.js
-      uses: actions/setup-node@v3
+      uses: actions/setup-node@v4
       with:
         node-version: '20.x'
     - name: Install sitespeed.io

package/.github/workflows/docker-scan.yml

@@ -10,7 +10,7 @@ jobs:
     runs-on: ubuntu-20.04
     steps:
       - name: Checkout code
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
       
       - name: Build an image from Dockerfile
         run: |

package/.github/workflows/docker.yml

@@ -10,7 +10,7 @@ jobs:
   build:
     runs-on: ubuntu-latest
     steps:
-    - uses: actions/checkout@v3
+    - uses: actions/checkout@v4
     - name: Build Docker containers
       run: |
         docker buildx install
@@ -28,8 +28,10 @@ jobs:
       run: docker run --rm -v "$(pwd)":/sitespeed.io --network=host sitespeedio/sitespeed.io http://127.0.0.1:3001  -n 1 -b edge
     - name: Run test on slim container  
       run: docker run --rm -v "$(pwd)":/sitespeed.io --network=host sitespeedio/sitespeed.io:slim http://127.0.0.1:3001 -n 1 --browsertime.firefox.preference "devtools.netmonitor.persistlog:true"
-    - name: Test WebPageReplay with Chrome  
+    - name: Test WebPageReplay with Chrome
       run: docker run --cap-add=NET_ADMIN --rm -v "$(pwd)":/sitespeed.io -e REPLAY=true -e LATENCY=100 sitespeedio/sitespeed.io https://www.sitespeed.io -n 3 -b chrome 
+    - name: Test WebPageReplay user journey with Chrome
+      run: docker run --cap-add=NET_ADMIN --rm -v "$(pwd)":/sitespeed.io -e REPLAY=true -e LATENCY=100 sitespeedio/sitespeed.io test/prepostscripts/multiWindows.cjs -n 1 -b chrome --multi
     - name: Test WebPageReplay with Firefox  
       run: docker run --cap-add=NET_ADMIN --rm -v "$(pwd)":/sitespeed.io --network=host -e REPLAY=true -e LATENCY=100 sitespeedio/sitespeed.io https://www.sitespeed.io -n 3 -b firefox --browsertime.firefox.acceptInsecureCerts true
     - name: Run Chrome test with config

package/.github/workflows/linux.yml

@@ -10,9 +10,9 @@ jobs:
   build:
     runs-on: ubuntu-latest
     steps:
-    - uses: actions/checkout@v3
+    - uses: actions/checkout@v4
     - name: Use Node.js
-      uses: actions/setup-node@v3
+      uses: actions/setup-node@v4
       with:
         node-version: '20.x'
     - name: Install sitespeed.io
@@ -24,6 +24,11 @@ jobs:
         sudo apt-get update
         sudo apt-get --only-upgrade install google-chrome-stable
         google-chrome --version
+    - name: Install dependencies
+      run: |
+          python -m pip install --upgrade --user pip
+          python -m pip install --user scipy
+          python -m pip show scipy
     - name: Install Firefox
       uses: browser-actions/setup-firefox@latest
       #with:
@@ -69,4 +74,6 @@ jobs:
     - name: Run test with Influx 2.6.1
       run: bin/sitespeed.js http://127.0.0.1:3001/simple/ -n 1 --influxdb.host 127.0.0.1 --influxdb.port 8087 --influxdb.version 2 --influxdb.organisation sitespeed --influxdb.token sitespeed --xvfb
     - name: Run Chrome test with config
-      run: node bin/sitespeed.js --config test/exampleConfig.json http://127.0.0.1:3001/simple/ --xvfb
+      run: node bin/sitespeed.js --config test/exampleConfig.json http://127.0.0.1:3001/simple/ --xvfb
+    - name: Run Chrome test using compare plugin
+      run: node bin/sitespeed.js --compare.id compare --compare.saveBaseline --compare.baselinePath test/ http://127.0.0.1:3001/simple/ --xvfb

package/.github/workflows/safari.yml

@@ -10,9 +10,9 @@ jobs:
   build:
     runs-on: macos-latest
     steps:
-    - uses: actions/checkout@v3
+    - uses: actions/checkout@v4
     - name: Use Node.js
-      uses: actions/setup-node@v3
+      uses: actions/setup-node@v4
       with:
         node-version: '20.x'
     - name: Install dependencies 

package/.github/workflows/sitespeed-io-action-example.yml

@@ -9,7 +9,7 @@ jobs:
     name: running sitespeed.io
     steps:
       - name: code checkout
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
       # Here we build our own container to make sure we test against our latest code
       # but YOU can just used the latest version by specifying 
       # sitespeedio/sitespeed.io:VERSION

package/.github/workflows/unittests.yml

@@ -13,9 +13,9 @@ jobs:
       matrix:
         node-version: [18.x, 20.x]
     steps:
-    - uses: actions/checkout@v3
+    - uses: actions/checkout@v4
     - name: Use Node.js ${{ matrix.node-version }}
-      uses: actions/setup-node@v3
+      uses: actions/setup-node@v4
       with:
         node-version: ${{ matrix.node-version }}
     - name: Install dependencies and Chrome

package/.github/workflows/windows.yml

@@ -10,9 +10,9 @@ jobs:
   build:
     runs-on: windows-latest
     steps:
-    - uses: actions/checkout@v3
+    - uses: actions/checkout@v4
     - name: Use Node.js
-      uses: actions/setup-node@v3
+      uses: actions/setup-node@v4
       with:
         node-version: '20.x'
     - name: Install sitespeed.io

package/CHANGELOG.md

@@ -1,5 +1,20 @@
 # CHANGELOG - sitespeed.io  (we use [semantic versioning](https://semver.org))
 
+## 30.4.0 - 2023-11-27
+### Fixed
+* Upgrade to Browsretime 19.1.0 with a fix for Geckodriver so that the correct ARM version is installed on Mac Arm machines.
+### Added
+* The new compare plugin makes it easy to use Mann Whitney U/Wilcox for support to find performance egressions. Read all about the plugin in the [documentation](https://www.sitespeed.io/documentation/sitespeed.io/compare/).
+
+## 30.3.0 - 2023-11-09
+
+### Added
+* Upgrade to Browsertime 18.0.0.
+* Added support to run user journeys with WebPageReplay [#4005](https://github.com/sitespeedio/sitespeed.io/pull/4005).
+
+### Fixed
+* Downgrade puppeteer in the +1 container for Lighthouse, thank you [bairov pavel](https://github.com/Amerousful) for PR [#123](https://github.com/sitespeedio/plugin-lighthouse/pull/123).
+
 ## 30.2.1 - 2023-11-03
 ### Fixed
 * The Docker+1 container got a fix for Lighthouse that didn't work in 30.2.0 (adding dev-shm when starting Chrome).

package/Dockerfile

@@ -1,4 +1,4 @@
-FROM sitespeedio/webbrowsers:chrome-119.0-firefox-119.0-edge-119.0
+FROM sitespeedio/webbrowsers:chrome-119.0-firefox-120.0-edge-119.0
 
 ARG TARGETPLATFORM=linux/amd64
 
@@ -44,4 +44,6 @@ RUN echo 'ALL ALL=NOPASSWD: /usr/sbin/tc, /usr/sbin/route, /usr/sbin/ip' > /etc/
 
 ENTRYPOINT ["/start.sh"]
 VOLUME /sitespeed.io
+VOLUME /baseline
+
 WORKDIR /sitespeed.io

package/bin/browsertimeWebPageReplay.js

@@ -36,9 +36,18 @@ try {
   throw e;
 }
 
-async function testURLs(engine, urls) {
+async function testURLs(engine, urls, isMulti) {
   try {
     await engine.start();
+
+    if(isMulti) {
+      const result = await engine.runMultiple(urls);
+      for (let errors of result[0].errors) {
+        if (errors.length > 0) {
+          process.exitCode = 1;
+        }
+      }
+    } else {
     for (let url of urls) {
       const result = await engine.run(url);
       for (let errors of result[0].errors) {
@@ -47,6 +56,7 @@ async function testURLs(engine, urls) {
         }
       }
     }
+    }
   } finally {
     engine.stop();
   }
@@ -203,11 +213,12 @@ async function runBrowsertime() {
       );
     }
   }
+
   const engine = new BrowsertimeEngine(btOptions);
-  const urls = getURLs(parsed.argv._);
+  const urls = parsed.argv.multi ? parsed.argv._ : getURLs(parsed.argv._);
 
   try {
-    await testURLs(engine, urls);
+    await testURLs(engine, urls,  parsed.argv.multi);
   } catch (e) {
     console.error('Could not run ' + e);
     process.exit(1);

package/lib/cli/cli.js

@@ -1870,6 +1870,68 @@ export async function parseCommandLine() {
       group: 'API'
     });
 
+  parsed
+    .option('compare.id', {
+      type: 'string',
+      describe:
+        'The id of the test. Will be used to find the baseline test, that is using the id as a part of the name.',
+      group: 'compare'
+    })
+    .option('compare.baselinePath', {
+      type: 'string',
+      describe:
+        'Specifies the path to the baseline data file. This file is used as a reference for comparison against the current test data.',
+      group: 'compare'
+    })
+    .option('compare.saveBaseline', {
+      type: 'boolean',
+      default: false,
+      describe:
+        'Determines whether to save the current test data as the new baseline. Set to true to save the current data as baseline for future comparisons.',
+      group: 'compare'
+    })
+    .option('compare.testType', {
+      describe:
+        'Selects the statistical test type to be used for comparison. Options are mannwhitneyu for the Mann-Whitney U test and wilcoxon for the Wilcoxon signed-rank test.',
+      choices: ['mannwhitneyu', ' wilcoxon'],
+      default: 'mannwhitneyu',
+      group: 'compare'
+    })
+    .option('compare.alternative', {
+      choices: ['less', ' greater', 'two-sided'],
+      default: 'less',
+      describe:
+        'Specifies the alternative hypothesis to be tested. Options are less for one-sided test where the first group is expected to be less than the second, greater for one-sided test with the first group expected to be greater, or two-sided for a two-sided test.',
+      group: 'compare'
+    })
+    .option('compare.wilcoxon.correction', {
+      type: 'boolean',
+      describe:
+        'Enables or disables the continuity correction in the Wilcoxon signed-rank test. Set to true to enable the correction.',
+      default: false,
+      group: 'compare'
+    })
+    .option('compare.wilcoxon.zeroMethod', {
+      choices: ['wilcox', ' pratt', 'zsplit'],
+      describe:
+        'Specifies the method for handling zero differences in the Wilcoxon test. wilcox discards all zero-difference pairs, pratt includes all, and zsplit splits them evenly among positive and negative ranks.',
+      default: 'zsplit',
+      group: 'compare'
+    })
+    .option('compare.mannwhitneyu.useContinuity', {
+      type: 'boolean',
+      default: false,
+      describe:
+        'Determines whether to use continuity correction in the Mann-Whitney U test. Set to true to apply the correction.',
+      group: 'compare'
+    })
+    .option('compare.mannwhitneyu.method', {
+      choices: ['auto', ' exact', 'symptotic'],
+      escribe:
+        'Selects the method for calculating the Mann-Whitney U test. auto automatically selects between exact and asymptotic based on sample size, exact uses the exact distribution of U, and symptotic uses a normal approximation.',
+      default: 'auto',
+      group: 'compare'
+    });
   parsed
     .option('mobile', {
       describe:

package/lib/plugins/compare/baseline.js

@@ -0,0 +1,35 @@
+import fs from 'node:fs/promises';
+import { join, resolve } from 'node:path';
+
+export async function getBaseline(id, compareOptions) {
+  try {
+    return JSON.parse(
+      await fs.readFile(
+        resolve(
+          join(compareOptions.baselinePath || process.cwd(), `${id}.json`)
+        )
+      )
+    );
+  } catch {
+    return;
+  }
+}
+/*
+async function getBaselineFromInternet(url) {
+  try {
+    const response = await fetch(url);
+    return response.json();
+  } catch (error) {
+    log.error('Could not fetch', error);
+  }
+}
+
+async function getBaselineFromFile(path) {}
+
+/*
+export async function saveBaseline(json, options) {
+
+}*/
+export async function saveBaseline(json, name) {
+  return fs.writeFile(resolve(name), JSON.stringify(json));
+}