sitespeed.io 36.4.0 → 37.0.0

package/package.json

@@ -5,7 +5,7 @@
     "sitespeed.io": "./bin/sitespeed.js",
     "sitespeed.io-wpr": "./bin/browsertimeWebPageReplay.js"
   },
-  "version": "36.4.0",
+  "version": "37.0.0",
   "description": "sitespeed.io is an open-source tool for comprehensive web performance analysis, enabling you to test, monitor, and optimize your website’s speed using real browsers in various environments.",
   "keywords": [
     "performance",
@@ -83,20 +83,17 @@
   "dependencies": {
     "@aws-sdk/client-s3": "3.717.0",
     "@google-cloud/storage": "7.14.0",
-    "@influxdata/influxdb-client": "1.33.2",
     "@sitespeed.io/log": "0.2.6",
     "@sitespeed.io/plugin": "1.0.0",
     "@tgwf/co2": "0.16.4",
     "@slack/webhook": "7.0.4",
     "axe-core": "4.10.2",
-    "browsertime": "24.2.0",
+    "browsertime": "24.3.0",
     "coach-core": "8.1.1",
     "dayjs": "1.11.11",
     "fast-crc32c": "2.0.0",
     "fast-stats": "0.0.7",
     "import-global": "1.1.1",
-    "influx": "5.9.3",
-    "junit-report-builder": "3.2.1",
     "lodash.get": "4.4.2",
     "lodash.merge": "4.6.2",
     "lodash.set": "4.3.2",
@@ -105,7 +102,6 @@
     "ora": "8.0.1",
     "pug": "3.0.3",
     "simplecrawler": "1.1.9",
-    "tape": "5.8.1",
     "yargs": "17.7.2"
   },
   "overrides": {

package/lib/plugins/influxdb/data-generator.js

@@ -1,262 +0,0 @@
-import merge from 'lodash.merge';
-
-import { flattenMessageData } from '../../support/flattenMessage.js';
-import {
-  getConnectivity,
-  getURLAndGroup,
-  toSafeKey
-} from '../../support/tsdbUtil.js';
-
-function getAdditionalTags(key, type) {
-  let tags = {};
-  const keyArray = key.split('.');
-  if (/(^contentTypes)/.test(key)) {
-    // contentTypes.favicon.requests.mean
-    // contentTypes.favicon.requests
-    // contentTypes.css.transferSize
-    tags.contentType = keyArray[1];
-  } else if (/(^pageTimings|^visualMetrics)/.test(key)) {
-    // pageTimings.serverResponseTime.max
-    // visualMetrics.SpeedIndex.median
-    tags.timings = keyArray[0];
-  } else
-    switch (type) {
-      case 'browsertime.pageSummary': {
-        // statistics.timings.pageTimings.backEndTime.median
-        // statistics.timings.userTimings.marks.logoTime.median
-        // statistics.visualMetrics.SpeedIndex.median
-        tags[keyArray[0]] = keyArray[1];
-        if (keyArray.length >= 5) {
-          tags[keyArray[2]] = keyArray[3];
-        }
-        if (key.includes('cpu.categories')) {
-          tags.cpu = 'category';
-        } else if (key.includes('cpu.events')) {
-          tags.cpu = 'event';
-        } else if (key.includes('cpu.longTasks')) {
-          tags.cpu = 'longTask';
-        }
-
-        break;
-      }
-      case 'browsertime.summary': {
-        // firstPaint.median
-        // userTimings.marks.logoTime.median
-        if (key.includes('userTimings')) {
-          tags[keyArray[0]] = keyArray[1];
-        }
-
-        break;
-      }
-      case 'axe.pageSummary': {
-        tags.axeType = keyArray[0];
-        break;
-      }
-      case 'coach.pageSummary': {
-        // advice.score
-        // advice.performance.score
-        if (keyArray.length > 2) {
-          tags.advice = keyArray[1];
-        }
-
-        // set the actual advice name
-        // advice.performance.adviceList.cacheHeaders.score
-        if (keyArray.length > 4) {
-          tags.adviceName = keyArray[3];
-        }
-
-        break;
-      }
-      case 'coach.summary': {
-        // score.max
-        // performance.score.median
-        if (keyArray.length === 3) {
-          tags.advice = keyArray[0];
-        }
-
-        break;
-      }
-      case 'pagexray.summary': {
-        // firstParty.requests.min pagexray.summary
-        // requests.median
-        // responseCodes.307.max pagexray.summary
-        // requests.min pagexray.summary
-        if (key.includes('responseCodes')) {
-          tags.responseCodes = 'response';
-        }
-
-        if (key.includes('firstParty') || key.includes('thirdParty')) {
-          tags.party = keyArray[0];
-        }
-
-        break;
-      }
-      case 'pagexray.pageSummary': {
-        // thirdParty.contentTypes.json.requests pagexray.pageSummary
-        // thirdParty.requests pagexray.pageSummary
-        // firstParty.cookieStats.max pagexray.pageSummary
-        // responseCodes.200 pagexray.pageSummary
-        // expireStats.max pagexray.pageSummary
-        // totalDomains pagexray.pageSummary
-        if (key.includes('firstParty') || key.includes('thirdParty')) {
-          tags.party = keyArray[0];
-        }
-        if (key.includes('responseCodes')) {
-          tags.responseCodes = 'response';
-        }
-        if (key.includes('contentTypes')) {
-          tags.contentType = keyArray[2];
-        }
-
-        break;
-      }
-      case 'thirdparty.pageSummary': {
-        tags.thirdPartyCategory = keyArray[1];
-        tags.thirdPartyType = keyArray[2];
-
-        break;
-      }
-      case 'lighthouse.pageSummary': {
-        // categories.seo.score
-        // categories.performance.score
-        if (key.includes('score')) {
-          tags.audit = keyArray[1];
-        }
-        if (key.includes('audits')) {
-          tags.audit = keyArray[1];
-        }
-
-        break;
-      }
-      case 'crux.pageSummary': {
-        tags.experience = keyArray[0];
-        tags.formFactor = keyArray[1];
-        tags.metric = keyArray[2];
-
-        break;
-      }
-      case 'gpsi.pageSummary': {
-        if (key.includes('googleWebVitals')) {
-          tags.testType = 'googleWebVitals';
-        } else if (key.includes('score')) {
-          tags.testType = 'score';
-        } else if (key.includes('loadingExperience')) {
-          tags.experience = keyArray[0];
-          tags.metric = keyArray[1];
-          tags.testType = 'crux';
-        }
-
-        break;
-      }
-      default:
-      // console.log('Missed added tags to ' + key + ' ' + type);
-    }
-  return tags;
-}
-
-function getFieldAndSeriesName(key) {
-  const functions = [
-    'min',
-    'p10',
-    'median',
-    'mean',
-    'avg',
-    'max',
-    'p90',
-    'p99',
-    'mdev',
-    'stddev',
-    'rsd'
-  ];
-  const keyArray = key.split('.');
-  const end = keyArray.pop();
-  if (functions.includes(end)) {
-    return { field: end, seriesName: keyArray.pop() };
-  }
-  return { field: 'value', seriesName: end };
-}
-export class InfluxDBDataGenerator {
-  constructor(includeQueryParameters, options) {
-    this.includeQueryParams = !!includeQueryParameters;
-    this.options = options;
-    this.defaultTags = {};
-    for (let row of options.influxdb.tags.split(',')) {
-      const keyAndValue = row.split('=');
-      this.defaultTags[keyAndValue[0]] = keyAndValue[1];
-    }
-  }
-
-  dataFromMessage(message, time, alias) {
-    function getTagsFromMessage(
-      message,
-      includeQueryParameters,
-      options,
-      defaultTags
-    ) {
-      const tags = merge({}, defaultTags);
-      let typeParts = message.type.split('.');
-      tags.origin = typeParts[0];
-      typeParts.push(typeParts.shift());
-      tags.summaryType = typeParts[0];
-
-      // always have browser and connectivity in Browsertime and related tools
-      if (
-        /(^pagexray|^coach|^browsertime|^thirdparty|^axe|^sustainable)/.test(
-          message.type
-        )
-      ) {
-        // if we have a friendly name for your connectivity, use that!
-        let connectivity = getConnectivity(options);
-        tags.connectivity = connectivity;
-        tags.browser = options.browser;
-      } else if (/(^gpsi)/.test(message.type)) {
-        tags.strategy = options.mobile ? 'mobile' : 'desktop';
-      }
-
-      // if we get a URL type, add the URL
-      if (message.url) {
-        const urlAndGroup = getURLAndGroup(
-          options,
-          message.group,
-          message.url,
-          includeQueryParameters,
-          alias
-        ).split('.');
-        tags.page = urlAndGroup[1];
-        tags.group = urlAndGroup[0];
-      } else if (message.group) {
-        // add the group of the summary message
-        tags.group = toSafeKey(message.group, options.influxdb.groupSeparator);
-      }
-
-      if (options.slug) {
-        tags.testName = options.slug;
-      }
-
-      return tags;
-    }
-    return Object.entries(flattenMessageData(message)).reduce(
-      (entries, [key, value]) => {
-        const fieldAndSeriesName = getFieldAndSeriesName(key);
-        let tags = getTagsFromMessage(
-          message,
-          this.includeQueryParams,
-          this.options,
-          this.defaultTags
-        );
-        tags = { ...getAdditionalTags(key, message.type), ...tags };
-        const point = {
-          time: time.valueOf(),
-          [fieldAndSeriesName.field]: value
-        };
-        entries.push({
-          tags,
-          seriesName: fieldAndSeriesName.seriesName,
-          point
-        });
-        return entries;
-      },
-      []
-    );
-  }
-}

package/lib/plugins/influxdb/index.js

@@ -1,176 +0,0 @@
-import { getLogger } from '@sitespeed.io/log';
-import dayjs from 'dayjs';
-
-import { SitespeedioPlugin } from '@sitespeed.io/plugin';
-import { InfluxDBSender as Sender } from './sender.js';
-import { InfluxDB2Sender as SenderV2 } from './senderV2.js';
-import { sendV1 } from './send-annotation.js';
-import { sendV2 } from './send-annotationV2.js';
-import { InfluxDBDataGenerator as DataGenerator } from './data-generator.js';
-import { throwIfMissing, isEmpty } from '../../support/util.js';
-
-const log = getLogger('sitespeedio.plugin.influxdb');
-export default class InfluxDBPlugin extends SitespeedioPlugin {
-  constructor(options, context, queue) {
-    super({ name: 'influxdb', options, context, queue });
-  }
-
-  open(context, options) {
-    throwIfMissing(options.influxdb, ['host', 'database'], 'influxdb');
-    this.filterRegistry = context.filterRegistry;
-    log.debug(
-      'Setup InfluxDB host %s and database %s',
-      options.influxdb.host,
-      options.influxdb.database
-    );
-
-    const options_ = options.influxdb;
-    this.options = options;
-    this.sender =
-      options_.version == 1 ? new Sender(options_) : new SenderV2(options_);
-    this.timestamp = context.timestamp;
-    this.resultUrls = context.resultUrls;
-    this.dataGenerator = new DataGenerator(
-      options_.includeQueryParams,
-      options
-    );
-    this.messageTypesToFireAnnotations = [];
-    this.receivedTypesThatFireAnnotations = {};
-    this.make = context.messageMaker('influxdb').make;
-    this.sendAnnotation = true;
-    this.alias = {};
-  }
-
-  processMessage(message, queue) {
-    const filterRegistry = this.filterRegistry;
-
-    // First catch if we are running Browsertime and/or WebPageTest
-    switch (message.type) {
-      case 'browsertime.setup': {
-        this.messageTypesToFireAnnotations.push('browsertime.pageSummary');
-        this.usingBrowsertime = true;
-
-        break;
-      }
-      case 'browsertime.config': {
-        if (message.data.screenshot) {
-          this.useScreenshots = message.data.screenshot;
-          this.screenshotType = message.data.screenshotType;
-        }
-
-        break;
-      }
-      case 'sitespeedio.setup': {
-        // Let other plugins know that the InfluxDB plugin is alive
-        queue.postMessage(this.make('influxdb.setup'));
-
-        break;
-      }
-      case 'grafana.setup': {
-        this.sendAnnotation = false;
-
-        break;
-      }
-      // No default
-    }
-
-    if (message.type === 'browsertime.alias') {
-      this.alias[message.url] = message.data;
-    }
-
-    if (
-      !(
-        message.type.endsWith('.summary') ||
-        message.type.endsWith('.pageSummary')
-      )
-    ) {
-      return;
-    }
-
-    if (this.messageTypesToFireAnnotations.includes(message.type)) {
-      this.receivedTypesThatFireAnnotations[message.url]
-        ? this.receivedTypesThatFireAnnotations[message.url]++
-        : (this.receivedTypesThatFireAnnotations[message.url] = 1);
-    }
-
-    // Let us skip this for a while and concentrate on the real deal
-    if (
-      /(^largestassets|^slowestassets|^aggregateassets|^domains)/.test(
-        message.type
-      )
-    )
-      return;
-
-    // we only sends individual groups to Influx, not the
-    // total of all groups (you can calculate that yourself)
-    if (message.group === 'total') {
-      return;
-    }
-
-    message = filterRegistry.filterMessage(message);
-    if (isEmpty(message.data)) return;
-
-    let data = this.dataGenerator.dataFromMessage(
-      message,
-      message.type === 'browsertime.pageSummary'
-        ? dayjs(message.runTime)
-        : this.timestamp,
-      this.alias
-    );
-
-    if (data.length > 0) {
-      log.debug('Send the following data to InfluxDB: %:2j', data);
-      return this.sender.send(data).then(() => {
-        // Make sure we only send the annotation once per URL:
-        // If we run browsertime, always send on browsertime.pageSummary
-        // If we run WebPageTest standalone, send on webPageTestSummary
-        // when we configured a base url
-        if (
-          this.receivedTypesThatFireAnnotations[message.url] ===
-            this.messageTypesToFireAnnotations.length &&
-          this.resultUrls.hasBaseUrl() &&
-          this.sendAnnotation
-        ) {
-          const absolutePagePath = this.resultUrls.absoluteSummaryPagePath(
-            message.url,
-            this.alias[message.url]
-          );
-          this.receivedTypesThatFireAnnotations[message.url] = 0;
-
-          return this.options.influxdb.version == 2
-            ? sendV2(
-                message.url,
-                message.group,
-                absolutePagePath,
-                this.useScreenshots,
-                this.screenshotType,
-                // Browsertime pass on when the first run was done for that URL
-                message.runTime,
-                this.alias,
-                this.usingBrowsertime,
-                this.options
-              )
-            : sendV1(
-                message.url,
-                message.group,
-                absolutePagePath,
-                this.useScreenshots,
-                this.screenshotType,
-                // Browsertime pass on when the first run was done for that URL
-                message.runTime,
-                this.alias,
-                this.usingBrowsertime,
-                this.options
-              );
-        }
-      });
-    } else {
-      return Promise.reject(
-        new Error(
-          'No data to send to influxdb for message:\n' +
-            JSON.stringify(message, undefined, 2)
-        )
-      );
-    }
-  }
-}

package/lib/plugins/influxdb/send-annotation.js

@@ -1,111 +0,0 @@
-import http from 'node:http';
-import https from 'node:https';
-import { stringify } from 'node:querystring';
-
-import { getLogger } from '@sitespeed.io/log';
-import dayjs from 'dayjs';
-
-import { getConnectivity, getURLAndGroup } from '../../support/tsdbUtil.js';
-import {
-  getAnnotationMessage,
-  getTagsAsString
-} from '../../support/annotationsHelper.js';
-
-const log = getLogger('sitespeedio.plugin.influxdb');
-
-export function sendV1(
-  url,
-  group,
-  absolutePagePath,
-  screenShotsEnabledInBrowsertime,
-  screenshotType,
-  runTime,
-  alias,
-  usingBrowsertime,
-  options
-) {
-  // The tags make it possible for the dashboard to use the
-  // templates to choose which annotations that will be showed.
-  // That's why we need to send tags that matches the template
-  // variables in Grafana.
-  const connectivity = getConnectivity(options);
-  const browser = options.browser;
-  const urlAndGroup = getURLAndGroup(
-    options,
-    group,
-    url,
-    options.influxdb.includeQueryParams,
-    alias
-  ).split('.');
-  let tags = [connectivity, browser, urlAndGroup[0], urlAndGroup[1]];
-
-  if (options.slug) {
-    tags.push(options.slug);
-  }
-
-  const message = getAnnotationMessage(
-    absolutePagePath,
-    screenShotsEnabledInBrowsertime,
-    screenshotType,
-    undefined,
-    usingBrowsertime,
-    options
-  );
-  const timestamp = runTime
-    ? Math.round(dayjs(runTime) / 1000)
-    : Math.round(dayjs() / 1000);
-  // if we have a category, let us send that category too
-  if (options.influxdb.tags) {
-    for (let row of options.influxdb.tags.split(',')) {
-      const keyAndValue = row.split('=');
-      tags.push(keyAndValue[1]);
-    }
-  }
-  const influxDBTags = getTagsAsString(tags);
-  const postData = `events title="Sitespeed.io",text="${message}",tags=${influxDBTags} ${timestamp}`;
-  const postOptions = {
-    hostname: options.influxdb.host,
-    port: options.influxdb.port,
-    path: '/write?db=' + options.influxdb.database + '&precision=s',
-    method: 'POST',
-    headers: {
-      'Content-Type': 'application/x-www-form-urlencoded',
-      'Content-Length': Buffer.byteLength(postData)
-    }
-  };
-
-  if (options.influxdb.username) {
-    postOptions.path =
-      postOptions.path +
-      '&' +
-      stringify({
-        u: options.influxdb.username,
-        p: options.influxdb.password
-      });
-  }
-
-  return new Promise((resolve, reject) => {
-    log.debug('Send annotation to Influx: %j', postData);
-    // not perfect but maybe work for us
-    const library = options.influxdb.protocol === 'https' ? https : http;
-    const request = library.request(postOptions, res => {
-      if (res.statusCode === 204) {
-        res.setEncoding('utf8');
-        log.debug('Sent annotation to InfluxDB');
-        resolve();
-      } else {
-        const e = new Error(
-          `Got ${res.statusCode} from InfluxDB when sending annotation ${res.statusMessage}`
-        );
-        log.warn(e.message);
-        reject(e);
-      }
-    });
-    request.on('error', error => {
-      log.error('Got error from InfluxDB when sending annotation', error);
-      reject(error);
-    });
-    request.write(postData);
-    request.end();
-  });
-}

package/lib/plugins/influxdb/send-annotationV2.js

@@ -1,106 +0,0 @@
-import http from 'node:http';
-import https from 'node:https';
-
-import { getLogger } from '@sitespeed.io/log';
-import dayjs from 'dayjs';
-
-import { getConnectivity, getURLAndGroup } from '../../support/tsdbUtil.js';
-import {
-  getAnnotationMessage,
-  getTagsAsString
-} from '../../support/annotationsHelper.js';
-
-const log = getLogger('sitespeedio.plugin.influxdb');
-
-export function sendV2(
-  url,
-  group,
-  absolutePagePath,
-  screenShotsEnabledInBrowsertime,
-  screenshotType,
-  runTime,
-  alias,
-  usingBrowsertime,
-  options
-) {
-  // The tags make it possible for the dashboard to use the
-  // templates to choose which annotations that will be showed.
-  // That's why we need to send tags that matches the template
-  // variables in Grafana.
-  const connectivity = getConnectivity(options);
-  const browser = options.browser;
-  const urlAndGroup = getURLAndGroup(
-    options,
-    group,
-    url,
-    options.influxdb.includeQueryParams,
-    alias
-  ).split('.');
-  let tags = [
-    `connectivity=${connectivity}`,
-    `browser=${browser}`,
-    `group=${urlAndGroup[0]}`,
-    `page=${urlAndGroup[1]}`
-  ];
-
-  if (options.slug) {
-    tags.push(`slug=${options.slug}`);
-  }
-
-  const message = getAnnotationMessage(
-    absolutePagePath,
-    screenShotsEnabledInBrowsertime,
-    screenshotType,
-    undefined,
-    usingBrowsertime,
-    options
-  );
-  const timestamp = runTime
-    ? Math.round(dayjs(runTime) / 1000)
-    : Math.round(dayjs() / 1000);
-  // if we have a category, let us send that category too
-  if (options.influxdb.tags) {
-    for (const tag of options.influxdb.tags.split(',')) {
-      tags.push(tag);
-    }
-  }
-  const influxDBTags = tags.join(',');
-  const grafanaTags = getTagsAsString(tags.map(pair => pair.split('=')[1]));
-  const postData = `annotations,${influxDBTags} title="Sitespeed.io",text="${message}",tags=${grafanaTags} ${timestamp}`;
-  const postOptions = {
-    hostname: options.influxdb.host,
-    port: options.influxdb.port,
-    path: `/api/v2/write?org=${options.influxdb.organisation}&bucket=${options.influxdb.database}&precision=s`,
-    method: 'POST',
-    headers: {
-      'Content-Type': 'application/x-www-form-urlencoded',
-      'Content-Length': Buffer.byteLength(postData),
-      Authorization: `Token ${options.influxdb.token}`
-    }
-  };
-
-  return new Promise((resolve, reject) => {
-    log.debug('Send annotation to Influx: %j', postData);
-    // not perfect but maybe work for us
-    const library = options.influxdb.protocol === 'https' ? https : http;
-    const request = library.request(postOptions, res => {
-      if (res.statusCode === 204) {
-        res.setEncoding('utf8');
-        log.debug('Sent annotation to InfluxDB');
-        resolve();
-      } else {
-        const e = new Error(
-          `Got ${res.statusCode} from InfluxDB when sending annotation ${res.statusMessage}`
-        );
-        log.warn(e.message);
-        reject(e);
-      }
-    });
-    request.on('error', error => {
-      log.error('Got error from InfluxDB when sending annotation', error);
-      reject(error);
-    });
-    request.write(postData);
-    request.end();
-  });
-}