sitespeed.io 35.7.5 → 36.0.0-alpha.2

package/lib/plugins/scp/index.js

@@ -3,11 +3,11 @@ import fs from 'node:fs';
 
 import { SitespeedioPlugin } from '@sitespeed.io/plugin';
 import { Client } from 'node-scp';
-import readdir from 'recursive-readdir';
-import intel from 'intel';
+import { getLogger } from '@sitespeed.io/log';
 import { throwIfMissing } from '../../support/util.js';
+import { recursiveReaddir } from '../../support/fileUtil.js';
 
-const log = intel.getLogger('sitespeedio.plugin.scp');
+const log = getLogger('sitespeedio.plugin.scp');
 
 async function getClient(scpOptions) {
   const options = {
@@ -46,7 +46,7 @@ async function upload(dir, scpOptions, prefix) {
     }
     await client.uploadDir(dir, path.join(scpOptions.destinationPath, prefix));
   } catch (error) {
-    log.error(error);
+    log.error(`Error uploading dir ` + error);
     throw error;
   } finally {
     if (client) {
@@ -66,7 +66,7 @@ async function uploadFiles(files, scpOptions, prefix) {
       );
     }
   } catch (error) {
-    log.error(error);
+    log.error(`Error uploading files ` + error);
     throw error;
   } finally {
     if (client) {
@@ -75,12 +75,8 @@ async function uploadFiles(files, scpOptions, prefix) {
   }
 }
 
-function ignoreDirectories(file, stats) {
-  return stats.isDirectory();
-}
-
 async function uploadLatestFiles(dir, scpOptions, prefix) {
-  const files = await readdir(dir, [ignoreDirectories]);
+  const files = await recursiveReaddir(dir, true);
 
   return uploadFiles(files, scpOptions, prefix);
 }

package/lib/plugins/slack/index.js

@@ -1,7 +1,5 @@
-import { promisify } from 'node:util';
-
-import intel from 'intel';
-import Slack from 'node-slack';
+import { getLogger } from '@sitespeed.io/log';
+import { IncomingWebhook } from '@slack/webhook';
 import merge from 'lodash.merge';
 import set from 'lodash.set';
 import { SitespeedioPlugin } from '@sitespeed.io/plugin';
@@ -11,7 +9,7 @@ import { getAttachements } from './attachements.js';
 import { getSummary } from './summary.js';
 import { throwIfMissing } from '../../support/util.js';
 
-const log = intel.getLogger('sitespeedio.plugin.slack');
+const log = getLogger('sitespeedio.plugin.slack');
 
 const defaultConfig = {
   userName: 'Sitespeed.io',
@@ -23,8 +21,7 @@ const defaultConfig = {
 
 function send(options, dataCollector, context, screenshotType, alias) {
   const slackOptions = merge({}, defaultConfig, options.slack);
-  const slack = new Slack(slackOptions.hookUrl);
-  const send = promisify(slack.send.bind(slack));
+  const slack = new IncomingWebhook(slackOptions.hookUrl);
   const type = slackOptions.type;
   const pageErrors = [];
   let logo = 'https://www.sitespeed.io/img/slack/sitespeed-logo-slack.png';
@@ -73,20 +70,22 @@ function send(options, dataCollector, context, screenshotType, alias) {
       slackOptions.channel,
       slackOptions.userName
     );
-    return send({
-      text,
-      icon_url: logo,
-      channel,
-      mrkdwn: true,
-      username: slackOptions.userName,
-      attachments
-    }).catch(error => {
-      if (error.errno === 'ETIMEDOUT') {
-        log.warn('Timeout sending Slack message.');
-      } else {
-        throw error;
-      }
-    });
+    return slack
+      .send({
+        text,
+        icon_url: logo,
+        channel,
+        mrkdwn: true,
+        username: slackOptions.userName,
+        attachments
+      })
+      .catch(error => {
+        if (error.errno === 'ETIMEDOUT') {
+          log.warn('Timeout sending Slack message.');
+        } else {
+          throw error;
+        }
+      });
   }
 }
 

package/lib/plugins/sustainable/index.js

@@ -4,7 +4,7 @@ import fs from 'node:fs';
 import zlib from 'node:zlib';
 import { promisify } from 'node:util';
 
-import intel from 'intel';
+import { getLogger } from '@sitespeed.io/log';
 import { co2, hosting } from '@tgwf/co2';
 import { SitespeedioPlugin } from '@sitespeed.io/plugin';
 import { Aggregator } from './aggregator.js';
@@ -27,7 +27,7 @@ const packageJson = JSON.parse(
 const co2Version = packageJson.dependencies['@tgwf/co2'];
 
 const gunzip = promisify(zlib.gunzip);
-const log = intel.getLogger('sitespeedio.plugin.sustainable');
+const log = getLogger('sitespeedio.plugin.sustainable');
 
 const DEFAULT_METRICS_PAGE_SUMMARY = [
   'co2PerPageView',
@@ -79,7 +79,30 @@ export default class SustainablePlugin extends SitespeedioPlugin {
       DEFAULT_METRICS_PAGE_SUMMARY,
       'sustainable.pageSummary'
     );
+
+    const greenDomainJSONpath = path.join(
+      __dirname,
+      'data',
+      'url2green.json.gz'
+    );
+
+    if (
+      !this.sustainableOptions.disableHosting &&
+      !this.sustainableOptions.useGreenWebHostingAPI
+    ) {
+      log.info('Reading green domain data from disk');
+      try {
+        this.data = await loadJSON(greenDomainJSONpath);
+      } catch {
+        log.error('Green domain local file is missing on disk.');
+        log.error(
+          'Run `DOWNLOAD_URL2GREEN=true npm install sitespeed.io` when you install to install the file locally or use the online API using `--sustainable.useGreenWebHostingAPI true` to check if a domain is green hosted.'
+        );
+        this.data = {};
+      }
+    }
   }
+
   async processMessage(message, queue) {
     const make = this.make;
     const aggregator = this.aggregator;
@@ -118,23 +141,21 @@ export default class SustainablePlugin extends SitespeedioPlugin {
       case 'pagexray.run': {
         // We got data for a URL, lets calculate co2, check green servers etc
         const listOfDomains = Object.keys(message.data.domains);
-        const greenDomainJSONpath = path.join(
-          __dirname,
-          'data',
-          'url2green.json.gz'
-        );
 
         let hostingGreenCheck;
         if (this.sustainableOptions.disableHosting === true) {
           hostingGreenCheck = [];
         } else {
-          hostingGreenCheck =
-            this.sustainableOptions.useGreenWebHostingAPI === true
-              ? await hosting.check(listOfDomains, undefined, 'sitespeed.io')
-              : await hosting.check(
-                  listOfDomains,
-                  await loadJSON(greenDomainJSONpath)
-                );
+          if (this.sustainableOptions.useGreenWebHostingAPI === true) {
+            log.info('Fetch hosting information from The Green Web Foundation');
+            hostingGreenCheck = await hosting(
+              listOfDomains,
+              undefined,
+              'sitespeed.io'
+            );
+          } else {
+            hostingGreenCheck = await hosting(listOfDomains, this.data);
+          }
         }
 
         const configuration = { model: this.sustainableOptions.model };

package/lib/plugins/text/color.js

@@ -0,0 +1,30 @@
+const ansi = {
+  reset: '\u001B[0m',
+  bold: '\u001B[1m',
+  green: '\u001B[32m',
+  yellow: '\u001B[33m',
+  red: '\u001B[31m',
+  blackBright: '\u001B[90m'
+};
+
+function stripAnsi(str) {
+  // eslint-disable-next-line no-control-regex
+  return str.replaceAll(/\u001B\[[0-9;]*[A-Za-z]/g, '');
+}
+
+export function colorise(str, clr) {
+  if (!clr || !ansi[clr]) return str;
+  return ansi[clr] + str + ansi.reset;
+}
+
+export function makeBold(str) {
+  return ansi.bold + str + ansi.reset;
+}
+
+export function getStrippedLength(str) {
+  return stripAnsi(str).length;
+}
+
+export function blackBright(str) {
+  return colorise(str, 'blackBright');
+}

package/lib/plugins/text/textBuilder.js

@@ -1,16 +1,42 @@
-import table from 'text-table';
-import flatten from 'lodash.flatten';
-import clic from 'cli-color';
-import chunk from 'lodash.chunk';
-
 import { plural, short, cap } from '../../support/helpers/index.js';
 import summaryBoxesSetup from '../html/setup/summaryBoxes.js';
+import { colorise, makeBold, getStrippedLength, blackBright } from './color.js';
 
-const { getStrippedLength, blackBright, bold } = clic;
 const tableOptions = {
   stringLength: getStrippedLength
 };
-const drab = blackBright;
+
+function table(rows, options = {}) {
+  const { stringLength = str => str.length } = options;
+
+  const colWidths = [];
+  for (const row of rows) {
+    for (const [i, cell] of row.entries()) {
+      const len = stringLength(String(cell));
+      colWidths[i] = Math.max(colWidths[i] || 0, len);
+    }
+  }
+
+  return rows
+    .map(row =>
+      row
+        .map((cell, i) => {
+          const cellStr = String(cell);
+          const padding = colWidths[i] - stringLength(cellStr);
+          return cellStr + ' '.repeat(padding);
+        })
+        .join('  ')
+    )
+    .join('\n');
+}
+
+function chunk(array, size) {
+  const out = [];
+  for (let i = 0; i < array.length; i += size) {
+    out.push(array.slice(i, i + size));
+  }
+  return out;
+}
 
 function getMarker(label) {
   return { ok: '√', warning: '!', error: '✗', info: '' }[label] || '';
@@ -25,22 +51,21 @@ function getColor(label) {
 
 function getHeader(context, options) {
   const noPages = options.urls.length;
-  return drab(
+  return blackBright(
     [
       `${plural(noPages, 'page')} analysed for ${short(context.name, 30)} `,
       `(${plural(options.browsertime.iterations, 'run')}, `,
       `${cap(options.browsertime.browser)}/${
         options.mobile ? 'mobile' : 'desktop'
-      }/${options.connectivity})`
+      })`
     ].join('')
   );
 }
 
 function getBoxes(metrics, html) {
-  return flatten(chunk(summaryBoxesSetup(metrics, html).filter(Boolean), 3));
+  return chunk(summaryBoxesSetup(metrics, html).filter(Boolean), 3).flat();
 }
 
-// foo bar -> fb
 function abbr(string_) {
   if (/total|overall/i.test(string_)) return string_.trim();
   return string_.replaceAll(/\w+\s?/g, a => a[0]);
@@ -54,9 +79,9 @@ export function renderSummary(metrics, context, options) {
   let out = getHeader(context, options);
   let rows = getBoxes(metrics, options.html).map(b => {
     const marker = getMarker(b.label),
-      c = getColor(b.label);
-    // color.xterm(ansi)(label),
-    return [clic[c](marker), clic[c](b.name), bold(b.median)];
+      clr = getColor(b.label);
+
+    return [colorise(marker, clr), colorise(b.name, clr), makeBold(b.median)];
   });
   rows.unshift(
     ['', 'Score / Metric', 'Median'],
@@ -72,13 +97,13 @@ export function renderBriefSummary(metrics, context, options) {
     reqs = '',
     rum = '';
   getBoxes(metrics, options.html).map(b => {
-    let c = getColor(b.label),
-      value = b.median,
-      name;
-    c = clic[c] || noop;
+    const clr = getColor(b.label);
+    let value = b.median;
+    let name;
+    const colorFn = clr ? str => colorise(str, clr) : noop;
     if (/score$/i.test(b.url)) {
       name = abbr(b.name.replace('score', ''));
-      scores.push(c(`${name}:${value}`));
+      scores.push(colorFn(`${name}:${value}`));
     } else if ('pageSize' === b.url) {
       value = value.replace(' ', ''); // 10 KB -> 10KB
       size = `${value}`;
@@ -86,6 +111,11 @@ export function renderBriefSummary(metrics, context, options) {
       reqs = `${value} reqs`;
     }
   });
-  lines.push(drab('Score: ') + scores.reverse().join(', '), size, reqs, rum);
+  lines.push(
+    blackBright('Score: ') + scores.reverse().join(', '),
+    size,
+    reqs,
+    rum
+  );
   options.summary = { out: `${out}\n` + lines.join(' / ') };
 }

package/lib/sitespeed.js

@@ -6,9 +6,7 @@ import fs from 'node:fs/promises';
 
 import dayjs from 'dayjs';
 import utc from 'dayjs/plugin/utc.js';
-import intel from 'intel';
-import pullAll from 'lodash.pullall';
-import union from 'lodash.union';
+import { getLogger } from '@sitespeed.io/log';
 
 import { configure } from './core/logging.js';
 import { toArray } from './support/util.js';
@@ -26,7 +24,7 @@ const packageJson = JSON.parse(
   await fs.readFile(path.resolve(path.join(__dirname, '..', 'package.json')))
 );
 
-const log = intel.getLogger('sitespeedio');
+const log = getLogger('sitespeedio');
 dayjs.extend(utc);
 
 const budgetResult = {
@@ -61,9 +59,6 @@ export async function run(options) {
     options
   );
 
-  // Setup logging
-  const logDir = await storageManager.createDirectory('logs');
-
   if (
     options.browsertime &&
     options.browsertime.tcpdump &&
@@ -74,7 +69,7 @@ export async function run(options) {
       'SSLKEYLOGFILE.txt'
     );
   }
-  configure(options, logDir);
+  configure(options);
 
   // Tell the world what we are using
   log.info(
@@ -85,9 +80,8 @@ export async function run(options) {
     packageJson.dependencies.browsertime,
     packageJson.dependencies['coach-core']
   );
-  if (log.isEnabledFor(log.DEBUG)) {
-    log.debug('Running with options: %:2j', options);
-  }
+
+  log.debug('Running with options: %:2j', options);
 
   let pluginNames = await parsePluginNames(options);
 
@@ -107,8 +101,11 @@ export async function run(options) {
     // Finalize the plugins that we wanna run
     // First we add the new ones and then remove, that means remove
     // always wins
-    pluginNames = union(pluginNames, toArray(plugins.add));
-    pullAll(pluginNames, toArray(plugins.remove));
+    pluginNames = [...new Set([...pluginNames, ...toArray(plugins.add)])];
+
+    const removeSet = new Set(toArray(plugins.remove));
+    pluginNames = pluginNames.filter(name => !removeSet.has(name));
+
     if (plugins.list) {
       log.info('The following plugins are enabled: %s', pluginNames.join(', '));
     }
@@ -122,12 +119,11 @@ export async function run(options) {
     budget: budgetResult,
     name: url,
     log,
-    intel,
+    getLogger,
     messageMaker,
     statsHelpers,
     filterRegistry
   };
-
   const queueHandler = new QueueHandler(options);
   const runningPlugins = await loadPlugins(
     pluginNames,

package/lib/support/fileUtil.js

@@ -0,0 +1,40 @@
+import { readdir } from 'node:fs/promises';
+import { existsSync } from 'node:fs';
+import path from 'node:path';
+
+export async function recursiveReaddir(dir, ignoreDirectories = false) {
+  const results = [];
+  const entries = await readdir(dir, { withFileTypes: true });
+
+  for (const entry of entries) {
+    const fullPath = path.join(dir, entry.name);
+
+    if (entry.isDirectory()) {
+      if (!ignoreDirectories) {
+        results.push(fullPath);
+      }
+      const subPaths = await recursiveReaddir(fullPath, ignoreDirectories);
+      results.push(...subPaths);
+    } else {
+      results.push(fullPath);
+    }
+  }
+
+  return results;
+}
+
+export function findUpSync(filenames, startDir = process.cwd()) {
+  let currentDir = startDir;
+
+  while (currentDir !== path.dirname(currentDir)) {
+    for (const filename of filenames) {
+      const filePath = path.resolve(currentDir, filename);
+      if (existsSync(filePath)) {
+        return filePath;
+      }
+    }
+    currentDir = path.dirname(currentDir);
+  }
+
+  return;
+}

package/lib/support/filterRegistry.js

@@ -1,4 +1,3 @@
-import clone from 'lodash.clonedeep';
 import { filterMetrics } from './metricsFilter.js';
 
 let filterForType = {};
@@ -34,7 +33,8 @@ export function filterMessage(message) {
     return message;
   }
 
-  const filteredMessage = clone(message);
-  filteredMessage.data = filterMetrics(filteredMessage.data, filterConfig);
-  return filteredMessage;
+  return {
+    ...message,
+    data: filterMetrics(message.data, filterConfig)
+  };
 }

package/lib/support/flattenMessage.js

@@ -1,6 +1,6 @@
 import { parse } from 'node:url';
-import intel from 'intel';
-const log = intel.getLogger('sitespeedio');
+import { getLogger } from '@sitespeed.io/log';
+const log = getLogger('sitespeedio');
 
 function joinNonEmpty(strings, delimeter) {
   return strings.filter(Boolean).join(delimeter);

package/lib/support/messageMaker.js

@@ -1,12 +1,12 @@
 import dayjs from 'dayjs';
 import merge from 'lodash.merge';
-import { v4 as makeUuid } from 'uuid';
+import { randomUUID } from 'node:crypto';
 
 export function messageMaker(source) {
   return {
     make(type, data, extras) {
       const timestamp = dayjs().format();
-      const uuid = makeUuid();
+      const uuid = randomUUID();
 
       return merge({ uuid, type, timestamp, source, data }, extras);
     }

package/lib/support/metricsFilter.js

@@ -1,10 +1,8 @@
-import isEmpty from 'lodash.isempty';
 import get from 'lodash.get';
 import set from 'lodash.set';
 import merge from 'lodash.merge';
-import reduce from 'lodash.reduce';
 
-import { toArray } from './util.js';
+import { toArray, isEmpty } from './util.js';
 
 function normalizePath(path) {
   if (path.endsWith('.*')) return path.slice(0, -2);
@@ -31,20 +29,15 @@ export function filterMetrics(json, metricPaths) {
     } else if (firstWildcard === 0) {
       const leafPath = path.slice(2);
 
-      reduce(
-        json,
-        (result, value, key) => {
-          if (typeof value === 'object') {
-            const leaf = this.filterMetrics(value, leafPath);
-
-            if (!isEmpty(leaf)) {
-              result[key] = leaf;
-            }
+      Object.entries(json).reduce((result, [key, value]) => {
+        if (typeof value === 'object') {
+          const leaf = this.filterMetrics(value, leafPath);
+          if (leaf && Object.keys(leaf).length > 0) {
+            result[key] = leaf;
           }
-          return result;
-        },
-        result
-      );
+        }
+        return result;
+      }, result);
     } else {
       let branchPath = path.slice(0, Math.max(0, firstWildcard));
       if (branchPath.endsWith('.')) branchPath = branchPath.slice(0, -1);

package/lib/support/osUtil.js

@@ -0,0 +1,36 @@
+import os from 'node:os';
+import { promises as fsPromises } from 'node:fs';
+
+export function osName() {
+  const platform = os.platform();
+  if (platform === 'win32') return 'Windows';
+  if (platform === 'darwin') return 'macOS';
+  if (platform === 'linux') return 'Linux';
+  return platform;
+}
+
+export async function getOS() {
+  if (os.platform() !== 'linux') {
+    return { dist: osName(), release: '' };
+  }
+
+  try {
+    const data = await fsPromises.readFile('/etc/os-release', 'utf8');
+    const lines = data.split('\n');
+    let dist = 'Linux';
+    let release = '';
+
+    for (const line of lines) {
+      if (line.startsWith('NAME=')) {
+        dist = line.split('=')[1].replaceAll('"', '');
+      } else if (line.startsWith('VERSION_ID=')) {
+        release = line.split('=')[1].replaceAll('"', '');
+      }
+    }
+
+    return { dist, release };
+  } catch {
+    // Return fallback values if reading fails
+    return { dist: 'Linux', release: '' };
+  }
+}

package/lib/support/util.js

@@ -21,3 +21,27 @@ export function throwIfMissing(options, keys, namespace) {
     );
   }
 }
+
+export function isEmpty(value) {
+  if (value === null) return true;
+
+  if (value === undefined) return true;
+
+  if (typeof value === 'boolean') return false;
+
+  if (typeof value === 'number') return false;
+
+  if (typeof value === 'string') return value.length === 0;
+
+  if (typeof value === 'function') return false;
+
+  if (Array.isArray(value)) return value.length === 0;
+
+  if (value instanceof Map || value instanceof Set) return value.size === 0;
+
+  if (typeof value === 'object') {
+    return Object.keys(value).length === 0;
+  }
+
+  return false;
+}