sitespeed.io 35.7.5 → 36.0.0-alpha.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (104) hide show
  1. package/.dockerignore +4 -1
  2. package/CHANGELOG.md +48 -0
  3. package/Dockerfile +10 -6
  4. package/Dockerfile-slim +9 -7
  5. package/LICENSE +1 -1
  6. package/bin/browsertimeWebPageReplay.js +1 -1
  7. package/lib/cli/cli.js +17 -25
  8. package/lib/core/logging.js +6 -98
  9. package/lib/core/queue.js +112 -0
  10. package/lib/core/queueHandler.js +20 -28
  11. package/lib/core/resultsStorage/pathToFolder.js +3 -3
  12. package/lib/core/resultsStorage/storageManager.js +7 -5
  13. package/lib/plugins/assets/index.js +1 -1
  14. package/lib/plugins/axe/axePostScript.cjs +4 -6
  15. package/lib/plugins/axe/index.js +2 -2
  16. package/lib/plugins/browsertime/analyzer.js +6 -7
  17. package/lib/plugins/browsertime/browsertimeAggregator.js +64 -42
  18. package/lib/plugins/browsertime/filmstrip.js +2 -2
  19. package/lib/plugins/browsertime/index.js +3 -3
  20. package/lib/plugins/budget/deprecatedVerify.js +2 -2
  21. package/lib/plugins/budget/index.js +2 -2
  22. package/lib/plugins/budget/json.js +2 -2
  23. package/lib/plugins/budget/junit.js +2 -2
  24. package/lib/plugins/budget/tap.js +2 -2
  25. package/lib/plugins/budget/verify.js +2 -2
  26. package/lib/plugins/coach/aggregator.js +7 -9
  27. package/lib/plugins/coach/index.js +2 -2
  28. package/lib/plugins/compare/helper.js +2 -2
  29. package/lib/plugins/compare/index.js +2 -2
  30. package/lib/plugins/crawler/index.js +2 -2
  31. package/lib/plugins/crux/index.js +2 -2
  32. package/lib/plugins/crux/send.js +2 -2
  33. package/lib/plugins/domains/aggregator.js +9 -11
  34. package/lib/plugins/domains/index.js +1 -1
  35. package/lib/plugins/gcs/index.js +17 -16
  36. package/lib/plugins/grafana/send-annotation.js +2 -2
  37. package/lib/plugins/graphite/data-generator.js +13 -17
  38. package/lib/plugins/graphite/index.js +4 -3
  39. package/lib/plugins/graphite/send-annotation.js +2 -2
  40. package/lib/plugins/graphite/sender.js +2 -2
  41. package/lib/plugins/html/dataCollector.js +8 -14
  42. package/lib/plugins/html/htmlBuilder.js +12 -10
  43. package/lib/plugins/html/index.js +2 -5
  44. package/lib/plugins/html/renderer.js +2 -2
  45. package/lib/plugins/html/setup/summaryBoxes.js +2 -2
  46. package/lib/plugins/html/templates/url/coach/technology.pug +10 -8
  47. package/lib/plugins/influxdb/data-generator.js +6 -8
  48. package/lib/plugins/influxdb/index.js +3 -4
  49. package/lib/plugins/influxdb/send-annotation.js +2 -2
  50. package/lib/plugins/influxdb/send-annotationV2.js +2 -2
  51. package/lib/plugins/lateststorer/index.js +1 -4
  52. package/lib/plugins/matrix/index.js +2 -2
  53. package/lib/plugins/matrix/send.js +2 -2
  54. package/lib/plugins/messagelogger/index.js +4 -3
  55. package/lib/plugins/pagexray/index.js +2 -2
  56. package/lib/plugins/pagexray/pagexrayAggregator.js +11 -10
  57. package/lib/plugins/remove/index.js +2 -2
  58. package/lib/plugins/s3/index.js +29 -28
  59. package/lib/plugins/s3/limit.js +34 -0
  60. package/lib/plugins/scp/index.js +6 -10
  61. package/lib/plugins/slack/index.js +20 -21
  62. package/lib/plugins/sustainable/index.js +35 -14
  63. package/lib/plugins/text/color.js +30 -0
  64. package/lib/plugins/text/textBuilder.js +50 -20
  65. package/lib/sitespeed.js +11 -15
  66. package/lib/support/fileUtil.js +40 -0
  67. package/lib/support/filterRegistry.js +4 -4
  68. package/lib/support/flattenMessage.js +2 -2
  69. package/lib/support/messageMaker.js +2 -2
  70. package/lib/support/metricsFilter.js +9 -16
  71. package/lib/support/osUtil.js +36 -0
  72. package/lib/support/util.js +24 -0
  73. package/npm-shrinkwrap.json +1103 -2595
  74. package/package.json +13 -31
  75. package/tools/postinstall.js +61 -0
  76. package/.github/CONTRIBUTING.md +0 -24
  77. package/.github/FUNDING.yml +0 -12
  78. package/.github/ISSUE_TEMPLATE/BUG_REPORT.yml +0 -65
  79. package/.github/ISSUE_TEMPLATE/FEATURE_IMPROVEMENT.yml +0 -15
  80. package/.github/ISSUE_TEMPLATE/QUESTION.yml +0 -15
  81. package/.github/PULL_REQUEST_TEMPLATE.md +0 -14
  82. package/.github/budget.json +0 -13
  83. package/.github/workflows/building-docker-autobuild.yml +0 -30
  84. package/.github/workflows/building-docker-release.yml +0 -76
  85. package/.github/workflows/crux-test.yml +0 -23
  86. package/.github/workflows/docker-scan.yml +0 -29
  87. package/.github/workflows/docker.yml +0 -39
  88. package/.github/workflows/linux.yml +0 -80
  89. package/.github/workflows/safari.yml +0 -30
  90. package/.github/workflows/sitespeed-io-action-example.yml +0 -22
  91. package/.github/workflows/unittests.yml +0 -41
  92. package/.github/workflows/windows.yml +0 -39
  93. package/.github/workflows/windowsFull.yml +0 -36
  94. package/.netlify +0 -1
  95. package/.spelling +0 -58
  96. package/Gemfile +0 -4
  97. package/Gemfile.lock +0 -53
  98. package/docs/README.md +0 -10
  99. package/lib/plugins/sustainable/data/url2green.json.gz +0 -0
  100. package/release/feed.js +0 -198
  101. package/release/friendlyNames.js +0 -9
  102. package/release/friendlyNamesBudget.js +0 -15
  103. package/wpr-record.log +0 -102
  104. package/wpr-replay.log +0 -96
package/.dockerignore CHANGED
@@ -4,4 +4,7 @@
4
4
  !package.json
5
5
  !LICENSE
6
6
  !npm-shrinkwrap.json
7
- !docker
7
+ !docker/adb
8
+ !docker/scripts
9
+ !tools/postinstall.js
10
+ !docker/webpagereplay/
package/CHANGELOG.md CHANGED
@@ -1,5 +1,53 @@
1
1
  # CHANGELOG - sitespeed.io (we use [semantic versioning](https://semver.org))
2
2
 
3
+ ## 36.0.0 - UNRELEASED
4
+
5
+ 36.0.0 will be released late January 2025.
6
+
7
+ ### Breaking
8
+ * Only download green2url data when you specifically ask for it [#4354](https://github.com/sitespeedio/sitespeed.io/pull/4354). To install you need to run `DOWNLOAD_URL2GREEN=true npm install sitespeed.io`. The green2url is also updated to use the latest availible data by late 2024. This saves 80 mb in default downloading.
9
+ * Make sure you can't run with both `--cpu` and `--collectProfileRun` since that do not make any sense [#4298](https://github.com/sitespeedio/sitespeed.io/pull/4298)
10
+ * Use correct name in Browsertime: userTimingAllowList instead of whitelist [#4346](https://github.com/sitespeedio/sitespeed.io/pull/4346).
11
+ * Plugins need to update the plugin dependency to @sitespeed.io/plugin 1.0.0 or higher.
12
+ * Replace intel (log) with sitespeed.io/log [#4381](https://github.com/sitespeedio/sitespeed.io/pull/4381). This remove the logToFile option in the cli. Instead of use that option, pipe the output to the file you want.
13
+
14
+ ### Added
15
+ * Update to Coach-core 8.1.1 [#4363](https://github.com/sitespeedio/sitespeed.io/pull/4363)
16
+ * Use the offical Slack plugin instead of node-slack [#4360](https://github.com/sitespeedio/sitespeed.io/pull/4360).
17
+ * Firefox 134 and NodeJS 22 in the Docker container [#4395](https://github.com/sitespeedio/sitespeed.io/pull/4395) and [#4396](https://github.com/sitespeedio/sitespeed.io/pull/4396).
18
+
19
+ ### Fixed
20
+ * Replace dependencies with local code:
21
+ * Replace lodash.forEach [#4378](https://github.com/sitespeedio/sitespeed.io/pull/4378).
22
+ * Replace recursive-readdir [#4377](https://github.com/sitespeedio/sitespeed.io/pull/4377).
23
+ * Replace cli-color [#4374](https://github.com/sitespeedio/sitespeed.io/pull/4374).
24
+ * Replace text-table [#4373](https://github.com/sitespeedio/sitespeed.io/pull/4373).
25
+ * Replace lodash.chunk [#4372](https://github.com/sitespeedio/sitespeed.io/pull/4372).
26
+ * Replace lodash.flatten [#4371](https://github.com/sitespeedio/sitespeed.io/pull/4371).
27
+ * Replace fs-extra [#4370](https://github.com/sitespeedio/sitespeed.io/pull/4370.)
28
+ * Replace uuid [#4369](https://github.com/sitespeedio/sitespeed.io/pull/4369).
29
+ * Replace lodash.clonedeep [#4388](https://github.com/sitespeedio/sitespeed.io/pull/4388).
30
+ * Remove unused lodash.pick [#4387](https://github.com/sitespeedio/sitespeed.io/pull/4387).
31
+ * Replace lodash.pullAll and lodash.union [#4386](https://github.com/sitespeedio/sitespeed.io/pull/4386).
32
+ * Replace find-up [#4385](https://github.com/sitespeedio/sitespeed.io/pull/4385).
33
+ * Replace getos and osname [#4384](https://github.com/sitespeedio/sitespeed.io/pull/4384).
34
+ * Replace p-limit [#4394](https://github.com/sitespeedio/sitespeed.io/pull/4394).
35
+ * Replace concurrent-queue [#4393](https://github.com/sitespeedio/sitespeed.io/pull/4393).
36
+ * Replace lodash.isEmpty [#4391](https://github.com/sitespeedio/sitespeed.io/pull/4391).
37
+ * Remove unused jstransformer-markdown-it [#4392](https://github.com/sitespeedio/sitespeed.io/pull/4392)
38
+
39
+ * Fix cli command: Use `--summaryDetail` (not summary-detail) [#4376](https://github.com/sitespeedio/sitespeed.io/pull/4376).
40
+ * Remove connectivity output from text since it was broken [#4375](https://github.com/sitespeedio/sitespeed.io/pull/4375).
41
+ * Upgrade to co2.js 0.16.4 [#4353](https://github.com/sitespeedio/sitespeed.io/pull/4353).
42
+ * Make sure co2 data is only read once at startup [#4352](https://github.com/sitespeedio/sitespeed.io/pull/4352).
43
+ * Making the slim container a little smaller [#4355](https://github.com/sitespeedio/sitespeed.io/pull/4355).
44
+ * Ugrade to google-cloud/storage-7.14.0 [#4361](https://github.com/sitespeedio/sitespeed.io/pull/4361).
45
+ * Upgrade to AWS client 3.717.0 [#4368](https://github.com/sitespeedio/sitespeed.io/pull/4368)
46
+ * Removed the webdriver manager in the Docker container [4390](https://github.com/sitespeedio/sitespeed.io/pull/4390). We don't use it but on MacOS Selenium still uses it to find the driver for Safari so we can only remove it in Docker.
47
+
48
+ ### Tech
49
+ * New GitHub actions that test uploading to S3, GCS and SCP.
50
+
3
51
  ## 35.7.5 - 2024-12-23
4
52
  ### Fixed
5
53
  * Update to faststat 0.0.7 [#4347](https://github.com/sitespeedio/sitespeed.io/pull/4347).
package/Dockerfile CHANGED
@@ -1,9 +1,10 @@
1
- FROM sitespeedio/webbrowsers:chrome-131.0-firefox-133.0-edge-131.0
1
+ FROM sitespeedio/webbrowsers:chrome-131.0-firefox-134.0-edge-131.0-b
2
2
 
3
3
  ARG TARGETPLATFORM=linux/amd64
4
4
 
5
- ENV SITESPEED_IO_BROWSERTIME__XVFB true
6
- ENV SITESPEED_IO_BROWSERTIME__DOCKER true
5
+ ENV SITESPEED_IO_BROWSERTIME__XVFB=true
6
+ ENV SITESPEED_IO_BROWSERTIME__DOCKER=true
7
+ ENV PYTHON=python3
7
8
 
8
9
  COPY docker/webpagereplay/$TARGETPLATFORM/wpr /usr/local/bin/
9
10
  COPY docker/webpagereplay/wpr_cert.pem /webpagereplay/certs/
@@ -11,7 +12,6 @@ COPY docker/webpagereplay/wpr_key.pem /webpagereplay/certs/
11
12
  COPY docker/webpagereplay/deterministic.js /webpagereplay/scripts/deterministic.js
12
13
  COPY docker/webpagereplay/LICENSE /webpagereplay/
13
14
 
14
-
15
15
  RUN sudo apt-get update && sudo apt-get install libnss3-tools python2 \
16
16
  net-tools \
17
17
  build-essential \
@@ -28,8 +28,12 @@ WORKDIR /usr/src/app
28
28
 
29
29
  COPY package.json /usr/src/app/
30
30
  COPY npm-shrinkwrap.json /usr/src/app/
31
- RUN npm install --production && npm cache clean --force
32
- COPY . /usr/src/app
31
+ COPY tools/postinstall.js /usr/src/app/tools/postinstall.js
32
+ RUN npm install --production && npm cache clean --force
33
+
34
+ COPY ./bin/ /usr/src/app/bin/
35
+ COPY ./lib/ /usr/src/app/lib/
36
+ RUN rm -fR /usr/src/app/node_modules/selenium-webdriver/bin
33
37
 
34
38
  COPY docker/scripts/start.sh /start.sh
35
39
 
package/Dockerfile-slim CHANGED
@@ -1,12 +1,12 @@
1
- FROM node:20.9.0-bookworm-slim
1
+ FROM node:22.13.0-bookworm-slim
2
2
 
3
3
  ARG TARGETPLATFORM=linux/amd64
4
4
 
5
- ENV SITESPEED_IO_BROWSERTIME__DOCKER true
6
- ENV SITESPEED_IO_BROWSERTIME__VIDEO false
7
- ENV SITESPEED_IO_BROWSERTIME__BROWSER firefox
8
- ENV SITESPEED_IO_BROWSERTIME__VISUAL_METRICS false
9
- ENV SITESPEED_IO_BROWSERTIME__HEADLESS true
5
+ ENV SITESPEED_IO_BROWSERTIME__DOCKER=true
6
+ ENV SITESPEED_IO_BROWSERTIME__VIDEO=false
7
+ ENV SITESPEED_IO_BROWSERTIME__BROWSER=firefox
8
+ ENV SITESPEED_IO_BROWSERTIME__VISUAL_METRICS=false
9
+ ENV SITESPEED_IO_BROWSERTIME__HEADLESS=true
10
10
 
11
11
  ENV PATH="/usr/local/bin:${PATH}"
12
12
 
@@ -21,7 +21,9 @@ RUN echo "deb http://deb.debian.org/debian/ unstable main contrib non-free" >> /
21
21
  RUN mkdir -p /usr/src/app
22
22
  WORKDIR /usr/src/app
23
23
  COPY . /usr/src/app
24
- RUN CHROMEDRIVER_SKIP_DOWNLOAD=true EGDEDRIVER_SKIP_DOWNLOAD=true npm install --production && npm cache clean --force && npm uninstall npm -g
24
+ COPY tools/postinstall.js /usr/src/app/tools/postinstall.js
25
+ RUN CHROMEDRIVER_SKIP_DOWNLOAD=true EGDEDRIVER_SKIP_DOWNLOAD=true npm install --omit=dev --omit=optional && npm cache clean --force && npm uninstall npm npx -g && rm -fR /usr/src/app/node_modules/selenium-webdriver/bin
26
+
25
27
  WORKDIR /usr/src/app
26
28
  COPY docker/scripts/start-slim.sh /start.sh
27
29
 
package/LICENSE CHANGED
@@ -1,6 +1,6 @@
1
1
  The MIT License (MIT)
2
2
 
3
- Copyright (c) 2012-2023 Peter Hedenskog
3
+ Copyright (c) 2012-2025 Peter Hedenskog
4
4
 
5
5
  Permission is hereby granted, free of charge, to any person obtaining a copy
6
6
  of this software and associated documentation files (the "Software"), to deal
@@ -8,10 +8,10 @@ import get from 'lodash.get';
8
8
  import yargs from 'yargs';
9
9
  import { hideBin } from 'yargs/helpers';
10
10
 
11
- import { findUpSync } from 'find-up';
12
11
  import { BrowsertimeEngine, configureLogging } from 'browsertime';
13
12
 
14
13
  import { getURLs } from '../lib/cli/util.js';
14
+ import { findUpSync } from '../lib/support/fileUtil.js';
15
15
 
16
16
  import {config as browsertimeConfig} from '../lib/plugins/browsertime/index.js';
17
17
 
package/lib/cli/cli.js CHANGED
@@ -6,10 +6,8 @@ import { readFileSync, statSync } from 'node:fs';
6
6
  import yargs from 'yargs';
7
7
  import { hideBin } from 'yargs/helpers';
8
8
  import merge from 'lodash.merge';
9
- import reduce from 'lodash.reduce';
10
9
  import set from 'lodash.set';
11
10
  import get from 'lodash.get';
12
- import { findUpSync } from 'find-up';
13
11
 
14
12
  import { getURLs, getAliases } from './util.js';
15
13
  import { toArray } from '../support/util.js';
@@ -19,6 +17,7 @@ import { config as metricsConfig } from '../plugins/metrics/index.js';
19
17
  import { config as slackConfig } from '../plugins/slack/index.js';
20
18
  import { config as htmlConfig } from '../plugins/html/index.js';
21
19
  import { messageTypes as matrixMessageTypes } from '../plugins/matrix/index.js';
20
+ import { findUpSync } from '../support/fileUtil.js';
22
21
 
23
22
  const metricList = Object.keys(friendlynames);
24
23
  const require = createRequire(import.meta.url);
@@ -98,6 +97,10 @@ function validateInput(argv) {
98
97
  return 'Error: Getting CrUx data do not work running in multi mode.';
99
98
  }
100
99
 
100
+ if (argv.browsertime.cpu && argv.browsertime.enableProfileRun) {
101
+ return 'Error: Use either --cpu or --enableProfileRun. Profile run will run one extra iteration to collect cpu/trace data.';
102
+ }
103
+
101
104
  if (
102
105
  argv.urlAlias &&
103
106
  argv._ &&
@@ -573,13 +576,13 @@ export async function parseCommandLine() {
573
576
  alias: 'enableProfileRun',
574
577
  type: 'boolean',
575
578
  describe:
576
- 'Make one extra run that collects the profiling trace log (no other metrics is collected). For Chrome it will collect the timeline trace, for Firefox it will get the Geckoprofiler trace. This means you do not need to get the trace for all runs and can skip the overhead it produces.'
579
+ 'Make one extra run that collects the profiling trace log (no other metrics is collected). For Chrome it will collect the timeline trace, for Firefox it will get the Geckoprofiler trace. This means you do not need to get the trace for all runs and can skip the overhead it produces. You should not run this together with --cpu since that will get a trace for every iteration.'
577
580
  })
578
581
  .option('browsertime.enableVideoRun', {
579
582
  alias: 'enableVideoRun',
580
583
  type: 'boolean',
581
584
  describe:
582
- 'Make one extra run that collects video and visual metrics. This means you can do your runs with --visualMetrics true --video false --enableVideoRun true to collect visual metrics from all runs and save a video from the profile/video run. If you run it together with --enableProfileRun it will also collect profiling trace.'
585
+ 'Make one extra run that collects video and visual metrics. This means you can do your runs with --visualMetrics true --video false --enableVideoRun true to collect visual metrics from all runs and save a video from the profile/video run. If you run it together with --enableProfileRun it will also collect profiling race.'
583
586
  })
584
587
  .option('browsertime.videoParams.filmstripFullSize', {
585
588
  alias: 'videoParams.filmstripFullSize',
@@ -616,10 +619,10 @@ export async function parseCommandLine() {
616
619
  'Show all screenshots in the filmstrip, independent if they have changed or not.',
617
620
  group: 'Filmstrip'
618
621
  })
619
- .option('browsertime.userTimingWhitelist', {
620
- alias: 'userTimingWhitelist',
622
+ .option('browsertime.userTimingAllowList', {
623
+ alias: 'userTimingAllowList',
621
624
  describe:
622
- 'This option takes a regex that will whitelist which userTimings to capture in the results. All userTimings are captured by default. T',
625
+ 'This option takes a regex that will whitelist which userTimings to capture in the results. All userTimings are captured by default.',
623
626
  group: 'Browser'
624
627
  })
625
628
  .option('axe.enable', {
@@ -1831,7 +1834,7 @@ export async function parseCommandLine() {
1831
1834
  type: 'boolean',
1832
1835
  group: 'Text'
1833
1836
  })
1834
- .option('summary-detail', {
1837
+ .option('summaryDetail', {
1835
1838
  describe: 'Show longer text summary to stdout',
1836
1839
  default: false,
1837
1840
  type: 'boolean',
@@ -2036,12 +2039,6 @@ export async function parseCommandLine() {
2036
2039
  default: false,
2037
2040
  type: 'boolean'
2038
2041
  })
2039
- .option('logToFile', {
2040
- describe:
2041
- 'Store the log for your run into a file in logs/sitespeed.io.log',
2042
- default: false,
2043
- type: 'boolean'
2044
- })
2045
2042
  .option('useHash', {
2046
2043
  describe:
2047
2044
  'If your site uses # for URLs and # give you unique URLs you need to turn on useHash. By default is it turned off, meaning URLs with hash and without hash are treated as the same URL',
@@ -2138,14 +2135,10 @@ export async function parseCommandLine() {
2138
2135
  argv = parsed.argv;
2139
2136
 
2140
2137
  // aliases are long options -> short option
2141
- const aliasLookup = reduce(
2142
- aliases,
2143
- (lookup, value, key) => {
2144
- lookup.set(value[0], key);
2145
- return lookup;
2146
- },
2147
- new Map()
2148
- );
2138
+ const aliasLookup = new Map();
2139
+ for (const [key, value] of Object.entries(aliases)) {
2140
+ aliasLookup.set(value[0], key);
2141
+ }
2149
2142
 
2150
2143
  let explicitOptions = yargs(hideBin(process.argv)).argv;
2151
2144
 
@@ -2154,9 +2147,8 @@ export async function parseCommandLine() {
2154
2147
  yargsInstance.getOptions().configObjects[0]
2155
2148
  );
2156
2149
 
2157
- explicitOptions = reduce(
2158
- explicitOptions,
2159
- (result, value, key) => {
2150
+ explicitOptions = Object.entries(explicitOptions).reduce(
2151
+ (result, [key, value]) => {
2160
2152
  if (aliasLookup.has(key)) {
2161
2153
  const fullKey = aliasLookup.get(key);
2162
2154
  result = set(result, fullKey, value);
@@ -1,100 +1,8 @@
1
- import intel from 'intel';
2
- import { createWriteStream } from 'node:fs';
3
- import { inherits } from 'node:util';
4
- const {
5
- INFO,
6
- DEBUG,
7
- VERBOSE,
8
- TRACE,
9
- NONE,
10
- basicConfig,
11
- Logger,
12
- Handler,
13
- Formatter
14
- } = intel;
1
+ import { configureLog } from '@sitespeed.io/log';
15
2
 
16
- // FileHandler isn't exposed in Intel when we moved to ESM.
17
- // To fix that for now we just use the same code as Intel.
18
-
19
- function StreamHandler(options) {
20
- options = options || {};
21
- if (!options.stream) {
22
- options = { stream: options };
23
- }
24
- Handler.call(this, options);
25
- this._stream = options.stream;
26
- }
27
-
28
- inherits(StreamHandler, Handler);
29
-
30
- StreamHandler.prototype.emit = function streamEmit(record) {
31
- this._stream.write(this.format(record) + '\n');
32
- };
33
-
34
- function FileHandler(options) {
35
- if (typeof options === 'string') {
36
- options = { file: options };
37
- }
38
- this._file = options.file;
39
-
40
- options.stream = this._open();
41
- StreamHandler.call(this, options);
42
- }
43
- inherits(FileHandler, StreamHandler);
44
-
45
- FileHandler.prototype._open = function open() {
46
- return createWriteStream(this._file, { flags: 'a' });
47
- };
48
-
49
- export function configure(options, logDir) {
50
- options = options || {};
51
-
52
- let level = INFO;
53
-
54
- switch (options.verbose) {
55
- case 1: {
56
- level = DEBUG;
57
- break;
58
- }
59
- case 2: {
60
- level = VERBOSE;
61
- break;
62
- }
63
- case 3: {
64
- level = TRACE;
65
- break;
66
- }
67
- default: {
68
- break;
69
- }
70
- }
71
-
72
- if (options.silent) {
73
- level = NONE;
74
- }
75
-
76
- if (level === INFO) {
77
- basicConfig({
78
- format: '[%(date)s] %(levelname)s: %(message)s',
79
- level: level
80
- });
81
- } else {
82
- basicConfig({
83
- format: '[%(date)s] %(levelname)s: [%(name)s] %(message)s',
84
- level: level
85
- });
86
- }
87
-
88
- if (options.logToFile) {
89
- let logger = new Logger();
90
- logger.addHandler(
91
- new FileHandler({
92
- file: logDir + '/sitespeed.io.log',
93
- formatter: new Formatter({
94
- format: '[%(date)s] %(levelname)s: [%(name)s] %(message)s',
95
- level: level
96
- })
97
- })
98
- );
99
- }
3
+ export function configure(options = {}) {
4
+ configureLog({
5
+ verbose: options.verbose ?? 0,
6
+ silent: options.silent ?? false
7
+ });
100
8
  }
@@ -0,0 +1,112 @@
1
+ // Simplified implementation of concurrent-queue
2
+ // https://www.npmjs.com/package/concurrent-queue
3
+
4
+ export function createQueue() {
5
+ let concurrency = Number.POSITIVE_INFINITY;
6
+ let processor;
7
+
8
+ const tasks = [];
9
+ let runningCount = 0;
10
+
11
+ const enqueuedCallbacks = [];
12
+ const processingStartedCallbacks = [];
13
+ const processingEndedCallbacks = [];
14
+ const drainedCallbacks = [];
15
+
16
+ /**
17
+ * We will trigger drained callbacks when:
18
+ * tasks.length === 0 and runningCount === 0
19
+ */
20
+ function checkDrained() {
21
+ if (tasks.length === 0 && runningCount === 0) {
22
+ for (const cb of drainedCallbacks) {
23
+ cb();
24
+ }
25
+ }
26
+ }
27
+
28
+ /**
29
+ * Attempt to start processing more tasks if we have
30
+ * capacity (runningCount < concurrency).
31
+ */
32
+ function tryProcessNext() {
33
+ while (tasks.length > 0 && runningCount < concurrency) {
34
+ const item = tasks.shift();
35
+ runningCount++;
36
+
37
+ for (const cb of processingStartedCallbacks) {
38
+ cb({ item });
39
+ }
40
+
41
+ const promise = Promise.resolve(processor(item));
42
+
43
+ promise
44
+ .then(() => {
45
+ // Fire processingEnded callbacks
46
+ for (const cb of processingEndedCallbacks) {
47
+ cb({ item, err: undefined });
48
+ }
49
+ })
50
+ .catch(error => {
51
+ // Fire processingEnded callbacks with an error
52
+ for (const cb of processingEndedCallbacks) {
53
+ cb({ item, err: error });
54
+ }
55
+ })
56
+ .finally(() => {
57
+ runningCount--;
58
+ checkDrained();
59
+ tryProcessNext();
60
+ });
61
+ }
62
+ }
63
+
64
+ const queue = function enqueue(item) {
65
+ for (const cb of enqueuedCallbacks) {
66
+ cb({ item });
67
+ }
68
+ tasks.push(item);
69
+
70
+ tryProcessNext();
71
+ };
72
+
73
+ queue.limit = options => {
74
+ if (options && typeof options.concurrency === 'number') {
75
+ concurrency = options.concurrency;
76
+ }
77
+ return queue;
78
+ };
79
+
80
+ queue.process = fn => {
81
+ processor = fn;
82
+ return queue;
83
+ };
84
+
85
+ queue.enqueued = callback => {
86
+ enqueuedCallbacks.push(callback);
87
+ return queue;
88
+ };
89
+
90
+ queue.processingStarted = callback => {
91
+ processingStartedCallbacks.push(callback);
92
+ return queue;
93
+ };
94
+
95
+ queue.processingEnded = callback => {
96
+ processingEndedCallbacks.push(callback);
97
+ return queue;
98
+ };
99
+
100
+ queue.drained = callback => {
101
+ drainedCallbacks.push(callback);
102
+ return queue;
103
+ };
104
+
105
+ Object.defineProperty(queue, 'isDrained', {
106
+ get() {
107
+ return tasks.length === 0 && runningCount === 0;
108
+ }
109
+ });
110
+
111
+ return queue;
112
+ }
@@ -1,17 +1,14 @@
1
- /* eslint no-console:0 */
2
-
3
- import cq from 'concurrent-queue';
4
- import intel from 'intel';
5
-
1
+ import { getLogger } from '@sitespeed.io/log';
6
2
  import { messageMaker } from '../support/messageMaker.js';
7
3
  import {
8
4
  registerQueueTime,
9
5
  registerProcessingTime,
10
6
  generateStatistics
11
7
  } from './queueStatistics.js';
8
+ import { createQueue } from './queue.js';
12
9
 
13
10
  const make = messageMaker('queueHandler').make;
14
- const log = intel.getLogger('sitespeedio.queuehandler');
11
+ const log = getLogger('sitespeedio.queuehandler');
15
12
 
16
13
  function shortenData(key, value) {
17
14
  if (key === 'data') {
@@ -103,36 +100,32 @@ export class QueueHandler {
103
100
  this.queues = plugins
104
101
  .filter(plugin => plugin.processMessage)
105
102
  .map(plugin => {
106
- const concurrency = plugin.concurrency || Number.POSITIVE_INFINITY;
107
- const queue = cq().limit({ concurrency });
103
+ const concurrency = plugin.concurrency ?? Number.POSITIVE_INFINITY;
104
+ // Create a queue with that concurrency
105
+ const queue = createQueue().limit({ concurrency });
108
106
 
109
107
  queue.plugin = plugin;
110
108
 
111
- const messageWaitingStart = {},
112
- messageProcessingStart = {};
109
+ const messageWaitingStart = {};
110
+ const messageProcessingStart = {};
113
111
 
114
112
  queue.enqueued(object => {
115
- const message = object.item;
113
+ const { item: message } = object;
116
114
  messageWaitingStart[message.uuid] = process.hrtime();
117
115
  });
118
116
 
119
117
  queue.processingStarted(object => {
120
- const message = object.item;
121
-
118
+ const { item: message } = object;
122
119
  const waitingDuration = process.hrtime(
123
- messageWaitingStart[message.uuid]
124
- ),
125
- waitingNanos = waitingDuration[0] * 1e9 + waitingDuration[1];
126
-
120
+ messageWaitingStart[message.uuid]
121
+ );
122
+ const waitingNanos = waitingDuration[0] * 1e9 + waitingDuration[1];
127
123
  registerQueueTime(message, queue.plugin, waitingNanos);
128
-
129
124
  messageProcessingStart[message.uuid] = process.hrtime();
130
125
  });
131
126
 
132
- // FIXME handle rejections (i.e. failures while processing messages) properly
133
127
  queue.processingEnded(object => {
134
- const message = object.item;
135
- const error = object.err;
128
+ const { item: message, err: error } = object;
136
129
  if (error) {
137
130
  let rejectionMessage =
138
131
  'Rejected ' +
@@ -140,9 +133,9 @@ export class QueueHandler {
140
133
  ' for plugin: ' +
141
134
  plugin.getName();
142
135
 
143
- if (message && message.url)
144
- rejectionMessage += ', url: ' + message.url;
145
-
136
+ if (message?.url) {
137
+ rejectionMessage += `, url: ${message.url}`;
138
+ }
146
139
  if (error.stack) {
147
140
  log.error(error.stack);
148
141
  }
@@ -154,7 +147,6 @@ export class QueueHandler {
154
147
  );
155
148
  const processingNanos =
156
149
  processingDuration[0] * 1e9 + processingDuration[1];
157
-
158
150
  registerProcessingTime(message, queue.plugin, processingNanos);
159
151
  });
160
152
 
@@ -221,9 +213,9 @@ export class QueueHandler {
221
213
  }
222
214
 
223
215
  async startProcessingQueues() {
224
- for (let item of this.queues) {
225
- const queue = item.queue,
226
- plugin = item.plugin;
216
+ for (const item of this.queues) {
217
+ const { queue, plugin } = item;
218
+ // For each queue, set up the processor that handles messages
227
219
  queue.process(message =>
228
220
  Promise.resolve(plugin.processMessage(message, this))
229
221
  );
@@ -1,10 +1,10 @@
1
1
  import { parse } from 'node:url';
2
2
  import { createHash } from 'node:crypto';
3
3
 
4
- import isEmpty from 'lodash.isempty';
5
- import intel from 'intel';
4
+ import { getLogger } from '@sitespeed.io/log';
6
5
 
7
- const log = intel.getLogger('sitespeedio.file');
6
+ import { isEmpty } from '../../support/util.js';
7
+ const log = getLogger('sitespeedio.file');
8
8
 
9
9
  function toSafeKey(key) {
10
10
  // U+2013 : EN DASH – as used on https://en.wikipedia.org/wiki/2019–20_coronavirus_pandemic
@@ -9,12 +9,12 @@ import {
9
9
  writeFile as _writeFile
10
10
  } from 'node:fs';
11
11
 
12
- import { copy } from 'fs-extra/esm';
13
- import intel from 'intel';
12
+ import { cp } from 'node:fs/promises';
13
+ import { getLogger } from '@sitespeed.io/log';
14
14
 
15
15
  import { pathToFolder } from './pathToFolder.js';
16
16
 
17
- const log = intel.getLogger('sitespeedio.storageManager');
17
+ const log = getLogger('sitespeedio.storageManager');
18
18
  const mkdir = promisify(_mkdir);
19
19
  const readdir = promisify(_readdir);
20
20
  const lstat = promisify(_lstat);
@@ -69,10 +69,12 @@ export function storageManager(baseDir, storagePathPrefix, options) {
69
69
  return storagePathPrefix;
70
70
  },
71
71
  copyToResultDir(filename) {
72
- return this.createDirectory().then(dir => copy(filename, dir));
72
+ return this.createDirectory().then(dir =>
73
+ cp(filename, dir, { recursive: true })
74
+ );
73
75
  },
74
76
  copyFileToDir(filename, dir) {
75
- return copy(filename, dir);
77
+ return cp(filename, dir, { recursive: true });
76
78
  },
77
79
  // TODO is missing alias
78
80
  removeDataForUrl(url) {
@@ -1,6 +1,6 @@
1
- import isEmpty from 'lodash.isempty';
2
1
  import { SitespeedioPlugin } from '@sitespeed.io/plugin';
3
2
  import { AssetsAggregator } from './aggregator.js';
3
+ import { isEmpty } from '../../support/util.js';
4
4
  const DEFAULT_METRICS_LARGEST_ASSETS = ['image.0.transferSize'];
5
5
 
6
6
  export default class AssetsPlugin extends SitespeedioPlugin {