sitespeed.io 35.7.5 → 36.0.0-alpha.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.dockerignore +4 -1
- package/CHANGELOG.md +48 -0
- package/Dockerfile +10 -6
- package/Dockerfile-slim +9 -7
- package/LICENSE +1 -1
- package/bin/browsertimeWebPageReplay.js +1 -1
- package/lib/cli/cli.js +17 -25
- package/lib/core/logging.js +6 -98
- package/lib/core/queue.js +112 -0
- package/lib/core/queueHandler.js +20 -28
- package/lib/core/resultsStorage/pathToFolder.js +3 -3
- package/lib/core/resultsStorage/storageManager.js +7 -5
- package/lib/plugins/assets/index.js +1 -1
- package/lib/plugins/axe/axePostScript.cjs +4 -6
- package/lib/plugins/axe/index.js +2 -2
- package/lib/plugins/browsertime/analyzer.js +6 -7
- package/lib/plugins/browsertime/browsertimeAggregator.js +64 -42
- package/lib/plugins/browsertime/filmstrip.js +2 -2
- package/lib/plugins/browsertime/index.js +3 -3
- package/lib/plugins/budget/deprecatedVerify.js +2 -2
- package/lib/plugins/budget/index.js +2 -2
- package/lib/plugins/budget/json.js +2 -2
- package/lib/plugins/budget/junit.js +2 -2
- package/lib/plugins/budget/tap.js +2 -2
- package/lib/plugins/budget/verify.js +2 -2
- package/lib/plugins/coach/aggregator.js +7 -9
- package/lib/plugins/coach/index.js +2 -2
- package/lib/plugins/compare/helper.js +2 -2
- package/lib/plugins/compare/index.js +2 -2
- package/lib/plugins/crawler/index.js +2 -2
- package/lib/plugins/crux/index.js +2 -2
- package/lib/plugins/crux/send.js +2 -2
- package/lib/plugins/domains/aggregator.js +9 -11
- package/lib/plugins/domains/index.js +1 -1
- package/lib/plugins/gcs/index.js +17 -16
- package/lib/plugins/grafana/send-annotation.js +2 -2
- package/lib/plugins/graphite/data-generator.js +13 -17
- package/lib/plugins/graphite/index.js +4 -3
- package/lib/plugins/graphite/send-annotation.js +2 -2
- package/lib/plugins/graphite/sender.js +2 -2
- package/lib/plugins/html/dataCollector.js +8 -14
- package/lib/plugins/html/htmlBuilder.js +12 -10
- package/lib/plugins/html/index.js +2 -5
- package/lib/plugins/html/renderer.js +2 -2
- package/lib/plugins/html/setup/summaryBoxes.js +2 -2
- package/lib/plugins/html/templates/url/coach/technology.pug +10 -8
- package/lib/plugins/influxdb/data-generator.js +6 -8
- package/lib/plugins/influxdb/index.js +3 -4
- package/lib/plugins/influxdb/send-annotation.js +2 -2
- package/lib/plugins/influxdb/send-annotationV2.js +2 -2
- package/lib/plugins/lateststorer/index.js +1 -4
- package/lib/plugins/matrix/index.js +2 -2
- package/lib/plugins/matrix/send.js +2 -2
- package/lib/plugins/messagelogger/index.js +4 -3
- package/lib/plugins/pagexray/index.js +2 -2
- package/lib/plugins/pagexray/pagexrayAggregator.js +11 -10
- package/lib/plugins/remove/index.js +2 -2
- package/lib/plugins/s3/index.js +29 -28
- package/lib/plugins/s3/limit.js +34 -0
- package/lib/plugins/scp/index.js +6 -10
- package/lib/plugins/slack/index.js +20 -21
- package/lib/plugins/sustainable/index.js +35 -14
- package/lib/plugins/text/color.js +30 -0
- package/lib/plugins/text/textBuilder.js +50 -20
- package/lib/sitespeed.js +11 -15
- package/lib/support/fileUtil.js +40 -0
- package/lib/support/filterRegistry.js +4 -4
- package/lib/support/flattenMessage.js +2 -2
- package/lib/support/messageMaker.js +2 -2
- package/lib/support/metricsFilter.js +9 -16
- package/lib/support/osUtil.js +36 -0
- package/lib/support/util.js +24 -0
- package/npm-shrinkwrap.json +1103 -2595
- package/package.json +13 -31
- package/tools/postinstall.js +61 -0
- package/.github/CONTRIBUTING.md +0 -24
- package/.github/FUNDING.yml +0 -12
- package/.github/ISSUE_TEMPLATE/BUG_REPORT.yml +0 -65
- package/.github/ISSUE_TEMPLATE/FEATURE_IMPROVEMENT.yml +0 -15
- package/.github/ISSUE_TEMPLATE/QUESTION.yml +0 -15
- package/.github/PULL_REQUEST_TEMPLATE.md +0 -14
- package/.github/budget.json +0 -13
- package/.github/workflows/building-docker-autobuild.yml +0 -30
- package/.github/workflows/building-docker-release.yml +0 -76
- package/.github/workflows/crux-test.yml +0 -23
- package/.github/workflows/docker-scan.yml +0 -29
- package/.github/workflows/docker.yml +0 -39
- package/.github/workflows/linux.yml +0 -80
- package/.github/workflows/safari.yml +0 -30
- package/.github/workflows/sitespeed-io-action-example.yml +0 -22
- package/.github/workflows/unittests.yml +0 -41
- package/.github/workflows/windows.yml +0 -39
- package/.github/workflows/windowsFull.yml +0 -36
- package/.netlify +0 -1
- package/.spelling +0 -58
- package/Gemfile +0 -4
- package/Gemfile.lock +0 -53
- package/docs/README.md +0 -10
- package/lib/plugins/sustainable/data/url2green.json.gz +0 -0
- package/release/feed.js +0 -198
- package/release/friendlyNames.js +0 -9
- package/release/friendlyNamesBudget.js +0 -15
- package/wpr-record.log +0 -102
- package/wpr-replay.log +0 -96
package/.dockerignore
CHANGED
package/CHANGELOG.md
CHANGED
|
@@ -1,5 +1,53 @@
|
|
|
1
1
|
# CHANGELOG - sitespeed.io (we use [semantic versioning](https://semver.org))
|
|
2
2
|
|
|
3
|
+
## 36.0.0 - UNRELEASED
|
|
4
|
+
|
|
5
|
+
36.0.0 will be released late January 2025.
|
|
6
|
+
|
|
7
|
+
### Breaking
|
|
8
|
+
* Only download green2url data when you specifically ask for it [#4354](https://github.com/sitespeedio/sitespeed.io/pull/4354). To install you need to run `DOWNLOAD_URL2GREEN=true npm install sitespeed.io`. The green2url is also updated to use the latest availible data by late 2024. This saves 80 mb in default downloading.
|
|
9
|
+
* Make sure you can't run with both `--cpu` and `--collectProfileRun` since that do not make any sense [#4298](https://github.com/sitespeedio/sitespeed.io/pull/4298)
|
|
10
|
+
* Use correct name in Browsertime: userTimingAllowList instead of whitelist [#4346](https://github.com/sitespeedio/sitespeed.io/pull/4346).
|
|
11
|
+
* Plugins need to update the plugin dependency to @sitespeed.io/plugin 1.0.0 or higher.
|
|
12
|
+
* Replace intel (log) with sitespeed.io/log [#4381](https://github.com/sitespeedio/sitespeed.io/pull/4381). This remove the logToFile option in the cli. Instead of use that option, pipe the output to the file you want.
|
|
13
|
+
|
|
14
|
+
### Added
|
|
15
|
+
* Update to Coach-core 8.1.1 [#4363](https://github.com/sitespeedio/sitespeed.io/pull/4363)
|
|
16
|
+
* Use the offical Slack plugin instead of node-slack [#4360](https://github.com/sitespeedio/sitespeed.io/pull/4360).
|
|
17
|
+
* Firefox 134 and NodeJS 22 in the Docker container [#4395](https://github.com/sitespeedio/sitespeed.io/pull/4395) and [#4396](https://github.com/sitespeedio/sitespeed.io/pull/4396).
|
|
18
|
+
|
|
19
|
+
### Fixed
|
|
20
|
+
* Replace dependencies with local code:
|
|
21
|
+
* Replace lodash.forEach [#4378](https://github.com/sitespeedio/sitespeed.io/pull/4378).
|
|
22
|
+
* Replace recursive-readdir [#4377](https://github.com/sitespeedio/sitespeed.io/pull/4377).
|
|
23
|
+
* Replace cli-color [#4374](https://github.com/sitespeedio/sitespeed.io/pull/4374).
|
|
24
|
+
* Replace text-table [#4373](https://github.com/sitespeedio/sitespeed.io/pull/4373).
|
|
25
|
+
* Replace lodash.chunk [#4372](https://github.com/sitespeedio/sitespeed.io/pull/4372).
|
|
26
|
+
* Replace lodash.flatten [#4371](https://github.com/sitespeedio/sitespeed.io/pull/4371).
|
|
27
|
+
* Replace fs-extra [#4370](https://github.com/sitespeedio/sitespeed.io/pull/4370.)
|
|
28
|
+
* Replace uuid [#4369](https://github.com/sitespeedio/sitespeed.io/pull/4369).
|
|
29
|
+
* Replace lodash.clonedeep [#4388](https://github.com/sitespeedio/sitespeed.io/pull/4388).
|
|
30
|
+
* Remove unused lodash.pick [#4387](https://github.com/sitespeedio/sitespeed.io/pull/4387).
|
|
31
|
+
* Replace lodash.pullAll and lodash.union [#4386](https://github.com/sitespeedio/sitespeed.io/pull/4386).
|
|
32
|
+
* Replace find-up [#4385](https://github.com/sitespeedio/sitespeed.io/pull/4385).
|
|
33
|
+
* Replace getos and osname [#4384](https://github.com/sitespeedio/sitespeed.io/pull/4384).
|
|
34
|
+
* Replace p-limit [#4394](https://github.com/sitespeedio/sitespeed.io/pull/4394).
|
|
35
|
+
* Replace concurrent-queue [#4393](https://github.com/sitespeedio/sitespeed.io/pull/4393).
|
|
36
|
+
* Replace lodash.isEmpty [#4391](https://github.com/sitespeedio/sitespeed.io/pull/4391).
|
|
37
|
+
* Remove unused jstransformer-markdown-it [#4392](https://github.com/sitespeedio/sitespeed.io/pull/4392)
|
|
38
|
+
|
|
39
|
+
* Fix cli command: Use `--summaryDetail` (not summary-detail) [#4376](https://github.com/sitespeedio/sitespeed.io/pull/4376).
|
|
40
|
+
* Remove connectivity output from text since it was broken [#4375](https://github.com/sitespeedio/sitespeed.io/pull/4375).
|
|
41
|
+
* Upgrade to co2.js 0.16.4 [#4353](https://github.com/sitespeedio/sitespeed.io/pull/4353).
|
|
42
|
+
* Make sure co2 data is only read once at startup [#4352](https://github.com/sitespeedio/sitespeed.io/pull/4352).
|
|
43
|
+
* Making the slim container a little smaller [#4355](https://github.com/sitespeedio/sitespeed.io/pull/4355).
|
|
44
|
+
* Ugrade to google-cloud/storage-7.14.0 [#4361](https://github.com/sitespeedio/sitespeed.io/pull/4361).
|
|
45
|
+
* Upgrade to AWS client 3.717.0 [#4368](https://github.com/sitespeedio/sitespeed.io/pull/4368)
|
|
46
|
+
* Removed the webdriver manager in the Docker container [4390](https://github.com/sitespeedio/sitespeed.io/pull/4390). We don't use it but on MacOS Selenium still uses it to find the driver for Safari so we can only remove it in Docker.
|
|
47
|
+
|
|
48
|
+
### Tech
|
|
49
|
+
* New GitHub actions that test uploading to S3, GCS and SCP.
|
|
50
|
+
|
|
3
51
|
## 35.7.5 - 2024-12-23
|
|
4
52
|
### Fixed
|
|
5
53
|
* Update to faststat 0.0.7 [#4347](https://github.com/sitespeedio/sitespeed.io/pull/4347).
|
package/Dockerfile
CHANGED
|
@@ -1,9 +1,10 @@
|
|
|
1
|
-
FROM sitespeedio/webbrowsers:chrome-131.0-firefox-
|
|
1
|
+
FROM sitespeedio/webbrowsers:chrome-131.0-firefox-134.0-edge-131.0-b
|
|
2
2
|
|
|
3
3
|
ARG TARGETPLATFORM=linux/amd64
|
|
4
4
|
|
|
5
|
-
ENV SITESPEED_IO_BROWSERTIME__XVFB
|
|
6
|
-
ENV SITESPEED_IO_BROWSERTIME__DOCKER
|
|
5
|
+
ENV SITESPEED_IO_BROWSERTIME__XVFB=true
|
|
6
|
+
ENV SITESPEED_IO_BROWSERTIME__DOCKER=true
|
|
7
|
+
ENV PYTHON=python3
|
|
7
8
|
|
|
8
9
|
COPY docker/webpagereplay/$TARGETPLATFORM/wpr /usr/local/bin/
|
|
9
10
|
COPY docker/webpagereplay/wpr_cert.pem /webpagereplay/certs/
|
|
@@ -11,7 +12,6 @@ COPY docker/webpagereplay/wpr_key.pem /webpagereplay/certs/
|
|
|
11
12
|
COPY docker/webpagereplay/deterministic.js /webpagereplay/scripts/deterministic.js
|
|
12
13
|
COPY docker/webpagereplay/LICENSE /webpagereplay/
|
|
13
14
|
|
|
14
|
-
|
|
15
15
|
RUN sudo apt-get update && sudo apt-get install libnss3-tools python2 \
|
|
16
16
|
net-tools \
|
|
17
17
|
build-essential \
|
|
@@ -28,8 +28,12 @@ WORKDIR /usr/src/app
|
|
|
28
28
|
|
|
29
29
|
COPY package.json /usr/src/app/
|
|
30
30
|
COPY npm-shrinkwrap.json /usr/src/app/
|
|
31
|
-
|
|
32
|
-
|
|
31
|
+
COPY tools/postinstall.js /usr/src/app/tools/postinstall.js
|
|
32
|
+
RUN npm install --production && npm cache clean --force
|
|
33
|
+
|
|
34
|
+
COPY ./bin/ /usr/src/app/bin/
|
|
35
|
+
COPY ./lib/ /usr/src/app/lib/
|
|
36
|
+
RUN rm -fR /usr/src/app/node_modules/selenium-webdriver/bin
|
|
33
37
|
|
|
34
38
|
COPY docker/scripts/start.sh /start.sh
|
|
35
39
|
|
package/Dockerfile-slim
CHANGED
|
@@ -1,12 +1,12 @@
|
|
|
1
|
-
FROM node:
|
|
1
|
+
FROM node:22.13.0-bookworm-slim
|
|
2
2
|
|
|
3
3
|
ARG TARGETPLATFORM=linux/amd64
|
|
4
4
|
|
|
5
|
-
ENV SITESPEED_IO_BROWSERTIME__DOCKER
|
|
6
|
-
ENV SITESPEED_IO_BROWSERTIME__VIDEO
|
|
7
|
-
ENV SITESPEED_IO_BROWSERTIME__BROWSER
|
|
8
|
-
ENV SITESPEED_IO_BROWSERTIME__VISUAL_METRICS
|
|
9
|
-
ENV SITESPEED_IO_BROWSERTIME__HEADLESS
|
|
5
|
+
ENV SITESPEED_IO_BROWSERTIME__DOCKER=true
|
|
6
|
+
ENV SITESPEED_IO_BROWSERTIME__VIDEO=false
|
|
7
|
+
ENV SITESPEED_IO_BROWSERTIME__BROWSER=firefox
|
|
8
|
+
ENV SITESPEED_IO_BROWSERTIME__VISUAL_METRICS=false
|
|
9
|
+
ENV SITESPEED_IO_BROWSERTIME__HEADLESS=true
|
|
10
10
|
|
|
11
11
|
ENV PATH="/usr/local/bin:${PATH}"
|
|
12
12
|
|
|
@@ -21,7 +21,9 @@ RUN echo "deb http://deb.debian.org/debian/ unstable main contrib non-free" >> /
|
|
|
21
21
|
RUN mkdir -p /usr/src/app
|
|
22
22
|
WORKDIR /usr/src/app
|
|
23
23
|
COPY . /usr/src/app
|
|
24
|
-
|
|
24
|
+
COPY tools/postinstall.js /usr/src/app/tools/postinstall.js
|
|
25
|
+
RUN CHROMEDRIVER_SKIP_DOWNLOAD=true EGDEDRIVER_SKIP_DOWNLOAD=true npm install --omit=dev --omit=optional && npm cache clean --force && npm uninstall npm npx -g && rm -fR /usr/src/app/node_modules/selenium-webdriver/bin
|
|
26
|
+
|
|
25
27
|
WORKDIR /usr/src/app
|
|
26
28
|
COPY docker/scripts/start-slim.sh /start.sh
|
|
27
29
|
|
package/LICENSE
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
The MIT License (MIT)
|
|
2
2
|
|
|
3
|
-
Copyright (c) 2012-
|
|
3
|
+
Copyright (c) 2012-2025 Peter Hedenskog
|
|
4
4
|
|
|
5
5
|
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
6
|
of this software and associated documentation files (the "Software"), to deal
|
|
@@ -8,10 +8,10 @@ import get from 'lodash.get';
|
|
|
8
8
|
import yargs from 'yargs';
|
|
9
9
|
import { hideBin } from 'yargs/helpers';
|
|
10
10
|
|
|
11
|
-
import { findUpSync } from 'find-up';
|
|
12
11
|
import { BrowsertimeEngine, configureLogging } from 'browsertime';
|
|
13
12
|
|
|
14
13
|
import { getURLs } from '../lib/cli/util.js';
|
|
14
|
+
import { findUpSync } from '../lib/support/fileUtil.js';
|
|
15
15
|
|
|
16
16
|
import {config as browsertimeConfig} from '../lib/plugins/browsertime/index.js';
|
|
17
17
|
|
package/lib/cli/cli.js
CHANGED
|
@@ -6,10 +6,8 @@ import { readFileSync, statSync } from 'node:fs';
|
|
|
6
6
|
import yargs from 'yargs';
|
|
7
7
|
import { hideBin } from 'yargs/helpers';
|
|
8
8
|
import merge from 'lodash.merge';
|
|
9
|
-
import reduce from 'lodash.reduce';
|
|
10
9
|
import set from 'lodash.set';
|
|
11
10
|
import get from 'lodash.get';
|
|
12
|
-
import { findUpSync } from 'find-up';
|
|
13
11
|
|
|
14
12
|
import { getURLs, getAliases } from './util.js';
|
|
15
13
|
import { toArray } from '../support/util.js';
|
|
@@ -19,6 +17,7 @@ import { config as metricsConfig } from '../plugins/metrics/index.js';
|
|
|
19
17
|
import { config as slackConfig } from '../plugins/slack/index.js';
|
|
20
18
|
import { config as htmlConfig } from '../plugins/html/index.js';
|
|
21
19
|
import { messageTypes as matrixMessageTypes } from '../plugins/matrix/index.js';
|
|
20
|
+
import { findUpSync } from '../support/fileUtil.js';
|
|
22
21
|
|
|
23
22
|
const metricList = Object.keys(friendlynames);
|
|
24
23
|
const require = createRequire(import.meta.url);
|
|
@@ -98,6 +97,10 @@ function validateInput(argv) {
|
|
|
98
97
|
return 'Error: Getting CrUx data do not work running in multi mode.';
|
|
99
98
|
}
|
|
100
99
|
|
|
100
|
+
if (argv.browsertime.cpu && argv.browsertime.enableProfileRun) {
|
|
101
|
+
return 'Error: Use either --cpu or --enableProfileRun. Profile run will run one extra iteration to collect cpu/trace data.';
|
|
102
|
+
}
|
|
103
|
+
|
|
101
104
|
if (
|
|
102
105
|
argv.urlAlias &&
|
|
103
106
|
argv._ &&
|
|
@@ -573,13 +576,13 @@ export async function parseCommandLine() {
|
|
|
573
576
|
alias: 'enableProfileRun',
|
|
574
577
|
type: 'boolean',
|
|
575
578
|
describe:
|
|
576
|
-
'Make one extra run that collects the profiling trace log (no other metrics is collected). For Chrome it will collect the timeline trace, for Firefox it will get the Geckoprofiler trace. This means you do not need to get the trace for all runs and can skip the overhead it produces.'
|
|
579
|
+
'Make one extra run that collects the profiling trace log (no other metrics is collected). For Chrome it will collect the timeline trace, for Firefox it will get the Geckoprofiler trace. This means you do not need to get the trace for all runs and can skip the overhead it produces. You should not run this together with --cpu since that will get a trace for every iteration.'
|
|
577
580
|
})
|
|
578
581
|
.option('browsertime.enableVideoRun', {
|
|
579
582
|
alias: 'enableVideoRun',
|
|
580
583
|
type: 'boolean',
|
|
581
584
|
describe:
|
|
582
|
-
'Make one extra run that collects video and visual metrics. This means you can do your runs with --visualMetrics true --video false --enableVideoRun true to collect visual metrics from all runs and save a video from the profile/video run. If you run it together with --enableProfileRun it will also collect profiling
|
|
585
|
+
'Make one extra run that collects video and visual metrics. This means you can do your runs with --visualMetrics true --video false --enableVideoRun true to collect visual metrics from all runs and save a video from the profile/video run. If you run it together with --enableProfileRun it will also collect profiling race.'
|
|
583
586
|
})
|
|
584
587
|
.option('browsertime.videoParams.filmstripFullSize', {
|
|
585
588
|
alias: 'videoParams.filmstripFullSize',
|
|
@@ -616,10 +619,10 @@ export async function parseCommandLine() {
|
|
|
616
619
|
'Show all screenshots in the filmstrip, independent if they have changed or not.',
|
|
617
620
|
group: 'Filmstrip'
|
|
618
621
|
})
|
|
619
|
-
.option('browsertime.
|
|
620
|
-
alias: '
|
|
622
|
+
.option('browsertime.userTimingAllowList', {
|
|
623
|
+
alias: 'userTimingAllowList',
|
|
621
624
|
describe:
|
|
622
|
-
'This option takes a regex that will whitelist which userTimings to capture in the results. All userTimings are captured by default.
|
|
625
|
+
'This option takes a regex that will whitelist which userTimings to capture in the results. All userTimings are captured by default.',
|
|
623
626
|
group: 'Browser'
|
|
624
627
|
})
|
|
625
628
|
.option('axe.enable', {
|
|
@@ -1831,7 +1834,7 @@ export async function parseCommandLine() {
|
|
|
1831
1834
|
type: 'boolean',
|
|
1832
1835
|
group: 'Text'
|
|
1833
1836
|
})
|
|
1834
|
-
.option('
|
|
1837
|
+
.option('summaryDetail', {
|
|
1835
1838
|
describe: 'Show longer text summary to stdout',
|
|
1836
1839
|
default: false,
|
|
1837
1840
|
type: 'boolean',
|
|
@@ -2036,12 +2039,6 @@ export async function parseCommandLine() {
|
|
|
2036
2039
|
default: false,
|
|
2037
2040
|
type: 'boolean'
|
|
2038
2041
|
})
|
|
2039
|
-
.option('logToFile', {
|
|
2040
|
-
describe:
|
|
2041
|
-
'Store the log for your run into a file in logs/sitespeed.io.log',
|
|
2042
|
-
default: false,
|
|
2043
|
-
type: 'boolean'
|
|
2044
|
-
})
|
|
2045
2042
|
.option('useHash', {
|
|
2046
2043
|
describe:
|
|
2047
2044
|
'If your site uses # for URLs and # give you unique URLs you need to turn on useHash. By default is it turned off, meaning URLs with hash and without hash are treated as the same URL',
|
|
@@ -2138,14 +2135,10 @@ export async function parseCommandLine() {
|
|
|
2138
2135
|
argv = parsed.argv;
|
|
2139
2136
|
|
|
2140
2137
|
// aliases are long options -> short option
|
|
2141
|
-
const aliasLookup =
|
|
2142
|
-
|
|
2143
|
-
(
|
|
2144
|
-
|
|
2145
|
-
return lookup;
|
|
2146
|
-
},
|
|
2147
|
-
new Map()
|
|
2148
|
-
);
|
|
2138
|
+
const aliasLookup = new Map();
|
|
2139
|
+
for (const [key, value] of Object.entries(aliases)) {
|
|
2140
|
+
aliasLookup.set(value[0], key);
|
|
2141
|
+
}
|
|
2149
2142
|
|
|
2150
2143
|
let explicitOptions = yargs(hideBin(process.argv)).argv;
|
|
2151
2144
|
|
|
@@ -2154,9 +2147,8 @@ export async function parseCommandLine() {
|
|
|
2154
2147
|
yargsInstance.getOptions().configObjects[0]
|
|
2155
2148
|
);
|
|
2156
2149
|
|
|
2157
|
-
explicitOptions = reduce(
|
|
2158
|
-
|
|
2159
|
-
(result, value, key) => {
|
|
2150
|
+
explicitOptions = Object.entries(explicitOptions).reduce(
|
|
2151
|
+
(result, [key, value]) => {
|
|
2160
2152
|
if (aliasLookup.has(key)) {
|
|
2161
2153
|
const fullKey = aliasLookup.get(key);
|
|
2162
2154
|
result = set(result, fullKey, value);
|
package/lib/core/logging.js
CHANGED
|
@@ -1,100 +1,8 @@
|
|
|
1
|
-
import
|
|
2
|
-
import { createWriteStream } from 'node:fs';
|
|
3
|
-
import { inherits } from 'node:util';
|
|
4
|
-
const {
|
|
5
|
-
INFO,
|
|
6
|
-
DEBUG,
|
|
7
|
-
VERBOSE,
|
|
8
|
-
TRACE,
|
|
9
|
-
NONE,
|
|
10
|
-
basicConfig,
|
|
11
|
-
Logger,
|
|
12
|
-
Handler,
|
|
13
|
-
Formatter
|
|
14
|
-
} = intel;
|
|
1
|
+
import { configureLog } from '@sitespeed.io/log';
|
|
15
2
|
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
if (!options.stream) {
|
|
22
|
-
options = { stream: options };
|
|
23
|
-
}
|
|
24
|
-
Handler.call(this, options);
|
|
25
|
-
this._stream = options.stream;
|
|
26
|
-
}
|
|
27
|
-
|
|
28
|
-
inherits(StreamHandler, Handler);
|
|
29
|
-
|
|
30
|
-
StreamHandler.prototype.emit = function streamEmit(record) {
|
|
31
|
-
this._stream.write(this.format(record) + '\n');
|
|
32
|
-
};
|
|
33
|
-
|
|
34
|
-
function FileHandler(options) {
|
|
35
|
-
if (typeof options === 'string') {
|
|
36
|
-
options = { file: options };
|
|
37
|
-
}
|
|
38
|
-
this._file = options.file;
|
|
39
|
-
|
|
40
|
-
options.stream = this._open();
|
|
41
|
-
StreamHandler.call(this, options);
|
|
42
|
-
}
|
|
43
|
-
inherits(FileHandler, StreamHandler);
|
|
44
|
-
|
|
45
|
-
FileHandler.prototype._open = function open() {
|
|
46
|
-
return createWriteStream(this._file, { flags: 'a' });
|
|
47
|
-
};
|
|
48
|
-
|
|
49
|
-
export function configure(options, logDir) {
|
|
50
|
-
options = options || {};
|
|
51
|
-
|
|
52
|
-
let level = INFO;
|
|
53
|
-
|
|
54
|
-
switch (options.verbose) {
|
|
55
|
-
case 1: {
|
|
56
|
-
level = DEBUG;
|
|
57
|
-
break;
|
|
58
|
-
}
|
|
59
|
-
case 2: {
|
|
60
|
-
level = VERBOSE;
|
|
61
|
-
break;
|
|
62
|
-
}
|
|
63
|
-
case 3: {
|
|
64
|
-
level = TRACE;
|
|
65
|
-
break;
|
|
66
|
-
}
|
|
67
|
-
default: {
|
|
68
|
-
break;
|
|
69
|
-
}
|
|
70
|
-
}
|
|
71
|
-
|
|
72
|
-
if (options.silent) {
|
|
73
|
-
level = NONE;
|
|
74
|
-
}
|
|
75
|
-
|
|
76
|
-
if (level === INFO) {
|
|
77
|
-
basicConfig({
|
|
78
|
-
format: '[%(date)s] %(levelname)s: %(message)s',
|
|
79
|
-
level: level
|
|
80
|
-
});
|
|
81
|
-
} else {
|
|
82
|
-
basicConfig({
|
|
83
|
-
format: '[%(date)s] %(levelname)s: [%(name)s] %(message)s',
|
|
84
|
-
level: level
|
|
85
|
-
});
|
|
86
|
-
}
|
|
87
|
-
|
|
88
|
-
if (options.logToFile) {
|
|
89
|
-
let logger = new Logger();
|
|
90
|
-
logger.addHandler(
|
|
91
|
-
new FileHandler({
|
|
92
|
-
file: logDir + '/sitespeed.io.log',
|
|
93
|
-
formatter: new Formatter({
|
|
94
|
-
format: '[%(date)s] %(levelname)s: [%(name)s] %(message)s',
|
|
95
|
-
level: level
|
|
96
|
-
})
|
|
97
|
-
})
|
|
98
|
-
);
|
|
99
|
-
}
|
|
3
|
+
export function configure(options = {}) {
|
|
4
|
+
configureLog({
|
|
5
|
+
verbose: options.verbose ?? 0,
|
|
6
|
+
silent: options.silent ?? false
|
|
7
|
+
});
|
|
100
8
|
}
|
|
@@ -0,0 +1,112 @@
|
|
|
1
|
+
// Simplified implementation of concurrent-queue
|
|
2
|
+
// https://www.npmjs.com/package/concurrent-queue
|
|
3
|
+
|
|
4
|
+
export function createQueue() {
|
|
5
|
+
let concurrency = Number.POSITIVE_INFINITY;
|
|
6
|
+
let processor;
|
|
7
|
+
|
|
8
|
+
const tasks = [];
|
|
9
|
+
let runningCount = 0;
|
|
10
|
+
|
|
11
|
+
const enqueuedCallbacks = [];
|
|
12
|
+
const processingStartedCallbacks = [];
|
|
13
|
+
const processingEndedCallbacks = [];
|
|
14
|
+
const drainedCallbacks = [];
|
|
15
|
+
|
|
16
|
+
/**
|
|
17
|
+
* We will trigger drained callbacks when:
|
|
18
|
+
* tasks.length === 0 and runningCount === 0
|
|
19
|
+
*/
|
|
20
|
+
function checkDrained() {
|
|
21
|
+
if (tasks.length === 0 && runningCount === 0) {
|
|
22
|
+
for (const cb of drainedCallbacks) {
|
|
23
|
+
cb();
|
|
24
|
+
}
|
|
25
|
+
}
|
|
26
|
+
}
|
|
27
|
+
|
|
28
|
+
/**
|
|
29
|
+
* Attempt to start processing more tasks if we have
|
|
30
|
+
* capacity (runningCount < concurrency).
|
|
31
|
+
*/
|
|
32
|
+
function tryProcessNext() {
|
|
33
|
+
while (tasks.length > 0 && runningCount < concurrency) {
|
|
34
|
+
const item = tasks.shift();
|
|
35
|
+
runningCount++;
|
|
36
|
+
|
|
37
|
+
for (const cb of processingStartedCallbacks) {
|
|
38
|
+
cb({ item });
|
|
39
|
+
}
|
|
40
|
+
|
|
41
|
+
const promise = Promise.resolve(processor(item));
|
|
42
|
+
|
|
43
|
+
promise
|
|
44
|
+
.then(() => {
|
|
45
|
+
// Fire processingEnded callbacks
|
|
46
|
+
for (const cb of processingEndedCallbacks) {
|
|
47
|
+
cb({ item, err: undefined });
|
|
48
|
+
}
|
|
49
|
+
})
|
|
50
|
+
.catch(error => {
|
|
51
|
+
// Fire processingEnded callbacks with an error
|
|
52
|
+
for (const cb of processingEndedCallbacks) {
|
|
53
|
+
cb({ item, err: error });
|
|
54
|
+
}
|
|
55
|
+
})
|
|
56
|
+
.finally(() => {
|
|
57
|
+
runningCount--;
|
|
58
|
+
checkDrained();
|
|
59
|
+
tryProcessNext();
|
|
60
|
+
});
|
|
61
|
+
}
|
|
62
|
+
}
|
|
63
|
+
|
|
64
|
+
const queue = function enqueue(item) {
|
|
65
|
+
for (const cb of enqueuedCallbacks) {
|
|
66
|
+
cb({ item });
|
|
67
|
+
}
|
|
68
|
+
tasks.push(item);
|
|
69
|
+
|
|
70
|
+
tryProcessNext();
|
|
71
|
+
};
|
|
72
|
+
|
|
73
|
+
queue.limit = options => {
|
|
74
|
+
if (options && typeof options.concurrency === 'number') {
|
|
75
|
+
concurrency = options.concurrency;
|
|
76
|
+
}
|
|
77
|
+
return queue;
|
|
78
|
+
};
|
|
79
|
+
|
|
80
|
+
queue.process = fn => {
|
|
81
|
+
processor = fn;
|
|
82
|
+
return queue;
|
|
83
|
+
};
|
|
84
|
+
|
|
85
|
+
queue.enqueued = callback => {
|
|
86
|
+
enqueuedCallbacks.push(callback);
|
|
87
|
+
return queue;
|
|
88
|
+
};
|
|
89
|
+
|
|
90
|
+
queue.processingStarted = callback => {
|
|
91
|
+
processingStartedCallbacks.push(callback);
|
|
92
|
+
return queue;
|
|
93
|
+
};
|
|
94
|
+
|
|
95
|
+
queue.processingEnded = callback => {
|
|
96
|
+
processingEndedCallbacks.push(callback);
|
|
97
|
+
return queue;
|
|
98
|
+
};
|
|
99
|
+
|
|
100
|
+
queue.drained = callback => {
|
|
101
|
+
drainedCallbacks.push(callback);
|
|
102
|
+
return queue;
|
|
103
|
+
};
|
|
104
|
+
|
|
105
|
+
Object.defineProperty(queue, 'isDrained', {
|
|
106
|
+
get() {
|
|
107
|
+
return tasks.length === 0 && runningCount === 0;
|
|
108
|
+
}
|
|
109
|
+
});
|
|
110
|
+
|
|
111
|
+
return queue;
|
|
112
|
+
}
|
package/lib/core/queueHandler.js
CHANGED
|
@@ -1,17 +1,14 @@
|
|
|
1
|
-
|
|
2
|
-
|
|
3
|
-
import cq from 'concurrent-queue';
|
|
4
|
-
import intel from 'intel';
|
|
5
|
-
|
|
1
|
+
import { getLogger } from '@sitespeed.io/log';
|
|
6
2
|
import { messageMaker } from '../support/messageMaker.js';
|
|
7
3
|
import {
|
|
8
4
|
registerQueueTime,
|
|
9
5
|
registerProcessingTime,
|
|
10
6
|
generateStatistics
|
|
11
7
|
} from './queueStatistics.js';
|
|
8
|
+
import { createQueue } from './queue.js';
|
|
12
9
|
|
|
13
10
|
const make = messageMaker('queueHandler').make;
|
|
14
|
-
const log =
|
|
11
|
+
const log = getLogger('sitespeedio.queuehandler');
|
|
15
12
|
|
|
16
13
|
function shortenData(key, value) {
|
|
17
14
|
if (key === 'data') {
|
|
@@ -103,36 +100,32 @@ export class QueueHandler {
|
|
|
103
100
|
this.queues = plugins
|
|
104
101
|
.filter(plugin => plugin.processMessage)
|
|
105
102
|
.map(plugin => {
|
|
106
|
-
const concurrency = plugin.concurrency
|
|
107
|
-
|
|
103
|
+
const concurrency = plugin.concurrency ?? Number.POSITIVE_INFINITY;
|
|
104
|
+
// Create a queue with that concurrency
|
|
105
|
+
const queue = createQueue().limit({ concurrency });
|
|
108
106
|
|
|
109
107
|
queue.plugin = plugin;
|
|
110
108
|
|
|
111
|
-
const messageWaitingStart = {}
|
|
112
|
-
|
|
109
|
+
const messageWaitingStart = {};
|
|
110
|
+
const messageProcessingStart = {};
|
|
113
111
|
|
|
114
112
|
queue.enqueued(object => {
|
|
115
|
-
const message = object
|
|
113
|
+
const { item: message } = object;
|
|
116
114
|
messageWaitingStart[message.uuid] = process.hrtime();
|
|
117
115
|
});
|
|
118
116
|
|
|
119
117
|
queue.processingStarted(object => {
|
|
120
|
-
const message = object
|
|
121
|
-
|
|
118
|
+
const { item: message } = object;
|
|
122
119
|
const waitingDuration = process.hrtime(
|
|
123
|
-
|
|
124
|
-
|
|
125
|
-
|
|
126
|
-
|
|
120
|
+
messageWaitingStart[message.uuid]
|
|
121
|
+
);
|
|
122
|
+
const waitingNanos = waitingDuration[0] * 1e9 + waitingDuration[1];
|
|
127
123
|
registerQueueTime(message, queue.plugin, waitingNanos);
|
|
128
|
-
|
|
129
124
|
messageProcessingStart[message.uuid] = process.hrtime();
|
|
130
125
|
});
|
|
131
126
|
|
|
132
|
-
// FIXME handle rejections (i.e. failures while processing messages) properly
|
|
133
127
|
queue.processingEnded(object => {
|
|
134
|
-
const message = object
|
|
135
|
-
const error = object.err;
|
|
128
|
+
const { item: message, err: error } = object;
|
|
136
129
|
if (error) {
|
|
137
130
|
let rejectionMessage =
|
|
138
131
|
'Rejected ' +
|
|
@@ -140,9 +133,9 @@ export class QueueHandler {
|
|
|
140
133
|
' for plugin: ' +
|
|
141
134
|
plugin.getName();
|
|
142
135
|
|
|
143
|
-
if (message
|
|
144
|
-
rejectionMessage +=
|
|
145
|
-
|
|
136
|
+
if (message?.url) {
|
|
137
|
+
rejectionMessage += `, url: ${message.url}`;
|
|
138
|
+
}
|
|
146
139
|
if (error.stack) {
|
|
147
140
|
log.error(error.stack);
|
|
148
141
|
}
|
|
@@ -154,7 +147,6 @@ export class QueueHandler {
|
|
|
154
147
|
);
|
|
155
148
|
const processingNanos =
|
|
156
149
|
processingDuration[0] * 1e9 + processingDuration[1];
|
|
157
|
-
|
|
158
150
|
registerProcessingTime(message, queue.plugin, processingNanos);
|
|
159
151
|
});
|
|
160
152
|
|
|
@@ -221,9 +213,9 @@ export class QueueHandler {
|
|
|
221
213
|
}
|
|
222
214
|
|
|
223
215
|
async startProcessingQueues() {
|
|
224
|
-
for (
|
|
225
|
-
const queue = item
|
|
226
|
-
|
|
216
|
+
for (const item of this.queues) {
|
|
217
|
+
const { queue, plugin } = item;
|
|
218
|
+
// For each queue, set up the processor that handles messages
|
|
227
219
|
queue.process(message =>
|
|
228
220
|
Promise.resolve(plugin.processMessage(message, this))
|
|
229
221
|
);
|
|
@@ -1,10 +1,10 @@
|
|
|
1
1
|
import { parse } from 'node:url';
|
|
2
2
|
import { createHash } from 'node:crypto';
|
|
3
3
|
|
|
4
|
-
import
|
|
5
|
-
import intel from 'intel';
|
|
4
|
+
import { getLogger } from '@sitespeed.io/log';
|
|
6
5
|
|
|
7
|
-
|
|
6
|
+
import { isEmpty } from '../../support/util.js';
|
|
7
|
+
const log = getLogger('sitespeedio.file');
|
|
8
8
|
|
|
9
9
|
function toSafeKey(key) {
|
|
10
10
|
// U+2013 : EN DASH – as used on https://en.wikipedia.org/wiki/2019–20_coronavirus_pandemic
|
|
@@ -9,12 +9,12 @@ import {
|
|
|
9
9
|
writeFile as _writeFile
|
|
10
10
|
} from 'node:fs';
|
|
11
11
|
|
|
12
|
-
import {
|
|
13
|
-
import
|
|
12
|
+
import { cp } from 'node:fs/promises';
|
|
13
|
+
import { getLogger } from '@sitespeed.io/log';
|
|
14
14
|
|
|
15
15
|
import { pathToFolder } from './pathToFolder.js';
|
|
16
16
|
|
|
17
|
-
const log =
|
|
17
|
+
const log = getLogger('sitespeedio.storageManager');
|
|
18
18
|
const mkdir = promisify(_mkdir);
|
|
19
19
|
const readdir = promisify(_readdir);
|
|
20
20
|
const lstat = promisify(_lstat);
|
|
@@ -69,10 +69,12 @@ export function storageManager(baseDir, storagePathPrefix, options) {
|
|
|
69
69
|
return storagePathPrefix;
|
|
70
70
|
},
|
|
71
71
|
copyToResultDir(filename) {
|
|
72
|
-
return this.createDirectory().then(dir =>
|
|
72
|
+
return this.createDirectory().then(dir =>
|
|
73
|
+
cp(filename, dir, { recursive: true })
|
|
74
|
+
);
|
|
73
75
|
},
|
|
74
76
|
copyFileToDir(filename, dir) {
|
|
75
|
-
return
|
|
77
|
+
return cp(filename, dir, { recursive: true });
|
|
76
78
|
},
|
|
77
79
|
// TODO is missing alias
|
|
78
80
|
removeDataForUrl(url) {
|
|
@@ -1,6 +1,6 @@
|
|
|
1
|
-
import isEmpty from 'lodash.isempty';
|
|
2
1
|
import { SitespeedioPlugin } from '@sitespeed.io/plugin';
|
|
3
2
|
import { AssetsAggregator } from './aggregator.js';
|
|
3
|
+
import { isEmpty } from '../../support/util.js';
|
|
4
4
|
const DEFAULT_METRICS_LARGEST_ASSETS = ['image.0.transferSize'];
|
|
5
5
|
|
|
6
6
|
export default class AssetsPlugin extends SitespeedioPlugin {
|