thirdparty-pulse 0.0.0-stage → 1.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +12 -0
- package/LICENSE +21 -0
- package/README.md +194 -2
- package/action.yml +47 -0
- package/bin/thirdparty-pulse.js +18 -0
- package/package.json +52 -4
- package/src/core.d.ts +16 -0
- package/src/core.js +176 -0
- package/src/index.d.ts +10 -0
- package/src/index.js +86 -0
- package/src/site.js +326 -0
- package/src/vendors.js +77 -0
package/CHANGELOG.md
ADDED
|
@@ -0,0 +1,12 @@
|
|
|
1
|
+
# Changelog
|
|
2
|
+
|
|
3
|
+
## 1.0.0 (2026-10-14)
|
|
4
|
+
|
|
5
|
+
First release, by Louis Wright.
|
|
6
|
+
|
|
7
|
+
- Per-page third-party script weight (estimated KB), request count and estimated main-thread time, with budgets (`maxKb`, `maxRequests`, `maxMainThreadMs`) and per-vendor budgets (`vendorBudgets`).
|
|
8
|
+
- Built-in table of ~45 common vendors with typical script weights; `weights` config for real transfer sizes and unknown hosts.
|
|
9
|
+
- Loading checks: synchronous third-party scripts, duplicate includes, heavy vendors that have facades, missing and unused preconnects.
|
|
10
|
+
- Per-vendor totals in the JSON report.
|
|
11
|
+
- Text/JSON/SARIF/Markdown output, `--fail-on`, exit codes 0/1/2, allowlist with expiry, severity overrides.
|
|
12
|
+
- Composite GitHub Action with job summary.
|
package/LICENSE
ADDED
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 Louis Wright
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
package/README.md
CHANGED
|
@@ -1,3 +1,195 @@
|
|
|
1
|
-
#
|
|
1
|
+
# thirdparty-pulse
|
|
2
2
|
|
|
3
|
-
|
|
3
|
+
> How much does each third-party script cost your pages? Estimated bytes, requests and main-thread time per page and per vendor, checked against budgets.
|
|
4
|
+
|
|
5
|
+
Tag managers, chat widgets, pixels and embeds are usually the heaviest JavaScript on a static site, and nobody owns their weight. `thirdparty-pulse` reads your built HTML, finds every third-party script, inline vendor loader, iframe and asset, estimates its transfer size and main-thread cost from a table of common vendors (or the sizes you configure), and fails CI when a page goes over budget. It also catches the loading mistakes that make third parties expensive: synchronous scripts in `<head>`, the same script included twice, and missing or unused preconnects.
|
|
6
|
+
|
|
7
|
+
CLI, library and GitHub Action. Node.js 18+, **zero runtime dependencies**. Works on any static build output (Astro, Next.js `output: 'export'`, Eleventy, Hugo, Jekyll, Docusaurus, VitePress, plain HTML) and runs offline: point it at the folder your build writes. By Louis Wright.
|
|
8
|
+
|
|
9
|
+
## Features
|
|
10
|
+
|
|
11
|
+
- **Budgets per page:** third-party script KB (`maxKb`, default 150), requests (`maxRequests`, default 10) and estimated main-thread time (`maxMainThreadMs`, default 250 ms, at `msPerKb` 2.5 ms per compressed KB).
|
|
12
|
+
- **Budgets per vendor:** `vendorBudgets` caps a single vendor (e.g. `{"google-tag-manager": 80}`).
|
|
13
|
+
- **Weights:** a built-in table of ~45 vendors (analytics, tag managers, chat, video, maps, payments, CDNs) with typical compressed sizes; override or add hosts with `weights` (host, URL glob or vendor id). Unknown hosts get `unknownKb` (30) and an info finding.
|
|
14
|
+
- **Loading:** synchronous third-party scripts (high in `<head>`, medium in `<body>`), duplicate includes, heavy vendors that have a facade (YouTube, Maps, chat), early origins without `preconnect`, and preconnects that nothing uses.
|
|
15
|
+
- **Report:** the JSON report lists every vendor/origin with its weight, requests and the number of pages it is on.
|
|
16
|
+
|
|
17
|
+
### Scope
|
|
18
|
+
|
|
19
|
+
Performance cost only. This is not a privacy, cookie or consent inventory and makes no judgement about what vendors do with data. Weights are estimates from typical transfer sizes; the tool reads files and does not load the site, so confirm big numbers with a lab trace (Lighthouse, WebPageTest).
|
|
20
|
+
|
|
21
|
+
## Rules
|
|
22
|
+
|
|
23
|
+
| Rule | Default severity | What it means |
|
|
24
|
+
|---|---|---|
|
|
25
|
+
| `tp-budget-bytes` | high | Third-party script weight on a page exceeds the byte budget |
|
|
26
|
+
| `tp-vendor-budget` | high | A single vendor exceeds its configured byte budget |
|
|
27
|
+
| `tp-sync-script` | high | Third-party script loads synchronously (no async/defer) and blocks parsing |
|
|
28
|
+
| `tp-budget-main-thread` | medium | Estimated third-party main-thread time on a page exceeds the budget |
|
|
29
|
+
| `tp-budget-requests` | medium | Number of third-party requests on a page exceeds the budget |
|
|
30
|
+
| `tp-duplicate-script` | medium | The same third-party script or vendor loader is included more than once on a page |
|
|
31
|
+
| `tp-heavy-vendor` | low | Heavy vendor script (100 KB+) where a lighter alternative or facade usually exists |
|
|
32
|
+
| `tp-no-preconnect` | low | Early third-party origin has no preconnect / dns-prefetch hint |
|
|
33
|
+
| `tp-preconnect-unused` | low | preconnect to a third-party origin that the page never uses |
|
|
34
|
+
| `tp-unknown-weight` | info | Unrecognised third-party script; a default weight was assumed (set weights in config) |
|
|
35
|
+
| `allowlist-expired` | medium | An allowlist entry is past its `expires` date |
|
|
36
|
+
| `allowlist-unused` | info | An allowlist entry matched nothing |
|
|
37
|
+
|
|
38
|
+
## Install and run
|
|
39
|
+
|
|
40
|
+
```bash
|
|
41
|
+
npx thirdparty-pulse dist
|
|
42
|
+
npx thirdparty-pulse _site --max-kb 100 --max-requests 6
|
|
43
|
+
npx thirdparty-pulse public -f json > third-party.json
|
|
44
|
+
```
|
|
45
|
+
|
|
46
|
+
Pages are mapped to URLs the way static hosts serve them: `index.html` → `/`, `blog/index.html` → `/blog/`, `about.html` → `/about.html` (or `/about` with `--clean-urls`). Pass `--base-url` with your public URL, or let it be inferred from canonical / `og:url` tags. Meta-refresh redirect stubs and `404.html` are skipped. `X-Robots-Tag` and `Link` headers are read from Netlify/Cloudflare `_headers` and `vercel.json`.
|
|
47
|
+
|
|
48
|
+
### Sample output
|
|
49
|
+
|
|
50
|
+
```text
|
|
51
|
+
thirdparty-pulse — https://example.com
|
|
52
|
+
1 page(s), 1 with third-party content, 6 vendor(s)/origin(s), heaviest page ~465 KB of third-party script
|
|
53
|
+
|
|
54
|
+
HIGH index.html / loads ~465 KB of third-party script (budget 150 KB): intercom 200 KB, google-tag-manager 90 KB, meta-pixel 75 KB, hotjar 70 KB, cdn.example-widgets.net 30 KB [tp-budget-bytes]
|
|
55
|
+
HIGH index.html:5 Synchronous third-party script https://www.googletagmanager.com/gtm.js?id=GTM-ABC123 on / in <head> [tp-sync-script]
|
|
56
|
+
MEDIUM index.html /: estimated ~1163 ms of third-party main-thread work (budget 250 ms) [tp-budget-main-thread]
|
|
57
|
+
MEDIUM index.html:11 https://static.hotjar.com/c/hotjar-1.js is included 2 times on / [tp-duplicate-script]
|
|
58
|
+
LOW index.html www.googletagmanager.com is requested from <head> on / without a preconnect [tp-no-preconnect]
|
|
59
|
+
LOW index.html connect.facebook.net is requested from <head> on / without a preconnect [tp-no-preconnect]
|
|
60
|
+
LOW index.html static.hotjar.com is requested from <head> on / without a preconnect [tp-no-preconnect]
|
|
61
|
+
LOW index.html widget.intercom.io is requested from <head> on / without a preconnect [tp-no-preconnect]
|
|
62
|
+
LOW index.html cdn.example-widgets.net is requested from <head> on / without a preconnect [tp-no-preconnect]
|
|
63
|
+
LOW index.html:4 preconnect to fonts.unused-cdn.example on /, but nothing on the page loads from it [tp-preconnect-unused]
|
|
64
|
+
LOW index.html:8 Intercom (~200 KB) on / [tp-heavy-vendor]
|
|
65
|
+
INFO index.html:9 cdn.example-widgets.net script on / is not in the vendor table; assumed 30 KB [tp-unknown-weight]
|
|
66
|
+
|
|
67
|
+
2 high, 2 medium, 7 low, 1 info
|
|
68
|
+
✔ Passing (fail-on: none).
|
|
69
|
+
```
|
|
70
|
+
|
|
71
|
+
## Options
|
|
72
|
+
|
|
73
|
+
```text
|
|
74
|
+
thirdparty-pulse 1.0.0 — third-party script weight and load-cost report with budgets
|
|
75
|
+
|
|
76
|
+
Usage: thirdparty-pulse [site-dir] [options]
|
|
77
|
+
|
|
78
|
+
Site
|
|
79
|
+
--base-url <url> Public URL of the site, e.g. https://example.com (default: inferred from canonical/og:url)
|
|
80
|
+
--include <globs> Only check these pages (file or URL path globs, comma-separated)
|
|
81
|
+
--exclude <globs> Skip these pages
|
|
82
|
+
--clean-urls Pages are served without .html (about.html -> /about)
|
|
83
|
+
Budgets
|
|
84
|
+
--max-kb <n> Third-party script KB per page (default 150)
|
|
85
|
+
--max-requests <n> Third-party requests per page (default 10)
|
|
86
|
+
--max-main-thread <ms> Estimated third-party main-thread ms per page (default 250)
|
|
87
|
+
|
|
88
|
+
Common
|
|
89
|
+
-c, --config <file> JSON config (default: .thirdparty-pulse.json or "thirdpartyPulse" in package.json)
|
|
90
|
+
-f, --format <fmt> text (default) | json | sarif | markdown
|
|
91
|
+
--json <file> Also write a JSON report
|
|
92
|
+
--sarif <file> Also write SARIF 2.1.0
|
|
93
|
+
--markdown <file> Also write Markdown (e.g. for a job summary)
|
|
94
|
+
--fail-on <sev> critical | high | medium | low | info | none (default: high)
|
|
95
|
+
--list-rules Print rules and default severities
|
|
96
|
+
--no-fixes Hide fix suggestions in text output
|
|
97
|
+
--no-color Disable colours
|
|
98
|
+
-v, --version / -h, --help
|
|
99
|
+
|
|
100
|
+
Exit codes: 0 = passed, 1 = findings at or above --fail-on, 2 = usage, config or read error.
|
|
101
|
+
```
|
|
102
|
+
|
|
103
|
+
## Configuration and allowlist
|
|
104
|
+
|
|
105
|
+
Options can live in `.thirdparty-pulse.json` (or a `"thirdpartyPulse"` key in `package.json`, or `--config <file>`):
|
|
106
|
+
|
|
107
|
+
```json
|
|
108
|
+
{
|
|
109
|
+
"failOn": "medium",
|
|
110
|
+
"severity": {
|
|
111
|
+
"tp-budget-bytes": "low"
|
|
112
|
+
},
|
|
113
|
+
"allowlist": [
|
|
114
|
+
{
|
|
115
|
+
"rule": "tp-heavy-vendor",
|
|
116
|
+
"subject": "intercom",
|
|
117
|
+
"reason": "Support chat is a product requirement; facade planned",
|
|
118
|
+
"expires": "2027-03-31"
|
|
119
|
+
}
|
|
120
|
+
],
|
|
121
|
+
"maxKb": 120,
|
|
122
|
+
"maxMainThreadMs": 200,
|
|
123
|
+
"vendorBudgets": {
|
|
124
|
+
"google-tag-manager": 80
|
|
125
|
+
},
|
|
126
|
+
"weights": {
|
|
127
|
+
"cdn.example-widgets.net": 45
|
|
128
|
+
}
|
|
129
|
+
}
|
|
130
|
+
```
|
|
131
|
+
|
|
132
|
+
- **`severity`** changes a rule's severity, or turns it `"off"`.
|
|
133
|
+
- **`allowlist`** entries suppress matching findings. Each needs a `reason` and matches on any combination of `rule`, `file`, `subject` (globs with `*`) and `match` (a substring of the message). Add `expires` (`YYYY-MM-DD`) so the exception is time-boxed: once it lapses the finding comes back along with an `allowlist-expired` finding (medium). Entries that no longer match anything are reported as `allowlist-unused` (info).
|
|
134
|
+
|
|
135
|
+
Config can also set `baseUrl`, `include`, `exclude` and `cleanUrls`. The config file is looked up in the site directory first, then in the current directory, so `.thirdparty-pulse.json` can live at your repository root. Other keys: `maxKb`, `maxRequests`, `maxMainThreadMs`, `msPerKb`, `unknownKb`, `vendorBudgets` (vendor id or host → KB) and `weights` (host, URL glob or vendor id → KB).
|
|
136
|
+
|
|
137
|
+
## Output and exit codes
|
|
138
|
+
|
|
139
|
+
- `--format text` (default), `json`, `sarif` (SARIF 2.1.0 for GitHub code scanning) or `markdown` (for job summaries and PR comments). `--json`, `--sarif` and `--markdown` write extra report files in the same run.
|
|
140
|
+
- `--fail-on <severity>` sets the threshold (`critical`, `high`, `medium`, `low`, `info`, `none`; default `high`).
|
|
141
|
+
- Exit codes: **0** passed, **1** findings at or above `--fail-on`, **2** usage, config or read error.
|
|
142
|
+
|
|
143
|
+
## GitHub Action
|
|
144
|
+
|
|
145
|
+
```yaml
|
|
146
|
+
permissions:
|
|
147
|
+
contents: read
|
|
148
|
+
security-events: write # only needed for the SARIF upload
|
|
149
|
+
jobs:
|
|
150
|
+
thirdparty-pulse:
|
|
151
|
+
runs-on: ubuntu-latest
|
|
152
|
+
steps:
|
|
153
|
+
- uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4
|
|
154
|
+
with: { persist-credentials: false }
|
|
155
|
+
- run: npm ci && npm run build # produce the static site first
|
|
156
|
+
- uses: MrLouisWright/thirdparty-pulse@v1
|
|
157
|
+
with:
|
|
158
|
+
path: dist
|
|
159
|
+
fail-on: high
|
|
160
|
+
sarif: thirdparty-pulse.sarif
|
|
161
|
+
- uses: github/codeql-action/upload-sarif@v3
|
|
162
|
+
if: always()
|
|
163
|
+
with: { sarif_file: thirdparty-pulse.sarif }
|
|
164
|
+
```
|
|
165
|
+
|
|
166
|
+
| Input | Description | Default |
|
|
167
|
+
|---|---|---|
|
|
168
|
+
| `path` | Built site directory (e.g. dist, public, _site) | `.` |
|
|
169
|
+
| `base-url` | Public URL of the site, e.g. https://example.com | |
|
|
170
|
+
| `fail-on` | Failure threshold | `high` |
|
|
171
|
+
| `config` | Config file path | |
|
|
172
|
+
| `sarif` / `json` | Write SARIF / JSON report to this path | |
|
|
173
|
+
| `job-summary` | Add the Markdown report to the job summary | `true` |
|
|
174
|
+
| `args` | Extra CLI arguments | |
|
|
175
|
+
|
|
176
|
+
The action runs the bundled CLI with Node 20; it has no dependencies to install. Pin it to a commit SHA if your policy requires it.
|
|
177
|
+
|
|
178
|
+
## Library
|
|
179
|
+
|
|
180
|
+
```js
|
|
181
|
+
import { check, weightOf, vendorForUrl } from "thirdparty-pulse";
|
|
182
|
+
const report = check("dist", { maxKb: 120 });
|
|
183
|
+
report.info.vendors; // [{ id, vendor, category, kb, pages, requests }]
|
|
184
|
+
vendorForUrl("https://www.googletagmanager.com/gtm.js").kb; // 90
|
|
185
|
+
```
|
|
186
|
+
|
|
187
|
+
Types are included.
|
|
188
|
+
|
|
189
|
+
## Requirements
|
|
190
|
+
|
|
191
|
+
Node.js 18 or newer. Zero runtime dependencies.
|
|
192
|
+
|
|
193
|
+
## Licence
|
|
194
|
+
|
|
195
|
+
MIT © Louis Wright. See [LICENSE](LICENSE).
|
package/action.yml
ADDED
|
@@ -0,0 +1,47 @@
|
|
|
1
|
+
name: 'thirdparty-pulse'
|
|
2
|
+
description: 'Third-party script weight and load-cost report with budgets'
|
|
3
|
+
author: 'Louis Wright'
|
|
4
|
+
branding: { icon: 'package', color: 'blue' }
|
|
5
|
+
inputs:
|
|
6
|
+
path: { description: 'Built site directory (e.g. dist, public, _site)', default: '.' }
|
|
7
|
+
base-url: { description: 'Public URL of the site, e.g. https://example.com', default: '' }
|
|
8
|
+
fail-on: { description: 'critical | high | medium | low | info | none', default: 'high' }
|
|
9
|
+
config: { description: 'Path to JSON config file', default: '' }
|
|
10
|
+
sarif: { description: 'Write SARIF to this path', default: '' }
|
|
11
|
+
json: { description: 'Write JSON report to this path', default: '' }
|
|
12
|
+
job-summary: { description: 'Write the Markdown report to the job summary (true/false)', default: 'true' }
|
|
13
|
+
args: { description: 'Extra CLI arguments', default: '' }
|
|
14
|
+
outputs:
|
|
15
|
+
markdown-file: { description: 'Path of the Markdown report', value: '${{ steps.run.outputs.markdown-file }}' }
|
|
16
|
+
runs:
|
|
17
|
+
using: 'composite'
|
|
18
|
+
steps:
|
|
19
|
+
- uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4
|
|
20
|
+
with: { node-version: '20' }
|
|
21
|
+
- id: run
|
|
22
|
+
shell: bash
|
|
23
|
+
env:
|
|
24
|
+
XB_PATH: ${{ inputs.path }}
|
|
25
|
+
XB_BASE_URL: ${{ inputs.base-url }}
|
|
26
|
+
XB_FAIL_ON: ${{ inputs.fail-on }}
|
|
27
|
+
XB_CONFIG: ${{ inputs.config }}
|
|
28
|
+
XB_SARIF: ${{ inputs.sarif }}
|
|
29
|
+
XB_JSON: ${{ inputs.json }}
|
|
30
|
+
XB_SUMMARY: ${{ inputs.job-summary }}
|
|
31
|
+
XB_ARGS: ${{ inputs.args }}
|
|
32
|
+
XB_BIN: ${{ github.action_path }}/bin/thirdparty-pulse.js
|
|
33
|
+
run: |
|
|
34
|
+
md="$RUNNER_TEMP/thirdparty-pulse.md"
|
|
35
|
+
args=("$XB_PATH" --fail-on "$XB_FAIL_ON" --markdown "$md")
|
|
36
|
+
[ -n "$XB_BASE_URL" ] && args+=(--base-url "$XB_BASE_URL")
|
|
37
|
+
[ -n "$XB_CONFIG" ] && args+=(--config "$XB_CONFIG")
|
|
38
|
+
[ -n "$XB_SARIF" ] && args+=(--sarif "$XB_SARIF")
|
|
39
|
+
[ -n "$XB_JSON" ] && args+=(--json "$XB_JSON")
|
|
40
|
+
set +e
|
|
41
|
+
# shellcheck disable=SC2086
|
|
42
|
+
node "$XB_BIN" "${args[@]}" $XB_ARGS
|
|
43
|
+
code=$?
|
|
44
|
+
set -e
|
|
45
|
+
echo "markdown-file=$md" >> "$GITHUB_OUTPUT"
|
|
46
|
+
if [ "$XB_SUMMARY" = "true" ] && [ -f "$md" ]; then cat "$md" >> "$GITHUB_STEP_SUMMARY"; fi
|
|
47
|
+
exit $code
|
|
@@ -0,0 +1,18 @@
|
|
|
1
|
+
#!/usr/bin/env node
|
|
2
|
+
import { readFileSync } from 'node:fs';
|
|
3
|
+
import { mainCli, check, RULES, SITE_OPTIONS, SITE_HELP } from '../src/index.js';
|
|
4
|
+
|
|
5
|
+
const pkg = JSON.parse(readFileSync(new URL('../package.json', import.meta.url), 'utf8'));
|
|
6
|
+
const num = (v) => (v == null ? undefined : Number(v));
|
|
7
|
+
mainCli({
|
|
8
|
+
name: 'thirdparty-pulse', version: pkg.version, configKey: 'thirdpartyPulse', rules: RULES,
|
|
9
|
+
tagline: 'third-party script weight and load-cost report with budgets',
|
|
10
|
+
usage: 'Usage: thirdparty-pulse [site-dir] [options]',
|
|
11
|
+
help: `${SITE_HELP}Budgets
|
|
12
|
+
--max-kb <n> Third-party script KB per page (default 150)
|
|
13
|
+
--max-requests <n> Third-party requests per page (default 10)
|
|
14
|
+
--max-main-thread <ms> Estimated third-party main-thread ms per page (default 250)
|
|
15
|
+
`,
|
|
16
|
+
options: { ...SITE_OPTIONS, 'max-kb': { type: 'string' }, 'max-requests': { type: 'string' }, 'max-main-thread': { type: 'string' } },
|
|
17
|
+
run: async ({ dir, values: o, config, configFile }) => check(dir, { cli: o, maxKb: num(o['max-kb']), maxRequests: num(o['max-requests']), maxMainThreadMs: num(o['max-main-thread']), toolVersion: pkg.version }, config, configFile),
|
|
18
|
+
});
|
package/package.json
CHANGED
|
@@ -1,6 +1,54 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "thirdparty-pulse",
|
|
3
|
-
"version": "
|
|
4
|
-
"
|
|
5
|
-
"
|
|
6
|
-
|
|
3
|
+
"version": "1.0.0",
|
|
4
|
+
"description": "Third-party script weight and load-cost report for static sites: estimated bytes, requests and main-thread time per page and per vendor, with budgets, plus sync-script, duplicate and preconnect checks. SARIF, Action. Zero deps.",
|
|
5
|
+
"type": "module",
|
|
6
|
+
"main": "./src/index.js",
|
|
7
|
+
"types": "./src/index.d.ts",
|
|
8
|
+
"exports": {
|
|
9
|
+
".": {
|
|
10
|
+
"types": "./src/index.d.ts",
|
|
11
|
+
"default": "./src/index.js"
|
|
12
|
+
},
|
|
13
|
+
"./package.json": "./package.json"
|
|
14
|
+
},
|
|
15
|
+
"bin": {
|
|
16
|
+
"thirdparty-pulse": "bin/thirdparty-pulse.js"
|
|
17
|
+
},
|
|
18
|
+
"files": [
|
|
19
|
+
"src",
|
|
20
|
+
"bin",
|
|
21
|
+
"action.yml",
|
|
22
|
+
"README.md",
|
|
23
|
+
"CHANGELOG.md",
|
|
24
|
+
"LICENSE"
|
|
25
|
+
],
|
|
26
|
+
"engines": {
|
|
27
|
+
"node": ">=18"
|
|
28
|
+
},
|
|
29
|
+
"scripts": {
|
|
30
|
+
"test": "node --test test/*.test.js"
|
|
31
|
+
},
|
|
32
|
+
"keywords": [
|
|
33
|
+
"third-party",
|
|
34
|
+
"performance",
|
|
35
|
+
"web-performance",
|
|
36
|
+
"performance-budget",
|
|
37
|
+
"javascript",
|
|
38
|
+
"core-web-vitals",
|
|
39
|
+
"static-site",
|
|
40
|
+
"lighthouse",
|
|
41
|
+
"sarif",
|
|
42
|
+
"github-action"
|
|
43
|
+
],
|
|
44
|
+
"author": "Louis Wright <hello@louiswright.co.uk> (https://github.com/MrLouisWright)",
|
|
45
|
+
"license": "MIT",
|
|
46
|
+
"repository": {
|
|
47
|
+
"type": "git",
|
|
48
|
+
"url": "git+https://github.com/MrLouisWright/thirdparty-pulse.git"
|
|
49
|
+
},
|
|
50
|
+
"bugs": {
|
|
51
|
+
"url": "https://github.com/MrLouisWright/thirdparty-pulse/issues"
|
|
52
|
+
},
|
|
53
|
+
"homepage": "https://github.com/MrLouisWright/thirdparty-pulse#readme"
|
|
54
|
+
}
|
package/src/core.d.ts
ADDED
|
@@ -0,0 +1,16 @@
|
|
|
1
|
+
export type Severity = 'critical' | 'high' | 'medium' | 'low' | 'info';
|
|
2
|
+
export interface Finding { rule: string; severity: Severity; message: string; fix?: string; file?: string; line?: number; subject?: string; [k: string]: unknown }
|
|
3
|
+
export interface AllowlistEntry { rule?: string; file?: string; subject?: string; match?: string; reason: string; expires?: string }
|
|
4
|
+
export interface Config { failOn?: Severity | 'none'; allowlist?: AllowlistEntry[]; severity?: Record<string, Severity | 'off'>; [k: string]: unknown }
|
|
5
|
+
export interface Rule { severity: Severity; title: string }
|
|
6
|
+
export interface Report { tool: { name: string; version: string; rules: Record<string, Rule> }; target: string; summaryLine?: string; counts: Record<Severity, number>; findings: Finding[]; suppressed: (Finding & { reason: string; expires: string | null })[]; [k: string]: unknown }
|
|
7
|
+
export const SEVERITIES: Severity[];
|
|
8
|
+
export const POLICY_RULES: Record<string, Rule>;
|
|
9
|
+
export class UsageError extends Error {}
|
|
10
|
+
export function applyPolicy(raw: Finding[], config?: Config, opts?: { now?: Date; configFile?: string }): { findings: Finding[]; suppressed: Report['suppressed'] };
|
|
11
|
+
export function validateConfig(config: unknown): Config;
|
|
12
|
+
export function exceeds(report: Report, failOn: Severity | 'none'): boolean;
|
|
13
|
+
export function formatText(report: Report, opts?: { color?: boolean; failOn?: string; fixes?: boolean }): string;
|
|
14
|
+
export function formatJson(report: Report): string;
|
|
15
|
+
export function formatSarif(report: Report): string;
|
|
16
|
+
export function formatMarkdown(report: Report, opts?: { failOn?: string }): string;
|
package/src/core.js
ADDED
|
@@ -0,0 +1,176 @@
|
|
|
1
|
+
// Shared reporting / policy / CLI core. Zero dependencies. Louis Wright, MIT.
|
|
2
|
+
import { readFileSync, writeFileSync, existsSync } from 'node:fs';
|
|
3
|
+
import { join, resolve } from 'node:path';
|
|
4
|
+
import { parseArgs } from 'node:util';
|
|
5
|
+
|
|
6
|
+
export const SEVERITIES = ['critical', 'high', 'medium', 'low', 'info'];
|
|
7
|
+
export const sevRank = (s) => { const i = SEVERITIES.indexOf(s); return i === -1 ? 99 : i; };
|
|
8
|
+
export class UsageError extends Error {}
|
|
9
|
+
|
|
10
|
+
export const POLICY_RULES = {
|
|
11
|
+
'allowlist-expired': { severity: 'medium', title: 'Allowlist entry has expired' },
|
|
12
|
+
'allowlist-unused': { severity: 'info', title: 'Allowlist entry matched nothing' },
|
|
13
|
+
};
|
|
14
|
+
|
|
15
|
+
function globRe(p) { return new RegExp('^' + String(p).replace(/[.+^${}()|[\]\\]/g, '\\$&').replace(/\*/g, '.*').replace(/\?/g, '.') + '$', 'i'); }
|
|
16
|
+
|
|
17
|
+
/** Does an allowlist entry ({rule, file, match, subject}) cover a finding? */
|
|
18
|
+
export function entryMatches(e, f) {
|
|
19
|
+
if (e.rule && !globRe(e.rule).test(f.rule)) return false;
|
|
20
|
+
if (e.file && !globRe(e.file).test(f.file || '')) return false;
|
|
21
|
+
if (e.subject && !globRe(e.subject).test(f.subject || '')) return false;
|
|
22
|
+
if (e.match && !String(f.message).toLowerCase().includes(String(e.match).toLowerCase())) return false;
|
|
23
|
+
return !!(e.rule || e.file || e.subject || e.match);
|
|
24
|
+
}
|
|
25
|
+
|
|
26
|
+
export function validateConfig(config) {
|
|
27
|
+
if (config == null || typeof config !== 'object' || Array.isArray(config)) throw new UsageError('config must be a JSON object');
|
|
28
|
+
if (config.allowlist != null && !Array.isArray(config.allowlist)) throw new UsageError('"allowlist" must be an array');
|
|
29
|
+
for (const [i, e] of (config.allowlist || []).entries()) {
|
|
30
|
+
if (!e || typeof e !== 'object') throw new UsageError(`allowlist[${i}] must be an object`);
|
|
31
|
+
if (!e.rule && !e.file && !e.subject && !e.match) throw new UsageError(`allowlist[${i}] needs at least one of rule, file, subject, match`);
|
|
32
|
+
if (!e.reason) throw new UsageError(`allowlist[${i}] needs a "reason"`);
|
|
33
|
+
if (e.expires != null && (!/^\d{4}-\d{2}-\d{2}$/.test(e.expires) || Number.isNaN(Date.parse(e.expires)))) throw new UsageError(`allowlist[${i}].expires must be YYYY-MM-DD`);
|
|
34
|
+
}
|
|
35
|
+
for (const [k, v] of Object.entries(config.severity || {})) if (v !== 'off' && !SEVERITIES.includes(v)) throw new UsageError(`severity.${k} must be one of ${SEVERITIES.join(', ')}, off`);
|
|
36
|
+
return config;
|
|
37
|
+
}
|
|
38
|
+
|
|
39
|
+
/** Apply severity overrides and the allowlist. Returns { findings, suppressed }. */
|
|
40
|
+
export function applyPolicy(raw, config = {}, { now = new Date(), configFile = 'config' } = {}) {
|
|
41
|
+
const today = now.toISOString().slice(0, 10);
|
|
42
|
+
const sev = config.severity || {};
|
|
43
|
+
const list = (config.allowlist || []).map((e, i) => ({ ...e, index: i, used: 0, expired: !!(e.expires && e.expires < today) }));
|
|
44
|
+
const findings = []; const suppressed = [];
|
|
45
|
+
for (const f0 of raw) {
|
|
46
|
+
const o = sev[f0.rule];
|
|
47
|
+
if (o === 'off') continue;
|
|
48
|
+
const f = o ? { ...f0, severity: o } : f0;
|
|
49
|
+
const hit = list.find((e) => entryMatches(e, f));
|
|
50
|
+
if (hit) { hit.used++; if (!hit.expired) { suppressed.push({ ...f, reason: hit.reason, expires: hit.expires || null }); continue; } }
|
|
51
|
+
findings.push(f);
|
|
52
|
+
}
|
|
53
|
+
for (const e of list) {
|
|
54
|
+
const label = [e.rule, e.file, e.subject, e.match].filter(Boolean).join(' / ');
|
|
55
|
+
if (e.expired && sev['allowlist-expired'] !== 'off') findings.push({ rule: 'allowlist-expired', severity: sev['allowlist-expired'] || 'medium', file: configFile, message: `Allowlist entry #${e.index + 1} (${label}) expired on ${e.expires}`, fix: 'Fix the underlying finding, or renew the entry with a new "expires" date and reason.' });
|
|
56
|
+
else if (!e.used && sev['allowlist-unused'] !== 'off') findings.push({ rule: 'allowlist-unused', severity: sev['allowlist-unused'] || 'info', file: configFile, message: `Allowlist entry #${e.index + 1} (${label}) matched no finding`, fix: 'Remove the stale entry.' });
|
|
57
|
+
}
|
|
58
|
+
findings.sort((a, b) => sevRank(a.severity) - sevRank(b.severity) || String(a.file || '').localeCompare(String(b.file || '')) || (a.line || 0) - (b.line || 0));
|
|
59
|
+
return { findings, suppressed };
|
|
60
|
+
}
|
|
61
|
+
|
|
62
|
+
export function countBy(findings) { const c = {}; for (const s of SEVERITIES) c[s] = 0; for (const f of findings) c[f.severity] = (c[f.severity] || 0) + 1; return c; }
|
|
63
|
+
export const exceeds = (r, failOn) => !!failOn && failOn !== 'none' && r.findings.some((f) => sevRank(f.severity) <= sevRank(failOn));
|
|
64
|
+
|
|
65
|
+
const C = { critical: '\x1b[35;1m', high: '\x1b[31;1m', medium: '\x1b[33m', low: '\x1b[36m', info: '\x1b[2m', dim: '\x1b[2m', bold: '\x1b[1m', reset: '\x1b[0m', red: '\x1b[31m', green: '\x1b[32m' };
|
|
66
|
+
const sumText = (counts) => SEVERITIES.filter((k) => counts[k]).map((k) => `${counts[k]} ${k}`).join(', ') || 'no findings';
|
|
67
|
+
const loc = (f) => (f.file ? `${f.file}${f.line ? ':' + f.line : ''}` : '');
|
|
68
|
+
|
|
69
|
+
export function formatText(r, { color = false, failOn, fixes = true } = {}) {
|
|
70
|
+
const c = (k, s) => (color ? `${C[k]}${s}${C.reset}` : s);
|
|
71
|
+
const l = [c('bold', `${r.tool.name} — ${r.target}`)];
|
|
72
|
+
if (r.summaryLine) l.push(c('dim', r.summaryLine));
|
|
73
|
+
l.push('');
|
|
74
|
+
for (const f of r.findings) {
|
|
75
|
+
l.push(`${c(f.severity, f.severity.toUpperCase().padEnd(8))} ${loc(f) ? c('dim', loc(f) + ' ') : ''}${f.message} [${f.rule}]`);
|
|
76
|
+
if (fixes && f.fix) l.push(c('dim', ` ↳ fix: ${f.fix}`));
|
|
77
|
+
}
|
|
78
|
+
if (r.findings.length) l.push('');
|
|
79
|
+
if (r.suppressed.length) l.push(c('dim', `${r.suppressed.length} finding(s) covered by the allowlist`));
|
|
80
|
+
l.push(sumText(r.counts));
|
|
81
|
+
if (failOn) l.push(exceeds(r, failOn) ? c('red', `✖ Failing (fail-on: ${failOn}).`) : c('green', `✔ Passing (fail-on: ${failOn}).`));
|
|
82
|
+
return l.join('\n') + '\n';
|
|
83
|
+
}
|
|
84
|
+
|
|
85
|
+
export const formatJson = (r) => JSON.stringify(r, null, 2) + '\n';
|
|
86
|
+
|
|
87
|
+
const LEVEL = { critical: 'error', high: 'error', medium: 'warning', low: 'note', info: 'note' };
|
|
88
|
+
const SCORE = { critical: '9.0', high: '7.5', medium: '5.0', low: '3.0', info: '1.0' };
|
|
89
|
+
export function formatSarif(r) {
|
|
90
|
+
const rules = { ...r.tool.rules, ...POLICY_RULES };
|
|
91
|
+
const ids = Object.keys(rules);
|
|
92
|
+
const repo = `https://github.com/MrLouisWright/${r.tool.name}`;
|
|
93
|
+
return JSON.stringify({
|
|
94
|
+
$schema: 'https://json.schemastore.org/sarif-2.1.0.json', version: '2.1.0',
|
|
95
|
+
runs: [{
|
|
96
|
+
tool: { driver: { name: r.tool.name, version: r.tool.version, informationUri: repo, rules: ids.map((id) => ({ id, name: id.replace(/(^|-)(\w)/g, (_, __, ch) => ch.toUpperCase()), shortDescription: { text: rules[id].title }, helpUri: `${repo}#rules`, defaultConfiguration: { level: LEVEL[rules[id].severity] }, properties: { 'security-severity': SCORE[rules[id].severity] } })) } },
|
|
97
|
+
results: r.findings.map((f) => ({ ruleId: f.rule, ruleIndex: ids.indexOf(f.rule), level: LEVEL[f.severity], message: { text: f.fix ? `${f.message}. Fix: ${f.fix}` : f.message }, locations: [{ physicalLocation: { artifactLocation: { uri: f.file || r.tool.defaultFile || '.' }, region: { startLine: f.line || 1 } } }], partialFingerprints: { ruleSubject: `${f.rule}:${f.file || ''}:${f.subject || f.message}` }, properties: { severity: f.severity } })),
|
|
98
|
+
}],
|
|
99
|
+
}, null, 2) + '\n';
|
|
100
|
+
}
|
|
101
|
+
|
|
102
|
+
const esc = (s) => String(s ?? '').replace(/\|/g, '\\|').replace(/\n/g, ' ');
|
|
103
|
+
export function formatMarkdown(r, { failOn } = {}) {
|
|
104
|
+
const l = [`## ${r.tool.emoji || '📏'} ${r.tool.name}`, '', `**${esc(r.target)}**: ${r.summaryLine ? esc(r.summaryLine) + '; ' : ''}${sumText(r.counts)}.`];
|
|
105
|
+
if (failOn) l.push('', exceeds(r, failOn) ? `❌ Failing at \`--fail-on ${failOn}\`.` : `✅ Passing at \`--fail-on ${failOn}\`.`);
|
|
106
|
+
if (r.findings.length) {
|
|
107
|
+
l.push('', '| Severity | Location | Rule | Finding | Fix |', '|---|---|---|---|---|');
|
|
108
|
+
for (const f of r.findings) l.push(`| ${f.severity} | ${loc(f) ? '`' + esc(loc(f)) + '`' : ''} | \`${f.rule}\` | ${esc(f.message)} | ${esc(f.fix)} |`);
|
|
109
|
+
}
|
|
110
|
+
if (r.suppressed.length) l.push('', `_${r.suppressed.length} finding(s) covered by the allowlist._`);
|
|
111
|
+
return l.join('\n') + '\n';
|
|
112
|
+
}
|
|
113
|
+
|
|
114
|
+
/** Build a report object from raw findings. */
|
|
115
|
+
export function buildReport(tool, target, raw, config, extra = {}) {
|
|
116
|
+
const { findings, suppressed } = applyPolicy(raw, config, { now: extra.now, configFile: extra.configFile });
|
|
117
|
+
const { now, configFile, ...rest } = extra;
|
|
118
|
+
return { tool: { name: tool.name, version: tool.version, rules: tool.rules, emoji: tool.emoji, defaultFile: tool.defaultFile }, target, ...rest, counts: countBy(findings), findings, suppressed };
|
|
119
|
+
}
|
|
120
|
+
|
|
121
|
+
export function loadConfig(dir, name, key, explicit) {
|
|
122
|
+
const file = explicit ? resolve(explicit) : join(dir, `.${name}.json`);
|
|
123
|
+
if (explicit || existsSync(file)) {
|
|
124
|
+
let c; try { c = JSON.parse(readFileSync(file, 'utf8')); } catch (e) { throw new UsageError(`Cannot read config ${file}: ${e.message}`); }
|
|
125
|
+
return { config: validateConfig(c), configFile: explicit || `.${name}.json` };
|
|
126
|
+
}
|
|
127
|
+
for (const d of [dir, process.cwd()]) {
|
|
128
|
+
if (d !== dir && existsSync(join(d, `.${name}.json`))) return loadConfig(d, name, key, join(d, `.${name}.json`));
|
|
129
|
+
try { const pj = JSON.parse(readFileSync(join(d, 'package.json'), 'utf8')); if (pj[key]) return { config: validateConfig(pj[key]), configFile: 'package.json' }; } catch (e) { if (e instanceof UsageError) throw e; }
|
|
130
|
+
}
|
|
131
|
+
return { config: {}, configFile: null };
|
|
132
|
+
}
|
|
133
|
+
|
|
134
|
+
const list = (v) => (v || []).flatMap((x) => String(x).split(',')).map((x) => x.trim()).filter(Boolean);
|
|
135
|
+
export { list as splitList };
|
|
136
|
+
|
|
137
|
+
/**
|
|
138
|
+
* Generic CLI. spec = { name, version, configKey, rules, help (tool-specific options text), options (parseArgs), defaultFailOn,
|
|
139
|
+
* run: async ({ dir, positionals, values, config, configFile, stdin }) => report }
|
|
140
|
+
*/
|
|
141
|
+
export async function runCli(spec, argv = process.argv.slice(2), io = { stdout: process.stdout, stderr: process.stderr }) {
|
|
142
|
+
const usage = (m) => { io.stderr.write(`${spec.name}: ${m}\nRun with --help for usage.\n`); return 2; };
|
|
143
|
+
let a;
|
|
144
|
+
try {
|
|
145
|
+
a = parseArgs({ args: argv, allowPositionals: true, options: { ...spec.options, config: { type: 'string', short: 'c' }, format: { type: 'string', short: 'f' }, json: { type: 'string' }, sarif: { type: 'string' }, markdown: { type: 'string' }, 'fail-on': { type: 'string' }, 'no-color': { type: 'boolean' }, 'no-fixes': { type: 'boolean' }, 'list-rules': { type: 'boolean' }, version: { type: 'boolean', short: 'v' }, help: { type: 'boolean', short: 'h' } } });
|
|
146
|
+
} catch (e) { return usage(e.message); }
|
|
147
|
+
const { values: o, positionals } = a;
|
|
148
|
+
const all = { ...spec.rules, ...POLICY_RULES };
|
|
149
|
+
if (o.help) { io.stdout.write(`${spec.name} ${spec.version} — ${spec.tagline}\n\n${spec.usage}\n\n${spec.help}\nCommon\n -c, --config <file> JSON config (default: .${spec.name}.json or "${spec.configKey}" in package.json)\n -f, --format <fmt> text (default) | json | sarif | markdown\n --json <file> Also write a JSON report\n --sarif <file> Also write SARIF 2.1.0\n --markdown <file> Also write Markdown (e.g. for a job summary)\n --fail-on <sev> critical | high | medium | low | info | none (default: ${spec.defaultFailOn || 'high'})\n --list-rules Print rules and default severities\n --no-fixes Hide fix suggestions in text output\n --no-color Disable colours\n -v, --version / -h, --help\n\nExit codes: 0 = passed, 1 = findings at or above --fail-on, 2 = usage, config or read error.\n`); return 0; }
|
|
150
|
+
if (o.version) { io.stdout.write(spec.version + '\n'); return 0; }
|
|
151
|
+
if (o['list-rules']) { for (const [id, r] of Object.entries(all)) io.stdout.write(`${id.padEnd(28)} ${r.severity.padEnd(9)} ${r.title}\n`); return 0; }
|
|
152
|
+
const format = o.format || 'text';
|
|
153
|
+
if (!['text', 'json', 'sarif', 'markdown'].includes(format)) return usage(`Invalid --format: ${format}`);
|
|
154
|
+
let report; let failOn;
|
|
155
|
+
try {
|
|
156
|
+
const dir = resolve(spec.dirFrom ? spec.dirFrom(o, positionals) : (positionals[0] || '.'));
|
|
157
|
+
if (!existsSync(dir)) return usage(`No such directory: ${dir}`);
|
|
158
|
+
const { config, configFile } = loadConfig(dir, spec.name, spec.configKey, o.config);
|
|
159
|
+
failOn = o['fail-on'] || config.failOn || spec.defaultFailOn || 'high';
|
|
160
|
+
if (failOn !== 'none' && !SEVERITIES.includes(failOn)) return usage(`Invalid --fail-on: ${failOn}`);
|
|
161
|
+
report = await spec.run({ dir, positionals, values: o, config, configFile: configFile || `.${spec.name}.json` });
|
|
162
|
+
} catch (e) { if (e instanceof UsageError || e.code === 'ENOENT' || e.code === 'EACCES') return usage(e.message); throw e; }
|
|
163
|
+
const color = !o['no-color'] && io.stdout.isTTY && !process.env.NO_COLOR;
|
|
164
|
+
const out = { text: () => formatText(report, { color, failOn, fixes: !o['no-fixes'] }), json: () => formatJson(report), sarif: () => formatSarif(report), markdown: () => formatMarkdown(report, { failOn }) };
|
|
165
|
+
io.stdout.write(out[format]());
|
|
166
|
+
try {
|
|
167
|
+
if (o.json) writeFileSync(o.json, formatJson(report));
|
|
168
|
+
if (o.sarif) writeFileSync(o.sarif, formatSarif(report));
|
|
169
|
+
if (o.markdown) writeFileSync(o.markdown, formatMarkdown(report, { failOn }));
|
|
170
|
+
} catch (e) { return usage(`Cannot write output: ${e.message}`); }
|
|
171
|
+
return exceeds(report, failOn) ? 1 : 0;
|
|
172
|
+
}
|
|
173
|
+
|
|
174
|
+
export function mainCli(spec) {
|
|
175
|
+
runCli(spec).then((c) => { process.exitCode = c; }, (e) => { process.stderr.write(`${spec.name}: ${e.stack || e}\n`); process.exitCode = 2; });
|
|
176
|
+
}
|
package/src/index.d.ts
ADDED
|
@@ -0,0 +1,10 @@
|
|
|
1
|
+
import type { Report, Rule } from './core.js';
|
|
2
|
+
export * from './core.js';
|
|
3
|
+
export interface Vendor { id: string; name: string; company: string; category: string; domains: string[]; kb: number }
|
|
4
|
+
export interface ThirdPartyOptions { baseUrl?: string; include?: string[]; exclude?: string[]; cleanUrls?: boolean; maxKb?: number; maxRequests?: number; maxMainThreadMs?: number; toolVersion?: string; now?: Date }
|
|
5
|
+
export declare const RULES: Record<string, Rule>;
|
|
6
|
+
export declare const DEFAULTS: { maxKb: number; maxRequests: number; maxMainThreadMs: number; msPerKb: number; unknownKb: number; heavyKb: number };
|
|
7
|
+
export declare const VENDORS: Vendor[];
|
|
8
|
+
export declare function vendorForUrl(url: string): Vendor | null;
|
|
9
|
+
export declare function weightOf(res: { url?: string | null; host: string; vendor?: Vendor | null }, config?: Record<string, unknown>): { kb: number; known: boolean };
|
|
10
|
+
export declare function check(dir: string, opts?: ThirdPartyOptions, config?: Record<string, unknown>, configFile?: string): Report;
|
package/src/index.js
ADDED
|
@@ -0,0 +1,86 @@
|
|
|
1
|
+
// thirdparty-pulse: third-party script weight and load-cost report for a built static site.
|
|
2
|
+
// Per page and per vendor: requests, estimated bytes and estimated main-thread time, checked against budgets,
|
|
3
|
+
// plus loading mistakes (synchronous third-party scripts, duplicates, missing/unused preconnects).
|
|
4
|
+
// Performance only: this is not a privacy or consent inventory. Zero dependencies. Louis Wright, MIT.
|
|
5
|
+
import { runBeacon, finder, globToRe, contentPages } from './site.js';
|
|
6
|
+
import { buildReport, UsageError } from './core.js';
|
|
7
|
+
import { thirdPartyResources, hostOf } from './vendors.js';
|
|
8
|
+
|
|
9
|
+
export * from './core.js';
|
|
10
|
+
export { loadSite, parseHtml, SITE_OPTIONS, SITE_HELP } from './site.js';
|
|
11
|
+
export { VENDORS, vendorForUrl } from './vendors.js';
|
|
12
|
+
|
|
13
|
+
export const RULES = {
|
|
14
|
+
'tp-budget-bytes': { severity: 'high', title: 'Third-party script weight on a page exceeds the byte budget' },
|
|
15
|
+
'tp-vendor-budget': { severity: 'high', title: 'A single vendor exceeds its configured byte budget' },
|
|
16
|
+
'tp-sync-script': { severity: 'high', title: 'Third-party script loads synchronously (no async/defer) and blocks parsing' },
|
|
17
|
+
'tp-budget-main-thread': { severity: 'medium', title: 'Estimated third-party main-thread time on a page exceeds the budget' },
|
|
18
|
+
'tp-budget-requests': { severity: 'medium', title: 'Number of third-party requests on a page exceeds the budget' },
|
|
19
|
+
'tp-duplicate-script': { severity: 'medium', title: 'The same third-party script or vendor loader is included more than once on a page' },
|
|
20
|
+
'tp-heavy-vendor': { severity: 'low', title: 'Heavy vendor script (100 KB+) where a lighter alternative or facade usually exists' },
|
|
21
|
+
'tp-no-preconnect': { severity: 'low', title: 'Early third-party origin has no preconnect / dns-prefetch hint' },
|
|
22
|
+
'tp-preconnect-unused': { severity: 'low', title: 'preconnect to a third-party origin that the page never uses' },
|
|
23
|
+
'tp-unknown-weight': { severity: 'info', title: 'Unrecognised third-party script; a default weight was assumed (set weights in config)' },
|
|
24
|
+
};
|
|
25
|
+
export const DEFAULTS = { maxKb: 150, maxRequests: 10, maxMainThreadMs: 250, msPerKb: 2.5, unknownKb: 30, heavyKb: 100 };
|
|
26
|
+
const HEAVY_TIP = { youtube: 'Use a click-to-load facade (lite-youtube-embed).', 'google-maps': 'Use a static map image that loads the interactive map on click.', intercom: 'Load the chat widget on first interaction.', zendesk: 'Load the chat widget on first interaction.', drift: 'Load the chat widget on first interaction.', hubspot: 'Load forms/chat on interaction.', disqus: 'Load comments when scrolled into view.', 'google-tag-manager': 'Audit tags in the container; remove unused ones.' };
|
|
27
|
+
|
|
28
|
+
/** Estimated weight (KB, compressed) of a third-party resource: config.weights (host or URL glob) > vendor table > unknownKb. */
|
|
29
|
+
export function weightOf(res, config = {}) {
|
|
30
|
+
for (const [k, kb] of Object.entries(config.weights || {})) { const re = globToRe(k); if ((res.url && re.test(res.url.replace(/^https?:\/\//, '').replace(/^\/\//, ''))) || re.test(res.host) || (res.vendor && res.vendor.id === k)) return { kb: Number(kb), known: true }; }
|
|
31
|
+
if (res.vendor) return { kb: res.vendor.kb, known: true };
|
|
32
|
+
return { kb: config.unknownKb ?? DEFAULTS.unknownKb, known: false };
|
|
33
|
+
}
|
|
34
|
+
|
|
35
|
+
function analyze(site, opts, config) {
|
|
36
|
+
const { findings, add } = finder(RULES);
|
|
37
|
+
const c = { ...DEFAULTS, ...config };
|
|
38
|
+
const siteHost = site.origin ? hostOf(site.origin) : null;
|
|
39
|
+
const totals = new Map(); let pagesWithTp = 0; let maxPage = 0;
|
|
40
|
+
const pages = contentPages(site);
|
|
41
|
+
for (const pg of pages) {
|
|
42
|
+
const doc = pg.doc; const res = thirdPartyResources(doc, siteHost);
|
|
43
|
+
const scripts = res.filter((r) => r.kind === 'script' || r.kind === 'inline');
|
|
44
|
+
const requests = res.filter((r) => r.kind !== 'hint' && r.kind !== 'inline');
|
|
45
|
+
if (!scripts.length && !requests.length) continue; pagesWithTp++;
|
|
46
|
+
let kb = 0; const vendorKb = new Map(); const seen = new Map();
|
|
47
|
+
for (const s of scripts) {
|
|
48
|
+
const key = s.url ? s.url.replace(/^https?:/, '') : `inline:${s.vendor.id}`;
|
|
49
|
+
// An inline loader for a vendor whose script tag is also present counts once.
|
|
50
|
+
if (s.inline && scripts.some((o) => !o.inline && o.vendor?.id === s.vendor.id)) continue;
|
|
51
|
+
if (seen.has(key)) { add('tp-duplicate-script', pg.path, `${s.url || s.vendor.name + ' loader'} is included ${seen.get(key) + 1} times on ${pg.path}`, 'Include each third-party script once (check layouts, partials and tag manager tags).', { file: pg.file, line: s.line }); seen.set(key, seen.get(key) + 1); continue; }
|
|
52
|
+
seen.set(key, 1);
|
|
53
|
+
const w = weightOf(s, c); kb += w.kb; const vid = s.vendor?.id || s.host;
|
|
54
|
+
vendorKb.set(vid, (vendorKb.get(vid) || 0) + w.kb);
|
|
55
|
+
const t = totals.get(vid) || { vendor: s.vendor?.name || s.host, category: s.vendor?.category || 'unknown', kb: w.kb, pages: 0, requests: 0 }; t.pages++; t.requests++; totals.set(vid, t);
|
|
56
|
+
if (!w.known && s.url) add('tp-unknown-weight', s.host, `${s.host} script on ${pg.path} is not in the vendor table; assumed ${w.kb} KB`, `Add "weights": { "${s.host}": <KB> } to the config with the real transfer size.`, { file: pg.file, line: s.line });
|
|
57
|
+
if (s.kind === 'script') {
|
|
58
|
+
const a = s.el.attrs;
|
|
59
|
+
if (a.async === undefined && a.defer === undefined && !/module/i.test(a.type || '')) add('tp-sync-script', s.url, `Synchronous third-party script ${s.url} on ${pg.path}${s.el.inHead ? ' in <head>' : ''}`, 'Add async (independent scripts) or defer (order-dependent), or load it after the page is interactive.', { file: pg.file, line: s.line, severity: s.el.inHead ? undefined : 'medium' });
|
|
60
|
+
}
|
|
61
|
+
if (s.vendor && s.vendor.kb >= c.heavyKb) add('tp-heavy-vendor', s.vendor.id, `${s.vendor.name} (~${s.vendor.kb} KB) on ${pg.path}`, HEAVY_TIP[s.vendor.id] || 'Load it on interaction or after the load event, or use a lighter alternative.', { file: pg.file, line: s.line });
|
|
62
|
+
}
|
|
63
|
+
for (const r of requests) if (r.kind !== 'script') { const vid = r.vendor?.id || r.host; const t = totals.get(vid) || { vendor: r.vendor?.name || r.host, category: r.vendor?.category || 'unknown', kb: 0, pages: 0, requests: 0 }; t.requests++; totals.set(vid, t); }
|
|
64
|
+
const ms = Math.round(kb * c.msPerKb); maxPage = Math.max(maxPage, kb);
|
|
65
|
+
if (kb > c.maxKb) add('tp-budget-bytes', pg.path, `${pg.path} loads ~${Math.round(kb)} KB of third-party script (budget ${c.maxKb} KB): ${[...vendorKb].sort((a, b) => b[1] - a[1]).map(([v, k]) => `${v} ${Math.round(k)} KB`).join(', ')}`, 'Remove, defer or replace the heaviest vendors, or raise maxKb deliberately.', { file: pg.file });
|
|
66
|
+
if (ms > c.maxMainThreadMs) add('tp-budget-main-thread', pg.path, `${pg.path}: estimated ~${ms} ms of third-party main-thread work (budget ${c.maxMainThreadMs} ms)`, 'Delay non-essential vendors until after load or user interaction; measure with a lab trace to confirm.', { file: pg.file });
|
|
67
|
+
if (requests.length > c.maxRequests) add('tp-budget-requests', pg.path, `${pg.path} makes ${requests.length} third-party requests (budget ${c.maxRequests})`, 'Consolidate vendors or self-host static assets (fonts, libraries).', { file: pg.file });
|
|
68
|
+
for (const [vid, k] of vendorKb) { const b = (c.vendorBudgets || {})[vid]; if (b != null && k > b) add('tp-vendor-budget', vid, `${vid} uses ~${Math.round(k)} KB on ${pg.path} (budget ${b} KB)`, 'Trim this vendor\'s configuration or raise its budget.', { file: pg.file }); }
|
|
69
|
+
// preconnect hints
|
|
70
|
+
const hints = doc.all.filter((e) => e.tag === 'link' && /preconnect|dns-prefetch/i.test(e.attrs.rel || '') && e.attrs.href);
|
|
71
|
+
const hinted = new Set(hints.map((e) => hostOf(e.attrs.href)));
|
|
72
|
+
const used = new Set(res.filter((r) => r.kind !== 'hint').map((r) => r.host));
|
|
73
|
+
for (const h of hints) { const host = hostOf(h.attrs.href); if (/preconnect/i.test(h.attrs.rel) && host && !used.has(host) && !(siteHost && host === siteHost)) add('tp-preconnect-unused', host, `preconnect to ${host} on ${pg.path}, but nothing on the page loads from it`, 'Remove unused preconnects: each opens a connection that competes with critical requests.', { file: pg.file, line: h.line }); }
|
|
74
|
+
const early = new Set(scripts.filter((s) => s.kind === 'script' && s.el.inHead && s.el.attrs.defer === undefined).map((s) => s.host));
|
|
75
|
+
for (const host of early) if (!hinted.has(host)) add('tp-no-preconnect', host, `${host} is requested from <head> on ${pg.path} without a preconnect`, `Add <link rel="preconnect" href="https://${host}" crossorigin> early in <head>.`, { file: pg.file });
|
|
76
|
+
}
|
|
77
|
+
const vendors = [...totals.entries()].map(([id, t]) => ({ id, ...t })).sort((a, b) => b.kb - a.kb);
|
|
78
|
+
return { findings, info: { pagesWithThirdParty: pagesWithTp, maxPageKb: Math.round(maxPage), vendors }, summary: `${pages.length} page(s), ${pagesWithTp} with third-party content, ${vendors.length} vendor(s)/origin(s), heaviest page ~${Math.round(maxPage)} KB of third-party script` };
|
|
79
|
+
}
|
|
80
|
+
|
|
81
|
+
export function check(dir, opts = {}, config = {}, configFile) {
|
|
82
|
+
const tool = { name: 'thirdparty-pulse', version: opts.toolVersion || '1.0.0', rules: RULES, emoji: '📦', defaultFile: 'index.html' };
|
|
83
|
+
const cfg = { ...config }; for (const [k, o] of [['maxKb', 'maxKb'], ['maxRequests', 'maxRequests'], ['maxMainThreadMs', 'maxMainThreadMs']]) if (opts[o] != null) cfg[k] = opts[o];
|
|
84
|
+
for (const k of ['maxKb', 'maxRequests', 'maxMainThreadMs', 'msPerKb', 'unknownKb']) if (cfg[k] != null && !(Number(cfg[k]) >= 0)) throw new UsageError(`${k} must be a non-negative number`);
|
|
85
|
+
return runBeacon(tool, dir, { ...opts, UsageError }, cfg, configFile, analyze, buildReport);
|
|
86
|
+
}
|
package/src/site.js
ADDED
|
@@ -0,0 +1,326 @@
|
|
|
1
|
+
// Shared static-site reader for the *-beacon packages: a small, forgiving HTML parser,
|
|
2
|
+
// a build-output walker that maps files to URLs, and _headers / vercel.json header rules.
|
|
3
|
+
import { readFileSync, readdirSync, statSync, existsSync } from 'node:fs';
|
|
4
|
+
import { join, relative, sep } from 'node:path';
|
|
5
|
+
|
|
6
|
+
const VOID = new Set(['area', 'base', 'br', 'col', 'embed', 'hr', 'img', 'input', 'link', 'meta', 'param', 'source', 'track', 'wbr']);
|
|
7
|
+
const RAW = new Set(['script', 'style', 'textarea', 'title', 'xmp', 'noscript', 'template']);
|
|
8
|
+
const HEADISH = new Set(['html', 'head', 'meta', 'link', 'title', 'base', 'script', 'style', 'noscript', 'template']);
|
|
9
|
+
const BLOCK = new Set(['address', 'article', 'aside', 'blockquote', 'br', 'dd', 'details', 'div', 'dl', 'dt', 'fieldset', 'figcaption', 'figure', 'footer', 'form', 'h1', 'h2', 'h3', 'h4', 'h5', 'h6', 'header', 'hr', 'li', 'main', 'nav', 'ol', 'p', 'pre', 'section', 'summary', 'table', 'td', 'th', 'tr', 'ul', 'option', 'caption']);
|
|
10
|
+
const CLOSES_P = new Set(['address', 'article', 'aside', 'blockquote', 'details', 'div', 'dl', 'fieldset', 'figure', 'footer', 'form', 'h1', 'h2', 'h3', 'h4', 'h5', 'h6', 'header', 'hr', 'main', 'nav', 'ol', 'p', 'pre', 'section', 'table', 'ul']);
|
|
11
|
+
const ENT = { amp: '&', lt: '<', gt: '>', quot: '"', apos: "'", nbsp: '\u00a0', mdash: '—', ndash: '–', hellip: '…', rsquo: '’', lsquo: '‘', ldquo: '“', rdquo: '”', copy: '©', reg: '®', trade: '™', laquo: '«', raquo: '»', middot: '·', bull: '•', times: '×', eacute: 'é', egrave: 'è', aacute: 'á', uuml: 'ü', ouml: 'ö', auml: 'ä', szlig: 'ß', ccedil: 'ç', ntilde: 'ñ', euro: '€', pound: '£', yen: '¥', deg: '°', shy: '' };
|
|
12
|
+
export const decode = (s) => String(s).replace(/&(#x[0-9a-f]+|#\d+|[a-z]+\d*);?/gi, (m, e) => { if (e[0] === '#') { const n = e[1] === 'x' || e[1] === 'X' ? parseInt(e.slice(2), 16) : parseInt(e.slice(1), 10); try { return String.fromCodePoint(n); } catch { return m; } } const v = ENT[e.toLowerCase()]; return v ?? m; });
|
|
13
|
+
export const squash = (s) => String(s).replace(/[\s\u00a0]+/g, ' ').trim();
|
|
14
|
+
|
|
15
|
+
function parseAttrs(src) {
|
|
16
|
+
const attrs = {};
|
|
17
|
+
for (const m of src.matchAll(/([^\s"'<>\/=]+)(?:\s*=\s*(?:"([^"]*)"|'([^']*)'|([^\s"'=<>`]+)))?/g)) { const k = m[1].toLowerCase(); if (!(k in attrs)) attrs[k] = decode(m[2] ?? m[3] ?? m[4] ?? ''); }
|
|
18
|
+
return attrs;
|
|
19
|
+
}
|
|
20
|
+
|
|
21
|
+
/** Parse HTML into a light tree. Elements: { tag, attrs, line, children, parent, inHead }. Text nodes: { text }. */
|
|
22
|
+
export function parseHtml(html) {
|
|
23
|
+
html = String(html).replace(/^\uFEFF/, '');
|
|
24
|
+
const root = { tag: '#document', attrs: {}, children: [], line: 1, parent: null };
|
|
25
|
+
const all = []; let cur = root; let bodyStarted = false; let i = 0; let line = 1;
|
|
26
|
+
const lineAt = (to) => { for (let k = i; k < to; k++) if (html.charCodeAt(k) === 10) line++; i = to; };
|
|
27
|
+
const inHeadEl = () => { for (let n = cur; n; n = n.parent) if (n.tag === 'head') return true; return false; };
|
|
28
|
+
const close = (tag) => { for (let n = cur; n && n !== root; n = n.parent) if (n.tag === tag) { cur = n.parent; return true; } return false; };
|
|
29
|
+
const open = (tag, attrs, ln, selfClose) => {
|
|
30
|
+
if (tag === 'body') bodyStarted = true;
|
|
31
|
+
else if (!HEADISH.has(tag) && !inHeadEl()) bodyStarted = true;
|
|
32
|
+
if (cur.tag === 'head' && !HEADISH.has(tag)) cur = cur.parent; // implicit </head>
|
|
33
|
+
if (CLOSES_P.has(tag)) { for (let n = cur; n && n !== root && !['div', 'section', 'article', 'main', 'body', 'td', 'li', 'blockquote'].includes(n.tag) || n === cur; n = n.parent) { if (n === root) break; if (n.tag === 'p') { cur = n.parent; break; } if (n !== cur) break; } }
|
|
34
|
+
if (tag === 'li') { for (let n = cur; n && n !== root; n = n.parent) { if (n.tag === 'li') { cur = n.parent; break; } if (n.tag === 'ul' || n.tag === 'ol') break; } }
|
|
35
|
+
if (tag === 'dt' || tag === 'dd') { for (let n = cur; n && n !== root; n = n.parent) { if (n.tag === 'dt' || n.tag === 'dd') { cur = n.parent; break; } if (n.tag === 'dl') break; } }
|
|
36
|
+
if (tag === 'option' && cur.tag === 'option') cur = cur.parent;
|
|
37
|
+
if ((tag === 'tr' || tag === 'td' || tag === 'th') && (cur.tag === 'td' || cur.tag === 'th')) cur = cur.parent;
|
|
38
|
+
if (tag === 'tr' && cur.tag === 'tr') cur = cur.parent;
|
|
39
|
+
const el = { tag, attrs, line: ln, children: [], parent: cur, inHead: !bodyStarted || inHeadEl() };
|
|
40
|
+
cur.children.push(el); all.push(el);
|
|
41
|
+
if (!VOID.has(tag) && !selfClose) cur = el;
|
|
42
|
+
return el;
|
|
43
|
+
};
|
|
44
|
+
while (i < html.length) {
|
|
45
|
+
const lt = html.indexOf('<', i);
|
|
46
|
+
if (lt === -1) { const t = html.slice(i); if (t.trim() && !inHeadEl()) bodyStarted = true; cur.children.push({ text: decode(t) }); lineAt(html.length); break; }
|
|
47
|
+
if (lt > i) { const t = html.slice(i, lt); if (t.trim()) { if (!inHeadEl() && cur.tag !== 'html' || cur.tag === 'html' || cur === root) { if (!inHeadEl()) bodyStarted = true; } } cur.children.push({ text: decode(t) }); lineAt(lt); }
|
|
48
|
+
if (html.startsWith('<!--', lt)) { const e = html.indexOf('-->', lt + 4); lineAt(e === -1 ? html.length : e + 3); continue; }
|
|
49
|
+
if (html[lt + 1] === '!' || html[lt + 1] === '?') { const e = html.indexOf('>', lt); lineAt(e === -1 ? html.length : e + 1); continue; }
|
|
50
|
+
const m = /^<(\/?)([a-zA-Z][a-zA-Z0-9:-]*)((?:[^>"']|"[^"]*"|'[^']*')*)>/.exec(html.slice(lt, lt + 20000));
|
|
51
|
+
if (!m) { cur.children.push({ text: '<' }); lineAt(lt + 1); continue; }
|
|
52
|
+
const tag = m[2].toLowerCase(); const ln = line;
|
|
53
|
+
lineAt(lt + m[0].length);
|
|
54
|
+
if (m[1]) { if (tag === 'p' && !close('p')) { /* stray </p> */ } else if (tag !== 'p') close(tag); continue; }
|
|
55
|
+
const el = open(tag, parseAttrs(m[3]), ln, /\/\s*$/.test(m[3]));
|
|
56
|
+
if (RAW.has(tag) && !/\/\s*$/.test(m[3])) {
|
|
57
|
+
const re = new RegExp(`</${tag}\\s*>`, 'i'); re.lastIndex = 0; const rest = html.slice(i); const em = re.exec(rest);
|
|
58
|
+
const end = em ? i + em.index : html.length; const raw = html.slice(i, end);
|
|
59
|
+
if (tag === 'noscript' || tag === 'template') { const inner = parseHtml(raw); for (const c of inner.root.children) { c.parent = el; el.children.push(c); } for (const c of inner.all) { c.line += ln - 1; c.inHead = el.inHead; all.push(c); } }
|
|
60
|
+
else el.children.push({ text: tag === 'script' || tag === 'style' ? raw : decode(raw), raw: true });
|
|
61
|
+
el.rawText = raw;
|
|
62
|
+
lineAt(end); if (em) lineAt(end + em[0].length);
|
|
63
|
+
cur = el.parent;
|
|
64
|
+
}
|
|
65
|
+
}
|
|
66
|
+
return makeDoc(root, all);
|
|
67
|
+
}
|
|
68
|
+
|
|
69
|
+
/** Visible text of an element (skips script/style/template, adds spaces at block boundaries). opts.alt includes img alt text. */
|
|
70
|
+
export function textOf(node, opts = {}) {
|
|
71
|
+
const out = [];
|
|
72
|
+
const walk = (n) => {
|
|
73
|
+
if (n.text != null) { if (!n.raw || (n.parent?.tag !== 'script' && n.parent?.tag !== 'style')) out.push(n.text); return; }
|
|
74
|
+
if (n.tag === 'script' || n.tag === 'style' || n.tag === 'template' || (n.tag === 'noscript' && !opts.noscript)) return;
|
|
75
|
+
if (opts.skip && opts.skip.has(n.tag)) return;
|
|
76
|
+
if (n.attrs?.hidden != null || n.attrs?.['aria-hidden'] === 'true' && opts.skipHidden) return;
|
|
77
|
+
if (n.tag === 'img' && opts.alt) out.push(` ${n.attrs.alt || ''} `);
|
|
78
|
+
if (BLOCK.has(n.tag)) out.push(' ');
|
|
79
|
+
for (const c of n.children || []) walk(c);
|
|
80
|
+
if (BLOCK.has(n.tag)) out.push(' ');
|
|
81
|
+
};
|
|
82
|
+
if (node.text != null) return squash(node.text);
|
|
83
|
+
for (const c of node.children || []) walk(c);
|
|
84
|
+
return squash(out.join(''));
|
|
85
|
+
}
|
|
86
|
+
const has = (el, tag) => { for (let n = el.parent; n; n = n.parent) if (n.tag === tag) return true; return false; };
|
|
87
|
+
export const ancestor = (el, pred) => { for (let n = el.parent; n; n = n.parent) if (n.tag && pred(n)) return n; return null; };
|
|
88
|
+
|
|
89
|
+
function makeDoc(root, all) {
|
|
90
|
+
const doc = { root, all };
|
|
91
|
+
const by = (t) => all.filter((e) => e.tag === t);
|
|
92
|
+
doc.find = (t) => by(t);
|
|
93
|
+
const html = all.find((e) => e.tag === 'html');
|
|
94
|
+
doc.lang = html?.attrs.lang ?? null;
|
|
95
|
+
doc.titles = by('title').filter((e) => !has(e, 'svg')).map((e) => ({ text: squash(decode(e.rawText ?? textOf(e))), line: e.line, inHead: e.inHead, el: e }));
|
|
96
|
+
doc.title = doc.titles.find((t) => t.inHead)?.text ?? doc.titles[0]?.text ?? null;
|
|
97
|
+
doc.metas = by('meta').map((e) => ({ name: (e.attrs.name || '').toLowerCase() || null, property: (e.attrs.property || '').toLowerCase() || null, httpEquiv: (e.attrs['http-equiv'] || '').toLowerCase() || null, content: e.attrs.content ?? null, charset: e.attrs.charset ?? null, itemprop: e.attrs.itemprop ?? null, line: e.line, inHead: e.inHead, el: e }));
|
|
98
|
+
doc.meta = (key) => { key = key.toLowerCase(); return doc.metas.filter((m) => m.name === key || m.property === key); };
|
|
99
|
+
doc.metaContent = (key) => { const m = doc.meta(key).find((x) => x.inHead) || doc.meta(key)[0]; return m ? (m.content ?? '') : null; };
|
|
100
|
+
doc.links = by('link').map((e) => ({ rel: (e.attrs.rel || '').toLowerCase().split(/\s+/).filter(Boolean), href: e.attrs.href ?? null, hreflang: e.attrs.hreflang ?? null, type: e.attrs.type ?? null, line: e.line, inHead: e.inHead, el: e }));
|
|
101
|
+
doc.base = by('base').find((e) => e.attrs.href)?.attrs.href ?? null;
|
|
102
|
+
doc.anchors = by('a').filter((e) => e.attrs.href != null).map((e) => ({ href: e.attrs.href, rel: (e.attrs.rel || '').toLowerCase().split(/\s+/).filter(Boolean), text: textOf(e, { alt: true }), line: e.line, el: e }));
|
|
103
|
+
doc.headings = [];
|
|
104
|
+
for (const e of all) {
|
|
105
|
+
let level = /^h([1-6])$/.exec(e.tag)?.[1];
|
|
106
|
+
if (!level && e.attrs.role === 'heading') level = e.attrs['aria-level'] || '2';
|
|
107
|
+
if (level && !has(e, 'svg') && !has(e, 'template')) doc.headings.push({ level: +level, text: textOf(e, { alt: true }), line: e.line, el: e, hidden: e.attrs.hidden != null || !!ancestor(e, (n) => n.attrs?.hidden != null) });
|
|
108
|
+
}
|
|
109
|
+
doc.jsonld = by('script').filter((e) => /^application\/ld\+json$/i.test((e.attrs.type || '').trim())).map((e) => { const raw = e.rawText ?? ''; try { return { raw, line: e.line, data: JSON.parse(raw.trim().replace(/^<!--|-->$/g, '')) }; } catch (err) { return { raw, line: e.line, error: err.message }; } });
|
|
110
|
+
const body = by('body')[0] || root;
|
|
111
|
+
let _text; let _main;
|
|
112
|
+
Object.defineProperty(doc, 'text', { get: () => (_text ??= textOf(body)) });
|
|
113
|
+
Object.defineProperty(doc, 'mainText', { get: () => {
|
|
114
|
+
if (_main != null) return _main;
|
|
115
|
+
const mains = all.filter((e) => e.tag === 'main' || e.attrs.role === 'main');
|
|
116
|
+
const arts = mains.length ? mains : all.filter((e) => e.tag === 'article' && !has(e, 'article'));
|
|
117
|
+
_main = arts.length ? squash(arts.map((e) => textOf(e, { skip: new Set(['nav', 'aside']) })).join(' ')) : textOf(body, { skip: new Set(['nav', 'header', 'footer', 'aside']) });
|
|
118
|
+
return _main;
|
|
119
|
+
} });
|
|
120
|
+
doc.words = (s) => (s ?? doc.mainText).split(/[^\p{L}\p{N}'’-]+/u).filter((w) => /[\p{L}\p{N}]/u.test(w));
|
|
121
|
+
doc.robots = (bots = ['robots']) => doc.metas.filter((m) => m.name && bots.includes(m.name)).map((m) => ({ ...m, directives: String(m.content || '').toLowerCase().split(',').map((d) => d.trim()).filter(Boolean) }));
|
|
122
|
+
return doc;
|
|
123
|
+
}
|
|
124
|
+
|
|
125
|
+
// ---------------------------------------------------------------------------------------------- site
|
|
126
|
+
export function globToRe(g) {
|
|
127
|
+
let s = ''; g = String(g);
|
|
128
|
+
for (let k = 0; k < g.length; k++) {
|
|
129
|
+
const c = g[k];
|
|
130
|
+
if (c === '*') { if (g[k + 1] === '*') { s += '.*'; k++; if (g[k + 1] === '/') k++; } else s += '[^/]*'; }
|
|
131
|
+
else if (c === '?') s += '[^/]';
|
|
132
|
+
else s += c.replace(/[.+^${}()|[\]\\]/g, '\\$&');
|
|
133
|
+
}
|
|
134
|
+
return new RegExp(`^${s}$`);
|
|
135
|
+
}
|
|
136
|
+
export const matchAny = (globs, s) => (globs || []).some((g) => globToRe(g).test(s));
|
|
137
|
+
|
|
138
|
+
const SKIP_DIRS = new Set(['node_modules', '.git', '.svn', '.hg']);
|
|
139
|
+
function walk(dir, base = dir, out = []) {
|
|
140
|
+
let ents; try { ents = readdirSync(dir, { withFileTypes: true }); } catch { return out; }
|
|
141
|
+
for (const e of ents) {
|
|
142
|
+
if (SKIP_DIRS.has(e.name)) continue;
|
|
143
|
+
const p = join(dir, e.name);
|
|
144
|
+
if (e.isDirectory()) walk(p, base, out);
|
|
145
|
+
else if (e.isFile()) out.push(relative(base, p).split(sep).join('/'));
|
|
146
|
+
}
|
|
147
|
+
return out;
|
|
148
|
+
}
|
|
149
|
+
|
|
150
|
+
/** URL path for a built file: index.html -> "/", a/index.html -> "/a/", a.html -> "/a.html" (or "/a" with cleanUrls). */
|
|
151
|
+
export function pathForFile(file, { cleanUrls = false } = {}) {
|
|
152
|
+
if (/(^|\/)index\.html?$/i.test(file)) return '/' + file.replace(/index\.html?$/i, '');
|
|
153
|
+
return '/' + (cleanUrls ? file.replace(/\.html?$/i, '') : file);
|
|
154
|
+
}
|
|
155
|
+
export function normPath(p) {
|
|
156
|
+
let s = String(p || '/').split('#')[0].split('?')[0];
|
|
157
|
+
try { s = decodeURI(s); } catch { /* keep */ }
|
|
158
|
+
if (!s.startsWith('/')) s = '/' + s;
|
|
159
|
+
return s.replace(/\/{2,}/g, '/');
|
|
160
|
+
}
|
|
161
|
+
|
|
162
|
+
/**
|
|
163
|
+
* Load a built static site. opts: { baseUrl, include, exclude, cleanUrls }.
|
|
164
|
+
* Returns { dir, files, pages, baseUrl, origin, find(urlOrPath), resolve(page, href), isInternal(url), headersFor(path), read(rel) }.
|
|
165
|
+
*/
|
|
166
|
+
export function loadSite(dir, opts = {}) {
|
|
167
|
+
if (!existsSync(dir) || !statSync(dir).isDirectory()) throw Object.assign(new Error(`No such directory: ${dir}`), { code: 'ENOENT' });
|
|
168
|
+
const files = walk(dir).sort();
|
|
169
|
+
const htmlFiles = files.filter((f) => /\.html?$/i.test(f) && !matchAny(opts.exclude, f) && !matchAny(opts.exclude, pathForFile(f, opts)) && (!opts.include?.length || matchAny(opts.include, f) || matchAny(opts.include, pathForFile(f, opts))));
|
|
170
|
+
const cache = new Map();
|
|
171
|
+
const pages = htmlFiles.map((file) => {
|
|
172
|
+
const page = { file, path: pathForFile(file, opts) };
|
|
173
|
+
Object.defineProperty(page, 'html', { get: () => readFileSync(join(dir, file), 'utf8') });
|
|
174
|
+
Object.defineProperty(page, 'doc', { get: () => { if (!cache.has(file)) cache.set(file, parseHtml(readFileSync(join(dir, file), 'utf8'))); return cache.get(file); }, enumerable: false });
|
|
175
|
+
return page;
|
|
176
|
+
});
|
|
177
|
+
const index = new Map();
|
|
178
|
+
const keyVariants = (path) => { const p = normPath(path).toLowerCase(); const out = [p]; if (p.endsWith('/')) { out.push(p + 'index.html', p + 'index.htm'); if (p !== '/') out.push(p.slice(0, -1)); } else if (/\.html?$/.test(p)) { out.push(p.replace(/\.html?$/, '')); if (/\/index\.html?$/.test(p)) out.push(p.replace(/index\.html?$/, '')); } else out.push(p + '/', p + '.html'); return out; };
|
|
179
|
+
for (const pg of pages) for (const k of keyVariants('/' + pg.file).concat(keyVariants(pg.path))) if (!index.has(k)) index.set(k, pg);
|
|
180
|
+
|
|
181
|
+
let baseUrl = opts.baseUrl ? String(opts.baseUrl).replace(/\/+$/, '') : null;
|
|
182
|
+
if (!baseUrl) {
|
|
183
|
+
const counts = new Map();
|
|
184
|
+
for (const pg of pages.slice(0, 60)) { const d = pg.doc; for (const v of [d.links.find((l) => l.rel.includes('canonical'))?.href, d.metaContent('og:url')]) { try { const u = new URL(v); if (/^https?:$/.test(u.protocol)) counts.set(u.origin, (counts.get(u.origin) || 0) + 1); } catch { /* relative */ } } }
|
|
185
|
+
baseUrl = [...counts.entries()].sort((a, b) => b[1] - a[1])[0]?.[0] ?? null;
|
|
186
|
+
}
|
|
187
|
+
let origin = null; let basePath = '';
|
|
188
|
+
if (baseUrl) { try { const u = new URL(baseUrl); origin = u.origin; basePath = u.pathname.replace(/\/+$/, ''); } catch { throw Object.assign(new Error(`Invalid base URL: ${baseUrl}`), { code: 'EINVAL', usage: true }); } }
|
|
189
|
+
const site = { dir, files, pages, baseUrl, origin, basePath, inferredBase: !opts.baseUrl && !!baseUrl };
|
|
190
|
+
site.urlOf = (pg) => (origin ? origin + basePath + pg.path : pg.path);
|
|
191
|
+
/** Absolute URL object for href as seen from page (honours <base href>). */
|
|
192
|
+
site.resolve = (pg, href) => {
|
|
193
|
+
if (href == null) return null; href = String(href).trim();
|
|
194
|
+
if (!href || /^(javascript|mailto|tel|data|sms|ftp|blob):/i.test(href)) return null;
|
|
195
|
+
const pageUrl = (origin || 'http://site.invalid') + basePath + pg.path;
|
|
196
|
+
try { const b = pg.doc?.base ? new URL(pg.doc.base, pageUrl).href : pageUrl; return new URL(href, b); } catch { return null; }
|
|
197
|
+
};
|
|
198
|
+
site.isInternal = (u) => { if (!u) return false; const o = origin || 'http://site.invalid'; if (u.origin === o) return true; if (origin) { try { const a = new URL(o); return u.hostname.replace(/^www\./, '') === a.hostname.replace(/^www\./, '') && false; } catch { return false; } } return false; };
|
|
199
|
+
/** Find the page for a URL / path / URL object. Returns undefined for external or unknown. */
|
|
200
|
+
site.find = (x) => {
|
|
201
|
+
let path;
|
|
202
|
+
if (x instanceof URL) { if (!(x.origin === (origin || 'http://site.invalid'))) return undefined; path = x.pathname; }
|
|
203
|
+
else { const s = String(x); if (/^[a-z][a-z0-9+.-]*:/i.test(s)) { try { const u = new URL(s); if (u.origin !== (origin || 'http://site.invalid')) return undefined; path = u.pathname; } catch { return undefined; } } else path = s; }
|
|
204
|
+
if (basePath && path.startsWith(basePath)) path = path.slice(basePath.length) || '/';
|
|
205
|
+
for (const k of keyVariants(path)) { const pg = index.get(k); if (pg) return pg; }
|
|
206
|
+
return undefined;
|
|
207
|
+
};
|
|
208
|
+
site.read = (rel) => { try { return readFileSync(join(dir, rel), 'utf8'); } catch { return null; } };
|
|
209
|
+
site.exists = (rel) => existsSync(join(dir, rel));
|
|
210
|
+
const rules = loadHeaderRules(dir);
|
|
211
|
+
site.headerRules = rules;
|
|
212
|
+
site.headersFor = (path) => { const out = {}; for (const r of rules) if (r.re.test(path) || r.re.test(path.replace(/\/$/, '')) ) for (const [k, v] of r.headers) (out[k] ||= []).push({ value: v, source: r.source, line: r.line }); return out; };
|
|
213
|
+
return site;
|
|
214
|
+
}
|
|
215
|
+
|
|
216
|
+
/** Header rules from Netlify / Cloudflare Pages `_headers` and `vercel.json`. */
|
|
217
|
+
export function loadHeaderRules(dir) {
|
|
218
|
+
const rules = [];
|
|
219
|
+
const hf = join(dir, '_headers');
|
|
220
|
+
if (existsSync(hf)) {
|
|
221
|
+
let cur = null;
|
|
222
|
+
readFileSync(hf, 'utf8').split(/\r?\n/).forEach((l, i) => {
|
|
223
|
+
if (!l.trim() || l.trim().startsWith('#')) return;
|
|
224
|
+
if (!/^\s/.test(l)) { const pat = l.trim(); cur = { source: '_headers', line: i + 1, pattern: pat, re: pathPatternRe(pat), headers: [] }; rules.push(cur); }
|
|
225
|
+
else if (cur) { const m = /^\s+([^:]+):\s*(.*)$/.exec(l); if (m) cur.headers.push([m[1].trim().toLowerCase(), m[2].trim()]); }
|
|
226
|
+
});
|
|
227
|
+
}
|
|
228
|
+
for (const f of ['vercel.json', '../vercel.json']) {
|
|
229
|
+
const vf = join(dir, f); if (!existsSync(vf)) continue;
|
|
230
|
+
try { const j = JSON.parse(readFileSync(vf, 'utf8')); const text = readFileSync(vf, 'utf8').split('\n'); for (const h of j.headers || []) { const ln = text.findIndex((l) => l.includes(JSON.stringify(h.source))) + 1 || undefined; rules.push({ source: f.replace('../', ''), line: ln, pattern: h.source, re: pathPatternRe(h.source), headers: (h.headers || []).map((x) => [String(x.key).toLowerCase(), String(x.value)]) }); } } catch { /* ignore */ }
|
|
231
|
+
break;
|
|
232
|
+
}
|
|
233
|
+
return rules;
|
|
234
|
+
}
|
|
235
|
+
function pathPatternRe(p) {
|
|
236
|
+
try { if (/^https?:\/\//.test(p)) p = new URL(p).pathname; } catch { /* keep */ }
|
|
237
|
+
if (/\(.*\)/.test(p) && !/:\w/.test(p.replace(/\(.*\)/g, ''))) { try { return new RegExp('^' + p.replace(/\//g, '\\/') + '$'); } catch { /* fall through */ } }
|
|
238
|
+
let s = '';
|
|
239
|
+
for (let k = 0; k < p.length; k++) {
|
|
240
|
+
const c = p[k];
|
|
241
|
+
if (c === '*') s += '.*';
|
|
242
|
+
else if (c === ':' && /\w/.test(p[k + 1] || '')) { const m = /^:(\w+)(\*|\+|\?)?(\([^)]*\))?/.exec(p.slice(k)); k += m[0].length - 1; s += m[2] === '*' || m[2] === '+' ? '.*' : m[3] ? m[3] : '[^/]+'; }
|
|
243
|
+
else s += c.replace(/[.+^${}()|[\]\\?]/g, '\\$&');
|
|
244
|
+
}
|
|
245
|
+
return new RegExp(`^${s}$`);
|
|
246
|
+
}
|
|
247
|
+
|
|
248
|
+
/** Robots directives that apply to a page from meta tags and X-Robots-Tag headers. bots: lowercased names. */
|
|
249
|
+
export function robotsDirectives(site, pg, bots = ['robots', 'googlebot']) {
|
|
250
|
+
const out = [];
|
|
251
|
+
for (const m of pg.doc.robots(bots)) out.push({ bot: m.name, directives: m.directives, source: 'meta', file: pg.file, line: m.line, inHead: m.inHead, raw: m.content });
|
|
252
|
+
for (const h of site.headersFor(pg.path)['x-robots-tag'] || []) {
|
|
253
|
+
const v = h.value; const mm = /^([a-z0-9_-]+)\s*:\s*(.*)$/i.exec(v);
|
|
254
|
+
const bot = mm && !/^(noindex|nofollow|none|all|index|follow|noarchive|nosnippet|unavailable_after|max-snippet|max-image-preview|max-video-preview|noimageindex|notranslate|indexifembedded)$/i.test(mm[1]) ? mm[1].toLowerCase() : 'robots';
|
|
255
|
+
const body = bot === 'robots' ? v : mm[2];
|
|
256
|
+
if (bot === 'robots' || bots.includes(bot)) out.push({ bot, directives: body.toLowerCase().split(',').map((d) => d.trim()).filter(Boolean), source: 'header', file: h.source, line: h.line, raw: v });
|
|
257
|
+
}
|
|
258
|
+
return out;
|
|
259
|
+
}
|
|
260
|
+
export const isNoindex = (dirs) => dirs.some((d) => d.directives.some((x) => x === 'noindex' || x === 'none'));
|
|
261
|
+
|
|
262
|
+
/** Page-level helpers shared across beacons. */
|
|
263
|
+
export function canonicalOf(site, pg) {
|
|
264
|
+
const c = pg.doc.links.filter((l) => l.rel.includes('canonical') && l.href != null);
|
|
265
|
+
const hdr = (site.headersFor(pg.path).link || []).map((h) => /<([^>]+)>\s*;\s*rel="?canonical"?/i.exec(h.value)?.[1]).filter(Boolean);
|
|
266
|
+
const href = c.find((l) => l.inHead)?.href ?? c[0]?.href ?? hdr[0] ?? null;
|
|
267
|
+
return { href, url: href != null ? site.resolve(pg, href) : null, tags: c, header: hdr };
|
|
268
|
+
}
|
|
269
|
+
|
|
270
|
+
/** Very small JSON-LD helpers. */
|
|
271
|
+
export function jsonldNodes(doc) {
|
|
272
|
+
const out = [];
|
|
273
|
+
const visit = (n, block, depth = 0) => {
|
|
274
|
+
if (Array.isArray(n)) { n.forEach((x) => visit(x, block, depth)); return; }
|
|
275
|
+
if (!n || typeof n !== 'object') return;
|
|
276
|
+
if (n['@graph']) visit(n['@graph'], block, depth);
|
|
277
|
+
if (n['@type']) out.push({ node: n, types: [].concat(n['@type']).map(String), line: block.line, depth });
|
|
278
|
+
for (const [k, v] of Object.entries(n)) if (k !== '@graph' && k !== '@context' && v && typeof v === 'object') visit(v, block, depth + 1);
|
|
279
|
+
};
|
|
280
|
+
for (const b of doc.jsonld) if (b.data) visit(b.data, b);
|
|
281
|
+
return out;
|
|
282
|
+
}
|
|
283
|
+
export const typeName = (t) => String(t).replace(/^https?:\/\/schema\.org\//, '').replace(/^schema:/, '');
|
|
284
|
+
|
|
285
|
+
// ---------------------------------------------------------------------------------------------- CLI glue
|
|
286
|
+
export const SITE_OPTIONS = { 'base-url': { type: 'string' }, include: { type: 'string', multiple: true }, exclude: { type: 'string', multiple: true }, 'clean-urls': { type: 'boolean' } };
|
|
287
|
+
export const SITE_HELP = `Site
|
|
288
|
+
--base-url <url> Public URL of the site, e.g. https://example.com (default: inferred from canonical/og:url)
|
|
289
|
+
--include <globs> Only check these pages (file or URL path globs, comma-separated)
|
|
290
|
+
--exclude <globs> Skip these pages
|
|
291
|
+
--clean-urls Pages are served without .html (about.html -> /about)
|
|
292
|
+
`;
|
|
293
|
+
const list = (v) => (v || []).flatMap((x) => String(x).split(',')).map((x) => x.trim()).filter(Boolean);
|
|
294
|
+
export function siteOptions(o = {}, config = {}) {
|
|
295
|
+
return { baseUrl: o['base-url'] ?? config.baseUrl, include: [...list(config.include), ...list(o.include)], exclude: [...list(config.exclude), ...list(o.exclude)], cleanUrls: o['clean-urls'] ?? config.cleanUrls ?? false };
|
|
296
|
+
}
|
|
297
|
+
/** Run a beacon: loads the site, calls analyze(site, opts) -> { findings, info, summary }, returns a report via buildReport. */
|
|
298
|
+
export function runBeacon(tool, dir, opts, config, configFile, analyze, buildReport) {
|
|
299
|
+
let site;
|
|
300
|
+
try { site = loadSite(dir, siteOptions(opts.cli || {}, { ...config, ...pick(opts, ['baseUrl', 'include', 'exclude', 'cleanUrls']) })); }
|
|
301
|
+
catch (e) { if (e.usage || e.code === 'ENOENT') { const U = opts.UsageError || Error; throw new U(e.message); } throw e; }
|
|
302
|
+
const { findings, info = {}, summary } = analyze(site, opts, config);
|
|
303
|
+
const line = summary ?? `${site.pages.length} page(s)${site.baseUrl ? ` at ${site.baseUrl}${site.inferredBase ? ' (inferred)' : ''}` : ' (no base URL; pass --base-url)'}`;
|
|
304
|
+
return buildReport(tool, site.baseUrl || dir, findings, config, { now: opts.now, configFile, summaryLine: line, info: { pages: site.pages.length, baseUrl: site.baseUrl, ...info } });
|
|
305
|
+
}
|
|
306
|
+
function pick(o, keys) { const r = {}; for (const k of keys) if (o[k] !== undefined && !(Array.isArray(o[k]) && !o[k].length)) r[k] = o[k]; return r; }
|
|
307
|
+
/** Finding factory bound to a rule table. */
|
|
308
|
+
export function finder(RULES, findings = []) {
|
|
309
|
+
const add = (rule, subject, message, fix, loc = {}) => { if (!RULES[rule]) throw new Error(`unknown rule ${rule}`); findings.push({ rule, severity: loc.severity || RULES[rule].severity, subject, file: loc.file, line: loc.line, message, fix }); };
|
|
310
|
+
return { findings, add };
|
|
311
|
+
}
|
|
312
|
+
/** Redirect stubs (meta refresh) and error pages are skipped by most checks. */
|
|
313
|
+
export const isRedirectStub = (pg) => pg.doc.metas.some((m) => m.httpEquiv === 'refresh');
|
|
314
|
+
export const isErrorPage = (pg) => /(^|\/)(404|500|50x|error)(\/index)?\.html?$/i.test(pg.file);
|
|
315
|
+
export const contentPages = (site) => site.pages.filter((p) => !isRedirectStub(p) && !isErrorPage(p));
|
|
316
|
+
/** Published / modified dates a page declares (JSON-LD, Open Graph article tags, <time itemprop>, meta). Values are Date or null. */
|
|
317
|
+
export function pageDates(doc) {
|
|
318
|
+
const pick = (vals) => { for (const v of vals) { if (!v) continue; const d = new Date(String(v).trim()); if (!Number.isNaN(d.getTime())) return d; } return null; };
|
|
319
|
+
const ld = jsonldNodes(doc).map((n) => n.node);
|
|
320
|
+
const ldv = (k) => ld.map((n) => [].concat(n[k] ?? [])[0]).filter(Boolean);
|
|
321
|
+
const timeProp = (k) => doc.all.filter((e) => e.attrs.itemprop === k).map((e) => e.attrs.datetime || e.attrs.content);
|
|
322
|
+
return {
|
|
323
|
+
published: pick([...ldv('datePublished'), doc.metaContent('article:published_time'), ...timeProp('datePublished'), doc.metaContent('date'), doc.metaContent('dc.date'), doc.metaContent('pubdate')]),
|
|
324
|
+
modified: pick([...ldv('dateModified'), doc.metaContent('article:modified_time'), doc.metaContent('og:updated_time'), ...timeProp('dateModified'), doc.metaContent('last-modified')]),
|
|
325
|
+
};
|
|
326
|
+
}
|
package/src/vendors.js
ADDED
|
@@ -0,0 +1,77 @@
|
|
|
1
|
+
// Known third-party vendors: domains, inline-loader signatures, category and typical script weight (KB, compressed). Louis Wright, MIT.
|
|
2
|
+
// category: analytics | advertising | social | tag-manager | session-replay | marketing | chat | consent | cdn | fonts | video | maps | payments | security | other
|
|
3
|
+
export const VENDORS = [
|
|
4
|
+
{ id: 'google-analytics', name: 'Google Analytics', company: 'Google', category: 'analytics', domains: ['google-analytics.com', 'analytics.google.com'], globals: ['gtag(', 'ga(', 'GoogleAnalyticsObject'], scripts: ['gtag/js', 'analytics.js'], sells: false, kb: 55, policyNames: ['google analytics'] },
|
|
5
|
+
{ id: 'google-tag-manager', name: 'Google Tag Manager', company: 'Google', category: 'tag-manager', domains: ['googletagmanager.com'], globals: ['dataLayer.push', 'GTM-'], sells: false, kb: 90, policyNames: ['google tag manager', 'google analytics'] },
|
|
6
|
+
{ id: 'google-ads', name: 'Google Ads / DoubleClick', company: 'Google', category: 'advertising', domains: ['doubleclick.net', 'googleadservices.com', 'googlesyndication.com', 'adservice.google.com', 'pagead2.googlesyndication.com'], globals: ['adsbygoogle', 'AW-'], sells: true, kb: 120, policyNames: ['google ads', 'doubleclick', 'adsense', 'google advertising'] },
|
|
7
|
+
{ id: 'meta-pixel', name: 'Meta Pixel', company: 'Meta', category: 'advertising', domains: ['connect.facebook.net', 'facebook.com/tr', 'facebook.net'], globals: ['fbq(', '_fbq'], sells: true, kb: 75, policyNames: ['facebook', 'meta pixel', 'meta platforms', 'meta'] },
|
|
8
|
+
{ id: 'tiktok-pixel', name: 'TikTok Pixel', company: 'TikTok', category: 'advertising', domains: ['analytics.tiktok.com'], globals: ['ttq.', 'TiktokAnalyticsObject'], sells: true, kb: 80, policyNames: ['tiktok'] },
|
|
9
|
+
{ id: 'linkedin-insight', name: 'LinkedIn Insight Tag', company: 'LinkedIn', category: 'advertising', domains: ['snap.licdn.com', 'px.ads.linkedin.com'], globals: ['_linkedin_partner_id', 'lintrk('], sells: true, kb: 30, policyNames: ['linkedin'] },
|
|
10
|
+
{ id: 'x-pixel', name: 'X (Twitter) Pixel', company: 'X', category: 'advertising', domains: ['static.ads-twitter.com', 'analytics.twitter.com', 't.co/i/adsct'], globals: ['twq('], sells: true, kb: 35, policyNames: ['twitter', 'x corp', 'x pixel'] },
|
|
11
|
+
{ id: 'pinterest-tag', name: 'Pinterest Tag', company: 'Pinterest', category: 'advertising', domains: ['s.pinimg.com/ct', 'ct.pinterest.com'], globals: ['pintrk('], sells: true, kb: 40, policyNames: ['pinterest'] },
|
|
12
|
+
{ id: 'snap-pixel', name: 'Snap Pixel', company: 'Snap', category: 'advertising', domains: ['sc-static.net', 'tr.snapchat.com'], globals: ['snaptr('], sells: true, kb: 40, policyNames: ['snapchat', 'snap inc'] },
|
|
13
|
+
{ id: 'microsoft-ads', name: 'Microsoft Advertising UET', company: 'Microsoft', category: 'advertising', domains: ['bat.bing.com'], globals: ['uetq'], sells: true, kb: 25, policyNames: ['microsoft advertising', 'bing ads', 'bing'] },
|
|
14
|
+
{ id: 'criteo', name: 'Criteo', company: 'Criteo', category: 'advertising', domains: ['criteo.com', 'criteo.net'], globals: ['criteo_q'], sells: true, kb: 45, policyNames: ['criteo'] },
|
|
15
|
+
{ id: 'taboola', name: 'Taboola', company: 'Taboola', category: 'advertising', domains: ['taboola.com'], globals: ['_tfa'], sells: true, kb: 90, policyNames: ['taboola'] },
|
|
16
|
+
{ id: 'hotjar', name: 'Hotjar', company: 'Hotjar', category: 'session-replay', domains: ['hotjar.com', 'hotjar.io'], globals: ['hj(', '_hjSettings'], sells: false, kb: 70, policyNames: ['hotjar'] },
|
|
17
|
+
{ id: 'clarity', name: 'Microsoft Clarity', company: 'Microsoft', category: 'session-replay', domains: ['clarity.ms'], globals: ['clarity('], sells: false, kb: 60, policyNames: ['clarity', 'microsoft clarity'] },
|
|
18
|
+
{ id: 'fullstory', name: 'FullStory', company: 'FullStory', category: 'session-replay', domains: ['fullstory.com'], globals: ['_fs_org', 'FS.identify'], sells: false, kb: 85, policyNames: ['fullstory'] },
|
|
19
|
+
{ id: 'mixpanel', name: 'Mixpanel', company: 'Mixpanel', category: 'analytics', domains: ['mixpanel.com', 'mxpnl.com'], globals: ['mixpanel.init'], sells: false, kb: 45, policyNames: ['mixpanel'] },
|
|
20
|
+
{ id: 'segment', name: 'Segment', company: 'Twilio', category: 'analytics', domains: ['cdn.segment.com', 'api.segment.io'], globals: ['analytics.load('], sells: false, kb: 60, policyNames: ['segment', 'twilio'] },
|
|
21
|
+
{ id: 'amplitude', name: 'Amplitude', company: 'Amplitude', category: 'analytics', domains: ['amplitude.com'], globals: ['amplitude.getInstance', 'amplitude.init'], sells: false, kb: 50, policyNames: ['amplitude'] },
|
|
22
|
+
{ id: 'heap', name: 'Heap', company: 'Heap', category: 'analytics', domains: ['heapanalytics.com', 'heap-api.com'], globals: ['heap.load('], sells: false, kb: 50, policyNames: ['heap'] },
|
|
23
|
+
{ id: 'plausible', name: 'Plausible', company: 'Plausible', category: 'analytics', domains: ['plausible.io'], globals: ['plausible('], sells: false, cookieless: true, kb: 1, policyNames: ['plausible'] },
|
|
24
|
+
{ id: 'fathom', name: 'Fathom', company: 'Fathom', category: 'analytics', domains: ['usefathom.com'], globals: [], sells: false, cookieless: true, kb: 2, policyNames: ['fathom'] },
|
|
25
|
+
{ id: 'hubspot', name: 'HubSpot', company: 'HubSpot', category: 'marketing', domains: ['hs-scripts.com', 'hs-analytics.net', 'hubspot.com', 'hsforms.net', 'hs-banner.com'], globals: ['_hsq'], sells: false, kb: 110, policyNames: ['hubspot'] },
|
|
26
|
+
{ id: 'mailchimp', name: 'Mailchimp', company: 'Intuit', category: 'marketing', domains: ['chimpstatic.com', 'list-manage.com'], globals: [], sells: false, kb: 30, policyNames: ['mailchimp'] },
|
|
27
|
+
{ id: 'intercom', name: 'Intercom', company: 'Intercom', category: 'chat', domains: ['intercom.io', 'intercomcdn.com', 'widget.intercom.io'], globals: ['Intercom('], sells: false, kb: 200, policyNames: ['intercom'] },
|
|
28
|
+
{ id: 'drift', name: 'Drift', company: 'Salesloft', category: 'chat', domains: ['drift.com', 'driftt.com'], globals: ['drift.load'], sells: false, kb: 180, policyNames: ['drift'] },
|
|
29
|
+
{ id: 'zendesk', name: 'Zendesk', company: 'Zendesk', category: 'chat', domains: ['zdassets.com', 'zendesk.com'], globals: ['zE('], sells: false, kb: 160, policyNames: ['zendesk'] },
|
|
30
|
+
{ id: 'youtube', name: 'YouTube embed', company: 'Google', category: 'video', domains: ['youtube.com', 'youtube-nocookie.com', 'ytimg.com'], globals: [], sells: false, kb: 500, policyNames: ['youtube'] },
|
|
31
|
+
{ id: 'vimeo', name: 'Vimeo embed', company: 'Vimeo', category: 'video', domains: ['player.vimeo.com', 'vimeocdn.com'], globals: [], sells: false, kb: 250, policyNames: ['vimeo'] },
|
|
32
|
+
{ id: 'google-maps', name: 'Google Maps', company: 'Google', category: 'maps', domains: ['maps.googleapis.com', 'maps.google.com', 'google.com/maps'], globals: [], sells: false, kb: 250, policyNames: ['google maps'] },
|
|
33
|
+
{ id: 'google-fonts', name: 'Google Fonts', company: 'Google', category: 'fonts', domains: ['fonts.googleapis.com', 'fonts.gstatic.com'], globals: [], sells: false, kb: 20, policyNames: ['google fonts'] },
|
|
34
|
+
{ id: 'adobe-fonts', name: 'Adobe Fonts', company: 'Adobe', category: 'fonts', domains: ['use.typekit.net', 'p.typekit.net'], globals: [], sells: false, kb: 30, policyNames: ['adobe fonts', 'typekit', 'adobe'] },
|
|
35
|
+
{ id: 'stripe', name: 'Stripe', company: 'Stripe', category: 'payments', domains: ['js.stripe.com', 'm.stripe.network'], globals: [], sells: false, essential: true, kb: 120, policyNames: ['stripe'] },
|
|
36
|
+
{ id: 'paypal', name: 'PayPal', company: 'PayPal', category: 'payments', domains: ['paypal.com', 'paypalobjects.com'], globals: [], sells: false, essential: true, kb: 100, policyNames: ['paypal'] },
|
|
37
|
+
{ id: 'recaptcha', name: 'Google reCAPTCHA', company: 'Google', category: 'security', domains: ['google.com/recaptcha', 'gstatic.com/recaptcha', 'recaptcha.net'], globals: ['grecaptcha'], sells: false, essential: true, kb: 150, policyNames: ['recaptcha'] },
|
|
38
|
+
{ id: 'hcaptcha', name: 'hCaptcha', company: 'Intuition Machines', category: 'security', domains: ['hcaptcha.com'], globals: ['hcaptcha'], sells: false, essential: true, kb: 90, policyNames: ['hcaptcha'] },
|
|
39
|
+
{ id: 'cloudflare-insights', name: 'Cloudflare Web Analytics', company: 'Cloudflare', category: 'analytics', domains: ['static.cloudflareinsights.com'], globals: [], sells: false, cookieless: true, kb: 6, policyNames: ['cloudflare'] },
|
|
40
|
+
{ id: 'cookiebot', name: 'Cookiebot', company: 'Usercentrics', category: 'consent', domains: ['consent.cookiebot.com', 'consentcdn.cookiebot.com'], globals: ['Cookiebot'], sells: false, essential: true, kb: 40, policyNames: ['cookiebot', 'usercentrics'] },
|
|
41
|
+
{ id: 'onetrust', name: 'OneTrust', company: 'OneTrust', category: 'consent', domains: ['cdn.cookielaw.org', 'onetrust.com'], globals: ['OneTrust', 'OptanonWrapper'], sells: false, essential: true, kb: 90, policyNames: ['onetrust'] },
|
|
42
|
+
{ id: 'usercentrics', name: 'Usercentrics', company: 'Usercentrics', category: 'consent', domains: ['app.usercentrics.eu', 'usercentrics.eu'], globals: ['UC_UI'], sells: false, essential: true, kb: 80, policyNames: ['usercentrics'] },
|
|
43
|
+
{ id: 'jsdelivr', name: 'jsDelivr CDN', company: 'jsDelivr', category: 'cdn', domains: ['cdn.jsdelivr.net'], globals: [], sells: false, essential: true, kb: 0, policyNames: ['jsdelivr'] },
|
|
44
|
+
{ id: 'unpkg', name: 'unpkg CDN', company: 'unpkg', category: 'cdn', domains: ['unpkg.com'], globals: [], sells: false, essential: true, kb: 0, policyNames: ['unpkg'] },
|
|
45
|
+
{ id: 'cdnjs', name: 'cdnjs', company: 'Cloudflare', category: 'cdn', domains: ['cdnjs.cloudflare.com'], globals: [], sells: false, essential: true, kb: 0, policyNames: ['cdnjs', 'cloudflare'] },
|
|
46
|
+
{ id: 'addthis', name: 'AddThis', company: 'Oracle', category: 'social', domains: ['addthis.com', 'addthisedge.com'], globals: ['addthis'], sells: true, kb: 150, policyNames: ['addthis'] },
|
|
47
|
+
{ id: 'sharethis', name: 'ShareThis', company: 'ShareThis', category: 'social', domains: ['sharethis.com'], globals: [], sells: true, kb: 100, policyNames: ['sharethis'] },
|
|
48
|
+
{ id: 'disqus', name: 'Disqus', company: 'Disqus', category: 'social', domains: ['disqus.com', 'disquscdn.com'], globals: ['disqus_config'], sells: true, kb: 300, policyNames: ['disqus'] },
|
|
49
|
+
];
|
|
50
|
+
export const TRACKING = new Set(['analytics', 'advertising', 'social', 'tag-manager', 'session-replay', 'marketing']);
|
|
51
|
+
const hostOf = (u) => { try { return new URL(u, 'https://x.invalid').host.toLowerCase(); } catch { return ''; } };
|
|
52
|
+
/** Match a URL to a vendor (domain suffix or domain+path prefix). */
|
|
53
|
+
export function vendorForUrl(url) {
|
|
54
|
+
let u; try { u = new URL(url, 'https://x.invalid'); } catch { return null; }
|
|
55
|
+
const host = u.host.toLowerCase().replace(/^www\./, ''); const hp = host + u.pathname;
|
|
56
|
+
for (const v of VENDORS) for (const d of v.domains) { if (d.includes('/') ? hp.startsWith(d) || hp.includes('.' + d) : host === d || host.endsWith('.' + d)) return v; }
|
|
57
|
+
return null;
|
|
58
|
+
}
|
|
59
|
+
/** Vendors referenced by inline script text. */
|
|
60
|
+
export function vendorsInScript(code) { const out = new Set(); for (const v of VENDORS) { if (v.globals.some((g) => code.includes(g))) out.add(v); for (const d of v.domains) if (code.includes(d)) out.add(v); } return [...out]; }
|
|
61
|
+
export { hostOf };
|
|
62
|
+
/** Third-party resources on a parsed page: [{ kind, url, host, vendor, line, el, inline }]. siteHost = first-party host (or null). */
|
|
63
|
+
export function thirdPartyResources(doc, siteHost) {
|
|
64
|
+
const out = []; const fp = (h) => !h || h === 'x.invalid' || (siteHost && (h === siteHost || h.endsWith('.' + siteHost.replace(/^www\./, '')) || h.replace(/^www\./, '') === siteHost.replace(/^www\./, '')));
|
|
65
|
+
const push = (kind, url, el, extra = {}) => { if (!url || /^(data|blob|javascript|mailto|tel):/i.test(url)) return; const host = hostOf(url); if (fp(host) && !/^\/\//.test(url) && !/^https?:/i.test(url)) return; if (fp(host)) return; out.push({ kind, url, host, vendor: vendorForUrl(url), line: el.line, el, ...extra }); };
|
|
66
|
+
const inNoscript = (el) => { for (let n = el.parent; n; n = n.parent) if (n.tag === 'noscript') return true; return false; };
|
|
67
|
+
for (const el of doc.all) {
|
|
68
|
+
const a = el.attrs;
|
|
69
|
+
if (el.tag === 'script' && a.src) push('script', a.src, el);
|
|
70
|
+
else if (el.tag === 'script' && el.rawText && !/json/i.test(a.type || '')) for (const v of vendorsInScript(el.rawText)) out.push({ kind: 'inline', url: null, host: v.domains[0], vendor: v, line: el.line, el, inline: true });
|
|
71
|
+
else if (el.tag === 'iframe' && a.src) push('iframe', a.src, el);
|
|
72
|
+
else if (el.tag === 'img' && a.src) { const px = (a.width === '1' && a.height === '1') || /display\s*:\s*none/.test(a.style || '') || inNoscript(el); push(px ? 'pixel' : 'image', a.src, el); }
|
|
73
|
+
else if (el.tag === 'link' && a.href && /stylesheet|preconnect|dns-prefetch|preload|prefetch/i.test(a.rel || '')) push(/stylesheet/i.test(a.rel) ? 'stylesheet' : 'hint', a.href, el);
|
|
74
|
+
else if ((el.tag === 'video' || el.tag === 'audio' || el.tag === 'source' || el.tag === 'embed' || el.tag === 'object') && (a.src || a.data)) push('media', a.src || a.data, el);
|
|
75
|
+
}
|
|
76
|
+
return out;
|
|
77
|
+
}
|