spiderlint 0.0.0 → 0.143.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +235 -0
- package/README.md +244 -0
- package/bin/spiderlint.js +11 -0
- package/dist/agent.js +46 -0
- package/dist/cache/extractors.js +37 -0
- package/dist/cache/http.js +99 -0
- package/dist/cache/index.js +101 -0
- package/dist/cache/purge.js +63 -0
- package/dist/cache/status.js +52 -0
- package/dist/cli.js +602 -0
- package/dist/color.js +10 -0
- package/dist/config/environment.js +88 -0
- package/dist/config/index.js +104 -0
- package/dist/config/policy.js +170 -0
- package/dist/config/schema.js +127 -0
- package/dist/crawl/body.js +92 -0
- package/dist/crawl/browser.js +498 -0
- package/dist/crawl/crawl.js +58 -0
- package/dist/crawl/diversity.js +153 -0
- package/dist/crawl/dns.js +194 -0
- package/dist/crawl/fetch.js +48 -0
- package/dist/crawl/frontier.js +370 -0
- package/dist/crawl/guard.js +69 -0
- package/dist/crawl/http.js +171 -0
- package/dist/crawl/links.js +89 -0
- package/dist/crawl/log.js +27 -0
- package/dist/crawl/network.js +88 -0
- package/dist/crawl/probe.js +107 -0
- package/dist/crawl/profile.js +33 -0
- package/dist/crawl/rdap.js +86 -0
- package/dist/crawl/resolve.js +132 -0
- package/dist/crawl/resources.js +233 -0
- package/dist/crawl/robots.js +100 -0
- package/dist/crawl/route.js +92 -0
- package/dist/crawl/scope.js +32 -0
- package/dist/crawl/scratch.js +15 -0
- package/dist/crawl/sitemap.js +187 -0
- package/dist/crawl/svcb.js +59 -0
- package/dist/crawl/tls-hello.js +297 -0
- package/dist/crawl/tls-probe.js +71 -0
- package/dist/facts/browser.js +138 -0
- package/dist/facts/byline.js +17 -0
- package/dist/facts/co2.js +27 -0
- package/dist/facts/csp.js +71 -0
- package/dist/facts/export.js +79 -0
- package/dist/facts/flatten.js +33 -0
- package/dist/facts/graph.js +73 -0
- package/dist/facts/headers.js +330 -0
- package/dist/facts/html.js +230 -0
- package/dist/facts/labels.js +89 -0
- package/dist/facts/language.js +32 -0
- package/dist/facts/parity.js +38 -0
- package/dist/facts/read-note.js +11 -0
- package/dist/facts/resources.js +46 -0
- package/dist/facts/robots.js +13 -0
- package/dist/facts/sites.js +140 -0
- package/dist/facts/transport.js +200 -0
- package/dist/facts/types.js +4 -0
- package/dist/facts/vendors.js +76 -0
- package/dist/fold/index.js +94 -0
- package/dist/groups/assign.js +45 -0
- package/dist/groups/sample.js +60 -0
- package/dist/i18n.js +92 -0
- package/dist/index.js +607 -0
- package/dist/interrupt.js +37 -0
- package/dist/logger.js +95 -0
- package/dist/plugins/axe.js +104 -0
- package/dist/plugins/carbon-txt.js +96 -0
- package/dist/plugins/cookies-registry.js +25 -0
- package/dist/plugins/cookies.js +168 -0
- package/dist/plugins/css.js +328 -0
- package/dist/plugins/dns.js +864 -0
- package/dist/plugins/feed-content.js +132 -0
- package/dist/plugins/feed-model.js +282 -0
- package/dist/plugins/feeds.js +836 -0
- package/dist/plugins/html-validate.js +191 -0
- package/dist/plugins/htmlhint.js +95 -0
- package/dist/plugins/icons.js +302 -0
- package/dist/plugins/images.js +269 -0
- package/dist/plugins/index.js +286 -0
- package/dist/plugins/keyboard.js +194 -0
- package/dist/plugins/lighthouse.js +110 -0
- package/dist/plugins/link-text.js +73 -0
- package/dist/plugins/list.js +22 -0
- package/dist/plugins/live.js +254 -0
- package/dist/plugins/mail.js +985 -0
- package/dist/plugins/manifest.js +97 -0
- package/dist/plugins/markup.js +98 -0
- package/dist/plugins/messages.js +25 -0
- package/dist/plugins/network.js +192 -0
- package/dist/plugins/origin.js +488 -0
- package/dist/plugins/profile-links.js +102 -0
- package/dist/plugins/schema-registry.js +101 -0
- package/dist/plugins/sshfp.js +159 -0
- package/dist/plugins/stack.js +161 -0
- package/dist/plugins/structured-data.js +358 -0
- package/dist/plugins/tls-probe.js +298 -0
- package/dist/plugins/tls-registry.js +418 -0
- package/dist/plugins/tls-scan.js +280 -0
- package/dist/plugins/trackers-registry.js +27 -0
- package/dist/plugins/trackers.js +56 -0
- package/dist/plugins/types.js +7 -0
- package/dist/plugins/visit.js +50 -0
- package/dist/plugins/well-known-registry.js +111 -0
- package/dist/plugins/well-known.js +732 -0
- package/dist/progress.js +162 -0
- package/dist/report/agent.js +122 -0
- package/dist/report/checkstyle.js +22 -0
- package/dist/report/csv.js +18 -0
- package/dist/report/html.js +266 -0
- package/dist/report/human.js +220 -0
- package/dist/report/index.js +16 -0
- package/dist/report/json.js +8 -0
- package/dist/report/printable.js +29 -0
- package/dist/report/rating.js +26 -0
- package/dist/report/sarif.js +100 -0
- package/dist/report/stats.js +46 -0
- package/dist/rules/builtin.js +465 -0
- package/dist/rules/caching.js +103 -0
- package/dist/rules/catalog.js +139 -0
- package/dist/rules/clock.js +48 -0
- package/dist/rules/declarative.js +234 -0
- package/dist/rules/deprecated.js +45 -0
- package/dist/rules/disclosure.js +99 -0
- package/dist/rules/fix.js +21 -0
- package/dist/rules/i18n.js +131 -0
- package/dist/rules/insights.js +100 -0
- package/dist/rules/length.js +52 -0
- package/dist/rules/message.js +48 -0
- package/dist/rules/relations.js +88 -0
- package/dist/rules/robots.js +75 -0
- package/dist/rules/rulesets.js +134 -0
- package/dist/rules/run.js +75 -0
- package/dist/rules/score.js +61 -0
- package/dist/rules/types.js +7 -0
- package/dist/rules/url.js +90 -0
- package/dist/server/analytics.js +49 -0
- package/dist/server/api.js +134 -0
- package/dist/server/clients.js +54 -0
- package/dist/server/clock.js +33 -0
- package/dist/server/compress.js +58 -0
- package/dist/server/health.js +14 -0
- package/dist/server/jobs.js +82 -0
- package/dist/server/main.js +83 -0
- package/dist/server/page.js +161 -0
- package/dist/server/policy.js +145 -0
- package/dist/server/providers.js +60 -0
- package/dist/server/queue.js +50 -0
- package/dist/server/scan.js +39 -0
- package/dist/server/settings.js +197 -0
- package/dist/server/web.js +353 -0
- package/dist/server/worker.js +129 -0
- package/dist/store/disk.js +215 -0
- package/dist/store/memory.js +18 -0
- package/dist/telemetry-environment.js +9 -0
- package/dist/telemetry.js +104 -0
- package/locales/es/LC_MESSAGES/messages.po +1864 -0
- package/locales/uk/LC_MESSAGES/messages.po +1864 -0
- package/package.json +134 -2
- package/presets/browser.yaml +26 -0
- package/presets/graph.yaml +35 -0
- package/presets/i18n.yaml +18 -0
- package/presets/insights.yaml +12 -0
- package/presets/links.yaml +22 -0
- package/presets/parity.yaml +59 -0
- package/presets/performance.yaml +271 -0
- package/presets/recommended.yaml +50 -0
- package/presets/redirects.yaml +23 -0
- package/presets/resources.yaml +29 -0
- package/presets/robots.yaml +13 -0
- package/presets/security-headers.yaml +257 -0
- package/presets/seo.yaml +380 -0
- package/presets/sitemap.yaml +34 -0
- package/presets/suggestions.yaml +97 -0
- package/presets/sustainability.yaml +16 -0
- package/presets/tls.yaml +72 -0
- package/presets/vendor.yaml +14 -0
- package/presets/web-comprehensive.yaml +17 -0
- package/presets/web-quick.yaml +15 -0
- package/src/server/apple-touch-icon.png +0 -0
- package/src/server/logo-36.png +0 -0
- package/src/server/logo-72.png +0 -0
- package/src/server/logo.png +0 -0
- package/vendors/dns.yaml +21 -0
- package/vendors/paths.yaml +55 -0
- package/vendors/stack.yaml +57 -0
|
@@ -0,0 +1,280 @@
|
|
|
1
|
+
// SPDX-FileCopyrightText: 2026 Damián Búho <damian.buho@proton.me>
|
|
2
|
+
//
|
|
3
|
+
// SPDX-License-Identifier: AGPL-3.0-only
|
|
4
|
+
import { connect } from "node:net";
|
|
5
|
+
import { setTimeout as sleep } from "node:timers/promises";
|
|
6
|
+
import { reason } from "../crawl/fetch.js";
|
|
7
|
+
import { pace } from "../crawl/network.js";
|
|
8
|
+
import { clientHello, curve, dhBits, FALLBACK_SCSV, INAPPROPRIATE_FALLBACK, keyShares, readFlight, SSL2_KINDS, sslv2Hello, TLS13_MANDATORY, VERSIONS } from "../crawl/tls-hello.js";
|
|
9
|
+
import { log } from "../logger.js";
|
|
10
|
+
import { GROUPS, SUITES } from "./tls-registry.js";
|
|
11
|
+
// Milliseconds one connection may take to connect and answer a hello.
|
|
12
|
+
const EXCHANGE_MS = 8000;
|
|
13
|
+
// Connections open to one address at a time.
|
|
14
|
+
const SOCKETS = 4;
|
|
15
|
+
// Retries of a connection that failed before its hello went out.
|
|
16
|
+
const RETRIES = 2;
|
|
17
|
+
// Hellos per enumeration, beyond any server’s suite or group count.
|
|
18
|
+
const MAX_ROUNDS = 128;
|
|
19
|
+
// Bytes read before a flight that never completes is cut off.
|
|
20
|
+
const MAX_BYTES = 256 * 1024;
|
|
21
|
+
// Protocols up to TLS 1.2, oldest first, each probed with its own ClientHello.
|
|
22
|
+
const LEGACY_HELLOS = ["SSLv3", "TLSv1", "TLSv1.1", "TLSv1.2"];
|
|
23
|
+
// Every protocol, oldest first, as `protocols` lists them.
|
|
24
|
+
const ORDER = ["SSLv2", ...LEGACY_HELLOS, "TLSv1.3"];
|
|
25
|
+
// Versions RFC 8996 and RFC 7568 retire.
|
|
26
|
+
const LEGACY = new Set(["SSLv2", "SSLv3", "TLSv1", "TLSv1.1"]);
|
|
27
|
+
const MODERN_SUITES = Object.keys(SUITES)
|
|
28
|
+
.map(Number)
|
|
29
|
+
.filter((code) => !SUITES[code]?.includes("_WITH_"));
|
|
30
|
+
const CLASSIC_SUITES = Object.keys(SUITES)
|
|
31
|
+
.map(Number)
|
|
32
|
+
.filter((code) => SUITES[code]?.includes("_WITH_"));
|
|
33
|
+
// Groups a TLS 1.2 ECDHE suite may use; finite-field ones are left out so a DHE suite shows the server’s own prime.
|
|
34
|
+
const CURVES = Object.keys(GROUPS)
|
|
35
|
+
.map(Number)
|
|
36
|
+
.filter((code) => code < 256);
|
|
37
|
+
const ALL_GROUPS = Object.keys(GROUPS).map(Number);
|
|
38
|
+
// Thrown when a connection failed before its hello went out, which a retry may cure.
|
|
39
|
+
class Unconnected extends Error {
|
|
40
|
+
}
|
|
41
|
+
// Connections open per address, and the callers waiting for one of them to close.
|
|
42
|
+
const busy = new Map();
|
|
43
|
+
// Waits for one of the address’s connection slots; the returned function frees it.
|
|
44
|
+
async function acquire(address) {
|
|
45
|
+
const state = busy.get(address) ?? { open: 0, waiting: [] };
|
|
46
|
+
busy.set(address, state);
|
|
47
|
+
if (state.open < SOCKETS)
|
|
48
|
+
state.open++;
|
|
49
|
+
else
|
|
50
|
+
await new Promise((resolve) => {
|
|
51
|
+
state.waiting.push(resolve);
|
|
52
|
+
});
|
|
53
|
+
return () => {
|
|
54
|
+
const next = state.waiting.shift();
|
|
55
|
+
if (next)
|
|
56
|
+
next();
|
|
57
|
+
else if (--state.open === 0)
|
|
58
|
+
busy.delete(address);
|
|
59
|
+
};
|
|
60
|
+
}
|
|
61
|
+
// One hello over a fresh connection and whatever the server answered before it closed, went quiet or said all it will.
|
|
62
|
+
function once(target, hello, client) {
|
|
63
|
+
return new Promise((resolve, reject) => {
|
|
64
|
+
const socket = connect({ host: target.address, port: target.port });
|
|
65
|
+
let data = Buffer.alloc(0);
|
|
66
|
+
let isConnected = false;
|
|
67
|
+
let isDone = false;
|
|
68
|
+
const finish = (error) => {
|
|
69
|
+
if (isDone)
|
|
70
|
+
return;
|
|
71
|
+
isDone = true;
|
|
72
|
+
clearTimeout(timer);
|
|
73
|
+
target.signal.removeEventListener("abort", abort);
|
|
74
|
+
socket.destroy();
|
|
75
|
+
if (error)
|
|
76
|
+
reject(error);
|
|
77
|
+
else
|
|
78
|
+
resolve(readFlight(data, client));
|
|
79
|
+
};
|
|
80
|
+
const abort = () => finish(new Error("aborted"));
|
|
81
|
+
const timer = setTimeout(() => finish(isConnected ? undefined : new Unconnected(`no connection within ${EXCHANGE_MS} ms`)), EXCHANGE_MS);
|
|
82
|
+
target.signal.addEventListener("abort", abort, { once: true });
|
|
83
|
+
socket.once("connect", () => {
|
|
84
|
+
isConnected = true;
|
|
85
|
+
socket.write(hello);
|
|
86
|
+
});
|
|
87
|
+
socket.on("data", (chunk) => {
|
|
88
|
+
data = Buffer.concat([data, chunk]);
|
|
89
|
+
if (data.length > MAX_BYTES || readFlight(data, client).complete)
|
|
90
|
+
finish();
|
|
91
|
+
});
|
|
92
|
+
socket.once("error", (error) => finish(isConnected ? undefined : new Unconnected(reason(error))));
|
|
93
|
+
socket.once("close", () => finish());
|
|
94
|
+
});
|
|
95
|
+
}
|
|
96
|
+
// One hello under the run’s pace and the address’s socket cap, retried with backoff and jitter while no connection opens.
|
|
97
|
+
async function exchange(target, hello, client) {
|
|
98
|
+
for (let attempt = 0;; attempt++) {
|
|
99
|
+
const release = await acquire(target.address);
|
|
100
|
+
let failure;
|
|
101
|
+
try {
|
|
102
|
+
await pace();
|
|
103
|
+
return await once(target, hello, client);
|
|
104
|
+
}
|
|
105
|
+
catch (error) {
|
|
106
|
+
failure = error;
|
|
107
|
+
}
|
|
108
|
+
finally {
|
|
109
|
+
release();
|
|
110
|
+
}
|
|
111
|
+
if (!(failure instanceof Unconnected) || attempt >= RETRIES || target.signal.aborted)
|
|
112
|
+
throw failure;
|
|
113
|
+
const wait = Math.round(250 * 2 ** attempt * (0.5 + Math.random()));
|
|
114
|
+
log.debug({ address: target.address, port: target.port, attempt, wait, error: reason(failure) }, "tls scan connection retried");
|
|
115
|
+
await sleep(wait, undefined, { signal: target.signal });
|
|
116
|
+
}
|
|
117
|
+
}
|
|
118
|
+
// Whether `flight` is a handshake at `protocol` on one of the `offered` suites.
|
|
119
|
+
function isAccepted(flight, protocol, offered) {
|
|
120
|
+
return flight.version === VERSIONS[protocol] && flight.suite !== undefined && offered.includes(flight.suite);
|
|
121
|
+
}
|
|
122
|
+
// The suites a protocol accepts, each found by offering the rest until the server refuses; `isFull` stops at the first.
|
|
123
|
+
async function enumerate(target, protocol, suites, isFull) {
|
|
124
|
+
const accepted = [];
|
|
125
|
+
const isModern = protocol === "TLSv1.3";
|
|
126
|
+
for (let offered = suites, round = 0; offered.length > 0 && round < MAX_ROUNDS; round++) {
|
|
127
|
+
const flight = await exchange(target, clientHello({ protocol, suites: offered, host: target.host, groups: isModern ? ALL_GROUPS : CURVES, ...(isModern ? { shares: [] } : { compression: true }) }));
|
|
128
|
+
const isTaken = isAccepted(flight, protocol, offered);
|
|
129
|
+
log.debug({ host: target.host, protocol, round, offered: offered.length, suite: flight.suite, version: flight.version, alert: flight.alert, error: flight.error, isTaken }, "tls suite offered");
|
|
130
|
+
if (!isTaken)
|
|
131
|
+
break;
|
|
132
|
+
accepted.push(flight);
|
|
133
|
+
if (!isFull)
|
|
134
|
+
break;
|
|
135
|
+
offered = offered.filter((suite) => suite !== flight.suite);
|
|
136
|
+
}
|
|
137
|
+
return accepted;
|
|
138
|
+
}
|
|
139
|
+
// The group a TLS 1.3 HelloRetryRequest or an ECDHE ServerKeyExchange names.
|
|
140
|
+
function namedGroup(flight, isModern) {
|
|
141
|
+
return isModern ? flight.retry && flight.group : flight.keyExchange && curve(flight.keyExchange);
|
|
142
|
+
}
|
|
143
|
+
// Groups the server takes, each named in a TLS 1.3 HelloRetryRequest or a ServerKeyExchange, offering the rest until it refuses.
|
|
144
|
+
async function enumerateGroups(target, protocol, suites, groups) {
|
|
145
|
+
const found = [];
|
|
146
|
+
const isModern = protocol === "TLSv1.3";
|
|
147
|
+
for (let offered = groups, round = 0; offered.length > 0 && round < MAX_ROUNDS; round++) {
|
|
148
|
+
const flight = await exchange(target, clientHello({ protocol, suites, host: target.host, groups: offered, ...(isModern && { shares: [] }) }));
|
|
149
|
+
const group = namedGroup(flight, isModern);
|
|
150
|
+
log.debug({ host: target.host, protocol, round, offered: offered.length, group, alert: flight.alert }, "tls group offered");
|
|
151
|
+
if (group === undefined || !offered.includes(group))
|
|
152
|
+
break;
|
|
153
|
+
found.push(group);
|
|
154
|
+
offered = offered.filter((entry) => entry !== group);
|
|
155
|
+
}
|
|
156
|
+
return found;
|
|
157
|
+
}
|
|
158
|
+
// Whether the server picks by its own order: the same suite when the accepted ones are offered forwards and reversed.
|
|
159
|
+
async function serverOrder(target, protocol, suites) {
|
|
160
|
+
if (suites.length < 2)
|
|
161
|
+
return;
|
|
162
|
+
const [forward, reverse] = [await exchange(target, clientHello({ protocol, suites, host: target.host, groups: CURVES })), await exchange(target, clientHello({ protocol, suites: suites.toReversed(), host: target.host, groups: CURVES }))];
|
|
163
|
+
log.debug({ host: target.host, protocol, forward: forward.suite, reverse: reverse.suite }, "tls server order tried");
|
|
164
|
+
return forward.suite === undefined || reverse.suite === undefined ? undefined : forward.suite === reverse.suite;
|
|
165
|
+
}
|
|
166
|
+
// Whether a retry below the best version is refused as a needless fallback (RFC 7507).
|
|
167
|
+
async function fallback(target, protocol, suites) {
|
|
168
|
+
const flight = await exchange(target, clientHello({ protocol, suites: [...suites, FALLBACK_SCSV], host: target.host, groups: CURVES }));
|
|
169
|
+
log.debug({ host: target.host, protocol, alert: flight.alert, suite: flight.suite }, "tls fallback tried");
|
|
170
|
+
if (flight.alert === INAPPROPRIATE_FALLBACK)
|
|
171
|
+
return true;
|
|
172
|
+
if (flight.suite !== undefined)
|
|
173
|
+
return false;
|
|
174
|
+
}
|
|
175
|
+
// Whether SSLv2 answers, with the cipher kinds it offers.
|
|
176
|
+
async function sslv2(target) {
|
|
177
|
+
const flight = await exchange(target, sslv2Hello());
|
|
178
|
+
log.debug({ host: target.host, kinds: flight.kinds, alert: flight.alert }, "sslv2 tried");
|
|
179
|
+
return flight.kinds ?? [];
|
|
180
|
+
}
|
|
181
|
+
// The certificates and stapled OCSP response of a TLS 1.3 handshake, decrypted up to its Certificate.
|
|
182
|
+
async function modernCertificates(target) {
|
|
183
|
+
const keys = keyShares();
|
|
184
|
+
const hello = clientHello({ protocol: "TLSv1.3", suites: [TLS13_MANDATORY], host: target.host, groups: keys.shares.map((share) => share.group), shares: keys.shares });
|
|
185
|
+
const flight = await exchange(target, hello, keys.client(hello));
|
|
186
|
+
log.debug({ host: target.host, certificates: flight.certificates?.length, ocsp: flight.ocsp !== undefined, retry: flight.retry, error: flight.error }, "tls 1.3 certificates read");
|
|
187
|
+
return flight;
|
|
188
|
+
}
|
|
189
|
+
// What a suite’s name says about its strength.
|
|
190
|
+
export function traits(name) {
|
|
191
|
+
const isLegacy = name.startsWith("SSL_CK_");
|
|
192
|
+
const isModern = !isLegacy && !name.includes("_WITH_");
|
|
193
|
+
const isNull = /_WITH_NULL_/.test(name) || /^TLS_SHA\d+_SHA\d+$/.test(name);
|
|
194
|
+
const isBroken = isLegacy || isNull || /_anon_|EXPORT|_RC4_|_RC2_|(?:^|_)DES(?:40)?_CBC_/.test(name);
|
|
195
|
+
const isForward = isModern || /^TLS_(?:EC)?DH(?:E_|_anon_)/.test(name);
|
|
196
|
+
const isCbc = name.includes("_CBC");
|
|
197
|
+
const isSmallBlock = /3DES|IDEA|(?:^|_)DES(?:40)?_|RC2|_28147_/.test(name);
|
|
198
|
+
return { isInsecure: isBroken, isWeak: !isBroken && (!isForward || isCbc || isSmallBlock), isForward, isCbc, isSmallBlock };
|
|
199
|
+
}
|
|
200
|
+
// The attacks negotiation alone shows the server open to.
|
|
201
|
+
function vulnerabilities(byProtocol, all, bits, isCompressed) {
|
|
202
|
+
const found = [
|
|
203
|
+
(byProtocol.SSLv3 ?? []).some((name) => traits(name).isCbc) && "poodle",
|
|
204
|
+
(byProtocol.TLSv1 ?? []).some((name) => traits(name).isCbc) && "beast",
|
|
205
|
+
all.some((name) => traits(name).isSmallBlock) && "sweet32",
|
|
206
|
+
all.some((name) => name.startsWith("TLS_RSA_EXPORT")) && "freak",
|
|
207
|
+
(all.some((name) => /_DHE?_(?:RSA|DSS|anon)_EXPORT/.test(name)) || (bits !== undefined && bits < 1024)) && "logjam",
|
|
208
|
+
all.some((name) => /_RC4_/.test(name)) && "rc4",
|
|
209
|
+
(byProtocol.SSLv2 ?? []).length > 0 && "drown",
|
|
210
|
+
isCompressed && "crime",
|
|
211
|
+
];
|
|
212
|
+
return found.filter((entry) => typeof entry === "string");
|
|
213
|
+
}
|
|
214
|
+
// Names of suite codes, deduplicated in order.
|
|
215
|
+
function names(codes) {
|
|
216
|
+
return [...new Set(codes.map((code) => SUITES[code] ?? `0x${code.toString(16).padStart(4, "0")}`))];
|
|
217
|
+
}
|
|
218
|
+
// Protocols, suites, groups and negotiation-level weaknesses of one origin, from hand-built hellos; `isFull` is false for one hello per protocol.
|
|
219
|
+
export async function scan(target, isFull) {
|
|
220
|
+
const [kinds, classic, modern] = await Promise.all([sslv2(target), Promise.all(LEGACY_HELLOS.map((protocol) => enumerate(target, protocol, CLASSIC_SUITES, isFull))), enumerate(target, "TLSv1.3", MODERN_SUITES, isFull)]);
|
|
221
|
+
const flights = new Map([...LEGACY_HELLOS.map((protocol, index) => [protocol, classic[index] ?? []]), ["TLSv1.3", modern]]);
|
|
222
|
+
const isSupported = (protocol) => (protocol === "SSLv2" ? kinds : (flights.get(protocol) ?? [])).length > 0;
|
|
223
|
+
const protocols = ORDER.filter((protocol) => isSupported(protocol));
|
|
224
|
+
const best = LEGACY_HELLOS.findLast((protocol) => isSupported(protocol));
|
|
225
|
+
const bestFlights = best ? (flights.get(best) ?? []) : [];
|
|
226
|
+
const [first] = bestFlights;
|
|
227
|
+
const hasModern = modern.length > 0;
|
|
228
|
+
const certified = hasModern ? await modernCertificates(target) : undefined;
|
|
229
|
+
const supported = LEGACY_HELLOS.filter((protocol) => protocols.includes(protocol));
|
|
230
|
+
const fallen = hasModern ? supported.at(-1) : supported.at(-2);
|
|
231
|
+
const legacyFlights = classic.flat();
|
|
232
|
+
const isCompressed = legacyFlights.some((flight) => flight.compression === 1);
|
|
233
|
+
const facts = {
|
|
234
|
+
protocols,
|
|
235
|
+
legacy: protocols.filter((protocol) => LEGACY.has(protocol)),
|
|
236
|
+
...(first && { "secure-renegotiation": first.extensions?.includes(0xff_01) ?? false, heartbeat: first.extensions?.includes(15) ?? false }),
|
|
237
|
+
...(legacyFlights.length > 0 && { compression: isCompressed }),
|
|
238
|
+
};
|
|
239
|
+
if (fallen) {
|
|
240
|
+
const scsv = await fallback(target, fallen, CLASSIC_SUITES);
|
|
241
|
+
if (scsv !== undefined)
|
|
242
|
+
facts["fallback-scsv"] = scsv;
|
|
243
|
+
}
|
|
244
|
+
const withCertificates = certified?.certificates?.length ? certified : [...bestFlights, ...legacyFlights].find((flight) => (flight.certificates?.length ?? 0) > 0);
|
|
245
|
+
if (withCertificates?.certificates)
|
|
246
|
+
facts.certificates = withCertificates.certificates;
|
|
247
|
+
if (withCertificates?.ocsp)
|
|
248
|
+
facts.ocsp = withCertificates.ocsp;
|
|
249
|
+
log.debug({ host: target.host, protocols, best, isFull, certificates: facts.certificates?.length }, "tls protocols scanned");
|
|
250
|
+
if (!isFull)
|
|
251
|
+
return facts;
|
|
252
|
+
const byProtocol = Object.fromEntries(protocols.map((protocol) => [protocol, protocol === "SSLv2" ? kinds.map((kind) => SSL2_KINDS[kind] ?? `0x${kind.toString(16)}`) : names((flights.get(protocol) ?? []).map((flight) => flight.suite ?? 0))]));
|
|
253
|
+
const all = [...new Set(ORDER.toReversed().flatMap((protocol) => byProtocol[protocol] ?? []))];
|
|
254
|
+
const dhe = legacyFlights.filter((flight) => flight.keyExchange && /^TLS_DH(?:E_(?:RSA|DSS)|_anon)_/.test(SUITES[flight.suite ?? 0] ?? ""));
|
|
255
|
+
const bits = dhe.length > 0 ? Math.min(...dhe.map((flight) => dhBits(flight.keyExchange))) : undefined;
|
|
256
|
+
const ecdhe = bestFlights.map((flight) => flight.suite ?? 0).filter((code) => /^TLS_ECDHE_(?:RSA|ECDSA)_/.test(SUITES[code] ?? ""));
|
|
257
|
+
const groups = [
|
|
258
|
+
...(hasModern
|
|
259
|
+
? await enumerateGroups(target, "TLSv1.3", modern.map((flight) => flight.suite ?? 0), ALL_GROUPS)
|
|
260
|
+
: []),
|
|
261
|
+
...(best && ecdhe.length > 0 ? await enumerateGroups(target, best, ecdhe, CURVES) : []),
|
|
262
|
+
];
|
|
263
|
+
const order = best
|
|
264
|
+
? await serverOrder(target, best, bestFlights.map((flight) => flight.suite ?? 0))
|
|
265
|
+
: undefined;
|
|
266
|
+
const forward = all.map((name) => traits(name).isForward);
|
|
267
|
+
log.debug({ host: target.host, ciphers: all.length, groups: groups.length, bits, order }, "tls suites scanned");
|
|
268
|
+
return {
|
|
269
|
+
...facts,
|
|
270
|
+
ciphers: all,
|
|
271
|
+
"ciphers-by-protocol": byProtocol,
|
|
272
|
+
"weak-ciphers": all.filter((name) => traits(name).isWeak),
|
|
273
|
+
"insecure-ciphers": all.filter((name) => traits(name).isInsecure),
|
|
274
|
+
...(order !== undefined && { "server-order": order }),
|
|
275
|
+
...(all.length > 0 && { "forward-secrecy": forward.every(Boolean) ? "all" : forward.some(Boolean) ? "some" : "none" }),
|
|
276
|
+
groups: [...new Set(groups.map((group) => GROUPS[group] ?? String(group)))],
|
|
277
|
+
...(bits !== undefined && { "dh-bits": bits }),
|
|
278
|
+
vulnerabilities: vulnerabilities(byProtocol, all, bits, isCompressed),
|
|
279
|
+
};
|
|
280
|
+
}
|
|
@@ -0,0 +1,27 @@
|
|
|
1
|
+
// SPDX-FileCopyrightText: 2026 Damián Búho <damian.buho@proton.me>
|
|
2
|
+
//
|
|
3
|
+
// SPDX-License-Identifier: AGPL-3.0-only
|
|
4
|
+
// Analytics and ad-tech hosts by vendor, hand-kept as of 2026-09-25; a host matches itself and its subdomains.
|
|
5
|
+
export const TRACKERS = {
|
|
6
|
+
"Adobe Analytics": ["omtrdc.net", "2o7.net", "demdex.net"],
|
|
7
|
+
Amplitude: ["amplitude.com"],
|
|
8
|
+
"Bing Ads": ["bat.bing.com"],
|
|
9
|
+
Criteo: ["criteo.com", "criteo.net"],
|
|
10
|
+
"Google Ads": ["doubleclick.net", "googleadservices.com", "googlesyndication.com", "adservice.google.com"],
|
|
11
|
+
"Google Analytics": ["google-analytics.com", "analytics.google.com", "googletagmanager.com"],
|
|
12
|
+
Heap: ["heapanalytics.com"],
|
|
13
|
+
Hotjar: ["hotjar.com", "hotjar.io"],
|
|
14
|
+
FullStory: ["fullstory.com"],
|
|
15
|
+
"LinkedIn Insight": ["snap.licdn.com", "px.ads.linkedin.com"],
|
|
16
|
+
"Meta Pixel": ["connect.facebook.net"],
|
|
17
|
+
"Microsoft Clarity": ["clarity.ms"],
|
|
18
|
+
Mixpanel: ["mixpanel.com", "mxpnl.com"],
|
|
19
|
+
Outbrain: ["outbrain.com"],
|
|
20
|
+
Quantcast: ["quantserve.com", "quantcount.com"],
|
|
21
|
+
Scorecard: ["scorecardresearch.com"],
|
|
22
|
+
Segment: ["segment.com", "segment.io"],
|
|
23
|
+
Taboola: ["taboola.com"],
|
|
24
|
+
"TikTok Pixel": ["analytics.tiktok.com"],
|
|
25
|
+
"X Ads": ["static.ads-twitter.com", "ads-api.twitter.com"],
|
|
26
|
+
"Yandex Metrica": ["mc.yandex.ru", "mc.yandex.com"],
|
|
27
|
+
};
|
|
@@ -0,0 +1,56 @@
|
|
|
1
|
+
// SPDX-FileCopyrightText: 2026 Damián Búho <damian.buho@proton.me>
|
|
2
|
+
//
|
|
3
|
+
// SPDX-License-Identifier: AGPL-3.0-only
|
|
4
|
+
import { log } from "../logger.js";
|
|
5
|
+
import { said } from "../rules/message.js";
|
|
6
|
+
import { TRACKERS } from "./trackers-registry.js";
|
|
7
|
+
import { definePlugin } from "./types.js";
|
|
8
|
+
const HOSTS = Object.entries(TRACKERS).flatMap(([vendor, hosts]) => hosts.map((host) => [host, vendor]));
|
|
9
|
+
// The vendor whose tracker host is `host` or a parent of it.
|
|
10
|
+
function vendorOf(host) {
|
|
11
|
+
return HOSTS.find(([tracker]) => host === tracker || host.endsWith(`.${tracker}`))?.[1];
|
|
12
|
+
}
|
|
13
|
+
// One finding per tracker vendor a page loads from, with its hosts and the pages loading it.
|
|
14
|
+
const inventory = (severity) => ({
|
|
15
|
+
meta: { id: "trackers/inventory", severity, scope: "site", facts: ["resources"], docs: "https://specification.website/spec/privacy/analytics-privacy/", fix: "Drop the vendor, or name it in the privacy policy and load it only after consent." },
|
|
16
|
+
check(pages) {
|
|
17
|
+
const found = new Map();
|
|
18
|
+
for (const page of pages) {
|
|
19
|
+
const resources = page.resources ?? [];
|
|
20
|
+
for (const resource of resources) {
|
|
21
|
+
const host = URL.canParse(resource.url) ? new URL(resource.url).hostname : "";
|
|
22
|
+
const vendor = vendorOf(host);
|
|
23
|
+
if (!vendor)
|
|
24
|
+
continue;
|
|
25
|
+
const entry = found.get(vendor) ?? { hosts: new Set(), urls: new Set() };
|
|
26
|
+
found.set(vendor, entry);
|
|
27
|
+
entry.hosts.add(host);
|
|
28
|
+
entry.urls.add(page.url.href);
|
|
29
|
+
}
|
|
30
|
+
}
|
|
31
|
+
log.debug({ rule: "trackers/inventory", pages: pages.length, vendors: found.keys().toArray() }, "trackers inventoried");
|
|
32
|
+
return found
|
|
33
|
+
.entries()
|
|
34
|
+
.map(([vendor, { hosts, urls }]) => ({
|
|
35
|
+
rule: "trackers/inventory",
|
|
36
|
+
severity,
|
|
37
|
+
scope: "site",
|
|
38
|
+
url: `https://${hosts.values().next().value}/`,
|
|
39
|
+
...said("{vendor} loads from these hosts on these pages", { vendor }),
|
|
40
|
+
data: { [`https://${hosts.values().next().value}/`]: { hosts: hosts.values().toArray().join(", ") } },
|
|
41
|
+
value: hosts.values().toArray(),
|
|
42
|
+
urls: urls.values().toArray(),
|
|
43
|
+
}))
|
|
44
|
+
.toArray();
|
|
45
|
+
},
|
|
46
|
+
});
|
|
47
|
+
export default definePlugin({
|
|
48
|
+
name: "trackers",
|
|
49
|
+
rules: { "trackers/inventory": inventory },
|
|
50
|
+
presets: {
|
|
51
|
+
trackers: {
|
|
52
|
+
description: "Inventory of the analytics and ad-tech vendors the site loads, from a hand-kept host list",
|
|
53
|
+
rules: { "trackers/inventory": { severity: "info", score: 2.6 } },
|
|
54
|
+
},
|
|
55
|
+
},
|
|
56
|
+
});
|
|
@@ -0,0 +1,50 @@
|
|
|
1
|
+
// SPDX-FileCopyrightText: 2026 Damián Búho <damian.buho@proton.me>
|
|
2
|
+
//
|
|
3
|
+
// SPDX-License-Identifier: AGPL-3.0-only
|
|
4
|
+
import { reason } from "../crawl/fetch.js";
|
|
5
|
+
import { log } from "../logger.js";
|
|
6
|
+
// Milliseconds a fresh page may take to load, then to fall quiet.
|
|
7
|
+
const LOAD_MS = 30_000;
|
|
8
|
+
const QUIET_MS = 5000;
|
|
9
|
+
// A page script defining `describe(element)`: a short CSS path and the opening tag, as locations print them.
|
|
10
|
+
export const DESCRIBE = `function describe(element) {
|
|
11
|
+
const parts = [];
|
|
12
|
+
for (let node = element; node && parts.length < 4; node = node.parentElement) {
|
|
13
|
+
if (node.id) { parts.unshift("#" + CSS.escape(node.id)); break; }
|
|
14
|
+
const same = node.parentElement ? [...node.parentElement.children].filter((sibling) => sibling.localName === node.localName) : [];
|
|
15
|
+
parts.unshift(same.length > 1 ? node.localName + ":nth-of-type(" + (same.indexOf(node) + 1) + ")" : node.localName);
|
|
16
|
+
if (node.localName === "body") break;
|
|
17
|
+
}
|
|
18
|
+
const tag = /^<[^>]*>/.exec(element.outerHTML)?.[0] ?? element.localName;
|
|
19
|
+
return { target: parts.join(" > "), html: tag.length > 120 ? tag.slice(0, 120) + "…" : tag };
|
|
20
|
+
}`;
|
|
21
|
+
// A navigation error from the network rather than the page, which one more attempt may cure.
|
|
22
|
+
const TRANSIENT = /net::ERR_(NETWORK_CHANGED|INTERNET_DISCONNECTED|CONNECTION_(RESET|CLOSED|REFUSED|TIMED_OUT)|TIMED_OUT|NAME_NOT_RESOLVED|ADDRESS_UNREACHABLE|HTTP2_PROTOCOL_ERROR)/;
|
|
23
|
+
// Loads `url`, once more after a transient network error, and waits for its network to fall quiet, reading it anyway once QUIET_MS pass.
|
|
24
|
+
export async function visit(page, url) {
|
|
25
|
+
try {
|
|
26
|
+
await page.goto(url, { waitUntil: "load", timeout: LOAD_MS });
|
|
27
|
+
}
|
|
28
|
+
catch (error) {
|
|
29
|
+
if (!TRANSIENT.test(reason(error)))
|
|
30
|
+
throw error;
|
|
31
|
+
log.warn({ url, error: reason(error) }, "navigation failed on the network, retrying once");
|
|
32
|
+
await page.goto(url, { waitUntil: "load", timeout: LOAD_MS });
|
|
33
|
+
}
|
|
34
|
+
try {
|
|
35
|
+
await page.waitForLoadState("networkidle", { timeout: QUIET_MS });
|
|
36
|
+
}
|
|
37
|
+
catch (error) {
|
|
38
|
+
log.debug({ url, error: reason(error) }, "fresh page never fell quiet; reading it anyway");
|
|
39
|
+
}
|
|
40
|
+
}
|
|
41
|
+
// `use` on a new page in the crawler’s context, closed after, so the crawler’s own page stays as rendered.
|
|
42
|
+
export async function withPage(live, use) {
|
|
43
|
+
const page = await live.context().newPage();
|
|
44
|
+
try {
|
|
45
|
+
return await use(page);
|
|
46
|
+
}
|
|
47
|
+
finally {
|
|
48
|
+
await page.close();
|
|
49
|
+
}
|
|
50
|
+
}
|
|
@@ -0,0 +1,111 @@
|
|
|
1
|
+
// SPDX-FileCopyrightText: 2026 Damián Búho <damian.buho@proton.me>
|
|
2
|
+
//
|
|
3
|
+
// SPDX-License-Identifier: AGPL-3.0-only
|
|
4
|
+
// IANA Well-Known URIs registry suffixes as of 2026-09-25, from https://www.iana.org/assignments/well-known-uris/well-known-uris-1.csv; refreshed by hand.
|
|
5
|
+
export const REGISTERED = new Set([
|
|
6
|
+
"acme-challenge",
|
|
7
|
+
"agent-card.json",
|
|
8
|
+
"amphtml",
|
|
9
|
+
"api-catalog",
|
|
10
|
+
"appspecific",
|
|
11
|
+
"ashrae",
|
|
12
|
+
"assetlinks.json",
|
|
13
|
+
"bluejetty",
|
|
14
|
+
"broadband-labels",
|
|
15
|
+
"brski",
|
|
16
|
+
"caldav",
|
|
17
|
+
"carddav",
|
|
18
|
+
"change-password",
|
|
19
|
+
"cmp",
|
|
20
|
+
"coap",
|
|
21
|
+
"coap-eap",
|
|
22
|
+
"core",
|
|
23
|
+
"csaf",
|
|
24
|
+
"csaf-aggregator",
|
|
25
|
+
"csipaus",
|
|
26
|
+
"csvm",
|
|
27
|
+
"cyclic-trigger",
|
|
28
|
+
"did.json",
|
|
29
|
+
"did-configuration.json",
|
|
30
|
+
"dnt",
|
|
31
|
+
"dnt-policy.txt",
|
|
32
|
+
"dots",
|
|
33
|
+
"easy-proxy",
|
|
34
|
+
"ecips",
|
|
35
|
+
"edhoc",
|
|
36
|
+
"enterprise-network-security",
|
|
37
|
+
"enterprise-transport-security",
|
|
38
|
+
"est",
|
|
39
|
+
"funding-manifest-urls",
|
|
40
|
+
"genid",
|
|
41
|
+
"gnap-as-rs",
|
|
42
|
+
"gpc.json",
|
|
43
|
+
"gs1resolver",
|
|
44
|
+
"hoba",
|
|
45
|
+
"host-meta",
|
|
46
|
+
"host-meta.json",
|
|
47
|
+
"hosting-provider",
|
|
48
|
+
"http-opportunistic",
|
|
49
|
+
"ic-domains",
|
|
50
|
+
"idp-proxy",
|
|
51
|
+
"jmap",
|
|
52
|
+
"keybase.txt",
|
|
53
|
+
"knx",
|
|
54
|
+
"looking-glass",
|
|
55
|
+
"masque",
|
|
56
|
+
"matrix",
|
|
57
|
+
"mercure",
|
|
58
|
+
"mta-sts.txt",
|
|
59
|
+
"mud",
|
|
60
|
+
"nfv-oauth-server-configuration",
|
|
61
|
+
"ni",
|
|
62
|
+
"nodeinfo",
|
|
63
|
+
"nostr.json",
|
|
64
|
+
"oauth-authorization-server",
|
|
65
|
+
"oauth-protected-resource",
|
|
66
|
+
"ohttp-gateway",
|
|
67
|
+
"ojobpub.json",
|
|
68
|
+
"openbindings",
|
|
69
|
+
"openid-federation",
|
|
70
|
+
"open-resource-discovery",
|
|
71
|
+
"openid-configuration",
|
|
72
|
+
"openorg",
|
|
73
|
+
"oslc",
|
|
74
|
+
"pki-validation",
|
|
75
|
+
"posh",
|
|
76
|
+
"privacy-sandbox-attestations.json",
|
|
77
|
+
"private-token-issuer-directory",
|
|
78
|
+
"probing.txt",
|
|
79
|
+
"pvd",
|
|
80
|
+
"rd",
|
|
81
|
+
"related-website-set.json",
|
|
82
|
+
"reload-config",
|
|
83
|
+
"repute-template",
|
|
84
|
+
"resourcesync",
|
|
85
|
+
"sbom",
|
|
86
|
+
"scitt-keys",
|
|
87
|
+
"security.txt",
|
|
88
|
+
"ssf-configuration",
|
|
89
|
+
"ssh-known-hosts",
|
|
90
|
+
"sshfp",
|
|
91
|
+
"stun-key",
|
|
92
|
+
"tea",
|
|
93
|
+
"terraform.json",
|
|
94
|
+
"thread",
|
|
95
|
+
"time",
|
|
96
|
+
"timezone",
|
|
97
|
+
"tdmrep.json",
|
|
98
|
+
"tor-relay",
|
|
99
|
+
"tpcd",
|
|
100
|
+
"traffic-advice",
|
|
101
|
+
"trust.txt",
|
|
102
|
+
"uma2-configuration",
|
|
103
|
+
"vacation-rental.json",
|
|
104
|
+
"void",
|
|
105
|
+
"webauthn",
|
|
106
|
+
"webfinger",
|
|
107
|
+
"webhook-authorized-senders.json",
|
|
108
|
+
"webweaver.json",
|
|
109
|
+
"wot",
|
|
110
|
+
"xregistry",
|
|
111
|
+
]);
|