spiderlint 0.0.0 → 0.143.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (186) hide show
  1. package/LICENSE +235 -0
  2. package/README.md +244 -0
  3. package/bin/spiderlint.js +11 -0
  4. package/dist/agent.js +46 -0
  5. package/dist/cache/extractors.js +37 -0
  6. package/dist/cache/http.js +99 -0
  7. package/dist/cache/index.js +101 -0
  8. package/dist/cache/purge.js +63 -0
  9. package/dist/cache/status.js +52 -0
  10. package/dist/cli.js +602 -0
  11. package/dist/color.js +10 -0
  12. package/dist/config/environment.js +88 -0
  13. package/dist/config/index.js +104 -0
  14. package/dist/config/policy.js +170 -0
  15. package/dist/config/schema.js +127 -0
  16. package/dist/crawl/body.js +92 -0
  17. package/dist/crawl/browser.js +498 -0
  18. package/dist/crawl/crawl.js +58 -0
  19. package/dist/crawl/diversity.js +153 -0
  20. package/dist/crawl/dns.js +194 -0
  21. package/dist/crawl/fetch.js +48 -0
  22. package/dist/crawl/frontier.js +370 -0
  23. package/dist/crawl/guard.js +69 -0
  24. package/dist/crawl/http.js +171 -0
  25. package/dist/crawl/links.js +89 -0
  26. package/dist/crawl/log.js +27 -0
  27. package/dist/crawl/network.js +88 -0
  28. package/dist/crawl/probe.js +107 -0
  29. package/dist/crawl/profile.js +33 -0
  30. package/dist/crawl/rdap.js +86 -0
  31. package/dist/crawl/resolve.js +132 -0
  32. package/dist/crawl/resources.js +233 -0
  33. package/dist/crawl/robots.js +100 -0
  34. package/dist/crawl/route.js +92 -0
  35. package/dist/crawl/scope.js +32 -0
  36. package/dist/crawl/scratch.js +15 -0
  37. package/dist/crawl/sitemap.js +187 -0
  38. package/dist/crawl/svcb.js +59 -0
  39. package/dist/crawl/tls-hello.js +297 -0
  40. package/dist/crawl/tls-probe.js +71 -0
  41. package/dist/facts/browser.js +138 -0
  42. package/dist/facts/byline.js +17 -0
  43. package/dist/facts/co2.js +27 -0
  44. package/dist/facts/csp.js +71 -0
  45. package/dist/facts/export.js +79 -0
  46. package/dist/facts/flatten.js +33 -0
  47. package/dist/facts/graph.js +73 -0
  48. package/dist/facts/headers.js +330 -0
  49. package/dist/facts/html.js +230 -0
  50. package/dist/facts/labels.js +89 -0
  51. package/dist/facts/language.js +32 -0
  52. package/dist/facts/parity.js +38 -0
  53. package/dist/facts/read-note.js +11 -0
  54. package/dist/facts/resources.js +46 -0
  55. package/dist/facts/robots.js +13 -0
  56. package/dist/facts/sites.js +140 -0
  57. package/dist/facts/transport.js +200 -0
  58. package/dist/facts/types.js +4 -0
  59. package/dist/facts/vendors.js +76 -0
  60. package/dist/fold/index.js +94 -0
  61. package/dist/groups/assign.js +45 -0
  62. package/dist/groups/sample.js +60 -0
  63. package/dist/i18n.js +92 -0
  64. package/dist/index.js +607 -0
  65. package/dist/interrupt.js +37 -0
  66. package/dist/logger.js +95 -0
  67. package/dist/plugins/axe.js +104 -0
  68. package/dist/plugins/carbon-txt.js +96 -0
  69. package/dist/plugins/cookies-registry.js +25 -0
  70. package/dist/plugins/cookies.js +168 -0
  71. package/dist/plugins/css.js +328 -0
  72. package/dist/plugins/dns.js +864 -0
  73. package/dist/plugins/feed-content.js +132 -0
  74. package/dist/plugins/feed-model.js +282 -0
  75. package/dist/plugins/feeds.js +836 -0
  76. package/dist/plugins/html-validate.js +191 -0
  77. package/dist/plugins/htmlhint.js +95 -0
  78. package/dist/plugins/icons.js +302 -0
  79. package/dist/plugins/images.js +269 -0
  80. package/dist/plugins/index.js +286 -0
  81. package/dist/plugins/keyboard.js +194 -0
  82. package/dist/plugins/lighthouse.js +110 -0
  83. package/dist/plugins/link-text.js +73 -0
  84. package/dist/plugins/list.js +22 -0
  85. package/dist/plugins/live.js +254 -0
  86. package/dist/plugins/mail.js +985 -0
  87. package/dist/plugins/manifest.js +97 -0
  88. package/dist/plugins/markup.js +98 -0
  89. package/dist/plugins/messages.js +25 -0
  90. package/dist/plugins/network.js +192 -0
  91. package/dist/plugins/origin.js +488 -0
  92. package/dist/plugins/profile-links.js +102 -0
  93. package/dist/plugins/schema-registry.js +101 -0
  94. package/dist/plugins/sshfp.js +159 -0
  95. package/dist/plugins/stack.js +161 -0
  96. package/dist/plugins/structured-data.js +358 -0
  97. package/dist/plugins/tls-probe.js +298 -0
  98. package/dist/plugins/tls-registry.js +418 -0
  99. package/dist/plugins/tls-scan.js +280 -0
  100. package/dist/plugins/trackers-registry.js +27 -0
  101. package/dist/plugins/trackers.js +56 -0
  102. package/dist/plugins/types.js +7 -0
  103. package/dist/plugins/visit.js +50 -0
  104. package/dist/plugins/well-known-registry.js +111 -0
  105. package/dist/plugins/well-known.js +732 -0
  106. package/dist/progress.js +162 -0
  107. package/dist/report/agent.js +122 -0
  108. package/dist/report/checkstyle.js +22 -0
  109. package/dist/report/csv.js +18 -0
  110. package/dist/report/html.js +266 -0
  111. package/dist/report/human.js +220 -0
  112. package/dist/report/index.js +16 -0
  113. package/dist/report/json.js +8 -0
  114. package/dist/report/printable.js +29 -0
  115. package/dist/report/rating.js +26 -0
  116. package/dist/report/sarif.js +100 -0
  117. package/dist/report/stats.js +46 -0
  118. package/dist/rules/builtin.js +465 -0
  119. package/dist/rules/caching.js +103 -0
  120. package/dist/rules/catalog.js +139 -0
  121. package/dist/rules/clock.js +48 -0
  122. package/dist/rules/declarative.js +234 -0
  123. package/dist/rules/deprecated.js +45 -0
  124. package/dist/rules/disclosure.js +99 -0
  125. package/dist/rules/fix.js +21 -0
  126. package/dist/rules/i18n.js +131 -0
  127. package/dist/rules/insights.js +100 -0
  128. package/dist/rules/length.js +52 -0
  129. package/dist/rules/message.js +48 -0
  130. package/dist/rules/relations.js +88 -0
  131. package/dist/rules/robots.js +75 -0
  132. package/dist/rules/rulesets.js +134 -0
  133. package/dist/rules/run.js +75 -0
  134. package/dist/rules/score.js +61 -0
  135. package/dist/rules/types.js +7 -0
  136. package/dist/rules/url.js +90 -0
  137. package/dist/server/analytics.js +49 -0
  138. package/dist/server/api.js +134 -0
  139. package/dist/server/clients.js +54 -0
  140. package/dist/server/clock.js +33 -0
  141. package/dist/server/compress.js +58 -0
  142. package/dist/server/health.js +14 -0
  143. package/dist/server/jobs.js +82 -0
  144. package/dist/server/main.js +83 -0
  145. package/dist/server/page.js +161 -0
  146. package/dist/server/policy.js +145 -0
  147. package/dist/server/providers.js +60 -0
  148. package/dist/server/queue.js +50 -0
  149. package/dist/server/scan.js +39 -0
  150. package/dist/server/settings.js +197 -0
  151. package/dist/server/web.js +353 -0
  152. package/dist/server/worker.js +129 -0
  153. package/dist/store/disk.js +215 -0
  154. package/dist/store/memory.js +18 -0
  155. package/dist/telemetry-environment.js +9 -0
  156. package/dist/telemetry.js +104 -0
  157. package/locales/es/LC_MESSAGES/messages.po +1864 -0
  158. package/locales/uk/LC_MESSAGES/messages.po +1864 -0
  159. package/package.json +134 -2
  160. package/presets/browser.yaml +26 -0
  161. package/presets/graph.yaml +35 -0
  162. package/presets/i18n.yaml +18 -0
  163. package/presets/insights.yaml +12 -0
  164. package/presets/links.yaml +22 -0
  165. package/presets/parity.yaml +59 -0
  166. package/presets/performance.yaml +271 -0
  167. package/presets/recommended.yaml +50 -0
  168. package/presets/redirects.yaml +23 -0
  169. package/presets/resources.yaml +29 -0
  170. package/presets/robots.yaml +13 -0
  171. package/presets/security-headers.yaml +257 -0
  172. package/presets/seo.yaml +380 -0
  173. package/presets/sitemap.yaml +34 -0
  174. package/presets/suggestions.yaml +97 -0
  175. package/presets/sustainability.yaml +16 -0
  176. package/presets/tls.yaml +72 -0
  177. package/presets/vendor.yaml +14 -0
  178. package/presets/web-comprehensive.yaml +17 -0
  179. package/presets/web-quick.yaml +15 -0
  180. package/src/server/apple-touch-icon.png +0 -0
  181. package/src/server/logo-36.png +0 -0
  182. package/src/server/logo-72.png +0 -0
  183. package/src/server/logo.png +0 -0
  184. package/vendors/dns.yaml +21 -0
  185. package/vendors/paths.yaml +55 -0
  186. package/vendors/stack.yaml +57 -0
@@ -0,0 +1,985 @@
1
+ // SPDX-FileCopyrightText: 2026 Damián Búho <damian.buho@proton.me>
2
+ //
3
+ // SPDX-License-Identifier: AGPL-3.0-only
4
+ import { createPublicKey, X509Certificate } from "node:crypto";
5
+ import { isIP, Socket } from "node:net";
6
+ import { connect as tlsConnect } from "node:tls";
7
+ import { Parser } from "htmlparser2";
8
+ import { getDomain } from "tldts";
9
+ import { reason } from "../crawl/fetch.js";
10
+ import { log } from "../logger.js";
11
+ import { pin } from "../rules/score.js";
12
+ import { MAIL_RULES, records, texts, warnOnce, zoneOf } from "./dns.js";
13
+ import { definePlugin } from "./types.js";
14
+ // Selectors common enough to probe, since DNS cannot list them.
15
+ const SELECTORS = ["default", "dkim", "mail", "selector1", "selector2", "google", "k1", "s1", "s2", "fm1", "fm2", "fm3", "protonmail", "protonmail2", "protonmail3"];
16
+ // A mail provider’s MX suffix and the selectors it signs with.
17
+ const PROVIDERS = [
18
+ [/\.zoho\.(com|eu|in)\.?$/i, ["zmail", "zoho"]],
19
+ [/\.mailgun\.org\.?$/i, ["mailo", "smtp", "mg"]],
20
+ [/\.mailbox\.org\.?$/i, ["MBO0001", "MBO0002"]],
21
+ [/\.migadu\.com\.?$/i, ["key1", "key2", "key3"]],
22
+ [/\.icloud\.com\.?$/i, ["sig1"]],
23
+ [/\.yandex\.(net|ru)\.?$/i, ["mail"]],
24
+ ];
25
+ // DNS-querying SPF terms RFC 7208 §4.6.4 allows, void lookups, and the walk’s own cap.
26
+ const SPF_LOOKUPS = 10;
27
+ const SPF_WALK = 30;
28
+ // Longest MTA-STS `max_age` RFC 8461 §3.2 allows.
29
+ const MAX_AGE = 31_557_600;
30
+ const SMTP_MS = 8000;
31
+ const DMARC_POLICIES = new Set(["none", "quarantine", "reject"]);
32
+ // `tag=value` pairs split on `;`, duplicates and malformed pairs as errors; `first` must lead with `version`.
33
+ function tagList(record, first, version) {
34
+ const tags = new Map();
35
+ const errors = [];
36
+ const parts = record
37
+ .split(";")
38
+ .map((part) => part.trim())
39
+ .filter(Boolean);
40
+ for (const [index, part] of parts.entries()) {
41
+ const match = /^([a-z][a-z0-9_]*)\s*=\s*(.*)$/is.exec(part);
42
+ if (!match)
43
+ errors.push(`malformed tag “${part}”`);
44
+ else if (tags.has(match[1].toLowerCase()))
45
+ errors.push(`duplicate tag ${match[1]}`);
46
+ else
47
+ tags.set(match[1].toLowerCase(), match[2].trim());
48
+ if (index === 0 && (match?.[1]?.toLowerCase() !== first || !version.test(match[2] ?? "")))
49
+ errors.push(`does not begin with ${first}=`);
50
+ }
51
+ return { tags, errors };
52
+ }
53
+ // A domain-spec with `%{d}` and `%{o}` filled; undefined when a sender macro is left, since no sender is known.
54
+ function expand(spec, domain) {
55
+ const filled = spec
56
+ .replaceAll(/%\{[do]r?\}/gi, () => domain)
57
+ .replaceAll("%%", "%")
58
+ .replaceAll("%_", " ")
59
+ .replaceAll("%-", "%20");
60
+ return filled.includes("%") ? undefined : filled.replace(/\.$/, "");
61
+ }
62
+ // Whether a reply is a void lookup: NXDOMAIN, or NOERROR with no answer (RFC 7208 §4.6.4).
63
+ function isVoid(reply) {
64
+ return reply.rcode === "NXDOMAIN" || (reply.rcode === "NOERROR" && reply.answers.length === 0);
65
+ }
66
+ // One SPF term split into qualifier, name and argument, with its grammar errors (RFC 7208 §12).
67
+ function spfTerm(text, errors) {
68
+ const modifier = /^([a-z][a-z0-9_.-]*)=(.*)$/i.exec(text);
69
+ if (modifier) {
70
+ const name = modifier[1].toLowerCase();
71
+ const value = modifier[2];
72
+ if (!value && ["redirect", "exp"].includes(name))
73
+ errors.push(`${name}= names no domain`);
74
+ if (/%(?![{%_-])/.test(value))
75
+ errors.push(`bad macro in ${text}`);
76
+ return { text, qualifier: "", name, value, modifier: true };
77
+ }
78
+ const match = /^([+\-~?]?)([a-z0-9]+)(.*)$/i.exec(text);
79
+ const qualifier = match?.[1] ?? "";
80
+ const name = (match?.[2] ?? text).toLowerCase();
81
+ const rest = match?.[3] ?? "";
82
+ const isPrefix = (value, most) => value === undefined || (/^\d{1,3}$/.test(value) && Number(value) <= most);
83
+ const ip = (family) => {
84
+ const [, address = "", prefix] = /^:([^/]+)(?:\/(.*))?$/.exec(rest) ?? [];
85
+ return isIP(address) === family && isPrefix(prefix, family === 4 ? 32 : 128);
86
+ };
87
+ const target = /^:([^/\s]+)$/.exec(rest)?.[1];
88
+ const validators = {
89
+ all: () => rest === "",
90
+ include: () => target !== undefined,
91
+ exists: () => target !== undefined,
92
+ ptr: () => rest === "" || target !== undefined,
93
+ a: () => /^(?::[^/\s]+)?(?:\/(\d{1,2}))?(?:\/\/(\d{1,3}))?$/.test(rest) && isPrefix(/\/(\d+)(?:\/\/|$)/.exec(rest)?.[1], 32) && isPrefix(/\/\/(\d+)$/.exec(rest)?.[1], 128),
94
+ ip4: () => ip(4),
95
+ ip6: () => ip(6),
96
+ };
97
+ validators.mx = validators.a;
98
+ const validate = validators[name];
99
+ if (validate === undefined)
100
+ errors.push(`unknown mechanism ${text}`);
101
+ else if (!validate())
102
+ errors.push(`malformed ${text}`);
103
+ const value = rest.startsWith(":") ? rest.slice(1).split("/", 1)[0] : undefined;
104
+ return { text, qualifier, name, ...(value && { value }), modifier: false };
105
+ }
106
+ // The terms of one `v=spf1` record, with the grammar errors found on the way.
107
+ function spfTerms(record) {
108
+ const errors = [];
109
+ const [version, ...rest] = record.trim().split(/ +/);
110
+ if (version?.toLowerCase() !== "v=spf1")
111
+ errors.push("does not begin with v=spf1");
112
+ const terms = rest.map((text) => spfTerm(text, errors));
113
+ for (const name of ["redirect", "exp"])
114
+ if (terms.filter((term) => term.modifier && term.name === name).length > 1)
115
+ errors.push(`${name}= appears twice`);
116
+ return { terms, errors };
117
+ }
118
+ // The SPF record of `domain`, when exactly one is published.
119
+ async function spfOf(domain, dns, walk) {
120
+ const reply = await dns.query(domain, "TXT");
121
+ const found = texts(reply).filter((entry) => /^v=spf1(\s|$)/i.test(entry));
122
+ if (isVoid(reply))
123
+ walk.voids += 1;
124
+ if (found.length !== 1)
125
+ walk.missing.push(domain);
126
+ return found.length === 1 ? found[0] : undefined;
127
+ }
128
+ // Counts the DNS-querying terms up to `all` of `record` and of every record it includes or redirects to, within the walk’s cap.
129
+ async function spfWalk(domain, record, dns, walk) {
130
+ walk.seen.add(domain.toLowerCase());
131
+ const { terms } = spfTerms(record);
132
+ const hasAll = terms.some((term) => term.name === "all");
133
+ for (const term of terms) {
134
+ if (walk.lookups >= SPF_WALK)
135
+ return log.debug({ domain, lookups: walk.lookups }, "spf walk capped");
136
+ if (term.name === "all" && !term.modifier)
137
+ return log.debug({ domain, lookups: walk.lookups }, "spf walk reached all");
138
+ const isQuerying = term.modifier ? term.name === "redirect" && !hasAll : ["include", "a", "mx", "ptr", "exists"].includes(term.name);
139
+ if (!isQuerying)
140
+ continue;
141
+ walk.lookups += 1;
142
+ if (term.name === "ptr")
143
+ walk.ptr = true;
144
+ const target = expand(term.value ?? domain, domain);
145
+ log.debug({ domain, term: term.text, target, lookups: walk.lookups }, "spf term counted");
146
+ if (!target || term.name === "ptr")
147
+ continue;
148
+ if (term.name === "include" || term.name === "redirect") {
149
+ if (walk.seen.has(target.toLowerCase()))
150
+ continue;
151
+ const next = await spfOf(target, dns, walk);
152
+ if (next)
153
+ await spfWalk(target, next, dns, walk);
154
+ }
155
+ else if (isVoid(await dns.query(target, term.name === "mx" ? "MX" : "A")))
156
+ walk.voids += 1;
157
+ }
158
+ }
159
+ // The one SPF record’s grammar, lookups, void lookups, `ptr`, terms after `all`, repeated terms and missing includes.
160
+ async function spfCheck(domain, record, dns) {
161
+ const { terms, errors } = spfTerms(record);
162
+ const walk = { lookups: 0, voids: 0, ptr: false, missing: [], seen: new Set() };
163
+ await spfWalk(domain, record, dns, walk);
164
+ const mechanisms = terms.filter((term) => !term.modifier);
165
+ const allAt = mechanisms.findIndex((term) => term.name === "all");
166
+ const keys = mechanisms.map((term) => term.text.toLowerCase().replace(/^\+/, ""));
167
+ const repeated = keys.filter((key, index) => keys.indexOf(key) !== index);
168
+ const ignored = allAt === -1 ? [] : terms.filter((term) => term.modifier && term.name === "redirect").map((term) => term.text);
169
+ const result = { errors, lookups: walk.lookups, "void-lookups": walk.voids, ptr: walk.ptr, "after-all": allAt === -1 ? [] : mechanisms.slice(allAt + 1).map((term) => term.text), redundant: [...new Set([...repeated, ...ignored])], "missing-includes": walk.missing };
170
+ log.debug({ domain, ...result }, "spf walked");
171
+ return result;
172
+ }
173
+ // The DMARC records at `_dmarc.<name>`.
174
+ async function dmarcRecords(name, dns) {
175
+ return texts(await dns.query(`_dmarc.${name}`, "TXT")).filter((entry) => /^v\s*=\s*DMARC1\s*(;|$)/i.test(entry));
176
+ }
177
+ // The DMARC record at the host, else at its organisational domain, with the policy that applies to the host (RFC 9989 §4.10).
178
+ export async function dmarc(host, dns) {
179
+ const names = new Set([host, getDomain(host, { allowPrivateDomains: true }) ?? host]);
180
+ for (const name of names) {
181
+ const found = await dmarcRecords(name, dns);
182
+ log.debug({ host, name, found: found.length }, "dmarc looked up");
183
+ const record = found[0];
184
+ if (!record)
185
+ continue;
186
+ const tags = new Map(record.split(";").map((tag) => tag.split("=", 2).map((part) => part.trim().toLowerCase())));
187
+ const policy = name === host ? tags.get("p") : (tags.get("sp") ?? tags.get("p"));
188
+ return { at: name, record, ...(policy && { policy }), count: found.length };
189
+ }
190
+ }
191
+ // The grammar of a DMARC record by RFC 9989 §4.7, `pct` read as the legacy tag older receivers still honour.
192
+ function dmarcTags(record) {
193
+ const parsed = tagList(record, "v", /^DMARC1$/);
194
+ const check = (tag, isValid) => {
195
+ const value = parsed.tags.get(tag);
196
+ if (value !== undefined && !isValid(value.toLowerCase()))
197
+ parsed.errors.push(`bad ${tag}=${value}`);
198
+ };
199
+ for (const tag of ["p", "sp", "np"])
200
+ check(tag, (value) => DMARC_POLICIES.has(value));
201
+ for (const tag of ["adkim", "aspf"])
202
+ check(tag, (value) => ["r", "s"].includes(value));
203
+ check("t", (value) => ["y", "n"].includes(value));
204
+ check("psd", (value) => ["y", "n", "u"].includes(value));
205
+ check("fo", (value) => /^[01ds](:[01ds])*$/.test(value));
206
+ check("pct", (value) => /^\d{1,3}$/.test(value) && Number(value) <= 100);
207
+ for (const tag of ["rua", "ruf"])
208
+ check(tag, (value) => value.split(",").every((uri) => URL.canParse(uri.trim().replace(/!\d+[kmgt]?$/, ""))));
209
+ return parsed;
210
+ }
211
+ // The report URIs of one tag, without a size limit suffix.
212
+ function reportUris(tags) {
213
+ return ["rua", "ruf"].flatMap((tag) => (tags.get(tag) ?? "")
214
+ .split(",")
215
+ .map((uri) => uri.trim().replace(/!\d+[kmgt]?$/i, ""))
216
+ .filter(Boolean));
217
+ }
218
+ // A DMARC record’s grammar, testing state, subdomain policy, non-mailto report URIs and external destinations that do not accept its reports (RFC 9990 §4).
219
+ async function dmarcCheck(subject, found, pages, dns) {
220
+ const { tags, errors } = dmarcTags(found.record);
221
+ const uris = reportUris(tags);
222
+ const notMailto = uris.filter((uri) => !/^mailto:[^@\s]+@[^@\s]+$/i.test(uri));
223
+ const organisation = getDomain(found.at, { allowPrivateDomains: true }) ?? found.at;
224
+ const destinations = [...new Set(uris.flatMap((uri) => (/^mailto:[^@]+@(.+)$/i.exec(uri)?.[1] ?? "").toLowerCase().split(/[?]/, 1)).filter((host) => host && getDomain(host, { allowPrivateDomains: true }) !== organisation))];
225
+ const unauthorized = [];
226
+ for (const destination of destinations) {
227
+ const isAuthorized = texts(await dns.query(`${found.at}._report._dmarc.${destination}`, "TXT")).some((entry) => /^v\s*=\s*DMARC1\s*(;|$)/i.test(entry));
228
+ log.debug({ subject, at: found.at, destination, isAuthorized }, "external dmarc destination checked");
229
+ if (!isAuthorized)
230
+ unauthorized.push(destination);
231
+ }
232
+ const subdomains = found.at === subject && pages.some((page) => new URL(page.url.href).hostname.endsWith(`.${subject}`));
233
+ const pct = tags.get("pct");
234
+ return { records: found.count, errors, testing: tags.get("t")?.toLowerCase() === "y" || (pct !== undefined && pct !== "100"), ...(found.at === subject && { subdomains }), ...(tags.has("sp") && { sp: tags.get("sp")?.toLowerCase() }), "not-mailto": notMailto, unauthorized };
235
+ }
236
+ // The grammar, key type, key size, testing flag and revocation of one DKIM key record (RFC 6376 §3.6.1).
237
+ function dkimKey(selector, record) {
238
+ const { tags, errors } = tagList(record.startsWith("v=") ? record : `v=DKIM1; ${record}`, "v", /^DKIM1$/);
239
+ const k = (tags.get("k") ?? "rsa").toLowerCase();
240
+ const p = (tags.get("p") ?? "").replaceAll(/\s/g, "");
241
+ if (!tags.has("p"))
242
+ errors.push("no p= tag");
243
+ if (!["rsa", "ed25519"].includes(k))
244
+ errors.push(`unknown k=${k}`);
245
+ let bits;
246
+ if (p && k === "rsa") {
247
+ try {
248
+ bits = createPublicKey({ key: Buffer.from(p, "base64"), format: "der", type: "spki" }).asymmetricKeyDetails?.modulusLength;
249
+ }
250
+ catch (error) {
251
+ errors.push(`p= is not an RSA public key: ${reason(error)}`);
252
+ }
253
+ }
254
+ const flags = (tags.get("t") ?? "")
255
+ .toLowerCase()
256
+ .split(":")
257
+ .map((flag) => flag.trim());
258
+ return { selector, errors, k, ...(bits !== undefined && { bits }), testing: flags.includes("y"), revoked: tags.has("p") && p === "" };
259
+ }
260
+ // Each probed selector that publishes a key, from the pinned list, the MX provider’s and the configured ones.
261
+ async function dkimCheck(domain, exchanges, extra, dns) {
262
+ const provided = PROVIDERS.filter(([pattern]) => exchanges.some((exchange) => pattern.test(exchange))).flatMap(([, selectors]) => selectors);
263
+ const selectors = [...new Set([...SELECTORS, ...provided, ...extra])];
264
+ const found = await Promise.all(selectors.map(async (selector) => [selector, texts(await dns.query(`${selector}._domainkey.${domain}`, "TXT")).join("")]));
265
+ const keys = found.filter(([, record]) => record).map(([selector, record]) => dkimKey(selector, record));
266
+ log.debug({ domain, probed: selectors.length, provided, found: keys.map((key) => key.selector) }, "dkim selectors probed");
267
+ return { probed: selectors.length, found: keys };
268
+ }
269
+ // Each exchange: its CNAME target, whether it is an address literal, and how many addresses it resolves to; and repeated exchanges.
270
+ async function exchangeCheck(exchanges, dns) {
271
+ const names = exchanges.map((exchange) => exchange.toLowerCase().replace(/\.$/, ""));
272
+ const checked = await Promise.all([...new Set(names)].map(async (exchange) => {
273
+ const isLiteral = isIP(exchange.replaceAll(/^\[|\]$/g, "")) !== 0;
274
+ if (isLiteral)
275
+ return { exchange, cname: false, literal: true };
276
+ const [a, aaaa] = await Promise.all([dns.query(exchange, "A"), dns.query(exchange, "AAAA")]);
277
+ const cname = records(a, "CNAME").find((record) => record.name.toLowerCase().replace(/\.$/, "") === exchange)?.data ?? false;
278
+ return { exchange, cname, literal: false, addresses: records(a, "A").length + records(aaaa, "AAAA").length };
279
+ }));
280
+ const duplicates = [...new Set(names.filter((name, index) => names.indexOf(name) !== index))];
281
+ log.debug({ exchanges: checked, duplicates }, "mail exchanges checked");
282
+ return { exchanges: checked, duplicates };
283
+ }
284
+ // Whether an MTA-STS `mx` pattern covers an exchange: exact, or `*.` for one leftmost label (RFC 8461 §4.1).
285
+ function isCovered(pattern, exchange) {
286
+ const [want, name] = [pattern.toLowerCase().replace(/\.$/, ""), exchange.toLowerCase().replace(/\.$/, "")];
287
+ return want.startsWith("*.") ? name.split(".").slice(1).join(".") === want.slice(2) && name.split(".").length === want.split(".").length : want === name;
288
+ }
289
+ // The `_mta-sts` record and, when it parses, the policy file at `mta-sts.<domain>` with the exchanges it does not cover (RFC 8461 §3).
290
+ async function mtaStsCheck(domain, exchanges, context) {
291
+ const found = texts(await context.dns.query(`_mta-sts.${domain}`, "TXT")).filter((entry) => /^v\s*=\s*STSv1\s*(;|$)/i.test(entry));
292
+ if (found.length === 0)
293
+ return {};
294
+ const { tags, errors } = tagList(found[0], "v", /^STSv1$/);
295
+ if (found.length > 1)
296
+ errors.push(`${found.length} records`);
297
+ if (!/^[A-Za-z0-9]{1,32}$/.test(tags.get("id") ?? ""))
298
+ errors.push("id= is not 1 to 32 letters and digits");
299
+ if (errors.length > 0)
300
+ return { record: found[0], errors };
301
+ const url = `https://mta-sts.${domain}/.well-known/mta-sts.txt`;
302
+ const policy = { url, errors: [] };
303
+ try {
304
+ const answer = await context.delegated(url, { method: "GET", redirect: "manual" });
305
+ const type = String([answer.headers["content-type"] ?? ""].flat()[0]);
306
+ policy.status = answer.status;
307
+ if (answer.status !== 200)
308
+ policy.errors.push(`answers ${answer.status}`);
309
+ if (!/^text\/plain\b/i.test(type))
310
+ policy.errors.push(`served as ${type || "no type"}`);
311
+ const fields = answer.body
312
+ .split(/\r?\n/)
313
+ .map((line) => /^([a-z_]+):\s*(.*?)\s*$/i.exec(line))
314
+ .filter((match) => match !== null);
315
+ const one = (key) => fields.find((match) => match[1] === key)?.[2];
316
+ const patterns = fields.filter((match) => match[1] === "mx").map((match) => match[2]);
317
+ const [version, mode, maxAge] = [one("version"), one("mode"), one("max_age")];
318
+ if (version !== "STSv1")
319
+ policy.errors.push(`version is ${version ?? "missing"}`);
320
+ if (!["enforce", "testing", "none"].includes(mode ?? ""))
321
+ policy.errors.push(`mode is ${mode ?? "missing"}`);
322
+ if (!/^\d{1,10}$/.test(maxAge ?? "") || Number(maxAge) > MAX_AGE)
323
+ policy.errors.push(`max_age is ${maxAge ?? "missing"}`);
324
+ if (mode !== "none" && patterns.length === 0)
325
+ policy.errors.push("no mx line");
326
+ Object.assign(policy, { mode, mx: patterns, "max-age": Number(maxAge), unmatched: mode === "none" ? [] : exchanges.filter((exchange) => patterns.every((pattern) => !isCovered(pattern, exchange))) });
327
+ }
328
+ catch (error) {
329
+ policy.errors.push(reason(error));
330
+ }
331
+ log.debug({ domain, url, errors: policy.errors, mode: policy.mode, unmatched: policy.unmatched }, "mta-sts policy read");
332
+ return { record: found[0], errors, policy };
333
+ }
334
+ // The `_smtp._tls` reporting record and its grammar (RFC 8460 §3).
335
+ async function tlsRptCheck(domain, dns) {
336
+ const found = texts(await dns.query(`_smtp._tls.${domain}`, "TXT")).filter((entry) => /^v\s*=\s*TLSRPTv1\s*(;|$)/i.test(entry));
337
+ if (found.length === 0)
338
+ return {};
339
+ const { tags, errors } = tagList(found[0], "v", /^TLSRPTv1$/);
340
+ if (found.length > 1)
341
+ errors.push(`${found.length} records`);
342
+ const uris = (tags.get("rua") ?? "")
343
+ .split(",")
344
+ .map((uri) => uri.trim())
345
+ .filter(Boolean);
346
+ if (uris.length === 0)
347
+ errors.push("no rua=");
348
+ for (const uri of uris)
349
+ if (!/^(mailto:[^@\s]+@[^@\s]+|https:\/\/\S+)$/i.test(uri))
350
+ errors.push(`rua ${uri} is neither mailto: nor https:`);
351
+ log.debug({ domain, errors }, "tls-rpt read");
352
+ return { record: found[0], errors };
353
+ }
354
+ // SVG Tiny PS profile errors of a BIMI logo: its root, a title, no script and no external reference.
355
+ function svgErrors(body) {
356
+ const errors = [];
357
+ let root;
358
+ let hasTitle = false;
359
+ const parser = new Parser({
360
+ onopentag(name, attributes) {
361
+ root ??= { name, attributes };
362
+ if (name === "title")
363
+ hasTitle = true;
364
+ else if (name === "script" || name === "foreignobject")
365
+ errors.push(`<${name}> is not allowed`);
366
+ for (const [key, value] of Object.entries(attributes)) {
367
+ if (/^on/i.test(key))
368
+ errors.push(`${key} is an event handler`);
369
+ else if (/^(xlink:)?href$/i.test(key) && !value.startsWith("#"))
370
+ errors.push(`${key} ${value} is an external reference`);
371
+ }
372
+ },
373
+ }, { xmlMode: true });
374
+ parser.end(body);
375
+ if (root?.name !== "svg")
376
+ return ["not an SVG document"];
377
+ const { version, baseProfile } = root.attributes;
378
+ if (version !== "1.2")
379
+ errors.push(`version is ${version ?? "missing"}, not 1.2`);
380
+ if (baseProfile?.toLowerCase() !== "tiny-ps")
381
+ errors.push(`baseProfile is ${baseProfile ?? "missing"}, not tiny-ps`);
382
+ for (const key of ["x", "y"])
383
+ if (Object.hasOwn(root.attributes, key))
384
+ errors.push(`the root carries ${key}=`);
385
+ if (!hasTitle)
386
+ errors.push("no <title>");
387
+ return errors;
388
+ }
389
+ // One GET through `delegated`, its status and body, or the error it failed with.
390
+ async function fetched(url, context) {
391
+ try {
392
+ const answer = await context.delegated(url, { method: "GET", redirect: "follow" });
393
+ return { status: answer.status, body: answer.body };
394
+ }
395
+ catch (error) {
396
+ return { error: reason(error) };
397
+ }
398
+ }
399
+ // The `default._bimi` record, its grammar, whether DMARC enforces, the logo’s SVG profile and whether the VMC parses.
400
+ async function bimiCheck(domain, isEnforced, context) {
401
+ const found = texts(await context.dns.query(`default._bimi.${domain}`, "TXT")).filter((entry) => /^v\s*=\s*BIMI1\s*(;|$)/i.test(entry));
402
+ if (found.length === 0)
403
+ return;
404
+ const { tags, errors } = tagList(found[0], "v", /^BIMI1$/);
405
+ const [logo, vmc] = [tags.get("l") ?? "", tags.get("a") ?? ""];
406
+ if (logo && !/^https:\/\/\S+$/i.test(logo))
407
+ errors.push(`l= ${logo} is not an https URL`);
408
+ if (vmc && !/^https:\/\/\S+$/i.test(vmc))
409
+ errors.push(`a= ${vmc} is not an https URL`);
410
+ const result = { record: found[0], errors, "dmarc-enforced": isEnforced };
411
+ if (logo && errors.length === 0) {
412
+ const answer = await fetched(logo, context);
413
+ result.logo = { url: logo, errors: answer.error ? [answer.error] : answer.status === 200 ? svgErrors(answer.body ?? "") : [`answers ${answer.status}`] };
414
+ }
415
+ if (vmc && errors.length === 0) {
416
+ const answer = await fetched(vmc, context);
417
+ const pem = /-----BEGIN CERTIFICATE-----[\s\S]+?-----END CERTIFICATE-----/.exec(answer.body ?? "")?.[0];
418
+ let isParsed = false;
419
+ try {
420
+ isParsed = pem !== undefined && new X509Certificate(pem).subject !== "";
421
+ }
422
+ catch (error) {
423
+ log.debug({ domain, vmc, error: reason(error) }, "vmc does not parse");
424
+ }
425
+ result.vmc = { url: vmc, status: answer.status, parses: isParsed };
426
+ }
427
+ log.debug({ domain, errors, isEnforced, logo: result.logo, vmc: result.vmc }, "bimi read");
428
+ return result;
429
+ }
430
+ // Each exchange’s `_25._tcp` TLSA records and whether the resolver validated them (RFC 7672 §2.2).
431
+ async function daneCheck(exchanges, dns) {
432
+ return Promise.all(exchanges.map(async (exchange) => {
433
+ const reply = await dns.query(`_25._tcp.${exchange}`, "TLSA");
434
+ const result = { exchange, tlsa: records(reply, "TLSA").length, signed: reply.ad };
435
+ log.debug(result, "dane read");
436
+ return result;
437
+ }));
438
+ }
439
+ // The reverse DNS name of an address.
440
+ function reverseName(address) {
441
+ if (isIP(address) === 4)
442
+ return `${address.split(".").toReversed().join(".")}.in-addr.arpa`;
443
+ const [head = "", tail = ""] = address.split("::", 2);
444
+ const [left, right] = [head ? head.split(":") : [], tail ? tail.split(":") : []];
445
+ const groups = address.includes("::") ? [...left, ...Array.from({ length: 8 - left.length - right.length }, () => "0"), ...right] : left;
446
+ return `${groups
447
+ .map((group) => group.padStart(4, "0"))
448
+ .join("")
449
+ .split("")
450
+ .toReversed()
451
+ .join(".")}.ip6.arpa`;
452
+ }
453
+ // The SMTP reply lines up to the last of one reply, which has a space after its code.
454
+ function smtpReply(socket) {
455
+ return new Promise((resolve, reject) => {
456
+ let buffer = "";
457
+ const onData = (chunk) => {
458
+ buffer += chunk.toString("latin1");
459
+ const lines = buffer.split("\r\n").filter(Boolean);
460
+ if (!/^\d{3} /m.test(buffer) || !buffer.endsWith("\r\n"))
461
+ return;
462
+ socket.off("data", onData);
463
+ resolve(lines);
464
+ };
465
+ socket.on("data", onData);
466
+ socket.once("error", reject);
467
+ socket.once("close", () => reject(new Error("connection closed")));
468
+ });
469
+ }
470
+ // The banner, STARTTLS offer, negotiated protocol and certificate validity of one exchange on port 25.
471
+ async function starttls(exchange, address, signal) {
472
+ const socket = new Socket();
473
+ const abort = AbortSignal.any([signal, AbortSignal.timeout(SMTP_MS)]);
474
+ const stop = () => socket.destroy(new Error(`port 25 of ${address} gave no answer in time`));
475
+ abort.addEventListener("abort", stop, { once: true });
476
+ try {
477
+ await new Promise((resolve, reject) => socket.once("error", reject).connect(25, address, resolve));
478
+ const [banner = ""] = await smtpReply(socket);
479
+ socket.write("EHLO spiderlint.invalid\r\n");
480
+ const extensions = await smtpReply(socket);
481
+ const offered = extensions.some((line) => /^250[- ]STARTTLS\b/i.test(line));
482
+ if (!offered)
483
+ return { banner, offered };
484
+ socket.write("STARTTLS\r\n");
485
+ const ready = await smtpReply(socket);
486
+ if (!ready[0]?.startsWith("220"))
487
+ return { banner, offered };
488
+ const secure = await new Promise((resolve, reject) => {
489
+ const tls = tlsConnect({ socket, servername: exchange, rejectUnauthorized: false }, () => {
490
+ resolve({ protocol: tls.getProtocol() ?? undefined, authorized: tls.authorized });
491
+ tls.end("QUIT\r\n");
492
+ });
493
+ tls.once("error", reject);
494
+ });
495
+ return { banner, offered, ...secure };
496
+ }
497
+ finally {
498
+ abort.removeEventListener("abort", stop);
499
+ socket.destroy();
500
+ }
501
+ }
502
+ // Each exchange’s port 25 answer and the PTR names of its address; one warning per run when the port does not answer.
503
+ async function starttlsCheck(exchanges, context) {
504
+ const results = [];
505
+ for (const exchange of exchanges) {
506
+ try {
507
+ const address = await context.address(exchange);
508
+ const ptr = records(await context.dns.query(reverseName(address), "PTR"), "PTR").map(({ data }) => data.toLowerCase().replace(/\.$/, ""));
509
+ const answer = await starttls(exchange, address, context.signal);
510
+ const name = (/^220[ -](\S+)/.exec(answer.banner)?.[1] ?? "").toLowerCase().replace(/\.$/, "");
511
+ const result = { exchange, address, ...answer, ptr, "banner-matches": ptr.includes(name) };
512
+ log.debug(result, "starttls probed");
513
+ results.push(result);
514
+ }
515
+ catch (error) {
516
+ warnOnce(context.dns, "SMTP port 25", { exchange, error: reason(error) });
517
+ }
518
+ }
519
+ return results;
520
+ }
521
+ // Whether an SPF record lets some host send: a mechanism with no `-`, `~` or `?` qualifier, or a `redirect=`.
522
+ const isAuthorizing = (record) => record
523
+ .split(/\s+/)
524
+ .slice(1)
525
+ .some((term) => /^\+?(all$|a\b|mx\b|ptr\b|ip4:|ip6:|include:|exists:)|^redirect=/i.test(term));
526
+ // MX, SPF and DMARC of one host or registrable domain, the `intent` its MX and SPF declare and the `mode` it is judged in; a mail name adds every authentication record.
527
+ const mail = {
528
+ id: "mail",
529
+ per: "host",
530
+ domains: true,
531
+ cached: false,
532
+ resolves: true,
533
+ timeout: 120_000,
534
+ async extract(host, context) {
535
+ const settings = (context.settings ?? {});
536
+ const zone = await zoneOf(host, context.dns);
537
+ if (!zone)
538
+ return;
539
+ const [mx, txt, policy] = await Promise.all([context.dns.query(host, "MX"), context.dns.query(host, "TXT"), dmarc(host, context.dns)]);
540
+ const exchanges = records(mx, "MX").map(({ data }) => ({ preference: data.preference ?? 0, exchange: data.exchange }));
541
+ const spf = texts(txt).filter((entry) => /^v=spf1(\s|$)/i.test(entry));
542
+ const receives = exchanges.some(({ exchange }) => exchange !== ".");
543
+ const sends = spf.some((record) => isAuthorizing(record));
544
+ const intent = sends ? (receives ? "both" : "sends") : receives ? "receives" : "none";
545
+ const mode = settings.mode && settings.mode !== "auto" ? settings.mode : intent === "none" ? "none" : "mail";
546
+ log.debug({ host, zone, mx: exchanges.length, spf: spf.length, dmarc: policy?.policy, intent, mode, configured: settings.mode ?? "auto" }, "mail mode decided");
547
+ const { count: _count, ...found } = policy ?? { count: 0 };
548
+ const base = { mx: exchanges, spf, ...(policy && { dmarc: found }), intent, mode };
549
+ if (mode === "none")
550
+ return base;
551
+ const real = exchanges.map(({ exchange }) => exchange).filter((exchange) => exchange !== ".");
552
+ const checkedDmarc = policy && (await dmarcCheck(host, policy, context.pages, context.dns));
553
+ const dmarcFacts = checkedDmarc && { ...found, ...checkedDmarc };
554
+ const isEnforced = ["quarantine", "reject"].includes(policy?.policy ?? "") && checkedDmarc?.testing === false;
555
+ const [walk, dkim, checked, mtaSts, tlsRpt, bimi, dane, smtp] = await Promise.all([
556
+ spf.length === 1 ? spfCheck(host, spf[0], context.dns) : undefined,
557
+ dkimCheck(host, real, settings["dkim-selectors"] ?? [], context.dns),
558
+ exchangeCheck(real, context.dns),
559
+ mtaStsCheck(host, real, context),
560
+ tlsRptCheck(host, context.dns),
561
+ bimiCheck(host, isEnforced, context),
562
+ settings.dane ? daneCheck(real, context.dns) : undefined,
563
+ settings.starttls ? starttlsCheck(real, context) : undefined,
564
+ ]);
565
+ return { ...base, ...(dmarcFacts && { dmarc: dmarcFacts }), ...(walk && { "spf-walk": walk }), dkim, ...checked, "mta-sts": mtaSts, "tls-rpt": tlsRpt, ...(bimi && { bimi }), ...(dane && { dane }), ...(smtp && { starttls: smtp }) };
566
+ },
567
+ };
568
+ // Guards a rule to names judged as mail names, and to facts present only for some.
569
+ const MAIL = { "site.hosts.*.mail.mode": "mail" };
570
+ const present = (path, type = "object") => ({ ...MAIL, [`site.hosts.*.mail.${path}`]: { type } });
571
+ const clean = (path) => ({ ...present(path), [`site.hosts.*.mail.${path}.errors`]: { maxItems: 0 } });
572
+ const RULES = {
573
+ "mail/spf-syntax": {
574
+ fact: "site.hosts.*.mail.spf-walk.errors",
575
+ when: present("spf-walk"),
576
+ expect: { maxItems: 0 },
577
+ message: "the SPF record breaks the RFC 7208 grammar, so receivers treat it as a permanent error and ignore it (got {got})",
578
+ severity: "warning",
579
+ score: 6.2,
580
+ docs: "https://www.rfc-editor.org/rfc/rfc7208#section-12",
581
+ fix: "Rewrite the TXT record at Name `{host}` as `v=spf1`, then mechanisms such as `mx ip4:192.0.2.1 include:_spf.example.net`, then `-all`, separated by single spaces.",
582
+ },
583
+ "mail/spf-lookups": {
584
+ fact: "site.hosts.*.mail.spf-walk.lookups",
585
+ when: clean("spf-walk"),
586
+ expect: { maximum: SPF_LOOKUPS },
587
+ message: "SPF needs {got} DNS lookups counting its includes, over the 10 RFC 7208 allows, so receivers fail it",
588
+ severity: "warning",
589
+ score: 5.8,
590
+ docs: "https://www.rfc-editor.org/rfc/rfc7208#section-4.6.4",
591
+ fix: "Cut the `include`, `a`, `mx`, `ptr` and `exists` terms of the SPF record at Name `{host}` to at most 10 lookups, replacing includes with the `ip4:` and `ip6:` ranges they name.",
592
+ },
593
+ "mail/spf-void-lookups": {
594
+ fact: "site.hosts.*.mail.spf-walk.void-lookups",
595
+ when: clean("spf-walk"),
596
+ expect: { maximum: 2 },
597
+ message: "SPF makes {got} lookups that find nothing, over the 2 RFC 7208 allows, so receivers fail it",
598
+ severity: "warning",
599
+ score: 5.2,
600
+ docs: "https://www.rfc-editor.org/rfc/rfc7208#section-4.6.4",
601
+ fix: "Delete the terms of the SPF record at Name `{host}` that name domains with no record, such as an `include:` of a provider you left.",
602
+ },
603
+ "mail/spf-ptr": {
604
+ fact: "site.hosts.*.mail.spf-walk.ptr",
605
+ when: clean("spf-walk"),
606
+ expect: { const: false },
607
+ message: "SPF uses the ptr mechanism, which RFC 7208 says not to use, since it is slow and unreliable",
608
+ severity: "warning",
609
+ score: 4.9,
610
+ docs: "https://www.rfc-editor.org/rfc/rfc7208#section-5.5",
611
+ fix: "Replace `ptr` in the SPF record at Name `{host}` with the `ip4:` and `ip6:` ranges of the servers it matched.",
612
+ },
613
+ "mail/spf-after-all": {
614
+ fact: "site.hosts.*.mail.spf-walk.after-all",
615
+ when: clean("spf-walk"),
616
+ expect: { maxItems: 0 },
617
+ message: "SPF lists mechanisms after all, which receivers never reach (got {got})",
618
+ severity: "warning",
619
+ score: 5.1,
620
+ docs: "https://www.rfc-editor.org/rfc/rfc7208#section-5.1",
621
+ fix: "Move every mechanism of the SPF record at Name `{host}` before its `all`, such as `v=spf1 mx include:_spf.example.net -all`.",
622
+ },
623
+ "mail/spf-redundant": {
624
+ fact: "site.hosts.*.mail.spf-walk.redundant",
625
+ when: clean("spf-walk"),
626
+ expect: { maxItems: 0 },
627
+ message: "SPF repeats a mechanism, or carries a redirect that all makes unreachable (got {got})",
628
+ severity: "info",
629
+ score: 1.4,
630
+ docs: "https://www.rfc-editor.org/rfc/rfc7208#section-6.1",
631
+ fix: "Delete the repeated terms from the SPF record at Name `{host}`, and drop `redirect=` where the record ends in `all`.",
632
+ },
633
+ "mail/spf-include": {
634
+ fact: "site.hosts.*.mail.spf-walk.missing-includes",
635
+ when: clean("spf-walk"),
636
+ expect: { maxItems: 0 },
637
+ message: "SPF includes or redirects to a name with no single SPF record, a permanent error to receivers (got {got})",
638
+ severity: "warning",
639
+ score: 5,
640
+ docs: "https://www.rfc-editor.org/rfc/rfc7208#section-5.2",
641
+ fix: "Delete the `include:` or `redirect=` of the SPF record at Name `{host}` that names no SPF record, or correct its spelling.",
642
+ },
643
+ "mail/dmarc-record": {
644
+ fact: "site.hosts.*.mail.dmarc.records",
645
+ when: present("dmarc"),
646
+ expect: { maximum: 1 },
647
+ message: "the name publishes {got} DMARC records, so receivers discard all of them",
648
+ severity: "warning",
649
+ score: 6.3,
650
+ docs: "https://www.rfc-editor.org/rfc/rfc9989#section-4.10",
651
+ fix: "Keep one TXT record with Content starting `v=DMARC1` at Name `_dmarc.{domain}`, merging the others into it.",
652
+ },
653
+ "mail/dmarc-syntax": {
654
+ fact: "site.hosts.*.mail.dmarc.errors",
655
+ when: present("dmarc"),
656
+ expect: { maxItems: 0 },
657
+ message: "the DMARC record breaks the RFC 9989 grammar (got {got})",
658
+ severity: "warning",
659
+ score: 6.1,
660
+ docs: "https://www.rfc-editor.org/rfc/rfc9989#section-4.7",
661
+ fix: "Rewrite the TXT record at Name `_dmarc.{domain}` as tag=value pairs after `v=DMARC1`, such as `v=DMARC1; p=reject; rua=mailto:dmarc@{domain}`.",
662
+ },
663
+ "mail/dmarc-subdomains": {
664
+ fact: "site.hosts.*.mail.dmarc",
665
+ when: { ...clean("dmarc"), "site.hosts.*.mail.dmarc.subdomains": true },
666
+ expect: { required: ["sp"] },
667
+ message: "the DMARC record sets no sp= policy although the domain serves subdomains, which then inherit p= silently",
668
+ severity: "info",
669
+ score: 2.6,
670
+ docs: "https://www.rfc-editor.org/rfc/rfc9989#section-4.7",
671
+ fix: "Add `sp=reject`, or the policy the subdomains need, to the TXT record at Name `_dmarc.{domain}`.",
672
+ },
673
+ "mail/dmarc-testing": {
674
+ fact: "site.hosts.*.mail.dmarc.testing",
675
+ when: clean("dmarc"),
676
+ expect: { const: false },
677
+ message: "the DMARC policy is in test mode, t=y or pct below 100, so receivers do not apply it to all mail",
678
+ severity: "info",
679
+ score: 2.4,
680
+ docs: "https://www.rfc-editor.org/rfc/rfc9989#section-4.7",
681
+ fix: "Remove `t=y` and `pct=` from the TXT record at Name `_dmarc.{domain}` once its reports show only your own servers.",
682
+ },
683
+ "mail/dmarc-report-uri": {
684
+ fact: "site.hosts.*.mail.dmarc.not-mailto",
685
+ when: clean("dmarc"),
686
+ expect: { maxItems: 0 },
687
+ message: "a DMARC report address is not a mailto: URI, so no report reaches it (got {got})",
688
+ severity: "warning",
689
+ score: 4.1,
690
+ docs: "https://www.rfc-editor.org/rfc/rfc9990#section-3",
691
+ fix: "Write each `rua=` and `ruf=` address in the TXT record at Name `_dmarc.{domain}` as `mailto:dmarc@{domain}`.",
692
+ },
693
+ "mail/dmarc-external": {
694
+ fact: "site.hosts.*.mail.dmarc.unauthorized",
695
+ when: clean("dmarc"),
696
+ expect: { maxItems: 0 },
697
+ message: "DMARC sends reports to a domain that does not accept them, so receivers drop them (got {got})",
698
+ severity: "warning",
699
+ score: 5.3,
700
+ docs: "https://www.rfc-editor.org/rfc/rfc9990#section-4",
701
+ fix: "Ask the report destination to publish a TXT record at Name `{domain}._report._dmarc.<destination>` with Content `v=DMARC1`, or report to an address under `{domain}`.",
702
+ },
703
+ "mail/dkim": {
704
+ fact: "site.hosts.*.mail.dkim.found",
705
+ when: present("dkim"),
706
+ expect: { minItems: 1 },
707
+ message: "no DKIM key found under any common selector; the probe list cannot be complete, so check your provider’s selector",
708
+ severity: "hint",
709
+ score: 0.7,
710
+ docs: "https://www.rfc-editor.org/rfc/rfc6376#section-3.6.2.1",
711
+ fix: "Publish the DKIM key your mail provider gives as a TXT record at Name `<selector>._domainkey.{domain}`, and list its selector under `org.spiderlint.mail.dkim-selectors`.",
712
+ },
713
+ "mail/dkim-syntax": {
714
+ fact: "site.hosts.*.mail.dkim.found",
715
+ when: present("dkim"),
716
+ expect: { items: { properties: { errors: { maxItems: 0 } } } },
717
+ message: "a DKIM key record breaks the RFC 6376 grammar, so signatures under it fail (got {got})",
718
+ severity: "warning",
719
+ score: 4,
720
+ docs: "https://www.rfc-editor.org/rfc/rfc6376#section-3.6.1",
721
+ fix: "Republish the TXT record at Name `<selector>._domainkey.{domain}` exactly as your mail provider gives it, `v=DKIM1; k=rsa; p=` and the base64 key.",
722
+ },
723
+ "mail/dkim-key": {
724
+ fact: "site.hosts.*.mail.dkim.found",
725
+ when: present("dkim"),
726
+ expect: { items: { properties: { bits: { minimum: 1024 } } } },
727
+ message: "a DKIM RSA key is shorter than the 1024 bits RFC 8301 requires, so receivers treat its signatures as absent (got {got})",
728
+ severity: "error",
729
+ score: 8.8,
730
+ docs: "https://www.rfc-editor.org/rfc/rfc8301#section-3.2",
731
+ fix: "Generate a 2048-bit DKIM key at your mail provider and publish it at a new selector under `_domainkey.{domain}`.",
732
+ },
733
+ "mail/dkim-key-size": {
734
+ fact: "site.hosts.*.mail.dkim.found",
735
+ when: present("dkim"),
736
+ expect: { items: { properties: { bits: { not: { minimum: 1024, maximum: 2047 } } } } },
737
+ message: "a DKIM RSA key is shorter than 2048 bits, which RFC 8301 recommends (got {got})",
738
+ severity: "warning",
739
+ score: 3.8,
740
+ docs: "https://www.rfc-editor.org/rfc/rfc8301#section-3.2",
741
+ fix: "Generate a 2048-bit DKIM key at your mail provider and publish it at a new selector under `_domainkey.{domain}`.",
742
+ },
743
+ "mail/dkim-testing": {
744
+ fact: "site.hosts.*.mail.dkim.found",
745
+ when: present("dkim"),
746
+ expect: { items: { properties: { testing: { const: false } } } },
747
+ message: "a DKIM key still carries the t=y testing flag, so receivers may ignore failed signatures (got {got})",
748
+ severity: "warning",
749
+ score: 3.9,
750
+ docs: "https://www.rfc-editor.org/rfc/rfc6376#section-3.6.1",
751
+ fix: "Remove `t=y` from the TXT record at Name `<selector>._domainkey.{domain}`.",
752
+ },
753
+ "mail/dkim-revoked": {
754
+ fact: "site.hosts.*.mail.dkim.found",
755
+ when: present("dkim"),
756
+ expect: { items: { properties: { revoked: { const: false } } } },
757
+ message: "a DKIM selector publishes an empty key, which revokes it; fine for a retired selector, not for one in use (got {got})",
758
+ severity: "info",
759
+ score: 1.6,
760
+ docs: "https://www.rfc-editor.org/rfc/rfc6376#section-3.6.1",
761
+ fix: "Delete the TXT record at Name `<selector>._domainkey.{domain}` once no mail signed with it is in transit.",
762
+ },
763
+ "mail/mx-resolves": {
764
+ fact: "site.hosts.*.mail.exchanges",
765
+ when: present("exchanges", "array"),
766
+ expect: { items: { properties: { addresses: { minimum: 1 } } } },
767
+ message: "an MX exchange has no A or AAAA record, so mail to it bounces (got {got})",
768
+ severity: "warning",
769
+ score: 5.9,
770
+ docs: "https://www.rfc-editor.org/rfc/rfc5321#section-5.1",
771
+ fix: "Add A and AAAA records for the Mail server the MX record of `{host}` names, or point the MX at a server that has them.",
772
+ },
773
+ "mail/mx-cname": {
774
+ fact: "site.hosts.*.mail.exchanges",
775
+ when: present("exchanges", "array"),
776
+ expect: { items: { properties: { cname: { const: false } } } },
777
+ message: "an MX exchange is a CNAME, which RFC 2181 forbids and some senders refuse (got {got})",
778
+ severity: "warning",
779
+ score: 5.6,
780
+ docs: "https://www.rfc-editor.org/rfc/rfc2181#section-10.3",
781
+ fix: "Set the Mail server of the MX record of `{host}` to the CNAME’s target, the name that has the A and AAAA records.",
782
+ },
783
+ "mail/mx-literal": {
784
+ fact: "site.hosts.*.mail.exchanges",
785
+ when: present("exchanges", "array"),
786
+ expect: { items: { properties: { literal: { const: false } } } },
787
+ message: "an MX exchange is an IP address, not a name, so senders cannot use it (got {got})",
788
+ severity: "warning",
789
+ score: 5.7,
790
+ docs: "https://www.rfc-editor.org/rfc/rfc7505#section-3",
791
+ fix: "Give the mail server a name with an A record, such as `mx.{domain}`, and set it as the Mail server of the MX record of `{host}`.",
792
+ },
793
+ "mail/mx-duplicate": {
794
+ fact: "site.hosts.*.mail.duplicates",
795
+ when: present("duplicates", "array"),
796
+ expect: { maxItems: 0 },
797
+ message: "the MX set names an exchange more than once (got {got})",
798
+ severity: "info",
799
+ score: 2.2,
800
+ docs: "https://www.rfc-editor.org/rfc/rfc5321#section-5.1",
801
+ fix: "Delete the repeated MX records of `{host}`, keeping one per Mail server.",
802
+ },
803
+ "mail/mta-sts": {
804
+ fact: "site.hosts.*.mail.mta-sts",
805
+ when: present("mta-sts"),
806
+ expect: { required: ["record"] },
807
+ message: "no MTA-STS record, so senders deliver over STARTTLS that an attacker on the path can strip",
808
+ severity: "info",
809
+ docs: "https://www.rfc-editor.org/rfc/rfc8461",
810
+ fix: "Serve `version: STSv1`, `mode: enforce`, one `mx:` line per Mail server and `max_age: 604800` at `https://mta-sts.{domain}/.well-known/mta-sts.txt`, then add a TXT record at Name `_mta-sts.{domain}` with Content `v=STSv1; id=20260101`.",
811
+ },
812
+ "mail/mta-sts-syntax": {
813
+ fact: "site.hosts.*.mail.mta-sts.errors",
814
+ when: present("mta-sts"),
815
+ expect: { maxItems: 0 },
816
+ message: "the _mta-sts record breaks the RFC 8461 grammar, so senders ignore the policy (got {got})",
817
+ severity: "warning",
818
+ score: 5.5,
819
+ docs: "https://www.rfc-editor.org/rfc/rfc8461#section-3.1",
820
+ fix: "Keep one TXT record at Name `_mta-sts.{domain}` with Content `v=STSv1; id=20260101`, the id letters and digits only.",
821
+ },
822
+ "mail/mta-sts-policy": {
823
+ fact: "site.hosts.*.mail.mta-sts.policy.errors",
824
+ when: { ...present("mta-sts"), "site.hosts.*.mail.mta-sts.policy": { type: "object" } },
825
+ expect: { maxItems: 0 },
826
+ message: "the MTA-STS policy file does not answer 200 as text/plain or does not parse, so senders ignore it (got {got})",
827
+ severity: "warning",
828
+ score: 5.4,
829
+ docs: "https://www.rfc-editor.org/rfc/rfc8461#section-3.2",
830
+ fix: "Serve `https://mta-sts.{domain}/.well-known/mta-sts.txt` with status 200 as `text/plain`, carrying `version: STSv1`, `mode: enforce`, one `mx:` line per Mail server and `max_age: 604800`.",
831
+ },
832
+ "mail/mta-sts-mode": {
833
+ fact: "site.hosts.*.mail.mta-sts.policy.mode",
834
+ when: { ...present("mta-sts"), "site.hosts.*.mail.mta-sts.policy.errors": { maxItems: 0 } },
835
+ expect: { const: "enforce" },
836
+ message: "the MTA-STS policy is in {got} mode, so senders still deliver when TLS fails",
837
+ severity: "hint",
838
+ score: 0.6,
839
+ docs: "https://www.rfc-editor.org/rfc/rfc8461#section-5",
840
+ fix: "Set `mode: enforce` in `https://mta-sts.{domain}/.well-known/mta-sts.txt` once TLS reports show no failures, and change the id of the TXT record at Name `_mta-sts.{domain}`.",
841
+ },
842
+ "mail/mta-sts-mx": {
843
+ fact: "site.hosts.*.mail.mta-sts.policy.unmatched",
844
+ when: { ...present("mta-sts"), "site.hosts.*.mail.mta-sts.policy.errors": { maxItems: 0 } },
845
+ expect: { maxItems: 0 },
846
+ message: "the MTA-STS policy covers no mx: pattern for an exchange, so enforcing senders refuse to deliver to it (got {got})",
847
+ severity: "warning",
848
+ score: 4.8,
849
+ docs: "https://www.rfc-editor.org/rfc/rfc8461#section-4.1",
850
+ fix: "Add an `mx:` line for every Mail server of `{host}` to `https://mta-sts.{domain}/.well-known/mta-sts.txt`, and change the id of the TXT record at Name `_mta-sts.{domain}`.",
851
+ },
852
+ "mail/mta-sts-max-age": {
853
+ fact: "site.hosts.*.mail.mta-sts.policy.max-age",
854
+ when: { ...present("mta-sts"), "site.hosts.*.mail.mta-sts.policy.errors": { maxItems: 0 } },
855
+ expect: { minimum: 86_400 },
856
+ message: "the MTA-STS max_age is {got} seconds, under a day, so senders forget the policy between messages",
857
+ severity: "warning",
858
+ score: 4.7,
859
+ docs: "https://www.rfc-editor.org/rfc/rfc8461#section-3.2",
860
+ fix: "Set `max_age: 604800` or more in `https://mta-sts.{domain}/.well-known/mta-sts.txt`.",
861
+ },
862
+ "mail/tls-rpt": {
863
+ fact: "site.hosts.*.mail.tls-rpt",
864
+ when: present("tls-rpt"),
865
+ expect: { required: ["record"] },
866
+ message: "no TLS-RPT record, so senders cannot report failed TLS deliveries",
867
+ severity: "info",
868
+ score: 1.8,
869
+ docs: "https://www.rfc-editor.org/rfc/rfc8460",
870
+ fix: "Add a TXT record at Name `_smtp._tls.{domain}` with Content `v=TLSRPTv1; rua=mailto:tls-reports@{domain}`.",
871
+ },
872
+ "mail/tls-rpt-syntax": {
873
+ fact: "site.hosts.*.mail.tls-rpt.errors",
874
+ when: present("tls-rpt"),
875
+ expect: { maxItems: 0 },
876
+ message: "the TLS-RPT record breaks the RFC 8460 grammar (got {got})",
877
+ severity: "warning",
878
+ score: 4.6,
879
+ docs: "https://www.rfc-editor.org/rfc/rfc8460#section-3",
880
+ fix: "Keep one TXT record at Name `_smtp._tls.{domain}` with Content `v=TLSRPTv1; rua=mailto:tls-reports@{domain}`.",
881
+ },
882
+ "mail/bimi-syntax": {
883
+ fact: "site.hosts.*.mail.bimi.errors",
884
+ when: present("bimi"),
885
+ expect: { maxItems: 0 },
886
+ message: "the BIMI record breaks its grammar (got {got})",
887
+ severity: "warning",
888
+ score: 4.5,
889
+ docs: "https://datatracker.ietf.org/doc/draft-brand-indicators-for-message-identification/",
890
+ fix: "Keep one TXT record at Name `default._bimi.{domain}` with Content `v=BIMI1; l=https://{domain}/logo.svg; a=https://{domain}/vmc.pem`.",
891
+ },
892
+ "mail/bimi-dmarc": {
893
+ fact: "site.hosts.*.mail.bimi.dmarc-enforced",
894
+ when: present("bimi"),
895
+ expect: { const: true },
896
+ message: "BIMI is published but DMARC does not quarantine or reject all mail, so no receiver shows the logo",
897
+ severity: "warning",
898
+ score: 4.4,
899
+ docs: "https://datatracker.ietf.org/doc/draft-brand-indicators-for-message-identification/",
900
+ fix: "Set `p=quarantine` or `p=reject` without `t=y` or `pct=` in the TXT record at Name `_dmarc.{domain}`.",
901
+ },
902
+ "mail/bimi-logo": {
903
+ fact: "site.hosts.*.mail.bimi.logo.errors",
904
+ when: { ...present("bimi"), "site.hosts.*.mail.bimi.logo": { type: "object" } },
905
+ expect: { maxItems: 0 },
906
+ message: "the BIMI logo does not load or breaks the SVG Tiny PS profile, so receivers show none (got {got})",
907
+ severity: "warning",
908
+ score: 4.3,
909
+ docs: "https://datatracker.ietf.org/doc/draft-svg-tiny-ps-abrotman/",
910
+ fix: 'Serve the logo the `l=` tag names as an SVG with `version="1.2" baseProfile="tiny-ps"`, a `<title>`, no script and no external reference.',
911
+ },
912
+ "mail/bimi-vmc": {
913
+ fact: "site.hosts.*.mail.bimi.vmc.parses",
914
+ when: { ...present("bimi"), "site.hosts.*.mail.bimi.vmc": { type: "object" } },
915
+ expect: { const: true },
916
+ message: "the BIMI certificate the a= tag names does not load or is not a PEM certificate",
917
+ severity: "warning",
918
+ score: 4.2,
919
+ docs: "https://datatracker.ietf.org/doc/draft-brand-indicators-for-message-identification/",
920
+ fix: "Serve the mark certificate your CA issued, in PEM form, at the URL the `a=` tag of `default._bimi.{domain}` names.",
921
+ },
922
+ "mail/dane": {
923
+ fact: "site.hosts.*.mail.dane",
924
+ when: present("dane", "array"),
925
+ expect: { items: { properties: { tlsa: { minimum: 1 }, signed: { const: true } } } },
926
+ message: "an exchange has no DNSSEC-signed TLSA record at _25._tcp, so senders cannot authenticate its certificate (got {got})",
927
+ severity: "hint",
928
+ score: 0.8,
929
+ docs: "https://www.rfc-editor.org/rfc/rfc7672",
930
+ fix: "Sign the zone of each Mail server of `{host}` and add a TLSA record at Name `_25._tcp.<mail server>` with Usage `3`, Selector `1`, Matching type `1` and the hash of its key.",
931
+ },
932
+ "mail/starttls": {
933
+ fact: "site.hosts.*.mail.starttls",
934
+ when: present("starttls", "array"),
935
+ expect: { items: { properties: { offered: { const: true } } } },
936
+ message: "an exchange does not offer STARTTLS on port 25, so mail to it travels in the clear (got {got})",
937
+ severity: "warning",
938
+ score: 6.4,
939
+ docs: "https://www.rfc-editor.org/rfc/rfc3207",
940
+ fix: "Turn on STARTTLS with a certificate for the Mail server name on every MX of `{host}`.",
941
+ },
942
+ "mail/starttls-certificate": {
943
+ fact: "site.hosts.*.mail.starttls",
944
+ when: present("starttls", "array"),
945
+ expect: { items: { anyOf: [{ properties: { offered: { const: false } } }, { properties: { authorized: { const: true } } }] } },
946
+ message: "an exchange’s STARTTLS certificate does not validate for its name, so MTA-STS and DANE senders refuse it (got {got})",
947
+ severity: "warning",
948
+ score: 6,
949
+ docs: "https://www.rfc-editor.org/rfc/rfc8461#section-4.2",
950
+ fix: "Install a certificate from a public CA covering the Mail server name, with its full chain, on every MX of `{host}`.",
951
+ },
952
+ "mail/starttls-banner": {
953
+ fact: "site.hosts.*.mail.starttls",
954
+ when: present("starttls", "array"),
955
+ expect: { items: { properties: { "banner-matches": { const: true } } } },
956
+ message: "an exchange’s 220 banner names a host its address’s PTR does not, which some receivers score as spam (got {got})",
957
+ severity: "info",
958
+ score: 1.2,
959
+ docs: "https://www.rfc-editor.org/rfc/rfc5321#section-4.3.1",
960
+ fix: "Make the SMTP greeting name of each MX of `{host}` and the PTR record of its address the same name.",
961
+ },
962
+ };
963
+ // The dns:mail rules a mail name answers to, kept under their IDs.
964
+ const CARRIED = ["dns/null-mx-mixed", "dns/spf-record", "dns/spf-all", "dns/dmarc-policy"];
965
+ const RECOMMENDED = [...CARRIED, "mail/spf-syntax", "mail/spf-lookups", "mail/spf-void-lookups", "mail/spf-include", "mail/dmarc-record", "mail/dmarc-syntax", "mail/dmarc-report-uri", "mail/dmarc-external", "mail/mx-resolves", "mail/mx-cname", "mail/mx-literal", "mail/mx-duplicate"];
966
+ const ALL = { ...Object.fromEntries(CARRIED.map((id) => [id, MAIL_RULES[id]])), ...RULES };
967
+ // Every recommended rule as a warning; a score outside the warning band is pinned into it.
968
+ function asWarning(id) {
969
+ const spec = ALL[id];
970
+ return { ...spec, severity: "warning", ...(typeof spec.score === "number" && { score: pin("warning", spec.score) }) };
971
+ }
972
+ // Mail authentication of every crawled host and its registrable domain: SPF, DMARC, DKIM, MX, MTA-STS, TLS-RPT, BIMI, and opt-in DANE and STARTTLS.
973
+ export default definePlugin({
974
+ name: "mail",
975
+ settings: {
976
+ type: "object",
977
+ additionalProperties: false,
978
+ properties: { mode: { enum: ["auto", "mail", "none"], default: "auto" }, "dkim-selectors": { type: "array", items: { type: "string", pattern: "^[A-Za-z0-9._-]+$" }, default: [] }, dane: { type: "boolean", default: false }, starttls: { type: "boolean", default: false } },
979
+ },
980
+ sites: [mail],
981
+ presets: {
982
+ mail: { description: "Mail authentication of names that take or send mail: SPF grammar and lookups, DMARC, DKIM keys, MX, MTA-STS, TLS-RPT and BIMI; DANE and STARTTLS once turned on", rules: ALL },
983
+ "mail:recommended": { description: "SPF, DMARC and MX correctness of names that take or send mail", rules: Object.fromEntries(RECOMMENDED.map((id) => [id, asWarning(id)])) },
984
+ },
985
+ });