spiderlint 0.0.0 → 0.143.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +235 -0
- package/README.md +244 -0
- package/bin/spiderlint.js +11 -0
- package/dist/agent.js +46 -0
- package/dist/cache/extractors.js +37 -0
- package/dist/cache/http.js +99 -0
- package/dist/cache/index.js +101 -0
- package/dist/cache/purge.js +63 -0
- package/dist/cache/status.js +52 -0
- package/dist/cli.js +602 -0
- package/dist/color.js +10 -0
- package/dist/config/environment.js +88 -0
- package/dist/config/index.js +104 -0
- package/dist/config/policy.js +170 -0
- package/dist/config/schema.js +127 -0
- package/dist/crawl/body.js +92 -0
- package/dist/crawl/browser.js +498 -0
- package/dist/crawl/crawl.js +58 -0
- package/dist/crawl/diversity.js +153 -0
- package/dist/crawl/dns.js +194 -0
- package/dist/crawl/fetch.js +48 -0
- package/dist/crawl/frontier.js +370 -0
- package/dist/crawl/guard.js +69 -0
- package/dist/crawl/http.js +171 -0
- package/dist/crawl/links.js +89 -0
- package/dist/crawl/log.js +27 -0
- package/dist/crawl/network.js +88 -0
- package/dist/crawl/probe.js +107 -0
- package/dist/crawl/profile.js +33 -0
- package/dist/crawl/rdap.js +86 -0
- package/dist/crawl/resolve.js +132 -0
- package/dist/crawl/resources.js +233 -0
- package/dist/crawl/robots.js +100 -0
- package/dist/crawl/route.js +92 -0
- package/dist/crawl/scope.js +32 -0
- package/dist/crawl/scratch.js +15 -0
- package/dist/crawl/sitemap.js +187 -0
- package/dist/crawl/svcb.js +59 -0
- package/dist/crawl/tls-hello.js +297 -0
- package/dist/crawl/tls-probe.js +71 -0
- package/dist/facts/browser.js +138 -0
- package/dist/facts/byline.js +17 -0
- package/dist/facts/co2.js +27 -0
- package/dist/facts/csp.js +71 -0
- package/dist/facts/export.js +79 -0
- package/dist/facts/flatten.js +33 -0
- package/dist/facts/graph.js +73 -0
- package/dist/facts/headers.js +330 -0
- package/dist/facts/html.js +230 -0
- package/dist/facts/labels.js +89 -0
- package/dist/facts/language.js +32 -0
- package/dist/facts/parity.js +38 -0
- package/dist/facts/read-note.js +11 -0
- package/dist/facts/resources.js +46 -0
- package/dist/facts/robots.js +13 -0
- package/dist/facts/sites.js +140 -0
- package/dist/facts/transport.js +200 -0
- package/dist/facts/types.js +4 -0
- package/dist/facts/vendors.js +76 -0
- package/dist/fold/index.js +94 -0
- package/dist/groups/assign.js +45 -0
- package/dist/groups/sample.js +60 -0
- package/dist/i18n.js +92 -0
- package/dist/index.js +607 -0
- package/dist/interrupt.js +37 -0
- package/dist/logger.js +95 -0
- package/dist/plugins/axe.js +104 -0
- package/dist/plugins/carbon-txt.js +96 -0
- package/dist/plugins/cookies-registry.js +25 -0
- package/dist/plugins/cookies.js +168 -0
- package/dist/plugins/css.js +328 -0
- package/dist/plugins/dns.js +864 -0
- package/dist/plugins/feed-content.js +132 -0
- package/dist/plugins/feed-model.js +282 -0
- package/dist/plugins/feeds.js +836 -0
- package/dist/plugins/html-validate.js +191 -0
- package/dist/plugins/htmlhint.js +95 -0
- package/dist/plugins/icons.js +302 -0
- package/dist/plugins/images.js +269 -0
- package/dist/plugins/index.js +286 -0
- package/dist/plugins/keyboard.js +194 -0
- package/dist/plugins/lighthouse.js +110 -0
- package/dist/plugins/link-text.js +73 -0
- package/dist/plugins/list.js +22 -0
- package/dist/plugins/live.js +254 -0
- package/dist/plugins/mail.js +985 -0
- package/dist/plugins/manifest.js +97 -0
- package/dist/plugins/markup.js +98 -0
- package/dist/plugins/messages.js +25 -0
- package/dist/plugins/network.js +192 -0
- package/dist/plugins/origin.js +488 -0
- package/dist/plugins/profile-links.js +102 -0
- package/dist/plugins/schema-registry.js +101 -0
- package/dist/plugins/sshfp.js +159 -0
- package/dist/plugins/stack.js +161 -0
- package/dist/plugins/structured-data.js +358 -0
- package/dist/plugins/tls-probe.js +298 -0
- package/dist/plugins/tls-registry.js +418 -0
- package/dist/plugins/tls-scan.js +280 -0
- package/dist/plugins/trackers-registry.js +27 -0
- package/dist/plugins/trackers.js +56 -0
- package/dist/plugins/types.js +7 -0
- package/dist/plugins/visit.js +50 -0
- package/dist/plugins/well-known-registry.js +111 -0
- package/dist/plugins/well-known.js +732 -0
- package/dist/progress.js +162 -0
- package/dist/report/agent.js +122 -0
- package/dist/report/checkstyle.js +22 -0
- package/dist/report/csv.js +18 -0
- package/dist/report/html.js +266 -0
- package/dist/report/human.js +220 -0
- package/dist/report/index.js +16 -0
- package/dist/report/json.js +8 -0
- package/dist/report/printable.js +29 -0
- package/dist/report/rating.js +26 -0
- package/dist/report/sarif.js +100 -0
- package/dist/report/stats.js +46 -0
- package/dist/rules/builtin.js +465 -0
- package/dist/rules/caching.js +103 -0
- package/dist/rules/catalog.js +139 -0
- package/dist/rules/clock.js +48 -0
- package/dist/rules/declarative.js +234 -0
- package/dist/rules/deprecated.js +45 -0
- package/dist/rules/disclosure.js +99 -0
- package/dist/rules/fix.js +21 -0
- package/dist/rules/i18n.js +131 -0
- package/dist/rules/insights.js +100 -0
- package/dist/rules/length.js +52 -0
- package/dist/rules/message.js +48 -0
- package/dist/rules/relations.js +88 -0
- package/dist/rules/robots.js +75 -0
- package/dist/rules/rulesets.js +134 -0
- package/dist/rules/run.js +75 -0
- package/dist/rules/score.js +61 -0
- package/dist/rules/types.js +7 -0
- package/dist/rules/url.js +90 -0
- package/dist/server/analytics.js +49 -0
- package/dist/server/api.js +134 -0
- package/dist/server/clients.js +54 -0
- package/dist/server/clock.js +33 -0
- package/dist/server/compress.js +58 -0
- package/dist/server/health.js +14 -0
- package/dist/server/jobs.js +82 -0
- package/dist/server/main.js +83 -0
- package/dist/server/page.js +161 -0
- package/dist/server/policy.js +145 -0
- package/dist/server/providers.js +60 -0
- package/dist/server/queue.js +50 -0
- package/dist/server/scan.js +39 -0
- package/dist/server/settings.js +197 -0
- package/dist/server/web.js +353 -0
- package/dist/server/worker.js +129 -0
- package/dist/store/disk.js +215 -0
- package/dist/store/memory.js +18 -0
- package/dist/telemetry-environment.js +9 -0
- package/dist/telemetry.js +104 -0
- package/locales/es/LC_MESSAGES/messages.po +1864 -0
- package/locales/uk/LC_MESSAGES/messages.po +1864 -0
- package/package.json +134 -2
- package/presets/browser.yaml +26 -0
- package/presets/graph.yaml +35 -0
- package/presets/i18n.yaml +18 -0
- package/presets/insights.yaml +12 -0
- package/presets/links.yaml +22 -0
- package/presets/parity.yaml +59 -0
- package/presets/performance.yaml +271 -0
- package/presets/recommended.yaml +50 -0
- package/presets/redirects.yaml +23 -0
- package/presets/resources.yaml +29 -0
- package/presets/robots.yaml +13 -0
- package/presets/security-headers.yaml +257 -0
- package/presets/seo.yaml +380 -0
- package/presets/sitemap.yaml +34 -0
- package/presets/suggestions.yaml +97 -0
- package/presets/sustainability.yaml +16 -0
- package/presets/tls.yaml +72 -0
- package/presets/vendor.yaml +14 -0
- package/presets/web-comprehensive.yaml +17 -0
- package/presets/web-quick.yaml +15 -0
- package/src/server/apple-touch-icon.png +0 -0
- package/src/server/logo-36.png +0 -0
- package/src/server/logo-72.png +0 -0
- package/src/server/logo.png +0 -0
- package/vendors/dns.yaml +21 -0
- package/vendors/paths.yaml +55 -0
- package/vendors/stack.yaml +57 -0
|
@@ -0,0 +1,220 @@
|
|
|
1
|
+
// SPDX-FileCopyrightText: 2026 Damián Búho <damian.buho@proton.me>
|
|
2
|
+
//
|
|
3
|
+
// SPDX-License-Identifier: AGPL-3.0-only
|
|
4
|
+
import { relative, singleOrigin } from "../crawl/scope.js";
|
|
5
|
+
import { environmentLocale } from "../i18n.js";
|
|
6
|
+
import { bytes as sized, isLabelled, label, withUnit } from "../facts/labels.js";
|
|
7
|
+
import { fixFor } from "../rules/fix.js";
|
|
8
|
+
import { cachedReads, valuesAt } from "../rules/message.js";
|
|
9
|
+
import { byImportance, scoreOf } from "../rules/score.js";
|
|
10
|
+
import { plain } from "../color.js";
|
|
11
|
+
import { printable, printableFinding } from "./printable.js";
|
|
12
|
+
import { passing } from "./rating.js";
|
|
13
|
+
const ORDER = { error: 0, warning: 1, info: 2, hint: 3 };
|
|
14
|
+
const TONE = { error: "red", warning: "yellow", info: "blue", hint: "dim" };
|
|
15
|
+
const LIST = 5;
|
|
16
|
+
const DETAIL = " ".repeat(10);
|
|
17
|
+
const NESTED = " ".repeat(12);
|
|
18
|
+
const LABEL = 11;
|
|
19
|
+
const PLURAL = { rule: "rules", error: "errors", warning: "warnings", info: "info", hint: "hints", page: "pages", launch: "launches", fetch: "fetches", request: "requests", "TLS probe": "TLS probes", "cached read": "cached reads" };
|
|
20
|
+
const ORANGE = "#ff8700";
|
|
21
|
+
const GRADE_TONE = { S: "green", A: "green", B: "yellow", C: ORANGE, D: ORANGE, E: "red", F: "red" };
|
|
22
|
+
const MINUS = "\u{2212}";
|
|
23
|
+
// The locale’s digits and decimal mark, groups split by a narrow no-break space as SI writes them.
|
|
24
|
+
function number(value, options = {}) {
|
|
25
|
+
return new Intl.NumberFormat(environmentLocale(), { maximumFractionDigits: 1, ...options })
|
|
26
|
+
.formatToParts(value)
|
|
27
|
+
.map((part) => (part.type === "group" ? "\u{202F}" : part.value))
|
|
28
|
+
.join("");
|
|
29
|
+
}
|
|
30
|
+
// A count and its noun, plural unless it is exactly one.
|
|
31
|
+
function counted(count, noun) {
|
|
32
|
+
return `${number(count)} ${count === 1 ? noun : (PLURAL[noun] ?? noun)}`;
|
|
33
|
+
}
|
|
34
|
+
// One summary row: a padded label, then its value.
|
|
35
|
+
function row(label, value) {
|
|
36
|
+
return `${label.padEnd(LABEL)}${value}`;
|
|
37
|
+
}
|
|
38
|
+
// At most `limit` URLs on the detail line, the rest as a count.
|
|
39
|
+
function list(urls, origin, limit) {
|
|
40
|
+
const shown = urls
|
|
41
|
+
.slice(0, limit)
|
|
42
|
+
.map((url) => relative(url, origin))
|
|
43
|
+
.join(", ");
|
|
44
|
+
return urls.length > limit ? `${shown} … and ${urls.length - limit} more` : shown;
|
|
45
|
+
}
|
|
46
|
+
// A URL, then what was measured there, each relative to the origin.
|
|
47
|
+
function shown(finding, url, origin) {
|
|
48
|
+
return [url, ...valuesAt(finding, url, number)].map((text) => relative(text, origin)).join(" ");
|
|
49
|
+
}
|
|
50
|
+
// At most `limit` URLs one per line with their values, the rest as a count.
|
|
51
|
+
function measured(finding, urls, indent, origin, paint, limit) {
|
|
52
|
+
const lines = urls.slice(0, limit).map((url) => paint("dim", `${indent}${shown(finding, url, origin)}`));
|
|
53
|
+
return urls.length > limit ? [...lines, paint("dim", `${indent}… and ${urls.length - limit} more`)] : lines;
|
|
54
|
+
}
|
|
55
|
+
// At most `limit` locations, each on its own line under what it locates, the rest as a count.
|
|
56
|
+
function located(locations, indent, paint, limit) {
|
|
57
|
+
if (!locations || locations.length === 0)
|
|
58
|
+
return [];
|
|
59
|
+
const shown = locations.slice(0, limit).map((location) => paint("dim", `${indent}at ${location}`));
|
|
60
|
+
return locations.length > limit ? [...shown, paint("dim", `${indent}… and ${locations.length - limit} more`)] : shown;
|
|
61
|
+
}
|
|
62
|
+
// A fold’s samples; pages whose locations all match share one list under the URLs.
|
|
63
|
+
function sampled(finding, origin, paint, limit) {
|
|
64
|
+
const samples = finding.samples ?? [];
|
|
65
|
+
const byPage = finding.sampleLocations;
|
|
66
|
+
const isShared = !finding.data && new Set(samples.map((url) => JSON.stringify(byPage?.[url] ?? []))).size === 1;
|
|
67
|
+
return isShared ? [paint("dim", `${DETAIL}e.g. ${list(samples, origin, limit)}`), ...located(byPage?.[samples[0]], NESTED, paint, limit)] : samples.flatMap((url) => [paint("dim", `${DETAIL}e.g. ${shown(finding, url, origin)}`), ...located(byPage?.[url], NESTED, paint, limit)]);
|
|
68
|
+
}
|
|
69
|
+
function heading(finding, paint) {
|
|
70
|
+
return ` ${paint(TONE[finding.severity], `${finding.severity.padEnd(7)} ${scoreOf(finding).toFixed(1)}`)} ${paint("bold", finding.rule)}`;
|
|
71
|
+
}
|
|
72
|
+
function shortMessage(finding, origin) {
|
|
73
|
+
return origin ? finding.message.replaceAll(`${origin}/`, "/") : finding.message;
|
|
74
|
+
}
|
|
75
|
+
// Page findings sharing severity, score, rule and sentence bundle together, by template when they carry one; folds and aggregates stay alone.
|
|
76
|
+
export function bundle(findings) {
|
|
77
|
+
const bundles = new Map();
|
|
78
|
+
for (const [index, finding] of findings.entries()) {
|
|
79
|
+
const isPlain = finding.occurrences === undefined && !finding.urls;
|
|
80
|
+
const key = isPlain ? `${finding.severity}\t${scoreOf(finding)}\t${finding.rule}\t${finding.text === undefined ? finding.message : `${finding.text}\t${JSON.stringify(finding.variables ?? {})}`}` : String(index);
|
|
81
|
+
bundles.set(key, [...(bundles.get(key) ?? []), finding]);
|
|
82
|
+
}
|
|
83
|
+
return bundles.values().toArray();
|
|
84
|
+
}
|
|
85
|
+
// A bundle prints its message once, then every page on its own line.
|
|
86
|
+
function bundled(same, origin, paint, limit) {
|
|
87
|
+
const first = same[0];
|
|
88
|
+
return [`${heading(first, paint)} — ${same.length} pages: ${shortMessage(first, origin)}`, ...same.flatMap((finding) => [paint("dim", `${DETAIL}${shown(finding, finding.url, origin)}`), ...located(finding.locations, NESTED, paint, limit)])];
|
|
89
|
+
}
|
|
90
|
+
// A fold shows its samples; an aggregate its URL list, and its own URL when that is not one of them.
|
|
91
|
+
function line(finding, origin, paint, limit) {
|
|
92
|
+
const url = relative(finding.url, origin);
|
|
93
|
+
const message = shortMessage(finding, origin);
|
|
94
|
+
const head = heading(finding, paint);
|
|
95
|
+
if (finding.occurrences !== undefined) {
|
|
96
|
+
const pages = `${finding.sampled === undefined ? `${finding.occurrences} pages` : `${finding.occurrences} of ${finding.sampled} sampled pages`}${finding.groups ? ` in ${finding.groups.join(", ")}` : ""}`;
|
|
97
|
+
return [`${head} — ${pages} (${Math.round((finding.coverage ?? 0) * 100)}%): ${message}`, ...sampled(finding, origin, paint, limit)];
|
|
98
|
+
}
|
|
99
|
+
if (!finding.urls)
|
|
100
|
+
return [`${head} ${url}: ${message}`, ...measured(finding, Object.hasOwn(finding.data ?? {}, finding.url) ? [finding.url] : [], DETAIL, origin, paint, limit), ...located(finding.locations, DETAIL, paint, limit)];
|
|
101
|
+
const subject = finding.urls.includes(finding.url) ? "—" : `${url}:`;
|
|
102
|
+
const urls = finding.data ? measured(finding, finding.urls, DETAIL, origin, paint, limit) : [paint("dim", `${DETAIL}${list(finding.urls, origin, limit)}`)];
|
|
103
|
+
return [`${head} ${subject} ${message}`, ...urls, ...located(finding.locations, DETAIL, paint, limit)];
|
|
104
|
+
}
|
|
105
|
+
// The grade and the rulesets it was earned under; a dash when nothing was judged.
|
|
106
|
+
function ratingValue(rating, paint) {
|
|
107
|
+
return rating ? `${paint(GRADE_TONE[rating.grade], paint("bold", rating.grade))} (${rating.rulesets.join(", ")})` : "– (no checks ran)";
|
|
108
|
+
}
|
|
109
|
+
// A signed change against the last run, green when it fell and red when it grew; nothing when equal or unknown.
|
|
110
|
+
function change(now, before, paint) {
|
|
111
|
+
return before === undefined || now === before ? "" : ` ${paint(now < before ? "green" : "red", `${now < before ? MINUS : "+"}${number(Math.abs(now - before))}`)}`;
|
|
112
|
+
}
|
|
113
|
+
// The rule’s fix filled for this finding, then its docs, under the finding.
|
|
114
|
+
function explanation(guide, finding, paint) {
|
|
115
|
+
return [guide?.fix ? `${paint("dim", `${DETAIL}fix `)}${fixFor(guide.fix, finding)}` : "", guide?.docs ? paint("dim", `${DETAIL}docs ${guide.docs}`) : ""].filter(Boolean);
|
|
116
|
+
}
|
|
117
|
+
// Findings by importance (score times the share of pages touched) then rule then URL, bundled, each followed by its explanation when `guides` is given.
|
|
118
|
+
function listed(findings, origin, paint, limit, total, guides) {
|
|
119
|
+
findings.sort(byImportance(total));
|
|
120
|
+
return bundle(findings).flatMap((same) => {
|
|
121
|
+
const first = same[0];
|
|
122
|
+
return [...(same.length > 1 ? bundled(same, origin, paint, limit) : line(first, origin, paint, limit)), ...(guides ? explanation(guides[first.rule], first, paint) : [])];
|
|
123
|
+
});
|
|
124
|
+
}
|
|
125
|
+
// The shared origin once on top, findings grouped by group then rule, site-wide ones next, then each vendor’s, hints last and only counted unless `isHintListed`, the fact statistics with `isStats`, then the totals; `isFull` lists every URL and location, `isExplained` each fix.
|
|
126
|
+
export function formatHuman(report, paint = plain, isFull = false, _lang, isHintListed = false, isExplained = false, isStats = false) {
|
|
127
|
+
const limit = isFull ? Infinity : LIST;
|
|
128
|
+
const guides = isExplained ? (report.rules ?? {}) : undefined;
|
|
129
|
+
const origin = singleOrigin(report.pages.map((page) => page.url.href));
|
|
130
|
+
const out = origin ? [paint(["bold", "underline"], origin)] : [];
|
|
131
|
+
const groups = new Map();
|
|
132
|
+
const vendors = new Map();
|
|
133
|
+
const findings = report.findings.map((finding) => printableFinding(finding));
|
|
134
|
+
const hints = findings.filter((finding) => finding.severity === "hint");
|
|
135
|
+
for (const finding of findings) {
|
|
136
|
+
if (finding.severity === "hint")
|
|
137
|
+
continue;
|
|
138
|
+
const [sections, key] = finding.vendor ? [vendors, finding.vendor] : [groups, finding.scope === "site" ? "site" : finding.group];
|
|
139
|
+
sections.set(key, [...(sections.get(key) ?? []), finding]);
|
|
140
|
+
}
|
|
141
|
+
for (const [group, findings] of groups) {
|
|
142
|
+
const pages = report.summary.groups[group] ?? 0;
|
|
143
|
+
out.push(group === "site" ? paint("bold", "site") : `${paint("bold", group)} ${paint("dim", `(${counted(pages, "page")})`)}`, ...listed(findings, origin, paint, limit, report.pages.length, guides));
|
|
144
|
+
}
|
|
145
|
+
for (const [vendor, findings] of vendors)
|
|
146
|
+
out.push(`${paint("bold", vendor)} ${paint("dim", "(vendor)")}`, ...listed(findings, origin, paint, limit, report.pages.length, guides));
|
|
147
|
+
if (hints.length > 0)
|
|
148
|
+
out.push(`${paint("bold", "hints")} ${paint("dim", `(${counted(hints.length, "hint")})`)}`, ...(isHintListed ? listed(hints, origin, paint, limit, report.pages.length, guides) : [paint("dim", `${DETAIL}--show-hints lists them`)]));
|
|
149
|
+
out.push(...cachedRow(findings, paint));
|
|
150
|
+
if (isFull)
|
|
151
|
+
out.push("", ...passedRows(report.summary, paint));
|
|
152
|
+
if (isStats)
|
|
153
|
+
out.push("", ...statRows(report.summary.stats ?? {}, paint));
|
|
154
|
+
out.push("", ...totals(report.summary, paint), ...costRows(report.summary.cost).map((line) => paint("dim", line)));
|
|
155
|
+
return out.join("\n");
|
|
156
|
+
}
|
|
157
|
+
// How many observations behind the findings came from a cache, by bucket, and how to read them again; nothing when none did.
|
|
158
|
+
function cachedRow(findings, paint) {
|
|
159
|
+
const reads = cachedReads(findings);
|
|
160
|
+
if (reads.length === 0)
|
|
161
|
+
return [];
|
|
162
|
+
const buckets = [...new Set(reads.map((read) => read.bucket))].join(", ");
|
|
163
|
+
return ["", paint("dim", `${counted(reads.length, "cached read")} behind these findings (${buckets}); --refresh reads them again`)];
|
|
164
|
+
}
|
|
165
|
+
// Every rule that failed nowhere with the pages it covered, under a count; nothing when none ran.
|
|
166
|
+
function passedRows({ checked }, paint) {
|
|
167
|
+
const rules = passing(checked);
|
|
168
|
+
return rules.length === 0 ? [] : [paint("bold", `passed ${counted(rules.length, "rule")}`), ...rules.map(([id, rule]) => ` ${paint("green", "✓")} ${id} ${paint("dim", counted(rule.pages, "page"))}`)];
|
|
169
|
+
}
|
|
170
|
+
// Pages, size, time, rules, checks, findings with their change since the last run, and the rating, one row each; findings are counted before folding.
|
|
171
|
+
function totals({ pages, bytes, durationMs, statuses, rules, checks, findings, rating, previous }, paint) {
|
|
172
|
+
const answers = Object.entries(statuses).map(([status, count]) => `${number(count)} × ${status}`);
|
|
173
|
+
const shown = Object.keys(ORDER).filter((severity) => severity !== "hint" || (findings.hint ?? 0) > 0);
|
|
174
|
+
const severities = shown.map((severity) => `${(findings[severity] ?? 0) > 0 ? paint(TONE[severity], counted(findings[severity], severity)) : counted(0, severity)}${change(findings[severity], previous?.findings[severity], paint)}`);
|
|
175
|
+
const since = previous ? paint("dim", ` since ${new Date(previous.started).toLocaleString(environmentLocale(), { dateStyle: "short", timeStyle: "short" })}`) : "";
|
|
176
|
+
return [
|
|
177
|
+
row("pages", answers.length > 0 ? `${number(pages)} (${answers.join(", ")})` : number(pages)),
|
|
178
|
+
row("size", sized(bytes, number)),
|
|
179
|
+
row("time", number(durationMs / 1000, { style: "unit", unit: "second" })),
|
|
180
|
+
row("rules", number(rules)),
|
|
181
|
+
row("checks", `${number(checks.passed)} of ${number(checks.total)} passed`),
|
|
182
|
+
row("findings", `${number(findings.total)}${change(findings.total, previous?.findings.total, paint)} (${severities.join(", ")})${since}`),
|
|
183
|
+
row("rating", ratingValue(rating, paint)),
|
|
184
|
+
];
|
|
185
|
+
}
|
|
186
|
+
// A measured value in the unit `path` carries, if any.
|
|
187
|
+
export function measure(value, path = "") {
|
|
188
|
+
return withUnit(path, value, number);
|
|
189
|
+
}
|
|
190
|
+
// Rows as aligned columns, the first padded at its end and the rest at their start, the header bold.
|
|
191
|
+
export function aligned(rows, paint) {
|
|
192
|
+
const widths = rows[0].map((_, column) => Math.max(...rows.map((row) => (row[column] ?? "").length)));
|
|
193
|
+
const lines = rows.map((row) => row
|
|
194
|
+
.map((cell, column) => (column === 0 ? cell.padEnd(widths[0]) : cell.padStart(widths[column])))
|
|
195
|
+
.join(" ")
|
|
196
|
+
.trimEnd());
|
|
197
|
+
return [paint("bold", lines[0]), ...lines.slice(1)];
|
|
198
|
+
}
|
|
199
|
+
// One aligned row per labelled numeric fact by its label: pages, then min, median, p95, max and total in its unit, under a header.
|
|
200
|
+
export function statRows(stats, paint) {
|
|
201
|
+
const cells = Object.entries(stats)
|
|
202
|
+
.filter(([path]) => isLabelled(path))
|
|
203
|
+
.map(([path, stat]) => [printable(label(path) ?? path), measure(stat.count), ...[stat.min, stat.median, stat.p95, stat.max, stat.total].map((value) => (value === undefined ? "–" : measure(value, path)))]);
|
|
204
|
+
return cells.length === 0 ? [row("stats", "none")] : aligned([["stats", "pages", "min", "median", "p95", "max", "total"], ...cells], paint);
|
|
205
|
+
}
|
|
206
|
+
// Browsers launched and pages they rendered, plain HTTP fetches, resource requests, extractor runs and cache hits, one row each.
|
|
207
|
+
function costRows({ browser, http, resources, extractors, extractorsCached = {} }) {
|
|
208
|
+
const runs = Object.entries(extractors).map(([id, count]) => `${id} ×${number(count)}`);
|
|
209
|
+
const hits = Object.entries(extractorsCached).map(([id, count]) => `${id} ×${number(count)}`);
|
|
210
|
+
return [
|
|
211
|
+
browser ? row("browser", `${browser.name}, ${counted(browser.pages, "page")} in ${counted(browser.launches, "launch")}${browser.tlsProbes > 0 ? `, ${counted(browser.tlsProbes, "TLS probe")}` : ""}`) : "",
|
|
212
|
+
http ? row("http", `${counted(http.pages, "fetch")}${http.revalidated > 0 ? ` (${number(http.revalidated)} revalidated)` : ""}`) : "",
|
|
213
|
+
browser || http ? "" : row("fetch", "none"),
|
|
214
|
+
resources
|
|
215
|
+
? row("resources", `${counted(resources.requests, "request")}${resources.cached > 0 ? ` (${number(resources.cached)} more from cache${resources.failuresCached ? `, ${counted(resources.failuresCached, "failure")}` : ""})` : ""}${resources.logged > 0 ? ` (${number(resources.logged)} more from the browser)` : ""}`)
|
|
216
|
+
: "",
|
|
217
|
+
runs.length > 0 ? row("extractors", runs.join(", ")) : "",
|
|
218
|
+
hits.length > 0 ? row("cached", hits.join(", ")) : "",
|
|
219
|
+
].filter(Boolean);
|
|
220
|
+
}
|
|
@@ -0,0 +1,16 @@
|
|
|
1
|
+
// SPDX-FileCopyrightText: 2026 Damián Búho <damian.buho@proton.me>
|
|
2
|
+
//
|
|
3
|
+
// SPDX-License-Identifier: AGPL-3.0-only
|
|
4
|
+
import { definePlugin } from "../plugins/types.js";
|
|
5
|
+
import { formatAgent } from "./agent.js";
|
|
6
|
+
import { formatCheckstyle } from "./checkstyle.js";
|
|
7
|
+
import { formatCsv } from "./csv.js";
|
|
8
|
+
import { formatHtml } from "./html.js";
|
|
9
|
+
import { formatHuman } from "./human.js";
|
|
10
|
+
import { formatJson } from "./json.js";
|
|
11
|
+
import { formatSarif } from "./sarif.js";
|
|
12
|
+
// The bundled report formats.
|
|
13
|
+
export default definePlugin({
|
|
14
|
+
name: "report",
|
|
15
|
+
formatters: { human: formatHuman, json: formatJson, sarif: formatSarif, checkstyle: formatCheckstyle, csv: formatCsv, html: formatHtml, agent: formatAgent },
|
|
16
|
+
});
|
|
@@ -0,0 +1,8 @@
|
|
|
1
|
+
// SPDX-FileCopyrightText: 2026 Damián Búho <damian.buho@proton.me>
|
|
2
|
+
//
|
|
3
|
+
// SPDX-License-Identifier: AGPL-3.0-only
|
|
4
|
+
import { scoreOf } from "../rules/score.js";
|
|
5
|
+
// Summary, findings and their rules’ docs; facts stay behind `spiderlint show-facts <url>`.
|
|
6
|
+
export function formatJson(report) {
|
|
7
|
+
return JSON.stringify({ summary: report.summary, findings: report.findings.map((finding) => ({ ...finding, score: scoreOf(finding) })), ...(report.rules && { rules: report.rules }) }, undefined, 2);
|
|
8
|
+
}
|
|
@@ -0,0 +1,29 @@
|
|
|
1
|
+
// SPDX-FileCopyrightText: 2026 Damián Búho <damian.buho@proton.me>
|
|
2
|
+
//
|
|
3
|
+
// SPDX-License-Identifier: AGPL-3.0-only
|
|
4
|
+
// C0 and C1 controls, newline and tab included, and the bidi overrides and isolates.
|
|
5
|
+
const UNPRINTABLE = /[\p{Cc}\u{202A}-\u{202E}\u{2066}-\u{2069}]/gu;
|
|
6
|
+
// Site text with every control shown as a `\u{…}` escape, so it can neither drive a terminal nor forge a line.
|
|
7
|
+
export function printable(text) {
|
|
8
|
+
return text.replaceAll(UNPRINTABLE, (char) => String.raw `\u{${char.codePointAt(0).toString(16)}}`);
|
|
9
|
+
}
|
|
10
|
+
// Every string datum of a record printable.
|
|
11
|
+
function datums(data) {
|
|
12
|
+
return Object.fromEntries(Object.entries(data).map(([key, datum]) => [key, typeof datum === "string" ? printable(datum) : datum]));
|
|
13
|
+
}
|
|
14
|
+
// A finding whose site-derived strings are all printable.
|
|
15
|
+
export function printableFinding(finding) {
|
|
16
|
+
const lines = (values) => values?.map((value) => printable(value));
|
|
17
|
+
return {
|
|
18
|
+
...finding,
|
|
19
|
+
url: printable(finding.url),
|
|
20
|
+
message: printable(finding.message),
|
|
21
|
+
...(finding.locations && { locations: lines(finding.locations) }),
|
|
22
|
+
...(finding.urls && { urls: lines(finding.urls) }),
|
|
23
|
+
...(finding.samples && { samples: lines(finding.samples) }),
|
|
24
|
+
...(finding.variables && { variables: datums(finding.variables) }),
|
|
25
|
+
...(finding.data && { data: Object.fromEntries(Object.entries(finding.data).map(([url, data]) => [printable(url), datums(data)])) }),
|
|
26
|
+
...(finding.evidence && { evidence: finding.evidence.map((read) => ({ ...read, key: printable(read.key) })) }),
|
|
27
|
+
...(finding.sampleLocations && { sampleLocations: Object.fromEntries(Object.entries(finding.sampleLocations).map(([url, found]) => [printable(url), lines(found)])) }),
|
|
28
|
+
};
|
|
29
|
+
}
|
|
@@ -0,0 +1,26 @@
|
|
|
1
|
+
// SPDX-FileCopyrightText: 2026 Damián Búho <damian.buho@proton.me>
|
|
2
|
+
//
|
|
3
|
+
// SPDX-License-Identifier: AGPL-3.0-only
|
|
4
|
+
// The rules that ran and failed nowhere, by ID.
|
|
5
|
+
export function passing(checked = {}) {
|
|
6
|
+
return Object.entries(checked)
|
|
7
|
+
.filter(([, rule]) => rule.failed === 0)
|
|
8
|
+
.toSorted(([a], [b]) => a.localeCompare(b));
|
|
9
|
+
}
|
|
10
|
+
// Lowest score in tenths each grade still reaches, best first; F takes the rest.
|
|
11
|
+
const FLOORS = [
|
|
12
|
+
[9, "A"],
|
|
13
|
+
[7, "B"],
|
|
14
|
+
[6, "C"],
|
|
15
|
+
[4, "D"],
|
|
16
|
+
[2, "E"],
|
|
17
|
+
];
|
|
18
|
+
// Grade by share passed, a failed check counting for its cost, 1 at a warning and more as its score rises; S needs no failed check, an error caps at B, nothing judged means no grade.
|
|
19
|
+
export function rate(checks, rulesets) {
|
|
20
|
+
if (checks.total === 0)
|
|
21
|
+
return undefined;
|
|
22
|
+
const floors = checks.errored > 0 ? FLOORS.slice(1) : FLOORS;
|
|
23
|
+
const share = Math.max(0, checks.total - (checks.cost ?? checks.failed)) / checks.total;
|
|
24
|
+
const grade = checks.failed === 0 ? "S" : (floors.find(([tenths]) => share * 10 >= tenths)?.[1] ?? "F");
|
|
25
|
+
return { grade, score: Number(share.toFixed(4)), rulesets };
|
|
26
|
+
}
|
|
@@ -0,0 +1,100 @@
|
|
|
1
|
+
// SPDX-FileCopyrightText: 2026 Damián Búho <damian.buho@proton.me>
|
|
2
|
+
//
|
|
3
|
+
// SPDX-License-Identifier: AGPL-3.0-only
|
|
4
|
+
import { VERSION } from "../agent.js";
|
|
5
|
+
import { fixFor } from "../rules/fix.js";
|
|
6
|
+
import { inEnglish, valuesAt, writtenAll } from "../rules/message.js";
|
|
7
|
+
import { scoreOf } from "../rules/score.js";
|
|
8
|
+
const SCHEMA = "https://raw.githubusercontent.com/oasis-tcs/sarif-spec/master/Schemata/sarif-schema-2.1.0.json";
|
|
9
|
+
const SECURITY = /^(tls|cookies|dns|mail|sshfp)\/|^http\/(hsts|csp|content-type-options|referrer-policy|frame-options|server-disclosure|permissions-policy|coop|coep|corp|reporting|no-x-xss|deprecated-header)/;
|
|
10
|
+
const LEVEL = { error: "error", warning: "warning", info: "note", hint: "none" };
|
|
11
|
+
function location(uri) {
|
|
12
|
+
return { physicalLocation: { artifactLocation: { uri } } };
|
|
13
|
+
}
|
|
14
|
+
// A location carrying the values measured there as its message, when there are any.
|
|
15
|
+
function measured(finding, uri) {
|
|
16
|
+
const values = valuesAt(finding, uri, inEnglish);
|
|
17
|
+
return { ...location(uri), ...(values.length > 0 && { message: { text: values.join(", ") } }) };
|
|
18
|
+
}
|
|
19
|
+
// Samples beyond the primary location, for a folded or site-wide finding.
|
|
20
|
+
function relatedLocations(finding) {
|
|
21
|
+
const extra = (finding.samples ?? finding.urls ?? []).filter((url) => url !== finding.url);
|
|
22
|
+
return extra.length === 0 ? undefined : extra.map((uri, id) => ({ id, ...measured(finding, uri) }));
|
|
23
|
+
}
|
|
24
|
+
// A template’s placeholders as SARIF’s positional `{0}`, in the order its variables are named.
|
|
25
|
+
function positional(text, names) {
|
|
26
|
+
return text.replaceAll(/\{(\w+)\}/g, (match, name) => (names.includes(name) ? `{${names.indexOf(name)}}` : match));
|
|
27
|
+
}
|
|
28
|
+
// The message: the English text, and for a template its message string ID and arguments (SARIF 2.1.0 §3.11.7).
|
|
29
|
+
function message(finding, ids) {
|
|
30
|
+
const id = finding.text === undefined ? undefined : ids.get(`${finding.rule}\t${finding.text}`);
|
|
31
|
+
const variables = writtenAll(finding.variables, inEnglish);
|
|
32
|
+
return { text: finding.message, ...(id && { id, arguments: Object.values(variables) }) };
|
|
33
|
+
}
|
|
34
|
+
function toResult(finding, ids) {
|
|
35
|
+
const related = relatedLocations(finding);
|
|
36
|
+
return {
|
|
37
|
+
ruleId: finding.rule,
|
|
38
|
+
level: LEVEL[finding.severity],
|
|
39
|
+
rank: scoreOf(finding) * 10,
|
|
40
|
+
message: message(finding, ids),
|
|
41
|
+
locations: [measured(finding, finding.url)],
|
|
42
|
+
...(finding.occurrences !== undefined && { occurrenceCount: finding.occurrences }),
|
|
43
|
+
...(related && { relatedLocations: related }),
|
|
44
|
+
...(finding.evidence && { properties: { evidence: finding.evidence } }),
|
|
45
|
+
};
|
|
46
|
+
}
|
|
47
|
+
// One message string ID per distinct template, numbered within its rule, and the rule’s strings by ID.
|
|
48
|
+
function messageStrings(findings) {
|
|
49
|
+
const ids = new Map();
|
|
50
|
+
const byRule = new Map();
|
|
51
|
+
for (const finding of findings) {
|
|
52
|
+
if (finding.text === undefined || ids.has(`${finding.rule}\t${finding.text}`))
|
|
53
|
+
continue;
|
|
54
|
+
const strings = byRule.get(finding.rule) ?? {};
|
|
55
|
+
const id = `m${Object.keys(strings).length}`;
|
|
56
|
+
strings[id] = { text: positional(finding.text, Object.keys(finding.variables ?? {})) };
|
|
57
|
+
ids.set(`${finding.rule}\t${finding.text}`, id);
|
|
58
|
+
byRule.set(finding.rule, strings);
|
|
59
|
+
}
|
|
60
|
+
return { ids, byRule };
|
|
61
|
+
}
|
|
62
|
+
// The rule’s generic fix and docs link, which code scanning shows beside each alert.
|
|
63
|
+
function help({ fix, docs }) {
|
|
64
|
+
const text = [fix && fixFor(fix), docs].filter(Boolean);
|
|
65
|
+
const markdown = [fix && fixFor(fix), docs && `[Documentation](${docs})`].filter(Boolean);
|
|
66
|
+
return text.length === 0 ? {} : { help: { text: text.join("\n\n"), markdown: markdown.join("\n\n") }, ...(docs && { helpUri: docs }) };
|
|
67
|
+
}
|
|
68
|
+
// Code scanning reads the worst score of a security rule’s findings as its CVSS-style severity, "0.0" to "10.0".
|
|
69
|
+
function security(id, findings) {
|
|
70
|
+
if (!SECURITY.test(id))
|
|
71
|
+
return {};
|
|
72
|
+
const worst = Math.max(...findings.filter((finding) => finding.rule === id).map((finding) => scoreOf(finding)));
|
|
73
|
+
return { properties: { "security-severity": worst.toFixed(1), tags: ["security"] } };
|
|
74
|
+
}
|
|
75
|
+
// One driver rule per distinct rule ID seen in the findings, with its help when the report carries its guide.
|
|
76
|
+
function toRules(findings, strings, guides = {}) {
|
|
77
|
+
const ids = [...new Set(findings.map((finding) => finding.rule))].toSorted((a, b) => a.localeCompare(b));
|
|
78
|
+
return ids.map((id) => ({ id, shortDescription: { text: id }, ...(strings.has(id) && { messageStrings: strings.get(id) }), ...(guides[id] && help(guides[id])), ...security(id, findings) }));
|
|
79
|
+
}
|
|
80
|
+
// The run's totals as a SARIF invocation; the summary rides in its property bag.
|
|
81
|
+
function toInvocation({ started, ...summary }) {
|
|
82
|
+
const endTimeUtc = new Date(Date.parse(started) + summary.durationMs).toISOString();
|
|
83
|
+
return { executionSuccessful: true, startTimeUtc: started, endTimeUtc, properties: summary };
|
|
84
|
+
}
|
|
85
|
+
// SARIF 2.1.0: one run, one tool. Folded findings carry occurrenceCount + relatedLocations (AGENTS.md ## Folding).
|
|
86
|
+
export function formatSarif(report) {
|
|
87
|
+
const { ids, byRule } = messageStrings(report.findings);
|
|
88
|
+
const sarif = {
|
|
89
|
+
$schema: SCHEMA,
|
|
90
|
+
version: "2.1.0",
|
|
91
|
+
runs: [
|
|
92
|
+
{
|
|
93
|
+
tool: { driver: { name: "spiderlint", version: VERSION, rules: toRules(report.findings, byRule, report.rules) } },
|
|
94
|
+
invocations: [toInvocation(report.summary)],
|
|
95
|
+
results: report.findings.map((finding) => toResult(finding, ids)),
|
|
96
|
+
},
|
|
97
|
+
],
|
|
98
|
+
};
|
|
99
|
+
return JSON.stringify(sarif, undefined, 2);
|
|
100
|
+
}
|
|
@@ -0,0 +1,46 @@
|
|
|
1
|
+
// SPDX-FileCopyrightText: 2026 Damián Búho <damian.buho@proton.me>
|
|
2
|
+
//
|
|
3
|
+
// SPDX-License-Identifier: AGPL-3.0-only
|
|
4
|
+
import { byRank, flatten } from "../facts/flatten.js";
|
|
5
|
+
import { log } from "../logger.js";
|
|
6
|
+
// Numbers that name something rather than measure it.
|
|
7
|
+
const NOMINAL = new Set(["http.status", "co2.version"]);
|
|
8
|
+
// Numbers a total of means nothing: a header’s configured limit, a position in the graph.
|
|
9
|
+
const UNSUMMED = new Set(["crawl.depth", "graph.depth", "graph.rank"]);
|
|
10
|
+
// Whether `path` measures something a total of is meaningful for.
|
|
11
|
+
function isSummable(path) {
|
|
12
|
+
return !UNSUMMED.has(path) && !path.endsWith(".max-age");
|
|
13
|
+
}
|
|
14
|
+
// Four decimals, enough for grams of CO2e and fractions of a millisecond.
|
|
15
|
+
function rounded(value) {
|
|
16
|
+
return Math.round(value * 10_000) / 10_000;
|
|
17
|
+
}
|
|
18
|
+
// The middle of an ascending list, the mean of the two middles when even.
|
|
19
|
+
export function median(sorted) {
|
|
20
|
+
const middle = sorted.length / 2;
|
|
21
|
+
return sorted.length % 2 === 1 ? sorted[Math.floor(middle)] : (sorted[middle - 1] + sorted[middle]) / 2;
|
|
22
|
+
}
|
|
23
|
+
// The nearest-rank `q` quantile of an ascending list.
|
|
24
|
+
function quantile(sorted, q) {
|
|
25
|
+
return sorted[Math.max(0, Math.ceil(q * sorted.length) - 1)];
|
|
26
|
+
}
|
|
27
|
+
// Count, min, median, p95, max and total of every numeric page fact, CO2, bytes, requests and timings first.
|
|
28
|
+
export function factStats(pages) {
|
|
29
|
+
const values = new Map();
|
|
30
|
+
for (const page of pages) {
|
|
31
|
+
const leaves = Object.entries(flatten(page));
|
|
32
|
+
for (const [path, value] of leaves)
|
|
33
|
+
if (typeof value === "number" && Number.isFinite(value) && !NOMINAL.has(path))
|
|
34
|
+
values
|
|
35
|
+
.set(path, values.get(path) ?? [])
|
|
36
|
+
.get(path)
|
|
37
|
+
?.push(value);
|
|
38
|
+
}
|
|
39
|
+
const paths = values.keys().toArray().toSorted(byRank);
|
|
40
|
+
log.debug({ pages: pages.length, facts: paths.length }, "fact statistics computed");
|
|
41
|
+
return Object.fromEntries(paths.map((path) => {
|
|
42
|
+
const sorted = values.get(path).toSorted((a, b) => a - b);
|
|
43
|
+
const total = sorted.reduce((sum, value) => sum + value, 0);
|
|
44
|
+
return [path, { count: sorted.length, min: sorted[0], median: rounded(median(sorted)), p95: quantile(sorted, 0.95), max: sorted.at(-1), ...(isSummable(path) && { total: rounded(total) }) }];
|
|
45
|
+
}));
|
|
46
|
+
}
|