spiderlint 0.0.0 → 0.143.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (186) hide show
  1. package/LICENSE +235 -0
  2. package/README.md +244 -0
  3. package/bin/spiderlint.js +11 -0
  4. package/dist/agent.js +46 -0
  5. package/dist/cache/extractors.js +37 -0
  6. package/dist/cache/http.js +99 -0
  7. package/dist/cache/index.js +101 -0
  8. package/dist/cache/purge.js +63 -0
  9. package/dist/cache/status.js +52 -0
  10. package/dist/cli.js +602 -0
  11. package/dist/color.js +10 -0
  12. package/dist/config/environment.js +88 -0
  13. package/dist/config/index.js +104 -0
  14. package/dist/config/policy.js +170 -0
  15. package/dist/config/schema.js +127 -0
  16. package/dist/crawl/body.js +92 -0
  17. package/dist/crawl/browser.js +498 -0
  18. package/dist/crawl/crawl.js +58 -0
  19. package/dist/crawl/diversity.js +153 -0
  20. package/dist/crawl/dns.js +194 -0
  21. package/dist/crawl/fetch.js +48 -0
  22. package/dist/crawl/frontier.js +370 -0
  23. package/dist/crawl/guard.js +69 -0
  24. package/dist/crawl/http.js +171 -0
  25. package/dist/crawl/links.js +89 -0
  26. package/dist/crawl/log.js +27 -0
  27. package/dist/crawl/network.js +88 -0
  28. package/dist/crawl/probe.js +107 -0
  29. package/dist/crawl/profile.js +33 -0
  30. package/dist/crawl/rdap.js +86 -0
  31. package/dist/crawl/resolve.js +132 -0
  32. package/dist/crawl/resources.js +233 -0
  33. package/dist/crawl/robots.js +100 -0
  34. package/dist/crawl/route.js +92 -0
  35. package/dist/crawl/scope.js +32 -0
  36. package/dist/crawl/scratch.js +15 -0
  37. package/dist/crawl/sitemap.js +187 -0
  38. package/dist/crawl/svcb.js +59 -0
  39. package/dist/crawl/tls-hello.js +297 -0
  40. package/dist/crawl/tls-probe.js +71 -0
  41. package/dist/facts/browser.js +138 -0
  42. package/dist/facts/byline.js +17 -0
  43. package/dist/facts/co2.js +27 -0
  44. package/dist/facts/csp.js +71 -0
  45. package/dist/facts/export.js +79 -0
  46. package/dist/facts/flatten.js +33 -0
  47. package/dist/facts/graph.js +73 -0
  48. package/dist/facts/headers.js +330 -0
  49. package/dist/facts/html.js +230 -0
  50. package/dist/facts/labels.js +89 -0
  51. package/dist/facts/language.js +32 -0
  52. package/dist/facts/parity.js +38 -0
  53. package/dist/facts/read-note.js +11 -0
  54. package/dist/facts/resources.js +46 -0
  55. package/dist/facts/robots.js +13 -0
  56. package/dist/facts/sites.js +140 -0
  57. package/dist/facts/transport.js +200 -0
  58. package/dist/facts/types.js +4 -0
  59. package/dist/facts/vendors.js +76 -0
  60. package/dist/fold/index.js +94 -0
  61. package/dist/groups/assign.js +45 -0
  62. package/dist/groups/sample.js +60 -0
  63. package/dist/i18n.js +92 -0
  64. package/dist/index.js +607 -0
  65. package/dist/interrupt.js +37 -0
  66. package/dist/logger.js +95 -0
  67. package/dist/plugins/axe.js +104 -0
  68. package/dist/plugins/carbon-txt.js +96 -0
  69. package/dist/plugins/cookies-registry.js +25 -0
  70. package/dist/plugins/cookies.js +168 -0
  71. package/dist/plugins/css.js +328 -0
  72. package/dist/plugins/dns.js +864 -0
  73. package/dist/plugins/feed-content.js +132 -0
  74. package/dist/plugins/feed-model.js +282 -0
  75. package/dist/plugins/feeds.js +836 -0
  76. package/dist/plugins/html-validate.js +191 -0
  77. package/dist/plugins/htmlhint.js +95 -0
  78. package/dist/plugins/icons.js +302 -0
  79. package/dist/plugins/images.js +269 -0
  80. package/dist/plugins/index.js +286 -0
  81. package/dist/plugins/keyboard.js +194 -0
  82. package/dist/plugins/lighthouse.js +110 -0
  83. package/dist/plugins/link-text.js +73 -0
  84. package/dist/plugins/list.js +22 -0
  85. package/dist/plugins/live.js +254 -0
  86. package/dist/plugins/mail.js +985 -0
  87. package/dist/plugins/manifest.js +97 -0
  88. package/dist/plugins/markup.js +98 -0
  89. package/dist/plugins/messages.js +25 -0
  90. package/dist/plugins/network.js +192 -0
  91. package/dist/plugins/origin.js +488 -0
  92. package/dist/plugins/profile-links.js +102 -0
  93. package/dist/plugins/schema-registry.js +101 -0
  94. package/dist/plugins/sshfp.js +159 -0
  95. package/dist/plugins/stack.js +161 -0
  96. package/dist/plugins/structured-data.js +358 -0
  97. package/dist/plugins/tls-probe.js +298 -0
  98. package/dist/plugins/tls-registry.js +418 -0
  99. package/dist/plugins/tls-scan.js +280 -0
  100. package/dist/plugins/trackers-registry.js +27 -0
  101. package/dist/plugins/trackers.js +56 -0
  102. package/dist/plugins/types.js +7 -0
  103. package/dist/plugins/visit.js +50 -0
  104. package/dist/plugins/well-known-registry.js +111 -0
  105. package/dist/plugins/well-known.js +732 -0
  106. package/dist/progress.js +162 -0
  107. package/dist/report/agent.js +122 -0
  108. package/dist/report/checkstyle.js +22 -0
  109. package/dist/report/csv.js +18 -0
  110. package/dist/report/html.js +266 -0
  111. package/dist/report/human.js +220 -0
  112. package/dist/report/index.js +16 -0
  113. package/dist/report/json.js +8 -0
  114. package/dist/report/printable.js +29 -0
  115. package/dist/report/rating.js +26 -0
  116. package/dist/report/sarif.js +100 -0
  117. package/dist/report/stats.js +46 -0
  118. package/dist/rules/builtin.js +465 -0
  119. package/dist/rules/caching.js +103 -0
  120. package/dist/rules/catalog.js +139 -0
  121. package/dist/rules/clock.js +48 -0
  122. package/dist/rules/declarative.js +234 -0
  123. package/dist/rules/deprecated.js +45 -0
  124. package/dist/rules/disclosure.js +99 -0
  125. package/dist/rules/fix.js +21 -0
  126. package/dist/rules/i18n.js +131 -0
  127. package/dist/rules/insights.js +100 -0
  128. package/dist/rules/length.js +52 -0
  129. package/dist/rules/message.js +48 -0
  130. package/dist/rules/relations.js +88 -0
  131. package/dist/rules/robots.js +75 -0
  132. package/dist/rules/rulesets.js +134 -0
  133. package/dist/rules/run.js +75 -0
  134. package/dist/rules/score.js +61 -0
  135. package/dist/rules/types.js +7 -0
  136. package/dist/rules/url.js +90 -0
  137. package/dist/server/analytics.js +49 -0
  138. package/dist/server/api.js +134 -0
  139. package/dist/server/clients.js +54 -0
  140. package/dist/server/clock.js +33 -0
  141. package/dist/server/compress.js +58 -0
  142. package/dist/server/health.js +14 -0
  143. package/dist/server/jobs.js +82 -0
  144. package/dist/server/main.js +83 -0
  145. package/dist/server/page.js +161 -0
  146. package/dist/server/policy.js +145 -0
  147. package/dist/server/providers.js +60 -0
  148. package/dist/server/queue.js +50 -0
  149. package/dist/server/scan.js +39 -0
  150. package/dist/server/settings.js +197 -0
  151. package/dist/server/web.js +353 -0
  152. package/dist/server/worker.js +129 -0
  153. package/dist/store/disk.js +215 -0
  154. package/dist/store/memory.js +18 -0
  155. package/dist/telemetry-environment.js +9 -0
  156. package/dist/telemetry.js +104 -0
  157. package/locales/es/LC_MESSAGES/messages.po +1864 -0
  158. package/locales/uk/LC_MESSAGES/messages.po +1864 -0
  159. package/package.json +134 -2
  160. package/presets/browser.yaml +26 -0
  161. package/presets/graph.yaml +35 -0
  162. package/presets/i18n.yaml +18 -0
  163. package/presets/insights.yaml +12 -0
  164. package/presets/links.yaml +22 -0
  165. package/presets/parity.yaml +59 -0
  166. package/presets/performance.yaml +271 -0
  167. package/presets/recommended.yaml +50 -0
  168. package/presets/redirects.yaml +23 -0
  169. package/presets/resources.yaml +29 -0
  170. package/presets/robots.yaml +13 -0
  171. package/presets/security-headers.yaml +257 -0
  172. package/presets/seo.yaml +380 -0
  173. package/presets/sitemap.yaml +34 -0
  174. package/presets/suggestions.yaml +97 -0
  175. package/presets/sustainability.yaml +16 -0
  176. package/presets/tls.yaml +72 -0
  177. package/presets/vendor.yaml +14 -0
  178. package/presets/web-comprehensive.yaml +17 -0
  179. package/presets/web-quick.yaml +15 -0
  180. package/src/server/apple-touch-icon.png +0 -0
  181. package/src/server/logo-36.png +0 -0
  182. package/src/server/logo-72.png +0 -0
  183. package/src/server/logo.png +0 -0
  184. package/vendors/dns.yaml +21 -0
  185. package/vendors/paths.yaml +55 -0
  186. package/vendors/stack.yaml +57 -0
package/package.json CHANGED
@@ -1,10 +1,142 @@
1
1
  {
2
2
  "name": "spiderlint",
3
- "version": "0.0.0",
4
- "description": "Placeholder reserving the name; real releases are published from CI.",
3
+ "version": "0.143.0",
4
+ "type": "module",
5
+ "description": "Site-wide linter for SEO tags, security headers, TLS and links",
5
6
  "license": "AGPL-3.0-only",
7
+ "engines": {
8
+ "node": ">=26"
9
+ },
10
+ "bin": {
11
+ "spiderlint": "bin/spiderlint.js"
12
+ },
13
+ "files": [
14
+ "bin/",
15
+ "dist/",
16
+ "src/server/*.png",
17
+ "presets/",
18
+ "vendors/",
19
+ "locales/"
20
+ ],
21
+ "scripts": {
22
+ "start": "node --experimental-strip-types src/cli.ts",
23
+ "lint": "eslint ./",
24
+ "fix": "eslint ./ --fix",
25
+ "format": "prettier --write \"src/**/*.ts\" \"tests/**/*.ts\" bin/spiderlint.js eslint.config.js",
26
+ "format:check": "prettier --check \"src/**/*.ts\" \"tests/**/*.ts\" bin/spiderlint.js eslint.config.js",
27
+ "test": "SPIDERLINT_LOG_LEVEL=warn node --test --test-timeout=120000 --experimental-strip-types tests/**/*.test.ts",
28
+ "typecheck": "tsc --noEmit",
29
+ "prepack": "tsc --project tsconfig.build.json",
30
+ "postpack": "rm --recursive --force dist"
31
+ },
32
+ "dependencies": {
33
+ "@axe-core/playwright": "4.13.0",
34
+ "@hono/node-server": "2.1.4",
35
+ "@opentelemetry/api": "1.9.1",
36
+ "@opentelemetry/exporter-metrics-otlp-http": "0.223.0",
37
+ "@opentelemetry/exporter-trace-otlp-http": "0.223.0",
38
+ "@opentelemetry/resources": "2.12.0",
39
+ "@opentelemetry/sdk-metrics": "2.12.0",
40
+ "@opentelemetry/sdk-trace-node": "2.12.0",
41
+ "@tgwf/co2": "0.19.0",
42
+ "ajv": "8.20.0",
43
+ "any-ascii": "0.3.3",
44
+ "axe-core": "4.14.0",
45
+ "browserslist": "4.29.3",
46
+ "bullmq": "6.3.12",
47
+ "caniuse-lite": "1.0.30001815",
48
+ "cheerio": "1.2.0",
49
+ "commander": "15.0.0",
50
+ "crawlee": "3.18.2",
51
+ "css-tree": "3.2.1",
52
+ "csstree-validator": "4.0.1",
53
+ "dns-packet": "5.6.1",
54
+ "doiuse": "6.0.6",
55
+ "eld": "2.1.0",
56
+ "esbuild": "0.28.2",
57
+ "gettext-parser": "9.1.1",
58
+ "hono": "4.13.13",
59
+ "html-validate": "11.16.2",
60
+ "htmlhint": "1.9.2",
61
+ "htmlparser2": "12.0.0",
62
+ "http-cache-semantics": "4.3.0",
63
+ "ioredis": "6.0.0",
64
+ "lighthouse": "13.5.0",
65
+ "lightningcss": "1.33.0",
66
+ "log-update": "8.0.0",
67
+ "marked": "18.1.0",
68
+ "picomatch": "4.0.7",
69
+ "pino": "10.4.0",
70
+ "pino-opentelemetry-transport": "4.0.2",
71
+ "pino-pretty": "13.2.0",
72
+ "playwright": "1.64.0",
73
+ "postcss": "8.5.29",
74
+ "proper-lockfile": "4.1.2",
75
+ "proxy-chain": "3.0.1",
76
+ "saxes": "6.0.0",
77
+ "sharp": "0.35.5",
78
+ "smol-toml": "1.9.1",
79
+ "structured-headers": "2.1.1",
80
+ "svgo": "4.1.0",
81
+ "tldts": "7.4.18",
82
+ "yaml": "2.9.1"
83
+ },
84
+ "devDependencies": {
85
+ "@eslint/js": "10.0.1",
86
+ "@types/caniuse-lite": "1.0.5",
87
+ "@types/css-tree": "3.2.0",
88
+ "@types/dns-packet": "5.6.5",
89
+ "@types/gettext-parser": "9.0.0",
90
+ "@types/http-cache-semantics": "4.2.0",
91
+ "@types/node": "26.6.5",
92
+ "@types/picomatch": "4.0.3",
93
+ "@types/proper-lockfile": "4.1.4",
94
+ "eslint": "10.12.0",
95
+ "eslint-config-prettier": "10.1.8",
96
+ "eslint-plugin-import-x": "4.17.1",
97
+ "eslint-plugin-n": "18.4.1",
98
+ "eslint-plugin-promise": "7.3.0",
99
+ "eslint-plugin-tsdoc": "0.5.4",
100
+ "eslint-plugin-unicorn": "76.0.0",
101
+ "globals": "17.13.0",
102
+ "prettier": "3.9.10",
103
+ "typescript": "6.0.3",
104
+ "typescript-eslint": "8.71.1"
105
+ },
106
+ "keywords": [
107
+ "spiderlint",
108
+ "linter",
109
+ "crawler",
110
+ "seo",
111
+ "security-headers",
112
+ "sarif"
113
+ ],
114
+ "author": {
115
+ "email": "damian.buho@proton.me",
116
+ "name": "Damián Búho",
117
+ "url": "https://dbuho.me"
118
+ },
119
+ "maintainers": [
120
+ {
121
+ "email": "damian.buho@proton.me",
122
+ "name": "Damián Búho",
123
+ "url": "https://dbuho.me"
124
+ }
125
+ ],
6
126
  "repository": {
7
127
  "type": "git",
8
128
  "url": "git+https://github.com/damian-buho/spiderlint.git"
129
+ },
130
+ "os": [
131
+ "linux",
132
+ "darwin"
133
+ ],
134
+ "cpu": [
135
+ "x64",
136
+ "arm64"
137
+ ],
138
+ "homepage": "https://dbuho.me/project/spiderlint/",
139
+ "bugs": {
140
+ "url": "https://github.com/damian-buho/spiderlint/issues"
9
141
  }
10
142
  }
@@ -0,0 +1,26 @@
1
+ # SPDX-FileCopyrightText: 2026 Damián Búho <damian.buho@proton.me>
2
+ #
3
+ # SPDX-License-Identifier: AGPL-3.0-only
4
+ ---
5
+ description: Checks that need a rendered page — console errors and unused preloads
6
+ extends:
7
+ - spiderlint:cookies:browser
8
+ when:
9
+ http.status: {minimum: 200, maximum: 299}
10
+ rules:
11
+ browser/console-errors:
12
+ fact: browser.console.errors
13
+ expect: {maxItems: 0}
14
+ severity: warning
15
+ score: 5.8
16
+ message: "the browser console logged errors ({got})"
17
+ docs: https://developer.mozilla.org/docs/Web/API/console/error_static
18
+ fix: "Fix the JavaScript error logged to the console."
19
+ browser/unused-preload:
20
+ fact: browser.console.warnings
21
+ expect: {not: {contains: {type: string, pattern: "preloaded using link preload but not used"}}}
22
+ severity: info
23
+ score: 1.8
24
+ message: "a preloaded resource went unused ({got})"
25
+ docs: https://developer.mozilla.org/docs/Web/HTML/Attributes/rel/preload
26
+ fix: "Remove the preload link, or use the resource it points at."
@@ -0,0 +1,35 @@
1
+ # SPDX-FileCopyrightText: 2026 Damián Búho <damian.buho@proton.me>
2
+ #
3
+ # SPDX-License-Identifier: AGPL-3.0-only
4
+ ---
5
+ description: Internal linking across the whole crawl, click depth, dead ends and weakly linked pages
6
+ when:
7
+ http.status: {minimum: 200, maximum: 299}
8
+ http.content-type: {pattern: "^text/html"}
9
+ rules:
10
+ links/click-depth:
11
+ fact: graph.depth
12
+ expect: {maximum: 3}
13
+ when: {graph.depth: {type: integer}}
14
+ severity: info
15
+ score: [[4, 1.6], [10, 3.0]]
16
+ message: "a page is {got} links away from the start page, more than 3"
17
+ fix: "Link the page from a hub, a section index or the navigation."
18
+ docs: https://developers.google.com/search/docs/crawling-indexing/links-crawlable
19
+ links/dead-end:
20
+ fact: graph.out-degree
21
+ expect: {minimum: 1}
22
+ severity: info
23
+ score: 1.6
24
+ message: "a page links to no other page of the site"
25
+ fix: "Add navigation or related links back into the site."
26
+ docs: https://developers.google.com/search/docs/crawling-indexing/links-crawlable
27
+ links/weakly-linked:
28
+ fact: graph.in-degree
29
+ expect: {minimum: 2}
30
+ when: {graph.in-degree: {minimum: 1}, crawl.discovered-via: {not: {const: seed}}}
31
+ severity: info
32
+ score: 1.8
33
+ message: "only {got} page of the site links here"
34
+ fix: "Link the page from its section index or related pages."
35
+ docs: https://developers.google.com/search/docs/crawling-indexing/links-crawlable
@@ -0,0 +1,18 @@
1
+ # SPDX-FileCopyrightText: 2026 Damián Búho <damian.buho@proton.me>
2
+ #
3
+ # SPDX-License-Identifier: AGPL-3.0-only
4
+ ---
5
+ description: hreflang alternates that name each other back and answer 2xx, and lang against Content-Language and the language of the title and description
6
+ rules:
7
+ i18n/hreflang-reciprocal:
8
+ severity: warning
9
+ score: 5.2
10
+ i18n/hreflang-status:
11
+ severity: warning
12
+ score: 5.6
13
+ i18n/content-language:
14
+ severity: info
15
+ score: 2.4
16
+ i18n/metadata-language:
17
+ severity: info
18
+ score: 2.8
@@ -0,0 +1,12 @@
1
+ # SPDX-FileCopyrightText: 2026 Damián Búho <damian.buho@proton.me>
2
+ #
3
+ # SPDX-License-Identifier: AGPL-3.0-only
4
+ ---
5
+ description: Pages that stand out from the rest of the site, all info
6
+ rules:
7
+ insight/numeric-outlier:
8
+ severity: info
9
+ score: 3.0
10
+ insight/minority-value:
11
+ severity: info
12
+ score: 2.0
@@ -0,0 +1,22 @@
1
+ # SPDX-FileCopyrightText: 2026 Damián Búho <damian.buho@proton.me>
2
+ #
3
+ # SPDX-License-Identifier: AGPL-3.0-only
4
+ ---
5
+ description: Links that answer an error, internal links that redirect or carry nofollow, rel=me profiles that do not link back
6
+ rules:
7
+ links/broken-internal:
8
+ severity: error
9
+ score: [[0, 8.4], [404, 7.4], [500, 8.4]]
10
+ links/redirected-internal:
11
+ severity: warning
12
+ score: 4.6
13
+ links/broken-external:
14
+ severity: warning
15
+ score: [[0, 6.0], [404, 4.6], [500, 6.0]]
16
+ links/internal-nofollow:
17
+ severity: warning
18
+ score: 4.2
19
+ links/external-rel: off
20
+ links/rel-me:
21
+ severity: info
22
+ score: 2.2
@@ -0,0 +1,59 @@
1
+ # SPDX-FileCopyrightText: 2026 Damián Búho <damian.buho@proton.me>
2
+ #
3
+ # SPDX-License-Identifier: AGPL-3.0-only
4
+ ---
5
+ description: What a page loses without JavaScript, from its raw HTML and its rendered DOM; browser mode only
6
+ when:
7
+ http.status: {minimum: 200, maximum: 299}
8
+ rules:
9
+ parity/title:
10
+ fact: parity.missing
11
+ expect: {properties: {title: false}}
12
+ severity: warning
13
+ score: 6.0
14
+ message: "the title exists only after JavaScript runs: {got}"
15
+ fix: "Render <title> on the server."
16
+ docs: https://developers.google.com/search/docs/crawling-indexing/javascript/javascript-seo-basics
17
+ parity/description:
18
+ fact: parity.missing
19
+ expect: {properties: {description: false}}
20
+ severity: info
21
+ score: 2.6
22
+ message: "the meta description exists only after JavaScript runs: {got}"
23
+ fix: "Render <meta name=\"description\"> on the server."
24
+ docs: https://developers.google.com/search/docs/crawling-indexing/javascript/javascript-seo-basics
25
+ parity/canonical:
26
+ fact: parity.missing
27
+ expect: {properties: {canonical: false}}
28
+ severity: warning
29
+ score: 5.5
30
+ message: "the canonical link exists only after JavaScript runs: {got}"
31
+ fix: "Render <link rel=\"canonical\"> on the server."
32
+ docs: https://developers.google.com/search/docs/crawling-indexing/javascript/javascript-seo-basics#properly-inject-canonical-links
33
+ parity/h1:
34
+ fact: parity.missing
35
+ expect: {properties: {h1: false}}
36
+ severity: info
37
+ score: 2.4
38
+ message: "the h1 exists only after JavaScript runs: {got}"
39
+ fix: "Render the page heading on the server."
40
+ docs: https://developers.google.com/search/docs/crawling-indexing/javascript/javascript-seo-basics
41
+ parity/text:
42
+ fact: parity
43
+ expect: {if: {properties: {rendered: {properties: {text: {minimum: 200}}}}}, then: {properties: {text-share: {minimum: 0.5}}}}
44
+ severity: warning
45
+ score: 5.2
46
+ message: "the raw HTML carries a {got} share of the text the rendered page shows"
47
+ fix: "Render the main content on the server, or give it a <noscript> fallback."
48
+ docs: https://developer.mozilla.org/docs/Glossary/SSR
49
+ parity/links:
50
+ fact: parity.missing
51
+ expect: {properties: {links: false}}
52
+ severity: warning
53
+ score: 5.8
54
+ message: "internal links exist only after JavaScript runs: {got}"
55
+ fix: "Render navigation links as <a href> on the server."
56
+ docs: https://developers.google.com/search/docs/crawling-indexing/links-crawlable
57
+ html/declaration-rewritten:
58
+ severity: info
59
+ score: 1.5
@@ -0,0 +1,271 @@
1
+ # SPDX-FileCopyrightText: 2026 Damián Búho <damian.buho@proton.me>
2
+ #
3
+ # SPDX-License-Identifier: AGPL-3.0-only
4
+ ---
5
+ description: Compression, caching, validators, protocol version and render-blocking HTML, no browser needed
6
+ rules:
7
+ http/compression:
8
+ fact: http.headers.content-encoding
9
+ expect: {type: string, pattern: "^\\s*(br|gzip|zstd)\\s*$"}
10
+ when:
11
+ http.status: {minimum: 200, maximum: 299}
12
+ http.content-type: {pattern: "^(text/|application/([\\w.-]+\\+)?(json|xml|javascript)\\b|image/svg\\+xml)"}
13
+ http.size.decoded: {minimum: 1024}
14
+ severity: warning
15
+ score: 6.3
16
+ message: "a text response over 1 KB should be sent br, zstd or gzip, found {got}"
17
+ fix: "Compress text responses with br, zstd or gzip at the edge or in the server."
18
+ docs: https://developer.mozilla.org/docs/Web/HTTP/Compression
19
+ http/vary-encoding:
20
+ fact: http.parsed.vary.value
21
+ expect: {type: array, contains: {enum: [accept-encoding, "*"]}}
22
+ when: {http.headers.content-encoding: {type: string}, http.parsed.vary.errors: {maxItems: 0}}
23
+ severity: warning
24
+ score: 5.4
25
+ message: "a compressed response should carry Vary: Accept-Encoding, found {got}"
26
+ fix: "Add Vary: Accept-Encoding to the response."
27
+ docs: https://developer.mozilla.org/docs/Web/HTTP/Headers/Vary
28
+ http/vary-star:
29
+ fact: http.parsed.vary.value
30
+ expect: {not: {contains: {const: "*"}}}
31
+ when: {http.parsed.vary.value: {type: array}}
32
+ severity: warning
33
+ score: 5.6
34
+ message: "Vary: * makes the response uncacheable, found {got}"
35
+ fix: "Replace Vary: * with the specific header names the response varies by."
36
+ docs: https://developer.mozilla.org/docs/Web/HTTP/Headers/Vary
37
+ http/validator:
38
+ fact: http.headers
39
+ expect: {anyOf: [{required: [etag]}, {required: [last-modified]}]}
40
+ when: {http.status: {minimum: 200, maximum: 299}}
41
+ severity: info
42
+ score: 2.6
43
+ message: "no ETag or Last-Modified to revalidate against"
44
+ fix: "Add an ETag or Last-Modified header to the response."
45
+ docs: https://developer.mozilla.org/docs/Web/HTTP/Caching#validation
46
+ http/html-cache-control:
47
+ fact: http.parsed.cache-control.value
48
+ expect: {type: object, minProperties: 1}
49
+ when: {http.status: {minimum: 200, maximum: 299}, http.content-type: {pattern: "^text/html"}, http.parsed.cache-control.errors: {maxItems: 0}}
50
+ severity: info
51
+ score: 2.7
52
+ message: "no Cache-Control directive on an HTML page, found {got}"
53
+ fix: "Add a Cache-Control header, e.g. no-cache to revalidate, or public with a max-age."
54
+ docs: https://developer.mozilla.org/docs/Web/HTTP/Headers/Cache-Control
55
+ http/no-store-bfcache:
56
+ fact: http.parsed.cache-control.value
57
+ expect: {not: {required: [no-store]}}
58
+ when: {http.content-type: {pattern: "^text/html"}, http.parsed.cache-control.value: {type: object}}
59
+ severity: info
60
+ score: 2.4
61
+ message: "Cache-Control no-store keeps the page out of the back/forward cache, found {got}"
62
+ fix: "Remove no-store from Cache-Control, or replace it with no-cache."
63
+ docs: https://web.dev/articles/bfcache#minimize-use-of-cache-control-no-store
64
+ caching/long-without-hash:
65
+ severity: warning
66
+ http/version:
67
+ fact: http.version
68
+ expect: {type: string, pattern: "^[23]"}
69
+ when: {http.version: {type: string}}
70
+ severity: warning
71
+ score: 5.8
72
+ message: "the server should speak HTTP/2 or HTTP/3, found {got}"
73
+ fix: "Enable HTTP/2 or HTTP/3 on the server."
74
+ docs: https://developer.mozilla.org/docs/Glossary/HTTP_2
75
+ http/alt-svc-h3:
76
+ fact: http.parsed.alt-svc.value
77
+ expect: {type: array, contains: {properties: {protocol: {const: h3}}, required: [protocol]}}
78
+ when: {url.protocol: "https:", http.version: {not: {const: "3.0"}}, http.parsed.alt-svc.errors: {maxItems: 0}}
79
+ severity: warning
80
+ score: 5.2
81
+ message: "Alt-Svc should advertise h3, found {got}"
82
+ fix: "Add h3 to the Alt-Svc header."
83
+ docs: https://developer.mozilla.org/docs/Web/HTTP/Headers/Alt-Svc
84
+ tls/ec-key:
85
+ fact: tls.cert.key.type
86
+ expect: {not: {const: RSA}}
87
+ when: {url.protocol: "https:", tls.cert.key: {type: object}}
88
+ severity: hint
89
+ score: 0.6
90
+ message: "the certificate carries an RSA key, where an EC key makes a smaller handshake"
91
+ fix: "Issue the certificate for an EC P-256 key."
92
+ docs: https://www.rfc-editor.org/rfc/rfc9325#section-4.5
93
+ http/retry-after:
94
+ fact: http.parsed.retry-after.value
95
+ expect: {type: object}
96
+ when: {http.status: 503, http.parsed.retry-after.errors: {maxItems: 0}}
97
+ severity: warning
98
+ score: 4.8
99
+ message: "a 503 should say when to retry with Retry-After"
100
+ fix: "Send Retry-After with a delay, or an HTTP-date for when to try again."
101
+ docs: https://developer.mozilla.org/docs/Web/HTTP/Headers/Retry-After
102
+ http/sunset-format:
103
+ fact: http.headers.sunset
104
+ expect: {type: string, pattern: "^(Mon|Tue|Wed|Thu|Fri|Sat|Sun), \\d{2} (Jan|Feb|Mar|Apr|May|Jun|Jul|Aug|Sep|Oct|Nov|Dec) \\d{4} \\d{2}:\\d{2}:\\d{2} GMT$"}
105
+ when: {http.headers.sunset: {type: string}}
106
+ severity: warning
107
+ score: 3.7
108
+ message: "Sunset should be an HTTP-date, found {got}"
109
+ fix: "Send Sunset as an HTTP-date, e.g. Sun, 01 Jan 2027 00:00:00 GMT."
110
+ docs: https://www.rfc-editor.org/rfc/rfc8594
111
+ http/deprecation-format:
112
+ fact: http.headers.deprecation
113
+ expect: {type: string, pattern: "^@-?\\d+$"}
114
+ when: {http.headers.deprecation: {type: string}}
115
+ severity: warning
116
+ score: 3.6
117
+ message: "Deprecation should be @ and Unix seconds, found {got}"
118
+ fix: "Send Deprecation: @1704067200 with the Unix timestamp of the removal."
119
+ docs: https://www.rfc-editor.org/rfc/rfc9745
120
+ html/render-blocking-script:
121
+ fact: html.scripts
122
+ expect: {items: {if: {required: [src], properties: {head: {const: true}, async: {const: false}, defer: {const: false}}}, then: {required: [type], properties: {type: {const: module}}}}}
123
+ severity: info
124
+ score: 3.0
125
+ message: "a script in <head> without async, defer or type=module blocks rendering: {got}"
126
+ fix: "Add async, defer, or type=module to the script, or move it to the end of <body>."
127
+ docs: https://developer.mozilla.org/docs/Web/HTML/Element/script#defer
128
+ html/first-img-lazy:
129
+ fact: html.images.0
130
+ expect: {not: {type: object, required: [loading], properties: {loading: {pattern: "^(?i:lazy)$"}}}}
131
+ severity: info
132
+ score: 2.9
133
+ message: "the first <img> is usually above the fold, and loading=lazy delays it: {got}"
134
+ fix: "Remove loading=lazy from the first <img>, or set it to eager."
135
+ docs: https://web.dev/articles/lcp-lazy-loading
136
+ html/img-dimensions:
137
+ fact: html.images
138
+ expect: {items: {if: {not: {required: [noscript]}}, then: {required: [width, height]}}}
139
+ severity: warning
140
+ score: 6.0
141
+ message: "an <img> without width and height shifts the layout as it loads: {got}"
142
+ fix: "Add width and height attributes to the <img>."
143
+ docs: https://web.dev/articles/optimize-cls#images-without-dimensions
144
+ http/early-hints-preload:
145
+ severity: info
146
+ score: 1.8
147
+ html/preconnect-unused:
148
+ severity: info
149
+ score: 1.4
150
+ html/preconnect-missing:
151
+ severity: info
152
+ score: 2.2
153
+ html/preconnect-crossorigin:
154
+ severity: info
155
+ score: 1.6
156
+ http/link-syntax:
157
+ fact: http.parsed.link.errors
158
+ expect: {maxItems: 0}
159
+ when: {http.parsed.link: {type: object}}
160
+ severity: warning
161
+ score: 4.6
162
+ message: "Link breaks RFC 8288 §3, so clients drop the links they cannot read: {got}"
163
+ fix: "Write each Link entry as <https://example.com/a>; rel=preload, separated by commas."
164
+ docs: https://www.rfc-editor.org/rfc/rfc8288#section-3
165
+ http/dictionary-format:
166
+ fact: http.parsed.use-as-dictionary.value
167
+ expect: {required: [match], properties: {match: {type: string}}}
168
+ when: {http.parsed.use-as-dictionary.value: {type: object}}
169
+ severity: warning
170
+ score: 3.5
171
+ message: "Use-As-Dictionary needs a match=\"<URL pattern>\" member, found {got}"
172
+ fix: "Send Use-As-Dictionary: match=\"/assets/app-*.js\"."
173
+ docs: https://www.rfc-editor.org/rfc/rfc9842#section-2.1
174
+ html/render-blocking-css:
175
+ fact: html.head.links
176
+ expect: {minContains: 0, maxContains: 1, contains: {required: [rel], properties: {rel: {pattern: "^\\s*(?i:stylesheet)\\s*$"}, media: {pattern: "^\\s*(?i:all|screen)?\\s*$"}}}}
177
+ severity: info
178
+ score: 2.3
179
+ message: "more than one stylesheet in <head> blocks rendering: {got}"
180
+ fix: "Inline the critical CSS and load the rest with media=print onload, or merge the stylesheets into one."
181
+ docs: https://web.dev/articles/defer-non-critical-css
182
+ http/cache-control-syntax:
183
+ fact: http.parsed.cache-control.errors
184
+ expect: {maxItems: 0}
185
+ when: {http.parsed.cache-control: {type: object}}
186
+ severity: warning
187
+ score: 5.0
188
+ message: "Cache-Control breaks RFC 9111 §5.2, and caches ignore what they cannot read: {got}"
189
+ fix: "Write Cache-Control with registered directives only, as in public, max-age=3600."
190
+ docs: https://www.rfc-editor.org/rfc/rfc9111#section-5.2
191
+ http/vary-syntax:
192
+ fact: http.parsed.vary.errors
193
+ expect: {maxItems: 0}
194
+ when: {http.parsed.vary: {type: object}}
195
+ severity: warning
196
+ score: 4.2
197
+ message: "Vary breaks RFC 9110 §12.5.5: {got}"
198
+ fix: "List the request header names the response varies by, as in Vary: Accept-Encoding."
199
+ docs: https://www.rfc-editor.org/rfc/rfc9110#section-12.5.5
200
+ http/content-type-syntax:
201
+ fact: http.parsed.content-type.errors
202
+ expect: {maxItems: 0}
203
+ when: {http.parsed.content-type: {type: object}}
204
+ severity: warning
205
+ score: 4.0
206
+ message: "Content-Type breaks RFC 9110 §8.3.1: {got}"
207
+ fix: "Send Content-Type as type/subtype with name=value parameters, as in text/html; charset=utf-8."
208
+ docs: https://www.rfc-editor.org/rfc/rfc9110#section-8.3.1
209
+ http/use-as-dictionary-syntax:
210
+ fact: http.parsed.use-as-dictionary.errors
211
+ expect: {maxItems: 0}
212
+ when: {http.parsed.use-as-dictionary: {type: object}}
213
+ severity: warning
214
+ score: 3.4
215
+ message: "Use-As-Dictionary breaks its structured-field grammar, so clients ignore it: {got}"
216
+ fix: "Send Use-As-Dictionary: match=\"/assets/app-*.js\"."
217
+ docs: https://www.rfc-editor.org/rfc/rfc9842#section-2.1
218
+ http/accept-ch-syntax:
219
+ fact: http.parsed.accept-ch.errors
220
+ expect: {maxItems: 0}
221
+ when: {http.parsed.accept-ch: {type: object}}
222
+ severity: warning
223
+ score: 3.9
224
+ message: "Accept-CH breaks its structured-field grammar, so browsers ignore it: {got}"
225
+ fix: "Send Accept-CH as a list of hint names, as in Accept-CH: Sec-CH-UA-Platform."
226
+ docs: https://www.rfc-editor.org/rfc/rfc8942#section-3.1
227
+ http/priority-syntax:
228
+ fact: http.parsed.priority.errors
229
+ expect: {maxItems: 0}
230
+ when: {http.parsed.priority: {type: object}}
231
+ severity: warning
232
+ score: 3.8
233
+ message: "Priority breaks its structured-field grammar, so clients ignore it: {got}"
234
+ fix: "Send Priority as a dictionary, as in Priority: u=1, i."
235
+ docs: https://www.rfc-editor.org/rfc/rfc9218#section-5
236
+ http/cache-status-syntax:
237
+ fact: http.parsed.cache-status.errors
238
+ expect: {maxItems: 0}
239
+ when: {http.parsed.cache-status: {type: object}}
240
+ severity: warning
241
+ score: 3.3
242
+ message: "Cache-Status breaks its structured-field grammar: {got}"
243
+ fix: "Send Cache-Status as a list of caches with parameters, as in Cache-Status: ExampleCache; hit."
244
+ docs: https://www.rfc-editor.org/rfc/rfc9211#section-2
245
+ http/retry-after-short:
246
+ fact: http.parsed.retry-after.value.seconds
247
+ expect: {minimum: 10}
248
+ when: {http.status: {enum: [429, 503]}, http.parsed.retry-after.value: {type: object}}
249
+ severity: info
250
+ score: [[0, 3.0], [9, 1.2]]
251
+ message: "Retry-After asks clients back in {got} seconds, which brings the load straight back to a server that is shedding it"
252
+ fix: "Send Retry-After: 120, or as long as the server needs to recover."
253
+ docs: https://www.rfc-editor.org/rfc/rfc9110#section-10.2.3
254
+ http/retry-after-syntax:
255
+ fact: http.parsed.retry-after.errors
256
+ expect: {maxItems: 0}
257
+ when: {http.parsed.retry-after: {type: object}}
258
+ severity: warning
259
+ score: 4.1
260
+ message: "Retry-After breaks RFC 9110 §10.2.3, so clients cannot tell when to retry: {got}"
261
+ fix: "Send Retry-After as a number of seconds, as in Retry-After: 120, or an IMF-fixdate."
262
+ docs: https://www.rfc-editor.org/rfc/rfc9110#section-10.2.3
263
+ http/alt-svc-syntax:
264
+ fact: http.parsed.alt-svc.errors
265
+ expect: {maxItems: 0}
266
+ when: {http.parsed.alt-svc: {type: object}}
267
+ severity: warning
268
+ score: 4.4
269
+ message: "Alt-Svc breaks RFC 7838 §3, so clients never try the alternative: {got}"
270
+ fix: "Send Alt-Svc as h3=\":443\"; ma=86400."
271
+ docs: https://www.rfc-editor.org/rfc/rfc7838#section-3
@@ -0,0 +1,50 @@
1
+ # SPDX-FileCopyrightText: 2026 Damián Búho <damian.buho@proton.me>
2
+ #
3
+ # SPDX-License-Identifier: AGPL-3.0-only
4
+ ---
5
+ description: The default ruleset of every group — all presets except browser
6
+ extends:
7
+ - spiderlint:seo
8
+ - spiderlint:security-headers
9
+ - spiderlint:performance
10
+ - spiderlint:links
11
+ - spiderlint:tls
12
+ - spiderlint:cookies
13
+ - spiderlint:redirects
14
+ - spiderlint:sitemap
15
+ - spiderlint:robots
16
+ - spiderlint:resources
17
+ - spiderlint:origin
18
+ - spiderlint:dns:recommended
19
+ - spiderlint:mail:recommended
20
+ - spiderlint:images
21
+ - spiderlint:icons
22
+ - spiderlint:insights
23
+ rules:
24
+ http/consistent-origin:
25
+ severity: info
26
+ score: 2.8
27
+ icons/favicon: warning
28
+ icons/apple-touch: warning
29
+ icons/declared-size: warning
30
+ icons/svg: off
31
+ icons/mask-icon: off
32
+ icons/ms-tile: off
33
+ manifest/parse:
34
+ severity: error
35
+ score: 7.2
36
+ origin/www-resolves: info
37
+ http/clock-skew:
38
+ severity: warning
39
+ score: [[-3600, 6.8], [-60, 3.4], [60, 3.4], [3600, 6.8]]
40
+ http/content-length:
41
+ severity: warning
42
+ score: 5.4
43
+ http/date:
44
+ fact: http.headers.date
45
+ expect: {type: string, pattern: "^(Mon|Tue|Wed|Thu|Fri|Sat|Sun), \\d{2} (Jan|Feb|Mar|Apr|May|Jun|Jul|Aug|Sep|Oct|Nov|Dec) \\d{4} \\d{2}:\\d{2}:\\d{2} GMT$"}
46
+ severity: warning
47
+ score: 4.2
48
+ message: "Date should be an IMF-fixdate, found {got}"
49
+ fix: "Send Date in IMF-fixdate form, as Sun, 06 Nov 1994 08:49:37 GMT."
50
+ docs: https://www.rfc-editor.org/rfc/rfc9110#section-5.6.7
@@ -0,0 +1,23 @@
1
+ # SPDX-FileCopyrightText: 2026 Damián Búho <damian.buho@proton.me>
2
+ #
3
+ # SPDX-License-Identifier: AGPL-3.0-only
4
+ ---
5
+ description: Redirect chain length and permanent redirects
6
+ rules:
7
+ redirects/chain-length:
8
+ fact: http.redirects
9
+ expect: {type: array, maxItems: 2}
10
+ severity: warning
11
+ score: 4.8
12
+ message: "takes more than 2 redirects to land ({got})"
13
+ fix: "Collapse the redirect chain so the final URL is reached in one or two hops."
14
+ docs: https://developers.google.com/search/docs/crawling-indexing/301-redirects
15
+ redirects/permanent:
16
+ fact: http.redirects
17
+ expect: {type: array, items: {properties: {status: {enum: [301, 308]}}}}
18
+ when: {http.status: {minimum: 200, maximum: 299}}
19
+ severity: info
20
+ score: 2.4
21
+ message: "reached through a temporary redirect ({got})"
22
+ fix: "Answer a moved page with 301 or 308, not 302, 303 or 307."
23
+ docs: https://developers.google.com/search/docs/crawling-indexing/301-redirects#permanent-server-side