spiderlint 0.0.0 → 0.143.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (186) hide show
  1. package/LICENSE +235 -0
  2. package/README.md +244 -0
  3. package/bin/spiderlint.js +11 -0
  4. package/dist/agent.js +46 -0
  5. package/dist/cache/extractors.js +37 -0
  6. package/dist/cache/http.js +99 -0
  7. package/dist/cache/index.js +101 -0
  8. package/dist/cache/purge.js +63 -0
  9. package/dist/cache/status.js +52 -0
  10. package/dist/cli.js +602 -0
  11. package/dist/color.js +10 -0
  12. package/dist/config/environment.js +88 -0
  13. package/dist/config/index.js +104 -0
  14. package/dist/config/policy.js +170 -0
  15. package/dist/config/schema.js +127 -0
  16. package/dist/crawl/body.js +92 -0
  17. package/dist/crawl/browser.js +498 -0
  18. package/dist/crawl/crawl.js +58 -0
  19. package/dist/crawl/diversity.js +153 -0
  20. package/dist/crawl/dns.js +194 -0
  21. package/dist/crawl/fetch.js +48 -0
  22. package/dist/crawl/frontier.js +370 -0
  23. package/dist/crawl/guard.js +69 -0
  24. package/dist/crawl/http.js +171 -0
  25. package/dist/crawl/links.js +89 -0
  26. package/dist/crawl/log.js +27 -0
  27. package/dist/crawl/network.js +88 -0
  28. package/dist/crawl/probe.js +107 -0
  29. package/dist/crawl/profile.js +33 -0
  30. package/dist/crawl/rdap.js +86 -0
  31. package/dist/crawl/resolve.js +132 -0
  32. package/dist/crawl/resources.js +233 -0
  33. package/dist/crawl/robots.js +100 -0
  34. package/dist/crawl/route.js +92 -0
  35. package/dist/crawl/scope.js +32 -0
  36. package/dist/crawl/scratch.js +15 -0
  37. package/dist/crawl/sitemap.js +187 -0
  38. package/dist/crawl/svcb.js +59 -0
  39. package/dist/crawl/tls-hello.js +297 -0
  40. package/dist/crawl/tls-probe.js +71 -0
  41. package/dist/facts/browser.js +138 -0
  42. package/dist/facts/byline.js +17 -0
  43. package/dist/facts/co2.js +27 -0
  44. package/dist/facts/csp.js +71 -0
  45. package/dist/facts/export.js +79 -0
  46. package/dist/facts/flatten.js +33 -0
  47. package/dist/facts/graph.js +73 -0
  48. package/dist/facts/headers.js +330 -0
  49. package/dist/facts/html.js +230 -0
  50. package/dist/facts/labels.js +89 -0
  51. package/dist/facts/language.js +32 -0
  52. package/dist/facts/parity.js +38 -0
  53. package/dist/facts/read-note.js +11 -0
  54. package/dist/facts/resources.js +46 -0
  55. package/dist/facts/robots.js +13 -0
  56. package/dist/facts/sites.js +140 -0
  57. package/dist/facts/transport.js +200 -0
  58. package/dist/facts/types.js +4 -0
  59. package/dist/facts/vendors.js +76 -0
  60. package/dist/fold/index.js +94 -0
  61. package/dist/groups/assign.js +45 -0
  62. package/dist/groups/sample.js +60 -0
  63. package/dist/i18n.js +92 -0
  64. package/dist/index.js +607 -0
  65. package/dist/interrupt.js +37 -0
  66. package/dist/logger.js +95 -0
  67. package/dist/plugins/axe.js +104 -0
  68. package/dist/plugins/carbon-txt.js +96 -0
  69. package/dist/plugins/cookies-registry.js +25 -0
  70. package/dist/plugins/cookies.js +168 -0
  71. package/dist/plugins/css.js +328 -0
  72. package/dist/plugins/dns.js +864 -0
  73. package/dist/plugins/feed-content.js +132 -0
  74. package/dist/plugins/feed-model.js +282 -0
  75. package/dist/plugins/feeds.js +836 -0
  76. package/dist/plugins/html-validate.js +191 -0
  77. package/dist/plugins/htmlhint.js +95 -0
  78. package/dist/plugins/icons.js +302 -0
  79. package/dist/plugins/images.js +269 -0
  80. package/dist/plugins/index.js +286 -0
  81. package/dist/plugins/keyboard.js +194 -0
  82. package/dist/plugins/lighthouse.js +110 -0
  83. package/dist/plugins/link-text.js +73 -0
  84. package/dist/plugins/list.js +22 -0
  85. package/dist/plugins/live.js +254 -0
  86. package/dist/plugins/mail.js +985 -0
  87. package/dist/plugins/manifest.js +97 -0
  88. package/dist/plugins/markup.js +98 -0
  89. package/dist/plugins/messages.js +25 -0
  90. package/dist/plugins/network.js +192 -0
  91. package/dist/plugins/origin.js +488 -0
  92. package/dist/plugins/profile-links.js +102 -0
  93. package/dist/plugins/schema-registry.js +101 -0
  94. package/dist/plugins/sshfp.js +159 -0
  95. package/dist/plugins/stack.js +161 -0
  96. package/dist/plugins/structured-data.js +358 -0
  97. package/dist/plugins/tls-probe.js +298 -0
  98. package/dist/plugins/tls-registry.js +418 -0
  99. package/dist/plugins/tls-scan.js +280 -0
  100. package/dist/plugins/trackers-registry.js +27 -0
  101. package/dist/plugins/trackers.js +56 -0
  102. package/dist/plugins/types.js +7 -0
  103. package/dist/plugins/visit.js +50 -0
  104. package/dist/plugins/well-known-registry.js +111 -0
  105. package/dist/plugins/well-known.js +732 -0
  106. package/dist/progress.js +162 -0
  107. package/dist/report/agent.js +122 -0
  108. package/dist/report/checkstyle.js +22 -0
  109. package/dist/report/csv.js +18 -0
  110. package/dist/report/html.js +266 -0
  111. package/dist/report/human.js +220 -0
  112. package/dist/report/index.js +16 -0
  113. package/dist/report/json.js +8 -0
  114. package/dist/report/printable.js +29 -0
  115. package/dist/report/rating.js +26 -0
  116. package/dist/report/sarif.js +100 -0
  117. package/dist/report/stats.js +46 -0
  118. package/dist/rules/builtin.js +465 -0
  119. package/dist/rules/caching.js +103 -0
  120. package/dist/rules/catalog.js +139 -0
  121. package/dist/rules/clock.js +48 -0
  122. package/dist/rules/declarative.js +234 -0
  123. package/dist/rules/deprecated.js +45 -0
  124. package/dist/rules/disclosure.js +99 -0
  125. package/dist/rules/fix.js +21 -0
  126. package/dist/rules/i18n.js +131 -0
  127. package/dist/rules/insights.js +100 -0
  128. package/dist/rules/length.js +52 -0
  129. package/dist/rules/message.js +48 -0
  130. package/dist/rules/relations.js +88 -0
  131. package/dist/rules/robots.js +75 -0
  132. package/dist/rules/rulesets.js +134 -0
  133. package/dist/rules/run.js +75 -0
  134. package/dist/rules/score.js +61 -0
  135. package/dist/rules/types.js +7 -0
  136. package/dist/rules/url.js +90 -0
  137. package/dist/server/analytics.js +49 -0
  138. package/dist/server/api.js +134 -0
  139. package/dist/server/clients.js +54 -0
  140. package/dist/server/clock.js +33 -0
  141. package/dist/server/compress.js +58 -0
  142. package/dist/server/health.js +14 -0
  143. package/dist/server/jobs.js +82 -0
  144. package/dist/server/main.js +83 -0
  145. package/dist/server/page.js +161 -0
  146. package/dist/server/policy.js +145 -0
  147. package/dist/server/providers.js +60 -0
  148. package/dist/server/queue.js +50 -0
  149. package/dist/server/scan.js +39 -0
  150. package/dist/server/settings.js +197 -0
  151. package/dist/server/web.js +353 -0
  152. package/dist/server/worker.js +129 -0
  153. package/dist/store/disk.js +215 -0
  154. package/dist/store/memory.js +18 -0
  155. package/dist/telemetry-environment.js +9 -0
  156. package/dist/telemetry.js +104 -0
  157. package/locales/es/LC_MESSAGES/messages.po +1864 -0
  158. package/locales/uk/LC_MESSAGES/messages.po +1864 -0
  159. package/package.json +134 -2
  160. package/presets/browser.yaml +26 -0
  161. package/presets/graph.yaml +35 -0
  162. package/presets/i18n.yaml +18 -0
  163. package/presets/insights.yaml +12 -0
  164. package/presets/links.yaml +22 -0
  165. package/presets/parity.yaml +59 -0
  166. package/presets/performance.yaml +271 -0
  167. package/presets/recommended.yaml +50 -0
  168. package/presets/redirects.yaml +23 -0
  169. package/presets/resources.yaml +29 -0
  170. package/presets/robots.yaml +13 -0
  171. package/presets/security-headers.yaml +257 -0
  172. package/presets/seo.yaml +380 -0
  173. package/presets/sitemap.yaml +34 -0
  174. package/presets/suggestions.yaml +97 -0
  175. package/presets/sustainability.yaml +16 -0
  176. package/presets/tls.yaml +72 -0
  177. package/presets/vendor.yaml +14 -0
  178. package/presets/web-comprehensive.yaml +17 -0
  179. package/presets/web-quick.yaml +15 -0
  180. package/src/server/apple-touch-icon.png +0 -0
  181. package/src/server/logo-36.png +0 -0
  182. package/src/server/logo-72.png +0 -0
  183. package/src/server/logo.png +0 -0
  184. package/vendors/dns.yaml +21 -0
  185. package/vendors/paths.yaml +55 -0
  186. package/vendors/stack.yaml +57 -0
@@ -0,0 +1,380 @@
1
+ # SPDX-FileCopyrightText: 2026 Damián Búho <damian.buho@proton.me>
2
+ #
3
+ # SPDX-License-Identifier: AGPL-3.0-only
4
+ ---
5
+ description: Titles, descriptions, headings, canonical, Open Graph and alt text
6
+ when:
7
+ http.status: {minimum: 200, maximum: 299}
8
+ rules:
9
+ html/lang:
10
+ fact: html.lang
11
+ expect: {type: string, pattern: "^[A-Za-z]{2,3}(-[A-Za-z0-9]{1,8})*$"}
12
+ severity: error
13
+ score: 8.2
14
+ message: "<html lang> should be a BCP 47 language tag, found {got}"
15
+ docs: https://developer.mozilla.org/docs/Web/HTML/Global_attributes/lang
16
+ fix: "Set <html lang> to a BCP 47 language tag."
17
+ html/viewport:
18
+ fact: html.meta.viewport
19
+ expect: {type: string, pattern: "^(?!.*(?i:user-scalable)\\s*=\\s*(?i:no|0)\\b)(?!.*(?i:maximum-scale)\\s*=\\s*1(\\.0+)?(?![\\d.]))(?=.*(?i:width\\s*=\\s*device-width))"}
20
+ severity: warning
21
+ score: 5.3
22
+ message: "meta viewport should set width=device-width and allow zoom, found {got}"
23
+ docs: https://developer.mozilla.org/docs/Web/HTML/Viewport_meta_tag
24
+ fix: "Set <meta name=viewport content=\"width=device-width\"> and allow zoom."
25
+ html/viewport-syntax:
26
+ severity: info
27
+ score: 1.3
28
+ html/theme-color:
29
+ fact: html.meta.theme-color
30
+ expect: {type: string, minLength: 1}
31
+ severity: warning
32
+ score: 3.3
33
+ message: "no meta theme-color"
34
+ docs: https://developer.mozilla.org/docs/Web/HTML/Element/meta/name/theme-color
35
+ fix: "Add <meta name=theme-color> with a color value."
36
+ html/theme-color-syntax:
37
+ severity: warning
38
+ score: 3.3
39
+ html/color-scheme:
40
+ fact: html.meta.color-scheme
41
+ expect: {type: string, pattern: "^\\s*(?:(?i:normal)|(?:(?i:only)\\s+)?(?!(?i:normal|only)\\b)[A-Za-z_-][\\w-]*(?:\\s+(?!(?i:normal|only)\\b)[A-Za-z_-][\\w-]*)*(?:\\s+(?i:only))?)\\s*$"}
42
+ severity: info
43
+ score: 1.4
44
+ message: "meta color-scheme should be normal, or scheme names such as light dark, found {got}"
45
+ docs: https://developer.mozilla.org/docs/Web/HTML/Element/meta/name/color-scheme
46
+ fix: "Add <meta name=color-scheme content=\"light dark\">."
47
+ html/theme-color-schemes:
48
+ when: {html.meta.color-scheme: {type: string, allOf: [{pattern: "(?i:\\blight\\b)"}, {pattern: "(?i:\\bdark\\b)"}]}}
49
+ fact: html.metas
50
+ expect:
51
+ allOf:
52
+ - {contains: {properties: {name: {const: theme-color}}, required: [name]}, minContains: 2}
53
+ - {contains: {properties: {name: {const: theme-color}, media: {pattern: "(?i:prefers-color-scheme)"}}, required: [name, media]}}
54
+ severity: warning
55
+ score: 3.4
56
+ message: "color-scheme names light and dark, but no theme-color pair carries a prefers-color-scheme media"
57
+ fix: "Add <meta name=theme-color media=\"(prefers-color-scheme: dark)\"> beside the light one."
58
+ docs: https://developer.mozilla.org/docs/Web/HTML/Element/meta/name/theme-color#media
59
+ html/noindex-listed:
60
+ fact: robots.noindex
61
+ expect: {const: false}
62
+ when: {sitemap.listed: true, url.twin: {not: {type: string}}, site.role: production}
63
+ severity: error
64
+ score: 8.0
65
+ message: "a page the sitemap lists says noindex, in <meta name=robots> or X-Robots-Tag"
66
+ docs: https://developers.google.com/search/docs/crawling-indexing/block-indexing
67
+ fix: "Remove noindex from <meta name=robots> or X-Robots-Tag, or unlist the page from the sitemap."
68
+ html/dir-rtl:
69
+ fact: html.dir
70
+ expect: {const: rtl}
71
+ when: {html.lang: {type: string, pattern: "^(?i:ar|arc|ckb|dv|fa|he|ps|sd|ug|ur|yi)(-|$)"}}
72
+ severity: warning
73
+ score: 3.8
74
+ message: "a right-to-left language should set <html dir=rtl>, found {got}"
75
+ docs: https://www.w3.org/International/questions/qa-html-dir
76
+ fix: "Add dir=rtl to the <html> element."
77
+ html/charset:
78
+ fact: html.charset
79
+ expect: {type: object, required: [declared, offset], properties: {declared: {pattern: "^(?i:utf-?8)$"}, offset: {maximum: 1024}}}
80
+ severity: warning
81
+ score: 5.2
82
+ message: "<meta charset> should declare UTF-8 within the first 1024 bytes, found {got}"
83
+ docs: https://html.spec.whatwg.org/multipage/semantics.html#charset
84
+ fix: "Add <meta charset=utf-8> as the first element of <head>."
85
+ html/privacy-policy:
86
+ fact: html.rels.privacy-policy
87
+ expect: {type: array, minItems: 1}
88
+ severity: info
89
+ score: 1.6
90
+ message: "no link with rel=privacy-policy"
91
+ docs: https://html.spec.whatwg.org/multipage/links.html#link-type-privacy-policy
92
+ fix: "Add <link rel=privacy-policy href=/privacy> to the head."
93
+ html/favicon:
94
+ fact: html.head.links
95
+ expect: {contains: {required: [rel], properties: {rel: {pattern: "(?i:(^|\\s)icon(\\s|$))"}}}}
96
+ severity: info
97
+ score: 2.2
98
+ message: "no <link rel=icon> in the head"
99
+ docs: https://developer.mozilla.org/docs/Web/HTML/Attributes/rel#icon
100
+ fix: "Add <link rel=icon href=/favicon.ico> to the head."
101
+ html/hreflang-x-default:
102
+ fact: html.hreflang
103
+ expect: {contains: {required: [lang], properties: {lang: {const: x-default}}}}
104
+ when: {html.hreflang: {minItems: 1}}
105
+ severity: info
106
+ score: 2.6
107
+ message: "hreflang alternates should include x-default, found {got}"
108
+ docs: https://developers.google.com/search/docs/specialty/international/localized-versions#xdefault
109
+ fix: "Add <link rel=alternate hreflang=x-default> to the head."
110
+ html/jsonld-parses:
111
+ fact: html.jsonld
112
+ expect: {items: {not: {required: ["@error"]}}}
113
+ severity: warning
114
+ score: 4.5
115
+ message: "a JSON-LD block does not parse: {got}"
116
+ docs: https://json-ld.org/spec/latest/json-ld/
117
+ fix: "Fix the JSON-LD block so it is valid JSON."
118
+ url/shape:
119
+ fact: url.pathname
120
+ expect: {type: string, pattern: "^(?!.*//)(%[0-9A-Fa-f]{2}|[^A-Z_%])*$"}
121
+ severity: info
122
+ score: 2.8
123
+ message: "URL path should be lowercase, with hyphens and no empty segment, found {got}"
124
+ docs: https://developers.google.com/search/docs/crawling-indexing/url-structure
125
+ fix: "Rewrite the path to be lowercase, with hyphens and no empty segments."
126
+ html/title:
127
+ fact: html.title
128
+ expect: {type: string}
129
+ severity: error
130
+ score: 7.8
131
+ message: "no <title>"
132
+ docs: https://developers.google.com/search/docs/appearance/title-link
133
+ fix: "Add a <title> to the head."
134
+ html/description:
135
+ fact: html.meta.description
136
+ expect: {type: string}
137
+ severity: warning
138
+ score: 6.0
139
+ message: "no meta description"
140
+ docs: https://developers.google.com/search/docs/appearance/snippet
141
+ fix: "Add <meta name=description> with a summary to the head."
142
+ html/title-length:
143
+ when: {html.title: {type: string}}
144
+ fact: html.title
145
+ expect: {type: string, minLength: 30, maxLength: 60}
146
+ severity: warning
147
+ score: 5.7
148
+ message: "<title> should be 30–60 characters, found {got}"
149
+ docs: https://developers.google.com/search/docs/appearance/title-link
150
+ fix: "Shorten or lengthen <title> to 30–60 characters."
151
+ html/description-length:
152
+ when: {html.meta.description: {type: string}}
153
+ fact: html.meta.description
154
+ expect: {type: string, minLength: 50, maxLength: 160}
155
+ severity: warning
156
+ score: 5.5
157
+ message: "meta description should be 50–160 characters, found {got}"
158
+ docs: https://developers.google.com/search/docs/appearance/snippet
159
+ fix: "Shorten or lengthen the meta description to 50–160 characters."
160
+ html/title-redundant:
161
+ when: {html.title: {type: string}}
162
+ fact: html.title
163
+ expect: {type: string, pattern: "^(?!.*(.{2,120}?)\\s*[-|:–—·»]\\s*\\1$).*$"}
164
+ severity: warning
165
+ score: 4.8
166
+ message: "<title> repeats itself around a separator: {got}"
167
+ docs: https://developers.google.com/search/docs/appearance/title-link
168
+ fix: "Remove the repeated suffix from <title>."
169
+ html/description-redundant:
170
+ when: {html.meta.description: {type: string}}
171
+ fact: html.meta.description
172
+ expect: {type: string, pattern: "^(?!.*(.{2,120}?)\\s*[-|:–—·»]\\s*\\1$).*$"}
173
+ severity: warning
174
+ score: 4.7
175
+ message: "meta description repeats itself around a separator: {got}"
176
+ docs: https://developers.google.com/search/docs/appearance/snippet
177
+ fix: "Remove the repeated suffix from the meta description."
178
+ html/one-h1:
179
+ fact: html.h1
180
+ expect: {type: array, minItems: 1, maxItems: 1}
181
+ severity: error
182
+ score: 7.4
183
+ message: "should have exactly one <h1>, found {got}"
184
+ docs: https://developer.mozilla.org/docs/Web/HTML/Reference/Elements/Heading_Elements
185
+ fix: "Use exactly one <h1> per page."
186
+ html/canonical:
187
+ fact: html.canonical
188
+ expect: {type: string, pattern: "^https?://"}
189
+ severity: warning
190
+ score: 5.4
191
+ message: "canonical link should be an absolute URL, found {got}"
192
+ docs: https://developers.google.com/search/docs/crawling-indexing/consolidate-duplicate-urls
193
+ fix: "Make the canonical link an absolute URL."
194
+ html/canonical-self:
195
+ severity: warning
196
+ score: 6.4
197
+ html/og-url-self:
198
+ severity: info
199
+ score: 2.4
200
+ html/img-alt:
201
+ fact: html.images
202
+ expect: {type: array, items: {required: [alt]}}
203
+ severity: warning
204
+ score: 5.0
205
+ message: "an <img> has no alt attribute: {got}"
206
+ docs: https://developers.google.com/search/docs/appearance/google-images
207
+ fix: "Add an alt attribute to every <img>."
208
+ html/og-title:
209
+ fact: html.property.og:title
210
+ expect: {type: string, minLength: 10, maxLength: 90}
211
+ severity: warning
212
+ score: 4.0
213
+ message: "og:title should be 10–90 characters, found {got}"
214
+ docs: https://ogp.me/#metadata
215
+ fix: "Set og:title to 10–90 characters."
216
+ html/og-description:
217
+ fact: html.property.og:description
218
+ expect: {type: string, minLength: 50, maxLength: 200}
219
+ severity: warning
220
+ score: 3.9
221
+ message: "og:description should be 50–200 characters, found {got}"
222
+ docs: https://ogp.me/#optional
223
+ fix: "Set og:description to 50–200 characters."
224
+ html/og-image:
225
+ fact: html.property.og:image
226
+ expect: {type: string, format: uri, pattern: "^(?i:https?)://"}
227
+ severity: warning
228
+ score: 4.6
229
+ message: "og:image should be an absolute http(s) URL, found {got}"
230
+ docs: https://ogp.me/#metadata
231
+ fix: "Make og:image an absolute URL."
232
+ html/og-type:
233
+ fact: html.property.og:type
234
+ expect: {type: string, pattern: "^(?:website|article|book|profile|music\\.(?:song|album|playlist|radio_station)|video\\.(?:movie|episode|tv_show|other)|[a-z][\\w-]*:[\\w.-]+)$"}
235
+ severity: info
236
+ score: 1.1
237
+ message: "og:type should be an Open Graph type or namespace:type, found {got}"
238
+ docs: https://ogp.me/#types
239
+ fix: "Set og:type to website, or article on a post."
240
+ html/og-url:
241
+ fact: html.property.og:url
242
+ expect: {type: string, format: uri, pattern: "^(?i:https?)://"}
243
+ severity: info
244
+ score: 2.5
245
+ message: "og:url should be an absolute http(s) URL, found {got}"
246
+ docs: https://ogp.me/#metadata
247
+ fix: "Make og:url an absolute URL of this page."
248
+ html/author:
249
+ fact: html.author
250
+ expect: {type: string, minLength: 1}
251
+ when: {html.property.og:type: article}
252
+ severity: warning
253
+ score: 3.7
254
+ message: "an article names no author for its link preview"
255
+ fix: "Add <meta name=author>, article:author, or a JSON-LD author to the article head."
256
+ docs: https://github.com/mastodon/mastodon/blob/423d0ca875fccbcd02ed4201999cec1625dee9e6/app/lib/link_details_extractor.rb#L195-L197
257
+ html/published-date:
258
+ fact: html.published
259
+ expect: {type: string, minLength: 1}
260
+ when: {html.property.og:type: article}
261
+ severity: warning
262
+ score: 3.6
263
+ message: "an article gives no publication date for its link preview"
264
+ fix: "Add <meta property=article:published_time> or a JSON-LD datePublished in ISO 8601."
265
+ docs: https://ogp.me/#type_article
266
+ html/unique-title:
267
+ unique: html.title
268
+ scope: site
269
+ severity: warning
270
+ score: 5.9
271
+ message: "pages share one <title>: {got}"
272
+ docs: https://developers.google.com/search/docs/appearance/title-link
273
+ fix: "Give each page a distinct <title>."
274
+ html/unique-description:
275
+ unique: html.meta.description
276
+ scope: site
277
+ severity: warning
278
+ score: 5.6
279
+ message: "pages share one meta description: {got}"
280
+ docs: https://developers.google.com/search/docs/appearance/snippet
281
+ fix: "Give each page a distinct meta description."
282
+ html/unique-og-title:
283
+ unique: html.property.og:title
284
+ scope: site
285
+ severity: warning
286
+ score: 4.2
287
+ message: "pages share one og:title: {got}"
288
+ docs: https://ogp.me/#metadata
289
+ fix: "Give each page a distinct og:title."
290
+ html/unique-og-description:
291
+ unique: html.property.og:description
292
+ scope: site
293
+ severity: warning
294
+ score: 4.1
295
+ message: "pages share one og:description: {got}"
296
+ docs: https://ogp.me/#optional
297
+ fix: "Give each page a distinct og:description."
298
+ html/noindex-twin:
299
+ fact: robots.noindex
300
+ expect: {const: true}
301
+ when: {url.twin: {type: string}}
302
+ severity: info
303
+ score: 3.0
304
+ message: "a page served off the canonical origin is indexable; mark the staging twin noindex"
305
+ fix: "Send X-Robots-Tag: noindex from every origin other than the canonical one."
306
+ docs: https://developers.google.com/search/docs/crawling-indexing/block-indexing
307
+ html/noindex-staging:
308
+ fact: robots.noindex
309
+ expect: {const: true}
310
+ when: {site.role: {enum: [development, staging]}, url.twin: {not: {type: string}}}
311
+ severity: warning
312
+ score: 6.2
313
+ message: "a page of a staging or development site is indexable"
314
+ fix: "Send X-Robots-Tag: noindex from every non-production host."
315
+ docs: https://developers.google.com/search/docs/crawling-indexing/block-indexing
316
+ http/tdm-reservation:
317
+ fact: http.headers.tdm-reservation
318
+ expect: {type: string, pattern: "^\\s*[01]\\s*$"}
319
+ when: {http.headers.tdm-reservation: {type: string}}
320
+ severity: warning
321
+ score: 3.5
322
+ message: "tdm-reservation should be 0 or 1, found {got}"
323
+ fix: "Send tdm-reservation: 1 to reserve text and data mining rights, 0 to waive them."
324
+ docs: https://www.w3.org/community/reports/tdmrep/CG-FINAL-tdmrep-20240510/
325
+ html/tdm-reservation:
326
+ fact: html.meta.tdm-reservation
327
+ expect: {type: string, pattern: "^\\s*[01]\\s*$"}
328
+ when: {html.meta.tdm-reservation: {type: string}}
329
+ severity: warning
330
+ score: 3.5
331
+ message: "<meta name=tdm-reservation> should be 0 or 1, found {got}"
332
+ fix: "Set <meta name=tdm-reservation content=1> to reserve text and data mining rights, 0 to waive them."
333
+ docs: https://www.w3.org/community/reports/tdmrep/CG-FINAL-tdmrep-20240510/
334
+ url/length:
335
+ fact: url.href
336
+ expect: {type: string, maxLength: 100}
337
+ severity: info
338
+ score: 1.8
339
+ message: "the URL is longer than 100 characters: {got}"
340
+ fix: "Shorten the path to the few words that name the page."
341
+ docs: https://developers.google.com/search/docs/crawling-indexing/url-structure
342
+ url/uppercase:
343
+ fact: url.pathname
344
+ expect: {type: string, not: {pattern: "(?<!%)(?<!%[0-9A-Fa-f])[A-Z]"}}
345
+ severity: warning
346
+ score: 5.1
347
+ message: "the path has uppercase letters, which a server may treat as another URL: {got}"
348
+ fix: "Use lowercase paths, and redirect each mixed-case path to its lowercase form."
349
+ docs: https://developers.google.com/search/docs/crawling-indexing/url-structure
350
+ url/session-param:
351
+ fact: url.href
352
+ expect: {type: string, not: {pattern: "(?i:[?&;](sid|sessionid|jsessionid|phpsessid)=)"}}
353
+ severity: warning
354
+ score: 5.8
355
+ message: "the URL carries a session ID: {got}"
356
+ fix: "Keep the session in a cookie, never in the URL."
357
+ docs: https://developers.google.com/search/docs/crawling-indexing/url-structure
358
+ url/double-slash:
359
+ fact: url.pathname
360
+ expect: {type: string, not: {pattern: "//"}}
361
+ severity: warning
362
+ score: 4.9
363
+ message: "the path has an empty segment: {got}"
364
+ fix: "Remove the doubled slash from the links, and redirect the doubled path to the single one."
365
+ docs: https://developers.google.com/search/docs/crawling-indexing/url-structure
366
+ url/extension:
367
+ fact: url.pathname
368
+ expect: {type: string, not: {pattern: "(?i:\\.(php|aspx?|jsp|cgi|html?)$)"}}
369
+ severity: hint
370
+ score: 0.6
371
+ message: "the path exposes the server technology in its extension: {got}"
372
+ fix: "Serve the page at an extensionless path, and redirect the old one to it."
373
+ docs: https://www.w3.org/Provider/Style/URI
374
+ url/readable-slug: info
375
+ url/separators:
376
+ severity: warning
377
+ score: 4.4
378
+ url/trailing-slash:
379
+ severity: warning
380
+ score: 4.3
@@ -0,0 +1,34 @@
1
+ # SPDX-FileCopyrightText: 2026 Damián Búho <damian.buho@proton.me>
2
+ #
3
+ # SPDX-License-Identifier: AGPL-3.0-only
4
+ ---
5
+ description: Sitemap readability, its agreement with the links and hreflang, and its media answering
6
+ when:
7
+ http.status: {minimum: 200, maximum: 299}
8
+ rules:
9
+ sitemap/orphan:
10
+ fact: crawl.referrers
11
+ expect: {type: array, minItems: 1}
12
+ when: {crawl.discovered-via: sitemap, site.crawl.complete: {const: true}}
13
+ severity: warning
14
+ score: 5.4
15
+ message: "listed in the sitemap, yet no crawled page links here"
16
+ fix: "Add a link to this page from somewhere on the site, or remove it from the sitemap."
17
+ docs: https://developers.google.com/search/docs/crawling-indexing/links-crawlable
18
+ sitemap/unlisted:
19
+ fact: sitemap.listed
20
+ expect: {const: true}
21
+ when: {html: {type: object}}
22
+ severity: info
23
+ message: "missing from the sitemap"
24
+ fix: "Add this URL to the sitemap."
25
+ docs: https://developers.google.com/search/docs/crawling-indexing/sitemaps/overview
26
+ sitemap/unreadable:
27
+ severity: warning
28
+ score: [[0, 5.8], [404, 4.4], [500, 5.8]]
29
+ sitemap/hreflang:
30
+ severity: warning
31
+ score: 4.6
32
+ sitemap/media:
33
+ severity: warning
34
+ score: [[0, 5.2], [404, 4.0], [500, 5.2]]
@@ -0,0 +1,97 @@
1
+ # SPDX-FileCopyrightText: 2026 Damián Búho <damian.buho@proton.me>
2
+ #
3
+ # SPDX-License-Identifier: AGPL-3.0-only
4
+ ---
5
+ description: Optional headers and head links a site may adopt, all hints, outside recommended
6
+ when:
7
+ http.status: {minimum: 200, maximum: 299}
8
+ http.content-type: {pattern: "^text/html"}
9
+ rules:
10
+ http/coep:
11
+ fact: http.parsed.cross-origin-embedder-policy.value
12
+ expect: {enum: [require-corp, credentialless]}
13
+ when: {http.parsed.cross-origin-embedder-policy.errors: {maxItems: 0}}
14
+ severity: hint
15
+ score: 0.8
16
+ message: "Cross-Origin-Embedder-Policy should be require-corp or credentialless, found {got}"
17
+ fix: "Send Cross-Origin-Embedder-Policy: credentialless once every embed loads under it."
18
+ docs: https://developer.mozilla.org/docs/Web/HTTP/Headers/Cross-Origin-Embedder-Policy
19
+ http/digest:
20
+ fact: http.headers
21
+ expect: {anyOf: [{required: [content-digest]}, {required: [repr-digest]}]}
22
+ severity: hint
23
+ score: 0.6
24
+ message: "no Content-Digest or Repr-Digest lets a client check the body"
25
+ fix: "Send Repr-Digest: sha-256=:<base64 digest>: with each response."
26
+ docs: https://www.rfc-editor.org/rfc/rfc9530
27
+ http/server-timing:
28
+ fact: http.headers.server-timing
29
+ expect: {type: string, minLength: 1}
30
+ when: {site.role: {enum: [development, staging]}}
31
+ severity: hint
32
+ score: 0.2
33
+ message: "no Server-Timing header shows where the server spent its time"
34
+ fix: "Send Server-Timing: app;dur=<milliseconds> with durations that name no internals."
35
+ docs: https://developer.mozilla.org/docs/Web/HTTP/Headers/Server-Timing
36
+ http/no-vary-search:
37
+ fact: http.headers.no-vary-search
38
+ expect: {type: string, minLength: 1}
39
+ when: {url.search: {minLength: 1}}
40
+ severity: hint
41
+ score: 0.4
42
+ message: "a URL with a query has no No-Vary-Search to say which parameters change the page"
43
+ fix: "Send No-Vary-Search: params=(\"utm_source\" \"utm_medium\") naming the parameters that do not change the page."
44
+ docs: https://developer.mozilla.org/docs/Web/HTTP/Headers/No-Vary-Search
45
+ html/speculation-rules:
46
+ fact: html.scripts
47
+ expect: {contains: {required: [type], properties: {type: {pattern: "^(?i:speculationrules)$"}}}}
48
+ when: {http.headers.speculation-rules: {not: {type: string}}}
49
+ severity: hint
50
+ score: 0.7
51
+ message: "no Speculation Rules script or header prefetches likely next pages"
52
+ fix: "Add <script type=speculationrules> with a prefetch rule for same-origin links."
53
+ docs: https://developer.mozilla.org/docs/Web/API/Speculation_Rules_API
54
+ html/feed-discovery:
55
+ fact: html.head.links
56
+ expect: {contains: {required: [rel, type], properties: {rel: {pattern: "(?i:(^|\\s)alternate(\\s|$))"}, type: {pattern: "^(?i:application/(rss\\+xml|atom\\+xml|feed\\+json))"}}}}
57
+ when: {url.pathname: "/"}
58
+ severity: hint
59
+ score: 0.5
60
+ message: "the home page links no RSS, Atom or JSON feed"
61
+ fix: "Add <link rel=alternate type=application/atom+xml href=/feed.xml> to the head."
62
+ docs: https://www.rssboard.org/rss-autodiscovery
63
+ html/alternate-formats:
64
+ fact: html.head.links
65
+ expect: {contains: {required: [rel, type], properties: {rel: {pattern: "(?i:(^|\\s)alternate(\\s|$))"}, type: {not: {pattern: "^(?i:text/html)"}}}}}
66
+ when: {url.pathname: "/"}
67
+ severity: hint
68
+ score: 0.5
69
+ message: "the home page links no machine-readable alternate (feed, JSON, Markdown)"
70
+ fix: "Add <link rel=alternate type=text/markdown href=/index.md> or a feed to the head."
71
+ docs: https://html.spec.whatwg.org/multipage/links.html#rel-alternate
72
+ html/nlweb:
73
+ fact: html.rels.nlweb
74
+ expect: {type: array, minItems: 1}
75
+ when: {url.pathname: "/"}
76
+ severity: hint
77
+ score: 0.3
78
+ message: "the home page links no NLWeb endpoint with rel=nlweb"
79
+ fix: "Add <link rel=nlweb href=/ask> pointing at the site’s NLWeb endpoint."
80
+ docs: https://github.com/nlweb-ai/NLWeb
81
+ html/fediverse-creator:
82
+ fact: html.meta.fediverse:creator
83
+ expect: {type: string, pattern: "^@?[^@\\s]+@[^@\\s]+\\.[^@\\s]+$"}
84
+ when: {html.property.og:type: article}
85
+ severity: hint
86
+ score: 0.3
87
+ message: "an article names no fediverse:creator account for Mastodon previews, found {got}"
88
+ fix: "Add <meta name=fediverse:creator content=\"@user@instance.example\"> to the article head."
89
+ docs: https://blog.joinmastodon.org/2024/07/highlighting-journalism-on-mastodon/
90
+ redirects/by:
91
+ fact: http.redirects
92
+ expect: {type: array, items: {required: [by]}}
93
+ severity: hint
94
+ score: 0.1
95
+ message: "a redirect does not say which component sent it with Redirect-By ({got})"
96
+ fix: "Send Redirect-By: <component> with every redirect."
97
+ docs: https://specification.website/spec/
@@ -0,0 +1,16 @@
1
+ # SPDX-FileCopyrightText: 2026 Damián Búho <damian.buho@proton.me>
2
+ #
3
+ # SPDX-License-Identifier: AGPL-3.0-only
4
+ ---
5
+ description: What a site discloses about its footprint in carbon.txt, and what a view of each page emits
6
+ extends: [spiderlint:carbon-txt]
7
+ rules:
8
+ green/page-weight-co2:
9
+ fact: co2.grams
10
+ expect: {maximum: 0.209}
11
+ when: {http.status: {minimum: 200, maximum: 299}}
12
+ severity: info
13
+ score: [[0.209, 1.4], [1.0, 2.6], [3.0, 3.2]]
14
+ message: "one view emits {got} g CO2e by the Sustainable Web Design model, above the 0.209 g budget (rating C)"
15
+ fix: "Cut the page’s transfer size: compress and resize images, drop unused scripts and fonts."
16
+ docs: https://developers.thegreenwebfoundation.org/co2js/explainer/methodologies-for-calculating-website-carbon/
@@ -0,0 +1,72 @@
1
+ # SPDX-FileCopyrightText: 2026 Damián Búho <damian.buho@proton.me>
2
+ #
3
+ # SPDX-License-Identifier: AGPL-3.0-only
4
+ ---
5
+ description: Certificate validity, protocol version and expiry
6
+ when:
7
+ url.protocol: "https:"
8
+ rules:
9
+ tls/authorized:
10
+ fact: tls.authorized
11
+ expect: {const: true}
12
+ severity: error
13
+ score: 9.5
14
+ message: "certificate is not trusted"
15
+ fix: "Install the CA bundle the server needs, or request a certificate from a trusted CA."
16
+ docs: https://developer.mozilla.org/docs/Web/Security/Transport_Layer_Security
17
+ tls/protocol:
18
+ fact: tls.protocol
19
+ expect: {enum: [TLSv1.2, TLSv1.3]}
20
+ severity: warning
21
+ score: 6.1
22
+ message: "TLS should be 1.2 or 1.3, found {got}"
23
+ fix: "Disable TLS 1.0 and 1.1 in the server configuration; keep only TLS 1.2 and 1.3."
24
+ docs: https://developer.mozilla.org/docs/Web/Security/Transport_Layer_Security
25
+ tls/cert-expiry:
26
+ fact: tls.cert.days-left
27
+ expect: {type: integer, minimum: 14}
28
+ when: {tls.cert.days-left: {minimum: 2}}
29
+ severity: info
30
+ score: [[2, 3.2], [14, 1.0]]
31
+ message: "certificate expires within 14 days ({got} left)"
32
+ fix: "Check that certificate renewal is scheduled and has succeeded before."
33
+ docs: https://developer.mozilla.org/docs/Web/Security/Transport_Layer_Security
34
+ tls/cert-expiring:
35
+ fact: tls.cert.days-left
36
+ expect: {type: integer, minimum: 2}
37
+ when: {tls.cert.days-left: {minimum: 0}}
38
+ severity: warning
39
+ score: [[0, 6.8], [2, 5.2]]
40
+ message: "certificate expires within 2 days ({got} left)"
41
+ fix: "Renew the certificate now and find out why automatic renewal did not run."
42
+ docs: https://developer.mozilla.org/docs/Web/Security/Transport_Layer_Security
43
+ tls/cert-expired:
44
+ fact: tls.cert.days-left
45
+ expect: {type: integer, minimum: 0}
46
+ severity: error
47
+ score: 9.3
48
+ message: "certificate has expired ({got} days left)"
49
+ fix: "Renew the certificate and reload the server that serves it."
50
+ docs: https://developer.mozilla.org/docs/Web/Security/Transport_Layer_Security
51
+ tls/key-strength:
52
+ fact: tls.cert.key
53
+ expect:
54
+ anyOf:
55
+ - {properties: {type: {const: RSA}, bits: {minimum: 2048}}, required: [bits]}
56
+ - {properties: {type: {const: EC}, curve: {enum: [P-256, P-384]}}, required: [curve]}
57
+ - {properties: {type: {const: Ed25519}}}
58
+ when: {tls.cert.key: {type: object}}
59
+ severity: error
60
+ score: 8.8
61
+ message: "certificate key is weaker than RSA 2048 or EC P-256: {got}"
62
+ fix: "Issue the certificate for a new EC P-256 key, or an RSA key of at least 2048 bits."
63
+ docs: https://www.rfc-editor.org/rfc/rfc9325#section-4.5
64
+ tls/signature:
65
+ fact: tls.cert.signatures
66
+ expect: {items: {not: {pattern: "[Ss][Hh][Aa]1|[Mm][Dd][245]"}}}
67
+ when: {tls.cert.signatures: {type: array}}
68
+ severity: error
69
+ score: 8.6
70
+ message: "the certificate chain is signed with SHA-1 or MD5: {got}"
71
+ fix: "Reissue the certificate from a CA that signs with SHA-256 or stronger, and serve its current intermediates."
72
+ docs: https://www.rfc-editor.org/rfc/rfc9155
@@ -0,0 +1,14 @@
1
+ # SPDX-FileCopyrightText: 2026 Damián Búho <damian.buho@proton.me>
2
+ #
3
+ # SPDX-License-Identifier: AGPL-3.0-only
4
+ ---
5
+ description: Features a CDN or host switched on for the site, read from the paths it adds, all info
6
+ rules:
7
+ vendor/cloudflare-email-obfuscation:
8
+ fact: site.origins.*.vendor.features
9
+ expect: {not: {contains: {const: cloudflare/email-obfuscation}}}
10
+ severity: info
11
+ score: 2.4
12
+ message: "Cloudflare’s Email Address Obfuscation rewrites every address into a link only JavaScript decodes, so a visitor without JavaScript sees no address"
13
+ fix: "Turn off Email Address Obfuscation in the Cloudflare dashboard under Security › Settings, or accept that visitors without JavaScript see no address."
14
+ docs: https://developers.cloudflare.com/waf/tools/scrape-shield/email-address-obfuscation/
@@ -0,0 +1,17 @@
1
+ # SPDX-FileCopyrightText: 2026 Damián Búho <damian.buho@proton.me>
2
+ #
3
+ # SPDX-License-Identifier: AGPL-3.0-only
4
+ ---
5
+ description: The recommended rules plus link graph, language, structured data, manifest, markup, well-known files, trackers and footprint, all over HTTP — for the scan server’s form
6
+ extends:
7
+ - spiderlint:recommended
8
+ - spiderlint:graph
9
+ - spiderlint:i18n
10
+ - spiderlint:structured-data
11
+ - spiderlint:manifest
12
+ - spiderlint:markup
13
+ - spiderlint:link-text
14
+ - spiderlint:well-known
15
+ - spiderlint:trackers
16
+ - spiderlint:sustainability
17
+ - spiderlint:vendor
@@ -0,0 +1,15 @@
1
+ # SPDX-FileCopyrightText: 2026 Damián Búho <damian.buho@proton.me>
2
+ #
3
+ # SPDX-License-Identifier: AGPL-3.0-only
4
+ ---
5
+ description: A fast look at the basics over HTTP — search tags, security headers, caching, TLS, redirects and robots.txt — for the scan server’s form
6
+ extends:
7
+ - spiderlint:seo
8
+ - spiderlint:security-headers
9
+ - spiderlint:performance
10
+ - spiderlint:tls
11
+ - spiderlint:redirects
12
+ - spiderlint:robots
13
+ - spiderlint:links
14
+ rules:
15
+ links/broken-external: off
Binary file
Binary file
Binary file
Binary file