recker 1.0.43 → 1.0.44

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (459) hide show
  1. package/README.md +47 -0
  2. package/dist/bin/recker-linux-x64 +0 -0
  3. package/dist/bin/recker-macos-x64 +0 -0
  4. package/dist/bin/recker-win-x64.exe +0 -0
  5. package/dist/bin/rek.cjs +85152 -100207
  6. package/dist/browser/ai/adaptive-timeout.d.ts +50 -0
  7. package/dist/browser/ai/adaptive-timeout.js +208 -0
  8. package/dist/browser/ai/client.d.ts +22 -0
  9. package/dist/browser/ai/client.js +294 -0
  10. package/dist/browser/ai/index.d.ts +14 -0
  11. package/dist/browser/ai/index.js +11 -0
  12. package/dist/browser/ai/providers/anthropic.d.ts +63 -0
  13. package/dist/browser/ai/providers/anthropic.js +370 -0
  14. package/dist/browser/ai/providers/base.d.ts +48 -0
  15. package/dist/browser/ai/providers/base.js +150 -0
  16. package/dist/browser/ai/providers/google.d.ts +59 -0
  17. package/dist/browser/ai/providers/google.js +305 -0
  18. package/dist/browser/ai/providers/ollama.d.ts +44 -0
  19. package/dist/browser/ai/providers/ollama.js +240 -0
  20. package/dist/browser/ai/providers/openai.d.ts +64 -0
  21. package/dist/browser/ai/providers/openai.js +298 -0
  22. package/dist/browser/ai/rate-limiter.d.ts +43 -0
  23. package/dist/browser/ai/rate-limiter.js +215 -0
  24. package/dist/browser/ai/vector/index.d.ts +2 -0
  25. package/dist/browser/ai/vector/index.js +2 -0
  26. package/dist/browser/ai/vector/similarity.d.ts +2 -0
  27. package/dist/browser/ai/vector/similarity.js +27 -0
  28. package/dist/browser/ai/vector/store.d.ts +27 -0
  29. package/dist/browser/ai/vector/store.js +82 -0
  30. package/dist/browser/browser/cache.d.ts +2 -40
  31. package/dist/browser/browser/cache.js +2 -199
  32. package/dist/browser/browser/index.d.ts +8 -0
  33. package/dist/browser/browser/index.js +8 -0
  34. package/dist/browser/browser/recker.d.ts +8 -1
  35. package/dist/browser/browser/recker.js +8 -2
  36. package/dist/browser/cache/indexed-db.d.ts +10 -0
  37. package/dist/browser/cache/indexed-db.js +88 -0
  38. package/dist/browser/cache/service-worker-cache.d.ts +18 -0
  39. package/dist/browser/cache/service-worker-cache.js +103 -0
  40. package/dist/browser/cache.d.ts +2 -40
  41. package/dist/browser/cache.js +2 -199
  42. package/dist/browser/constants/user-agents.d.ts +7 -0
  43. package/dist/browser/constants/user-agents.js +7 -0
  44. package/dist/browser/core/client.d.ts +2 -0
  45. package/dist/browser/core/client.js +19 -1
  46. package/dist/browser/index.d.ts +8 -0
  47. package/dist/browser/index.js +8 -0
  48. package/dist/browser/plugins/har-recorder.d.ts +40 -0
  49. package/dist/browser/plugins/har-recorder.js +120 -0
  50. package/dist/browser/plugins/network-simulation.d.ts +7 -0
  51. package/dist/browser/plugins/network-simulation.js +13 -0
  52. package/dist/browser/presets/android.d.ts +2 -0
  53. package/dist/browser/presets/android.js +16 -0
  54. package/dist/browser/presets/anthropic.d.ts +8 -0
  55. package/dist/browser/presets/anthropic.js +27 -0
  56. package/dist/browser/presets/aws.d.ts +19 -0
  57. package/dist/browser/presets/aws.js +68 -0
  58. package/dist/browser/presets/azure-openai.d.ts +10 -0
  59. package/dist/browser/presets/azure-openai.js +35 -0
  60. package/dist/browser/presets/azure.d.ts +41 -0
  61. package/dist/browser/presets/azure.js +104 -0
  62. package/dist/browser/presets/chaturbate.d.ts +2 -0
  63. package/dist/browser/presets/chaturbate.js +17 -0
  64. package/dist/browser/presets/cloudflare.d.ts +12 -0
  65. package/dist/browser/presets/cloudflare.js +39 -0
  66. package/dist/browser/presets/cohere.d.ts +7 -0
  67. package/dist/browser/presets/cohere.js +22 -0
  68. package/dist/browser/presets/deepseek.d.ts +7 -0
  69. package/dist/browser/presets/deepseek.js +22 -0
  70. package/dist/browser/presets/digitalocean.d.ts +5 -0
  71. package/dist/browser/presets/digitalocean.js +16 -0
  72. package/dist/browser/presets/discord.d.ts +6 -0
  73. package/dist/browser/presets/discord.js +17 -0
  74. package/dist/browser/presets/elevenlabs.d.ts +6 -0
  75. package/dist/browser/presets/elevenlabs.js +20 -0
  76. package/dist/browser/presets/enhancers.d.ts +20 -0
  77. package/dist/browser/presets/enhancers.js +85 -0
  78. package/dist/browser/presets/fireworks.d.ts +7 -0
  79. package/dist/browser/presets/fireworks.js +22 -0
  80. package/dist/browser/presets/gcp.d.ts +34 -0
  81. package/dist/browser/presets/gcp.js +91 -0
  82. package/dist/browser/presets/gemini.d.ts +7 -0
  83. package/dist/browser/presets/gemini.js +23 -0
  84. package/dist/browser/presets/github.d.ts +6 -0
  85. package/dist/browser/presets/github.js +17 -0
  86. package/dist/browser/presets/gitlab.d.ts +6 -0
  87. package/dist/browser/presets/gitlab.js +16 -0
  88. package/dist/browser/presets/groq.d.ts +7 -0
  89. package/dist/browser/presets/groq.js +22 -0
  90. package/dist/browser/presets/hubspot.d.ts +9 -0
  91. package/dist/browser/presets/hubspot.js +28 -0
  92. package/dist/browser/presets/huggingface.d.ts +7 -0
  93. package/dist/browser/presets/huggingface.js +23 -0
  94. package/dist/browser/presets/index.d.ts +47 -0
  95. package/dist/browser/presets/index.js +47 -0
  96. package/dist/browser/presets/ios.d.ts +2 -0
  97. package/dist/browser/presets/ios.js +13 -0
  98. package/dist/browser/presets/linear.d.ts +5 -0
  99. package/dist/browser/presets/linear.js +16 -0
  100. package/dist/browser/presets/mailgun.d.ts +7 -0
  101. package/dist/browser/presets/mailgun.js +20 -0
  102. package/dist/browser/presets/meta.d.ts +10 -0
  103. package/dist/browser/presets/meta.js +33 -0
  104. package/dist/browser/presets/mistral.d.ts +7 -0
  105. package/dist/browser/presets/mistral.js +22 -0
  106. package/dist/browser/presets/notion.d.ts +6 -0
  107. package/dist/browser/presets/notion.js +17 -0
  108. package/dist/browser/presets/openai.d.ts +9 -0
  109. package/dist/browser/presets/openai.js +30 -0
  110. package/dist/browser/presets/oracle.d.ts +19 -0
  111. package/dist/browser/presets/oracle.js +117 -0
  112. package/dist/browser/presets/perplexity.d.ts +7 -0
  113. package/dist/browser/presets/perplexity.js +22 -0
  114. package/dist/browser/presets/pinecone.d.ts +8 -0
  115. package/dist/browser/presets/pinecone.js +42 -0
  116. package/dist/browser/presets/registry.d.ts +23 -0
  117. package/dist/browser/presets/registry.js +519 -0
  118. package/dist/browser/presets/replicate.d.ts +7 -0
  119. package/dist/browser/presets/replicate.js +23 -0
  120. package/dist/browser/presets/sendgrid.d.ts +6 -0
  121. package/dist/browser/presets/sendgrid.js +20 -0
  122. package/dist/browser/presets/sentry.d.ts +11 -0
  123. package/dist/browser/presets/sentry.js +48 -0
  124. package/dist/browser/presets/sinch.d.ts +9 -0
  125. package/dist/browser/presets/sinch.js +39 -0
  126. package/dist/browser/presets/slack.d.ts +5 -0
  127. package/dist/browser/presets/slack.js +16 -0
  128. package/dist/browser/presets/square.d.ts +10 -0
  129. package/dist/browser/presets/square.js +33 -0
  130. package/dist/browser/presets/stripe.d.ts +7 -0
  131. package/dist/browser/presets/stripe.js +23 -0
  132. package/dist/browser/presets/supabase.d.ts +6 -0
  133. package/dist/browser/presets/supabase.js +18 -0
  134. package/dist/browser/presets/tiktok.d.ts +10 -0
  135. package/dist/browser/presets/tiktok.js +38 -0
  136. package/dist/browser/presets/together.d.ts +7 -0
  137. package/dist/browser/presets/together.js +22 -0
  138. package/dist/browser/presets/twilio.d.ts +6 -0
  139. package/dist/browser/presets/twilio.js +17 -0
  140. package/dist/browser/presets/vercel.d.ts +6 -0
  141. package/dist/browser/presets/vercel.js +23 -0
  142. package/dist/browser/presets/vultr.d.ts +5 -0
  143. package/dist/browser/presets/vultr.js +16 -0
  144. package/dist/browser/presets/xai.d.ts +8 -0
  145. package/dist/browser/presets/xai.js +23 -0
  146. package/dist/browser/presets/youtube.d.ts +5 -0
  147. package/dist/browser/presets/youtube.js +20 -0
  148. package/dist/browser/recker.d.ts +8 -1
  149. package/dist/browser/recker.js +8 -2
  150. package/dist/browser/scrape/document.d.ts +5 -4
  151. package/dist/browser/scrape/document.js +89 -76
  152. package/dist/browser/scrape/element.d.ts +10 -8
  153. package/dist/browser/scrape/element.js +295 -81
  154. package/dist/browser/scrape/extractors.d.ts +11 -11
  155. package/dist/browser/scrape/extractors.js +145 -113
  156. package/dist/browser/scrape/parser/back.d.ts +1 -0
  157. package/dist/browser/scrape/parser/back.js +3 -0
  158. package/dist/browser/scrape/parser/index.d.ts +20 -0
  159. package/dist/browser/scrape/parser/index.js +19 -0
  160. package/dist/browser/scrape/parser/matcher.d.ts +30 -0
  161. package/dist/browser/scrape/parser/matcher.js +99 -0
  162. package/dist/browser/scrape/parser/nodes/comment.d.ts +12 -0
  163. package/dist/browser/scrape/parser/nodes/comment.js +21 -0
  164. package/dist/browser/scrape/parser/nodes/html.d.ts +110 -0
  165. package/dist/browser/scrape/parser/nodes/html.js +978 -0
  166. package/dist/browser/scrape/parser/nodes/node.d.ts +18 -0
  167. package/dist/browser/scrape/parser/nodes/node.js +31 -0
  168. package/dist/browser/scrape/parser/nodes/text.d.ts +14 -0
  169. package/dist/browser/scrape/parser/nodes/text.js +30 -0
  170. package/dist/browser/scrape/parser/nodes/type.d.ts +6 -0
  171. package/dist/browser/scrape/parser/nodes/type.js +7 -0
  172. package/dist/browser/scrape/parser/parse.d.ts +1 -0
  173. package/dist/browser/scrape/parser/parse.js +1 -0
  174. package/dist/browser/scrape/parser/valid.d.ts +2 -0
  175. package/dist/browser/scrape/parser/valid.js +5 -0
  176. package/dist/browser/scrape/parser/void-tag.d.ts +7 -0
  177. package/dist/browser/scrape/parser/void-tag.js +43 -0
  178. package/dist/browser/scrape/types.d.ts +7 -0
  179. package/dist/browser/seo/analyzer.d.ts +59 -0
  180. package/dist/browser/seo/analyzer.js +1399 -0
  181. package/dist/browser/seo/keywords.d.ts +16 -0
  182. package/dist/browser/seo/keywords.js +55 -0
  183. package/dist/browser/seo/rules/accessibility.d.ts +2 -0
  184. package/dist/browser/seo/rules/accessibility.js +733 -0
  185. package/dist/browser/seo/rules/ai-search.d.ts +2 -0
  186. package/dist/browser/seo/rules/ai-search.js +436 -0
  187. package/dist/browser/seo/rules/analytics.d.ts +2 -0
  188. package/dist/browser/seo/rules/analytics.js +306 -0
  189. package/dist/browser/seo/rules/best-practices.d.ts +2 -0
  190. package/dist/browser/seo/rules/best-practices.js +195 -0
  191. package/dist/browser/seo/rules/canonical.d.ts +12 -0
  192. package/dist/browser/seo/rules/canonical.js +270 -0
  193. package/dist/browser/seo/rules/content.d.ts +2 -0
  194. package/dist/browser/seo/rules/content.js +522 -0
  195. package/dist/browser/seo/rules/crawl.d.ts +2 -0
  196. package/dist/browser/seo/rules/crawl.js +435 -0
  197. package/dist/browser/seo/rules/cwv.d.ts +2 -0
  198. package/dist/browser/seo/rules/cwv.js +248 -0
  199. package/dist/browser/seo/rules/ecommerce.d.ts +2 -0
  200. package/dist/browser/seo/rules/ecommerce.js +312 -0
  201. package/dist/browser/seo/rules/i18n.d.ts +2 -0
  202. package/dist/browser/seo/rules/i18n.js +288 -0
  203. package/dist/browser/seo/rules/images.d.ts +2 -0
  204. package/dist/browser/seo/rules/images.js +255 -0
  205. package/dist/browser/seo/rules/index.d.ts +52 -0
  206. package/dist/browser/seo/rules/index.js +159 -0
  207. package/dist/browser/seo/rules/internal-linking.d.ts +2 -0
  208. package/dist/browser/seo/rules/internal-linking.js +394 -0
  209. package/dist/browser/seo/rules/links.d.ts +2 -0
  210. package/dist/browser/seo/rules/links.js +498 -0
  211. package/dist/browser/seo/rules/local.d.ts +2 -0
  212. package/dist/browser/seo/rules/local.js +289 -0
  213. package/dist/browser/seo/rules/meta.d.ts +2 -0
  214. package/dist/browser/seo/rules/meta.js +805 -0
  215. package/dist/browser/seo/rules/mobile.d.ts +2 -0
  216. package/dist/browser/seo/rules/mobile.js +161 -0
  217. package/dist/browser/seo/rules/performance.d.ts +2 -0
  218. package/dist/browser/seo/rules/performance.js +738 -0
  219. package/dist/browser/seo/rules/pwa.d.ts +2 -0
  220. package/dist/browser/seo/rules/pwa.js +299 -0
  221. package/dist/browser/seo/rules/readability.d.ts +2 -0
  222. package/dist/browser/seo/rules/readability.js +264 -0
  223. package/dist/browser/seo/rules/redirects.d.ts +16 -0
  224. package/dist/browser/seo/rules/redirects.js +199 -0
  225. package/dist/browser/seo/rules/resources.d.ts +2 -0
  226. package/dist/browser/seo/rules/resources.js +390 -0
  227. package/dist/browser/seo/rules/schema.d.ts +2 -0
  228. package/dist/browser/seo/rules/schema.js +379 -0
  229. package/dist/browser/seo/rules/security.d.ts +2 -0
  230. package/dist/browser/seo/rules/security.js +877 -0
  231. package/dist/browser/seo/rules/social.d.ts +2 -0
  232. package/dist/browser/seo/rules/social.js +603 -0
  233. package/dist/browser/seo/rules/structural.d.ts +2 -0
  234. package/dist/browser/seo/rules/structural.js +223 -0
  235. package/dist/browser/seo/rules/technical-advanced.d.ts +10 -0
  236. package/dist/browser/seo/rules/technical-advanced.js +289 -0
  237. package/dist/browser/seo/rules/technical.d.ts +2 -0
  238. package/dist/browser/seo/rules/technical.js +480 -0
  239. package/dist/browser/seo/rules/thresholds.d.ts +196 -0
  240. package/dist/browser/seo/rules/thresholds.js +118 -0
  241. package/dist/browser/seo/rules/types.d.ts +498 -0
  242. package/dist/browser/seo/rules/types.js +11 -0
  243. package/dist/browser/seo/types.d.ts +211 -0
  244. package/dist/browser/seo/types.js +1 -0
  245. package/dist/browser/transport/curl.d.ts +4 -0
  246. package/dist/browser/transport/curl.js +101 -0
  247. package/dist/browser/transport/undici.js +1 -2
  248. package/dist/browser/transport/worker.d.ts +18 -0
  249. package/dist/browser/transport/worker.js +278 -0
  250. package/dist/browser/types/index.d.ts +4 -1
  251. package/dist/browser/utils/binary-manager.d.ts +4 -0
  252. package/dist/browser/utils/binary-manager.js +72 -0
  253. package/dist/browser/utils/user-agent.js +2 -13
  254. package/dist/cache/indexed-db.d.ts +10 -0
  255. package/dist/cache/indexed-db.js +88 -0
  256. package/dist/cache/service-worker-cache.d.ts +18 -0
  257. package/dist/cache/service-worker-cache.js +103 -0
  258. package/dist/cli/commands/ai.d.ts +2 -0
  259. package/dist/cli/commands/ai.js +162 -0
  260. package/dist/cli/commands/bench.d.ts +2 -0
  261. package/dist/cli/commands/bench.js +51 -0
  262. package/dist/cli/commands/dns.d.ts +2 -0
  263. package/dist/cli/commands/dns.js +295 -0
  264. package/dist/cli/commands/har.d.ts +2 -0
  265. package/dist/cli/commands/har.js +171 -0
  266. package/dist/cli/commands/hls.d.ts +2 -0
  267. package/dist/cli/commands/hls.js +192 -0
  268. package/dist/cli/commands/network.d.ts +2 -0
  269. package/dist/cli/commands/network.js +288 -0
  270. package/dist/cli/commands/protocols.d.ts +2 -0
  271. package/dist/cli/commands/protocols.js +344 -0
  272. package/dist/cli/commands/scrape.d.ts +2 -0
  273. package/dist/cli/commands/scrape.js +176 -0
  274. package/dist/cli/commands/security.d.ts +2 -0
  275. package/dist/cli/commands/security.js +57 -0
  276. package/dist/cli/commands/seo.d.ts +2 -0
  277. package/dist/cli/commands/seo.js +125 -0
  278. package/dist/cli/commands/serve.d.ts +2 -0
  279. package/dist/cli/commands/serve.js +531 -0
  280. package/dist/cli/commands/spider.d.ts +3 -0
  281. package/dist/cli/commands/spider.js +456 -0
  282. package/dist/cli/commands/utils.d.ts +2 -0
  283. package/dist/cli/commands/utils.js +176 -0
  284. package/dist/cli/commands/vector.d.ts +2 -0
  285. package/dist/cli/commands/vector.js +158 -0
  286. package/dist/cli/handler.d.ts +2 -2
  287. package/dist/cli/handler.js +6 -6
  288. package/dist/cli/helpers.d.ts +7 -0
  289. package/dist/cli/helpers.js +128 -0
  290. package/dist/cli/index.js +96 -5228
  291. package/dist/cli/parser/help.d.ts +2 -0
  292. package/dist/cli/parser/help.js +52 -0
  293. package/dist/cli/parser/index.d.ts +3 -0
  294. package/dist/cli/parser/index.js +3 -0
  295. package/dist/cli/parser/parser.d.ts +4 -0
  296. package/dist/cli/parser/parser.js +146 -0
  297. package/dist/cli/parser/types.d.ts +41 -0
  298. package/dist/cli/parser/types.js +1 -0
  299. package/dist/cli/presets.d.ts +1 -1
  300. package/dist/cli/presets.js +1 -1
  301. package/dist/cli/router.d.ts +36 -0
  302. package/dist/cli/router.js +195 -0
  303. package/dist/cli/tui/ai-chat.js +1 -1
  304. package/dist/cli/tui/commands/context.d.ts +9 -0
  305. package/dist/cli/tui/commands/context.js +1 -0
  306. package/dist/cli/tui/commands/dns.d.ts +10 -0
  307. package/dist/cli/tui/commands/dns.js +461 -0
  308. package/dist/cli/tui/commands/hls.d.ts +2 -0
  309. package/dist/cli/tui/commands/hls.js +162 -0
  310. package/dist/cli/tui/commands/ip.d.ts +2 -0
  311. package/dist/cli/tui/commands/ip.js +45 -0
  312. package/dist/cli/tui/commands/network.d.ts +3 -0
  313. package/dist/cli/tui/commands/network.js +81 -0
  314. package/dist/cli/tui/commands/protocols.d.ts +6 -0
  315. package/dist/cli/tui/commands/protocols.js +531 -0
  316. package/dist/cli/tui/commands/security.d.ts +2 -0
  317. package/dist/cli/tui/commands/security.js +48 -0
  318. package/dist/cli/tui/commands/seo.d.ts +2 -0
  319. package/dist/cli/tui/commands/seo.js +74 -0
  320. package/dist/cli/tui/context.d.ts +12 -0
  321. package/dist/cli/tui/context.js +1 -0
  322. package/dist/cli/tui/shell.d.ts +11 -20
  323. package/dist/cli/tui/shell.js +216 -1873
  324. package/dist/constants/user-agents.d.ts +7 -0
  325. package/dist/constants/user-agents.js +7 -0
  326. package/dist/core/client.d.ts +2 -0
  327. package/dist/core/client.js +19 -1
  328. package/dist/index.d.ts +1 -0
  329. package/dist/index.js +1 -0
  330. package/dist/mcp/cli.js +2 -3
  331. package/dist/mcp/data/embeddings.json +1 -1
  332. package/dist/mcp/tools/network.js +298 -158
  333. package/dist/plugins/har-player.d.ts +23 -0
  334. package/dist/plugins/har-player.js +49 -0
  335. package/dist/plugins/har-recorder.d.ts +37 -3
  336. package/dist/plugins/har-recorder.js +116 -63
  337. package/dist/plugins/network-simulation.d.ts +7 -0
  338. package/dist/plugins/network-simulation.js +13 -0
  339. package/dist/presets/android.d.ts +2 -0
  340. package/dist/presets/android.js +16 -0
  341. package/dist/presets/chaturbate.d.ts +2 -0
  342. package/dist/presets/chaturbate.js +17 -0
  343. package/dist/presets/elevenlabs.d.ts +6 -0
  344. package/dist/presets/elevenlabs.js +20 -0
  345. package/dist/presets/enhancers.d.ts +20 -0
  346. package/dist/presets/enhancers.js +85 -0
  347. package/dist/presets/hubspot.d.ts +9 -0
  348. package/dist/presets/hubspot.js +28 -0
  349. package/dist/presets/index.d.ts +10 -0
  350. package/dist/presets/index.js +10 -0
  351. package/dist/presets/ios.d.ts +2 -0
  352. package/dist/presets/ios.js +13 -0
  353. package/dist/presets/pinecone.d.ts +8 -0
  354. package/dist/presets/pinecone.js +42 -0
  355. package/dist/presets/registry.js +60 -0
  356. package/dist/presets/sendgrid.d.ts +6 -0
  357. package/dist/presets/sendgrid.js +20 -0
  358. package/dist/presets/sentry.d.ts +11 -0
  359. package/dist/presets/sentry.js +48 -0
  360. package/dist/presets/square.d.ts +10 -0
  361. package/dist/presets/square.js +33 -0
  362. package/dist/recker.d.ts +3 -0
  363. package/dist/recker.js +4 -0
  364. package/dist/scrape/document.d.ts +5 -4
  365. package/dist/scrape/document.js +89 -76
  366. package/dist/scrape/element.d.ts +10 -8
  367. package/dist/scrape/element.js +295 -81
  368. package/dist/scrape/extractors.d.ts +11 -11
  369. package/dist/scrape/extractors.js +145 -113
  370. package/dist/scrape/index.d.ts +2 -0
  371. package/dist/scrape/index.js +1 -0
  372. package/dist/scrape/parser/back.d.ts +1 -0
  373. package/dist/scrape/parser/back.js +3 -0
  374. package/dist/scrape/parser/index.d.ts +20 -0
  375. package/dist/scrape/parser/index.js +19 -0
  376. package/dist/scrape/parser/matcher.d.ts +30 -0
  377. package/dist/scrape/parser/matcher.js +99 -0
  378. package/dist/scrape/parser/nodes/comment.d.ts +12 -0
  379. package/dist/scrape/parser/nodes/comment.js +21 -0
  380. package/dist/scrape/parser/nodes/html.d.ts +110 -0
  381. package/dist/scrape/parser/nodes/html.js +978 -0
  382. package/dist/scrape/parser/nodes/node.d.ts +18 -0
  383. package/dist/scrape/parser/nodes/node.js +31 -0
  384. package/dist/scrape/parser/nodes/text.d.ts +14 -0
  385. package/dist/scrape/parser/nodes/text.js +30 -0
  386. package/dist/scrape/parser/nodes/type.d.ts +6 -0
  387. package/dist/scrape/parser/nodes/type.js +7 -0
  388. package/dist/scrape/parser/parse.d.ts +1 -0
  389. package/dist/scrape/parser/parse.js +1 -0
  390. package/dist/scrape/parser/valid.d.ts +2 -0
  391. package/dist/scrape/parser/valid.js +5 -0
  392. package/dist/scrape/parser/void-tag.d.ts +7 -0
  393. package/dist/scrape/parser/void-tag.js +43 -0
  394. package/dist/scrape/spider.d.ts +19 -0
  395. package/dist/scrape/spider.js +28 -3
  396. package/dist/scrape/types.d.ts +7 -0
  397. package/dist/seo/analyzer.d.ts +15 -5
  398. package/dist/seo/analyzer.js +636 -175
  399. package/dist/seo/formatter.d.ts +16 -0
  400. package/dist/seo/formatter.js +228 -0
  401. package/dist/seo/index.d.ts +2 -0
  402. package/dist/seo/index.js +1 -0
  403. package/dist/seo/keywords.d.ts +16 -0
  404. package/dist/seo/keywords.js +55 -0
  405. package/dist/seo/rules/accessibility.js +96 -57
  406. package/dist/seo/rules/ai-search.js +44 -31
  407. package/dist/seo/rules/analytics.d.ts +2 -0
  408. package/dist/seo/rules/analytics.js +306 -0
  409. package/dist/seo/rules/best-practices.js +21 -14
  410. package/dist/seo/rules/canonical.js +53 -32
  411. package/dist/seo/rules/content.js +317 -31
  412. package/dist/seo/rules/crawl.js +55 -40
  413. package/dist/seo/rules/cwv.js +21 -15
  414. package/dist/seo/rules/ecommerce.js +82 -22
  415. package/dist/seo/rules/i18n.js +75 -36
  416. package/dist/seo/rules/images.js +109 -30
  417. package/dist/seo/rules/index.js +2 -0
  418. package/dist/seo/rules/internal-linking.js +58 -39
  419. package/dist/seo/rules/links.js +79 -52
  420. package/dist/seo/rules/local.js +49 -25
  421. package/dist/seo/rules/meta.js +339 -81
  422. package/dist/seo/rules/mobile.js +112 -2
  423. package/dist/seo/rules/performance.js +434 -66
  424. package/dist/seo/rules/pwa.js +36 -39
  425. package/dist/seo/rules/readability.js +31 -22
  426. package/dist/seo/rules/redirects.js +21 -15
  427. package/dist/seo/rules/resources.js +59 -42
  428. package/dist/seo/rules/schema.js +333 -8
  429. package/dist/seo/rules/security.js +142 -80
  430. package/dist/seo/rules/social.js +277 -47
  431. package/dist/seo/rules/structural.js +87 -19
  432. package/dist/seo/rules/technical-advanced.js +30 -24
  433. package/dist/seo/rules/technical.js +243 -42
  434. package/dist/seo/rules/types.d.ts +53 -1
  435. package/dist/seo/seo-spider.d.ts +22 -0
  436. package/dist/seo/seo-spider.js +77 -13
  437. package/dist/seo/types.d.ts +8 -1
  438. package/dist/seo/validators/llms-txt.js +19 -0
  439. package/dist/seo/validators/rss.d.ts +11 -0
  440. package/dist/seo/validators/rss.js +93 -0
  441. package/dist/seo/validators/sitemap.js +36 -26
  442. package/dist/transport/curl.d.ts +4 -0
  443. package/dist/transport/curl.js +101 -0
  444. package/dist/transport/udp.js +0 -1
  445. package/dist/transport/undici.js +1 -2
  446. package/dist/transport/worker.d.ts +18 -0
  447. package/dist/transport/worker.js +278 -0
  448. package/dist/types/index.d.ts +4 -1
  449. package/dist/utils/binary-manager.d.ts +4 -0
  450. package/dist/utils/binary-manager.js +72 -0
  451. package/dist/utils/optional-require.d.ts +7 -8
  452. package/dist/utils/optional-require.js +2 -21
  453. package/dist/utils/upload.d.ts +6 -0
  454. package/dist/utils/upload.js +11 -0
  455. package/dist/utils/user-agent.js +2 -13
  456. package/dist/version.js +1 -1
  457. package/package.json +12 -6
  458. package/dist/browser/utils/optional-require.d.ts +0 -19
  459. package/dist/browser/utils/optional-require.js +0 -105
@@ -7,8 +7,9 @@ export const technicalAdvancedRules = [
7
7
  severity: 'warning',
8
8
  description: 'Pages should not use meta refresh redirects',
9
9
  check: (ctx) => {
10
- if (!ctx.metaRefresh)
11
- return null;
10
+ if (!ctx.metaRefresh) {
11
+ return createResult({ id: 'meta-refresh-redirect', name: 'Meta Refresh Redirect', category: 'technical', severity: 'warning' }, 'info', 'Not applicable (no meta refresh detected)', { recommendation: 'This rule checks for meta refresh redirects which can impact SEO' });
12
+ }
12
13
  const { delay, url } = ctx.metaRefresh;
13
14
  if (url) {
14
15
  return createResult({ id: 'meta-refresh-redirect', name: 'Meta Refresh Redirect', category: 'technical', severity: 'warning' }, delay === 0 ? 'warn' : 'fail', `Page uses meta refresh redirect (${delay}s delay)`, {
@@ -35,7 +36,7 @@ export const technicalAdvancedRules = [
35
36
  }
36
37
  });
37
38
  }
38
- return null;
39
+ return createResult({ id: 'meta-refresh-redirect', name: 'Meta Refresh Redirect', category: 'technical', severity: 'warning' }, 'info', 'Not applicable (no meta refresh detected)', { recommendation: 'This rule checks for meta refresh redirects which can impact SEO' });
39
40
  },
40
41
  },
41
42
  {
@@ -45,8 +46,9 @@ export const technicalAdvancedRules = [
45
46
  severity: 'warning',
46
47
  description: 'HTML page should not exceed reasonable size limits',
47
48
  check: (ctx) => {
48
- if (ctx.htmlSize === undefined)
49
- return null;
49
+ if (ctx.htmlSize === undefined) {
50
+ return createResult({ id: 'html-page-size', name: 'HTML Page Size', category: 'performance', severity: 'warning' }, 'info', 'Not applicable (HTML size data unavailable)', { recommendation: 'This rule checks if HTML size is within recommended limits for optimal performance' });
51
+ }
50
52
  const sizeKb = ctx.htmlSize / 1024;
51
53
  const sizeMb = sizeKb / 1024;
52
54
  if (sizeMb > 2) {
@@ -81,8 +83,9 @@ export const technicalAdvancedRules = [
81
83
  severity: 'warning',
82
84
  description: 'Total page weight should be optimized for performance',
83
85
  check: (ctx) => {
84
- if (ctx.totalPageSize === undefined)
85
- return null;
86
+ if (ctx.totalPageSize === undefined) {
87
+ return createResult({ id: 'total-page-size', name: 'Total Page Size', category: 'performance', severity: 'warning' }, 'info', 'Not applicable (total page size data unavailable)', { recommendation: 'This rule checks total page weight including all resources for optimal load time' });
88
+ }
86
89
  const sizeMb = ctx.totalPageSize / (1024 * 1024);
87
90
  if (sizeMb > 5) {
88
91
  return createResult({ id: 'total-page-size', name: 'Total Page Size', category: 'performance', severity: 'warning' }, 'fail', `Total page size ${sizeMb.toFixed(2)}MB exceeds 5MB`, {
@@ -117,8 +120,9 @@ export const technicalAdvancedRules = [
117
120
  description: 'Server should respond within acceptable time limits',
118
121
  check: (ctx) => {
119
122
  const ttfb = ctx.timings?.ttfb;
120
- if (ttfb === undefined)
121
- return null;
123
+ if (ttfb === undefined) {
124
+ return createResult({ id: 'server-response-time', name: 'Server Response Time', category: 'performance', severity: 'warning' }, 'info', 'Not applicable (TTFB timing data unavailable)', { recommendation: 'This rule checks server response time (TTFB) for performance optimization' });
125
+ }
122
126
  if (ttfb > 5000) {
123
127
  return createResult({ id: 'server-response-time', name: 'Server Response Time', category: 'performance', severity: 'warning' }, 'fail', `TTFB ${(ttfb / 1000).toFixed(2)}s exceeds 5s`, {
124
128
  value: ttfb,
@@ -151,8 +155,9 @@ export const technicalAdvancedRules = [
151
155
  severity: 'warning',
152
156
  description: 'URLs should not be excessively long',
153
157
  check: (ctx) => {
154
- if (!ctx.url)
155
- return null;
158
+ if (!ctx.url) {
159
+ return createResult({ id: 'url-length', name: 'URL Length', category: 'technical', severity: 'warning' }, 'info', 'Not applicable (URL data unavailable)', { recommendation: 'This rule checks URL length to ensure it is within recommended limits' });
160
+ }
156
161
  const length = ctx.url.length;
157
162
  if (length > 2000) {
158
163
  return createResult({ id: 'url-length', name: 'URL Length', category: 'technical', severity: 'warning' }, 'fail', `URL length ${length} chars exceeds browser limits`, {
@@ -186,8 +191,9 @@ export const technicalAdvancedRules = [
186
191
  severity: 'warning',
187
192
  description: 'URLs should not contain problematic special characters',
188
193
  check: (ctx) => {
189
- if (!ctx.url)
190
- return null;
194
+ if (!ctx.url) {
195
+ return createResult({ id: 'url-special-chars', name: 'URL Special Characters', category: 'technical', severity: 'warning' }, 'info', 'Not applicable (URL data unavailable)', { recommendation: 'This rule checks for problematic special characters in URLs' });
196
+ }
191
197
  try {
192
198
  const urlObj = new URL(ctx.url);
193
199
  const path = urlObj.pathname + urlObj.search;
@@ -224,8 +230,9 @@ export const technicalAdvancedRules = [
224
230
  recommendation: 'Fix URL syntax',
225
231
  evidence: {
226
232
  found: ctx.url,
227
- expected: 'Valid URL',
228
- impact: 'Invalid URLs cannot be crawled or indexed'
233
+ expected: 'Valid URL format (e.g., https://example.com/path)',
234
+ impact: 'Invalid URLs cannot be crawled or indexed',
235
+ example: 'https://example.com/valid-path-with-hyphens',
229
236
  }
230
237
  });
231
238
  }
@@ -238,8 +245,9 @@ export const technicalAdvancedRules = [
238
245
  severity: 'error',
239
246
  description: 'Pages with password fields must use HTTPS',
240
247
  check: (ctx) => {
241
- if (!ctx.hasPasswordField || ctx.isHttps === undefined)
242
- return null;
248
+ if (!ctx.hasPasswordField || ctx.isHttps === undefined) {
249
+ return createResult({ id: 'password-on-http', name: 'Password Fields on HTTP', category: 'security', severity: 'error' }, 'info', 'Not applicable (no password fields detected)', { recommendation: 'This rule ensures password fields are served over HTTPS for security' });
250
+ }
243
251
  if (ctx.hasPasswordField && !ctx.isHttps) {
244
252
  return createResult({ id: 'password-on-http', name: 'Password Fields on HTTP', category: 'security', severity: 'error' }, 'fail', 'Password field detected on non-HTTPS page', {
245
253
  recommendation: 'Serve login/registration pages over HTTPS only',
@@ -251,10 +259,7 @@ export const technicalAdvancedRules = [
251
259
  }
252
260
  });
253
261
  }
254
- if (ctx.hasPasswordField && ctx.isHttps) {
255
- return createResult({ id: 'password-on-http', name: 'Password Fields on HTTP', category: 'security', severity: 'error' }, 'pass', 'Password field properly served over HTTPS');
256
- }
257
- return null;
262
+ return createResult({ id: 'password-on-http', name: 'Password Fields on HTTP', category: 'security', severity: 'error' }, 'pass', 'Password field properly served over HTTPS');
258
263
  },
259
264
  },
260
265
  {
@@ -264,8 +269,9 @@ export const technicalAdvancedRules = [
264
269
  severity: 'warning',
265
270
  description: 'Forms should submit data over HTTPS',
266
271
  check: (ctx) => {
267
- if (ctx.formsOnHttp === undefined)
268
- return null;
272
+ if (ctx.formsOnHttp === undefined) {
273
+ return createResult({ id: 'forms-on-http', name: 'Forms on HTTP', category: 'security', severity: 'warning' }, 'info', 'Not applicable (form data unavailable)', { recommendation: 'This rule checks if forms submit data over secure HTTPS connections' });
274
+ }
269
275
  if (ctx.formsOnHttp > 0) {
270
276
  return createResult({ id: 'forms-on-http', name: 'Forms on HTTP', category: 'security', severity: 'warning' }, 'warn', `${ctx.formsOnHttp} form(s) submit to HTTP URLs`, {
271
277
  value: ctx.formsOnHttp,
@@ -277,7 +283,7 @@ export const technicalAdvancedRules = [
277
283
  }
278
284
  });
279
285
  }
280
- return null;
286
+ return createResult({ id: 'forms-on-http', name: 'Forms on HTTP', category: 'security', severity: 'warning' }, 'pass', 'All forms submit over HTTPS or no forms detected');
281
287
  },
282
288
  },
283
289
  ];
@@ -8,7 +8,16 @@ export const technicalRules = [
8
8
  description: 'Page should have a canonical URL',
9
9
  check: (ctx) => {
10
10
  if (!ctx.hasCanonical) {
11
- return createResult({ id: 'canonical-exists', name: 'Canonical URL', category: 'technical', severity: 'warning' }, 'warn', 'No canonical URL defined', { recommendation: 'Add <link rel="canonical" href="..."> to prevent duplicate content' });
11
+ return createResult({ id: 'canonical-exists', name: 'Canonical URL', category: 'technical', severity: 'warning' }, 'warn', 'No canonical URL defined', {
12
+ recommendation: 'Add a canonical URL to indicate the preferred version of this page.',
13
+ evidence: {
14
+ found: 'No <link rel="canonical"> tag',
15
+ expected: 'A canonical URL pointing to the preferred version',
16
+ impact: 'Without canonical, search engines may split ranking signals across duplicate URLs (www vs non-www, HTTP vs HTTPS, with/without trailing slash).',
17
+ example: '<link rel="canonical" href="https://example.com/page/">',
18
+ learnMore: 'https://developers.google.com/search/docs/crawling-indexing/canonicalization'
19
+ }
20
+ });
12
21
  }
13
22
  return createResult({ id: 'canonical-exists', name: 'Canonical URL', category: 'technical', severity: 'warning' }, 'pass', 'Canonical URL is defined', { value: ctx.canonicalUrl });
14
23
  },
@@ -21,7 +30,16 @@ export const technicalRules = [
21
30
  description: 'HTML should have lang attribute',
22
31
  check: (ctx) => {
23
32
  if (!ctx.hasLang) {
24
- return createResult({ id: 'lang-exists', name: 'Language', category: 'technical', severity: 'warning' }, 'warn', 'Missing lang attribute on <html>', { recommendation: 'Add lang attribute: <html lang="en">' });
33
+ return createResult({ id: 'lang-exists', name: 'Language', category: 'technical', severity: 'warning' }, 'warn', 'Missing lang attribute on <html>', {
34
+ recommendation: 'Declare the page language for better accessibility and SEO.',
35
+ evidence: {
36
+ found: '<html> tag without lang attribute',
37
+ expected: '<html lang="en"> (or appropriate language code)',
38
+ impact: 'The lang attribute helps screen readers pronounce content correctly and helps search engines serve the right audience. It\'s required for WCAG accessibility compliance.',
39
+ example: '<html lang="en">\n<html lang="pt-BR">\n<html lang="es">',
40
+ learnMore: 'https://developer.mozilla.org/en-US/docs/Web/HTML/Global_attributes/lang'
41
+ }
42
+ });
25
43
  }
26
44
  return createResult({ id: 'lang-exists', name: 'Language', category: 'technical', severity: 'warning' }, 'pass', `Language attribute set (${ctx.langValue})`);
27
45
  },
@@ -34,10 +52,27 @@ export const technicalRules = [
34
52
  description: 'Page should declare character encoding',
35
53
  check: (ctx) => {
36
54
  if (!ctx.hasCharset) {
37
- return createResult({ id: 'charset-exists', name: 'Charset', category: 'technical', severity: 'warning' }, 'warn', 'Missing charset declaration', { recommendation: 'Add <meta charset="UTF-8"> in <head>' });
55
+ return createResult({ id: 'charset-exists', name: 'Charset', category: 'technical', severity: 'warning' }, 'warn', 'Missing charset declaration', {
56
+ recommendation: 'Declare character encoding to ensure proper text rendering.',
57
+ evidence: {
58
+ found: 'No charset meta tag',
59
+ expected: '<meta charset="UTF-8"> as the first element in <head>',
60
+ impact: 'Without charset declaration, browsers may guess incorrectly, causing garbled text (mojibake). UTF-8 supports all languages and special characters.',
61
+ example: '<head>\n <meta charset="UTF-8">\n <!-- other tags -->\n</head>',
62
+ learnMore: 'https://developer.mozilla.org/en-US/docs/Web/HTML/Element/meta#charset'
63
+ }
64
+ });
38
65
  }
39
66
  if (ctx.charset && ctx.charset.toLowerCase() !== 'utf-8') {
40
- return createResult({ id: 'charset-exists', name: 'Charset', category: 'technical', severity: 'warning' }, 'warn', `Non-UTF-8 charset: ${ctx.charset}`, { recommendation: 'Use UTF-8 charset for best compatibility' });
67
+ return createResult({ id: 'charset-exists', name: 'Charset', category: 'technical', severity: 'warning' }, 'warn', `Non-UTF-8 charset: ${ctx.charset}`, {
68
+ recommendation: 'Switch to UTF-8 encoding for universal compatibility.',
69
+ evidence: {
70
+ found: `charset="${ctx.charset}"`,
71
+ expected: 'charset="UTF-8"',
72
+ impact: 'UTF-8 is the standard encoding that supports all languages. Legacy encodings like ISO-8859-1 or Windows-1252 cannot display all characters.',
73
+ learnMore: 'https://www.w3.org/International/questions/qa-choosing-encodings'
74
+ }
75
+ });
41
76
  }
42
77
  return createResult({ id: 'charset-exists', name: 'Charset', category: 'technical', severity: 'warning' }, 'pass', 'UTF-8 charset declared');
43
78
  },
@@ -49,10 +84,21 @@ export const technicalRules = [
49
84
  severity: 'warning',
50
85
  description: 'Check if page is set to noindex',
51
86
  check: (ctx) => {
52
- if (!ctx.metaRobots || ctx.metaRobots.length === 0)
53
- return null;
87
+ if (!ctx.metaRobots || ctx.metaRobots.length === 0) {
88
+ return createResult({ id: 'robots-noindex', name: 'Robots', category: 'technical', severity: 'warning' }, 'info', 'Not applicable (no robots meta tag)', { recommendation: 'This rule checks for noindex directive in robots meta tag' });
89
+ }
54
90
  if (ctx.metaRobots.includes('noindex')) {
55
- return createResult({ id: 'robots-noindex', name: 'Robots', category: 'technical', severity: 'warning' }, 'warn', 'Page is set to noindex', { value: ctx.metaRobots.join(', '), recommendation: 'Remove noindex if you want the page to be indexed' });
91
+ return createResult({ id: 'robots-noindex', name: 'Robots', category: 'technical', severity: 'warning' }, 'warn', 'Page is set to noindex', {
92
+ value: ctx.metaRobots.join(', '),
93
+ recommendation: 'Verify that noindex is intentional. This page will NOT appear in search results.',
94
+ evidence: {
95
+ found: `<meta name="robots" content="${ctx.metaRobots.join(', ')}">`,
96
+ expected: 'No noindex directive (for pages you want indexed)',
97
+ impact: 'The noindex directive completely prevents this page from appearing in Google, Bing, and other search engines. This is useful for thank-you pages or admin areas, but harmful if applied accidentally.',
98
+ example: 'To allow indexing: <meta name="robots" content="index, follow">',
99
+ learnMore: 'https://developers.google.com/search/docs/crawling-indexing/block-indexing'
100
+ }
101
+ });
56
102
  }
57
103
  return createResult({ id: 'robots-noindex', name: 'Robots', category: 'technical', severity: 'info' }, 'info', `Robots meta: ${ctx.metaRobots.join(', ')}`);
58
104
  },
@@ -65,7 +111,16 @@ export const technicalRules = [
65
111
  description: 'Page should have a favicon defined',
66
112
  check: (ctx) => {
67
113
  if (!ctx.hasFavicon) {
68
- return createResult({ id: 'favicon-exists', name: 'Favicon', category: 'technical', severity: 'warning' }, 'warn', 'No favicon defined', { recommendation: 'Add <link rel="icon" href="/favicon.ico"> for browser tab icon' });
114
+ return createResult({ id: 'favicon-exists', name: 'Favicon', category: 'technical', severity: 'warning' }, 'warn', 'No favicon defined', {
115
+ recommendation: 'Add a favicon to improve brand recognition and user experience.',
116
+ evidence: {
117
+ found: 'No favicon link tag',
118
+ expected: '<link rel="icon"> pointing to your favicon',
119
+ impact: 'Favicons appear in browser tabs, bookmarks, history, and search results. Missing favicons look unprofessional and generate 404 errors in server logs.',
120
+ example: '<link rel="icon" href="/favicon.ico" sizes="32x32">\n<link rel="icon" href="/icon.svg" type="image/svg+xml">\n<link rel="apple-touch-icon" href="/apple-touch-icon.png">',
121
+ learnMore: 'https://web.dev/articles/add-manifest#icons'
122
+ }
123
+ });
69
124
  }
70
125
  return createResult({ id: 'favicon-exists', name: 'Favicon', category: 'technical', severity: 'warning' }, 'pass', 'Favicon is defined', { value: ctx.faviconUrl });
71
126
  },
@@ -78,9 +133,18 @@ export const technicalRules = [
78
133
  description: 'URLs should be lowercase for consistency',
79
134
  check: (ctx) => {
80
135
  if (ctx.urlHasUppercase) {
81
- return createResult({ id: 'url-lowercase', name: 'URL Lowercase', category: 'technical', severity: 'warning' }, 'warn', 'URL contains uppercase characters', { recommendation: 'Use lowercase URLs for better SEO consistency' });
136
+ return createResult({ id: 'url-lowercase', name: 'URL Lowercase', category: 'technical', severity: 'warning' }, 'warn', 'URL contains uppercase characters', {
137
+ recommendation: 'Convert URL to lowercase for consistency and to avoid duplicate content.',
138
+ evidence: {
139
+ found: ctx.url,
140
+ expected: 'All lowercase URL path',
141
+ impact: 'URLs are case-sensitive on most servers. /Page and /page are different URLs, which can split ranking signals and cause duplicate content issues.',
142
+ example: 'Instead of /About-Us, use /about-us',
143
+ learnMore: 'https://developers.google.com/search/docs/crawling-indexing/url-structure'
144
+ }
145
+ });
82
146
  }
83
- return null;
147
+ return createResult({ id: 'url-lowercase', name: 'URL Lowercase', category: 'technical', severity: 'warning' }, 'info', 'Not applicable (URL is properly lowercase)', { recommendation: 'This rule ensures URLs use lowercase to avoid duplicate content issues' });
84
148
  },
85
149
  },
86
150
  {
@@ -96,9 +160,18 @@ export const technicalRules = [
96
160
  issues.push('accents');
97
161
  if (ctx.urlHasSpecialChars)
98
162
  issues.push('special characters');
99
- return createResult({ id: 'url-clean', name: 'URL Clean', category: 'technical', severity: 'warning' }, 'warn', `URL contains ${issues.join(' and ')}`, { recommendation: 'Use clean URLs without accents or special characters' });
163
+ return createResult({ id: 'url-clean', name: 'URL Clean', category: 'technical', severity: 'warning' }, 'warn', `URL contains ${issues.join(' and ')}`, {
164
+ recommendation: 'Use clean, ASCII-only URLs for maximum compatibility.',
165
+ evidence: {
166
+ found: ctx.url,
167
+ expected: 'URL with only letters, numbers, and hyphens',
168
+ impact: 'Accented characters and special characters get percent-encoded (%C3%A9 instead of é), making URLs ugly and hard to share. Some systems may not handle them correctly.',
169
+ example: 'Instead of /café-menü, use /cafe-menu',
170
+ learnMore: 'https://developers.google.com/search/docs/crawling-indexing/url-structure'
171
+ }
172
+ });
100
173
  }
101
- return null;
174
+ return createResult({ id: 'url-clean', name: 'URL Clean', category: 'technical', severity: 'warning' }, 'info', 'Not applicable (URL is clean)', { recommendation: 'This rule checks for accents and special characters in URLs' });
102
175
  },
103
176
  },
104
177
  {
@@ -108,17 +181,28 @@ export const technicalRules = [
108
181
  severity: 'warning',
109
182
  description: 'URLs should not contain query parameters',
110
183
  check: (ctx) => {
111
- if (!ctx.url)
112
- return null;
184
+ if (!ctx.url) {
185
+ return createResult({ id: 'url-no-params', name: 'URL Parameters', category: 'technical', severity: 'warning' }, 'info', 'Not applicable (URL data unavailable)', { recommendation: 'This rule checks for query parameters that can cause duplicate content' });
186
+ }
113
187
  try {
114
188
  const urlObj = new URL(ctx.url);
115
189
  if (urlObj.search && urlObj.search.length > 1) {
116
- return createResult({ id: 'url-no-params', name: 'URL Parameters', category: 'technical', severity: 'warning' }, 'warn', 'URL contains query parameters', { value: urlObj.search, recommendation: 'Use clean URLs (rewritten paths) without query parameters' });
190
+ return createResult({ id: 'url-no-params', name: 'URL Parameters', category: 'technical', severity: 'warning' }, 'warn', 'URL contains query parameters', {
191
+ value: urlObj.search,
192
+ recommendation: 'Consider using clean, path-based URLs instead of query parameters.',
193
+ evidence: {
194
+ found: urlObj.search,
195
+ expected: 'Clean URL without query strings',
196
+ impact: 'Query parameters make URLs harder to read and share. Google recommends using simple, descriptive URLs. Parameters can also cause crawl budget waste if they create many URL variations.',
197
+ example: 'Instead of /products?category=shoes&color=red\nUse /products/shoes/red',
198
+ learnMore: 'https://developers.google.com/search/docs/crawling-indexing/url-structure#use-simple-urls'
199
+ }
200
+ });
117
201
  }
118
202
  }
119
203
  catch {
120
204
  }
121
- return null;
205
+ return createResult({ id: 'url-no-params', name: 'URL Parameters', category: 'technical', severity: 'warning' }, 'info', 'Not applicable (no query parameters)', { recommendation: 'Clean URLs without parameters are preferred for SEO' });
122
206
  },
123
207
  },
124
208
  {
@@ -128,14 +212,24 @@ export const technicalRules = [
128
212
  severity: 'warning',
129
213
  description: 'Check for restrictive meta robots directives like noindex, nofollow, noarchive etc.',
130
214
  check: (ctx) => {
131
- if (!ctx.metaRobots || ctx.metaRobots.length === 0)
132
- return null;
215
+ if (!ctx.metaRobots || ctx.metaRobots.length === 0) {
216
+ return createResult({ id: 'technical-meta-robots-directives', name: 'Meta Robots Directives', category: 'technical', severity: 'warning' }, 'info', 'Not applicable (no robots meta tag)', { recommendation: 'This rule checks for restrictive robots directives' });
217
+ }
133
218
  const restrictiveDirectives = ['noindex', 'nofollow', 'noarchive', 'nosnippet', 'noimageindex'];
134
219
  const foundRestrictive = ctx.metaRobots.filter(directive => restrictiveDirectives.includes(directive));
135
220
  if (foundRestrictive.length > 0) {
136
- return createResult({ id: 'technical-meta-robots-directives', name: 'Meta Robots Directives', category: 'technical', severity: 'warning' }, 'warn', `Restrictive meta robots directives found: ${foundRestrictive.join(', ')}`, { recommendation: 'Ensure these directives are intentional to prevent unintended blocking of indexing or crawling.' });
221
+ return createResult({ id: 'technical-meta-robots-directives', name: 'Meta Robots Directives', category: 'technical', severity: 'warning' }, 'warn', `Restrictive meta robots directives found: ${foundRestrictive.join(', ')}`, {
222
+ recommendation: 'Verify these restrictive directives are intentional.',
223
+ evidence: {
224
+ found: foundRestrictive.join(', '),
225
+ expected: 'index, follow (for pages you want in search results)',
226
+ impact: 'noindex: page won\'t appear in search. nofollow: links won\'t pass PageRank. noarchive: no cached version. nosnippet: no description in results. noimageindex: images not indexed.',
227
+ example: 'To allow everything: <meta name="robots" content="index, follow, archive, snippet">',
228
+ learnMore: 'https://developers.google.com/search/docs/crawling-indexing/robots-meta-tag'
229
+ }
230
+ });
137
231
  }
138
- return null;
232
+ return createResult({ id: 'technical-meta-robots-directives', name: 'Meta Robots Directives', category: 'technical', severity: 'warning' }, 'info', 'Not applicable (no restrictive directives found)', { recommendation: 'Meta robots directives allow full indexing' });
139
233
  },
140
234
  },
141
235
  {
@@ -145,13 +239,21 @@ export const technicalRules = [
145
239
  severity: 'info',
146
240
  description: 'X-Robots-Tag header can be used to control indexing, especially for non-HTML content.',
147
241
  check: (ctx) => {
148
- if (!ctx.responseHeaders)
149
- return null;
242
+ if (!ctx.responseHeaders) {
243
+ return createResult({ id: 'technical-x-robots-tag', name: 'X-Robots-Tag Header', category: 'technical', severity: 'info' }, 'info', 'Not applicable (response headers unavailable)', { recommendation: 'This rule checks for X-Robots-Tag HTTP header' });
244
+ }
150
245
  const xRobotsTag = ctx.responseHeaders['x-robots-tag'] || ctx.responseHeaders['X-Robots-Tag'];
151
246
  if (xRobotsTag) {
152
- return createResult({ id: 'technical-x-robots-tag', name: 'X-Robots-Tag Header', category: 'technical', severity: 'info' }, 'info', `X-Robots-Tag header found: ${xRobotsTag}`, { recommendation: 'Ensure X-Robots-Tag directives (e.g., noindex) are intentional.' });
247
+ return createResult({ id: 'technical-x-robots-tag', name: 'X-Robots-Tag Header', category: 'technical', severity: 'info' }, 'info', `X-Robots-Tag header found: ${xRobotsTag}`, {
248
+ recommendation: 'Verify this server-level robots directive is intentional.',
249
+ evidence: {
250
+ found: `X-Robots-Tag: ${xRobotsTag}`,
251
+ impact: 'X-Robots-Tag HTTP header works like meta robots but at server level. It\'s commonly used for non-HTML files (PDFs, images) but can also override meta tags for HTML pages.',
252
+ learnMore: 'https://developers.google.com/search/docs/crawling-indexing/robots-meta-tag#xrobotstag'
253
+ }
254
+ });
153
255
  }
154
- return null;
256
+ return createResult({ id: 'technical-x-robots-tag', name: 'X-Robots-Tag Header', category: 'technical', severity: 'info' }, 'info', 'Not applicable (no X-Robots-Tag header)', { recommendation: 'X-Robots-Tag header is not present' });
155
257
  },
156
258
  },
157
259
  {
@@ -171,9 +273,18 @@ export const technicalRules = [
171
273
  if (!ctx.hasTermsOfServiceLink)
172
274
  missingSignals.push('Terms of Service');
173
275
  if (missingSignals.length > 0) {
174
- return createResult({ id: 'technical-trust-signals', name: 'Trust Signals (Links)', category: 'technical', severity: 'info' }, 'info', `Missing links to key trust pages: ${missingSignals.join(', ')}`, { recommendation: 'Add clear links to "About Us", "Contact", "Privacy Policy", and "Terms of Service" pages to build user and search engine trust.' });
276
+ return createResult({ id: 'technical-trust-signals', name: 'Trust Signals (Links)', category: 'technical', severity: 'info' }, 'info', `Missing links to key trust pages: ${missingSignals.join(', ')}`, {
277
+ recommendation: 'Add links to trust-building pages to improve E-E-A-T signals.',
278
+ evidence: {
279
+ found: `Missing: ${missingSignals.join(', ')}`,
280
+ expected: 'Links to About, Contact, Privacy Policy, and Terms of Service',
281
+ impact: 'Google\'s E-E-A-T (Experience, Expertise, Authoritativeness, Trustworthiness) guidelines favor sites that clearly identify who they are. These pages are essential for YMYL (Your Money Your Life) sites.',
282
+ example: '<footer>\n <a href="/about">About Us</a>\n <a href="/contact">Contact</a>\n <a href="/privacy">Privacy Policy</a>\n <a href="/terms">Terms of Service</a>\n</footer>',
283
+ learnMore: 'https://developers.google.com/search/docs/fundamentals/creating-helpful-content'
284
+ }
285
+ });
175
286
  }
176
- return null;
287
+ return createResult({ id: 'technical-trust-signals', name: 'Trust Signals (Links)', category: 'technical', severity: 'info' }, 'info', 'Not applicable (all trust signals present)', { recommendation: 'All key trust pages are linked' });
177
288
  },
178
289
  },
179
290
  {
@@ -183,13 +294,22 @@ export const technicalRules = [
183
294
  severity: 'info',
184
295
  description: 'A higher text to HTML ratio indicates more content relative to code, which is good for SEO.',
185
296
  check: (ctx) => {
186
- if (ctx.textHtmlRatio === undefined)
187
- return null;
297
+ if (ctx.textHtmlRatio === undefined) {
298
+ return createResult({ id: 'technical-text-html-ratio', name: 'Text/HTML Ratio', category: 'technical', severity: 'info' }, 'info', 'Not applicable (text/HTML ratio data unavailable)', { recommendation: 'This rule checks the ratio of visible text to HTML code' });
299
+ }
188
300
  const threshold = 15;
189
301
  if (ctx.textHtmlRatio < threshold) {
190
- return createResult({ id: 'technical-text-html-ratio', name: 'Text/HTML Ratio', category: 'technical', severity: 'info' }, 'warn', `Low Text/HTML ratio: ${ctx.textHtmlRatio.toFixed(2)}% (target > ${threshold}%)`, { recommendation: 'Increase text content relative to HTML code. Reduce unnecessary markup, or add more textual content.' });
302
+ return createResult({ id: 'technical-text-html-ratio', name: 'Text/HTML Ratio', category: 'technical', severity: 'info' }, 'warn', `Low Text/HTML ratio: ${ctx.textHtmlRatio.toFixed(2)}% (target > ${threshold}%)`, {
303
+ recommendation: 'Increase visible text content relative to HTML code.',
304
+ evidence: {
305
+ found: `${ctx.textHtmlRatio.toFixed(2)}% text content`,
306
+ expected: `At least ${threshold}% text content`,
307
+ impact: 'Pages with low text-to-HTML ratio may appear thin or low-quality to search engines. This can indicate excessive scripts, inline CSS, or insufficient content.',
308
+ learnMore: 'https://developers.google.com/search/docs/fundamentals/creating-helpful-content#content-and-quality'
309
+ }
310
+ });
191
311
  }
192
- return null;
312
+ return createResult({ id: 'technical-text-html-ratio', name: 'Text/HTML Ratio', category: 'technical', severity: 'info' }, 'info', 'Not applicable (text/HTML ratio is good)', { recommendation: `Text/HTML ratio is ${ctx.textHtmlRatio.toFixed(2)}%, which is healthy` });
193
313
  },
194
314
  },
195
315
  {
@@ -199,7 +319,15 @@ export const technicalRules = [
199
319
  severity: 'info',
200
320
  description: 'Ensure a robots.txt file exists at the root of the domain to guide crawlers.',
201
321
  check: (ctx) => {
202
- return createResult({ id: 'technical-robots-txt-hint', name: 'Robots.txt Hint', category: 'technical', severity: 'info' }, 'info', 'Robots.txt existence cannot be verified from HTML alone.', { recommendation: 'Ensure a valid `robots.txt` file is present at your domain root (e.g., `https://example.com/robots.txt`) to guide search engine crawlers and define your sitemap location.' });
322
+ return createResult({ id: 'technical-robots-txt-hint', name: 'Robots.txt Hint', category: 'technical', severity: 'info' }, 'info', 'Robots.txt existence cannot be verified from HTML alone.', {
323
+ recommendation: 'Ensure you have a valid robots.txt file at your domain root.',
324
+ evidence: {
325
+ expected: 'A robots.txt file at /robots.txt',
326
+ impact: 'robots.txt tells search engines which pages to crawl. It should also point to your sitemap.xml for efficient discovery of all pages.',
327
+ example: '# Example robots.txt\nUser-agent: *\nAllow: /\nDisallow: /admin/\nDisallow: /private/\n\nSitemap: https://example.com/sitemap.xml',
328
+ learnMore: 'https://developers.google.com/search/docs/crawling-indexing/robots/create-robots-txt'
329
+ }
330
+ });
203
331
  },
204
332
  },
205
333
  {
@@ -209,8 +337,9 @@ export const technicalRules = [
209
337
  severity: 'warning',
210
338
  description: 'URLs should not have more than 3 query parameters',
211
339
  check: (ctx) => {
212
- if (!ctx.url)
213
- return null;
340
+ if (!ctx.url) {
341
+ return createResult({ id: 'url-many-parameters', name: 'Too Many URL Parameters', category: 'technical', severity: 'warning' }, 'info', 'Not applicable (URL data unavailable)', { recommendation: 'This rule checks for excessive URL parameters' });
342
+ }
214
343
  try {
215
344
  const url = new URL(ctx.url);
216
345
  const paramCount = Array.from(url.searchParams.keys()).length;
@@ -221,14 +350,15 @@ export const technicalRules = [
221
350
  evidence: {
222
351
  found: url.search,
223
352
  expected: '3 or fewer parameters',
224
- impact: 'Multiple parameters make URLs less enticing and may cause indexing issues'
353
+ impact: 'Multiple parameters make URLs less enticing to click and may cause indexing issues. Search engines may treat parameter variations as duplicate content, wasting crawl budget.',
354
+ learnMore: 'https://developers.google.com/search/docs/crawling-indexing/url-structure#use-simple-urls'
225
355
  }
226
356
  });
227
357
  }
228
358
  }
229
359
  catch {
230
360
  }
231
- return null;
361
+ return createResult({ id: 'url-many-parameters', name: 'Too Many URL Parameters', category: 'technical', severity: 'warning' }, 'info', 'Not applicable (3 or fewer URL parameters)', { recommendation: 'URL parameters are within acceptable limits' });
232
362
  },
233
363
  },
234
364
  {
@@ -238,8 +368,9 @@ export const technicalRules = [
238
368
  severity: 'error',
239
369
  description: 'Pages should not use Flash, Java Applets, or Silverlight',
240
370
  check: (ctx) => {
241
- if (ctx.hasDeprecatedPlugins === undefined)
242
- return null;
371
+ if (ctx.hasDeprecatedPlugins === undefined) {
372
+ return createResult({ id: 'deprecated-plugins', name: 'Deprecated Plugins', category: 'technical', severity: 'error' }, 'info', 'Not applicable (deprecated plugins data unavailable)', { recommendation: 'This rule checks for Flash, Java, and Silverlight usage' });
373
+ }
243
374
  if (ctx.hasDeprecatedPlugins) {
244
375
  return createResult({ id: 'deprecated-plugins', name: 'Deprecated Plugins', category: 'technical', severity: 'error' }, 'fail', 'Page uses deprecated plugins (Flash, Java, or Silverlight)', {
245
376
  recommendation: 'Convert plugin content to HTML5',
@@ -251,7 +382,7 @@ export const technicalRules = [
251
382
  }
252
383
  });
253
384
  }
254
- return null;
385
+ return createResult({ id: 'deprecated-plugins', name: 'Deprecated Plugins', category: 'technical', severity: 'error' }, 'info', 'Not applicable (no deprecated plugins detected)', { recommendation: 'No Flash, Java, or Silverlight plugins found' });
255
386
  },
256
387
  },
257
388
  {
@@ -261,19 +392,89 @@ export const technicalRules = [
261
392
  severity: 'error',
262
393
  description: 'Pages should not use <frame> or <frameset> tags',
263
394
  check: (ctx) => {
264
- if (ctx.hasFrameTags === undefined)
265
- return null;
395
+ if (ctx.hasFrameTags === undefined) {
396
+ return createResult({ id: 'frame-tags', name: 'Frame Tags', category: 'technical', severity: 'error' }, 'info', 'Not applicable (frame tags data unavailable)', { recommendation: 'This rule checks for obsolete <frame> and <frameset> tags' });
397
+ }
266
398
  if (ctx.hasFrameTags) {
267
399
  return createResult({ id: 'frame-tags', name: 'Frame Tags', category: 'technical', severity: 'error' }, 'fail', 'Page uses <frame> or <frameset> tags', {
268
400
  recommendation: 'Remove frame tags and restructure using modern HTML',
269
401
  evidence: {
270
402
  found: '<frame> or <frameset> tags detected',
271
403
  expected: 'No frame tags',
272
- impact: 'Search engines have difficulty indexing content within frames, affecting rankings'
404
+ impact: 'Frames are obsolete HTML. Search engines have difficulty indexing content within frames. Mobile devices don\'t support them well. Use modern CSS layouts instead.',
405
+ learnMore: 'https://developer.mozilla.org/en-US/docs/Web/HTML/Element/frame'
406
+ }
407
+ });
408
+ }
409
+ return createResult({ id: 'frame-tags', name: 'Frame Tags', category: 'technical', severity: 'error' }, 'info', 'Not applicable (no frame tags detected)', { recommendation: 'No obsolete frame tags found' });
410
+ },
411
+ },
412
+ {
413
+ id: 'avoid-iframes',
414
+ name: 'Avoid iFrames',
415
+ category: 'technical',
416
+ severity: 'info',
417
+ description: 'iFrames can cause SEO and usability issues.',
418
+ check: (ctx) => {
419
+ if (ctx.iframeCount && ctx.iframeCount > 0) {
420
+ return createResult({ id: 'avoid-iframes', name: 'iFrames Usage', category: 'technical', severity: 'info' }, 'info', `${ctx.iframeCount} iFrame(s) detected`, {
421
+ value: ctx.iframeCount,
422
+ recommendation: 'Avoid iFrames for main content. They can be hard to index and navigate.',
423
+ evidence: {
424
+ found: `${ctx.iframeCount} iframes`,
425
+ expected: 'Minimal iframe usage, only for embeds (videos, maps)',
426
+ impact: 'Content inside iframes is treated as separate pages and may not contribute to the parent page\'s SEO. Iframes can also slow page load and cause layout shifts.',
427
+ example: 'Acceptable: <iframe src="https://www.youtube.com/embed/..." loading="lazy">\nAvoid: Main content inside iframes',
428
+ learnMore: 'https://developers.google.com/search/docs/crawling-indexing/mobile/mobile-sites-mobile-first-indexing#iframes'
429
+ }
430
+ });
431
+ }
432
+ return createResult({ id: 'avoid-iframes', name: 'Avoid iFrames', category: 'technical', severity: 'info' }, 'info', 'Not applicable (no iframes detected)', { recommendation: 'No iframes found on this page' });
433
+ },
434
+ },
435
+ {
436
+ id: 'deprecated-html-tags',
437
+ name: 'Deprecated HTML Tags',
438
+ category: 'technical',
439
+ severity: 'warning',
440
+ description: 'Avoid deprecated HTML tags like <center>, <font>, <marquee>',
441
+ check: (ctx) => {
442
+ if (ctx.deprecatedTagsCount && ctx.deprecatedTagsCount > 0) {
443
+ return createResult({ id: 'deprecated-html-tags', name: 'Deprecated HTML Tags', category: 'technical', severity: 'warning' }, 'warn', `${ctx.deprecatedTagsCount} deprecated HTML tags found`, {
444
+ value: ctx.deprecatedTagsCount,
445
+ recommendation: 'Replace deprecated tags (center, font, marquee, etc.) with CSS.',
446
+ evidence: {
447
+ found: ctx.deprecatedTagsFound?.join(', '),
448
+ expected: 'Modern HTML5 elements with CSS styling',
449
+ impact: 'Deprecated tags like <center>, <font>, and <marquee> are obsolete. They indicate outdated code that may not render correctly in all browsers.',
450
+ example: 'Instead of <center>text</center>, use <div style="text-align: center">text</div>\nInstead of <font color="red">, use <span style="color: red">',
451
+ learnMore: 'https://developer.mozilla.org/en-US/docs/Web/HTML/Element#obsolete_and_deprecated_elements'
452
+ }
453
+ });
454
+ }
455
+ return createResult({ id: 'deprecated-html-tags', name: 'Deprecated HTML Tags', category: 'technical', severity: 'warning' }, 'info', 'Not applicable (no deprecated tags detected)', { recommendation: 'No obsolete HTML tags found' });
456
+ },
457
+ },
458
+ {
459
+ id: 'apple-touch-icon',
460
+ name: 'Apple Touch Icon',
461
+ category: 'technical',
462
+ severity: 'info',
463
+ description: 'Add an apple-touch-icon for iOS devices',
464
+ check: (ctx) => {
465
+ if (ctx.hasAppleTouchIcon === false) {
466
+ return createResult({ id: 'apple-touch-icon', name: 'Apple Touch Icon', category: 'technical', severity: 'info' }, 'info', 'Missing apple-touch-icon', {
467
+ recommendation: 'Add an Apple Touch Icon for iOS devices.',
468
+ evidence: {
469
+ found: 'No apple-touch-icon link tag',
470
+ expected: '<link rel="apple-touch-icon"> pointing to a 180x180 PNG',
471
+ impact: 'Apple Touch Icons appear when users add your site to their iOS home screen. Without one, iOS will use a screenshot of your page, which often looks poor.',
472
+ example: '<link rel="apple-touch-icon" sizes="180x180" href="/apple-touch-icon.png">',
473
+ learnMore: 'https://developer.apple.com/library/archive/documentation/AppleApplications/Reference/SafariWebContent/ConfiguringWebApplications/ConfiguringWebApplications.html'
273
474
  }
274
475
  });
275
476
  }
276
- return null;
477
+ return createResult({ id: 'apple-touch-icon', name: 'Apple Touch Icon', category: 'technical', severity: 'info' }, 'pass', 'Apple Touch Icon present');
277
478
  },
278
479
  },
279
480
  ];