recker 1.0.43 → 1.0.44

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (459) hide show
  1. package/README.md +47 -0
  2. package/dist/bin/recker-linux-x64 +0 -0
  3. package/dist/bin/recker-macos-x64 +0 -0
  4. package/dist/bin/recker-win-x64.exe +0 -0
  5. package/dist/bin/rek.cjs +85152 -100207
  6. package/dist/browser/ai/adaptive-timeout.d.ts +50 -0
  7. package/dist/browser/ai/adaptive-timeout.js +208 -0
  8. package/dist/browser/ai/client.d.ts +22 -0
  9. package/dist/browser/ai/client.js +294 -0
  10. package/dist/browser/ai/index.d.ts +14 -0
  11. package/dist/browser/ai/index.js +11 -0
  12. package/dist/browser/ai/providers/anthropic.d.ts +63 -0
  13. package/dist/browser/ai/providers/anthropic.js +370 -0
  14. package/dist/browser/ai/providers/base.d.ts +48 -0
  15. package/dist/browser/ai/providers/base.js +150 -0
  16. package/dist/browser/ai/providers/google.d.ts +59 -0
  17. package/dist/browser/ai/providers/google.js +305 -0
  18. package/dist/browser/ai/providers/ollama.d.ts +44 -0
  19. package/dist/browser/ai/providers/ollama.js +240 -0
  20. package/dist/browser/ai/providers/openai.d.ts +64 -0
  21. package/dist/browser/ai/providers/openai.js +298 -0
  22. package/dist/browser/ai/rate-limiter.d.ts +43 -0
  23. package/dist/browser/ai/rate-limiter.js +215 -0
  24. package/dist/browser/ai/vector/index.d.ts +2 -0
  25. package/dist/browser/ai/vector/index.js +2 -0
  26. package/dist/browser/ai/vector/similarity.d.ts +2 -0
  27. package/dist/browser/ai/vector/similarity.js +27 -0
  28. package/dist/browser/ai/vector/store.d.ts +27 -0
  29. package/dist/browser/ai/vector/store.js +82 -0
  30. package/dist/browser/browser/cache.d.ts +2 -40
  31. package/dist/browser/browser/cache.js +2 -199
  32. package/dist/browser/browser/index.d.ts +8 -0
  33. package/dist/browser/browser/index.js +8 -0
  34. package/dist/browser/browser/recker.d.ts +8 -1
  35. package/dist/browser/browser/recker.js +8 -2
  36. package/dist/browser/cache/indexed-db.d.ts +10 -0
  37. package/dist/browser/cache/indexed-db.js +88 -0
  38. package/dist/browser/cache/service-worker-cache.d.ts +18 -0
  39. package/dist/browser/cache/service-worker-cache.js +103 -0
  40. package/dist/browser/cache.d.ts +2 -40
  41. package/dist/browser/cache.js +2 -199
  42. package/dist/browser/constants/user-agents.d.ts +7 -0
  43. package/dist/browser/constants/user-agents.js +7 -0
  44. package/dist/browser/core/client.d.ts +2 -0
  45. package/dist/browser/core/client.js +19 -1
  46. package/dist/browser/index.d.ts +8 -0
  47. package/dist/browser/index.js +8 -0
  48. package/dist/browser/plugins/har-recorder.d.ts +40 -0
  49. package/dist/browser/plugins/har-recorder.js +120 -0
  50. package/dist/browser/plugins/network-simulation.d.ts +7 -0
  51. package/dist/browser/plugins/network-simulation.js +13 -0
  52. package/dist/browser/presets/android.d.ts +2 -0
  53. package/dist/browser/presets/android.js +16 -0
  54. package/dist/browser/presets/anthropic.d.ts +8 -0
  55. package/dist/browser/presets/anthropic.js +27 -0
  56. package/dist/browser/presets/aws.d.ts +19 -0
  57. package/dist/browser/presets/aws.js +68 -0
  58. package/dist/browser/presets/azure-openai.d.ts +10 -0
  59. package/dist/browser/presets/azure-openai.js +35 -0
  60. package/dist/browser/presets/azure.d.ts +41 -0
  61. package/dist/browser/presets/azure.js +104 -0
  62. package/dist/browser/presets/chaturbate.d.ts +2 -0
  63. package/dist/browser/presets/chaturbate.js +17 -0
  64. package/dist/browser/presets/cloudflare.d.ts +12 -0
  65. package/dist/browser/presets/cloudflare.js +39 -0
  66. package/dist/browser/presets/cohere.d.ts +7 -0
  67. package/dist/browser/presets/cohere.js +22 -0
  68. package/dist/browser/presets/deepseek.d.ts +7 -0
  69. package/dist/browser/presets/deepseek.js +22 -0
  70. package/dist/browser/presets/digitalocean.d.ts +5 -0
  71. package/dist/browser/presets/digitalocean.js +16 -0
  72. package/dist/browser/presets/discord.d.ts +6 -0
  73. package/dist/browser/presets/discord.js +17 -0
  74. package/dist/browser/presets/elevenlabs.d.ts +6 -0
  75. package/dist/browser/presets/elevenlabs.js +20 -0
  76. package/dist/browser/presets/enhancers.d.ts +20 -0
  77. package/dist/browser/presets/enhancers.js +85 -0
  78. package/dist/browser/presets/fireworks.d.ts +7 -0
  79. package/dist/browser/presets/fireworks.js +22 -0
  80. package/dist/browser/presets/gcp.d.ts +34 -0
  81. package/dist/browser/presets/gcp.js +91 -0
  82. package/dist/browser/presets/gemini.d.ts +7 -0
  83. package/dist/browser/presets/gemini.js +23 -0
  84. package/dist/browser/presets/github.d.ts +6 -0
  85. package/dist/browser/presets/github.js +17 -0
  86. package/dist/browser/presets/gitlab.d.ts +6 -0
  87. package/dist/browser/presets/gitlab.js +16 -0
  88. package/dist/browser/presets/groq.d.ts +7 -0
  89. package/dist/browser/presets/groq.js +22 -0
  90. package/dist/browser/presets/hubspot.d.ts +9 -0
  91. package/dist/browser/presets/hubspot.js +28 -0
  92. package/dist/browser/presets/huggingface.d.ts +7 -0
  93. package/dist/browser/presets/huggingface.js +23 -0
  94. package/dist/browser/presets/index.d.ts +47 -0
  95. package/dist/browser/presets/index.js +47 -0
  96. package/dist/browser/presets/ios.d.ts +2 -0
  97. package/dist/browser/presets/ios.js +13 -0
  98. package/dist/browser/presets/linear.d.ts +5 -0
  99. package/dist/browser/presets/linear.js +16 -0
  100. package/dist/browser/presets/mailgun.d.ts +7 -0
  101. package/dist/browser/presets/mailgun.js +20 -0
  102. package/dist/browser/presets/meta.d.ts +10 -0
  103. package/dist/browser/presets/meta.js +33 -0
  104. package/dist/browser/presets/mistral.d.ts +7 -0
  105. package/dist/browser/presets/mistral.js +22 -0
  106. package/dist/browser/presets/notion.d.ts +6 -0
  107. package/dist/browser/presets/notion.js +17 -0
  108. package/dist/browser/presets/openai.d.ts +9 -0
  109. package/dist/browser/presets/openai.js +30 -0
  110. package/dist/browser/presets/oracle.d.ts +19 -0
  111. package/dist/browser/presets/oracle.js +117 -0
  112. package/dist/browser/presets/perplexity.d.ts +7 -0
  113. package/dist/browser/presets/perplexity.js +22 -0
  114. package/dist/browser/presets/pinecone.d.ts +8 -0
  115. package/dist/browser/presets/pinecone.js +42 -0
  116. package/dist/browser/presets/registry.d.ts +23 -0
  117. package/dist/browser/presets/registry.js +519 -0
  118. package/dist/browser/presets/replicate.d.ts +7 -0
  119. package/dist/browser/presets/replicate.js +23 -0
  120. package/dist/browser/presets/sendgrid.d.ts +6 -0
  121. package/dist/browser/presets/sendgrid.js +20 -0
  122. package/dist/browser/presets/sentry.d.ts +11 -0
  123. package/dist/browser/presets/sentry.js +48 -0
  124. package/dist/browser/presets/sinch.d.ts +9 -0
  125. package/dist/browser/presets/sinch.js +39 -0
  126. package/dist/browser/presets/slack.d.ts +5 -0
  127. package/dist/browser/presets/slack.js +16 -0
  128. package/dist/browser/presets/square.d.ts +10 -0
  129. package/dist/browser/presets/square.js +33 -0
  130. package/dist/browser/presets/stripe.d.ts +7 -0
  131. package/dist/browser/presets/stripe.js +23 -0
  132. package/dist/browser/presets/supabase.d.ts +6 -0
  133. package/dist/browser/presets/supabase.js +18 -0
  134. package/dist/browser/presets/tiktok.d.ts +10 -0
  135. package/dist/browser/presets/tiktok.js +38 -0
  136. package/dist/browser/presets/together.d.ts +7 -0
  137. package/dist/browser/presets/together.js +22 -0
  138. package/dist/browser/presets/twilio.d.ts +6 -0
  139. package/dist/browser/presets/twilio.js +17 -0
  140. package/dist/browser/presets/vercel.d.ts +6 -0
  141. package/dist/browser/presets/vercel.js +23 -0
  142. package/dist/browser/presets/vultr.d.ts +5 -0
  143. package/dist/browser/presets/vultr.js +16 -0
  144. package/dist/browser/presets/xai.d.ts +8 -0
  145. package/dist/browser/presets/xai.js +23 -0
  146. package/dist/browser/presets/youtube.d.ts +5 -0
  147. package/dist/browser/presets/youtube.js +20 -0
  148. package/dist/browser/recker.d.ts +8 -1
  149. package/dist/browser/recker.js +8 -2
  150. package/dist/browser/scrape/document.d.ts +5 -4
  151. package/dist/browser/scrape/document.js +89 -76
  152. package/dist/browser/scrape/element.d.ts +10 -8
  153. package/dist/browser/scrape/element.js +295 -81
  154. package/dist/browser/scrape/extractors.d.ts +11 -11
  155. package/dist/browser/scrape/extractors.js +145 -113
  156. package/dist/browser/scrape/parser/back.d.ts +1 -0
  157. package/dist/browser/scrape/parser/back.js +3 -0
  158. package/dist/browser/scrape/parser/index.d.ts +20 -0
  159. package/dist/browser/scrape/parser/index.js +19 -0
  160. package/dist/browser/scrape/parser/matcher.d.ts +30 -0
  161. package/dist/browser/scrape/parser/matcher.js +99 -0
  162. package/dist/browser/scrape/parser/nodes/comment.d.ts +12 -0
  163. package/dist/browser/scrape/parser/nodes/comment.js +21 -0
  164. package/dist/browser/scrape/parser/nodes/html.d.ts +110 -0
  165. package/dist/browser/scrape/parser/nodes/html.js +978 -0
  166. package/dist/browser/scrape/parser/nodes/node.d.ts +18 -0
  167. package/dist/browser/scrape/parser/nodes/node.js +31 -0
  168. package/dist/browser/scrape/parser/nodes/text.d.ts +14 -0
  169. package/dist/browser/scrape/parser/nodes/text.js +30 -0
  170. package/dist/browser/scrape/parser/nodes/type.d.ts +6 -0
  171. package/dist/browser/scrape/parser/nodes/type.js +7 -0
  172. package/dist/browser/scrape/parser/parse.d.ts +1 -0
  173. package/dist/browser/scrape/parser/parse.js +1 -0
  174. package/dist/browser/scrape/parser/valid.d.ts +2 -0
  175. package/dist/browser/scrape/parser/valid.js +5 -0
  176. package/dist/browser/scrape/parser/void-tag.d.ts +7 -0
  177. package/dist/browser/scrape/parser/void-tag.js +43 -0
  178. package/dist/browser/scrape/types.d.ts +7 -0
  179. package/dist/browser/seo/analyzer.d.ts +59 -0
  180. package/dist/browser/seo/analyzer.js +1399 -0
  181. package/dist/browser/seo/keywords.d.ts +16 -0
  182. package/dist/browser/seo/keywords.js +55 -0
  183. package/dist/browser/seo/rules/accessibility.d.ts +2 -0
  184. package/dist/browser/seo/rules/accessibility.js +733 -0
  185. package/dist/browser/seo/rules/ai-search.d.ts +2 -0
  186. package/dist/browser/seo/rules/ai-search.js +436 -0
  187. package/dist/browser/seo/rules/analytics.d.ts +2 -0
  188. package/dist/browser/seo/rules/analytics.js +306 -0
  189. package/dist/browser/seo/rules/best-practices.d.ts +2 -0
  190. package/dist/browser/seo/rules/best-practices.js +195 -0
  191. package/dist/browser/seo/rules/canonical.d.ts +12 -0
  192. package/dist/browser/seo/rules/canonical.js +270 -0
  193. package/dist/browser/seo/rules/content.d.ts +2 -0
  194. package/dist/browser/seo/rules/content.js +522 -0
  195. package/dist/browser/seo/rules/crawl.d.ts +2 -0
  196. package/dist/browser/seo/rules/crawl.js +435 -0
  197. package/dist/browser/seo/rules/cwv.d.ts +2 -0
  198. package/dist/browser/seo/rules/cwv.js +248 -0
  199. package/dist/browser/seo/rules/ecommerce.d.ts +2 -0
  200. package/dist/browser/seo/rules/ecommerce.js +312 -0
  201. package/dist/browser/seo/rules/i18n.d.ts +2 -0
  202. package/dist/browser/seo/rules/i18n.js +288 -0
  203. package/dist/browser/seo/rules/images.d.ts +2 -0
  204. package/dist/browser/seo/rules/images.js +255 -0
  205. package/dist/browser/seo/rules/index.d.ts +52 -0
  206. package/dist/browser/seo/rules/index.js +159 -0
  207. package/dist/browser/seo/rules/internal-linking.d.ts +2 -0
  208. package/dist/browser/seo/rules/internal-linking.js +394 -0
  209. package/dist/browser/seo/rules/links.d.ts +2 -0
  210. package/dist/browser/seo/rules/links.js +498 -0
  211. package/dist/browser/seo/rules/local.d.ts +2 -0
  212. package/dist/browser/seo/rules/local.js +289 -0
  213. package/dist/browser/seo/rules/meta.d.ts +2 -0
  214. package/dist/browser/seo/rules/meta.js +805 -0
  215. package/dist/browser/seo/rules/mobile.d.ts +2 -0
  216. package/dist/browser/seo/rules/mobile.js +161 -0
  217. package/dist/browser/seo/rules/performance.d.ts +2 -0
  218. package/dist/browser/seo/rules/performance.js +738 -0
  219. package/dist/browser/seo/rules/pwa.d.ts +2 -0
  220. package/dist/browser/seo/rules/pwa.js +299 -0
  221. package/dist/browser/seo/rules/readability.d.ts +2 -0
  222. package/dist/browser/seo/rules/readability.js +264 -0
  223. package/dist/browser/seo/rules/redirects.d.ts +16 -0
  224. package/dist/browser/seo/rules/redirects.js +199 -0
  225. package/dist/browser/seo/rules/resources.d.ts +2 -0
  226. package/dist/browser/seo/rules/resources.js +390 -0
  227. package/dist/browser/seo/rules/schema.d.ts +2 -0
  228. package/dist/browser/seo/rules/schema.js +379 -0
  229. package/dist/browser/seo/rules/security.d.ts +2 -0
  230. package/dist/browser/seo/rules/security.js +877 -0
  231. package/dist/browser/seo/rules/social.d.ts +2 -0
  232. package/dist/browser/seo/rules/social.js +603 -0
  233. package/dist/browser/seo/rules/structural.d.ts +2 -0
  234. package/dist/browser/seo/rules/structural.js +223 -0
  235. package/dist/browser/seo/rules/technical-advanced.d.ts +10 -0
  236. package/dist/browser/seo/rules/technical-advanced.js +289 -0
  237. package/dist/browser/seo/rules/technical.d.ts +2 -0
  238. package/dist/browser/seo/rules/technical.js +480 -0
  239. package/dist/browser/seo/rules/thresholds.d.ts +196 -0
  240. package/dist/browser/seo/rules/thresholds.js +118 -0
  241. package/dist/browser/seo/rules/types.d.ts +498 -0
  242. package/dist/browser/seo/rules/types.js +11 -0
  243. package/dist/browser/seo/types.d.ts +211 -0
  244. package/dist/browser/seo/types.js +1 -0
  245. package/dist/browser/transport/curl.d.ts +4 -0
  246. package/dist/browser/transport/curl.js +101 -0
  247. package/dist/browser/transport/undici.js +1 -2
  248. package/dist/browser/transport/worker.d.ts +18 -0
  249. package/dist/browser/transport/worker.js +278 -0
  250. package/dist/browser/types/index.d.ts +4 -1
  251. package/dist/browser/utils/binary-manager.d.ts +4 -0
  252. package/dist/browser/utils/binary-manager.js +72 -0
  253. package/dist/browser/utils/user-agent.js +2 -13
  254. package/dist/cache/indexed-db.d.ts +10 -0
  255. package/dist/cache/indexed-db.js +88 -0
  256. package/dist/cache/service-worker-cache.d.ts +18 -0
  257. package/dist/cache/service-worker-cache.js +103 -0
  258. package/dist/cli/commands/ai.d.ts +2 -0
  259. package/dist/cli/commands/ai.js +162 -0
  260. package/dist/cli/commands/bench.d.ts +2 -0
  261. package/dist/cli/commands/bench.js +51 -0
  262. package/dist/cli/commands/dns.d.ts +2 -0
  263. package/dist/cli/commands/dns.js +295 -0
  264. package/dist/cli/commands/har.d.ts +2 -0
  265. package/dist/cli/commands/har.js +171 -0
  266. package/dist/cli/commands/hls.d.ts +2 -0
  267. package/dist/cli/commands/hls.js +192 -0
  268. package/dist/cli/commands/network.d.ts +2 -0
  269. package/dist/cli/commands/network.js +288 -0
  270. package/dist/cli/commands/protocols.d.ts +2 -0
  271. package/dist/cli/commands/protocols.js +344 -0
  272. package/dist/cli/commands/scrape.d.ts +2 -0
  273. package/dist/cli/commands/scrape.js +176 -0
  274. package/dist/cli/commands/security.d.ts +2 -0
  275. package/dist/cli/commands/security.js +57 -0
  276. package/dist/cli/commands/seo.d.ts +2 -0
  277. package/dist/cli/commands/seo.js +125 -0
  278. package/dist/cli/commands/serve.d.ts +2 -0
  279. package/dist/cli/commands/serve.js +531 -0
  280. package/dist/cli/commands/spider.d.ts +3 -0
  281. package/dist/cli/commands/spider.js +456 -0
  282. package/dist/cli/commands/utils.d.ts +2 -0
  283. package/dist/cli/commands/utils.js +176 -0
  284. package/dist/cli/commands/vector.d.ts +2 -0
  285. package/dist/cli/commands/vector.js +158 -0
  286. package/dist/cli/handler.d.ts +2 -2
  287. package/dist/cli/handler.js +6 -6
  288. package/dist/cli/helpers.d.ts +7 -0
  289. package/dist/cli/helpers.js +128 -0
  290. package/dist/cli/index.js +96 -5228
  291. package/dist/cli/parser/help.d.ts +2 -0
  292. package/dist/cli/parser/help.js +52 -0
  293. package/dist/cli/parser/index.d.ts +3 -0
  294. package/dist/cli/parser/index.js +3 -0
  295. package/dist/cli/parser/parser.d.ts +4 -0
  296. package/dist/cli/parser/parser.js +146 -0
  297. package/dist/cli/parser/types.d.ts +41 -0
  298. package/dist/cli/parser/types.js +1 -0
  299. package/dist/cli/presets.d.ts +1 -1
  300. package/dist/cli/presets.js +1 -1
  301. package/dist/cli/router.d.ts +36 -0
  302. package/dist/cli/router.js +195 -0
  303. package/dist/cli/tui/ai-chat.js +1 -1
  304. package/dist/cli/tui/commands/context.d.ts +9 -0
  305. package/dist/cli/tui/commands/context.js +1 -0
  306. package/dist/cli/tui/commands/dns.d.ts +10 -0
  307. package/dist/cli/tui/commands/dns.js +461 -0
  308. package/dist/cli/tui/commands/hls.d.ts +2 -0
  309. package/dist/cli/tui/commands/hls.js +162 -0
  310. package/dist/cli/tui/commands/ip.d.ts +2 -0
  311. package/dist/cli/tui/commands/ip.js +45 -0
  312. package/dist/cli/tui/commands/network.d.ts +3 -0
  313. package/dist/cli/tui/commands/network.js +81 -0
  314. package/dist/cli/tui/commands/protocols.d.ts +6 -0
  315. package/dist/cli/tui/commands/protocols.js +531 -0
  316. package/dist/cli/tui/commands/security.d.ts +2 -0
  317. package/dist/cli/tui/commands/security.js +48 -0
  318. package/dist/cli/tui/commands/seo.d.ts +2 -0
  319. package/dist/cli/tui/commands/seo.js +74 -0
  320. package/dist/cli/tui/context.d.ts +12 -0
  321. package/dist/cli/tui/context.js +1 -0
  322. package/dist/cli/tui/shell.d.ts +11 -20
  323. package/dist/cli/tui/shell.js +216 -1873
  324. package/dist/constants/user-agents.d.ts +7 -0
  325. package/dist/constants/user-agents.js +7 -0
  326. package/dist/core/client.d.ts +2 -0
  327. package/dist/core/client.js +19 -1
  328. package/dist/index.d.ts +1 -0
  329. package/dist/index.js +1 -0
  330. package/dist/mcp/cli.js +2 -3
  331. package/dist/mcp/data/embeddings.json +1 -1
  332. package/dist/mcp/tools/network.js +298 -158
  333. package/dist/plugins/har-player.d.ts +23 -0
  334. package/dist/plugins/har-player.js +49 -0
  335. package/dist/plugins/har-recorder.d.ts +37 -3
  336. package/dist/plugins/har-recorder.js +116 -63
  337. package/dist/plugins/network-simulation.d.ts +7 -0
  338. package/dist/plugins/network-simulation.js +13 -0
  339. package/dist/presets/android.d.ts +2 -0
  340. package/dist/presets/android.js +16 -0
  341. package/dist/presets/chaturbate.d.ts +2 -0
  342. package/dist/presets/chaturbate.js +17 -0
  343. package/dist/presets/elevenlabs.d.ts +6 -0
  344. package/dist/presets/elevenlabs.js +20 -0
  345. package/dist/presets/enhancers.d.ts +20 -0
  346. package/dist/presets/enhancers.js +85 -0
  347. package/dist/presets/hubspot.d.ts +9 -0
  348. package/dist/presets/hubspot.js +28 -0
  349. package/dist/presets/index.d.ts +10 -0
  350. package/dist/presets/index.js +10 -0
  351. package/dist/presets/ios.d.ts +2 -0
  352. package/dist/presets/ios.js +13 -0
  353. package/dist/presets/pinecone.d.ts +8 -0
  354. package/dist/presets/pinecone.js +42 -0
  355. package/dist/presets/registry.js +60 -0
  356. package/dist/presets/sendgrid.d.ts +6 -0
  357. package/dist/presets/sendgrid.js +20 -0
  358. package/dist/presets/sentry.d.ts +11 -0
  359. package/dist/presets/sentry.js +48 -0
  360. package/dist/presets/square.d.ts +10 -0
  361. package/dist/presets/square.js +33 -0
  362. package/dist/recker.d.ts +3 -0
  363. package/dist/recker.js +4 -0
  364. package/dist/scrape/document.d.ts +5 -4
  365. package/dist/scrape/document.js +89 -76
  366. package/dist/scrape/element.d.ts +10 -8
  367. package/dist/scrape/element.js +295 -81
  368. package/dist/scrape/extractors.d.ts +11 -11
  369. package/dist/scrape/extractors.js +145 -113
  370. package/dist/scrape/index.d.ts +2 -0
  371. package/dist/scrape/index.js +1 -0
  372. package/dist/scrape/parser/back.d.ts +1 -0
  373. package/dist/scrape/parser/back.js +3 -0
  374. package/dist/scrape/parser/index.d.ts +20 -0
  375. package/dist/scrape/parser/index.js +19 -0
  376. package/dist/scrape/parser/matcher.d.ts +30 -0
  377. package/dist/scrape/parser/matcher.js +99 -0
  378. package/dist/scrape/parser/nodes/comment.d.ts +12 -0
  379. package/dist/scrape/parser/nodes/comment.js +21 -0
  380. package/dist/scrape/parser/nodes/html.d.ts +110 -0
  381. package/dist/scrape/parser/nodes/html.js +978 -0
  382. package/dist/scrape/parser/nodes/node.d.ts +18 -0
  383. package/dist/scrape/parser/nodes/node.js +31 -0
  384. package/dist/scrape/parser/nodes/text.d.ts +14 -0
  385. package/dist/scrape/parser/nodes/text.js +30 -0
  386. package/dist/scrape/parser/nodes/type.d.ts +6 -0
  387. package/dist/scrape/parser/nodes/type.js +7 -0
  388. package/dist/scrape/parser/parse.d.ts +1 -0
  389. package/dist/scrape/parser/parse.js +1 -0
  390. package/dist/scrape/parser/valid.d.ts +2 -0
  391. package/dist/scrape/parser/valid.js +5 -0
  392. package/dist/scrape/parser/void-tag.d.ts +7 -0
  393. package/dist/scrape/parser/void-tag.js +43 -0
  394. package/dist/scrape/spider.d.ts +19 -0
  395. package/dist/scrape/spider.js +28 -3
  396. package/dist/scrape/types.d.ts +7 -0
  397. package/dist/seo/analyzer.d.ts +15 -5
  398. package/dist/seo/analyzer.js +636 -175
  399. package/dist/seo/formatter.d.ts +16 -0
  400. package/dist/seo/formatter.js +228 -0
  401. package/dist/seo/index.d.ts +2 -0
  402. package/dist/seo/index.js +1 -0
  403. package/dist/seo/keywords.d.ts +16 -0
  404. package/dist/seo/keywords.js +55 -0
  405. package/dist/seo/rules/accessibility.js +96 -57
  406. package/dist/seo/rules/ai-search.js +44 -31
  407. package/dist/seo/rules/analytics.d.ts +2 -0
  408. package/dist/seo/rules/analytics.js +306 -0
  409. package/dist/seo/rules/best-practices.js +21 -14
  410. package/dist/seo/rules/canonical.js +53 -32
  411. package/dist/seo/rules/content.js +317 -31
  412. package/dist/seo/rules/crawl.js +55 -40
  413. package/dist/seo/rules/cwv.js +21 -15
  414. package/dist/seo/rules/ecommerce.js +82 -22
  415. package/dist/seo/rules/i18n.js +75 -36
  416. package/dist/seo/rules/images.js +109 -30
  417. package/dist/seo/rules/index.js +2 -0
  418. package/dist/seo/rules/internal-linking.js +58 -39
  419. package/dist/seo/rules/links.js +79 -52
  420. package/dist/seo/rules/local.js +49 -25
  421. package/dist/seo/rules/meta.js +339 -81
  422. package/dist/seo/rules/mobile.js +112 -2
  423. package/dist/seo/rules/performance.js +434 -66
  424. package/dist/seo/rules/pwa.js +36 -39
  425. package/dist/seo/rules/readability.js +31 -22
  426. package/dist/seo/rules/redirects.js +21 -15
  427. package/dist/seo/rules/resources.js +59 -42
  428. package/dist/seo/rules/schema.js +333 -8
  429. package/dist/seo/rules/security.js +142 -80
  430. package/dist/seo/rules/social.js +277 -47
  431. package/dist/seo/rules/structural.js +87 -19
  432. package/dist/seo/rules/technical-advanced.js +30 -24
  433. package/dist/seo/rules/technical.js +243 -42
  434. package/dist/seo/rules/types.d.ts +53 -1
  435. package/dist/seo/seo-spider.d.ts +22 -0
  436. package/dist/seo/seo-spider.js +77 -13
  437. package/dist/seo/types.d.ts +8 -1
  438. package/dist/seo/validators/llms-txt.js +19 -0
  439. package/dist/seo/validators/rss.d.ts +11 -0
  440. package/dist/seo/validators/rss.js +93 -0
  441. package/dist/seo/validators/sitemap.js +36 -26
  442. package/dist/transport/curl.d.ts +4 -0
  443. package/dist/transport/curl.js +101 -0
  444. package/dist/transport/udp.js +0 -1
  445. package/dist/transport/undici.js +1 -2
  446. package/dist/transport/worker.d.ts +18 -0
  447. package/dist/transport/worker.js +278 -0
  448. package/dist/types/index.d.ts +4 -1
  449. package/dist/utils/binary-manager.d.ts +4 -0
  450. package/dist/utils/binary-manager.js +72 -0
  451. package/dist/utils/optional-require.d.ts +7 -8
  452. package/dist/utils/optional-require.js +2 -21
  453. package/dist/utils/upload.d.ts +6 -0
  454. package/dist/utils/upload.js +11 -0
  455. package/dist/utils/user-agent.js +2 -13
  456. package/dist/version.js +1 -1
  457. package/package.json +12 -6
  458. package/dist/browser/utils/optional-require.d.ts +0 -19
  459. package/dist/browser/utils/optional-require.js +0 -105
package/dist/browser/seo/rules/technical.js ADDED
@@ -0,0 +1,480 @@
1
+ import { createResult } from './types.js';
2
+ export const technicalRules = [
3
+ {
4
+ id: 'canonical-exists',
5
+ name: 'Canonical URL',
6
+ category: 'technical',
7
+ severity: 'warning',
8
+ description: 'Page should have a canonical URL',
9
+ check: (ctx) => {
10
+ if (!ctx.hasCanonical) {
11
+ return createResult({ id: 'canonical-exists', name: 'Canonical URL', category: 'technical', severity: 'warning' }, 'warn', 'No canonical URL defined', {
12
+ recommendation: 'Add a canonical URL to indicate the preferred version of this page.',
13
+ evidence: {
14
+ found: 'No <link rel="canonical"> tag',
15
+ expected: 'A canonical URL pointing to the preferred version',
16
+ impact: 'Without canonical, search engines may split ranking signals across duplicate URLs (www vs non-www, HTTP vs HTTPS, with/without trailing slash).',
17
+ example: '<link rel="canonical" href="https://example.com/page/">',
18
+ learnMore: 'https://developers.google.com/search/docs/crawling-indexing/canonicalization'
19
+ }
20
+ });
21
+ }
22
+ return createResult({ id: 'canonical-exists', name: 'Canonical URL', category: 'technical', severity: 'warning' }, 'pass', 'Canonical URL is defined', { value: ctx.canonicalUrl });
23
+ },
24
+ },
25
+ {
26
+ id: 'lang-exists',
27
+ name: 'Language',
28
+ category: 'technical',
29
+ severity: 'warning',
30
+ description: 'HTML should have lang attribute',
31
+ check: (ctx) => {
32
+ if (!ctx.hasLang) {
33
+ return createResult({ id: 'lang-exists', name: 'Language', category: 'technical', severity: 'warning' }, 'warn', 'Missing lang attribute on <html>', {
34
+ recommendation: 'Declare the page language for better accessibility and SEO.',
35
+ evidence: {
36
+ found: '<html> tag without lang attribute',
37
+ expected: '<html lang="en"> (or appropriate language code)',
38
+ impact: 'The lang attribute helps screen readers pronounce content correctly and helps search engines serve the right audience. It\'s required for WCAG accessibility compliance.',
39
+ example: '<html lang="en">\n<html lang="pt-BR">\n<html lang="es">',
40
+ learnMore: 'https://developer.mozilla.org/en-US/docs/Web/HTML/Global_attributes/lang'
41
+ }
42
+ });
43
+ }
44
+ return createResult({ id: 'lang-exists', name: 'Language', category: 'technical', severity: 'warning' }, 'pass', `Language attribute set (${ctx.langValue})`);
45
+ },
46
+ },
47
+ {
48
+ id: 'charset-exists',
49
+ name: 'Charset',
50
+ category: 'technical',
51
+ severity: 'warning',
52
+ description: 'Page should declare character encoding',
53
+ check: (ctx) => {
54
+ if (!ctx.hasCharset) {
55
+ return createResult({ id: 'charset-exists', name: 'Charset', category: 'technical', severity: 'warning' }, 'warn', 'Missing charset declaration', {
56
+ recommendation: 'Declare character encoding to ensure proper text rendering.',
57
+ evidence: {
58
+ found: 'No charset meta tag',
59
+ expected: '<meta charset="UTF-8"> as the first element in <head>',
60
+ impact: 'Without charset declaration, browsers may guess incorrectly, causing garbled text (mojibake). UTF-8 supports all languages and special characters.',
61
+ example: '<head>\n <meta charset="UTF-8">\n <!-- other tags -->\n</head>',
62
+ learnMore: 'https://developer.mozilla.org/en-US/docs/Web/HTML/Element/meta#charset'
63
+ }
64
+ });
65
+ }
66
+ if (ctx.charset && ctx.charset.toLowerCase() !== 'utf-8') {
67
+ return createResult({ id: 'charset-exists', name: 'Charset', category: 'technical', severity: 'warning' }, 'warn', `Non-UTF-8 charset: ${ctx.charset}`, {
68
+ recommendation: 'Switch to UTF-8 encoding for universal compatibility.',
69
+ evidence: {
70
+ found: `charset="${ctx.charset}"`,
71
+ expected: 'charset="UTF-8"',
72
+ impact: 'UTF-8 is the standard encoding that supports all languages. Legacy encodings like ISO-8859-1 or Windows-1252 cannot display all characters.',
73
+ learnMore: 'https://www.w3.org/International/questions/qa-choosing-encodings'
74
+ }
75
+ });
76
+ }
77
+ return createResult({ id: 'charset-exists', name: 'Charset', category: 'technical', severity: 'warning' }, 'pass', 'UTF-8 charset declared');
78
+ },
79
+ },
80
+ {
81
+ id: 'robots-noindex',
82
+ name: 'Robots Noindex',
83
+ category: 'technical',
84
+ severity: 'warning',
85
+ description: 'Check if page is set to noindex',
86
+ check: (ctx) => {
87
+ if (!ctx.metaRobots || ctx.metaRobots.length === 0) {
88
+ return createResult({ id: 'robots-noindex', name: 'Robots', category: 'technical', severity: 'warning' }, 'info', 'Not applicable (no robots meta tag)', { recommendation: 'This rule checks for noindex directive in robots meta tag' });
89
+ }
90
+ if (ctx.metaRobots.includes('noindex')) {
91
+ return createResult({ id: 'robots-noindex', name: 'Robots', category: 'technical', severity: 'warning' }, 'warn', 'Page is set to noindex', {
92
+ value: ctx.metaRobots.join(', '),
93
+ recommendation: 'Verify that noindex is intentional. This page will NOT appear in search results.',
94
+ evidence: {
95
+ found: `<meta name="robots" content="${ctx.metaRobots.join(', ')}">`,
96
+ expected: 'No noindex directive (for pages you want indexed)',
97
+ impact: 'The noindex directive completely prevents this page from appearing in Google, Bing, and other search engines. This is useful for thank-you pages or admin areas, but harmful if applied accidentally.',
98
+ example: 'To allow indexing: <meta name="robots" content="index, follow">',
99
+ learnMore: 'https://developers.google.com/search/docs/crawling-indexing/block-indexing'
100
+ }
101
+ });
102
+ }
103
+ return createResult({ id: 'robots-noindex', name: 'Robots', category: 'technical', severity: 'info' }, 'info', `Robots meta: ${ctx.metaRobots.join(', ')}`);
104
+ },
105
+ },
106
+ {
107
+ id: 'favicon-exists',
108
+ name: 'Favicon',
109
+ category: 'technical',
110
+ severity: 'warning',
111
+ description: 'Page should have a favicon defined',
112
+ check: (ctx) => {
113
+ if (!ctx.hasFavicon) {
114
+ return createResult({ id: 'favicon-exists', name: 'Favicon', category: 'technical', severity: 'warning' }, 'warn', 'No favicon defined', {
115
+ recommendation: 'Add a favicon to improve brand recognition and user experience.',
116
+ evidence: {
117
+ found: 'No favicon link tag',
118
+ expected: '<link rel="icon"> pointing to your favicon',
119
+ impact: 'Favicons appear in browser tabs, bookmarks, history, and search results. Missing favicons look unprofessional and generate 404 errors in server logs.',
120
+ example: '<link rel="icon" href="/favicon.ico" sizes="32x32">\n<link rel="icon" href="/icon.svg" type="image/svg+xml">\n<link rel="apple-touch-icon" href="/apple-touch-icon.png">',
121
+ learnMore: 'https://web.dev/articles/add-manifest#icons'
122
+ }
123
+ });
124
+ }
125
+ return createResult({ id: 'favicon-exists', name: 'Favicon', category: 'technical', severity: 'warning' }, 'pass', 'Favicon is defined', { value: ctx.faviconUrl });
126
+ },
127
+ },
128
+ {
129
+ id: 'url-lowercase',
130
+ name: 'URL Lowercase',
131
+ category: 'technical',
132
+ severity: 'warning',
133
+ description: 'URLs should be lowercase for consistency',
134
+ check: (ctx) => {
135
+ if (ctx.urlHasUppercase) {
136
+ return createResult({ id: 'url-lowercase', name: 'URL Lowercase', category: 'technical', severity: 'warning' }, 'warn', 'URL contains uppercase characters', {
137
+ recommendation: 'Convert URL to lowercase for consistency and to avoid duplicate content.',
138
+ evidence: {
139
+ found: ctx.url,
140
+ expected: 'All lowercase URL path',
141
+ impact: 'URLs are case-sensitive on most servers. /Page and /page are different URLs, which can split ranking signals and cause duplicate content issues.',
142
+ example: 'Instead of /About-Us, use /about-us',
143
+ learnMore: 'https://developers.google.com/search/docs/crawling-indexing/url-structure'
144
+ }
145
+ });
146
+ }
147
+ return createResult({ id: 'url-lowercase', name: 'URL Lowercase', category: 'technical', severity: 'warning' }, 'info', 'Not applicable (URL is properly lowercase)', { recommendation: 'This rule ensures URLs use lowercase to avoid duplicate content issues' });
148
+ },
149
+ },
150
+ {
151
+ id: 'url-clean',
152
+ name: 'URL Clean',
153
+ category: 'technical',
154
+ severity: 'warning',
155
+ description: 'URLs should not contain special characters or accents',
156
+ check: (ctx) => {
157
+ if (ctx.urlHasAccents || ctx.urlHasSpecialChars) {
158
+ const issues = [];
159
+ if (ctx.urlHasAccents)
160
+ issues.push('accents');
161
+ if (ctx.urlHasSpecialChars)
162
+ issues.push('special characters');
163
+ return createResult({ id: 'url-clean', name: 'URL Clean', category: 'technical', severity: 'warning' }, 'warn', `URL contains ${issues.join(' and ')}`, {
164
+ recommendation: 'Use clean, ASCII-only URLs for maximum compatibility.',
165
+ evidence: {
166
+ found: ctx.url,
167
+ expected: 'URL with only letters, numbers, and hyphens',
168
+ impact: 'Accented characters and special characters get percent-encoded (%C3%A9 instead of é), making URLs ugly and hard to share. Some systems may not handle them correctly.',
169
+ example: 'Instead of /café-menü, use /cafe-menu',
170
+ learnMore: 'https://developers.google.com/search/docs/crawling-indexing/url-structure'
171
+ }
172
+ });
173
+ }
174
+ return createResult({ id: 'url-clean', name: 'URL Clean', category: 'technical', severity: 'warning' }, 'info', 'Not applicable (URL is clean)', { recommendation: 'This rule checks for accents and special characters in URLs' });
175
+ },
176
+ },
177
+ {
178
+ id: 'url-no-params',
179
+ name: 'URL Parameters',
180
+ category: 'technical',
181
+ severity: 'warning',
182
+ description: 'URLs should not contain query parameters',
183
+ check: (ctx) => {
184
+ if (!ctx.url) {
185
+ return createResult({ id: 'url-no-params', name: 'URL Parameters', category: 'technical', severity: 'warning' }, 'info', 'Not applicable (URL data unavailable)', { recommendation: 'This rule checks for query parameters that can cause duplicate content' });
186
+ }
187
+ try {
188
+ const urlObj = new URL(ctx.url);
189
+ if (urlObj.search && urlObj.search.length > 1) {
190
+ return createResult({ id: 'url-no-params', name: 'URL Parameters', category: 'technical', severity: 'warning' }, 'warn', 'URL contains query parameters', {
191
+ value: urlObj.search,
192
+ recommendation: 'Consider using clean, path-based URLs instead of query parameters.',
193
+ evidence: {
194
+ found: urlObj.search,
195
+ expected: 'Clean URL without query strings',
196
+ impact: 'Query parameters make URLs harder to read and share. Google recommends using simple, descriptive URLs. Parameters can also cause crawl budget waste if they create many URL variations.',
197
+ example: 'Instead of /products?category=shoes&color=red\nUse /products/shoes/red',
198
+ learnMore: 'https://developers.google.com/search/docs/crawling-indexing/url-structure#use-simple-urls'
199
+ }
200
+ });
201
+ }
202
+ }
203
+ catch {
204
+ }
205
+ return createResult({ id: 'url-no-params', name: 'URL Parameters', category: 'technical', severity: 'warning' }, 'info', 'Not applicable (no query parameters)', { recommendation: 'Clean URLs without parameters are preferred for SEO' });
206
+ },
207
+ },
208
+ {
209
+ id: 'technical-meta-robots-directives',
210
+ name: 'Meta Robots Directives',
211
+ category: 'technical',
212
+ severity: 'warning',
213
+ description: 'Check for restrictive meta robots directives like noindex, nofollow, noarchive etc.',
214
+ check: (ctx) => {
215
+ if (!ctx.metaRobots || ctx.metaRobots.length === 0) {
216
+ return createResult({ id: 'technical-meta-robots-directives', name: 'Meta Robots Directives', category: 'technical', severity: 'warning' }, 'info', 'Not applicable (no robots meta tag)', { recommendation: 'This rule checks for restrictive robots directives' });
217
+ }
218
+ const restrictiveDirectives = ['noindex', 'nofollow', 'noarchive', 'nosnippet', 'noimageindex'];
219
+ const foundRestrictive = ctx.metaRobots.filter(directive => restrictiveDirectives.includes(directive));
220
+ if (foundRestrictive.length > 0) {
221
+ return createResult({ id: 'technical-meta-robots-directives', name: 'Meta Robots Directives', category: 'technical', severity: 'warning' }, 'warn', `Restrictive meta robots directives found: ${foundRestrictive.join(', ')}`, {
222
+ recommendation: 'Verify these restrictive directives are intentional.',
223
+ evidence: {
224
+ found: foundRestrictive.join(', '),
225
+ expected: 'index, follow (for pages you want in search results)',
226
+ impact: 'noindex: page won\'t appear in search. nofollow: links won\'t pass PageRank. noarchive: no cached version. nosnippet: no description in results. noimageindex: images not indexed.',
227
+ example: 'To allow everything: <meta name="robots" content="index, follow, archive, snippet">',
228
+ learnMore: 'https://developers.google.com/search/docs/crawling-indexing/robots-meta-tag'
229
+ }
230
+ });
231
+ }
232
+ return createResult({ id: 'technical-meta-robots-directives', name: 'Meta Robots Directives', category: 'technical', severity: 'warning' }, 'info', 'Not applicable (no restrictive directives found)', { recommendation: 'Meta robots directives allow full indexing' });
233
+ },
234
+ },
235
+ {
236
+ id: 'technical-x-robots-tag',
237
+ name: 'X-Robots-Tag Header',
238
+ category: 'technical',
239
+ severity: 'info',
240
+ description: 'X-Robots-Tag header can be used to control indexing, especially for non-HTML content.',
241
+ check: (ctx) => {
242
+ if (!ctx.responseHeaders) {
243
+ return createResult({ id: 'technical-x-robots-tag', name: 'X-Robots-Tag Header', category: 'technical', severity: 'info' }, 'info', 'Not applicable (response headers unavailable)', { recommendation: 'This rule checks for X-Robots-Tag HTTP header' });
244
+ }
245
+ const xRobotsTag = ctx.responseHeaders['x-robots-tag'] || ctx.responseHeaders['X-Robots-Tag'];
246
+ if (xRobotsTag) {
247
+ return createResult({ id: 'technical-x-robots-tag', name: 'X-Robots-Tag Header', category: 'technical', severity: 'info' }, 'info', `X-Robots-Tag header found: ${xRobotsTag}`, {
248
+ recommendation: 'Verify this server-level robots directive is intentional.',
249
+ evidence: {
250
+ found: `X-Robots-Tag: ${xRobotsTag}`,
251
+ impact: 'X-Robots-Tag HTTP header works like meta robots but at server level. It\'s commonly used for non-HTML files (PDFs, images) but can also override meta tags for HTML pages.',
252
+ learnMore: 'https://developers.google.com/search/docs/crawling-indexing/robots-meta-tag#xrobotstag'
253
+ }
254
+ });
255
+ }
256
+ return createResult({ id: 'technical-x-robots-tag', name: 'X-Robots-Tag Header', category: 'technical', severity: 'info' }, 'info', 'Not applicable (no X-Robots-Tag header)', { recommendation: 'X-Robots-Tag header is not present' });
257
+ },
258
+ },
259
+ {
260
+ id: 'technical-trust-signals',
261
+ name: 'Trust Signals (Links)',
262
+ category: 'technical',
263
+ severity: 'info',
264
+ description: 'Presence of links to "About", "Contact", "Privacy Policy", "Terms of Service" pages builds trust.',
265
+ check: (ctx) => {
266
+ const missingSignals = [];
267
+ if (!ctx.hasAboutPageLink)
268
+ missingSignals.push('About Page');
269
+ if (!ctx.hasContactPageLink)
270
+ missingSignals.push('Contact Page');
271
+ if (!ctx.hasPrivacyPolicyLink)
272
+ missingSignals.push('Privacy Policy');
273
+ if (!ctx.hasTermsOfServiceLink)
274
+ missingSignals.push('Terms of Service');
275
+ if (missingSignals.length > 0) {
276
+ return createResult({ id: 'technical-trust-signals', name: 'Trust Signals (Links)', category: 'technical', severity: 'info' }, 'info', `Missing links to key trust pages: ${missingSignals.join(', ')}`, {
277
+ recommendation: 'Add links to trust-building pages to improve E-E-A-T signals.',
278
+ evidence: {
279
+ found: `Missing: ${missingSignals.join(', ')}`,
280
+ expected: 'Links to About, Contact, Privacy Policy, and Terms of Service',
281
+ impact: 'Google\'s E-E-A-T (Experience, Expertise, Authoritativeness, Trustworthiness) guidelines favor sites that clearly identify who they are. These pages are essential for YMYL (Your Money Your Life) sites.',
282
+ example: '<footer>\n <a href="/about">About Us</a>\n <a href="/contact">Contact</a>\n <a href="/privacy">Privacy Policy</a>\n <a href="/terms">Terms of Service</a>\n</footer>',
283
+ learnMore: 'https://developers.google.com/search/docs/fundamentals/creating-helpful-content'
284
+ }
285
+ });
286
+ }
287
+ return createResult({ id: 'technical-trust-signals', name: 'Trust Signals (Links)', category: 'technical', severity: 'info' }, 'info', 'Not applicable (all trust signals present)', { recommendation: 'All key trust pages are linked' });
288
+ },
289
+ },
290
+ {
291
+ id: 'technical-text-html-ratio',
292
+ name: 'Text/HTML Ratio',
293
+ category: 'technical',
294
+ severity: 'info',
295
+ description: 'A higher text to HTML ratio indicates more content relative to code, which is good for SEO.',
296
+ check: (ctx) => {
297
+ if (ctx.textHtmlRatio === undefined) {
298
+ return createResult({ id: 'technical-text-html-ratio', name: 'Text/HTML Ratio', category: 'technical', severity: 'info' }, 'info', 'Not applicable (text/HTML ratio data unavailable)', { recommendation: 'This rule checks the ratio of visible text to HTML code' });
299
+ }
300
+ const threshold = 15;
301
+ if (ctx.textHtmlRatio < threshold) {
302
+ return createResult({ id: 'technical-text-html-ratio', name: 'Text/HTML Ratio', category: 'technical', severity: 'info' }, 'warn', `Low Text/HTML ratio: ${ctx.textHtmlRatio.toFixed(2)}% (target > ${threshold}%)`, {
303
+ recommendation: 'Increase visible text content relative to HTML code.',
304
+ evidence: {
305
+ found: `${ctx.textHtmlRatio.toFixed(2)}% text content`,
306
+ expected: `At least ${threshold}% text content`,
307
+ impact: 'Pages with low text-to-HTML ratio may appear thin or low-quality to search engines. This can indicate excessive scripts, inline CSS, or insufficient content.',
308
+ learnMore: 'https://developers.google.com/search/docs/fundamentals/creating-helpful-content#content-and-quality'
309
+ }
310
+ });
311
+ }
312
+ return createResult({ id: 'technical-text-html-ratio', name: 'Text/HTML Ratio', category: 'technical', severity: 'info' }, 'info', 'Not applicable (text/HTML ratio is good)', { recommendation: `Text/HTML ratio is ${ctx.textHtmlRatio.toFixed(2)}%, which is healthy` });
313
+ },
314
+ },
315
+ {
316
+ id: 'technical-robots-txt-hint',
317
+ name: 'Robots.txt Hint',
318
+ category: 'technical',
319
+ severity: 'info',
320
+ description: 'Ensure a robots.txt file exists at the root of the domain to guide crawlers.',
321
+ check: (ctx) => {
322
+ return createResult({ id: 'technical-robots-txt-hint', name: 'Robots.txt Hint', category: 'technical', severity: 'info' }, 'info', 'Robots.txt existence cannot be verified from HTML alone.', {
323
+ recommendation: 'Ensure you have a valid robots.txt file at your domain root.',
324
+ evidence: {
325
+ expected: 'A robots.txt file at /robots.txt',
326
+ impact: 'robots.txt tells search engines which pages to crawl. It should also point to your sitemap.xml for efficient discovery of all pages.',
327
+ example: '# Example robots.txt\nUser-agent: *\nAllow: /\nDisallow: /admin/\nDisallow: /private/\n\nSitemap: https://example.com/sitemap.xml',
328
+ learnMore: 'https://developers.google.com/search/docs/crawling-indexing/robots/create-robots-txt'
329
+ }
330
+ });
331
+ },
332
+ },
333
+ {
334
+ id: 'url-many-parameters',
335
+ name: 'Too Many URL Parameters',
336
+ category: 'technical',
337
+ severity: 'warning',
338
+ description: 'URLs should not have more than 3 query parameters',
339
+ check: (ctx) => {
340
+ if (!ctx.url) {
341
+ return createResult({ id: 'url-many-parameters', name: 'Too Many URL Parameters', category: 'technical', severity: 'warning' }, 'info', 'Not applicable (URL data unavailable)', { recommendation: 'This rule checks for excessive URL parameters' });
342
+ }
343
+ try {
344
+ const url = new URL(ctx.url);
345
+ const paramCount = Array.from(url.searchParams.keys()).length;
346
+ if (paramCount > 3) {
347
+ return createResult({ id: 'url-many-parameters', name: 'Too Many URL Parameters', category: 'technical', severity: 'warning' }, 'warn', `URL has ${paramCount} query parameters`, {
348
+ value: paramCount,
349
+ recommendation: 'Reduce URL parameters to 3 or fewer for better crawlability',
350
+ evidence: {
351
+ found: url.search,
352
+ expected: '3 or fewer parameters',
353
+ impact: 'Multiple parameters make URLs less enticing to click and may cause indexing issues. Search engines may treat parameter variations as duplicate content, wasting crawl budget.',
354
+ learnMore: 'https://developers.google.com/search/docs/crawling-indexing/url-structure#use-simple-urls'
355
+ }
356
+ });
357
+ }
358
+ }
359
+ catch {
360
+ }
361
+ return createResult({ id: 'url-many-parameters', name: 'Too Many URL Parameters', category: 'technical', severity: 'warning' }, 'info', 'Not applicable (3 or fewer URL parameters)', { recommendation: 'URL parameters are within acceptable limits' });
362
+ },
363
+ },
364
+ {
365
+ id: 'deprecated-plugins',
366
+ name: 'Deprecated Plugins',
367
+ category: 'technical',
368
+ severity: 'error',
369
+ description: 'Pages should not use Flash, Java Applets, or Silverlight',
370
+ check: (ctx) => {
371
+ if (ctx.hasDeprecatedPlugins === undefined) {
372
+ return createResult({ id: 'deprecated-plugins', name: 'Deprecated Plugins', category: 'technical', severity: 'error' }, 'info', 'Not applicable (deprecated plugins data unavailable)', { recommendation: 'This rule checks for Flash, Java, and Silverlight usage' });
373
+ }
374
+ if (ctx.hasDeprecatedPlugins) {
375
+ return createResult({ id: 'deprecated-plugins', name: 'Deprecated Plugins', category: 'technical', severity: 'error' }, 'fail', 'Page uses deprecated plugins (Flash, Java, or Silverlight)', {
376
+ recommendation: 'Convert plugin content to HTML5',
377
+ evidence: {
378
+ found: ctx.deprecatedPluginTypes?.join(', ') || 'Flash/Java/Silverlight detected',
379
+ expected: 'No deprecated plugins',
380
+ impact: 'These plugins do not work on mobile devices and cannot be crawled properly',
381
+ learnMore: 'https://developers.google.com/search/docs/crawling-indexing/mobile/mobile-sites-mobile-first-indexing'
382
+ }
383
+ });
384
+ }
385
+ return createResult({ id: 'deprecated-plugins', name: 'Deprecated Plugins', category: 'technical', severity: 'error' }, 'info', 'Not applicable (no deprecated plugins detected)', { recommendation: 'No Flash, Java, or Silverlight plugins found' });
386
+ },
387
+ },
388
+ {
389
+ id: 'frame-tags',
390
+ name: 'Frame Tags',
391
+ category: 'technical',
392
+ severity: 'error',
393
+ description: 'Pages should not use <frame> or <frameset> tags',
394
+ check: (ctx) => {
395
+ if (ctx.hasFrameTags === undefined) {
396
+ return createResult({ id: 'frame-tags', name: 'Frame Tags', category: 'technical', severity: 'error' }, 'info', 'Not applicable (frame tags data unavailable)', { recommendation: 'This rule checks for obsolete <frame> and <frameset> tags' });
397
+ }
398
+ if (ctx.hasFrameTags) {
399
+ return createResult({ id: 'frame-tags', name: 'Frame Tags', category: 'technical', severity: 'error' }, 'fail', 'Page uses <frame> or <frameset> tags', {
400
+ recommendation: 'Remove frame tags and restructure using modern HTML',
401
+ evidence: {
402
+ found: '<frame> or <frameset> tags detected',
403
+ expected: 'No frame tags',
404
+ impact: 'Frames are obsolete HTML. Search engines have difficulty indexing content within frames. Mobile devices don\'t support them well. Use modern CSS layouts instead.',
405
+ learnMore: 'https://developer.mozilla.org/en-US/docs/Web/HTML/Element/frame'
406
+ }
407
+ });
408
+ }
409
+ return createResult({ id: 'frame-tags', name: 'Frame Tags', category: 'technical', severity: 'error' }, 'info', 'Not applicable (no frame tags detected)', { recommendation: 'No obsolete frame tags found' });
410
+ },
411
+ },
412
+ {
413
+ id: 'avoid-iframes',
414
+ name: 'Avoid iFrames',
415
+ category: 'technical',
416
+ severity: 'info',
417
+ description: 'iFrames can cause SEO and usability issues.',
418
+ check: (ctx) => {
419
+ if (ctx.iframeCount && ctx.iframeCount > 0) {
420
+ return createResult({ id: 'avoid-iframes', name: 'iFrames Usage', category: 'technical', severity: 'info' }, 'info', `${ctx.iframeCount} iFrame(s) detected`, {
421
+ value: ctx.iframeCount,
422
+ recommendation: 'Avoid iFrames for main content. They can be hard to index and navigate.',
423
+ evidence: {
424
+ found: `${ctx.iframeCount} iframes`,
425
+ expected: 'Minimal iframe usage, only for embeds (videos, maps)',
426
+ impact: 'Content inside iframes is treated as separate pages and may not contribute to the parent page\'s SEO. Iframes can also slow page load and cause layout shifts.',
427
+ example: 'Acceptable: <iframe src="https://www.youtube.com/embed/..." loading="lazy">\nAvoid: Main content inside iframes',
428
+ learnMore: 'https://developers.google.com/search/docs/crawling-indexing/mobile/mobile-sites-mobile-first-indexing#iframes'
429
+ }
430
+ });
431
+ }
432
+ return createResult({ id: 'avoid-iframes', name: 'Avoid iFrames', category: 'technical', severity: 'info' }, 'info', 'Not applicable (no iframes detected)', { recommendation: 'No iframes found on this page' });
433
+ },
434
+ },
435
+ {
436
+ id: 'deprecated-html-tags',
437
+ name: 'Deprecated HTML Tags',
438
+ category: 'technical',
439
+ severity: 'warning',
440
+ description: 'Avoid deprecated HTML tags like <center>, <font>, <marquee>',
441
+ check: (ctx) => {
442
+ if (ctx.deprecatedTagsCount && ctx.deprecatedTagsCount > 0) {
443
+ return createResult({ id: 'deprecated-html-tags', name: 'Deprecated HTML Tags', category: 'technical', severity: 'warning' }, 'warn', `${ctx.deprecatedTagsCount} deprecated HTML tags found`, {
444
+ value: ctx.deprecatedTagsCount,
445
+ recommendation: 'Replace deprecated tags (center, font, marquee, etc.) with CSS.',
446
+ evidence: {
447
+ found: ctx.deprecatedTagsFound?.join(', '),
448
+ expected: 'Modern HTML5 elements with CSS styling',
449
+ impact: 'Deprecated tags like <center>, <font>, and <marquee> are obsolete. They indicate outdated code that may not render correctly in all browsers.',
450
+ example: 'Instead of <center>text</center>, use <div style="text-align: center">text</div>\nInstead of <font color="red">, use <span style="color: red">',
451
+ learnMore: 'https://developer.mozilla.org/en-US/docs/Web/HTML/Element#obsolete_and_deprecated_elements'
452
+ }
453
+ });
454
+ }
455
+ return createResult({ id: 'deprecated-html-tags', name: 'Deprecated HTML Tags', category: 'technical', severity: 'warning' }, 'info', 'Not applicable (no deprecated tags detected)', { recommendation: 'No obsolete HTML tags found' });
456
+ },
457
+ },
458
+ {
459
+ id: 'apple-touch-icon',
460
+ name: 'Apple Touch Icon',
461
+ category: 'technical',
462
+ severity: 'info',
463
+ description: 'Add an apple-touch-icon for iOS devices',
464
+ check: (ctx) => {
465
+ if (ctx.hasAppleTouchIcon === false) {
466
+ return createResult({ id: 'apple-touch-icon', name: 'Apple Touch Icon', category: 'technical', severity: 'info' }, 'info', 'Missing apple-touch-icon', {
467
+ recommendation: 'Add an Apple Touch Icon for iOS devices.',
468
+ evidence: {
469
+ found: 'No apple-touch-icon link tag',
470
+ expected: '<link rel="apple-touch-icon"> pointing to a 180x180 PNG',
471
+ impact: 'Apple Touch Icons appear when users add your site to their iOS home screen. Without one, iOS will use a screenshot of your page, which often looks poor.',
472
+ example: '<link rel="apple-touch-icon" sizes="180x180" href="/apple-touch-icon.png">',
473
+ learnMore: 'https://developer.apple.com/library/archive/documentation/AppleApplications/Reference/SafariWebContent/ConfiguringWebApplications/ConfiguringWebApplications.html'
474
+ }
475
+ });
476
+ }
477
+ return createResult({ id: 'apple-touch-icon', name: 'Apple Touch Icon', category: 'technical', severity: 'info' }, 'pass', 'Apple Touch Icon present');
478
+ },
479
+ },
480
+ ];