recker 1.0.43 → 1.0.44

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (459) hide show
  1. package/README.md +47 -0
  2. package/dist/bin/recker-linux-x64 +0 -0
  3. package/dist/bin/recker-macos-x64 +0 -0
  4. package/dist/bin/recker-win-x64.exe +0 -0
  5. package/dist/bin/rek.cjs +85152 -100207
  6. package/dist/browser/ai/adaptive-timeout.d.ts +50 -0
  7. package/dist/browser/ai/adaptive-timeout.js +208 -0
  8. package/dist/browser/ai/client.d.ts +22 -0
  9. package/dist/browser/ai/client.js +294 -0
  10. package/dist/browser/ai/index.d.ts +14 -0
  11. package/dist/browser/ai/index.js +11 -0
  12. package/dist/browser/ai/providers/anthropic.d.ts +63 -0
  13. package/dist/browser/ai/providers/anthropic.js +370 -0
  14. package/dist/browser/ai/providers/base.d.ts +48 -0
  15. package/dist/browser/ai/providers/base.js +150 -0
  16. package/dist/browser/ai/providers/google.d.ts +59 -0
  17. package/dist/browser/ai/providers/google.js +305 -0
  18. package/dist/browser/ai/providers/ollama.d.ts +44 -0
  19. package/dist/browser/ai/providers/ollama.js +240 -0
  20. package/dist/browser/ai/providers/openai.d.ts +64 -0
  21. package/dist/browser/ai/providers/openai.js +298 -0
  22. package/dist/browser/ai/rate-limiter.d.ts +43 -0
  23. package/dist/browser/ai/rate-limiter.js +215 -0
  24. package/dist/browser/ai/vector/index.d.ts +2 -0
  25. package/dist/browser/ai/vector/index.js +2 -0
  26. package/dist/browser/ai/vector/similarity.d.ts +2 -0
  27. package/dist/browser/ai/vector/similarity.js +27 -0
  28. package/dist/browser/ai/vector/store.d.ts +27 -0
  29. package/dist/browser/ai/vector/store.js +82 -0
  30. package/dist/browser/browser/cache.d.ts +2 -40
  31. package/dist/browser/browser/cache.js +2 -199
  32. package/dist/browser/browser/index.d.ts +8 -0
  33. package/dist/browser/browser/index.js +8 -0
  34. package/dist/browser/browser/recker.d.ts +8 -1
  35. package/dist/browser/browser/recker.js +8 -2
  36. package/dist/browser/cache/indexed-db.d.ts +10 -0
  37. package/dist/browser/cache/indexed-db.js +88 -0
  38. package/dist/browser/cache/service-worker-cache.d.ts +18 -0
  39. package/dist/browser/cache/service-worker-cache.js +103 -0
  40. package/dist/browser/cache.d.ts +2 -40
  41. package/dist/browser/cache.js +2 -199
  42. package/dist/browser/constants/user-agents.d.ts +7 -0
  43. package/dist/browser/constants/user-agents.js +7 -0
  44. package/dist/browser/core/client.d.ts +2 -0
  45. package/dist/browser/core/client.js +19 -1
  46. package/dist/browser/index.d.ts +8 -0
  47. package/dist/browser/index.js +8 -0
  48. package/dist/browser/plugins/har-recorder.d.ts +40 -0
  49. package/dist/browser/plugins/har-recorder.js +120 -0
  50. package/dist/browser/plugins/network-simulation.d.ts +7 -0
  51. package/dist/browser/plugins/network-simulation.js +13 -0
  52. package/dist/browser/presets/android.d.ts +2 -0
  53. package/dist/browser/presets/android.js +16 -0
  54. package/dist/browser/presets/anthropic.d.ts +8 -0
  55. package/dist/browser/presets/anthropic.js +27 -0
  56. package/dist/browser/presets/aws.d.ts +19 -0
  57. package/dist/browser/presets/aws.js +68 -0
  58. package/dist/browser/presets/azure-openai.d.ts +10 -0
  59. package/dist/browser/presets/azure-openai.js +35 -0
  60. package/dist/browser/presets/azure.d.ts +41 -0
  61. package/dist/browser/presets/azure.js +104 -0
  62. package/dist/browser/presets/chaturbate.d.ts +2 -0
  63. package/dist/browser/presets/chaturbate.js +17 -0
  64. package/dist/browser/presets/cloudflare.d.ts +12 -0
  65. package/dist/browser/presets/cloudflare.js +39 -0
  66. package/dist/browser/presets/cohere.d.ts +7 -0
  67. package/dist/browser/presets/cohere.js +22 -0
  68. package/dist/browser/presets/deepseek.d.ts +7 -0
  69. package/dist/browser/presets/deepseek.js +22 -0
  70. package/dist/browser/presets/digitalocean.d.ts +5 -0
  71. package/dist/browser/presets/digitalocean.js +16 -0
  72. package/dist/browser/presets/discord.d.ts +6 -0
  73. package/dist/browser/presets/discord.js +17 -0
  74. package/dist/browser/presets/elevenlabs.d.ts +6 -0
  75. package/dist/browser/presets/elevenlabs.js +20 -0
  76. package/dist/browser/presets/enhancers.d.ts +20 -0
  77. package/dist/browser/presets/enhancers.js +85 -0
  78. package/dist/browser/presets/fireworks.d.ts +7 -0
  79. package/dist/browser/presets/fireworks.js +22 -0
  80. package/dist/browser/presets/gcp.d.ts +34 -0
  81. package/dist/browser/presets/gcp.js +91 -0
  82. package/dist/browser/presets/gemini.d.ts +7 -0
  83. package/dist/browser/presets/gemini.js +23 -0
  84. package/dist/browser/presets/github.d.ts +6 -0
  85. package/dist/browser/presets/github.js +17 -0
  86. package/dist/browser/presets/gitlab.d.ts +6 -0
  87. package/dist/browser/presets/gitlab.js +16 -0
  88. package/dist/browser/presets/groq.d.ts +7 -0
  89. package/dist/browser/presets/groq.js +22 -0
  90. package/dist/browser/presets/hubspot.d.ts +9 -0
  91. package/dist/browser/presets/hubspot.js +28 -0
  92. package/dist/browser/presets/huggingface.d.ts +7 -0
  93. package/dist/browser/presets/huggingface.js +23 -0
  94. package/dist/browser/presets/index.d.ts +47 -0
  95. package/dist/browser/presets/index.js +47 -0
  96. package/dist/browser/presets/ios.d.ts +2 -0
  97. package/dist/browser/presets/ios.js +13 -0
  98. package/dist/browser/presets/linear.d.ts +5 -0
  99. package/dist/browser/presets/linear.js +16 -0
  100. package/dist/browser/presets/mailgun.d.ts +7 -0
  101. package/dist/browser/presets/mailgun.js +20 -0
  102. package/dist/browser/presets/meta.d.ts +10 -0
  103. package/dist/browser/presets/meta.js +33 -0
  104. package/dist/browser/presets/mistral.d.ts +7 -0
  105. package/dist/browser/presets/mistral.js +22 -0
  106. package/dist/browser/presets/notion.d.ts +6 -0
  107. package/dist/browser/presets/notion.js +17 -0
  108. package/dist/browser/presets/openai.d.ts +9 -0
  109. package/dist/browser/presets/openai.js +30 -0
  110. package/dist/browser/presets/oracle.d.ts +19 -0
  111. package/dist/browser/presets/oracle.js +117 -0
  112. package/dist/browser/presets/perplexity.d.ts +7 -0
  113. package/dist/browser/presets/perplexity.js +22 -0
  114. package/dist/browser/presets/pinecone.d.ts +8 -0
  115. package/dist/browser/presets/pinecone.js +42 -0
  116. package/dist/browser/presets/registry.d.ts +23 -0
  117. package/dist/browser/presets/registry.js +519 -0
  118. package/dist/browser/presets/replicate.d.ts +7 -0
  119. package/dist/browser/presets/replicate.js +23 -0
  120. package/dist/browser/presets/sendgrid.d.ts +6 -0
  121. package/dist/browser/presets/sendgrid.js +20 -0
  122. package/dist/browser/presets/sentry.d.ts +11 -0
  123. package/dist/browser/presets/sentry.js +48 -0
  124. package/dist/browser/presets/sinch.d.ts +9 -0
  125. package/dist/browser/presets/sinch.js +39 -0
  126. package/dist/browser/presets/slack.d.ts +5 -0
  127. package/dist/browser/presets/slack.js +16 -0
  128. package/dist/browser/presets/square.d.ts +10 -0
  129. package/dist/browser/presets/square.js +33 -0
  130. package/dist/browser/presets/stripe.d.ts +7 -0
  131. package/dist/browser/presets/stripe.js +23 -0
  132. package/dist/browser/presets/supabase.d.ts +6 -0
  133. package/dist/browser/presets/supabase.js +18 -0
  134. package/dist/browser/presets/tiktok.d.ts +10 -0
  135. package/dist/browser/presets/tiktok.js +38 -0
  136. package/dist/browser/presets/together.d.ts +7 -0
  137. package/dist/browser/presets/together.js +22 -0
  138. package/dist/browser/presets/twilio.d.ts +6 -0
  139. package/dist/browser/presets/twilio.js +17 -0
  140. package/dist/browser/presets/vercel.d.ts +6 -0
  141. package/dist/browser/presets/vercel.js +23 -0
  142. package/dist/browser/presets/vultr.d.ts +5 -0
  143. package/dist/browser/presets/vultr.js +16 -0
  144. package/dist/browser/presets/xai.d.ts +8 -0
  145. package/dist/browser/presets/xai.js +23 -0
  146. package/dist/browser/presets/youtube.d.ts +5 -0
  147. package/dist/browser/presets/youtube.js +20 -0
  148. package/dist/browser/recker.d.ts +8 -1
  149. package/dist/browser/recker.js +8 -2
  150. package/dist/browser/scrape/document.d.ts +5 -4
  151. package/dist/browser/scrape/document.js +89 -76
  152. package/dist/browser/scrape/element.d.ts +10 -8
  153. package/dist/browser/scrape/element.js +295 -81
  154. package/dist/browser/scrape/extractors.d.ts +11 -11
  155. package/dist/browser/scrape/extractors.js +145 -113
  156. package/dist/browser/scrape/parser/back.d.ts +1 -0
  157. package/dist/browser/scrape/parser/back.js +3 -0
  158. package/dist/browser/scrape/parser/index.d.ts +20 -0
  159. package/dist/browser/scrape/parser/index.js +19 -0
  160. package/dist/browser/scrape/parser/matcher.d.ts +30 -0
  161. package/dist/browser/scrape/parser/matcher.js +99 -0
  162. package/dist/browser/scrape/parser/nodes/comment.d.ts +12 -0
  163. package/dist/browser/scrape/parser/nodes/comment.js +21 -0
  164. package/dist/browser/scrape/parser/nodes/html.d.ts +110 -0
  165. package/dist/browser/scrape/parser/nodes/html.js +978 -0
  166. package/dist/browser/scrape/parser/nodes/node.d.ts +18 -0
  167. package/dist/browser/scrape/parser/nodes/node.js +31 -0
  168. package/dist/browser/scrape/parser/nodes/text.d.ts +14 -0
  169. package/dist/browser/scrape/parser/nodes/text.js +30 -0
  170. package/dist/browser/scrape/parser/nodes/type.d.ts +6 -0
  171. package/dist/browser/scrape/parser/nodes/type.js +7 -0
  172. package/dist/browser/scrape/parser/parse.d.ts +1 -0
  173. package/dist/browser/scrape/parser/parse.js +1 -0
  174. package/dist/browser/scrape/parser/valid.d.ts +2 -0
  175. package/dist/browser/scrape/parser/valid.js +5 -0
  176. package/dist/browser/scrape/parser/void-tag.d.ts +7 -0
  177. package/dist/browser/scrape/parser/void-tag.js +43 -0
  178. package/dist/browser/scrape/types.d.ts +7 -0
  179. package/dist/browser/seo/analyzer.d.ts +59 -0
  180. package/dist/browser/seo/analyzer.js +1399 -0
  181. package/dist/browser/seo/keywords.d.ts +16 -0
  182. package/dist/browser/seo/keywords.js +55 -0
  183. package/dist/browser/seo/rules/accessibility.d.ts +2 -0
  184. package/dist/browser/seo/rules/accessibility.js +733 -0
  185. package/dist/browser/seo/rules/ai-search.d.ts +2 -0
  186. package/dist/browser/seo/rules/ai-search.js +436 -0
  187. package/dist/browser/seo/rules/analytics.d.ts +2 -0
  188. package/dist/browser/seo/rules/analytics.js +306 -0
  189. package/dist/browser/seo/rules/best-practices.d.ts +2 -0
  190. package/dist/browser/seo/rules/best-practices.js +195 -0
  191. package/dist/browser/seo/rules/canonical.d.ts +12 -0
  192. package/dist/browser/seo/rules/canonical.js +270 -0
  193. package/dist/browser/seo/rules/content.d.ts +2 -0
  194. package/dist/browser/seo/rules/content.js +522 -0
  195. package/dist/browser/seo/rules/crawl.d.ts +2 -0
  196. package/dist/browser/seo/rules/crawl.js +435 -0
  197. package/dist/browser/seo/rules/cwv.d.ts +2 -0
  198. package/dist/browser/seo/rules/cwv.js +248 -0
  199. package/dist/browser/seo/rules/ecommerce.d.ts +2 -0
  200. package/dist/browser/seo/rules/ecommerce.js +312 -0
  201. package/dist/browser/seo/rules/i18n.d.ts +2 -0
  202. package/dist/browser/seo/rules/i18n.js +288 -0
  203. package/dist/browser/seo/rules/images.d.ts +2 -0
  204. package/dist/browser/seo/rules/images.js +255 -0
  205. package/dist/browser/seo/rules/index.d.ts +52 -0
  206. package/dist/browser/seo/rules/index.js +159 -0
  207. package/dist/browser/seo/rules/internal-linking.d.ts +2 -0
  208. package/dist/browser/seo/rules/internal-linking.js +394 -0
  209. package/dist/browser/seo/rules/links.d.ts +2 -0
  210. package/dist/browser/seo/rules/links.js +498 -0
  211. package/dist/browser/seo/rules/local.d.ts +2 -0
  212. package/dist/browser/seo/rules/local.js +289 -0
  213. package/dist/browser/seo/rules/meta.d.ts +2 -0
  214. package/dist/browser/seo/rules/meta.js +805 -0
  215. package/dist/browser/seo/rules/mobile.d.ts +2 -0
  216. package/dist/browser/seo/rules/mobile.js +161 -0
  217. package/dist/browser/seo/rules/performance.d.ts +2 -0
  218. package/dist/browser/seo/rules/performance.js +738 -0
  219. package/dist/browser/seo/rules/pwa.d.ts +2 -0
  220. package/dist/browser/seo/rules/pwa.js +299 -0
  221. package/dist/browser/seo/rules/readability.d.ts +2 -0
  222. package/dist/browser/seo/rules/readability.js +264 -0
  223. package/dist/browser/seo/rules/redirects.d.ts +16 -0
  224. package/dist/browser/seo/rules/redirects.js +199 -0
  225. package/dist/browser/seo/rules/resources.d.ts +2 -0
  226. package/dist/browser/seo/rules/resources.js +390 -0
  227. package/dist/browser/seo/rules/schema.d.ts +2 -0
  228. package/dist/browser/seo/rules/schema.js +379 -0
  229. package/dist/browser/seo/rules/security.d.ts +2 -0
  230. package/dist/browser/seo/rules/security.js +877 -0
  231. package/dist/browser/seo/rules/social.d.ts +2 -0
  232. package/dist/browser/seo/rules/social.js +603 -0
  233. package/dist/browser/seo/rules/structural.d.ts +2 -0
  234. package/dist/browser/seo/rules/structural.js +223 -0
  235. package/dist/browser/seo/rules/technical-advanced.d.ts +10 -0
  236. package/dist/browser/seo/rules/technical-advanced.js +289 -0
  237. package/dist/browser/seo/rules/technical.d.ts +2 -0
  238. package/dist/browser/seo/rules/technical.js +480 -0
  239. package/dist/browser/seo/rules/thresholds.d.ts +196 -0
  240. package/dist/browser/seo/rules/thresholds.js +118 -0
  241. package/dist/browser/seo/rules/types.d.ts +498 -0
  242. package/dist/browser/seo/rules/types.js +11 -0
  243. package/dist/browser/seo/types.d.ts +211 -0
  244. package/dist/browser/seo/types.js +1 -0
  245. package/dist/browser/transport/curl.d.ts +4 -0
  246. package/dist/browser/transport/curl.js +101 -0
  247. package/dist/browser/transport/undici.js +1 -2
  248. package/dist/browser/transport/worker.d.ts +18 -0
  249. package/dist/browser/transport/worker.js +278 -0
  250. package/dist/browser/types/index.d.ts +4 -1
  251. package/dist/browser/utils/binary-manager.d.ts +4 -0
  252. package/dist/browser/utils/binary-manager.js +72 -0
  253. package/dist/browser/utils/user-agent.js +2 -13
  254. package/dist/cache/indexed-db.d.ts +10 -0
  255. package/dist/cache/indexed-db.js +88 -0
  256. package/dist/cache/service-worker-cache.d.ts +18 -0
  257. package/dist/cache/service-worker-cache.js +103 -0
  258. package/dist/cli/commands/ai.d.ts +2 -0
  259. package/dist/cli/commands/ai.js +162 -0
  260. package/dist/cli/commands/bench.d.ts +2 -0
  261. package/dist/cli/commands/bench.js +51 -0
  262. package/dist/cli/commands/dns.d.ts +2 -0
  263. package/dist/cli/commands/dns.js +295 -0
  264. package/dist/cli/commands/har.d.ts +2 -0
  265. package/dist/cli/commands/har.js +171 -0
  266. package/dist/cli/commands/hls.d.ts +2 -0
  267. package/dist/cli/commands/hls.js +192 -0
  268. package/dist/cli/commands/network.d.ts +2 -0
  269. package/dist/cli/commands/network.js +288 -0
  270. package/dist/cli/commands/protocols.d.ts +2 -0
  271. package/dist/cli/commands/protocols.js +344 -0
  272. package/dist/cli/commands/scrape.d.ts +2 -0
  273. package/dist/cli/commands/scrape.js +176 -0
  274. package/dist/cli/commands/security.d.ts +2 -0
  275. package/dist/cli/commands/security.js +57 -0
  276. package/dist/cli/commands/seo.d.ts +2 -0
  277. package/dist/cli/commands/seo.js +125 -0
  278. package/dist/cli/commands/serve.d.ts +2 -0
  279. package/dist/cli/commands/serve.js +531 -0
  280. package/dist/cli/commands/spider.d.ts +3 -0
  281. package/dist/cli/commands/spider.js +456 -0
  282. package/dist/cli/commands/utils.d.ts +2 -0
  283. package/dist/cli/commands/utils.js +176 -0
  284. package/dist/cli/commands/vector.d.ts +2 -0
  285. package/dist/cli/commands/vector.js +158 -0
  286. package/dist/cli/handler.d.ts +2 -2
  287. package/dist/cli/handler.js +6 -6
  288. package/dist/cli/helpers.d.ts +7 -0
  289. package/dist/cli/helpers.js +128 -0
  290. package/dist/cli/index.js +96 -5228
  291. package/dist/cli/parser/help.d.ts +2 -0
  292. package/dist/cli/parser/help.js +52 -0
  293. package/dist/cli/parser/index.d.ts +3 -0
  294. package/dist/cli/parser/index.js +3 -0
  295. package/dist/cli/parser/parser.d.ts +4 -0
  296. package/dist/cli/parser/parser.js +146 -0
  297. package/dist/cli/parser/types.d.ts +41 -0
  298. package/dist/cli/parser/types.js +1 -0
  299. package/dist/cli/presets.d.ts +1 -1
  300. package/dist/cli/presets.js +1 -1
  301. package/dist/cli/router.d.ts +36 -0
  302. package/dist/cli/router.js +195 -0
  303. package/dist/cli/tui/ai-chat.js +1 -1
  304. package/dist/cli/tui/commands/context.d.ts +9 -0
  305. package/dist/cli/tui/commands/context.js +1 -0
  306. package/dist/cli/tui/commands/dns.d.ts +10 -0
  307. package/dist/cli/tui/commands/dns.js +461 -0
  308. package/dist/cli/tui/commands/hls.d.ts +2 -0
  309. package/dist/cli/tui/commands/hls.js +162 -0
  310. package/dist/cli/tui/commands/ip.d.ts +2 -0
  311. package/dist/cli/tui/commands/ip.js +45 -0
  312. package/dist/cli/tui/commands/network.d.ts +3 -0
  313. package/dist/cli/tui/commands/network.js +81 -0
  314. package/dist/cli/tui/commands/protocols.d.ts +6 -0
  315. package/dist/cli/tui/commands/protocols.js +531 -0
  316. package/dist/cli/tui/commands/security.d.ts +2 -0
  317. package/dist/cli/tui/commands/security.js +48 -0
  318. package/dist/cli/tui/commands/seo.d.ts +2 -0
  319. package/dist/cli/tui/commands/seo.js +74 -0
  320. package/dist/cli/tui/context.d.ts +12 -0
  321. package/dist/cli/tui/context.js +1 -0
  322. package/dist/cli/tui/shell.d.ts +11 -20
  323. package/dist/cli/tui/shell.js +216 -1873
  324. package/dist/constants/user-agents.d.ts +7 -0
  325. package/dist/constants/user-agents.js +7 -0
  326. package/dist/core/client.d.ts +2 -0
  327. package/dist/core/client.js +19 -1
  328. package/dist/index.d.ts +1 -0
  329. package/dist/index.js +1 -0
  330. package/dist/mcp/cli.js +2 -3
  331. package/dist/mcp/data/embeddings.json +1 -1
  332. package/dist/mcp/tools/network.js +298 -158
  333. package/dist/plugins/har-player.d.ts +23 -0
  334. package/dist/plugins/har-player.js +49 -0
  335. package/dist/plugins/har-recorder.d.ts +37 -3
  336. package/dist/plugins/har-recorder.js +116 -63
  337. package/dist/plugins/network-simulation.d.ts +7 -0
  338. package/dist/plugins/network-simulation.js +13 -0
  339. package/dist/presets/android.d.ts +2 -0
  340. package/dist/presets/android.js +16 -0
  341. package/dist/presets/chaturbate.d.ts +2 -0
  342. package/dist/presets/chaturbate.js +17 -0
  343. package/dist/presets/elevenlabs.d.ts +6 -0
  344. package/dist/presets/elevenlabs.js +20 -0
  345. package/dist/presets/enhancers.d.ts +20 -0
  346. package/dist/presets/enhancers.js +85 -0
  347. package/dist/presets/hubspot.d.ts +9 -0
  348. package/dist/presets/hubspot.js +28 -0
  349. package/dist/presets/index.d.ts +10 -0
  350. package/dist/presets/index.js +10 -0
  351. package/dist/presets/ios.d.ts +2 -0
  352. package/dist/presets/ios.js +13 -0
  353. package/dist/presets/pinecone.d.ts +8 -0
  354. package/dist/presets/pinecone.js +42 -0
  355. package/dist/presets/registry.js +60 -0
  356. package/dist/presets/sendgrid.d.ts +6 -0
  357. package/dist/presets/sendgrid.js +20 -0
  358. package/dist/presets/sentry.d.ts +11 -0
  359. package/dist/presets/sentry.js +48 -0
  360. package/dist/presets/square.d.ts +10 -0
  361. package/dist/presets/square.js +33 -0
  362. package/dist/recker.d.ts +3 -0
  363. package/dist/recker.js +4 -0
  364. package/dist/scrape/document.d.ts +5 -4
  365. package/dist/scrape/document.js +89 -76
  366. package/dist/scrape/element.d.ts +10 -8
  367. package/dist/scrape/element.js +295 -81
  368. package/dist/scrape/extractors.d.ts +11 -11
  369. package/dist/scrape/extractors.js +145 -113
  370. package/dist/scrape/index.d.ts +2 -0
  371. package/dist/scrape/index.js +1 -0
  372. package/dist/scrape/parser/back.d.ts +1 -0
  373. package/dist/scrape/parser/back.js +3 -0
  374. package/dist/scrape/parser/index.d.ts +20 -0
  375. package/dist/scrape/parser/index.js +19 -0
  376. package/dist/scrape/parser/matcher.d.ts +30 -0
  377. package/dist/scrape/parser/matcher.js +99 -0
  378. package/dist/scrape/parser/nodes/comment.d.ts +12 -0
  379. package/dist/scrape/parser/nodes/comment.js +21 -0
  380. package/dist/scrape/parser/nodes/html.d.ts +110 -0
  381. package/dist/scrape/parser/nodes/html.js +978 -0
  382. package/dist/scrape/parser/nodes/node.d.ts +18 -0
  383. package/dist/scrape/parser/nodes/node.js +31 -0
  384. package/dist/scrape/parser/nodes/text.d.ts +14 -0
  385. package/dist/scrape/parser/nodes/text.js +30 -0
  386. package/dist/scrape/parser/nodes/type.d.ts +6 -0
  387. package/dist/scrape/parser/nodes/type.js +7 -0
  388. package/dist/scrape/parser/parse.d.ts +1 -0
  389. package/dist/scrape/parser/parse.js +1 -0
  390. package/dist/scrape/parser/valid.d.ts +2 -0
  391. package/dist/scrape/parser/valid.js +5 -0
  392. package/dist/scrape/parser/void-tag.d.ts +7 -0
  393. package/dist/scrape/parser/void-tag.js +43 -0
  394. package/dist/scrape/spider.d.ts +19 -0
  395. package/dist/scrape/spider.js +28 -3
  396. package/dist/scrape/types.d.ts +7 -0
  397. package/dist/seo/analyzer.d.ts +15 -5
  398. package/dist/seo/analyzer.js +636 -175
  399. package/dist/seo/formatter.d.ts +16 -0
  400. package/dist/seo/formatter.js +228 -0
  401. package/dist/seo/index.d.ts +2 -0
  402. package/dist/seo/index.js +1 -0
  403. package/dist/seo/keywords.d.ts +16 -0
  404. package/dist/seo/keywords.js +55 -0
  405. package/dist/seo/rules/accessibility.js +96 -57
  406. package/dist/seo/rules/ai-search.js +44 -31
  407. package/dist/seo/rules/analytics.d.ts +2 -0
  408. package/dist/seo/rules/analytics.js +306 -0
  409. package/dist/seo/rules/best-practices.js +21 -14
  410. package/dist/seo/rules/canonical.js +53 -32
  411. package/dist/seo/rules/content.js +317 -31
  412. package/dist/seo/rules/crawl.js +55 -40
  413. package/dist/seo/rules/cwv.js +21 -15
  414. package/dist/seo/rules/ecommerce.js +82 -22
  415. package/dist/seo/rules/i18n.js +75 -36
  416. package/dist/seo/rules/images.js +109 -30
  417. package/dist/seo/rules/index.js +2 -0
  418. package/dist/seo/rules/internal-linking.js +58 -39
  419. package/dist/seo/rules/links.js +79 -52
  420. package/dist/seo/rules/local.js +49 -25
  421. package/dist/seo/rules/meta.js +339 -81
  422. package/dist/seo/rules/mobile.js +112 -2
  423. package/dist/seo/rules/performance.js +434 -66
  424. package/dist/seo/rules/pwa.js +36 -39
  425. package/dist/seo/rules/readability.js +31 -22
  426. package/dist/seo/rules/redirects.js +21 -15
  427. package/dist/seo/rules/resources.js +59 -42
  428. package/dist/seo/rules/schema.js +333 -8
  429. package/dist/seo/rules/security.js +142 -80
  430. package/dist/seo/rules/social.js +277 -47
  431. package/dist/seo/rules/structural.js +87 -19
  432. package/dist/seo/rules/technical-advanced.js +30 -24
  433. package/dist/seo/rules/technical.js +243 -42
  434. package/dist/seo/rules/types.d.ts +53 -1
  435. package/dist/seo/seo-spider.d.ts +22 -0
  436. package/dist/seo/seo-spider.js +77 -13
  437. package/dist/seo/types.d.ts +8 -1
  438. package/dist/seo/validators/llms-txt.js +19 -0
  439. package/dist/seo/validators/rss.d.ts +11 -0
  440. package/dist/seo/validators/rss.js +93 -0
  441. package/dist/seo/validators/sitemap.js +36 -26
  442. package/dist/transport/curl.d.ts +4 -0
  443. package/dist/transport/curl.js +101 -0
  444. package/dist/transport/udp.js +0 -1
  445. package/dist/transport/undici.js +1 -2
  446. package/dist/transport/worker.d.ts +18 -0
  447. package/dist/transport/worker.js +278 -0
  448. package/dist/types/index.d.ts +4 -1
  449. package/dist/utils/binary-manager.d.ts +4 -0
  450. package/dist/utils/binary-manager.js +72 -0
  451. package/dist/utils/optional-require.d.ts +7 -8
  452. package/dist/utils/optional-require.js +2 -21
  453. package/dist/utils/upload.d.ts +6 -0
  454. package/dist/utils/upload.js +11 -0
  455. package/dist/utils/user-agent.js +2 -13
  456. package/dist/version.js +1 -1
  457. package/package.json +12 -6
  458. package/dist/browser/utils/optional-require.d.ts +0 -19
  459. package/dist/browser/utils/optional-require.js +0 -105
package/dist/browser/seo/rules/canonical.js ADDED
@@ -0,0 +1,270 @@
1
+ import { createResult } from './types.js';
2
+ export const canonicalRules = [
3
+ {
4
+ id: 'canonical-present',
5
+ name: 'Canonical Tag Present',
6
+ category: 'canonicalization',
7
+ severity: 'warning',
8
+ description: 'Pages should have a canonical URL defined',
9
+ check: (ctx) => {
10
+ if (ctx.hasCanonical === undefined) {
11
+ return createResult({ id: 'canonical-present', name: 'Canonical Tag Present', category: 'canonicalization', severity: 'warning' }, 'info', 'Not applicable (canonical data unavailable)', { recommendation: 'This rule checks if a canonical URL is defined to avoid duplicate content issues' });
12
+ }
13
+ if (!ctx.hasCanonical) {
14
+ return createResult({ id: 'canonical-present', name: 'Canonical Tag Present', category: 'canonicalization', severity: 'warning' }, 'warn', 'Page is missing canonical tag', {
15
+ recommendation: 'Add <link rel="canonical" href="..."> to specify the preferred URL',
16
+ evidence: {
17
+ found: 'No canonical tag',
18
+ expected: '<link rel="canonical" href="https://example.com/page">',
19
+ impact: 'Without canonical, search engines may index duplicate versions',
20
+ example: '<head>\n <link rel="canonical" href="https://example.com/page" />\n</head>',
21
+ }
22
+ });
23
+ }
24
+ return createResult({ id: 'canonical-present', name: 'Canonical Tag Present', category: 'canonicalization', severity: 'warning' }, 'pass', 'Canonical tag is present');
25
+ },
26
+ },
27
+ {
28
+ id: 'canonical-multiple',
29
+ name: 'Multiple Canonical Tags',
30
+ category: 'canonicalization',
31
+ severity: 'error',
32
+ description: 'Page should have only one canonical tag',
33
+ check: (ctx) => {
34
+ if (ctx.canonicalCount === undefined) {
35
+ return createResult({ id: 'canonical-multiple', name: 'Multiple Canonical Tags', category: 'canonicalization', severity: 'error' }, 'info', 'Not applicable (canonical count data unavailable)', { recommendation: 'This rule ensures only one canonical tag exists per page' });
36
+ }
37
+ if (ctx.canonicalCount > 1) {
38
+ return createResult({ id: 'canonical-multiple', name: 'Multiple Canonical Tags', category: 'canonicalization', severity: 'error' }, 'fail', `Page has ${ctx.canonicalCount} canonical tags`, {
39
+ value: ctx.canonicalCount,
40
+ recommendation: 'Remove duplicate canonical tags; keep only one',
41
+ evidence: {
42
+ found: ctx.canonicalUrls || [],
43
+ expected: 'Single canonical tag',
44
+ impact: 'Multiple canonicals confuse search engines about preferred URL',
45
+ example: '<link rel="canonical" href="https://example.com/page" /> <!-- Keep only one -->',
46
+ }
47
+ });
48
+ }
49
+ return createResult({ id: 'canonical-multiple', name: 'Multiple Canonical Tags', category: 'canonicalization', severity: 'error' }, 'pass', 'Single canonical tag found');
50
+ },
51
+ },
52
+ {
53
+ id: 'canonical-self-referencing',
54
+ name: 'Canonical Self-Reference',
55
+ category: 'canonicalization',
56
+ severity: 'info',
57
+ description: 'Canonical should typically point to the current page URL',
58
+ check: (ctx) => {
59
+ if (!ctx.canonicalUrl || !ctx.url) {
60
+ return createResult({ id: 'canonical-self-referencing', name: 'Canonical Self-Reference', category: 'canonicalization', severity: 'info' }, 'info', 'Not applicable (canonical or URL data unavailable)', { recommendation: 'This rule checks if canonical URL points to the current page' });
61
+ }
62
+ const isSelfReferencing = normalizeUrl(ctx.canonicalUrl) === normalizeUrl(ctx.url);
63
+ if (!isSelfReferencing) {
64
+ return createResult({ id: 'canonical-self-referencing', name: 'Canonical Self-Reference', category: 'canonicalization', severity: 'info' }, 'info', 'Canonical points to different URL', {
65
+ recommendation: 'Verify this is intentional canonicalization',
66
+ evidence: {
67
+ found: ctx.canonicalUrl,
68
+ expected: ctx.url,
69
+ impact: 'This page signals to search engines that another URL is preferred',
70
+ example: `Current URL: ${ctx.url}\nCanonical: ${ctx.canonicalUrl}`,
71
+ }
72
+ });
73
+ }
74
+ return createResult({ id: 'canonical-self-referencing', name: 'Canonical Self-Reference', category: 'canonicalization', severity: 'info' }, 'pass', 'Canonical is self-referencing');
75
+ },
76
+ },
77
+ {
78
+ id: 'canonical-broken',
79
+ name: 'Broken Canonical URL',
80
+ category: 'canonicalization',
81
+ severity: 'error',
82
+ description: 'Canonical URL should be accessible (not 404)',
83
+ check: (ctx) => {
84
+ if (ctx.canonicalStatus === undefined) {
85
+ return createResult({ id: 'canonical-broken', name: 'Broken Canonical URL', category: 'canonicalization', severity: 'error' }, 'info', 'Not applicable (canonical status data unavailable)', { recommendation: 'This rule validates that canonical URL is accessible and returns 200 OK' });
86
+ }
87
+ if (ctx.canonicalStatus === 404) {
88
+ return createResult({ id: 'canonical-broken', name: 'Broken Canonical URL', category: 'canonicalization', severity: 'error' }, 'fail', 'Canonical URL returns 404', {
89
+ value: ctx.canonicalUrl,
90
+ recommendation: 'Fix canonical to point to an existing, accessible page',
91
+ evidence: {
92
+ found: `${ctx.canonicalUrl} → 404`,
93
+ expected: '200 OK',
94
+ impact: 'Broken canonical prevents proper indexing',
95
+ example: '<link rel="canonical" href="https://example.com/existing-page" />',
96
+ }
97
+ });
98
+ }
99
+ if (ctx.canonicalStatus >= 400) {
100
+ return createResult({ id: 'canonical-broken', name: 'Broken Canonical URL', category: 'canonicalization', severity: 'error' }, 'fail', `Canonical URL returns error (${ctx.canonicalStatus})`, {
101
+ value: ctx.canonicalUrl,
102
+ recommendation: 'Fix canonical to point to an accessible page',
103
+ evidence: {
104
+ found: `${ctx.canonicalUrl} → ${ctx.canonicalStatus}`,
105
+ expected: '200 OK',
106
+ impact: 'Canonical errors prevent proper indexing',
107
+ example: '<link rel="canonical" href="https://example.com/accessible-page" />',
108
+ }
109
+ });
110
+ }
111
+ if (ctx.canonicalStatus >= 300 && ctx.canonicalFinalUrl) {
112
+ return createResult({ id: 'canonical-broken', name: 'Broken Canonical URL', category: 'canonicalization', severity: 'error' }, 'warn', 'Canonical URL redirects', {
113
+ value: ctx.canonicalUrl,
114
+ recommendation: 'Update canonical to final destination URL',
115
+ evidence: {
116
+ found: `${ctx.canonicalUrl} → ${ctx.canonicalFinalUrl}`,
117
+ expected: 'Direct URL without redirect',
118
+ impact: 'Canonical redirects waste crawl budget',
119
+ example: `<link rel="canonical" href="${ctx.canonicalFinalUrl}" />`,
120
+ }
121
+ });
122
+ }
123
+ return createResult({ id: 'canonical-broken', name: 'Broken Canonical URL', category: 'canonicalization', severity: 'error' }, 'pass', 'Canonical URL is accessible');
124
+ },
125
+ },
126
+ {
127
+ id: 'canonical-protocol',
128
+ name: 'Canonical Protocol',
129
+ category: 'canonicalization',
130
+ severity: 'warning',
131
+ description: 'Canonical should use HTTPS protocol',
132
+ check: (ctx) => {
133
+ if (!ctx.canonicalUrl) {
134
+ return createResult({ id: 'canonical-protocol', name: 'Canonical Protocol', category: 'canonicalization', severity: 'warning' }, 'info', 'Not applicable (no canonical URL detected)', { recommendation: 'This rule ensures canonical URLs use HTTPS protocol for security' });
135
+ }
136
+ if (ctx.canonicalUrl.startsWith('http://')) {
137
+ return createResult({ id: 'canonical-protocol', name: 'Canonical Protocol', category: 'canonicalization', severity: 'warning' }, 'warn', 'Canonical uses HTTP instead of HTTPS', {
138
+ value: ctx.canonicalUrl,
139
+ recommendation: 'Update canonical to use HTTPS',
140
+ evidence: {
141
+ found: ctx.canonicalUrl,
142
+ expected: ctx.canonicalUrl.replace('http://', 'https://'),
143
+ impact: 'HTTP canonicals may cause indexing preference issues',
144
+ example: `<link rel="canonical" href="${ctx.canonicalUrl.replace('http://', 'https://')}" />`,
145
+ }
146
+ });
147
+ }
148
+ return createResult({ id: 'canonical-protocol', name: 'Canonical Protocol', category: 'canonicalization', severity: 'warning' }, 'pass', 'Canonical uses HTTPS');
149
+ },
150
+ },
151
+ {
152
+ id: 'canonical-absolute',
153
+ name: 'Canonical Absolute URL',
154
+ category: 'canonicalization',
155
+ severity: 'warning',
156
+ description: 'Canonical URL should be absolute, not relative',
157
+ check: (ctx) => {
158
+ if (!ctx.canonicalUrl) {
159
+ return createResult({ id: 'canonical-absolute', name: 'Canonical Absolute URL', category: 'canonicalization', severity: 'warning' }, 'info', 'Not applicable (no canonical URL detected)', { recommendation: 'This rule checks that canonical URLs are absolute, not relative' });
160
+ }
161
+ const isRelative = !ctx.canonicalUrl.startsWith('http://') &&
162
+ !ctx.canonicalUrl.startsWith('https://') &&
163
+ !ctx.canonicalUrl.startsWith('//');
164
+ if (isRelative) {
165
+ return createResult({ id: 'canonical-absolute', name: 'Canonical Absolute URL', category: 'canonicalization', severity: 'warning' }, 'warn', 'Canonical URL is relative', {
166
+ value: ctx.canonicalUrl,
167
+ recommendation: 'Use absolute URL including protocol and domain',
168
+ evidence: {
169
+ found: ctx.canonicalUrl,
170
+ expected: 'https://example.com/page',
171
+ impact: 'Relative canonicals may be misinterpreted by crawlers',
172
+ example: '<link rel="canonical" href="https://example.com/page" />',
173
+ }
174
+ });
175
+ }
176
+ return createResult({ id: 'canonical-absolute', name: 'Canonical Absolute URL', category: 'canonicalization', severity: 'warning' }, 'pass', 'Canonical URL is absolute');
177
+ },
178
+ },
179
+ {
180
+ id: 'canonical-chain',
181
+ name: 'Canonical Chain',
182
+ category: 'canonicalization',
183
+ severity: 'warning',
184
+ description: 'Canonical should not create chains',
185
+ check: (ctx) => {
186
+ if (ctx.canonicalChainLength === undefined) {
187
+ return createResult({ id: 'canonical-chain', name: 'Canonical Chain', category: 'canonicalization', severity: 'warning' }, 'info', 'Not applicable (canonical chain data unavailable)', { recommendation: 'This rule detects canonical chains where one canonical points to another' });
188
+ }
189
+ if (ctx.canonicalChainLength > 1) {
190
+ return createResult({ id: 'canonical-chain', name: 'Canonical Chain', category: 'canonicalization', severity: 'warning' }, 'warn', `Canonical chain detected (${ctx.canonicalChainLength} hops)`, {
191
+ value: ctx.canonicalChainLength,
192
+ recommendation: 'Update canonical to point directly to final canonical URL',
193
+ evidence: {
194
+ found: ctx.canonicalChain || [],
195
+ expected: 'Direct canonical to final URL',
196
+ impact: 'Canonical chains may cause consolidation issues',
197
+ example: `<link rel="canonical" href="${ctx.canonicalChain?.[ctx.canonicalChain.length - 1] || 'https://example.com/final-page'}" />`,
198
+ }
199
+ });
200
+ }
201
+ return createResult({ id: 'canonical-chain', name: 'Canonical Chain', category: 'canonicalization', severity: 'warning' }, 'pass', 'No canonical chain detected');
202
+ },
203
+ },
204
+ {
205
+ id: 'canonical-parameters',
206
+ name: 'Canonical Query Parameters',
207
+ category: 'canonicalization',
208
+ severity: 'info',
209
+ description: 'Canonical URLs with query parameters should be intentional',
210
+ check: (ctx) => {
211
+ if (!ctx.canonicalUrl) {
212
+ return createResult({ id: 'canonical-parameters', name: 'Canonical Query Parameters', category: 'canonicalization', severity: 'info' }, 'info', 'Not applicable (no canonical URL detected)', { recommendation: 'This rule checks for query parameters in canonical URLs' });
213
+ }
214
+ try {
215
+ const url = new URL(ctx.canonicalUrl);
216
+ if (url.search && url.search.length > 1) {
217
+ return createResult({ id: 'canonical-parameters', name: 'Canonical Query Parameters', category: 'canonicalization', severity: 'info' }, 'info', 'Canonical contains query parameters', {
218
+ value: ctx.canonicalUrl,
219
+ recommendation: 'Verify query parameters in canonical are intentional',
220
+ evidence: {
221
+ found: url.search,
222
+ impact: 'Query parameters in canonicals may indicate tracking or filtering'
223
+ }
224
+ });
225
+ }
226
+ }
227
+ catch {
228
+ }
229
+ return createResult({ id: 'canonical-parameters', name: 'Canonical Query Parameters', category: 'canonicalization', severity: 'info' }, 'pass', 'Canonical URL has no query parameters');
230
+ },
231
+ },
232
+ {
233
+ id: 'canonical-noindex-conflict',
234
+ name: 'Canonical + Noindex Conflict',
235
+ category: 'canonicalization',
236
+ severity: 'warning',
237
+ description: 'Pages with noindex should not have canonical to indexed page',
238
+ check: (ctx) => {
239
+ if (!ctx.hasCanonical || ctx.metaRobots === undefined) {
240
+ return createResult({ id: 'canonical-noindex-conflict', name: 'Canonical + Noindex Conflict', category: 'canonicalization', severity: 'warning' }, 'info', 'Not applicable (canonical or robots meta tag data unavailable)', { recommendation: 'This rule detects conflicting noindex and canonical directives' });
241
+ }
242
+ const robots = Array.isArray(ctx.metaRobots) ? ctx.metaRobots : [ctx.metaRobots];
243
+ const hasNoindex = robots.some(r => r.toLowerCase().includes('noindex'));
244
+ if (hasNoindex && ctx.canonicalUrl) {
245
+ const isSelfReferencing = ctx.url && normalizeUrl(ctx.canonicalUrl) === normalizeUrl(ctx.url);
246
+ if (isSelfReferencing) {
247
+ return createResult({ id: 'canonical-noindex-conflict', name: 'Canonical + Noindex Conflict', category: 'canonicalization', severity: 'warning' }, 'warn', 'Page has both noindex and self-referencing canonical', {
248
+ recommendation: 'Remove canonical or noindex - conflicting signals',
249
+ evidence: {
250
+ found: `noindex + canonical to self (${ctx.canonicalUrl})`,
251
+ expected: 'Either noindex OR canonical, not both',
252
+ impact: 'Google ignores noindex if page has canonical to itself',
253
+ example: '<meta name="robots" content="noindex" /> <!-- Remove this -->\n<!-- OR -->\n<link rel="canonical" href="..." /> <!-- Remove this -->',
254
+ }
255
+ });
256
+ }
257
+ }
258
+ return createResult({ id: 'canonical-noindex-conflict', name: 'Canonical + Noindex Conflict', category: 'canonicalization', severity: 'warning' }, 'pass', 'No conflicting noindex and canonical directives');
259
+ },
260
+ },
261
+ ];
262
+ function normalizeUrl(url) {
263
+ try {
264
+ const u = new URL(url);
265
+ return `${u.protocol}//${u.hostname}${u.pathname.replace(/\/$/, '')}${u.search}`.toLowerCase();
266
+ }
267
+ catch {
268
+ return url.toLowerCase();
269
+ }
270
+ }
package/dist/browser/seo/rules/content.d.ts ADDED
@@ -0,0 +1,2 @@
1
+ import { SeoRule } from './types.js';
2
+ export declare const contentRules: SeoRule[];