recker 1.0.43 → 1.0.44

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (459) hide show
  1. package/README.md +47 -0
  2. package/dist/bin/recker-linux-x64 +0 -0
  3. package/dist/bin/recker-macos-x64 +0 -0
  4. package/dist/bin/recker-win-x64.exe +0 -0
  5. package/dist/bin/rek.cjs +85152 -100207
  6. package/dist/browser/ai/adaptive-timeout.d.ts +50 -0
  7. package/dist/browser/ai/adaptive-timeout.js +208 -0
  8. package/dist/browser/ai/client.d.ts +22 -0
  9. package/dist/browser/ai/client.js +294 -0
  10. package/dist/browser/ai/index.d.ts +14 -0
  11. package/dist/browser/ai/index.js +11 -0
  12. package/dist/browser/ai/providers/anthropic.d.ts +63 -0
  13. package/dist/browser/ai/providers/anthropic.js +370 -0
  14. package/dist/browser/ai/providers/base.d.ts +48 -0
  15. package/dist/browser/ai/providers/base.js +150 -0
  16. package/dist/browser/ai/providers/google.d.ts +59 -0
  17. package/dist/browser/ai/providers/google.js +305 -0
  18. package/dist/browser/ai/providers/ollama.d.ts +44 -0
  19. package/dist/browser/ai/providers/ollama.js +240 -0
  20. package/dist/browser/ai/providers/openai.d.ts +64 -0
  21. package/dist/browser/ai/providers/openai.js +298 -0
  22. package/dist/browser/ai/rate-limiter.d.ts +43 -0
  23. package/dist/browser/ai/rate-limiter.js +215 -0
  24. package/dist/browser/ai/vector/index.d.ts +2 -0
  25. package/dist/browser/ai/vector/index.js +2 -0
  26. package/dist/browser/ai/vector/similarity.d.ts +2 -0
  27. package/dist/browser/ai/vector/similarity.js +27 -0
  28. package/dist/browser/ai/vector/store.d.ts +27 -0
  29. package/dist/browser/ai/vector/store.js +82 -0
  30. package/dist/browser/browser/cache.d.ts +2 -40
  31. package/dist/browser/browser/cache.js +2 -199
  32. package/dist/browser/browser/index.d.ts +8 -0
  33. package/dist/browser/browser/index.js +8 -0
  34. package/dist/browser/browser/recker.d.ts +8 -1
  35. package/dist/browser/browser/recker.js +8 -2
  36. package/dist/browser/cache/indexed-db.d.ts +10 -0
  37. package/dist/browser/cache/indexed-db.js +88 -0
  38. package/dist/browser/cache/service-worker-cache.d.ts +18 -0
  39. package/dist/browser/cache/service-worker-cache.js +103 -0
  40. package/dist/browser/cache.d.ts +2 -40
  41. package/dist/browser/cache.js +2 -199
  42. package/dist/browser/constants/user-agents.d.ts +7 -0
  43. package/dist/browser/constants/user-agents.js +7 -0
  44. package/dist/browser/core/client.d.ts +2 -0
  45. package/dist/browser/core/client.js +19 -1
  46. package/dist/browser/index.d.ts +8 -0
  47. package/dist/browser/index.js +8 -0
  48. package/dist/browser/plugins/har-recorder.d.ts +40 -0
  49. package/dist/browser/plugins/har-recorder.js +120 -0
  50. package/dist/browser/plugins/network-simulation.d.ts +7 -0
  51. package/dist/browser/plugins/network-simulation.js +13 -0
  52. package/dist/browser/presets/android.d.ts +2 -0
  53. package/dist/browser/presets/android.js +16 -0
  54. package/dist/browser/presets/anthropic.d.ts +8 -0
  55. package/dist/browser/presets/anthropic.js +27 -0
  56. package/dist/browser/presets/aws.d.ts +19 -0
  57. package/dist/browser/presets/aws.js +68 -0
  58. package/dist/browser/presets/azure-openai.d.ts +10 -0
  59. package/dist/browser/presets/azure-openai.js +35 -0
  60. package/dist/browser/presets/azure.d.ts +41 -0
  61. package/dist/browser/presets/azure.js +104 -0
  62. package/dist/browser/presets/chaturbate.d.ts +2 -0
  63. package/dist/browser/presets/chaturbate.js +17 -0
  64. package/dist/browser/presets/cloudflare.d.ts +12 -0
  65. package/dist/browser/presets/cloudflare.js +39 -0
  66. package/dist/browser/presets/cohere.d.ts +7 -0
  67. package/dist/browser/presets/cohere.js +22 -0
  68. package/dist/browser/presets/deepseek.d.ts +7 -0
  69. package/dist/browser/presets/deepseek.js +22 -0
  70. package/dist/browser/presets/digitalocean.d.ts +5 -0
  71. package/dist/browser/presets/digitalocean.js +16 -0
  72. package/dist/browser/presets/discord.d.ts +6 -0
  73. package/dist/browser/presets/discord.js +17 -0
  74. package/dist/browser/presets/elevenlabs.d.ts +6 -0
  75. package/dist/browser/presets/elevenlabs.js +20 -0
  76. package/dist/browser/presets/enhancers.d.ts +20 -0
  77. package/dist/browser/presets/enhancers.js +85 -0
  78. package/dist/browser/presets/fireworks.d.ts +7 -0
  79. package/dist/browser/presets/fireworks.js +22 -0
  80. package/dist/browser/presets/gcp.d.ts +34 -0
  81. package/dist/browser/presets/gcp.js +91 -0
  82. package/dist/browser/presets/gemini.d.ts +7 -0
  83. package/dist/browser/presets/gemini.js +23 -0
  84. package/dist/browser/presets/github.d.ts +6 -0
  85. package/dist/browser/presets/github.js +17 -0
  86. package/dist/browser/presets/gitlab.d.ts +6 -0
  87. package/dist/browser/presets/gitlab.js +16 -0
  88. package/dist/browser/presets/groq.d.ts +7 -0
  89. package/dist/browser/presets/groq.js +22 -0
  90. package/dist/browser/presets/hubspot.d.ts +9 -0
  91. package/dist/browser/presets/hubspot.js +28 -0
  92. package/dist/browser/presets/huggingface.d.ts +7 -0
  93. package/dist/browser/presets/huggingface.js +23 -0
  94. package/dist/browser/presets/index.d.ts +47 -0
  95. package/dist/browser/presets/index.js +47 -0
  96. package/dist/browser/presets/ios.d.ts +2 -0
  97. package/dist/browser/presets/ios.js +13 -0
  98. package/dist/browser/presets/linear.d.ts +5 -0
  99. package/dist/browser/presets/linear.js +16 -0
  100. package/dist/browser/presets/mailgun.d.ts +7 -0
  101. package/dist/browser/presets/mailgun.js +20 -0
  102. package/dist/browser/presets/meta.d.ts +10 -0
  103. package/dist/browser/presets/meta.js +33 -0
  104. package/dist/browser/presets/mistral.d.ts +7 -0
  105. package/dist/browser/presets/mistral.js +22 -0
  106. package/dist/browser/presets/notion.d.ts +6 -0
  107. package/dist/browser/presets/notion.js +17 -0
  108. package/dist/browser/presets/openai.d.ts +9 -0
  109. package/dist/browser/presets/openai.js +30 -0
  110. package/dist/browser/presets/oracle.d.ts +19 -0
  111. package/dist/browser/presets/oracle.js +117 -0
  112. package/dist/browser/presets/perplexity.d.ts +7 -0
  113. package/dist/browser/presets/perplexity.js +22 -0
  114. package/dist/browser/presets/pinecone.d.ts +8 -0
  115. package/dist/browser/presets/pinecone.js +42 -0
  116. package/dist/browser/presets/registry.d.ts +23 -0
  117. package/dist/browser/presets/registry.js +519 -0
  118. package/dist/browser/presets/replicate.d.ts +7 -0
  119. package/dist/browser/presets/replicate.js +23 -0
  120. package/dist/browser/presets/sendgrid.d.ts +6 -0
  121. package/dist/browser/presets/sendgrid.js +20 -0
  122. package/dist/browser/presets/sentry.d.ts +11 -0
  123. package/dist/browser/presets/sentry.js +48 -0
  124. package/dist/browser/presets/sinch.d.ts +9 -0
  125. package/dist/browser/presets/sinch.js +39 -0
  126. package/dist/browser/presets/slack.d.ts +5 -0
  127. package/dist/browser/presets/slack.js +16 -0
  128. package/dist/browser/presets/square.d.ts +10 -0
  129. package/dist/browser/presets/square.js +33 -0
  130. package/dist/browser/presets/stripe.d.ts +7 -0
  131. package/dist/browser/presets/stripe.js +23 -0
  132. package/dist/browser/presets/supabase.d.ts +6 -0
  133. package/dist/browser/presets/supabase.js +18 -0
  134. package/dist/browser/presets/tiktok.d.ts +10 -0
  135. package/dist/browser/presets/tiktok.js +38 -0
  136. package/dist/browser/presets/together.d.ts +7 -0
  137. package/dist/browser/presets/together.js +22 -0
  138. package/dist/browser/presets/twilio.d.ts +6 -0
  139. package/dist/browser/presets/twilio.js +17 -0
  140. package/dist/browser/presets/vercel.d.ts +6 -0
  141. package/dist/browser/presets/vercel.js +23 -0
  142. package/dist/browser/presets/vultr.d.ts +5 -0
  143. package/dist/browser/presets/vultr.js +16 -0
  144. package/dist/browser/presets/xai.d.ts +8 -0
  145. package/dist/browser/presets/xai.js +23 -0
  146. package/dist/browser/presets/youtube.d.ts +5 -0
  147. package/dist/browser/presets/youtube.js +20 -0
  148. package/dist/browser/recker.d.ts +8 -1
  149. package/dist/browser/recker.js +8 -2
  150. package/dist/browser/scrape/document.d.ts +5 -4
  151. package/dist/browser/scrape/document.js +89 -76
  152. package/dist/browser/scrape/element.d.ts +10 -8
  153. package/dist/browser/scrape/element.js +295 -81
  154. package/dist/browser/scrape/extractors.d.ts +11 -11
  155. package/dist/browser/scrape/extractors.js +145 -113
  156. package/dist/browser/scrape/parser/back.d.ts +1 -0
  157. package/dist/browser/scrape/parser/back.js +3 -0
  158. package/dist/browser/scrape/parser/index.d.ts +20 -0
  159. package/dist/browser/scrape/parser/index.js +19 -0
  160. package/dist/browser/scrape/parser/matcher.d.ts +30 -0
  161. package/dist/browser/scrape/parser/matcher.js +99 -0
  162. package/dist/browser/scrape/parser/nodes/comment.d.ts +12 -0
  163. package/dist/browser/scrape/parser/nodes/comment.js +21 -0
  164. package/dist/browser/scrape/parser/nodes/html.d.ts +110 -0
  165. package/dist/browser/scrape/parser/nodes/html.js +978 -0
  166. package/dist/browser/scrape/parser/nodes/node.d.ts +18 -0
  167. package/dist/browser/scrape/parser/nodes/node.js +31 -0
  168. package/dist/browser/scrape/parser/nodes/text.d.ts +14 -0
  169. package/dist/browser/scrape/parser/nodes/text.js +30 -0
  170. package/dist/browser/scrape/parser/nodes/type.d.ts +6 -0
  171. package/dist/browser/scrape/parser/nodes/type.js +7 -0
  172. package/dist/browser/scrape/parser/parse.d.ts +1 -0
  173. package/dist/browser/scrape/parser/parse.js +1 -0
  174. package/dist/browser/scrape/parser/valid.d.ts +2 -0
  175. package/dist/browser/scrape/parser/valid.js +5 -0
  176. package/dist/browser/scrape/parser/void-tag.d.ts +7 -0
  177. package/dist/browser/scrape/parser/void-tag.js +43 -0
  178. package/dist/browser/scrape/types.d.ts +7 -0
  179. package/dist/browser/seo/analyzer.d.ts +59 -0
  180. package/dist/browser/seo/analyzer.js +1399 -0
  181. package/dist/browser/seo/keywords.d.ts +16 -0
  182. package/dist/browser/seo/keywords.js +55 -0
  183. package/dist/browser/seo/rules/accessibility.d.ts +2 -0
  184. package/dist/browser/seo/rules/accessibility.js +733 -0
  185. package/dist/browser/seo/rules/ai-search.d.ts +2 -0
  186. package/dist/browser/seo/rules/ai-search.js +436 -0
  187. package/dist/browser/seo/rules/analytics.d.ts +2 -0
  188. package/dist/browser/seo/rules/analytics.js +306 -0
  189. package/dist/browser/seo/rules/best-practices.d.ts +2 -0
  190. package/dist/browser/seo/rules/best-practices.js +195 -0
  191. package/dist/browser/seo/rules/canonical.d.ts +12 -0
  192. package/dist/browser/seo/rules/canonical.js +270 -0
  193. package/dist/browser/seo/rules/content.d.ts +2 -0
  194. package/dist/browser/seo/rules/content.js +522 -0
  195. package/dist/browser/seo/rules/crawl.d.ts +2 -0
  196. package/dist/browser/seo/rules/crawl.js +435 -0
  197. package/dist/browser/seo/rules/cwv.d.ts +2 -0
  198. package/dist/browser/seo/rules/cwv.js +248 -0
  199. package/dist/browser/seo/rules/ecommerce.d.ts +2 -0
  200. package/dist/browser/seo/rules/ecommerce.js +312 -0
  201. package/dist/browser/seo/rules/i18n.d.ts +2 -0
  202. package/dist/browser/seo/rules/i18n.js +288 -0
  203. package/dist/browser/seo/rules/images.d.ts +2 -0
  204. package/dist/browser/seo/rules/images.js +255 -0
  205. package/dist/browser/seo/rules/index.d.ts +52 -0
  206. package/dist/browser/seo/rules/index.js +159 -0
  207. package/dist/browser/seo/rules/internal-linking.d.ts +2 -0
  208. package/dist/browser/seo/rules/internal-linking.js +394 -0
  209. package/dist/browser/seo/rules/links.d.ts +2 -0
  210. package/dist/browser/seo/rules/links.js +498 -0
  211. package/dist/browser/seo/rules/local.d.ts +2 -0
  212. package/dist/browser/seo/rules/local.js +289 -0
  213. package/dist/browser/seo/rules/meta.d.ts +2 -0
  214. package/dist/browser/seo/rules/meta.js +805 -0
  215. package/dist/browser/seo/rules/mobile.d.ts +2 -0
  216. package/dist/browser/seo/rules/mobile.js +161 -0
  217. package/dist/browser/seo/rules/performance.d.ts +2 -0
  218. package/dist/browser/seo/rules/performance.js +738 -0
  219. package/dist/browser/seo/rules/pwa.d.ts +2 -0
  220. package/dist/browser/seo/rules/pwa.js +299 -0
  221. package/dist/browser/seo/rules/readability.d.ts +2 -0
  222. package/dist/browser/seo/rules/readability.js +264 -0
  223. package/dist/browser/seo/rules/redirects.d.ts +16 -0
  224. package/dist/browser/seo/rules/redirects.js +199 -0
  225. package/dist/browser/seo/rules/resources.d.ts +2 -0
  226. package/dist/browser/seo/rules/resources.js +390 -0
  227. package/dist/browser/seo/rules/schema.d.ts +2 -0
  228. package/dist/browser/seo/rules/schema.js +379 -0
  229. package/dist/browser/seo/rules/security.d.ts +2 -0
  230. package/dist/browser/seo/rules/security.js +877 -0
  231. package/dist/browser/seo/rules/social.d.ts +2 -0
  232. package/dist/browser/seo/rules/social.js +603 -0
  233. package/dist/browser/seo/rules/structural.d.ts +2 -0
  234. package/dist/browser/seo/rules/structural.js +223 -0
  235. package/dist/browser/seo/rules/technical-advanced.d.ts +10 -0
  236. package/dist/browser/seo/rules/technical-advanced.js +289 -0
  237. package/dist/browser/seo/rules/technical.d.ts +2 -0
  238. package/dist/browser/seo/rules/technical.js +480 -0
  239. package/dist/browser/seo/rules/thresholds.d.ts +196 -0
  240. package/dist/browser/seo/rules/thresholds.js +118 -0
  241. package/dist/browser/seo/rules/types.d.ts +498 -0
  242. package/dist/browser/seo/rules/types.js +11 -0
  243. package/dist/browser/seo/types.d.ts +211 -0
  244. package/dist/browser/seo/types.js +1 -0
  245. package/dist/browser/transport/curl.d.ts +4 -0
  246. package/dist/browser/transport/curl.js +101 -0
  247. package/dist/browser/transport/undici.js +1 -2
  248. package/dist/browser/transport/worker.d.ts +18 -0
  249. package/dist/browser/transport/worker.js +278 -0
  250. package/dist/browser/types/index.d.ts +4 -1
  251. package/dist/browser/utils/binary-manager.d.ts +4 -0
  252. package/dist/browser/utils/binary-manager.js +72 -0
  253. package/dist/browser/utils/user-agent.js +2 -13
  254. package/dist/cache/indexed-db.d.ts +10 -0
  255. package/dist/cache/indexed-db.js +88 -0
  256. package/dist/cache/service-worker-cache.d.ts +18 -0
  257. package/dist/cache/service-worker-cache.js +103 -0
  258. package/dist/cli/commands/ai.d.ts +2 -0
  259. package/dist/cli/commands/ai.js +162 -0
  260. package/dist/cli/commands/bench.d.ts +2 -0
  261. package/dist/cli/commands/bench.js +51 -0
  262. package/dist/cli/commands/dns.d.ts +2 -0
  263. package/dist/cli/commands/dns.js +295 -0
  264. package/dist/cli/commands/har.d.ts +2 -0
  265. package/dist/cli/commands/har.js +171 -0
  266. package/dist/cli/commands/hls.d.ts +2 -0
  267. package/dist/cli/commands/hls.js +192 -0
  268. package/dist/cli/commands/network.d.ts +2 -0
  269. package/dist/cli/commands/network.js +288 -0
  270. package/dist/cli/commands/protocols.d.ts +2 -0
  271. package/dist/cli/commands/protocols.js +344 -0
  272. package/dist/cli/commands/scrape.d.ts +2 -0
  273. package/dist/cli/commands/scrape.js +176 -0
  274. package/dist/cli/commands/security.d.ts +2 -0
  275. package/dist/cli/commands/security.js +57 -0
  276. package/dist/cli/commands/seo.d.ts +2 -0
  277. package/dist/cli/commands/seo.js +125 -0
  278. package/dist/cli/commands/serve.d.ts +2 -0
  279. package/dist/cli/commands/serve.js +531 -0
  280. package/dist/cli/commands/spider.d.ts +3 -0
  281. package/dist/cli/commands/spider.js +456 -0
  282. package/dist/cli/commands/utils.d.ts +2 -0
  283. package/dist/cli/commands/utils.js +176 -0
  284. package/dist/cli/commands/vector.d.ts +2 -0
  285. package/dist/cli/commands/vector.js +158 -0
  286. package/dist/cli/handler.d.ts +2 -2
  287. package/dist/cli/handler.js +6 -6
  288. package/dist/cli/helpers.d.ts +7 -0
  289. package/dist/cli/helpers.js +128 -0
  290. package/dist/cli/index.js +96 -5228
  291. package/dist/cli/parser/help.d.ts +2 -0
  292. package/dist/cli/parser/help.js +52 -0
  293. package/dist/cli/parser/index.d.ts +3 -0
  294. package/dist/cli/parser/index.js +3 -0
  295. package/dist/cli/parser/parser.d.ts +4 -0
  296. package/dist/cli/parser/parser.js +146 -0
  297. package/dist/cli/parser/types.d.ts +41 -0
  298. package/dist/cli/parser/types.js +1 -0
  299. package/dist/cli/presets.d.ts +1 -1
  300. package/dist/cli/presets.js +1 -1
  301. package/dist/cli/router.d.ts +36 -0
  302. package/dist/cli/router.js +195 -0
  303. package/dist/cli/tui/ai-chat.js +1 -1
  304. package/dist/cli/tui/commands/context.d.ts +9 -0
  305. package/dist/cli/tui/commands/context.js +1 -0
  306. package/dist/cli/tui/commands/dns.d.ts +10 -0
  307. package/dist/cli/tui/commands/dns.js +461 -0
  308. package/dist/cli/tui/commands/hls.d.ts +2 -0
  309. package/dist/cli/tui/commands/hls.js +162 -0
  310. package/dist/cli/tui/commands/ip.d.ts +2 -0
  311. package/dist/cli/tui/commands/ip.js +45 -0
  312. package/dist/cli/tui/commands/network.d.ts +3 -0
  313. package/dist/cli/tui/commands/network.js +81 -0
  314. package/dist/cli/tui/commands/protocols.d.ts +6 -0
  315. package/dist/cli/tui/commands/protocols.js +531 -0
  316. package/dist/cli/tui/commands/security.d.ts +2 -0
  317. package/dist/cli/tui/commands/security.js +48 -0
  318. package/dist/cli/tui/commands/seo.d.ts +2 -0
  319. package/dist/cli/tui/commands/seo.js +74 -0
  320. package/dist/cli/tui/context.d.ts +12 -0
  321. package/dist/cli/tui/context.js +1 -0
  322. package/dist/cli/tui/shell.d.ts +11 -20
  323. package/dist/cli/tui/shell.js +216 -1873
  324. package/dist/constants/user-agents.d.ts +7 -0
  325. package/dist/constants/user-agents.js +7 -0
  326. package/dist/core/client.d.ts +2 -0
  327. package/dist/core/client.js +19 -1
  328. package/dist/index.d.ts +1 -0
  329. package/dist/index.js +1 -0
  330. package/dist/mcp/cli.js +2 -3
  331. package/dist/mcp/data/embeddings.json +1 -1
  332. package/dist/mcp/tools/network.js +298 -158
  333. package/dist/plugins/har-player.d.ts +23 -0
  334. package/dist/plugins/har-player.js +49 -0
  335. package/dist/plugins/har-recorder.d.ts +37 -3
  336. package/dist/plugins/har-recorder.js +116 -63
  337. package/dist/plugins/network-simulation.d.ts +7 -0
  338. package/dist/plugins/network-simulation.js +13 -0
  339. package/dist/presets/android.d.ts +2 -0
  340. package/dist/presets/android.js +16 -0
  341. package/dist/presets/chaturbate.d.ts +2 -0
  342. package/dist/presets/chaturbate.js +17 -0
  343. package/dist/presets/elevenlabs.d.ts +6 -0
  344. package/dist/presets/elevenlabs.js +20 -0
  345. package/dist/presets/enhancers.d.ts +20 -0
  346. package/dist/presets/enhancers.js +85 -0
  347. package/dist/presets/hubspot.d.ts +9 -0
  348. package/dist/presets/hubspot.js +28 -0
  349. package/dist/presets/index.d.ts +10 -0
  350. package/dist/presets/index.js +10 -0
  351. package/dist/presets/ios.d.ts +2 -0
  352. package/dist/presets/ios.js +13 -0
  353. package/dist/presets/pinecone.d.ts +8 -0
  354. package/dist/presets/pinecone.js +42 -0
  355. package/dist/presets/registry.js +60 -0
  356. package/dist/presets/sendgrid.d.ts +6 -0
  357. package/dist/presets/sendgrid.js +20 -0
  358. package/dist/presets/sentry.d.ts +11 -0
  359. package/dist/presets/sentry.js +48 -0
  360. package/dist/presets/square.d.ts +10 -0
  361. package/dist/presets/square.js +33 -0
  362. package/dist/recker.d.ts +3 -0
  363. package/dist/recker.js +4 -0
  364. package/dist/scrape/document.d.ts +5 -4
  365. package/dist/scrape/document.js +89 -76
  366. package/dist/scrape/element.d.ts +10 -8
  367. package/dist/scrape/element.js +295 -81
  368. package/dist/scrape/extractors.d.ts +11 -11
  369. package/dist/scrape/extractors.js +145 -113
  370. package/dist/scrape/index.d.ts +2 -0
  371. package/dist/scrape/index.js +1 -0
  372. package/dist/scrape/parser/back.d.ts +1 -0
  373. package/dist/scrape/parser/back.js +3 -0
  374. package/dist/scrape/parser/index.d.ts +20 -0
  375. package/dist/scrape/parser/index.js +19 -0
  376. package/dist/scrape/parser/matcher.d.ts +30 -0
  377. package/dist/scrape/parser/matcher.js +99 -0
  378. package/dist/scrape/parser/nodes/comment.d.ts +12 -0
  379. package/dist/scrape/parser/nodes/comment.js +21 -0
  380. package/dist/scrape/parser/nodes/html.d.ts +110 -0
  381. package/dist/scrape/parser/nodes/html.js +978 -0
  382. package/dist/scrape/parser/nodes/node.d.ts +18 -0
  383. package/dist/scrape/parser/nodes/node.js +31 -0
  384. package/dist/scrape/parser/nodes/text.d.ts +14 -0
  385. package/dist/scrape/parser/nodes/text.js +30 -0
  386. package/dist/scrape/parser/nodes/type.d.ts +6 -0
  387. package/dist/scrape/parser/nodes/type.js +7 -0
  388. package/dist/scrape/parser/parse.d.ts +1 -0
  389. package/dist/scrape/parser/parse.js +1 -0
  390. package/dist/scrape/parser/valid.d.ts +2 -0
  391. package/dist/scrape/parser/valid.js +5 -0
  392. package/dist/scrape/parser/void-tag.d.ts +7 -0
  393. package/dist/scrape/parser/void-tag.js +43 -0
  394. package/dist/scrape/spider.d.ts +19 -0
  395. package/dist/scrape/spider.js +28 -3
  396. package/dist/scrape/types.d.ts +7 -0
  397. package/dist/seo/analyzer.d.ts +15 -5
  398. package/dist/seo/analyzer.js +636 -175
  399. package/dist/seo/formatter.d.ts +16 -0
  400. package/dist/seo/formatter.js +228 -0
  401. package/dist/seo/index.d.ts +2 -0
  402. package/dist/seo/index.js +1 -0
  403. package/dist/seo/keywords.d.ts +16 -0
  404. package/dist/seo/keywords.js +55 -0
  405. package/dist/seo/rules/accessibility.js +96 -57
  406. package/dist/seo/rules/ai-search.js +44 -31
  407. package/dist/seo/rules/analytics.d.ts +2 -0
  408. package/dist/seo/rules/analytics.js +306 -0
  409. package/dist/seo/rules/best-practices.js +21 -14
  410. package/dist/seo/rules/canonical.js +53 -32
  411. package/dist/seo/rules/content.js +317 -31
  412. package/dist/seo/rules/crawl.js +55 -40
  413. package/dist/seo/rules/cwv.js +21 -15
  414. package/dist/seo/rules/ecommerce.js +82 -22
  415. package/dist/seo/rules/i18n.js +75 -36
  416. package/dist/seo/rules/images.js +109 -30
  417. package/dist/seo/rules/index.js +2 -0
  418. package/dist/seo/rules/internal-linking.js +58 -39
  419. package/dist/seo/rules/links.js +79 -52
  420. package/dist/seo/rules/local.js +49 -25
  421. package/dist/seo/rules/meta.js +339 -81
  422. package/dist/seo/rules/mobile.js +112 -2
  423. package/dist/seo/rules/performance.js +434 -66
  424. package/dist/seo/rules/pwa.js +36 -39
  425. package/dist/seo/rules/readability.js +31 -22
  426. package/dist/seo/rules/redirects.js +21 -15
  427. package/dist/seo/rules/resources.js +59 -42
  428. package/dist/seo/rules/schema.js +333 -8
  429. package/dist/seo/rules/security.js +142 -80
  430. package/dist/seo/rules/social.js +277 -47
  431. package/dist/seo/rules/structural.js +87 -19
  432. package/dist/seo/rules/technical-advanced.js +30 -24
  433. package/dist/seo/rules/technical.js +243 -42
  434. package/dist/seo/rules/types.d.ts +53 -1
  435. package/dist/seo/seo-spider.d.ts +22 -0
  436. package/dist/seo/seo-spider.js +77 -13
  437. package/dist/seo/types.d.ts +8 -1
  438. package/dist/seo/validators/llms-txt.js +19 -0
  439. package/dist/seo/validators/rss.d.ts +11 -0
  440. package/dist/seo/validators/rss.js +93 -0
  441. package/dist/seo/validators/sitemap.js +36 -26
  442. package/dist/transport/curl.d.ts +4 -0
  443. package/dist/transport/curl.js +101 -0
  444. package/dist/transport/udp.js +0 -1
  445. package/dist/transport/undici.js +1 -2
  446. package/dist/transport/worker.d.ts +18 -0
  447. package/dist/transport/worker.js +278 -0
  448. package/dist/types/index.d.ts +4 -1
  449. package/dist/utils/binary-manager.d.ts +4 -0
  450. package/dist/utils/binary-manager.js +72 -0
  451. package/dist/utils/optional-require.d.ts +7 -8
  452. package/dist/utils/optional-require.js +2 -21
  453. package/dist/utils/upload.d.ts +6 -0
  454. package/dist/utils/upload.js +11 -0
  455. package/dist/utils/user-agent.js +2 -13
  456. package/dist/version.js +1 -1
  457. package/package.json +12 -6
  458. package/dist/browser/utils/optional-require.d.ts +0 -19
  459. package/dist/browser/utils/optional-require.js +0 -105
@@ -0,0 +1,344 @@
1
+ import colors from '../../utils/colors.js';
2
+ import { RekArgs, generateHelp } from '../parser/index.js';
3
+ import { promises as fs } from 'node:fs';
4
+ import pathMod from 'node:path';
5
+ const ftpSchema = {
6
+ name: 'ftp',
7
+ description: 'FTP client operations',
8
+ params: {
9
+ user: { type: 'string', default: 'anonymous', description: 'Username' },
10
+ pass: { type: 'string', default: 'anonymous@', description: 'Password' },
11
+ port: { type: 'number', default: 21, description: 'Port' },
12
+ },
13
+ flags: {
14
+ secure: { description: 'Use FTPS (explicit TLS)', default: false },
15
+ implicit: { description: 'Use implicit FTPS (port 990)', default: false }
16
+ },
17
+ examples: [
18
+ { cmd: 'rek ftp ls ftp.example.com', desc: 'List files' },
19
+ { cmd: 'rek ftp get ftp.example.com /file.txt', desc: 'Download file' }
20
+ ]
21
+ };
22
+ const telnetSchema = {
23
+ name: 'telnet',
24
+ description: 'Connect to a Telnet server',
25
+ params: {
26
+ timeout: { type: 'number', default: 30000, description: 'Timeout in ms' }
27
+ },
28
+ examples: [
29
+ { cmd: 'rek telnet towel.blinkenlights.nl', desc: 'Star Wars' },
30
+ { cmd: 'rek telnet localhost 23', desc: 'Local server' }
31
+ ]
32
+ };
33
+ const graphqlSchema = {
34
+ name: 'graphql',
35
+ description: 'Execute GraphQL queries and mutations',
36
+ params: {
37
+ query: { type: 'string', description: 'Inline query' },
38
+ file: { type: 'string', description: 'Query file path' },
39
+ variables: { type: 'json', description: 'Variables JSON' },
40
+ 'var-file': { type: 'string', description: 'Variables file path' }
41
+ },
42
+ examples: [
43
+ { cmd: 'rek graphql api.com/graphql query="{ me { name } }"', desc: 'Simple query' }
44
+ ]
45
+ };
46
+ const jsonRpcSchema = {
47
+ name: 'jsonrpc',
48
+ description: 'Make JSON-RPC 2.0 calls',
49
+ params: {
50
+ method: { type: 'string', description: 'RPC Method' },
51
+ params: { type: 'json', description: 'RPC Params (array or object)' }
52
+ },
53
+ examples: [
54
+ { cmd: 'rek jsonrpc api.com/rpc method=sum params=[1,2]', desc: 'Call method' }
55
+ ]
56
+ };
57
+ const soapSchema = {
58
+ name: 'soap',
59
+ description: 'Make a SOAP request',
60
+ params: {
61
+ namespace: { type: 'string', description: 'SOAP namespace' },
62
+ envelope: { type: 'string', default: '1.1', description: 'SOAP envelope version' },
63
+ },
64
+ examples: [
65
+ { cmd: 'rek soap api.com/ws GetWeather city="New York"', desc: 'SOAP Action' }
66
+ ]
67
+ };
68
+ const odataSchema = {
69
+ name: 'odata',
70
+ description: 'Query an OData service',
71
+ params: {
72
+ select: { type: 'string', description: 'Select fields' },
73
+ filter: { type: 'string', description: 'Filter expression' },
74
+ orderby: { type: 'string', description: 'Order by' },
75
+ top: { type: 'number', description: 'Limit results' },
76
+ skip: { type: 'number', description: 'Skip results' },
77
+ expand: { type: 'string', description: 'Expand navigation' },
78
+ },
79
+ examples: [
80
+ { cmd: 'rek odata api.com/odata Customers top=10', desc: 'Query' }
81
+ ]
82
+ };
83
+ export function registerProtocolCommands(program) {
84
+ const ftpCmd = program.command('ftp')
85
+ .description('FTP client operations')
86
+ .addHelpText('after', generateHelp(ftpSchema));
87
+ ftpCmd.command('ls')
88
+ .description('List files')
89
+ .argument('<host>', 'FTP Host')
90
+ .argument('[args...]', 'Path and options')
91
+ .action(async (host, rawArgs) => {
92
+ const { data, args } = RekArgs.parse(rawArgs, ftpSchema);
93
+ const path = args[0] || '/';
94
+ const secure = data.implicit ? 'implicit' : !!data.secure;
95
+ const { createFTP } = await import('../../protocols/ftp.js');
96
+ const client = createFTP({ host, port: data.port, user: data.user, password: data.pass, secure });
97
+ console.log(colors.gray(`Connecting to ${host}...`));
98
+ try {
99
+ const connectResult = await client.connect();
100
+ if (!connectResult.success)
101
+ throw new Error(connectResult.message);
102
+ console.log(colors.green('Connected'));
103
+ const result = await client.list(path);
104
+ if (!result.success || !result.data)
105
+ throw new Error(result.message);
106
+ console.log(`\n${colors.bold(`Contents of ${path}:`)}\n`);
107
+ for (const item of result.data) {
108
+ const typeChar = item.type === 'directory' ? 'd' : item.type === 'link' ? 'l' : '-';
109
+ const nameColor = item.type === 'directory' ? colors.blue : colors.white;
110
+ console.log(`${typeChar}${item.permissions || 'rwxr-xr-x'} ${item.size.toString().padStart(10)} ${item.rawModifiedAt || ''} ${nameColor(item.name)}`);
111
+ }
112
+ await client.close();
113
+ }
114
+ catch (err) {
115
+ console.error(colors.red(`FTP Error: ${err.message}`));
116
+ process.exit(1);
117
+ }
118
+ });
119
+ ftpCmd.command('get')
120
+ .description('Download file')
121
+ .argument('<host>', 'FTP Host')
122
+ .argument('<remote>', 'Remote path')
123
+ .argument('[args...]', 'Local path and options')
124
+ .action(async (host, remote, rawArgs) => {
125
+ const { data, args } = RekArgs.parse(rawArgs, ftpSchema);
126
+ const local = args[0] || pathMod.basename(remote);
127
+ const secure = data.implicit ? 'implicit' : !!data.secure;
128
+ const { createFTP } = await import('../../protocols/ftp.js');
129
+ const client = createFTP({ host, port: data.port, user: data.user, password: data.pass, secure });
130
+ console.log(colors.gray(`Connecting to ${host}...`));
131
+ try {
132
+ if (!(await client.connect()).success)
133
+ throw new Error('Connection failed');
134
+ console.log(colors.gray(`Downloading ${remote} -> ${local}...`));
135
+ let lastProgress = 0;
136
+ client.progress((p) => {
137
+ if (p.bytesOverall - lastProgress > 100000) {
138
+ process.stdout.write(`\r ${colors.cyan((p.bytesOverall / 1024 / 1024).toFixed(2) + ' MB')} downloaded...`);
139
+ lastProgress = p.bytesOverall;
140
+ }
141
+ });
142
+ const result = await client.download(remote, local);
143
+ console.log('');
144
+ if (!result.success)
145
+ throw new Error(result.message);
146
+ console.log(colors.green(`✔ Downloaded to ${local}`));
147
+ await client.close();
148
+ }
149
+ catch (err) {
150
+ console.error(colors.red(`FTP Error: ${err.message}`));
151
+ process.exit(1);
152
+ }
153
+ });
154
+ ftpCmd.command('put')
155
+ .description('Upload file')
156
+ .argument('<host>', 'FTP Host')
157
+ .argument('<local>', 'Local path')
158
+ .argument('[args...]', 'Remote path and options')
159
+ .action(async (host, local, rawArgs) => {
160
+ const { data, args } = RekArgs.parse(rawArgs, ftpSchema);
161
+ const remote = args[0] || '/' + pathMod.basename(local);
162
+ const secure = data.implicit ? 'implicit' : !!data.secure;
163
+ const { createFTP } = await import('../../protocols/ftp.js');
164
+ const client = createFTP({ host, port: data.port, user: data.user, password: data.pass, secure });
165
+ console.log(colors.gray(`Connecting to ${host}...`));
166
+ try {
167
+ if (!(await client.connect()).success)
168
+ throw new Error('Connection failed');
169
+ console.log(colors.gray(`Uploading ${local} -> ${remote}...`));
170
+ let lastProgress = 0;
171
+ client.progress((p) => {
172
+ if (p.bytesOverall - lastProgress > 100000) {
173
+ process.stdout.write(`\r ${colors.cyan((p.bytesOverall / 1024 / 1024).toFixed(2) + ' MB')} uploaded...`);
174
+ lastProgress = p.bytesOverall;
175
+ }
176
+ });
177
+ const result = await client.upload(local, remote);
178
+ console.log('');
179
+ if (!result.success)
180
+ throw new Error(result.message);
181
+ console.log(colors.green(`✔ Uploaded to ${remote}`));
182
+ await client.close();
183
+ }
184
+ catch (err) {
185
+ console.error(colors.red(`FTP Error: ${err.message}`));
186
+ process.exit(1);
187
+ }
188
+ });
189
+ program.command('telnet')
190
+ .description(telnetSchema.description)
191
+ .argument('<host>', 'Hostname')
192
+ .argument('[args...]', 'Port and options')
193
+ .addHelpText('after', generateHelp(telnetSchema))
194
+ .action(async (host, rawArgs) => {
195
+ const { data, args } = RekArgs.parse(rawArgs, telnetSchema);
196
+ let port = 23;
197
+ if (args.length > 0)
198
+ port = parseInt(String(args[0]));
199
+ const { createTelnet } = await import('../../protocols/telnet.js');
200
+ console.log(colors.gray(`Connecting to ${host}:${port}...`));
201
+ const client = createTelnet({ host, port, timeout: data.timeout });
202
+ try {
203
+ await client.connect();
204
+ console.log(colors.green(`Connected to ${host}:${port}`));
205
+ if (process.stdin.isTTY)
206
+ process.stdin.setRawMode(true);
207
+ process.stdin.resume();
208
+ process.stdin.on('data', async (d) => {
209
+ if (d[0] === 0x03) {
210
+ await client.close();
211
+ process.exit(0);
212
+ }
213
+ await client.send(d.toString());
214
+ });
215
+ client.on('data', (d) => process.stdout.write(d));
216
+ client.on('close', () => process.exit(0));
217
+ client.on('error', (e) => { console.error(colors.red(e.message)); process.exit(1); });
218
+ }
219
+ catch (err) {
220
+ console.error(colors.red(`Telnet Error: ${err.message}`));
221
+ process.exit(1);
222
+ }
223
+ });
224
+ program.command('graphql').alias('gql')
225
+ .description(graphqlSchema.description)
226
+ .argument('<url>', 'GraphQL Endpoint')
227
+ .argument('[args...]', 'Options')
228
+ .addHelpText('after', generateHelp(graphqlSchema))
229
+ .action(async (url, rawArgs) => {
230
+ const { data, headers } = RekArgs.parse(rawArgs, graphqlSchema);
231
+ let query = data.query;
232
+ let variables = data.variables || {};
233
+ if (data.file)
234
+ query = await fs.readFile(data.file, 'utf-8');
235
+ if (data['var-file'])
236
+ variables = JSON.parse(await fs.readFile(data['var-file'], 'utf-8'));
237
+ if (!query) {
238
+ console.error(colors.red('Error: Query is required via query= or file='));
239
+ process.exit(1);
240
+ }
241
+ if (!url.startsWith('http'))
242
+ url = `https://${url}`;
243
+ const { graphql } = await import('../../plugins/graphql.js');
244
+ const { createClient } = await import('../../core/client.js');
245
+ try {
246
+ const client = createClient({ baseUrl: url, headers: { 'Content-Type': 'application/json', ...headers } });
247
+ const result = await graphql(client, query, variables);
248
+ console.log(JSON.stringify(result, null, 2));
249
+ }
250
+ catch (err) {
251
+ console.error(colors.red(`GraphQL Error: ${err.message}`));
252
+ process.exit(1);
253
+ }
254
+ });
255
+ program.command('jsonrpc')
256
+ .description(jsonRpcSchema.description)
257
+ .argument('<url>', 'Endpoint')
258
+ .argument('<method>', 'Method')
259
+ .argument('[args...]', 'Params and options')
260
+ .addHelpText('after', generateHelp(jsonRpcSchema))
261
+ .action(async (url, method, rawArgs) => {
262
+ const { data, headers, args } = RekArgs.parse(rawArgs, jsonRpcSchema);
263
+ let params = data.params;
264
+ if (!params && args.length > 0) {
265
+ params = args;
266
+ }
267
+ if (!url.startsWith('http'))
268
+ url = `https://${url}`;
269
+ const { createJsonRpcClient } = await import('../../plugins/jsonrpc.js');
270
+ const { createClient } = await import('../../core/client.js');
271
+ try {
272
+ const client = createClient({ baseUrl: url, headers });
273
+ const rpc = createJsonRpcClient(client, { endpoint: url });
274
+ const result = await rpc.call(method, params);
275
+ console.log(JSON.stringify(result, null, 2));
276
+ }
277
+ catch (err) {
278
+ console.error(colors.red(`RPC Error: ${err.message}`));
279
+ process.exit(1);
280
+ }
281
+ });
282
+ program.command('soap')
283
+ .description(soapSchema.description)
284
+ .argument('<url>', 'SOAP Endpoint')
285
+ .argument('<action>', 'Action/Operation')
286
+ .argument('[args...]', 'Parameters')
287
+ .addHelpText('after', generateHelp(soapSchema))
288
+ .action(async (url, action, rawArgs) => {
289
+ const { data, headers } = RekArgs.parse(rawArgs, soapSchema);
290
+ if (!url.startsWith('http'))
291
+ url = `https://${url}`;
292
+ const { createClient } = await import('../../core/client.js');
293
+ const { createSoapClient } = await import('../../plugins/soap.js');
294
+ try {
295
+ const httpClient = createClient({ headers });
296
+ const soap = createSoapClient(httpClient, { endpoint: url, namespace: data.namespace });
297
+ const body = { ...data };
298
+ delete body.namespace;
299
+ delete body.envelope;
300
+ const result = await soap.call(action, body);
301
+ console.log(JSON.stringify(result, null, 2));
302
+ }
303
+ catch (err) {
304
+ console.error(colors.red(`SOAP Error: ${err.message}`));
305
+ process.exit(1);
306
+ }
307
+ });
308
+ program.command('odata')
309
+ .description(odataSchema.description)
310
+ .argument('<url>', 'OData Service URL')
311
+ .argument('<entity>', 'Entity Set')
312
+ .argument('[args...]', 'Query Options')
313
+ .addHelpText('after', generateHelp(odataSchema))
314
+ .action(async (url, entity, rawArgs) => {
315
+ const { data, headers } = RekArgs.parse(rawArgs, odataSchema);
316
+ if (!url.startsWith('http'))
317
+ url = `https://${url}`;
318
+ const { createClient } = await import('../../core/client.js');
319
+ const { createODataClient } = await import('../../plugins/odata.js');
320
+ try {
321
+ const httpClient = createClient({ headers });
322
+ const odata = createODataClient(httpClient, { serviceRoot: url });
323
+ let query = odata.query(entity);
324
+ if (data.select)
325
+ query = query.select(...data.select.split(','));
326
+ if (data.filter)
327
+ query = query.filter(data.filter);
328
+ if (data.orderby)
329
+ query = query.orderBy(data.orderby);
330
+ if (data.top)
331
+ query = query.top(data.top);
332
+ if (data.skip)
333
+ query = query.skip(data.skip);
334
+ if (data.expand)
335
+ query = query.expand(data.expand);
336
+ const results = await query.get();
337
+ console.log(JSON.stringify(results, null, 2));
338
+ }
339
+ catch (err) {
340
+ console.error(colors.red(`OData Error: ${err.message}`));
341
+ process.exit(1);
342
+ }
343
+ });
344
+ }
@@ -0,0 +1,2 @@
1
+ import { RekCommand as Command } from '../router.js';
2
+ export declare function registerScrapeCommand(program: Command): void;
@@ -0,0 +1,176 @@
1
+ import colors from '../../utils/colors.js';
2
+ import { RekArgs, generateHelp } from '../parser/index.js';
3
+ const schema = {
4
+ name: 'scrape',
5
+ description: 'Extract data from web pages with CSS selectors.\nFetches a web page and extracts data using CSS selectors or built-in extractors.',
6
+ params: {
7
+ select: { type: 'string', description: 'CSS selector to extract elements' },
8
+ attr: { type: 'string', description: 'Extract specific attribute (use with select)' },
9
+ },
10
+ keywords: {
11
+ links: { description: 'Extract all links with text and href' },
12
+ images: { description: 'Extract all images with src and alt' },
13
+ meta: { description: 'Extract all meta tags' },
14
+ tables: { description: 'Extract tables as structured JSON' },
15
+ scripts: { description: 'Extract all script sources' },
16
+ jsonld: { description: 'Extract JSON-LD structured data' },
17
+ },
18
+ examples: [
19
+ { cmd: 'rek scrape example.com', desc: 'Basic page info' },
20
+ { cmd: 'rek scrape example.com select="h1"', desc: 'Extract h1 text' },
21
+ { cmd: 'rek scrape example.com select="a" attr=href', desc: 'Extract link hrefs' },
22
+ { cmd: 'rek scrape example.com links', desc: 'All links' },
23
+ { cmd: 'rek scrape example.com tables', desc: 'All tables as JSON' }
24
+ ]
25
+ };
26
+ export function registerScrapeCommand(program) {
27
+ program
28
+ .command('scrape')
29
+ .alias('extract')
30
+ .description('Extract data from web pages with CSS selectors')
31
+ .argument('<url>', 'URL to scrape')
32
+ .argument('[args...]', 'Options: select=SELECTOR, attr=NAME, links, images...')
33
+ .addHelpText('after', generateHelp(schema))
34
+ .action(async (url, rawArgs) => {
35
+ const { data } = RekArgs.parse(rawArgs, schema);
36
+ const { ScrapeDocument } = await import('../../scrape/document.js');
37
+ const { Client } = await import('../../core/client.js');
38
+ if (!url.startsWith('http://') && !url.startsWith('https://')) {
39
+ url = `https://${url}`;
40
+ }
41
+ console.log(colors.gray(`Fetching ${url}...`));
42
+ try {
43
+ const client = new Client();
44
+ const response = await client.get(url);
45
+ const html = await response.text();
46
+ const doc = await ScrapeDocument.create(html, { baseUrl: url });
47
+ const hasExtraction = data.select || data.links || data.images || data.meta || data.tables || data.scripts || data.jsonld;
48
+ if (!hasExtraction) {
49
+ const title = doc.text('title') || 'N/A';
50
+ const description = doc.attr('meta[name="description"]', 'content') || 'N/A';
51
+ const h1 = doc.text('h1') || 'N/A';
52
+ const linkCount = doc.links().length;
53
+ const imageCount = doc.images().length;
54
+ console.log(`
55
+ ${colors.bold(colors.cyan('📄 Page Info'))}
56
+
57
+ ${colors.bold('Title:')} ${title}
58
+ ${colors.bold('Description:')} ${description.slice(0, 100)}${description.length > 100 ? '...' : ''}
59
+ ${colors.bold('H1:')} ${h1}
60
+ ${colors.bold('Links:')} ${linkCount}
61
+ ${colors.bold('Images:')} ${imageCount}
62
+ `);
63
+ return;
64
+ }
65
+ if (data.select) {
66
+ if (data.attr) {
67
+ const values = doc.attrs(data.select, data.attr);
68
+ console.log(`\n${colors.bold(`Found ${values.length} values for "${data.attr}" in "${data.select}"`)}\n`);
69
+ values.slice(0, 50).forEach((value, i) => {
70
+ if (value) {
71
+ console.log(`${colors.gray(`${i + 1}.`)} ${value}`);
72
+ }
73
+ });
74
+ if (values.length > 50) {
75
+ console.log(colors.gray(`\n... and ${values.length - 50} more`));
76
+ }
77
+ }
78
+ else {
79
+ const texts = doc.texts(data.select);
80
+ console.log(`\n${colors.bold(`Found ${texts.length} elements matching "${data.select}"`)}\n`);
81
+ texts.slice(0, 50).forEach((text, i) => {
82
+ const trimmed = text.trim();
83
+ if (trimmed) {
84
+ console.log(`${colors.gray(`${i + 1}.`)} ${trimmed.slice(0, 200)}`);
85
+ }
86
+ });
87
+ if (texts.length > 50) {
88
+ console.log(colors.gray(`\n... and ${texts.length - 50} more`));
89
+ }
90
+ }
91
+ return;
92
+ }
93
+ if (data.links) {
94
+ const links = doc.links();
95
+ console.log(`\n${colors.bold(`Found ${links.length} links`)}\n`);
96
+ links.slice(0, 50).forEach((link, i) => {
97
+ const text = (link.text || '').trim().slice(0, 50) || '[no text]';
98
+ console.log(`${colors.gray(`${i + 1}.`)} ${colors.cyan(text)}`);
99
+ console.log(` ${colors.gray(link.href)}`);
100
+ });
101
+ if (links.length > 50) {
102
+ console.log(colors.gray(`\n... and ${links.length - 50} more`));
103
+ }
104
+ return;
105
+ }
106
+ if (data.images) {
107
+ const images = doc.images();
108
+ console.log(`\n${colors.bold(`Found ${images.length} images`)}\n`);
109
+ images.slice(0, 30).forEach((img, i) => {
110
+ const alt = img.alt || '[no alt]';
111
+ console.log(`${colors.gray(`${i + 1}.`)} ${colors.cyan(alt.slice(0, 50))}`);
112
+ console.log(` ${colors.gray(img.src)}`);
113
+ });
114
+ if (images.length > 30) {
115
+ console.log(colors.gray(`\n... and ${images.length - 30} more`));
116
+ }
117
+ return;
118
+ }
119
+ if (data.meta) {
120
+ const meta = doc.meta();
121
+ const entries = Object.entries(meta);
122
+ console.log(`\n${colors.bold(`Found ${entries.length} meta entries`)}\n`);
123
+ entries.forEach(([name, content]) => {
124
+ if (name && content) {
125
+ const value = String(content);
126
+ console.log(`${colors.cyan(name)}: ${value.slice(0, 100)}${value.length > 100 ? '...' : ''}`);
127
+ }
128
+ });
129
+ return;
130
+ }
131
+ if (data.tables) {
132
+ const tables = doc.tables();
133
+ console.log(`\n${colors.bold(`Found ${tables.length} tables`)}\n`);
134
+ tables.slice(0, 5).forEach((table, tableIndex) => {
135
+ console.log(`${colors.bold(`Table ${tableIndex + 1}:`)} ${table.rows?.length || 0} rows`);
136
+ console.log(JSON.stringify((table.rows || []).slice(0, 10), null, 2));
137
+ if ((table.rows?.length || 0) > 10) {
138
+ console.log(colors.gray(`... and ${(table.rows?.length || 0) - 10} more rows`));
139
+ }
140
+ console.log('');
141
+ });
142
+ return;
143
+ }
144
+ if (data.scripts) {
145
+ const scripts = doc.scripts();
146
+ const external = scripts.filter(s => s.src);
147
+ const inline = scripts.filter(s => !s.src);
148
+ console.log(`\n${colors.bold(`Found ${external.length} external scripts, ${inline.length} inline`)}\n`);
149
+ if (external.length > 0) {
150
+ console.log(colors.bold('External Scripts:'));
151
+ external.slice(0, 20).forEach((script, i) => {
152
+ console.log(`${colors.gray(`${i + 1}.`)} ${script.src}`);
153
+ });
154
+ if (external.length > 20) {
155
+ console.log(colors.gray(`... and ${external.length - 20} more`));
156
+ }
157
+ }
158
+ return;
159
+ }
160
+ if (data.jsonld) {
161
+ const jsonld = doc.jsonLd();
162
+ console.log(`\n${colors.bold(`Found ${jsonld.length} JSON-LD blocks`)}\n`);
163
+ jsonld.forEach((data, i) => {
164
+ console.log(`${colors.bold(`Block ${i + 1}:`)} ${data['@type'] || 'Unknown type'}`);
165
+ console.log(JSON.stringify(data, null, 2));
166
+ console.log('');
167
+ });
168
+ return;
169
+ }
170
+ }
171
+ catch (error) {
172
+ console.error(colors.red(`Scrape failed: ${error.message}`));
173
+ process.exit(1);
174
+ }
175
+ });
176
+ }
@@ -0,0 +1,2 @@
1
+ import { RekCommand as Command } from '../router.js';
2
+ export declare function registerSecurityCommand(program: Command): void;
@@ -0,0 +1,57 @@
1
+ import colors from '../../utils/colors.js';
2
+ import { generateHelp } from '../parser/index.js';
3
+ const schema = {
4
+ name: 'security',
5
+ description: 'Grade a website\'s security headers (A+ to F).\nAnalyzes HTTP response headers for security best practices like HSTS, CSP, and more.',
6
+ examples: [
7
+ { cmd: 'rek security github.com', desc: 'Grade GitHub\'s headers' }
8
+ ]
9
+ };
10
+ export function registerSecurityCommand(program) {
11
+ program
12
+ .command('security')
13
+ .alias('headers')
14
+ .alias('grade')
15
+ .description('Grade a website\'s security headers')
16
+ .argument('<url>', 'URL to analyze')
17
+ .addHelpText('after', generateHelp(schema))
18
+ .action(async (url) => {
19
+ if (!url.startsWith('http'))
20
+ url = `https://${url}`;
21
+ const { createClient } = await import('../../core/client.js');
22
+ const { analyzeSecurityHeaders } = await import('../../utils/security-grader.js');
23
+ console.log(colors.gray(`Analyzing security headers for ${url}...`));
24
+ try {
25
+ const origin = new URL(url).origin;
26
+ const client = createClient({ baseUrl: origin });
27
+ const res = await client.get(url);
28
+ const report = analyzeSecurityHeaders(res.headers);
29
+ let gradeColor = colors.red;
30
+ if (report.grade.startsWith('A'))
31
+ gradeColor = colors.green;
32
+ if (report.grade.startsWith('B'))
33
+ gradeColor = colors.blue;
34
+ if (report.grade.startsWith('C'))
35
+ gradeColor = colors.yellow;
36
+ console.log(`
37
+ ${colors.bold(colors.cyan('🛡️ Security Headers Report'))}
38
+ Grade: ${gradeColor(colors.bold(report.grade))} (${report.score}/100)
39
+
40
+ ${colors.bold('Details:')}`);
41
+ report.details.forEach(item => {
42
+ const icon = item.status === 'pass' ? colors.green('✔') : item.status === 'warn' ? colors.yellow('⚠') : colors.red('✖');
43
+ const headerName = colors.bold(item.header);
44
+ const value = item.value ? colors.gray(`= ${item.value.length > 50 ? item.value.slice(0, 47) + '...' : item.value}`) : colors.gray('(missing)');
45
+ console.log(` ${icon} ${headerName} ${value}`);
46
+ if (item.status !== 'pass') {
47
+ console.log(` ${colors.red('→')} ${item.message}`);
48
+ }
49
+ });
50
+ console.log('');
51
+ }
52
+ catch (error) {
53
+ console.error(colors.red(`Analysis failed: ${error.message}`));
54
+ process.exit(1);
55
+ }
56
+ });
57
+ }
@@ -0,0 +1,2 @@
1
+ import { RekCommand as Command } from '../router.js';
2
+ export declare function registerSeoCommand(program: Command): void;