recker 1.0.43 → 1.0.44
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +47 -0
- package/dist/bin/recker-linux-x64 +0 -0
- package/dist/bin/recker-macos-x64 +0 -0
- package/dist/bin/recker-win-x64.exe +0 -0
- package/dist/bin/rek.cjs +85152 -100207
- package/dist/browser/ai/adaptive-timeout.d.ts +50 -0
- package/dist/browser/ai/adaptive-timeout.js +208 -0
- package/dist/browser/ai/client.d.ts +22 -0
- package/dist/browser/ai/client.js +294 -0
- package/dist/browser/ai/index.d.ts +14 -0
- package/dist/browser/ai/index.js +11 -0
- package/dist/browser/ai/providers/anthropic.d.ts +63 -0
- package/dist/browser/ai/providers/anthropic.js +370 -0
- package/dist/browser/ai/providers/base.d.ts +48 -0
- package/dist/browser/ai/providers/base.js +150 -0
- package/dist/browser/ai/providers/google.d.ts +59 -0
- package/dist/browser/ai/providers/google.js +305 -0
- package/dist/browser/ai/providers/ollama.d.ts +44 -0
- package/dist/browser/ai/providers/ollama.js +240 -0
- package/dist/browser/ai/providers/openai.d.ts +64 -0
- package/dist/browser/ai/providers/openai.js +298 -0
- package/dist/browser/ai/rate-limiter.d.ts +43 -0
- package/dist/browser/ai/rate-limiter.js +215 -0
- package/dist/browser/ai/vector/index.d.ts +2 -0
- package/dist/browser/ai/vector/index.js +2 -0
- package/dist/browser/ai/vector/similarity.d.ts +2 -0
- package/dist/browser/ai/vector/similarity.js +27 -0
- package/dist/browser/ai/vector/store.d.ts +27 -0
- package/dist/browser/ai/vector/store.js +82 -0
- package/dist/browser/browser/cache.d.ts +2 -40
- package/dist/browser/browser/cache.js +2 -199
- package/dist/browser/browser/index.d.ts +8 -0
- package/dist/browser/browser/index.js +8 -0
- package/dist/browser/browser/recker.d.ts +8 -1
- package/dist/browser/browser/recker.js +8 -2
- package/dist/browser/cache/indexed-db.d.ts +10 -0
- package/dist/browser/cache/indexed-db.js +88 -0
- package/dist/browser/cache/service-worker-cache.d.ts +18 -0
- package/dist/browser/cache/service-worker-cache.js +103 -0
- package/dist/browser/cache.d.ts +2 -40
- package/dist/browser/cache.js +2 -199
- package/dist/browser/constants/user-agents.d.ts +7 -0
- package/dist/browser/constants/user-agents.js +7 -0
- package/dist/browser/core/client.d.ts +2 -0
- package/dist/browser/core/client.js +19 -1
- package/dist/browser/index.d.ts +8 -0
- package/dist/browser/index.js +8 -0
- package/dist/browser/plugins/har-recorder.d.ts +40 -0
- package/dist/browser/plugins/har-recorder.js +120 -0
- package/dist/browser/plugins/network-simulation.d.ts +7 -0
- package/dist/browser/plugins/network-simulation.js +13 -0
- package/dist/browser/presets/android.d.ts +2 -0
- package/dist/browser/presets/android.js +16 -0
- package/dist/browser/presets/anthropic.d.ts +8 -0
- package/dist/browser/presets/anthropic.js +27 -0
- package/dist/browser/presets/aws.d.ts +19 -0
- package/dist/browser/presets/aws.js +68 -0
- package/dist/browser/presets/azure-openai.d.ts +10 -0
- package/dist/browser/presets/azure-openai.js +35 -0
- package/dist/browser/presets/azure.d.ts +41 -0
- package/dist/browser/presets/azure.js +104 -0
- package/dist/browser/presets/chaturbate.d.ts +2 -0
- package/dist/browser/presets/chaturbate.js +17 -0
- package/dist/browser/presets/cloudflare.d.ts +12 -0
- package/dist/browser/presets/cloudflare.js +39 -0
- package/dist/browser/presets/cohere.d.ts +7 -0
- package/dist/browser/presets/cohere.js +22 -0
- package/dist/browser/presets/deepseek.d.ts +7 -0
- package/dist/browser/presets/deepseek.js +22 -0
- package/dist/browser/presets/digitalocean.d.ts +5 -0
- package/dist/browser/presets/digitalocean.js +16 -0
- package/dist/browser/presets/discord.d.ts +6 -0
- package/dist/browser/presets/discord.js +17 -0
- package/dist/browser/presets/elevenlabs.d.ts +6 -0
- package/dist/browser/presets/elevenlabs.js +20 -0
- package/dist/browser/presets/enhancers.d.ts +20 -0
- package/dist/browser/presets/enhancers.js +85 -0
- package/dist/browser/presets/fireworks.d.ts +7 -0
- package/dist/browser/presets/fireworks.js +22 -0
- package/dist/browser/presets/gcp.d.ts +34 -0
- package/dist/browser/presets/gcp.js +91 -0
- package/dist/browser/presets/gemini.d.ts +7 -0
- package/dist/browser/presets/gemini.js +23 -0
- package/dist/browser/presets/github.d.ts +6 -0
- package/dist/browser/presets/github.js +17 -0
- package/dist/browser/presets/gitlab.d.ts +6 -0
- package/dist/browser/presets/gitlab.js +16 -0
- package/dist/browser/presets/groq.d.ts +7 -0
- package/dist/browser/presets/groq.js +22 -0
- package/dist/browser/presets/hubspot.d.ts +9 -0
- package/dist/browser/presets/hubspot.js +28 -0
- package/dist/browser/presets/huggingface.d.ts +7 -0
- package/dist/browser/presets/huggingface.js +23 -0
- package/dist/browser/presets/index.d.ts +47 -0
- package/dist/browser/presets/index.js +47 -0
- package/dist/browser/presets/ios.d.ts +2 -0
- package/dist/browser/presets/ios.js +13 -0
- package/dist/browser/presets/linear.d.ts +5 -0
- package/dist/browser/presets/linear.js +16 -0
- package/dist/browser/presets/mailgun.d.ts +7 -0
- package/dist/browser/presets/mailgun.js +20 -0
- package/dist/browser/presets/meta.d.ts +10 -0
- package/dist/browser/presets/meta.js +33 -0
- package/dist/browser/presets/mistral.d.ts +7 -0
- package/dist/browser/presets/mistral.js +22 -0
- package/dist/browser/presets/notion.d.ts +6 -0
- package/dist/browser/presets/notion.js +17 -0
- package/dist/browser/presets/openai.d.ts +9 -0
- package/dist/browser/presets/openai.js +30 -0
- package/dist/browser/presets/oracle.d.ts +19 -0
- package/dist/browser/presets/oracle.js +117 -0
- package/dist/browser/presets/perplexity.d.ts +7 -0
- package/dist/browser/presets/perplexity.js +22 -0
- package/dist/browser/presets/pinecone.d.ts +8 -0
- package/dist/browser/presets/pinecone.js +42 -0
- package/dist/browser/presets/registry.d.ts +23 -0
- package/dist/browser/presets/registry.js +519 -0
- package/dist/browser/presets/replicate.d.ts +7 -0
- package/dist/browser/presets/replicate.js +23 -0
- package/dist/browser/presets/sendgrid.d.ts +6 -0
- package/dist/browser/presets/sendgrid.js +20 -0
- package/dist/browser/presets/sentry.d.ts +11 -0
- package/dist/browser/presets/sentry.js +48 -0
- package/dist/browser/presets/sinch.d.ts +9 -0
- package/dist/browser/presets/sinch.js +39 -0
- package/dist/browser/presets/slack.d.ts +5 -0
- package/dist/browser/presets/slack.js +16 -0
- package/dist/browser/presets/square.d.ts +10 -0
- package/dist/browser/presets/square.js +33 -0
- package/dist/browser/presets/stripe.d.ts +7 -0
- package/dist/browser/presets/stripe.js +23 -0
- package/dist/browser/presets/supabase.d.ts +6 -0
- package/dist/browser/presets/supabase.js +18 -0
- package/dist/browser/presets/tiktok.d.ts +10 -0
- package/dist/browser/presets/tiktok.js +38 -0
- package/dist/browser/presets/together.d.ts +7 -0
- package/dist/browser/presets/together.js +22 -0
- package/dist/browser/presets/twilio.d.ts +6 -0
- package/dist/browser/presets/twilio.js +17 -0
- package/dist/browser/presets/vercel.d.ts +6 -0
- package/dist/browser/presets/vercel.js +23 -0
- package/dist/browser/presets/vultr.d.ts +5 -0
- package/dist/browser/presets/vultr.js +16 -0
- package/dist/browser/presets/xai.d.ts +8 -0
- package/dist/browser/presets/xai.js +23 -0
- package/dist/browser/presets/youtube.d.ts +5 -0
- package/dist/browser/presets/youtube.js +20 -0
- package/dist/browser/recker.d.ts +8 -1
- package/dist/browser/recker.js +8 -2
- package/dist/browser/scrape/document.d.ts +5 -4
- package/dist/browser/scrape/document.js +89 -76
- package/dist/browser/scrape/element.d.ts +10 -8
- package/dist/browser/scrape/element.js +295 -81
- package/dist/browser/scrape/extractors.d.ts +11 -11
- package/dist/browser/scrape/extractors.js +145 -113
- package/dist/browser/scrape/parser/back.d.ts +1 -0
- package/dist/browser/scrape/parser/back.js +3 -0
- package/dist/browser/scrape/parser/index.d.ts +20 -0
- package/dist/browser/scrape/parser/index.js +19 -0
- package/dist/browser/scrape/parser/matcher.d.ts +30 -0
- package/dist/browser/scrape/parser/matcher.js +99 -0
- package/dist/browser/scrape/parser/nodes/comment.d.ts +12 -0
- package/dist/browser/scrape/parser/nodes/comment.js +21 -0
- package/dist/browser/scrape/parser/nodes/html.d.ts +110 -0
- package/dist/browser/scrape/parser/nodes/html.js +978 -0
- package/dist/browser/scrape/parser/nodes/node.d.ts +18 -0
- package/dist/browser/scrape/parser/nodes/node.js +31 -0
- package/dist/browser/scrape/parser/nodes/text.d.ts +14 -0
- package/dist/browser/scrape/parser/nodes/text.js +30 -0
- package/dist/browser/scrape/parser/nodes/type.d.ts +6 -0
- package/dist/browser/scrape/parser/nodes/type.js +7 -0
- package/dist/browser/scrape/parser/parse.d.ts +1 -0
- package/dist/browser/scrape/parser/parse.js +1 -0
- package/dist/browser/scrape/parser/valid.d.ts +2 -0
- package/dist/browser/scrape/parser/valid.js +5 -0
- package/dist/browser/scrape/parser/void-tag.d.ts +7 -0
- package/dist/browser/scrape/parser/void-tag.js +43 -0
- package/dist/browser/scrape/types.d.ts +7 -0
- package/dist/browser/seo/analyzer.d.ts +59 -0
- package/dist/browser/seo/analyzer.js +1399 -0
- package/dist/browser/seo/keywords.d.ts +16 -0
- package/dist/browser/seo/keywords.js +55 -0
- package/dist/browser/seo/rules/accessibility.d.ts +2 -0
- package/dist/browser/seo/rules/accessibility.js +733 -0
- package/dist/browser/seo/rules/ai-search.d.ts +2 -0
- package/dist/browser/seo/rules/ai-search.js +436 -0
- package/dist/browser/seo/rules/analytics.d.ts +2 -0
- package/dist/browser/seo/rules/analytics.js +306 -0
- package/dist/browser/seo/rules/best-practices.d.ts +2 -0
- package/dist/browser/seo/rules/best-practices.js +195 -0
- package/dist/browser/seo/rules/canonical.d.ts +12 -0
- package/dist/browser/seo/rules/canonical.js +270 -0
- package/dist/browser/seo/rules/content.d.ts +2 -0
- package/dist/browser/seo/rules/content.js +522 -0
- package/dist/browser/seo/rules/crawl.d.ts +2 -0
- package/dist/browser/seo/rules/crawl.js +435 -0
- package/dist/browser/seo/rules/cwv.d.ts +2 -0
- package/dist/browser/seo/rules/cwv.js +248 -0
- package/dist/browser/seo/rules/ecommerce.d.ts +2 -0
- package/dist/browser/seo/rules/ecommerce.js +312 -0
- package/dist/browser/seo/rules/i18n.d.ts +2 -0
- package/dist/browser/seo/rules/i18n.js +288 -0
- package/dist/browser/seo/rules/images.d.ts +2 -0
- package/dist/browser/seo/rules/images.js +255 -0
- package/dist/browser/seo/rules/index.d.ts +52 -0
- package/dist/browser/seo/rules/index.js +159 -0
- package/dist/browser/seo/rules/internal-linking.d.ts +2 -0
- package/dist/browser/seo/rules/internal-linking.js +394 -0
- package/dist/browser/seo/rules/links.d.ts +2 -0
- package/dist/browser/seo/rules/links.js +498 -0
- package/dist/browser/seo/rules/local.d.ts +2 -0
- package/dist/browser/seo/rules/local.js +289 -0
- package/dist/browser/seo/rules/meta.d.ts +2 -0
- package/dist/browser/seo/rules/meta.js +805 -0
- package/dist/browser/seo/rules/mobile.d.ts +2 -0
- package/dist/browser/seo/rules/mobile.js +161 -0
- package/dist/browser/seo/rules/performance.d.ts +2 -0
- package/dist/browser/seo/rules/performance.js +738 -0
- package/dist/browser/seo/rules/pwa.d.ts +2 -0
- package/dist/browser/seo/rules/pwa.js +299 -0
- package/dist/browser/seo/rules/readability.d.ts +2 -0
- package/dist/browser/seo/rules/readability.js +264 -0
- package/dist/browser/seo/rules/redirects.d.ts +16 -0
- package/dist/browser/seo/rules/redirects.js +199 -0
- package/dist/browser/seo/rules/resources.d.ts +2 -0
- package/dist/browser/seo/rules/resources.js +390 -0
- package/dist/browser/seo/rules/schema.d.ts +2 -0
- package/dist/browser/seo/rules/schema.js +379 -0
- package/dist/browser/seo/rules/security.d.ts +2 -0
- package/dist/browser/seo/rules/security.js +877 -0
- package/dist/browser/seo/rules/social.d.ts +2 -0
- package/dist/browser/seo/rules/social.js +603 -0
- package/dist/browser/seo/rules/structural.d.ts +2 -0
- package/dist/browser/seo/rules/structural.js +223 -0
- package/dist/browser/seo/rules/technical-advanced.d.ts +10 -0
- package/dist/browser/seo/rules/technical-advanced.js +289 -0
- package/dist/browser/seo/rules/technical.d.ts +2 -0
- package/dist/browser/seo/rules/technical.js +480 -0
- package/dist/browser/seo/rules/thresholds.d.ts +196 -0
- package/dist/browser/seo/rules/thresholds.js +118 -0
- package/dist/browser/seo/rules/types.d.ts +498 -0
- package/dist/browser/seo/rules/types.js +11 -0
- package/dist/browser/seo/types.d.ts +211 -0
- package/dist/browser/seo/types.js +1 -0
- package/dist/browser/transport/curl.d.ts +4 -0
- package/dist/browser/transport/curl.js +101 -0
- package/dist/browser/transport/undici.js +1 -2
- package/dist/browser/transport/worker.d.ts +18 -0
- package/dist/browser/transport/worker.js +278 -0
- package/dist/browser/types/index.d.ts +4 -1
- package/dist/browser/utils/binary-manager.d.ts +4 -0
- package/dist/browser/utils/binary-manager.js +72 -0
- package/dist/browser/utils/user-agent.js +2 -13
- package/dist/cache/indexed-db.d.ts +10 -0
- package/dist/cache/indexed-db.js +88 -0
- package/dist/cache/service-worker-cache.d.ts +18 -0
- package/dist/cache/service-worker-cache.js +103 -0
- package/dist/cli/commands/ai.d.ts +2 -0
- package/dist/cli/commands/ai.js +162 -0
- package/dist/cli/commands/bench.d.ts +2 -0
- package/dist/cli/commands/bench.js +51 -0
- package/dist/cli/commands/dns.d.ts +2 -0
- package/dist/cli/commands/dns.js +295 -0
- package/dist/cli/commands/har.d.ts +2 -0
- package/dist/cli/commands/har.js +171 -0
- package/dist/cli/commands/hls.d.ts +2 -0
- package/dist/cli/commands/hls.js +192 -0
- package/dist/cli/commands/network.d.ts +2 -0
- package/dist/cli/commands/network.js +288 -0
- package/dist/cli/commands/protocols.d.ts +2 -0
- package/dist/cli/commands/protocols.js +344 -0
- package/dist/cli/commands/scrape.d.ts +2 -0
- package/dist/cli/commands/scrape.js +176 -0
- package/dist/cli/commands/security.d.ts +2 -0
- package/dist/cli/commands/security.js +57 -0
- package/dist/cli/commands/seo.d.ts +2 -0
- package/dist/cli/commands/seo.js +125 -0
- package/dist/cli/commands/serve.d.ts +2 -0
- package/dist/cli/commands/serve.js +531 -0
- package/dist/cli/commands/spider.d.ts +3 -0
- package/dist/cli/commands/spider.js +456 -0
- package/dist/cli/commands/utils.d.ts +2 -0
- package/dist/cli/commands/utils.js +176 -0
- package/dist/cli/commands/vector.d.ts +2 -0
- package/dist/cli/commands/vector.js +158 -0
- package/dist/cli/handler.d.ts +2 -2
- package/dist/cli/handler.js +6 -6
- package/dist/cli/helpers.d.ts +7 -0
- package/dist/cli/helpers.js +128 -0
- package/dist/cli/index.js +96 -5228
- package/dist/cli/parser/help.d.ts +2 -0
- package/dist/cli/parser/help.js +52 -0
- package/dist/cli/parser/index.d.ts +3 -0
- package/dist/cli/parser/index.js +3 -0
- package/dist/cli/parser/parser.d.ts +4 -0
- package/dist/cli/parser/parser.js +146 -0
- package/dist/cli/parser/types.d.ts +41 -0
- package/dist/cli/parser/types.js +1 -0
- package/dist/cli/presets.d.ts +1 -1
- package/dist/cli/presets.js +1 -1
- package/dist/cli/router.d.ts +36 -0
- package/dist/cli/router.js +195 -0
- package/dist/cli/tui/ai-chat.js +1 -1
- package/dist/cli/tui/commands/context.d.ts +9 -0
- package/dist/cli/tui/commands/context.js +1 -0
- package/dist/cli/tui/commands/dns.d.ts +10 -0
- package/dist/cli/tui/commands/dns.js +461 -0
- package/dist/cli/tui/commands/hls.d.ts +2 -0
- package/dist/cli/tui/commands/hls.js +162 -0
- package/dist/cli/tui/commands/ip.d.ts +2 -0
- package/dist/cli/tui/commands/ip.js +45 -0
- package/dist/cli/tui/commands/network.d.ts +3 -0
- package/dist/cli/tui/commands/network.js +81 -0
- package/dist/cli/tui/commands/protocols.d.ts +6 -0
- package/dist/cli/tui/commands/protocols.js +531 -0
- package/dist/cli/tui/commands/security.d.ts +2 -0
- package/dist/cli/tui/commands/security.js +48 -0
- package/dist/cli/tui/commands/seo.d.ts +2 -0
- package/dist/cli/tui/commands/seo.js +74 -0
- package/dist/cli/tui/context.d.ts +12 -0
- package/dist/cli/tui/context.js +1 -0
- package/dist/cli/tui/shell.d.ts +11 -20
- package/dist/cli/tui/shell.js +216 -1873
- package/dist/constants/user-agents.d.ts +7 -0
- package/dist/constants/user-agents.js +7 -0
- package/dist/core/client.d.ts +2 -0
- package/dist/core/client.js +19 -1
- package/dist/index.d.ts +1 -0
- package/dist/index.js +1 -0
- package/dist/mcp/cli.js +2 -3
- package/dist/mcp/data/embeddings.json +1 -1
- package/dist/mcp/tools/network.js +298 -158
- package/dist/plugins/har-player.d.ts +23 -0
- package/dist/plugins/har-player.js +49 -0
- package/dist/plugins/har-recorder.d.ts +37 -3
- package/dist/plugins/har-recorder.js +116 -63
- package/dist/plugins/network-simulation.d.ts +7 -0
- package/dist/plugins/network-simulation.js +13 -0
- package/dist/presets/android.d.ts +2 -0
- package/dist/presets/android.js +16 -0
- package/dist/presets/chaturbate.d.ts +2 -0
- package/dist/presets/chaturbate.js +17 -0
- package/dist/presets/elevenlabs.d.ts +6 -0
- package/dist/presets/elevenlabs.js +20 -0
- package/dist/presets/enhancers.d.ts +20 -0
- package/dist/presets/enhancers.js +85 -0
- package/dist/presets/hubspot.d.ts +9 -0
- package/dist/presets/hubspot.js +28 -0
- package/dist/presets/index.d.ts +10 -0
- package/dist/presets/index.js +10 -0
- package/dist/presets/ios.d.ts +2 -0
- package/dist/presets/ios.js +13 -0
- package/dist/presets/pinecone.d.ts +8 -0
- package/dist/presets/pinecone.js +42 -0
- package/dist/presets/registry.js +60 -0
- package/dist/presets/sendgrid.d.ts +6 -0
- package/dist/presets/sendgrid.js +20 -0
- package/dist/presets/sentry.d.ts +11 -0
- package/dist/presets/sentry.js +48 -0
- package/dist/presets/square.d.ts +10 -0
- package/dist/presets/square.js +33 -0
- package/dist/recker.d.ts +3 -0
- package/dist/recker.js +4 -0
- package/dist/scrape/document.d.ts +5 -4
- package/dist/scrape/document.js +89 -76
- package/dist/scrape/element.d.ts +10 -8
- package/dist/scrape/element.js +295 -81
- package/dist/scrape/extractors.d.ts +11 -11
- package/dist/scrape/extractors.js +145 -113
- package/dist/scrape/index.d.ts +2 -0
- package/dist/scrape/index.js +1 -0
- package/dist/scrape/parser/back.d.ts +1 -0
- package/dist/scrape/parser/back.js +3 -0
- package/dist/scrape/parser/index.d.ts +20 -0
- package/dist/scrape/parser/index.js +19 -0
- package/dist/scrape/parser/matcher.d.ts +30 -0
- package/dist/scrape/parser/matcher.js +99 -0
- package/dist/scrape/parser/nodes/comment.d.ts +12 -0
- package/dist/scrape/parser/nodes/comment.js +21 -0
- package/dist/scrape/parser/nodes/html.d.ts +110 -0
- package/dist/scrape/parser/nodes/html.js +978 -0
- package/dist/scrape/parser/nodes/node.d.ts +18 -0
- package/dist/scrape/parser/nodes/node.js +31 -0
- package/dist/scrape/parser/nodes/text.d.ts +14 -0
- package/dist/scrape/parser/nodes/text.js +30 -0
- package/dist/scrape/parser/nodes/type.d.ts +6 -0
- package/dist/scrape/parser/nodes/type.js +7 -0
- package/dist/scrape/parser/parse.d.ts +1 -0
- package/dist/scrape/parser/parse.js +1 -0
- package/dist/scrape/parser/valid.d.ts +2 -0
- package/dist/scrape/parser/valid.js +5 -0
- package/dist/scrape/parser/void-tag.d.ts +7 -0
- package/dist/scrape/parser/void-tag.js +43 -0
- package/dist/scrape/spider.d.ts +19 -0
- package/dist/scrape/spider.js +28 -3
- package/dist/scrape/types.d.ts +7 -0
- package/dist/seo/analyzer.d.ts +15 -5
- package/dist/seo/analyzer.js +636 -175
- package/dist/seo/formatter.d.ts +16 -0
- package/dist/seo/formatter.js +228 -0
- package/dist/seo/index.d.ts +2 -0
- package/dist/seo/index.js +1 -0
- package/dist/seo/keywords.d.ts +16 -0
- package/dist/seo/keywords.js +55 -0
- package/dist/seo/rules/accessibility.js +96 -57
- package/dist/seo/rules/ai-search.js +44 -31
- package/dist/seo/rules/analytics.d.ts +2 -0
- package/dist/seo/rules/analytics.js +306 -0
- package/dist/seo/rules/best-practices.js +21 -14
- package/dist/seo/rules/canonical.js +53 -32
- package/dist/seo/rules/content.js +317 -31
- package/dist/seo/rules/crawl.js +55 -40
- package/dist/seo/rules/cwv.js +21 -15
- package/dist/seo/rules/ecommerce.js +82 -22
- package/dist/seo/rules/i18n.js +75 -36
- package/dist/seo/rules/images.js +109 -30
- package/dist/seo/rules/index.js +2 -0
- package/dist/seo/rules/internal-linking.js +58 -39
- package/dist/seo/rules/links.js +79 -52
- package/dist/seo/rules/local.js +49 -25
- package/dist/seo/rules/meta.js +339 -81
- package/dist/seo/rules/mobile.js +112 -2
- package/dist/seo/rules/performance.js +434 -66
- package/dist/seo/rules/pwa.js +36 -39
- package/dist/seo/rules/readability.js +31 -22
- package/dist/seo/rules/redirects.js +21 -15
- package/dist/seo/rules/resources.js +59 -42
- package/dist/seo/rules/schema.js +333 -8
- package/dist/seo/rules/security.js +142 -80
- package/dist/seo/rules/social.js +277 -47
- package/dist/seo/rules/structural.js +87 -19
- package/dist/seo/rules/technical-advanced.js +30 -24
- package/dist/seo/rules/technical.js +243 -42
- package/dist/seo/rules/types.d.ts +53 -1
- package/dist/seo/seo-spider.d.ts +22 -0
- package/dist/seo/seo-spider.js +77 -13
- package/dist/seo/types.d.ts +8 -1
- package/dist/seo/validators/llms-txt.js +19 -0
- package/dist/seo/validators/rss.d.ts +11 -0
- package/dist/seo/validators/rss.js +93 -0
- package/dist/seo/validators/sitemap.js +36 -26
- package/dist/transport/curl.d.ts +4 -0
- package/dist/transport/curl.js +101 -0
- package/dist/transport/udp.js +0 -1
- package/dist/transport/undici.js +1 -2
- package/dist/transport/worker.d.ts +18 -0
- package/dist/transport/worker.js +278 -0
- package/dist/types/index.d.ts +4 -1
- package/dist/utils/binary-manager.d.ts +4 -0
- package/dist/utils/binary-manager.js +72 -0
- package/dist/utils/optional-require.d.ts +7 -8
- package/dist/utils/optional-require.js +2 -21
- package/dist/utils/upload.d.ts +6 -0
- package/dist/utils/upload.js +11 -0
- package/dist/utils/user-agent.js +2 -13
- package/dist/version.js +1 -1
- package/package.json +12 -6
- package/dist/browser/utils/optional-require.d.ts +0 -19
- package/dist/browser/utils/optional-require.js +0 -105
|
@@ -0,0 +1,344 @@
|
|
|
1
|
+
import colors from '../../utils/colors.js';
|
|
2
|
+
import { RekArgs, generateHelp } from '../parser/index.js';
|
|
3
|
+
import { promises as fs } from 'node:fs';
|
|
4
|
+
import pathMod from 'node:path';
|
|
5
|
+
const ftpSchema = {
|
|
6
|
+
name: 'ftp',
|
|
7
|
+
description: 'FTP client operations',
|
|
8
|
+
params: {
|
|
9
|
+
user: { type: 'string', default: 'anonymous', description: 'Username' },
|
|
10
|
+
pass: { type: 'string', default: 'anonymous@', description: 'Password' },
|
|
11
|
+
port: { type: 'number', default: 21, description: 'Port' },
|
|
12
|
+
},
|
|
13
|
+
flags: {
|
|
14
|
+
secure: { description: 'Use FTPS (explicit TLS)', default: false },
|
|
15
|
+
implicit: { description: 'Use implicit FTPS (port 990)', default: false }
|
|
16
|
+
},
|
|
17
|
+
examples: [
|
|
18
|
+
{ cmd: 'rek ftp ls ftp.example.com', desc: 'List files' },
|
|
19
|
+
{ cmd: 'rek ftp get ftp.example.com /file.txt', desc: 'Download file' }
|
|
20
|
+
]
|
|
21
|
+
};
|
|
22
|
+
const telnetSchema = {
|
|
23
|
+
name: 'telnet',
|
|
24
|
+
description: 'Connect to a Telnet server',
|
|
25
|
+
params: {
|
|
26
|
+
timeout: { type: 'number', default: 30000, description: 'Timeout in ms' }
|
|
27
|
+
},
|
|
28
|
+
examples: [
|
|
29
|
+
{ cmd: 'rek telnet towel.blinkenlights.nl', desc: 'Star Wars' },
|
|
30
|
+
{ cmd: 'rek telnet localhost 23', desc: 'Local server' }
|
|
31
|
+
]
|
|
32
|
+
};
|
|
33
|
+
const graphqlSchema = {
|
|
34
|
+
name: 'graphql',
|
|
35
|
+
description: 'Execute GraphQL queries and mutations',
|
|
36
|
+
params: {
|
|
37
|
+
query: { type: 'string', description: 'Inline query' },
|
|
38
|
+
file: { type: 'string', description: 'Query file path' },
|
|
39
|
+
variables: { type: 'json', description: 'Variables JSON' },
|
|
40
|
+
'var-file': { type: 'string', description: 'Variables file path' }
|
|
41
|
+
},
|
|
42
|
+
examples: [
|
|
43
|
+
{ cmd: 'rek graphql api.com/graphql query="{ me { name } }"', desc: 'Simple query' }
|
|
44
|
+
]
|
|
45
|
+
};
|
|
46
|
+
const jsonRpcSchema = {
|
|
47
|
+
name: 'jsonrpc',
|
|
48
|
+
description: 'Make JSON-RPC 2.0 calls',
|
|
49
|
+
params: {
|
|
50
|
+
method: { type: 'string', description: 'RPC Method' },
|
|
51
|
+
params: { type: 'json', description: 'RPC Params (array or object)' }
|
|
52
|
+
},
|
|
53
|
+
examples: [
|
|
54
|
+
{ cmd: 'rek jsonrpc api.com/rpc method=sum params=[1,2]', desc: 'Call method' }
|
|
55
|
+
]
|
|
56
|
+
};
|
|
57
|
+
const soapSchema = {
|
|
58
|
+
name: 'soap',
|
|
59
|
+
description: 'Make a SOAP request',
|
|
60
|
+
params: {
|
|
61
|
+
namespace: { type: 'string', description: 'SOAP namespace' },
|
|
62
|
+
envelope: { type: 'string', default: '1.1', description: 'SOAP envelope version' },
|
|
63
|
+
},
|
|
64
|
+
examples: [
|
|
65
|
+
{ cmd: 'rek soap api.com/ws GetWeather city="New York"', desc: 'SOAP Action' }
|
|
66
|
+
]
|
|
67
|
+
};
|
|
68
|
+
const odataSchema = {
|
|
69
|
+
name: 'odata',
|
|
70
|
+
description: 'Query an OData service',
|
|
71
|
+
params: {
|
|
72
|
+
select: { type: 'string', description: 'Select fields' },
|
|
73
|
+
filter: { type: 'string', description: 'Filter expression' },
|
|
74
|
+
orderby: { type: 'string', description: 'Order by' },
|
|
75
|
+
top: { type: 'number', description: 'Limit results' },
|
|
76
|
+
skip: { type: 'number', description: 'Skip results' },
|
|
77
|
+
expand: { type: 'string', description: 'Expand navigation' },
|
|
78
|
+
},
|
|
79
|
+
examples: [
|
|
80
|
+
{ cmd: 'rek odata api.com/odata Customers top=10', desc: 'Query' }
|
|
81
|
+
]
|
|
82
|
+
};
|
|
83
|
+
export function registerProtocolCommands(program) {
|
|
84
|
+
const ftpCmd = program.command('ftp')
|
|
85
|
+
.description('FTP client operations')
|
|
86
|
+
.addHelpText('after', generateHelp(ftpSchema));
|
|
87
|
+
ftpCmd.command('ls')
|
|
88
|
+
.description('List files')
|
|
89
|
+
.argument('<host>', 'FTP Host')
|
|
90
|
+
.argument('[args...]', 'Path and options')
|
|
91
|
+
.action(async (host, rawArgs) => {
|
|
92
|
+
const { data, args } = RekArgs.parse(rawArgs, ftpSchema);
|
|
93
|
+
const path = args[0] || '/';
|
|
94
|
+
const secure = data.implicit ? 'implicit' : !!data.secure;
|
|
95
|
+
const { createFTP } = await import('../../protocols/ftp.js');
|
|
96
|
+
const client = createFTP({ host, port: data.port, user: data.user, password: data.pass, secure });
|
|
97
|
+
console.log(colors.gray(`Connecting to ${host}...`));
|
|
98
|
+
try {
|
|
99
|
+
const connectResult = await client.connect();
|
|
100
|
+
if (!connectResult.success)
|
|
101
|
+
throw new Error(connectResult.message);
|
|
102
|
+
console.log(colors.green('Connected'));
|
|
103
|
+
const result = await client.list(path);
|
|
104
|
+
if (!result.success || !result.data)
|
|
105
|
+
throw new Error(result.message);
|
|
106
|
+
console.log(`\n${colors.bold(`Contents of ${path}:`)}\n`);
|
|
107
|
+
for (const item of result.data) {
|
|
108
|
+
const typeChar = item.type === 'directory' ? 'd' : item.type === 'link' ? 'l' : '-';
|
|
109
|
+
const nameColor = item.type === 'directory' ? colors.blue : colors.white;
|
|
110
|
+
console.log(`${typeChar}${item.permissions || 'rwxr-xr-x'} ${item.size.toString().padStart(10)} ${item.rawModifiedAt || ''} ${nameColor(item.name)}`);
|
|
111
|
+
}
|
|
112
|
+
await client.close();
|
|
113
|
+
}
|
|
114
|
+
catch (err) {
|
|
115
|
+
console.error(colors.red(`FTP Error: ${err.message}`));
|
|
116
|
+
process.exit(1);
|
|
117
|
+
}
|
|
118
|
+
});
|
|
119
|
+
ftpCmd.command('get')
|
|
120
|
+
.description('Download file')
|
|
121
|
+
.argument('<host>', 'FTP Host')
|
|
122
|
+
.argument('<remote>', 'Remote path')
|
|
123
|
+
.argument('[args...]', 'Local path and options')
|
|
124
|
+
.action(async (host, remote, rawArgs) => {
|
|
125
|
+
const { data, args } = RekArgs.parse(rawArgs, ftpSchema);
|
|
126
|
+
const local = args[0] || pathMod.basename(remote);
|
|
127
|
+
const secure = data.implicit ? 'implicit' : !!data.secure;
|
|
128
|
+
const { createFTP } = await import('../../protocols/ftp.js');
|
|
129
|
+
const client = createFTP({ host, port: data.port, user: data.user, password: data.pass, secure });
|
|
130
|
+
console.log(colors.gray(`Connecting to ${host}...`));
|
|
131
|
+
try {
|
|
132
|
+
if (!(await client.connect()).success)
|
|
133
|
+
throw new Error('Connection failed');
|
|
134
|
+
console.log(colors.gray(`Downloading ${remote} -> ${local}...`));
|
|
135
|
+
let lastProgress = 0;
|
|
136
|
+
client.progress((p) => {
|
|
137
|
+
if (p.bytesOverall - lastProgress > 100000) {
|
|
138
|
+
process.stdout.write(`\r ${colors.cyan((p.bytesOverall / 1024 / 1024).toFixed(2) + ' MB')} downloaded...`);
|
|
139
|
+
lastProgress = p.bytesOverall;
|
|
140
|
+
}
|
|
141
|
+
});
|
|
142
|
+
const result = await client.download(remote, local);
|
|
143
|
+
console.log('');
|
|
144
|
+
if (!result.success)
|
|
145
|
+
throw new Error(result.message);
|
|
146
|
+
console.log(colors.green(`✔ Downloaded to ${local}`));
|
|
147
|
+
await client.close();
|
|
148
|
+
}
|
|
149
|
+
catch (err) {
|
|
150
|
+
console.error(colors.red(`FTP Error: ${err.message}`));
|
|
151
|
+
process.exit(1);
|
|
152
|
+
}
|
|
153
|
+
});
|
|
154
|
+
ftpCmd.command('put')
|
|
155
|
+
.description('Upload file')
|
|
156
|
+
.argument('<host>', 'FTP Host')
|
|
157
|
+
.argument('<local>', 'Local path')
|
|
158
|
+
.argument('[args...]', 'Remote path and options')
|
|
159
|
+
.action(async (host, local, rawArgs) => {
|
|
160
|
+
const { data, args } = RekArgs.parse(rawArgs, ftpSchema);
|
|
161
|
+
const remote = args[0] || '/' + pathMod.basename(local);
|
|
162
|
+
const secure = data.implicit ? 'implicit' : !!data.secure;
|
|
163
|
+
const { createFTP } = await import('../../protocols/ftp.js');
|
|
164
|
+
const client = createFTP({ host, port: data.port, user: data.user, password: data.pass, secure });
|
|
165
|
+
console.log(colors.gray(`Connecting to ${host}...`));
|
|
166
|
+
try {
|
|
167
|
+
if (!(await client.connect()).success)
|
|
168
|
+
throw new Error('Connection failed');
|
|
169
|
+
console.log(colors.gray(`Uploading ${local} -> ${remote}...`));
|
|
170
|
+
let lastProgress = 0;
|
|
171
|
+
client.progress((p) => {
|
|
172
|
+
if (p.bytesOverall - lastProgress > 100000) {
|
|
173
|
+
process.stdout.write(`\r ${colors.cyan((p.bytesOverall / 1024 / 1024).toFixed(2) + ' MB')} uploaded...`);
|
|
174
|
+
lastProgress = p.bytesOverall;
|
|
175
|
+
}
|
|
176
|
+
});
|
|
177
|
+
const result = await client.upload(local, remote);
|
|
178
|
+
console.log('');
|
|
179
|
+
if (!result.success)
|
|
180
|
+
throw new Error(result.message);
|
|
181
|
+
console.log(colors.green(`✔ Uploaded to ${remote}`));
|
|
182
|
+
await client.close();
|
|
183
|
+
}
|
|
184
|
+
catch (err) {
|
|
185
|
+
console.error(colors.red(`FTP Error: ${err.message}`));
|
|
186
|
+
process.exit(1);
|
|
187
|
+
}
|
|
188
|
+
});
|
|
189
|
+
program.command('telnet')
|
|
190
|
+
.description(telnetSchema.description)
|
|
191
|
+
.argument('<host>', 'Hostname')
|
|
192
|
+
.argument('[args...]', 'Port and options')
|
|
193
|
+
.addHelpText('after', generateHelp(telnetSchema))
|
|
194
|
+
.action(async (host, rawArgs) => {
|
|
195
|
+
const { data, args } = RekArgs.parse(rawArgs, telnetSchema);
|
|
196
|
+
let port = 23;
|
|
197
|
+
if (args.length > 0)
|
|
198
|
+
port = parseInt(String(args[0]));
|
|
199
|
+
const { createTelnet } = await import('../../protocols/telnet.js');
|
|
200
|
+
console.log(colors.gray(`Connecting to ${host}:${port}...`));
|
|
201
|
+
const client = createTelnet({ host, port, timeout: data.timeout });
|
|
202
|
+
try {
|
|
203
|
+
await client.connect();
|
|
204
|
+
console.log(colors.green(`Connected to ${host}:${port}`));
|
|
205
|
+
if (process.stdin.isTTY)
|
|
206
|
+
process.stdin.setRawMode(true);
|
|
207
|
+
process.stdin.resume();
|
|
208
|
+
process.stdin.on('data', async (d) => {
|
|
209
|
+
if (d[0] === 0x03) {
|
|
210
|
+
await client.close();
|
|
211
|
+
process.exit(0);
|
|
212
|
+
}
|
|
213
|
+
await client.send(d.toString());
|
|
214
|
+
});
|
|
215
|
+
client.on('data', (d) => process.stdout.write(d));
|
|
216
|
+
client.on('close', () => process.exit(0));
|
|
217
|
+
client.on('error', (e) => { console.error(colors.red(e.message)); process.exit(1); });
|
|
218
|
+
}
|
|
219
|
+
catch (err) {
|
|
220
|
+
console.error(colors.red(`Telnet Error: ${err.message}`));
|
|
221
|
+
process.exit(1);
|
|
222
|
+
}
|
|
223
|
+
});
|
|
224
|
+
program.command('graphql').alias('gql')
|
|
225
|
+
.description(graphqlSchema.description)
|
|
226
|
+
.argument('<url>', 'GraphQL Endpoint')
|
|
227
|
+
.argument('[args...]', 'Options')
|
|
228
|
+
.addHelpText('after', generateHelp(graphqlSchema))
|
|
229
|
+
.action(async (url, rawArgs) => {
|
|
230
|
+
const { data, headers } = RekArgs.parse(rawArgs, graphqlSchema);
|
|
231
|
+
let query = data.query;
|
|
232
|
+
let variables = data.variables || {};
|
|
233
|
+
if (data.file)
|
|
234
|
+
query = await fs.readFile(data.file, 'utf-8');
|
|
235
|
+
if (data['var-file'])
|
|
236
|
+
variables = JSON.parse(await fs.readFile(data['var-file'], 'utf-8'));
|
|
237
|
+
if (!query) {
|
|
238
|
+
console.error(colors.red('Error: Query is required via query= or file='));
|
|
239
|
+
process.exit(1);
|
|
240
|
+
}
|
|
241
|
+
if (!url.startsWith('http'))
|
|
242
|
+
url = `https://${url}`;
|
|
243
|
+
const { graphql } = await import('../../plugins/graphql.js');
|
|
244
|
+
const { createClient } = await import('../../core/client.js');
|
|
245
|
+
try {
|
|
246
|
+
const client = createClient({ baseUrl: url, headers: { 'Content-Type': 'application/json', ...headers } });
|
|
247
|
+
const result = await graphql(client, query, variables);
|
|
248
|
+
console.log(JSON.stringify(result, null, 2));
|
|
249
|
+
}
|
|
250
|
+
catch (err) {
|
|
251
|
+
console.error(colors.red(`GraphQL Error: ${err.message}`));
|
|
252
|
+
process.exit(1);
|
|
253
|
+
}
|
|
254
|
+
});
|
|
255
|
+
program.command('jsonrpc')
|
|
256
|
+
.description(jsonRpcSchema.description)
|
|
257
|
+
.argument('<url>', 'Endpoint')
|
|
258
|
+
.argument('<method>', 'Method')
|
|
259
|
+
.argument('[args...]', 'Params and options')
|
|
260
|
+
.addHelpText('after', generateHelp(jsonRpcSchema))
|
|
261
|
+
.action(async (url, method, rawArgs) => {
|
|
262
|
+
const { data, headers, args } = RekArgs.parse(rawArgs, jsonRpcSchema);
|
|
263
|
+
let params = data.params;
|
|
264
|
+
if (!params && args.length > 0) {
|
|
265
|
+
params = args;
|
|
266
|
+
}
|
|
267
|
+
if (!url.startsWith('http'))
|
|
268
|
+
url = `https://${url}`;
|
|
269
|
+
const { createJsonRpcClient } = await import('../../plugins/jsonrpc.js');
|
|
270
|
+
const { createClient } = await import('../../core/client.js');
|
|
271
|
+
try {
|
|
272
|
+
const client = createClient({ baseUrl: url, headers });
|
|
273
|
+
const rpc = createJsonRpcClient(client, { endpoint: url });
|
|
274
|
+
const result = await rpc.call(method, params);
|
|
275
|
+
console.log(JSON.stringify(result, null, 2));
|
|
276
|
+
}
|
|
277
|
+
catch (err) {
|
|
278
|
+
console.error(colors.red(`RPC Error: ${err.message}`));
|
|
279
|
+
process.exit(1);
|
|
280
|
+
}
|
|
281
|
+
});
|
|
282
|
+
program.command('soap')
|
|
283
|
+
.description(soapSchema.description)
|
|
284
|
+
.argument('<url>', 'SOAP Endpoint')
|
|
285
|
+
.argument('<action>', 'Action/Operation')
|
|
286
|
+
.argument('[args...]', 'Parameters')
|
|
287
|
+
.addHelpText('after', generateHelp(soapSchema))
|
|
288
|
+
.action(async (url, action, rawArgs) => {
|
|
289
|
+
const { data, headers } = RekArgs.parse(rawArgs, soapSchema);
|
|
290
|
+
if (!url.startsWith('http'))
|
|
291
|
+
url = `https://${url}`;
|
|
292
|
+
const { createClient } = await import('../../core/client.js');
|
|
293
|
+
const { createSoapClient } = await import('../../plugins/soap.js');
|
|
294
|
+
try {
|
|
295
|
+
const httpClient = createClient({ headers });
|
|
296
|
+
const soap = createSoapClient(httpClient, { endpoint: url, namespace: data.namespace });
|
|
297
|
+
const body = { ...data };
|
|
298
|
+
delete body.namespace;
|
|
299
|
+
delete body.envelope;
|
|
300
|
+
const result = await soap.call(action, body);
|
|
301
|
+
console.log(JSON.stringify(result, null, 2));
|
|
302
|
+
}
|
|
303
|
+
catch (err) {
|
|
304
|
+
console.error(colors.red(`SOAP Error: ${err.message}`));
|
|
305
|
+
process.exit(1);
|
|
306
|
+
}
|
|
307
|
+
});
|
|
308
|
+
program.command('odata')
|
|
309
|
+
.description(odataSchema.description)
|
|
310
|
+
.argument('<url>', 'OData Service URL')
|
|
311
|
+
.argument('<entity>', 'Entity Set')
|
|
312
|
+
.argument('[args...]', 'Query Options')
|
|
313
|
+
.addHelpText('after', generateHelp(odataSchema))
|
|
314
|
+
.action(async (url, entity, rawArgs) => {
|
|
315
|
+
const { data, headers } = RekArgs.parse(rawArgs, odataSchema);
|
|
316
|
+
if (!url.startsWith('http'))
|
|
317
|
+
url = `https://${url}`;
|
|
318
|
+
const { createClient } = await import('../../core/client.js');
|
|
319
|
+
const { createODataClient } = await import('../../plugins/odata.js');
|
|
320
|
+
try {
|
|
321
|
+
const httpClient = createClient({ headers });
|
|
322
|
+
const odata = createODataClient(httpClient, { serviceRoot: url });
|
|
323
|
+
let query = odata.query(entity);
|
|
324
|
+
if (data.select)
|
|
325
|
+
query = query.select(...data.select.split(','));
|
|
326
|
+
if (data.filter)
|
|
327
|
+
query = query.filter(data.filter);
|
|
328
|
+
if (data.orderby)
|
|
329
|
+
query = query.orderBy(data.orderby);
|
|
330
|
+
if (data.top)
|
|
331
|
+
query = query.top(data.top);
|
|
332
|
+
if (data.skip)
|
|
333
|
+
query = query.skip(data.skip);
|
|
334
|
+
if (data.expand)
|
|
335
|
+
query = query.expand(data.expand);
|
|
336
|
+
const results = await query.get();
|
|
337
|
+
console.log(JSON.stringify(results, null, 2));
|
|
338
|
+
}
|
|
339
|
+
catch (err) {
|
|
340
|
+
console.error(colors.red(`OData Error: ${err.message}`));
|
|
341
|
+
process.exit(1);
|
|
342
|
+
}
|
|
343
|
+
});
|
|
344
|
+
}
|
|
@@ -0,0 +1,176 @@
|
|
|
1
|
+
import colors from '../../utils/colors.js';
|
|
2
|
+
import { RekArgs, generateHelp } from '../parser/index.js';
|
|
3
|
+
const schema = {
|
|
4
|
+
name: 'scrape',
|
|
5
|
+
description: 'Extract data from web pages with CSS selectors.\nFetches a web page and extracts data using CSS selectors or built-in extractors.',
|
|
6
|
+
params: {
|
|
7
|
+
select: { type: 'string', description: 'CSS selector to extract elements' },
|
|
8
|
+
attr: { type: 'string', description: 'Extract specific attribute (use with select)' },
|
|
9
|
+
},
|
|
10
|
+
keywords: {
|
|
11
|
+
links: { description: 'Extract all links with text and href' },
|
|
12
|
+
images: { description: 'Extract all images with src and alt' },
|
|
13
|
+
meta: { description: 'Extract all meta tags' },
|
|
14
|
+
tables: { description: 'Extract tables as structured JSON' },
|
|
15
|
+
scripts: { description: 'Extract all script sources' },
|
|
16
|
+
jsonld: { description: 'Extract JSON-LD structured data' },
|
|
17
|
+
},
|
|
18
|
+
examples: [
|
|
19
|
+
{ cmd: 'rek scrape example.com', desc: 'Basic page info' },
|
|
20
|
+
{ cmd: 'rek scrape example.com select="h1"', desc: 'Extract h1 text' },
|
|
21
|
+
{ cmd: 'rek scrape example.com select="a" attr=href', desc: 'Extract link hrefs' },
|
|
22
|
+
{ cmd: 'rek scrape example.com links', desc: 'All links' },
|
|
23
|
+
{ cmd: 'rek scrape example.com tables', desc: 'All tables as JSON' }
|
|
24
|
+
]
|
|
25
|
+
};
|
|
26
|
+
export function registerScrapeCommand(program) {
|
|
27
|
+
program
|
|
28
|
+
.command('scrape')
|
|
29
|
+
.alias('extract')
|
|
30
|
+
.description('Extract data from web pages with CSS selectors')
|
|
31
|
+
.argument('<url>', 'URL to scrape')
|
|
32
|
+
.argument('[args...]', 'Options: select=SELECTOR, attr=NAME, links, images...')
|
|
33
|
+
.addHelpText('after', generateHelp(schema))
|
|
34
|
+
.action(async (url, rawArgs) => {
|
|
35
|
+
const { data } = RekArgs.parse(rawArgs, schema);
|
|
36
|
+
const { ScrapeDocument } = await import('../../scrape/document.js');
|
|
37
|
+
const { Client } = await import('../../core/client.js');
|
|
38
|
+
if (!url.startsWith('http://') && !url.startsWith('https://')) {
|
|
39
|
+
url = `https://${url}`;
|
|
40
|
+
}
|
|
41
|
+
console.log(colors.gray(`Fetching ${url}...`));
|
|
42
|
+
try {
|
|
43
|
+
const client = new Client();
|
|
44
|
+
const response = await client.get(url);
|
|
45
|
+
const html = await response.text();
|
|
46
|
+
const doc = await ScrapeDocument.create(html, { baseUrl: url });
|
|
47
|
+
const hasExtraction = data.select || data.links || data.images || data.meta || data.tables || data.scripts || data.jsonld;
|
|
48
|
+
if (!hasExtraction) {
|
|
49
|
+
const title = doc.text('title') || 'N/A';
|
|
50
|
+
const description = doc.attr('meta[name="description"]', 'content') || 'N/A';
|
|
51
|
+
const h1 = doc.text('h1') || 'N/A';
|
|
52
|
+
const linkCount = doc.links().length;
|
|
53
|
+
const imageCount = doc.images().length;
|
|
54
|
+
console.log(`
|
|
55
|
+
${colors.bold(colors.cyan('📄 Page Info'))}
|
|
56
|
+
|
|
57
|
+
${colors.bold('Title:')} ${title}
|
|
58
|
+
${colors.bold('Description:')} ${description.slice(0, 100)}${description.length > 100 ? '...' : ''}
|
|
59
|
+
${colors.bold('H1:')} ${h1}
|
|
60
|
+
${colors.bold('Links:')} ${linkCount}
|
|
61
|
+
${colors.bold('Images:')} ${imageCount}
|
|
62
|
+
`);
|
|
63
|
+
return;
|
|
64
|
+
}
|
|
65
|
+
if (data.select) {
|
|
66
|
+
if (data.attr) {
|
|
67
|
+
const values = doc.attrs(data.select, data.attr);
|
|
68
|
+
console.log(`\n${colors.bold(`Found ${values.length} values for "${data.attr}" in "${data.select}"`)}\n`);
|
|
69
|
+
values.slice(0, 50).forEach((value, i) => {
|
|
70
|
+
if (value) {
|
|
71
|
+
console.log(`${colors.gray(`${i + 1}.`)} ${value}`);
|
|
72
|
+
}
|
|
73
|
+
});
|
|
74
|
+
if (values.length > 50) {
|
|
75
|
+
console.log(colors.gray(`\n... and ${values.length - 50} more`));
|
|
76
|
+
}
|
|
77
|
+
}
|
|
78
|
+
else {
|
|
79
|
+
const texts = doc.texts(data.select);
|
|
80
|
+
console.log(`\n${colors.bold(`Found ${texts.length} elements matching "${data.select}"`)}\n`);
|
|
81
|
+
texts.slice(0, 50).forEach((text, i) => {
|
|
82
|
+
const trimmed = text.trim();
|
|
83
|
+
if (trimmed) {
|
|
84
|
+
console.log(`${colors.gray(`${i + 1}.`)} ${trimmed.slice(0, 200)}`);
|
|
85
|
+
}
|
|
86
|
+
});
|
|
87
|
+
if (texts.length > 50) {
|
|
88
|
+
console.log(colors.gray(`\n... and ${texts.length - 50} more`));
|
|
89
|
+
}
|
|
90
|
+
}
|
|
91
|
+
return;
|
|
92
|
+
}
|
|
93
|
+
if (data.links) {
|
|
94
|
+
const links = doc.links();
|
|
95
|
+
console.log(`\n${colors.bold(`Found ${links.length} links`)}\n`);
|
|
96
|
+
links.slice(0, 50).forEach((link, i) => {
|
|
97
|
+
const text = (link.text || '').trim().slice(0, 50) || '[no text]';
|
|
98
|
+
console.log(`${colors.gray(`${i + 1}.`)} ${colors.cyan(text)}`);
|
|
99
|
+
console.log(` ${colors.gray(link.href)}`);
|
|
100
|
+
});
|
|
101
|
+
if (links.length > 50) {
|
|
102
|
+
console.log(colors.gray(`\n... and ${links.length - 50} more`));
|
|
103
|
+
}
|
|
104
|
+
return;
|
|
105
|
+
}
|
|
106
|
+
if (data.images) {
|
|
107
|
+
const images = doc.images();
|
|
108
|
+
console.log(`\n${colors.bold(`Found ${images.length} images`)}\n`);
|
|
109
|
+
images.slice(0, 30).forEach((img, i) => {
|
|
110
|
+
const alt = img.alt || '[no alt]';
|
|
111
|
+
console.log(`${colors.gray(`${i + 1}.`)} ${colors.cyan(alt.slice(0, 50))}`);
|
|
112
|
+
console.log(` ${colors.gray(img.src)}`);
|
|
113
|
+
});
|
|
114
|
+
if (images.length > 30) {
|
|
115
|
+
console.log(colors.gray(`\n... and ${images.length - 30} more`));
|
|
116
|
+
}
|
|
117
|
+
return;
|
|
118
|
+
}
|
|
119
|
+
if (data.meta) {
|
|
120
|
+
const meta = doc.meta();
|
|
121
|
+
const entries = Object.entries(meta);
|
|
122
|
+
console.log(`\n${colors.bold(`Found ${entries.length} meta entries`)}\n`);
|
|
123
|
+
entries.forEach(([name, content]) => {
|
|
124
|
+
if (name && content) {
|
|
125
|
+
const value = String(content);
|
|
126
|
+
console.log(`${colors.cyan(name)}: ${value.slice(0, 100)}${value.length > 100 ? '...' : ''}`);
|
|
127
|
+
}
|
|
128
|
+
});
|
|
129
|
+
return;
|
|
130
|
+
}
|
|
131
|
+
if (data.tables) {
|
|
132
|
+
const tables = doc.tables();
|
|
133
|
+
console.log(`\n${colors.bold(`Found ${tables.length} tables`)}\n`);
|
|
134
|
+
tables.slice(0, 5).forEach((table, tableIndex) => {
|
|
135
|
+
console.log(`${colors.bold(`Table ${tableIndex + 1}:`)} ${table.rows?.length || 0} rows`);
|
|
136
|
+
console.log(JSON.stringify((table.rows || []).slice(0, 10), null, 2));
|
|
137
|
+
if ((table.rows?.length || 0) > 10) {
|
|
138
|
+
console.log(colors.gray(`... and ${(table.rows?.length || 0) - 10} more rows`));
|
|
139
|
+
}
|
|
140
|
+
console.log('');
|
|
141
|
+
});
|
|
142
|
+
return;
|
|
143
|
+
}
|
|
144
|
+
if (data.scripts) {
|
|
145
|
+
const scripts = doc.scripts();
|
|
146
|
+
const external = scripts.filter(s => s.src);
|
|
147
|
+
const inline = scripts.filter(s => !s.src);
|
|
148
|
+
console.log(`\n${colors.bold(`Found ${external.length} external scripts, ${inline.length} inline`)}\n`);
|
|
149
|
+
if (external.length > 0) {
|
|
150
|
+
console.log(colors.bold('External Scripts:'));
|
|
151
|
+
external.slice(0, 20).forEach((script, i) => {
|
|
152
|
+
console.log(`${colors.gray(`${i + 1}.`)} ${script.src}`);
|
|
153
|
+
});
|
|
154
|
+
if (external.length > 20) {
|
|
155
|
+
console.log(colors.gray(`... and ${external.length - 20} more`));
|
|
156
|
+
}
|
|
157
|
+
}
|
|
158
|
+
return;
|
|
159
|
+
}
|
|
160
|
+
if (data.jsonld) {
|
|
161
|
+
const jsonld = doc.jsonLd();
|
|
162
|
+
console.log(`\n${colors.bold(`Found ${jsonld.length} JSON-LD blocks`)}\n`);
|
|
163
|
+
jsonld.forEach((data, i) => {
|
|
164
|
+
console.log(`${colors.bold(`Block ${i + 1}:`)} ${data['@type'] || 'Unknown type'}`);
|
|
165
|
+
console.log(JSON.stringify(data, null, 2));
|
|
166
|
+
console.log('');
|
|
167
|
+
});
|
|
168
|
+
return;
|
|
169
|
+
}
|
|
170
|
+
}
|
|
171
|
+
catch (error) {
|
|
172
|
+
console.error(colors.red(`Scrape failed: ${error.message}`));
|
|
173
|
+
process.exit(1);
|
|
174
|
+
}
|
|
175
|
+
});
|
|
176
|
+
}
|
|
@@ -0,0 +1,57 @@
|
|
|
1
|
+
import colors from '../../utils/colors.js';
|
|
2
|
+
import { generateHelp } from '../parser/index.js';
|
|
3
|
+
const schema = {
|
|
4
|
+
name: 'security',
|
|
5
|
+
description: 'Grade a website\'s security headers (A+ to F).\nAnalyzes HTTP response headers for security best practices like HSTS, CSP, and more.',
|
|
6
|
+
examples: [
|
|
7
|
+
{ cmd: 'rek security github.com', desc: 'Grade GitHub\'s headers' }
|
|
8
|
+
]
|
|
9
|
+
};
|
|
10
|
+
export function registerSecurityCommand(program) {
|
|
11
|
+
program
|
|
12
|
+
.command('security')
|
|
13
|
+
.alias('headers')
|
|
14
|
+
.alias('grade')
|
|
15
|
+
.description('Grade a website\'s security headers')
|
|
16
|
+
.argument('<url>', 'URL to analyze')
|
|
17
|
+
.addHelpText('after', generateHelp(schema))
|
|
18
|
+
.action(async (url) => {
|
|
19
|
+
if (!url.startsWith('http'))
|
|
20
|
+
url = `https://${url}`;
|
|
21
|
+
const { createClient } = await import('../../core/client.js');
|
|
22
|
+
const { analyzeSecurityHeaders } = await import('../../utils/security-grader.js');
|
|
23
|
+
console.log(colors.gray(`Analyzing security headers for ${url}...`));
|
|
24
|
+
try {
|
|
25
|
+
const origin = new URL(url).origin;
|
|
26
|
+
const client = createClient({ baseUrl: origin });
|
|
27
|
+
const res = await client.get(url);
|
|
28
|
+
const report = analyzeSecurityHeaders(res.headers);
|
|
29
|
+
let gradeColor = colors.red;
|
|
30
|
+
if (report.grade.startsWith('A'))
|
|
31
|
+
gradeColor = colors.green;
|
|
32
|
+
if (report.grade.startsWith('B'))
|
|
33
|
+
gradeColor = colors.blue;
|
|
34
|
+
if (report.grade.startsWith('C'))
|
|
35
|
+
gradeColor = colors.yellow;
|
|
36
|
+
console.log(`
|
|
37
|
+
${colors.bold(colors.cyan('🛡️ Security Headers Report'))}
|
|
38
|
+
Grade: ${gradeColor(colors.bold(report.grade))} (${report.score}/100)
|
|
39
|
+
|
|
40
|
+
${colors.bold('Details:')}`);
|
|
41
|
+
report.details.forEach(item => {
|
|
42
|
+
const icon = item.status === 'pass' ? colors.green('✔') : item.status === 'warn' ? colors.yellow('⚠') : colors.red('✖');
|
|
43
|
+
const headerName = colors.bold(item.header);
|
|
44
|
+
const value = item.value ? colors.gray(`= ${item.value.length > 50 ? item.value.slice(0, 47) + '...' : item.value}`) : colors.gray('(missing)');
|
|
45
|
+
console.log(` ${icon} ${headerName} ${value}`);
|
|
46
|
+
if (item.status !== 'pass') {
|
|
47
|
+
console.log(` ${colors.red('→')} ${item.message}`);
|
|
48
|
+
}
|
|
49
|
+
});
|
|
50
|
+
console.log('');
|
|
51
|
+
}
|
|
52
|
+
catch (error) {
|
|
53
|
+
console.error(colors.red(`Analysis failed: ${error.message}`));
|
|
54
|
+
process.exit(1);
|
|
55
|
+
}
|
|
56
|
+
});
|
|
57
|
+
}
|