recker 1.0.43 → 1.0.44-next.084c2ef
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +47 -0
- package/dist/bin/recker-linux-x64 +0 -0
- package/dist/bin/recker-macos-x64 +0 -0
- package/dist/bin/recker-win-x64.exe +0 -0
- package/dist/bin/rek.cjs +85617 -99560
- package/dist/browser/ai/adaptive-timeout.d.ts +50 -0
- package/dist/browser/ai/adaptive-timeout.js +208 -0
- package/dist/browser/ai/client.d.ts +22 -0
- package/dist/browser/ai/client.js +294 -0
- package/dist/browser/ai/index.d.ts +14 -0
- package/dist/browser/ai/index.js +11 -0
- package/dist/browser/ai/providers/anthropic.d.ts +63 -0
- package/dist/browser/ai/providers/anthropic.js +370 -0
- package/dist/browser/ai/providers/base.d.ts +48 -0
- package/dist/browser/ai/providers/base.js +150 -0
- package/dist/browser/ai/providers/google.d.ts +59 -0
- package/dist/browser/ai/providers/google.js +305 -0
- package/dist/browser/ai/providers/ollama.d.ts +44 -0
- package/dist/browser/ai/providers/ollama.js +240 -0
- package/dist/browser/ai/providers/openai.d.ts +64 -0
- package/dist/browser/ai/providers/openai.js +298 -0
- package/dist/browser/ai/rate-limiter.d.ts +43 -0
- package/dist/browser/ai/rate-limiter.js +215 -0
- package/dist/browser/ai/vector/index.d.ts +2 -0
- package/dist/browser/ai/vector/index.js +2 -0
- package/dist/browser/ai/vector/similarity.d.ts +2 -0
- package/dist/browser/ai/vector/similarity.js +27 -0
- package/dist/browser/ai/vector/store.d.ts +27 -0
- package/dist/browser/ai/vector/store.js +82 -0
- package/dist/browser/browser/cache.d.ts +2 -40
- package/dist/browser/browser/cache.js +2 -199
- package/dist/browser/browser/index.d.ts +8 -0
- package/dist/browser/browser/index.js +8 -0
- package/dist/browser/browser/recker.d.ts +8 -1
- package/dist/browser/browser/recker.js +8 -2
- package/dist/browser/cache/indexed-db.d.ts +10 -0
- package/dist/browser/cache/indexed-db.js +88 -0
- package/dist/browser/cache/service-worker-cache.d.ts +18 -0
- package/dist/browser/cache/service-worker-cache.js +103 -0
- package/dist/browser/cache.d.ts +2 -40
- package/dist/browser/cache.js +2 -199
- package/dist/browser/constants/user-agents.d.ts +7 -0
- package/dist/browser/constants/user-agents.js +7 -0
- package/dist/browser/core/client.d.ts +2 -0
- package/dist/browser/core/client.js +19 -1
- package/dist/browser/index.d.ts +8 -0
- package/dist/browser/index.js +8 -0
- package/dist/browser/plugins/har-recorder.d.ts +40 -0
- package/dist/browser/plugins/har-recorder.js +120 -0
- package/dist/browser/plugins/network-simulation.d.ts +7 -0
- package/dist/browser/plugins/network-simulation.js +13 -0
- package/dist/browser/presets/android.d.ts +2 -0
- package/dist/browser/presets/android.js +16 -0
- package/dist/browser/presets/anthropic.d.ts +8 -0
- package/dist/browser/presets/anthropic.js +27 -0
- package/dist/browser/presets/aws.d.ts +19 -0
- package/dist/browser/presets/aws.js +68 -0
- package/dist/browser/presets/azure-openai.d.ts +10 -0
- package/dist/browser/presets/azure-openai.js +35 -0
- package/dist/browser/presets/azure.d.ts +41 -0
- package/dist/browser/presets/azure.js +104 -0
- package/dist/browser/presets/chaturbate.d.ts +2 -0
- package/dist/browser/presets/chaturbate.js +17 -0
- package/dist/browser/presets/cloudflare.d.ts +12 -0
- package/dist/browser/presets/cloudflare.js +39 -0
- package/dist/browser/presets/cohere.d.ts +7 -0
- package/dist/browser/presets/cohere.js +22 -0
- package/dist/browser/presets/deepseek.d.ts +7 -0
- package/dist/browser/presets/deepseek.js +22 -0
- package/dist/browser/presets/digitalocean.d.ts +5 -0
- package/dist/browser/presets/digitalocean.js +16 -0
- package/dist/browser/presets/discord.d.ts +6 -0
- package/dist/browser/presets/discord.js +17 -0
- package/dist/browser/presets/elevenlabs.d.ts +6 -0
- package/dist/browser/presets/elevenlabs.js +20 -0
- package/dist/browser/presets/enhancers.d.ts +20 -0
- package/dist/browser/presets/enhancers.js +85 -0
- package/dist/browser/presets/fireworks.d.ts +7 -0
- package/dist/browser/presets/fireworks.js +22 -0
- package/dist/browser/presets/gcp.d.ts +34 -0
- package/dist/browser/presets/gcp.js +91 -0
- package/dist/browser/presets/gemini.d.ts +7 -0
- package/dist/browser/presets/gemini.js +23 -0
- package/dist/browser/presets/github.d.ts +6 -0
- package/dist/browser/presets/github.js +17 -0
- package/dist/browser/presets/gitlab.d.ts +6 -0
- package/dist/browser/presets/gitlab.js +16 -0
- package/dist/browser/presets/groq.d.ts +7 -0
- package/dist/browser/presets/groq.js +22 -0
- package/dist/browser/presets/hubspot.d.ts +9 -0
- package/dist/browser/presets/hubspot.js +28 -0
- package/dist/browser/presets/huggingface.d.ts +7 -0
- package/dist/browser/presets/huggingface.js +23 -0
- package/dist/browser/presets/index.d.ts +47 -0
- package/dist/browser/presets/index.js +47 -0
- package/dist/browser/presets/ios.d.ts +2 -0
- package/dist/browser/presets/ios.js +13 -0
- package/dist/browser/presets/linear.d.ts +5 -0
- package/dist/browser/presets/linear.js +16 -0
- package/dist/browser/presets/mailgun.d.ts +7 -0
- package/dist/browser/presets/mailgun.js +20 -0
- package/dist/browser/presets/meta.d.ts +10 -0
- package/dist/browser/presets/meta.js +33 -0
- package/dist/browser/presets/mistral.d.ts +7 -0
- package/dist/browser/presets/mistral.js +22 -0
- package/dist/browser/presets/notion.d.ts +6 -0
- package/dist/browser/presets/notion.js +17 -0
- package/dist/browser/presets/openai.d.ts +9 -0
- package/dist/browser/presets/openai.js +30 -0
- package/dist/browser/presets/oracle.d.ts +19 -0
- package/dist/browser/presets/oracle.js +117 -0
- package/dist/browser/presets/perplexity.d.ts +7 -0
- package/dist/browser/presets/perplexity.js +22 -0
- package/dist/browser/presets/pinecone.d.ts +8 -0
- package/dist/browser/presets/pinecone.js +42 -0
- package/dist/browser/presets/registry.d.ts +23 -0
- package/dist/browser/presets/registry.js +519 -0
- package/dist/browser/presets/replicate.d.ts +7 -0
- package/dist/browser/presets/replicate.js +23 -0
- package/dist/browser/presets/sendgrid.d.ts +6 -0
- package/dist/browser/presets/sendgrid.js +20 -0
- package/dist/browser/presets/sentry.d.ts +11 -0
- package/dist/browser/presets/sentry.js +48 -0
- package/dist/browser/presets/sinch.d.ts +9 -0
- package/dist/browser/presets/sinch.js +39 -0
- package/dist/browser/presets/slack.d.ts +5 -0
- package/dist/browser/presets/slack.js +16 -0
- package/dist/browser/presets/square.d.ts +10 -0
- package/dist/browser/presets/square.js +33 -0
- package/dist/browser/presets/stripe.d.ts +7 -0
- package/dist/browser/presets/stripe.js +23 -0
- package/dist/browser/presets/supabase.d.ts +6 -0
- package/dist/browser/presets/supabase.js +18 -0
- package/dist/browser/presets/tiktok.d.ts +10 -0
- package/dist/browser/presets/tiktok.js +38 -0
- package/dist/browser/presets/together.d.ts +7 -0
- package/dist/browser/presets/together.js +22 -0
- package/dist/browser/presets/twilio.d.ts +6 -0
- package/dist/browser/presets/twilio.js +17 -0
- package/dist/browser/presets/vercel.d.ts +6 -0
- package/dist/browser/presets/vercel.js +23 -0
- package/dist/browser/presets/vultr.d.ts +5 -0
- package/dist/browser/presets/vultr.js +16 -0
- package/dist/browser/presets/xai.d.ts +8 -0
- package/dist/browser/presets/xai.js +23 -0
- package/dist/browser/presets/youtube.d.ts +5 -0
- package/dist/browser/presets/youtube.js +20 -0
- package/dist/browser/recker.d.ts +8 -1
- package/dist/browser/recker.js +8 -2
- package/dist/browser/scrape/document.d.ts +5 -4
- package/dist/browser/scrape/document.js +89 -76
- package/dist/browser/scrape/element.d.ts +10 -8
- package/dist/browser/scrape/element.js +295 -81
- package/dist/browser/scrape/extractors.d.ts +11 -11
- package/dist/browser/scrape/extractors.js +145 -113
- package/dist/browser/scrape/parser/back.d.ts +1 -0
- package/dist/browser/scrape/parser/back.js +3 -0
- package/dist/browser/scrape/parser/index.d.ts +20 -0
- package/dist/browser/scrape/parser/index.js +19 -0
- package/dist/browser/scrape/parser/matcher.d.ts +30 -0
- package/dist/browser/scrape/parser/matcher.js +99 -0
- package/dist/browser/scrape/parser/nodes/comment.d.ts +12 -0
- package/dist/browser/scrape/parser/nodes/comment.js +21 -0
- package/dist/browser/scrape/parser/nodes/html.d.ts +110 -0
- package/dist/browser/scrape/parser/nodes/html.js +978 -0
- package/dist/browser/scrape/parser/nodes/node.d.ts +18 -0
- package/dist/browser/scrape/parser/nodes/node.js +31 -0
- package/dist/browser/scrape/parser/nodes/text.d.ts +14 -0
- package/dist/browser/scrape/parser/nodes/text.js +30 -0
- package/dist/browser/scrape/parser/nodes/type.d.ts +6 -0
- package/dist/browser/scrape/parser/nodes/type.js +7 -0
- package/dist/browser/scrape/parser/parse.d.ts +1 -0
- package/dist/browser/scrape/parser/parse.js +1 -0
- package/dist/browser/scrape/parser/valid.d.ts +2 -0
- package/dist/browser/scrape/parser/valid.js +5 -0
- package/dist/browser/scrape/parser/void-tag.d.ts +7 -0
- package/dist/browser/scrape/parser/void-tag.js +43 -0
- package/dist/browser/scrape/types.d.ts +7 -0
- package/dist/browser/seo/analyzer.d.ts +59 -0
- package/dist/browser/seo/analyzer.js +1399 -0
- package/dist/browser/seo/keywords.d.ts +16 -0
- package/dist/browser/seo/keywords.js +55 -0
- package/dist/browser/seo/rules/accessibility.d.ts +2 -0
- package/dist/browser/seo/rules/accessibility.js +733 -0
- package/dist/browser/seo/rules/ai-search.d.ts +2 -0
- package/dist/browser/seo/rules/ai-search.js +436 -0
- package/dist/browser/seo/rules/analytics.d.ts +2 -0
- package/dist/browser/seo/rules/analytics.js +306 -0
- package/dist/browser/seo/rules/best-practices.d.ts +2 -0
- package/dist/browser/seo/rules/best-practices.js +195 -0
- package/dist/browser/seo/rules/canonical.d.ts +12 -0
- package/dist/browser/seo/rules/canonical.js +270 -0
- package/dist/browser/seo/rules/content.d.ts +2 -0
- package/dist/browser/seo/rules/content.js +522 -0
- package/dist/browser/seo/rules/crawl.d.ts +2 -0
- package/dist/browser/seo/rules/crawl.js +435 -0
- package/dist/browser/seo/rules/cwv.d.ts +2 -0
- package/dist/browser/seo/rules/cwv.js +248 -0
- package/dist/browser/seo/rules/ecommerce.d.ts +2 -0
- package/dist/browser/seo/rules/ecommerce.js +312 -0
- package/dist/browser/seo/rules/i18n.d.ts +2 -0
- package/dist/browser/seo/rules/i18n.js +288 -0
- package/dist/browser/seo/rules/images.d.ts +2 -0
- package/dist/browser/seo/rules/images.js +255 -0
- package/dist/browser/seo/rules/index.d.ts +52 -0
- package/dist/browser/seo/rules/index.js +163 -0
- package/dist/browser/seo/rules/internal-linking.d.ts +2 -0
- package/dist/browser/seo/rules/internal-linking.js +394 -0
- package/dist/browser/seo/rules/links.d.ts +2 -0
- package/dist/browser/seo/rules/links.js +498 -0
- package/dist/browser/seo/rules/local.d.ts +2 -0
- package/dist/browser/seo/rules/local.js +289 -0
- package/dist/browser/seo/rules/manifest.d.ts +2 -0
- package/dist/browser/seo/rules/manifest.js +621 -0
- package/dist/browser/seo/rules/meta.d.ts +2 -0
- package/dist/browser/seo/rules/meta.js +805 -0
- package/dist/browser/seo/rules/mobile.d.ts +2 -0
- package/dist/browser/seo/rules/mobile.js +161 -0
- package/dist/browser/seo/rules/performance.d.ts +2 -0
- package/dist/browser/seo/rules/performance.js +738 -0
- package/dist/browser/seo/rules/pwa.d.ts +2 -0
- package/dist/browser/seo/rules/pwa.js +299 -0
- package/dist/browser/seo/rules/readability.d.ts +2 -0
- package/dist/browser/seo/rules/readability.js +264 -0
- package/dist/browser/seo/rules/redirects.d.ts +16 -0
- package/dist/browser/seo/rules/redirects.js +199 -0
- package/dist/browser/seo/rules/resources.d.ts +2 -0
- package/dist/browser/seo/rules/resources.js +390 -0
- package/dist/browser/seo/rules/schema.d.ts +2 -0
- package/dist/browser/seo/rules/schema.js +379 -0
- package/dist/browser/seo/rules/security.d.ts +2 -0
- package/dist/browser/seo/rules/security.js +877 -0
- package/dist/browser/seo/rules/social.d.ts +2 -0
- package/dist/browser/seo/rules/social.js +603 -0
- package/dist/browser/seo/rules/structural.d.ts +2 -0
- package/dist/browser/seo/rules/structural.js +223 -0
- package/dist/browser/seo/rules/technical-advanced.d.ts +10 -0
- package/dist/browser/seo/rules/technical-advanced.js +289 -0
- package/dist/browser/seo/rules/technical.d.ts +3 -0
- package/dist/browser/seo/rules/technical.js +509 -0
- package/dist/browser/seo/rules/thresholds.d.ts +196 -0
- package/dist/browser/seo/rules/thresholds.js +118 -0
- package/dist/browser/seo/rules/types.d.ts +518 -0
- package/dist/browser/seo/rules/types.js +11 -0
- package/dist/browser/seo/types.d.ts +211 -0
- package/dist/browser/seo/types.js +1 -0
- package/dist/browser/transport/curl.d.ts +4 -0
- package/dist/browser/transport/curl.js +101 -0
- package/dist/browser/transport/undici.js +1 -2
- package/dist/browser/transport/worker.d.ts +18 -0
- package/dist/browser/transport/worker.js +278 -0
- package/dist/browser/types/index.d.ts +4 -1
- package/dist/browser/utils/binary-manager.d.ts +4 -0
- package/dist/browser/utils/binary-manager.js +72 -0
- package/dist/browser/utils/user-agent.js +2 -13
- package/dist/cache/indexed-db.d.ts +10 -0
- package/dist/cache/indexed-db.js +88 -0
- package/dist/cache/service-worker-cache.d.ts +18 -0
- package/dist/cache/service-worker-cache.js +103 -0
- package/dist/cli/commands/ai.d.ts +2 -0
- package/dist/cli/commands/ai.js +162 -0
- package/dist/cli/commands/bench.d.ts +2 -0
- package/dist/cli/commands/bench.js +51 -0
- package/dist/cli/commands/dns.d.ts +2 -0
- package/dist/cli/commands/dns.js +295 -0
- package/dist/cli/commands/har.d.ts +2 -0
- package/dist/cli/commands/har.js +171 -0
- package/dist/cli/commands/hls.d.ts +2 -0
- package/dist/cli/commands/hls.js +192 -0
- package/dist/cli/commands/network.d.ts +2 -0
- package/dist/cli/commands/network.js +288 -0
- package/dist/cli/commands/protocols.d.ts +2 -0
- package/dist/cli/commands/protocols.js +344 -0
- package/dist/cli/commands/scrape.d.ts +2 -0
- package/dist/cli/commands/scrape.js +176 -0
- package/dist/cli/commands/security.d.ts +2 -0
- package/dist/cli/commands/security.js +57 -0
- package/dist/cli/commands/seo.d.ts +2 -0
- package/dist/cli/commands/seo.js +125 -0
- package/dist/cli/commands/serve.d.ts +2 -0
- package/dist/cli/commands/serve.js +531 -0
- package/dist/cli/commands/spider.d.ts +3 -0
- package/dist/cli/commands/spider.js +478 -0
- package/dist/cli/commands/utils.d.ts +2 -0
- package/dist/cli/commands/utils.js +176 -0
- package/dist/cli/commands/vector.d.ts +2 -0
- package/dist/cli/commands/vector.js +158 -0
- package/dist/cli/handler.d.ts +2 -2
- package/dist/cli/handler.js +6 -6
- package/dist/cli/helpers.d.ts +7 -0
- package/dist/cli/helpers.js +128 -0
- package/dist/cli/index.js +96 -5228
- package/dist/cli/parser/help.d.ts +2 -0
- package/dist/cli/parser/help.js +52 -0
- package/dist/cli/parser/index.d.ts +3 -0
- package/dist/cli/parser/index.js +3 -0
- package/dist/cli/parser/parser.d.ts +4 -0
- package/dist/cli/parser/parser.js +146 -0
- package/dist/cli/parser/types.d.ts +41 -0
- package/dist/cli/parser/types.js +1 -0
- package/dist/cli/presets.d.ts +1 -1
- package/dist/cli/presets.js +1 -1
- package/dist/cli/router.d.ts +36 -0
- package/dist/cli/router.js +195 -0
- package/dist/cli/tui/ai-chat.js +1 -1
- package/dist/cli/tui/commands/context.d.ts +9 -0
- package/dist/cli/tui/commands/context.js +1 -0
- package/dist/cli/tui/commands/dns.d.ts +10 -0
- package/dist/cli/tui/commands/dns.js +461 -0
- package/dist/cli/tui/commands/hls.d.ts +2 -0
- package/dist/cli/tui/commands/hls.js +162 -0
- package/dist/cli/tui/commands/ip.d.ts +2 -0
- package/dist/cli/tui/commands/ip.js +45 -0
- package/dist/cli/tui/commands/network.d.ts +3 -0
- package/dist/cli/tui/commands/network.js +81 -0
- package/dist/cli/tui/commands/protocols.d.ts +6 -0
- package/dist/cli/tui/commands/protocols.js +531 -0
- package/dist/cli/tui/commands/security.d.ts +2 -0
- package/dist/cli/tui/commands/security.js +48 -0
- package/dist/cli/tui/commands/seo.d.ts +2 -0
- package/dist/cli/tui/commands/seo.js +74 -0
- package/dist/cli/tui/context.d.ts +12 -0
- package/dist/cli/tui/context.js +1 -0
- package/dist/cli/tui/shell.d.ts +11 -20
- package/dist/cli/tui/shell.js +216 -1873
- package/dist/constants/user-agents.d.ts +7 -0
- package/dist/constants/user-agents.js +7 -0
- package/dist/core/client.d.ts +2 -0
- package/dist/core/client.js +19 -1
- package/dist/index.d.ts +1 -0
- package/dist/index.js +1 -0
- package/dist/mcp/cli.js +2 -3
- package/dist/mcp/data/embeddings.json +1 -1
- package/dist/mcp/tools/network.js +298 -158
- package/dist/plugins/har-player.d.ts +23 -0
- package/dist/plugins/har-player.js +49 -0
- package/dist/plugins/har-recorder.d.ts +37 -3
- package/dist/plugins/har-recorder.js +116 -63
- package/dist/plugins/network-simulation.d.ts +7 -0
- package/dist/plugins/network-simulation.js +13 -0
- package/dist/presets/android.d.ts +2 -0
- package/dist/presets/android.js +16 -0
- package/dist/presets/chaturbate.d.ts +2 -0
- package/dist/presets/chaturbate.js +17 -0
- package/dist/presets/elevenlabs.d.ts +6 -0
- package/dist/presets/elevenlabs.js +20 -0
- package/dist/presets/enhancers.d.ts +20 -0
- package/dist/presets/enhancers.js +85 -0
- package/dist/presets/hubspot.d.ts +9 -0
- package/dist/presets/hubspot.js +28 -0
- package/dist/presets/index.d.ts +10 -0
- package/dist/presets/index.js +10 -0
- package/dist/presets/ios.d.ts +2 -0
- package/dist/presets/ios.js +13 -0
- package/dist/presets/pinecone.d.ts +8 -0
- package/dist/presets/pinecone.js +42 -0
- package/dist/presets/registry.js +60 -0
- package/dist/presets/sendgrid.d.ts +6 -0
- package/dist/presets/sendgrid.js +20 -0
- package/dist/presets/sentry.d.ts +11 -0
- package/dist/presets/sentry.js +48 -0
- package/dist/presets/square.d.ts +10 -0
- package/dist/presets/square.js +33 -0
- package/dist/recker.d.ts +3 -0
- package/dist/recker.js +4 -0
- package/dist/scrape/document.d.ts +5 -4
- package/dist/scrape/document.js +89 -76
- package/dist/scrape/element.d.ts +10 -8
- package/dist/scrape/element.js +295 -81
- package/dist/scrape/extractors.d.ts +11 -11
- package/dist/scrape/extractors.js +145 -113
- package/dist/scrape/index.d.ts +2 -0
- package/dist/scrape/index.js +1 -0
- package/dist/scrape/parser/back.d.ts +1 -0
- package/dist/scrape/parser/back.js +3 -0
- package/dist/scrape/parser/index.d.ts +20 -0
- package/dist/scrape/parser/index.js +19 -0
- package/dist/scrape/parser/matcher.d.ts +30 -0
- package/dist/scrape/parser/matcher.js +99 -0
- package/dist/scrape/parser/nodes/comment.d.ts +12 -0
- package/dist/scrape/parser/nodes/comment.js +21 -0
- package/dist/scrape/parser/nodes/html.d.ts +110 -0
- package/dist/scrape/parser/nodes/html.js +978 -0
- package/dist/scrape/parser/nodes/node.d.ts +18 -0
- package/dist/scrape/parser/nodes/node.js +31 -0
- package/dist/scrape/parser/nodes/text.d.ts +14 -0
- package/dist/scrape/parser/nodes/text.js +30 -0
- package/dist/scrape/parser/nodes/type.d.ts +6 -0
- package/dist/scrape/parser/nodes/type.js +7 -0
- package/dist/scrape/parser/parse.d.ts +1 -0
- package/dist/scrape/parser/parse.js +1 -0
- package/dist/scrape/parser/valid.d.ts +2 -0
- package/dist/scrape/parser/valid.js +5 -0
- package/dist/scrape/parser/void-tag.d.ts +7 -0
- package/dist/scrape/parser/void-tag.js +43 -0
- package/dist/scrape/spider.d.ts +19 -0
- package/dist/scrape/spider.js +28 -3
- package/dist/scrape/types.d.ts +7 -0
- package/dist/seo/analyzer.d.ts +15 -5
- package/dist/seo/analyzer.js +636 -175
- package/dist/seo/formatter.d.ts +16 -0
- package/dist/seo/formatter.js +228 -0
- package/dist/seo/index.d.ts +2 -0
- package/dist/seo/index.js +1 -0
- package/dist/seo/keywords.d.ts +16 -0
- package/dist/seo/keywords.js +55 -0
- package/dist/seo/rules/accessibility.js +96 -57
- package/dist/seo/rules/ai-search.js +44 -31
- package/dist/seo/rules/analytics.d.ts +2 -0
- package/dist/seo/rules/analytics.js +306 -0
- package/dist/seo/rules/best-practices.js +21 -14
- package/dist/seo/rules/canonical.js +53 -32
- package/dist/seo/rules/content.js +317 -31
- package/dist/seo/rules/crawl.js +55 -40
- package/dist/seo/rules/cwv.js +21 -15
- package/dist/seo/rules/ecommerce.js +82 -22
- package/dist/seo/rules/i18n.js +75 -36
- package/dist/seo/rules/images.js +109 -30
- package/dist/seo/rules/index.js +7 -1
- package/dist/seo/rules/internal-linking.js +58 -39
- package/dist/seo/rules/links.js +79 -52
- package/dist/seo/rules/local.js +49 -25
- package/dist/seo/rules/manifest.d.ts +2 -0
- package/dist/seo/rules/manifest.js +621 -0
- package/dist/seo/rules/meta.js +339 -81
- package/dist/seo/rules/mobile.js +112 -2
- package/dist/seo/rules/performance.js +434 -66
- package/dist/seo/rules/pwa.js +36 -39
- package/dist/seo/rules/readability.js +31 -22
- package/dist/seo/rules/redirects.js +21 -15
- package/dist/seo/rules/resources.js +59 -42
- package/dist/seo/rules/schema.js +333 -8
- package/dist/seo/rules/security.js +142 -80
- package/dist/seo/rules/social.js +277 -47
- package/dist/seo/rules/structural.js +87 -19
- package/dist/seo/rules/technical-advanced.js +30 -24
- package/dist/seo/rules/technical.d.ts +1 -0
- package/dist/seo/rules/technical.js +272 -42
- package/dist/seo/rules/types.d.ts +74 -2
- package/dist/seo/seo-spider.d.ts +35 -0
- package/dist/seo/seo-spider.js +152 -13
- package/dist/seo/types.d.ts +8 -1
- package/dist/seo/validators/llms-txt.js +19 -0
- package/dist/seo/validators/rss.d.ts +11 -0
- package/dist/seo/validators/rss.js +93 -0
- package/dist/seo/validators/sitemap.js +36 -26
- package/dist/transport/curl.d.ts +4 -0
- package/dist/transport/curl.js +101 -0
- package/dist/transport/udp.js +0 -1
- package/dist/transport/undici.js +1 -2
- package/dist/transport/worker.d.ts +18 -0
- package/dist/transport/worker.js +278 -0
- package/dist/types/index.d.ts +4 -1
- package/dist/utils/binary-manager.d.ts +4 -0
- package/dist/utils/binary-manager.js +72 -0
- package/dist/utils/optional-require.d.ts +7 -8
- package/dist/utils/optional-require.js +2 -21
- package/dist/utils/upload.d.ts +6 -0
- package/dist/utils/upload.js +11 -0
- package/dist/utils/user-agent.js +2 -13
- package/dist/version.js +1 -1
- package/package.json +12 -6
- package/dist/browser/utils/optional-require.d.ts +0 -19
- package/dist/browser/utils/optional-require.js +0 -105
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import
|
|
1
|
+
import { parse } from '../../scrape/parser/index.js';
|
|
2
2
|
const VALID_CHANGEFREQ = ['always', 'hourly', 'daily', 'weekly', 'monthly', 'yearly', 'never'];
|
|
3
3
|
const MAX_URLS_PER_SITEMAP = 50000;
|
|
4
4
|
const MAX_SITEMAP_SIZE = 50 * 1024 * 1024;
|
|
@@ -9,12 +9,12 @@ export function parseSitemap(content, compressed = false) {
|
|
|
9
9
|
const sitemaps = [];
|
|
10
10
|
let type = 'unknown';
|
|
11
11
|
try {
|
|
12
|
-
const
|
|
13
|
-
if (
|
|
12
|
+
const root = parse(content, { lowerCaseTagName: false });
|
|
13
|
+
if (root.querySelectorAll('urlset').length > 0) {
|
|
14
14
|
type = 'urlset';
|
|
15
|
-
|
|
16
|
-
const
|
|
17
|
-
const loc =
|
|
15
|
+
root.querySelectorAll('url').forEach((urlElem) => {
|
|
16
|
+
const locElem = urlElem.querySelector('loc');
|
|
17
|
+
const loc = locElem ? locElem.text.trim() : '';
|
|
18
18
|
if (!loc) {
|
|
19
19
|
errors.push('URL entry missing <loc> element');
|
|
20
20
|
return;
|
|
@@ -27,7 +27,8 @@ export function parseSitemap(content, compressed = false) {
|
|
|
27
27
|
return;
|
|
28
28
|
}
|
|
29
29
|
const url = { loc };
|
|
30
|
-
const
|
|
30
|
+
const lastmodElem = urlElem.querySelector('lastmod');
|
|
31
|
+
const lastmod = lastmodElem ? lastmodElem.text.trim() : '';
|
|
31
32
|
if (lastmod) {
|
|
32
33
|
if (isValidDate(lastmod)) {
|
|
33
34
|
url.lastmod = lastmod;
|
|
@@ -36,7 +37,8 @@ export function parseSitemap(content, compressed = false) {
|
|
|
36
37
|
warnings.push(`Invalid lastmod date for ${loc}: ${lastmod}`);
|
|
37
38
|
}
|
|
38
39
|
}
|
|
39
|
-
const
|
|
40
|
+
const changefreqElem = urlElem.querySelector('changefreq');
|
|
41
|
+
const changefreq = changefreqElem ? changefreqElem.text.trim().toLowerCase() : '';
|
|
40
42
|
if (changefreq) {
|
|
41
43
|
if (VALID_CHANGEFREQ.includes(changefreq)) {
|
|
42
44
|
url.changefreq = changefreq;
|
|
@@ -45,7 +47,8 @@ export function parseSitemap(content, compressed = false) {
|
|
|
45
47
|
warnings.push(`Invalid changefreq for ${loc}: ${changefreq}`);
|
|
46
48
|
}
|
|
47
49
|
}
|
|
48
|
-
const
|
|
50
|
+
const priorityElem = urlElem.querySelector('priority');
|
|
51
|
+
const priority = priorityElem ? priorityElem.text.trim() : '';
|
|
49
52
|
if (priority) {
|
|
50
53
|
const p = parseFloat(priority);
|
|
51
54
|
if (isNaN(p) || p < 0 || p > 1) {
|
|
@@ -56,14 +59,17 @@ export function parseSitemap(content, compressed = false) {
|
|
|
56
59
|
}
|
|
57
60
|
}
|
|
58
61
|
const images = [];
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
const
|
|
62
|
+
const imageElems = urlElem.querySelectorAll('image\:image, image');
|
|
63
|
+
imageElems.forEach((imgElem) => {
|
|
64
|
+
const locEl = imgElem.querySelector('image\:loc, loc');
|
|
65
|
+
const imgLoc = locEl ? locEl.text.trim() : '';
|
|
62
66
|
if (imgLoc) {
|
|
67
|
+
const captionEl = imgElem.querySelector('image\:caption, caption');
|
|
68
|
+
const titleEl = imgElem.querySelector('image\:title, title');
|
|
63
69
|
images.push({
|
|
64
70
|
loc: imgLoc,
|
|
65
|
-
caption:
|
|
66
|
-
title:
|
|
71
|
+
caption: captionEl ? captionEl.text.trim() || undefined : undefined,
|
|
72
|
+
title: titleEl ? titleEl.text.trim() || undefined : undefined,
|
|
67
73
|
});
|
|
68
74
|
}
|
|
69
75
|
});
|
|
@@ -71,10 +77,10 @@ export function parseSitemap(content, compressed = false) {
|
|
|
71
77
|
url.images = images;
|
|
72
78
|
}
|
|
73
79
|
const alternates = [];
|
|
74
|
-
|
|
75
|
-
|
|
76
|
-
const hreflang =
|
|
77
|
-
const href =
|
|
80
|
+
const linkElems = urlElem.querySelectorAll('xhtml\:link[rel="alternate"], link[rel="alternate"]');
|
|
81
|
+
linkElems.forEach((linkElem) => {
|
|
82
|
+
const hreflang = linkElem.getAttribute('hreflang');
|
|
83
|
+
const href = linkElem.getAttribute('href');
|
|
78
84
|
if (hreflang && href) {
|
|
79
85
|
alternates.push({ hreflang, href });
|
|
80
86
|
}
|
|
@@ -85,17 +91,18 @@ export function parseSitemap(content, compressed = false) {
|
|
|
85
91
|
urls.push(url);
|
|
86
92
|
});
|
|
87
93
|
}
|
|
88
|
-
else if (
|
|
94
|
+
else if (root.querySelectorAll('sitemapindex').length > 0) {
|
|
89
95
|
type = 'sitemapindex';
|
|
90
|
-
|
|
91
|
-
const
|
|
92
|
-
const loc =
|
|
96
|
+
root.querySelectorAll('sitemap').forEach((sitemapElem) => {
|
|
97
|
+
const locElem = sitemapElem.querySelector('loc');
|
|
98
|
+
const loc = locElem ? locElem.text.trim() : '';
|
|
93
99
|
if (!loc) {
|
|
94
100
|
errors.push('Sitemap entry missing <loc> element');
|
|
95
101
|
return;
|
|
96
102
|
}
|
|
97
103
|
const sitemap = { loc };
|
|
98
|
-
const
|
|
104
|
+
const lastmodElem = sitemapElem.querySelector('lastmod');
|
|
105
|
+
const lastmod = lastmodElem ? lastmodElem.text.trim() : '';
|
|
99
106
|
if (lastmod) {
|
|
100
107
|
if (isValidDate(lastmod)) {
|
|
101
108
|
sitemap.lastmod = lastmod;
|
|
@@ -285,6 +292,9 @@ export async function discoverSitemaps(baseUrl, robotsTxtContent, fetcher) {
|
|
|
285
292
|
}
|
|
286
293
|
}
|
|
287
294
|
}
|
|
295
|
+
if (discovered.size > 0) {
|
|
296
|
+
return Array.from(discovered);
|
|
297
|
+
}
|
|
288
298
|
for (const path of commonLocations) {
|
|
289
299
|
const url = new URL(path, base).href;
|
|
290
300
|
try {
|
|
@@ -399,9 +409,9 @@ export async function fetchAndValidateSitemap(url, fetcher) {
|
|
|
399
409
|
}
|
|
400
410
|
function isValidDate(dateString) {
|
|
401
411
|
const patterns = [
|
|
402
|
-
|
|
403
|
-
|
|
404
|
-
|
|
412
|
+
/\d{4}-\d{2}-\d{2}$/,
|
|
413
|
+
/\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}([+-]\d{2}:\d{2}|Z)$/,
|
|
414
|
+
/\d{4}-\d{2}-\d{2}T\d{2}:\d{2}([+-]\d{2}:\d{2}|Z)$/,
|
|
405
415
|
];
|
|
406
416
|
if (!patterns.some(p => p.test(dateString))) {
|
|
407
417
|
return false;
|
|
@@ -0,0 +1,101 @@
|
|
|
1
|
+
import { spawn } from 'node:child_process';
|
|
2
|
+
import { HttpResponse } from '../core/response.js';
|
|
3
|
+
import { NetworkError } from '../core/errors.js';
|
|
4
|
+
import { getCurlPath, hasImpersonate } from '../utils/binary-manager.js';
|
|
5
|
+
export class CurlTransport {
|
|
6
|
+
async dispatch(req) {
|
|
7
|
+
return new Promise(async (resolve, reject) => {
|
|
8
|
+
const args = [
|
|
9
|
+
'-X', req.method,
|
|
10
|
+
req.url,
|
|
11
|
+
'-i',
|
|
12
|
+
'-s',
|
|
13
|
+
'--compressed',
|
|
14
|
+
'--no-keepalive'
|
|
15
|
+
];
|
|
16
|
+
req.headers.forEach((val, key) => {
|
|
17
|
+
args.push('-H', `${key}: ${val}`);
|
|
18
|
+
});
|
|
19
|
+
if (req.body && typeof req.body === 'string') {
|
|
20
|
+
args.push('-d', req.body);
|
|
21
|
+
}
|
|
22
|
+
let command = process.env.RECKER_CURL_BIN;
|
|
23
|
+
if (!command) {
|
|
24
|
+
if (await hasImpersonate()) {
|
|
25
|
+
command = getCurlPath();
|
|
26
|
+
}
|
|
27
|
+
else {
|
|
28
|
+
command = 'curl';
|
|
29
|
+
}
|
|
30
|
+
}
|
|
31
|
+
const child = spawn(command, args);
|
|
32
|
+
const stdoutChunks = [];
|
|
33
|
+
const stderrChunks = [];
|
|
34
|
+
child.stdout.on('data', (chunk) => stdoutChunks.push(chunk));
|
|
35
|
+
child.stderr.on('data', (chunk) => stderrChunks.push(chunk));
|
|
36
|
+
child.on('error', (err) => {
|
|
37
|
+
reject(new NetworkError(`Failed to spawn curl: ${err.message}`, 'ERR_CURL_SPAWN', req));
|
|
38
|
+
});
|
|
39
|
+
child.on('close', (code) => {
|
|
40
|
+
if (code !== 0) {
|
|
41
|
+
const stderr = Buffer.concat(stderrChunks).toString();
|
|
42
|
+
reject(new NetworkError(`Curl exited with code ${code}: ${stderr}`, 'ERR_CURL_EXIT', req));
|
|
43
|
+
return;
|
|
44
|
+
}
|
|
45
|
+
const fullOutput = Buffer.concat(stdoutChunks);
|
|
46
|
+
let headerEndIndex = fullOutput.indexOf('\r\n\r\n');
|
|
47
|
+
let offset = 4;
|
|
48
|
+
if (headerEndIndex === -1) {
|
|
49
|
+
headerEndIndex = fullOutput.indexOf('\n\n');
|
|
50
|
+
offset = 2;
|
|
51
|
+
}
|
|
52
|
+
if (headerEndIndex !== -1) {
|
|
53
|
+
const firstLine = fullOutput.subarray(0, Math.min(20, headerEndIndex)).toString();
|
|
54
|
+
if (firstLine.startsWith('HTTP/1.1 100') || firstLine.startsWith('HTTP/2 100')) {
|
|
55
|
+
const nextStart = headerEndIndex + offset;
|
|
56
|
+
const secondSplit = fullOutput.indexOf('\r\n\r\n', nextStart);
|
|
57
|
+
if (secondSplit !== -1) {
|
|
58
|
+
headerEndIndex = secondSplit;
|
|
59
|
+
offset = 4;
|
|
60
|
+
}
|
|
61
|
+
}
|
|
62
|
+
}
|
|
63
|
+
if (headerEndIndex === -1) {
|
|
64
|
+
const nativeResponse = new Response(fullOutput, { status: 200, statusText: 'OK' });
|
|
65
|
+
resolve(new HttpResponse(nativeResponse, { connection: { protocol: 'curl' } }));
|
|
66
|
+
return;
|
|
67
|
+
}
|
|
68
|
+
const headerBlock = fullOutput.subarray(0, headerEndIndex).toString();
|
|
69
|
+
const bodyBlock = fullOutput.subarray(headerEndIndex + offset);
|
|
70
|
+
const headerLines = headerBlock.split(/\r?\n/);
|
|
71
|
+
const statusLine = headerLines[0];
|
|
72
|
+
let status = 200;
|
|
73
|
+
let statusText = 'OK';
|
|
74
|
+
const statusMatch = statusLine.match(/HTTP\/[\d\.]+ (\d+) ?(.*)/);
|
|
75
|
+
if (statusMatch) {
|
|
76
|
+
status = parseInt(statusMatch[1], 10);
|
|
77
|
+
statusText = statusMatch[2] || '';
|
|
78
|
+
}
|
|
79
|
+
const headers = new Headers();
|
|
80
|
+
for (let i = 1; i < headerLines.length; i++) {
|
|
81
|
+
const line = headerLines[i];
|
|
82
|
+
const colon = line.indexOf(':');
|
|
83
|
+
if (colon > 0) {
|
|
84
|
+
const key = line.substring(0, colon).trim();
|
|
85
|
+
const val = line.substring(colon + 1).trim();
|
|
86
|
+
headers.append(key, val);
|
|
87
|
+
}
|
|
88
|
+
}
|
|
89
|
+
const nativeResponse = new Response(bodyBlock, {
|
|
90
|
+
status,
|
|
91
|
+
statusText,
|
|
92
|
+
headers
|
|
93
|
+
});
|
|
94
|
+
resolve(new HttpResponse(nativeResponse, {
|
|
95
|
+
timings: {},
|
|
96
|
+
connection: { protocol: 'curl' }
|
|
97
|
+
}));
|
|
98
|
+
});
|
|
99
|
+
});
|
|
100
|
+
}
|
|
101
|
+
}
|
package/dist/transport/udp.js
CHANGED
package/dist/transport/undici.js
CHANGED
|
@@ -852,8 +852,7 @@ function wrapDownloadResponse(response, onProgress) {
|
|
|
852
852
|
});
|
|
853
853
|
return new Response(webBody, {
|
|
854
854
|
status: response.statusCode,
|
|
855
|
-
|
|
856
|
-
headers: response.headers
|
|
855
|
+
headers: response.headers,
|
|
857
856
|
});
|
|
858
857
|
}
|
|
859
858
|
function wrapUploadBody(body, onProgress, total) {
|
|
@@ -0,0 +1,18 @@
|
|
|
1
|
+
import type { ReckerRequest, ReckerResponse, Transport } from '../types/index.js';
|
|
2
|
+
export interface WorkerTransportOptions {
|
|
3
|
+
poolSize?: number;
|
|
4
|
+
}
|
|
5
|
+
export declare class WorkerTransport implements Transport {
|
|
6
|
+
private options;
|
|
7
|
+
private workers;
|
|
8
|
+
private workerIndex;
|
|
9
|
+
private pendingRequests;
|
|
10
|
+
private workerUrl;
|
|
11
|
+
constructor(options?: WorkerTransportOptions);
|
|
12
|
+
static isSupported(): boolean;
|
|
13
|
+
private createWorker;
|
|
14
|
+
private getNextWorker;
|
|
15
|
+
dispatch(req: ReckerRequest): Promise<ReckerResponse>;
|
|
16
|
+
terminate(): void;
|
|
17
|
+
get pendingCount(): number;
|
|
18
|
+
}
|
|
@@ -0,0 +1,278 @@
|
|
|
1
|
+
const WORKER_SCRIPT = `
|
|
2
|
+
self.onmessage = async (event) => {
|
|
3
|
+
const { id, method, url, headers, body, timeout } = event.data;
|
|
4
|
+
|
|
5
|
+
try {
|
|
6
|
+
const controller = new AbortController();
|
|
7
|
+
let timeoutId;
|
|
8
|
+
|
|
9
|
+
if (timeout) {
|
|
10
|
+
timeoutId = setTimeout(() => controller.abort(), timeout);
|
|
11
|
+
}
|
|
12
|
+
|
|
13
|
+
const start = performance.now();
|
|
14
|
+
|
|
15
|
+
const response = await fetch(url, {
|
|
16
|
+
method,
|
|
17
|
+
headers,
|
|
18
|
+
body,
|
|
19
|
+
signal: controller.signal,
|
|
20
|
+
keepalive: true
|
|
21
|
+
});
|
|
22
|
+
|
|
23
|
+
if (timeoutId) clearTimeout(timeoutId);
|
|
24
|
+
|
|
25
|
+
const totalTime = performance.now() - start;
|
|
26
|
+
|
|
27
|
+
// Read response body as ArrayBuffer for transferability
|
|
28
|
+
const arrayBuffer = await response.arrayBuffer();
|
|
29
|
+
|
|
30
|
+
// Serialize headers
|
|
31
|
+
const responseHeaders = {};
|
|
32
|
+
response.headers.forEach((value, key) => {
|
|
33
|
+
responseHeaders[key] = value;
|
|
34
|
+
});
|
|
35
|
+
|
|
36
|
+
self.postMessage({
|
|
37
|
+
id,
|
|
38
|
+
success: true,
|
|
39
|
+
status: response.status,
|
|
40
|
+
statusText: response.statusText,
|
|
41
|
+
headers: responseHeaders,
|
|
42
|
+
ok: response.ok,
|
|
43
|
+
url: response.url,
|
|
44
|
+
body: arrayBuffer,
|
|
45
|
+
timings: { total: totalTime, firstByte: totalTime }
|
|
46
|
+
}, [arrayBuffer]);
|
|
47
|
+
|
|
48
|
+
} catch (error) {
|
|
49
|
+
self.postMessage({
|
|
50
|
+
id,
|
|
51
|
+
success: false,
|
|
52
|
+
error: {
|
|
53
|
+
name: error.name,
|
|
54
|
+
message: error.message
|
|
55
|
+
}
|
|
56
|
+
});
|
|
57
|
+
}
|
|
58
|
+
};
|
|
59
|
+
`;
|
|
60
|
+
export class WorkerTransport {
|
|
61
|
+
options;
|
|
62
|
+
workers = [];
|
|
63
|
+
workerIndex = 0;
|
|
64
|
+
pendingRequests = new Map();
|
|
65
|
+
workerUrl;
|
|
66
|
+
constructor(options = {}) {
|
|
67
|
+
this.options = options;
|
|
68
|
+
const poolSize = options.poolSize ?? (typeof navigator !== 'undefined' ? navigator.hardwareConcurrency : 4) ?? 4;
|
|
69
|
+
const blob = new Blob([WORKER_SCRIPT], { type: 'application/javascript' });
|
|
70
|
+
this.workerUrl = URL.createObjectURL(blob);
|
|
71
|
+
for (let i = 0; i < poolSize; i++) {
|
|
72
|
+
this.createWorker();
|
|
73
|
+
}
|
|
74
|
+
}
|
|
75
|
+
static isSupported() {
|
|
76
|
+
return typeof Worker !== 'undefined';
|
|
77
|
+
}
|
|
78
|
+
createWorker() {
|
|
79
|
+
const worker = new Worker(this.workerUrl);
|
|
80
|
+
worker.onmessage = (event) => {
|
|
81
|
+
const { id, success, error, ...response } = event.data;
|
|
82
|
+
const pending = this.pendingRequests.get(id);
|
|
83
|
+
if (!pending)
|
|
84
|
+
return;
|
|
85
|
+
this.pendingRequests.delete(id);
|
|
86
|
+
if (success) {
|
|
87
|
+
pending.resolve(new WorkerResponseWrapper(response));
|
|
88
|
+
}
|
|
89
|
+
else {
|
|
90
|
+
const err = new Error(error?.message || 'Worker request failed');
|
|
91
|
+
err.name = error?.name || 'WorkerError';
|
|
92
|
+
pending.reject(err);
|
|
93
|
+
}
|
|
94
|
+
};
|
|
95
|
+
worker.onerror = (event) => {
|
|
96
|
+
console.error('[WorkerTransport] Worker error:', event.message);
|
|
97
|
+
};
|
|
98
|
+
this.workers.push(worker);
|
|
99
|
+
}
|
|
100
|
+
getNextWorker() {
|
|
101
|
+
const worker = this.workers[this.workerIndex];
|
|
102
|
+
this.workerIndex = (this.workerIndex + 1) % this.workers.length;
|
|
103
|
+
return worker;
|
|
104
|
+
}
|
|
105
|
+
async dispatch(req) {
|
|
106
|
+
if (!WorkerTransport.isSupported()) {
|
|
107
|
+
throw new Error('Web Workers are not supported in this environment');
|
|
108
|
+
}
|
|
109
|
+
const id = crypto.randomUUID();
|
|
110
|
+
const worker = this.getNextWorker();
|
|
111
|
+
const timeoutMs = typeof req.timeout === 'number'
|
|
112
|
+
? req.timeout
|
|
113
|
+
: req.timeout?.request;
|
|
114
|
+
const headers = {};
|
|
115
|
+
if (req.headers) {
|
|
116
|
+
if (req.headers instanceof Headers) {
|
|
117
|
+
req.headers.forEach((value, key) => {
|
|
118
|
+
headers[key] = value;
|
|
119
|
+
});
|
|
120
|
+
}
|
|
121
|
+
else if (typeof req.headers === 'object') {
|
|
122
|
+
Object.assign(headers, req.headers);
|
|
123
|
+
}
|
|
124
|
+
}
|
|
125
|
+
let body;
|
|
126
|
+
if (req.body) {
|
|
127
|
+
if (typeof req.body === 'string') {
|
|
128
|
+
body = req.body;
|
|
129
|
+
}
|
|
130
|
+
else if (req.body instanceof ArrayBuffer) {
|
|
131
|
+
body = req.body;
|
|
132
|
+
}
|
|
133
|
+
else if (req.body instanceof Blob) {
|
|
134
|
+
body = await req.body.arrayBuffer();
|
|
135
|
+
}
|
|
136
|
+
else if (typeof req.body === 'object') {
|
|
137
|
+
body = JSON.stringify(req.body);
|
|
138
|
+
}
|
|
139
|
+
}
|
|
140
|
+
return new Promise((resolve, reject) => {
|
|
141
|
+
this.pendingRequests.set(id, { resolve, reject });
|
|
142
|
+
if (req.signal) {
|
|
143
|
+
req.signal.addEventListener('abort', () => {
|
|
144
|
+
const pending = this.pendingRequests.get(id);
|
|
145
|
+
if (pending) {
|
|
146
|
+
this.pendingRequests.delete(id);
|
|
147
|
+
const err = new Error('Request aborted');
|
|
148
|
+
err.name = 'AbortError';
|
|
149
|
+
pending.reject(err);
|
|
150
|
+
}
|
|
151
|
+
});
|
|
152
|
+
}
|
|
153
|
+
worker.postMessage({
|
|
154
|
+
id,
|
|
155
|
+
method: req.method,
|
|
156
|
+
url: req.url,
|
|
157
|
+
headers,
|
|
158
|
+
body,
|
|
159
|
+
timeout: timeoutMs
|
|
160
|
+
});
|
|
161
|
+
});
|
|
162
|
+
}
|
|
163
|
+
terminate() {
|
|
164
|
+
for (const worker of this.workers) {
|
|
165
|
+
worker.terminate();
|
|
166
|
+
}
|
|
167
|
+
this.workers = [];
|
|
168
|
+
URL.revokeObjectURL(this.workerUrl);
|
|
169
|
+
for (const [id, pending] of this.pendingRequests) {
|
|
170
|
+
pending.reject(new Error('Transport terminated'));
|
|
171
|
+
}
|
|
172
|
+
this.pendingRequests.clear();
|
|
173
|
+
}
|
|
174
|
+
get pendingCount() {
|
|
175
|
+
return this.pendingRequests.size;
|
|
176
|
+
}
|
|
177
|
+
}
|
|
178
|
+
class WorkerResponseWrapper {
|
|
179
|
+
status;
|
|
180
|
+
statusText;
|
|
181
|
+
headers;
|
|
182
|
+
ok;
|
|
183
|
+
url;
|
|
184
|
+
timings;
|
|
185
|
+
connection = {};
|
|
186
|
+
raw;
|
|
187
|
+
bodyBuffer;
|
|
188
|
+
bodyUsed = false;
|
|
189
|
+
constructor(data) {
|
|
190
|
+
this.status = data.status ?? 0;
|
|
191
|
+
this.statusText = data.statusText ?? '';
|
|
192
|
+
this.ok = data.ok ?? false;
|
|
193
|
+
this.url = data.url ?? '';
|
|
194
|
+
this.timings = data.timings ?? { total: 0 };
|
|
195
|
+
this.bodyBuffer = data.body ?? new ArrayBuffer(0);
|
|
196
|
+
this.headers = new Headers();
|
|
197
|
+
if (data.headers) {
|
|
198
|
+
for (const [key, value] of Object.entries(data.headers)) {
|
|
199
|
+
this.headers.set(key, value);
|
|
200
|
+
}
|
|
201
|
+
}
|
|
202
|
+
this.raw = new Response(this.bodyBuffer, {
|
|
203
|
+
status: this.status,
|
|
204
|
+
statusText: this.statusText,
|
|
205
|
+
headers: this.headers,
|
|
206
|
+
});
|
|
207
|
+
}
|
|
208
|
+
checkBodyUsed() {
|
|
209
|
+
if (this.bodyUsed) {
|
|
210
|
+
throw new Error('Body has already been consumed');
|
|
211
|
+
}
|
|
212
|
+
this.bodyUsed = true;
|
|
213
|
+
}
|
|
214
|
+
async json() {
|
|
215
|
+
this.checkBodyUsed();
|
|
216
|
+
const text = new TextDecoder().decode(this.bodyBuffer);
|
|
217
|
+
return JSON.parse(text);
|
|
218
|
+
}
|
|
219
|
+
async text() {
|
|
220
|
+
this.checkBodyUsed();
|
|
221
|
+
return new TextDecoder().decode(this.bodyBuffer);
|
|
222
|
+
}
|
|
223
|
+
async blob() {
|
|
224
|
+
this.checkBodyUsed();
|
|
225
|
+
return new Blob([this.bodyBuffer]);
|
|
226
|
+
}
|
|
227
|
+
async cleanText() {
|
|
228
|
+
const text = await this.text();
|
|
229
|
+
return text.replace(/<[^>]*>?/gm, '');
|
|
230
|
+
}
|
|
231
|
+
read() {
|
|
232
|
+
if (this.bodyUsed)
|
|
233
|
+
return null;
|
|
234
|
+
this.bodyUsed = true;
|
|
235
|
+
const buffer = this.bodyBuffer;
|
|
236
|
+
return new ReadableStream({
|
|
237
|
+
start(controller) {
|
|
238
|
+
controller.enqueue(new Uint8Array(buffer));
|
|
239
|
+
controller.close();
|
|
240
|
+
}
|
|
241
|
+
});
|
|
242
|
+
}
|
|
243
|
+
clone() {
|
|
244
|
+
const clonedBuffer = this.bodyBuffer.slice(0);
|
|
245
|
+
const headersObj = {};
|
|
246
|
+
this.headers.forEach((value, key) => {
|
|
247
|
+
headersObj[key] = value;
|
|
248
|
+
});
|
|
249
|
+
return new WorkerResponseWrapper({
|
|
250
|
+
status: this.status,
|
|
251
|
+
statusText: this.statusText,
|
|
252
|
+
ok: this.ok,
|
|
253
|
+
url: this.url,
|
|
254
|
+
timings: this.timings,
|
|
255
|
+
body: clonedBuffer,
|
|
256
|
+
headers: headersObj,
|
|
257
|
+
});
|
|
258
|
+
}
|
|
259
|
+
async *sse() {
|
|
260
|
+
throw new Error('SSE is not supported in WorkerTransport. Use FetchTransport for SSE.');
|
|
261
|
+
}
|
|
262
|
+
async *download() {
|
|
263
|
+
const total = this.bodyBuffer.byteLength;
|
|
264
|
+
yield {
|
|
265
|
+
loaded: total,
|
|
266
|
+
transferred: total,
|
|
267
|
+
total,
|
|
268
|
+
percent: 100,
|
|
269
|
+
direction: 'download'
|
|
270
|
+
};
|
|
271
|
+
}
|
|
272
|
+
async *[Symbol.asyncIterator]() {
|
|
273
|
+
if (this.bodyUsed)
|
|
274
|
+
return;
|
|
275
|
+
this.bodyUsed = true;
|
|
276
|
+
yield new Uint8Array(this.bodyBuffer);
|
|
277
|
+
}
|
|
278
|
+
}
|
package/dist/types/index.d.ts
CHANGED
|
@@ -58,8 +58,9 @@ export interface ReckerRequest {
|
|
|
58
58
|
maxRedirects?: number;
|
|
59
59
|
followRedirects?: boolean;
|
|
60
60
|
http2?: boolean;
|
|
61
|
+
useCurl?: boolean;
|
|
62
|
+
userAgent?: string;
|
|
61
63
|
withHeader(name: string, value: string): ReckerRequest;
|
|
62
|
-
withBody(body: BodyInit): ReckerRequest;
|
|
63
64
|
_hooks?: {
|
|
64
65
|
onDnsLookup?: (info: any) => void;
|
|
65
66
|
onTcpConnect?: (info: any) => void;
|
|
@@ -329,6 +330,8 @@ export interface ClientOptions {
|
|
|
329
330
|
jitter?: boolean;
|
|
330
331
|
statusCodes?: number[];
|
|
331
332
|
};
|
|
333
|
+
useCurl?: boolean;
|
|
334
|
+
userAgent?: string;
|
|
332
335
|
}
|
|
333
336
|
export interface ReckerResponse<T = unknown> {
|
|
334
337
|
status: number;
|
|
@@ -0,0 +1,72 @@
|
|
|
1
|
+
import { promises as fs } from 'node:fs';
|
|
2
|
+
import { join } from 'node:path';
|
|
3
|
+
import { homedir, platform, arch } from 'node:os';
|
|
4
|
+
import { spawn } from 'node:child_process';
|
|
5
|
+
import { createWriteStream } from 'node:fs';
|
|
6
|
+
import { Readable } from 'node:stream';
|
|
7
|
+
import { finished } from 'node:stream/promises';
|
|
8
|
+
const BIN_DIR = join(homedir(), '.recker', 'bin');
|
|
9
|
+
const VERSION = 'v0.6.1';
|
|
10
|
+
export function getCurlBinName() {
|
|
11
|
+
return 'curl-impersonate-chrome';
|
|
12
|
+
}
|
|
13
|
+
export function getCurlPath() {
|
|
14
|
+
return join(BIN_DIR, getCurlBinName());
|
|
15
|
+
}
|
|
16
|
+
export async function hasImpersonate() {
|
|
17
|
+
try {
|
|
18
|
+
await fs.access(getCurlPath());
|
|
19
|
+
return true;
|
|
20
|
+
}
|
|
21
|
+
catch {
|
|
22
|
+
return false;
|
|
23
|
+
}
|
|
24
|
+
}
|
|
25
|
+
function getDownloadUrl() {
|
|
26
|
+
const p = platform();
|
|
27
|
+
const a = arch();
|
|
28
|
+
const baseUrl = `https://github.com/lwthiker/curl-impersonate/releases/download/${VERSION}`;
|
|
29
|
+
if (p === 'linux') {
|
|
30
|
+
if (a === 'x64')
|
|
31
|
+
return `${baseUrl}/curl-impersonate-${VERSION}.x86_64-linux-gnu.tar.gz`;
|
|
32
|
+
if (a === 'arm64')
|
|
33
|
+
return `${baseUrl}/curl-impersonate-${VERSION}.aarch64-linux-gnu.tar.gz`;
|
|
34
|
+
}
|
|
35
|
+
if (p === 'darwin') {
|
|
36
|
+
throw new Error('Auto-install not yet supported on macOS. Please install curl-impersonate manually.');
|
|
37
|
+
}
|
|
38
|
+
if (p === 'win32') {
|
|
39
|
+
throw new Error('Auto-install not yet supported on Windows.');
|
|
40
|
+
}
|
|
41
|
+
throw new Error(`Unsupported platform: ${p} ${a}`);
|
|
42
|
+
}
|
|
43
|
+
export async function installCurlImpersonate(logger = console) {
|
|
44
|
+
const url = getDownloadUrl();
|
|
45
|
+
const tarPath = join(BIN_DIR, 'curl-impersonate.tar.gz');
|
|
46
|
+
await fs.mkdir(BIN_DIR, { recursive: true });
|
|
47
|
+
logger.log(`Downloading curl-impersonate from ${url}...`);
|
|
48
|
+
const res = await fetch(url);
|
|
49
|
+
if (!res.ok)
|
|
50
|
+
throw new Error(`Failed to download: ${res.statusText}`);
|
|
51
|
+
if (!res.body)
|
|
52
|
+
throw new Error('Empty body');
|
|
53
|
+
const fileStream = createWriteStream(tarPath);
|
|
54
|
+
await finished(Readable.fromWeb(res.body).pipe(fileStream));
|
|
55
|
+
logger.log('Extracting...');
|
|
56
|
+
await new Promise((resolve, reject) => {
|
|
57
|
+
const tar = spawn('tar', ['-xzf', tarPath, '-C', BIN_DIR]);
|
|
58
|
+
tar.on('close', (code) => {
|
|
59
|
+
if (code === 0)
|
|
60
|
+
resolve();
|
|
61
|
+
else
|
|
62
|
+
reject(new Error(`Tar exited with code ${code}`));
|
|
63
|
+
});
|
|
64
|
+
});
|
|
65
|
+
await fs.unlink(tarPath);
|
|
66
|
+
if (await hasImpersonate()) {
|
|
67
|
+
logger.log(`✅ Installed to ${getCurlPath()}`);
|
|
68
|
+
}
|
|
69
|
+
else {
|
|
70
|
+
throw new Error('Installation failed: Binary not found after extraction');
|
|
71
|
+
}
|
|
72
|
+
}
|
|
@@ -1,10 +1,10 @@
|
|
|
1
|
-
|
|
2
|
-
|
|
3
|
-
|
|
4
|
-
|
|
5
|
-
|
|
6
|
-
}
|
|
7
|
-
|
|
1
|
+
export declare const OPTIONAL_DEPENDENCIES: {
|
|
2
|
+
readonly cardinal: {
|
|
3
|
+
readonly package: "cardinal";
|
|
4
|
+
readonly feature: "syntax highlighting in CLI";
|
|
5
|
+
readonly version: "2.x";
|
|
6
|
+
};
|
|
7
|
+
};
|
|
8
8
|
export declare class MissingDependencyError extends Error {
|
|
9
9
|
readonly packageName: string;
|
|
10
10
|
readonly submodule: string;
|
|
@@ -16,4 +16,3 @@ export declare function requireOptional<T = unknown>(packageName: string, submod
|
|
|
16
16
|
export declare function isPackageAvailable(packageName: string): boolean;
|
|
17
17
|
export declare function getInstallCommand(packages: string[]): string;
|
|
18
18
|
export declare function requireOptionalMany<T extends Record<string, unknown>>(packages: string[], submodule: string): Promise<T>;
|
|
19
|
-
export {};
|