recker 1.0.43 → 1.0.44
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +47 -0
- package/dist/bin/recker-linux-x64 +0 -0
- package/dist/bin/recker-macos-x64 +0 -0
- package/dist/bin/recker-win-x64.exe +0 -0
- package/dist/bin/rek.cjs +85152 -100207
- package/dist/browser/ai/adaptive-timeout.d.ts +50 -0
- package/dist/browser/ai/adaptive-timeout.js +208 -0
- package/dist/browser/ai/client.d.ts +22 -0
- package/dist/browser/ai/client.js +294 -0
- package/dist/browser/ai/index.d.ts +14 -0
- package/dist/browser/ai/index.js +11 -0
- package/dist/browser/ai/providers/anthropic.d.ts +63 -0
- package/dist/browser/ai/providers/anthropic.js +370 -0
- package/dist/browser/ai/providers/base.d.ts +48 -0
- package/dist/browser/ai/providers/base.js +150 -0
- package/dist/browser/ai/providers/google.d.ts +59 -0
- package/dist/browser/ai/providers/google.js +305 -0
- package/dist/browser/ai/providers/ollama.d.ts +44 -0
- package/dist/browser/ai/providers/ollama.js +240 -0
- package/dist/browser/ai/providers/openai.d.ts +64 -0
- package/dist/browser/ai/providers/openai.js +298 -0
- package/dist/browser/ai/rate-limiter.d.ts +43 -0
- package/dist/browser/ai/rate-limiter.js +215 -0
- package/dist/browser/ai/vector/index.d.ts +2 -0
- package/dist/browser/ai/vector/index.js +2 -0
- package/dist/browser/ai/vector/similarity.d.ts +2 -0
- package/dist/browser/ai/vector/similarity.js +27 -0
- package/dist/browser/ai/vector/store.d.ts +27 -0
- package/dist/browser/ai/vector/store.js +82 -0
- package/dist/browser/browser/cache.d.ts +2 -40
- package/dist/browser/browser/cache.js +2 -199
- package/dist/browser/browser/index.d.ts +8 -0
- package/dist/browser/browser/index.js +8 -0
- package/dist/browser/browser/recker.d.ts +8 -1
- package/dist/browser/browser/recker.js +8 -2
- package/dist/browser/cache/indexed-db.d.ts +10 -0
- package/dist/browser/cache/indexed-db.js +88 -0
- package/dist/browser/cache/service-worker-cache.d.ts +18 -0
- package/dist/browser/cache/service-worker-cache.js +103 -0
- package/dist/browser/cache.d.ts +2 -40
- package/dist/browser/cache.js +2 -199
- package/dist/browser/constants/user-agents.d.ts +7 -0
- package/dist/browser/constants/user-agents.js +7 -0
- package/dist/browser/core/client.d.ts +2 -0
- package/dist/browser/core/client.js +19 -1
- package/dist/browser/index.d.ts +8 -0
- package/dist/browser/index.js +8 -0
- package/dist/browser/plugins/har-recorder.d.ts +40 -0
- package/dist/browser/plugins/har-recorder.js +120 -0
- package/dist/browser/plugins/network-simulation.d.ts +7 -0
- package/dist/browser/plugins/network-simulation.js +13 -0
- package/dist/browser/presets/android.d.ts +2 -0
- package/dist/browser/presets/android.js +16 -0
- package/dist/browser/presets/anthropic.d.ts +8 -0
- package/dist/browser/presets/anthropic.js +27 -0
- package/dist/browser/presets/aws.d.ts +19 -0
- package/dist/browser/presets/aws.js +68 -0
- package/dist/browser/presets/azure-openai.d.ts +10 -0
- package/dist/browser/presets/azure-openai.js +35 -0
- package/dist/browser/presets/azure.d.ts +41 -0
- package/dist/browser/presets/azure.js +104 -0
- package/dist/browser/presets/chaturbate.d.ts +2 -0
- package/dist/browser/presets/chaturbate.js +17 -0
- package/dist/browser/presets/cloudflare.d.ts +12 -0
- package/dist/browser/presets/cloudflare.js +39 -0
- package/dist/browser/presets/cohere.d.ts +7 -0
- package/dist/browser/presets/cohere.js +22 -0
- package/dist/browser/presets/deepseek.d.ts +7 -0
- package/dist/browser/presets/deepseek.js +22 -0
- package/dist/browser/presets/digitalocean.d.ts +5 -0
- package/dist/browser/presets/digitalocean.js +16 -0
- package/dist/browser/presets/discord.d.ts +6 -0
- package/dist/browser/presets/discord.js +17 -0
- package/dist/browser/presets/elevenlabs.d.ts +6 -0
- package/dist/browser/presets/elevenlabs.js +20 -0
- package/dist/browser/presets/enhancers.d.ts +20 -0
- package/dist/browser/presets/enhancers.js +85 -0
- package/dist/browser/presets/fireworks.d.ts +7 -0
- package/dist/browser/presets/fireworks.js +22 -0
- package/dist/browser/presets/gcp.d.ts +34 -0
- package/dist/browser/presets/gcp.js +91 -0
- package/dist/browser/presets/gemini.d.ts +7 -0
- package/dist/browser/presets/gemini.js +23 -0
- package/dist/browser/presets/github.d.ts +6 -0
- package/dist/browser/presets/github.js +17 -0
- package/dist/browser/presets/gitlab.d.ts +6 -0
- package/dist/browser/presets/gitlab.js +16 -0
- package/dist/browser/presets/groq.d.ts +7 -0
- package/dist/browser/presets/groq.js +22 -0
- package/dist/browser/presets/hubspot.d.ts +9 -0
- package/dist/browser/presets/hubspot.js +28 -0
- package/dist/browser/presets/huggingface.d.ts +7 -0
- package/dist/browser/presets/huggingface.js +23 -0
- package/dist/browser/presets/index.d.ts +47 -0
- package/dist/browser/presets/index.js +47 -0
- package/dist/browser/presets/ios.d.ts +2 -0
- package/dist/browser/presets/ios.js +13 -0
- package/dist/browser/presets/linear.d.ts +5 -0
- package/dist/browser/presets/linear.js +16 -0
- package/dist/browser/presets/mailgun.d.ts +7 -0
- package/dist/browser/presets/mailgun.js +20 -0
- package/dist/browser/presets/meta.d.ts +10 -0
- package/dist/browser/presets/meta.js +33 -0
- package/dist/browser/presets/mistral.d.ts +7 -0
- package/dist/browser/presets/mistral.js +22 -0
- package/dist/browser/presets/notion.d.ts +6 -0
- package/dist/browser/presets/notion.js +17 -0
- package/dist/browser/presets/openai.d.ts +9 -0
- package/dist/browser/presets/openai.js +30 -0
- package/dist/browser/presets/oracle.d.ts +19 -0
- package/dist/browser/presets/oracle.js +117 -0
- package/dist/browser/presets/perplexity.d.ts +7 -0
- package/dist/browser/presets/perplexity.js +22 -0
- package/dist/browser/presets/pinecone.d.ts +8 -0
- package/dist/browser/presets/pinecone.js +42 -0
- package/dist/browser/presets/registry.d.ts +23 -0
- package/dist/browser/presets/registry.js +519 -0
- package/dist/browser/presets/replicate.d.ts +7 -0
- package/dist/browser/presets/replicate.js +23 -0
- package/dist/browser/presets/sendgrid.d.ts +6 -0
- package/dist/browser/presets/sendgrid.js +20 -0
- package/dist/browser/presets/sentry.d.ts +11 -0
- package/dist/browser/presets/sentry.js +48 -0
- package/dist/browser/presets/sinch.d.ts +9 -0
- package/dist/browser/presets/sinch.js +39 -0
- package/dist/browser/presets/slack.d.ts +5 -0
- package/dist/browser/presets/slack.js +16 -0
- package/dist/browser/presets/square.d.ts +10 -0
- package/dist/browser/presets/square.js +33 -0
- package/dist/browser/presets/stripe.d.ts +7 -0
- package/dist/browser/presets/stripe.js +23 -0
- package/dist/browser/presets/supabase.d.ts +6 -0
- package/dist/browser/presets/supabase.js +18 -0
- package/dist/browser/presets/tiktok.d.ts +10 -0
- package/dist/browser/presets/tiktok.js +38 -0
- package/dist/browser/presets/together.d.ts +7 -0
- package/dist/browser/presets/together.js +22 -0
- package/dist/browser/presets/twilio.d.ts +6 -0
- package/dist/browser/presets/twilio.js +17 -0
- package/dist/browser/presets/vercel.d.ts +6 -0
- package/dist/browser/presets/vercel.js +23 -0
- package/dist/browser/presets/vultr.d.ts +5 -0
- package/dist/browser/presets/vultr.js +16 -0
- package/dist/browser/presets/xai.d.ts +8 -0
- package/dist/browser/presets/xai.js +23 -0
- package/dist/browser/presets/youtube.d.ts +5 -0
- package/dist/browser/presets/youtube.js +20 -0
- package/dist/browser/recker.d.ts +8 -1
- package/dist/browser/recker.js +8 -2
- package/dist/browser/scrape/document.d.ts +5 -4
- package/dist/browser/scrape/document.js +89 -76
- package/dist/browser/scrape/element.d.ts +10 -8
- package/dist/browser/scrape/element.js +295 -81
- package/dist/browser/scrape/extractors.d.ts +11 -11
- package/dist/browser/scrape/extractors.js +145 -113
- package/dist/browser/scrape/parser/back.d.ts +1 -0
- package/dist/browser/scrape/parser/back.js +3 -0
- package/dist/browser/scrape/parser/index.d.ts +20 -0
- package/dist/browser/scrape/parser/index.js +19 -0
- package/dist/browser/scrape/parser/matcher.d.ts +30 -0
- package/dist/browser/scrape/parser/matcher.js +99 -0
- package/dist/browser/scrape/parser/nodes/comment.d.ts +12 -0
- package/dist/browser/scrape/parser/nodes/comment.js +21 -0
- package/dist/browser/scrape/parser/nodes/html.d.ts +110 -0
- package/dist/browser/scrape/parser/nodes/html.js +978 -0
- package/dist/browser/scrape/parser/nodes/node.d.ts +18 -0
- package/dist/browser/scrape/parser/nodes/node.js +31 -0
- package/dist/browser/scrape/parser/nodes/text.d.ts +14 -0
- package/dist/browser/scrape/parser/nodes/text.js +30 -0
- package/dist/browser/scrape/parser/nodes/type.d.ts +6 -0
- package/dist/browser/scrape/parser/nodes/type.js +7 -0
- package/dist/browser/scrape/parser/parse.d.ts +1 -0
- package/dist/browser/scrape/parser/parse.js +1 -0
- package/dist/browser/scrape/parser/valid.d.ts +2 -0
- package/dist/browser/scrape/parser/valid.js +5 -0
- package/dist/browser/scrape/parser/void-tag.d.ts +7 -0
- package/dist/browser/scrape/parser/void-tag.js +43 -0
- package/dist/browser/scrape/types.d.ts +7 -0
- package/dist/browser/seo/analyzer.d.ts +59 -0
- package/dist/browser/seo/analyzer.js +1399 -0
- package/dist/browser/seo/keywords.d.ts +16 -0
- package/dist/browser/seo/keywords.js +55 -0
- package/dist/browser/seo/rules/accessibility.d.ts +2 -0
- package/dist/browser/seo/rules/accessibility.js +733 -0
- package/dist/browser/seo/rules/ai-search.d.ts +2 -0
- package/dist/browser/seo/rules/ai-search.js +436 -0
- package/dist/browser/seo/rules/analytics.d.ts +2 -0
- package/dist/browser/seo/rules/analytics.js +306 -0
- package/dist/browser/seo/rules/best-practices.d.ts +2 -0
- package/dist/browser/seo/rules/best-practices.js +195 -0
- package/dist/browser/seo/rules/canonical.d.ts +12 -0
- package/dist/browser/seo/rules/canonical.js +270 -0
- package/dist/browser/seo/rules/content.d.ts +2 -0
- package/dist/browser/seo/rules/content.js +522 -0
- package/dist/browser/seo/rules/crawl.d.ts +2 -0
- package/dist/browser/seo/rules/crawl.js +435 -0
- package/dist/browser/seo/rules/cwv.d.ts +2 -0
- package/dist/browser/seo/rules/cwv.js +248 -0
- package/dist/browser/seo/rules/ecommerce.d.ts +2 -0
- package/dist/browser/seo/rules/ecommerce.js +312 -0
- package/dist/browser/seo/rules/i18n.d.ts +2 -0
- package/dist/browser/seo/rules/i18n.js +288 -0
- package/dist/browser/seo/rules/images.d.ts +2 -0
- package/dist/browser/seo/rules/images.js +255 -0
- package/dist/browser/seo/rules/index.d.ts +52 -0
- package/dist/browser/seo/rules/index.js +159 -0
- package/dist/browser/seo/rules/internal-linking.d.ts +2 -0
- package/dist/browser/seo/rules/internal-linking.js +394 -0
- package/dist/browser/seo/rules/links.d.ts +2 -0
- package/dist/browser/seo/rules/links.js +498 -0
- package/dist/browser/seo/rules/local.d.ts +2 -0
- package/dist/browser/seo/rules/local.js +289 -0
- package/dist/browser/seo/rules/meta.d.ts +2 -0
- package/dist/browser/seo/rules/meta.js +805 -0
- package/dist/browser/seo/rules/mobile.d.ts +2 -0
- package/dist/browser/seo/rules/mobile.js +161 -0
- package/dist/browser/seo/rules/performance.d.ts +2 -0
- package/dist/browser/seo/rules/performance.js +738 -0
- package/dist/browser/seo/rules/pwa.d.ts +2 -0
- package/dist/browser/seo/rules/pwa.js +299 -0
- package/dist/browser/seo/rules/readability.d.ts +2 -0
- package/dist/browser/seo/rules/readability.js +264 -0
- package/dist/browser/seo/rules/redirects.d.ts +16 -0
- package/dist/browser/seo/rules/redirects.js +199 -0
- package/dist/browser/seo/rules/resources.d.ts +2 -0
- package/dist/browser/seo/rules/resources.js +390 -0
- package/dist/browser/seo/rules/schema.d.ts +2 -0
- package/dist/browser/seo/rules/schema.js +379 -0
- package/dist/browser/seo/rules/security.d.ts +2 -0
- package/dist/browser/seo/rules/security.js +877 -0
- package/dist/browser/seo/rules/social.d.ts +2 -0
- package/dist/browser/seo/rules/social.js +603 -0
- package/dist/browser/seo/rules/structural.d.ts +2 -0
- package/dist/browser/seo/rules/structural.js +223 -0
- package/dist/browser/seo/rules/technical-advanced.d.ts +10 -0
- package/dist/browser/seo/rules/technical-advanced.js +289 -0
- package/dist/browser/seo/rules/technical.d.ts +2 -0
- package/dist/browser/seo/rules/technical.js +480 -0
- package/dist/browser/seo/rules/thresholds.d.ts +196 -0
- package/dist/browser/seo/rules/thresholds.js +118 -0
- package/dist/browser/seo/rules/types.d.ts +498 -0
- package/dist/browser/seo/rules/types.js +11 -0
- package/dist/browser/seo/types.d.ts +211 -0
- package/dist/browser/seo/types.js +1 -0
- package/dist/browser/transport/curl.d.ts +4 -0
- package/dist/browser/transport/curl.js +101 -0
- package/dist/browser/transport/undici.js +1 -2
- package/dist/browser/transport/worker.d.ts +18 -0
- package/dist/browser/transport/worker.js +278 -0
- package/dist/browser/types/index.d.ts +4 -1
- package/dist/browser/utils/binary-manager.d.ts +4 -0
- package/dist/browser/utils/binary-manager.js +72 -0
- package/dist/browser/utils/user-agent.js +2 -13
- package/dist/cache/indexed-db.d.ts +10 -0
- package/dist/cache/indexed-db.js +88 -0
- package/dist/cache/service-worker-cache.d.ts +18 -0
- package/dist/cache/service-worker-cache.js +103 -0
- package/dist/cli/commands/ai.d.ts +2 -0
- package/dist/cli/commands/ai.js +162 -0
- package/dist/cli/commands/bench.d.ts +2 -0
- package/dist/cli/commands/bench.js +51 -0
- package/dist/cli/commands/dns.d.ts +2 -0
- package/dist/cli/commands/dns.js +295 -0
- package/dist/cli/commands/har.d.ts +2 -0
- package/dist/cli/commands/har.js +171 -0
- package/dist/cli/commands/hls.d.ts +2 -0
- package/dist/cli/commands/hls.js +192 -0
- package/dist/cli/commands/network.d.ts +2 -0
- package/dist/cli/commands/network.js +288 -0
- package/dist/cli/commands/protocols.d.ts +2 -0
- package/dist/cli/commands/protocols.js +344 -0
- package/dist/cli/commands/scrape.d.ts +2 -0
- package/dist/cli/commands/scrape.js +176 -0
- package/dist/cli/commands/security.d.ts +2 -0
- package/dist/cli/commands/security.js +57 -0
- package/dist/cli/commands/seo.d.ts +2 -0
- package/dist/cli/commands/seo.js +125 -0
- package/dist/cli/commands/serve.d.ts +2 -0
- package/dist/cli/commands/serve.js +531 -0
- package/dist/cli/commands/spider.d.ts +3 -0
- package/dist/cli/commands/spider.js +456 -0
- package/dist/cli/commands/utils.d.ts +2 -0
- package/dist/cli/commands/utils.js +176 -0
- package/dist/cli/commands/vector.d.ts +2 -0
- package/dist/cli/commands/vector.js +158 -0
- package/dist/cli/handler.d.ts +2 -2
- package/dist/cli/handler.js +6 -6
- package/dist/cli/helpers.d.ts +7 -0
- package/dist/cli/helpers.js +128 -0
- package/dist/cli/index.js +96 -5228
- package/dist/cli/parser/help.d.ts +2 -0
- package/dist/cli/parser/help.js +52 -0
- package/dist/cli/parser/index.d.ts +3 -0
- package/dist/cli/parser/index.js +3 -0
- package/dist/cli/parser/parser.d.ts +4 -0
- package/dist/cli/parser/parser.js +146 -0
- package/dist/cli/parser/types.d.ts +41 -0
- package/dist/cli/parser/types.js +1 -0
- package/dist/cli/presets.d.ts +1 -1
- package/dist/cli/presets.js +1 -1
- package/dist/cli/router.d.ts +36 -0
- package/dist/cli/router.js +195 -0
- package/dist/cli/tui/ai-chat.js +1 -1
- package/dist/cli/tui/commands/context.d.ts +9 -0
- package/dist/cli/tui/commands/context.js +1 -0
- package/dist/cli/tui/commands/dns.d.ts +10 -0
- package/dist/cli/tui/commands/dns.js +461 -0
- package/dist/cli/tui/commands/hls.d.ts +2 -0
- package/dist/cli/tui/commands/hls.js +162 -0
- package/dist/cli/tui/commands/ip.d.ts +2 -0
- package/dist/cli/tui/commands/ip.js +45 -0
- package/dist/cli/tui/commands/network.d.ts +3 -0
- package/dist/cli/tui/commands/network.js +81 -0
- package/dist/cli/tui/commands/protocols.d.ts +6 -0
- package/dist/cli/tui/commands/protocols.js +531 -0
- package/dist/cli/tui/commands/security.d.ts +2 -0
- package/dist/cli/tui/commands/security.js +48 -0
- package/dist/cli/tui/commands/seo.d.ts +2 -0
- package/dist/cli/tui/commands/seo.js +74 -0
- package/dist/cli/tui/context.d.ts +12 -0
- package/dist/cli/tui/context.js +1 -0
- package/dist/cli/tui/shell.d.ts +11 -20
- package/dist/cli/tui/shell.js +216 -1873
- package/dist/constants/user-agents.d.ts +7 -0
- package/dist/constants/user-agents.js +7 -0
- package/dist/core/client.d.ts +2 -0
- package/dist/core/client.js +19 -1
- package/dist/index.d.ts +1 -0
- package/dist/index.js +1 -0
- package/dist/mcp/cli.js +2 -3
- package/dist/mcp/data/embeddings.json +1 -1
- package/dist/mcp/tools/network.js +298 -158
- package/dist/plugins/har-player.d.ts +23 -0
- package/dist/plugins/har-player.js +49 -0
- package/dist/plugins/har-recorder.d.ts +37 -3
- package/dist/plugins/har-recorder.js +116 -63
- package/dist/plugins/network-simulation.d.ts +7 -0
- package/dist/plugins/network-simulation.js +13 -0
- package/dist/presets/android.d.ts +2 -0
- package/dist/presets/android.js +16 -0
- package/dist/presets/chaturbate.d.ts +2 -0
- package/dist/presets/chaturbate.js +17 -0
- package/dist/presets/elevenlabs.d.ts +6 -0
- package/dist/presets/elevenlabs.js +20 -0
- package/dist/presets/enhancers.d.ts +20 -0
- package/dist/presets/enhancers.js +85 -0
- package/dist/presets/hubspot.d.ts +9 -0
- package/dist/presets/hubspot.js +28 -0
- package/dist/presets/index.d.ts +10 -0
- package/dist/presets/index.js +10 -0
- package/dist/presets/ios.d.ts +2 -0
- package/dist/presets/ios.js +13 -0
- package/dist/presets/pinecone.d.ts +8 -0
- package/dist/presets/pinecone.js +42 -0
- package/dist/presets/registry.js +60 -0
- package/dist/presets/sendgrid.d.ts +6 -0
- package/dist/presets/sendgrid.js +20 -0
- package/dist/presets/sentry.d.ts +11 -0
- package/dist/presets/sentry.js +48 -0
- package/dist/presets/square.d.ts +10 -0
- package/dist/presets/square.js +33 -0
- package/dist/recker.d.ts +3 -0
- package/dist/recker.js +4 -0
- package/dist/scrape/document.d.ts +5 -4
- package/dist/scrape/document.js +89 -76
- package/dist/scrape/element.d.ts +10 -8
- package/dist/scrape/element.js +295 -81
- package/dist/scrape/extractors.d.ts +11 -11
- package/dist/scrape/extractors.js +145 -113
- package/dist/scrape/index.d.ts +2 -0
- package/dist/scrape/index.js +1 -0
- package/dist/scrape/parser/back.d.ts +1 -0
- package/dist/scrape/parser/back.js +3 -0
- package/dist/scrape/parser/index.d.ts +20 -0
- package/dist/scrape/parser/index.js +19 -0
- package/dist/scrape/parser/matcher.d.ts +30 -0
- package/dist/scrape/parser/matcher.js +99 -0
- package/dist/scrape/parser/nodes/comment.d.ts +12 -0
- package/dist/scrape/parser/nodes/comment.js +21 -0
- package/dist/scrape/parser/nodes/html.d.ts +110 -0
- package/dist/scrape/parser/nodes/html.js +978 -0
- package/dist/scrape/parser/nodes/node.d.ts +18 -0
- package/dist/scrape/parser/nodes/node.js +31 -0
- package/dist/scrape/parser/nodes/text.d.ts +14 -0
- package/dist/scrape/parser/nodes/text.js +30 -0
- package/dist/scrape/parser/nodes/type.d.ts +6 -0
- package/dist/scrape/parser/nodes/type.js +7 -0
- package/dist/scrape/parser/parse.d.ts +1 -0
- package/dist/scrape/parser/parse.js +1 -0
- package/dist/scrape/parser/valid.d.ts +2 -0
- package/dist/scrape/parser/valid.js +5 -0
- package/dist/scrape/parser/void-tag.d.ts +7 -0
- package/dist/scrape/parser/void-tag.js +43 -0
- package/dist/scrape/spider.d.ts +19 -0
- package/dist/scrape/spider.js +28 -3
- package/dist/scrape/types.d.ts +7 -0
- package/dist/seo/analyzer.d.ts +15 -5
- package/dist/seo/analyzer.js +636 -175
- package/dist/seo/formatter.d.ts +16 -0
- package/dist/seo/formatter.js +228 -0
- package/dist/seo/index.d.ts +2 -0
- package/dist/seo/index.js +1 -0
- package/dist/seo/keywords.d.ts +16 -0
- package/dist/seo/keywords.js +55 -0
- package/dist/seo/rules/accessibility.js +96 -57
- package/dist/seo/rules/ai-search.js +44 -31
- package/dist/seo/rules/analytics.d.ts +2 -0
- package/dist/seo/rules/analytics.js +306 -0
- package/dist/seo/rules/best-practices.js +21 -14
- package/dist/seo/rules/canonical.js +53 -32
- package/dist/seo/rules/content.js +317 -31
- package/dist/seo/rules/crawl.js +55 -40
- package/dist/seo/rules/cwv.js +21 -15
- package/dist/seo/rules/ecommerce.js +82 -22
- package/dist/seo/rules/i18n.js +75 -36
- package/dist/seo/rules/images.js +109 -30
- package/dist/seo/rules/index.js +2 -0
- package/dist/seo/rules/internal-linking.js +58 -39
- package/dist/seo/rules/links.js +79 -52
- package/dist/seo/rules/local.js +49 -25
- package/dist/seo/rules/meta.js +339 -81
- package/dist/seo/rules/mobile.js +112 -2
- package/dist/seo/rules/performance.js +434 -66
- package/dist/seo/rules/pwa.js +36 -39
- package/dist/seo/rules/readability.js +31 -22
- package/dist/seo/rules/redirects.js +21 -15
- package/dist/seo/rules/resources.js +59 -42
- package/dist/seo/rules/schema.js +333 -8
- package/dist/seo/rules/security.js +142 -80
- package/dist/seo/rules/social.js +277 -47
- package/dist/seo/rules/structural.js +87 -19
- package/dist/seo/rules/technical-advanced.js +30 -24
- package/dist/seo/rules/technical.js +243 -42
- package/dist/seo/rules/types.d.ts +53 -1
- package/dist/seo/seo-spider.d.ts +22 -0
- package/dist/seo/seo-spider.js +77 -13
- package/dist/seo/types.d.ts +8 -1
- package/dist/seo/validators/llms-txt.js +19 -0
- package/dist/seo/validators/rss.d.ts +11 -0
- package/dist/seo/validators/rss.js +93 -0
- package/dist/seo/validators/sitemap.js +36 -26
- package/dist/transport/curl.d.ts +4 -0
- package/dist/transport/curl.js +101 -0
- package/dist/transport/udp.js +0 -1
- package/dist/transport/undici.js +1 -2
- package/dist/transport/worker.d.ts +18 -0
- package/dist/transport/worker.js +278 -0
- package/dist/types/index.d.ts +4 -1
- package/dist/utils/binary-manager.d.ts +4 -0
- package/dist/utils/binary-manager.js +72 -0
- package/dist/utils/optional-require.d.ts +7 -8
- package/dist/utils/optional-require.js +2 -21
- package/dist/utils/upload.d.ts +6 -0
- package/dist/utils/upload.js +11 -0
- package/dist/utils/user-agent.js +2 -13
- package/dist/version.js +1 -1
- package/package.json +12 -6
- package/dist/browser/utils/optional-require.d.ts +0 -19
- package/dist/browser/utils/optional-require.js +0 -105
|
@@ -0,0 +1,436 @@
|
|
|
1
|
+
import { createResult } from './types.js';
|
|
2
|
+
export const aiSearchRules = [
|
|
3
|
+
{
|
|
4
|
+
id: 'ai-llms-txt-exists',
|
|
5
|
+
name: 'llms.txt File',
|
|
6
|
+
category: 'ai-search',
|
|
7
|
+
severity: 'info',
|
|
8
|
+
description: 'llms.txt helps AI systems understand your site (llmstxt.org)',
|
|
9
|
+
check: (ctx) => {
|
|
10
|
+
if (ctx.llmsTxt === undefined) {
|
|
11
|
+
return createResult({ id: 'ai-llms-txt-exists', name: 'llms.txt File', category: 'ai-search', severity: 'info' }, 'info', 'Not applicable (llms.txt data unavailable)', { recommendation: 'This rule checks for llms.txt file to optimize for AI/LLM discovery' });
|
|
12
|
+
}
|
|
13
|
+
if (!ctx.llmsTxt.exists) {
|
|
14
|
+
return createResult({ id: 'ai-llms-txt-exists', name: 'llms.txt File', category: 'ai-search', severity: 'info' }, 'info', 'llms.txt not found', {
|
|
15
|
+
recommendation: 'Create llms.txt to optimize for AI/LLM discovery',
|
|
16
|
+
evidence: {
|
|
17
|
+
expected: '/llms.txt file',
|
|
18
|
+
impact: 'AI systems may not understand your site structure and content',
|
|
19
|
+
learnMore: 'https://llmstxt.org/',
|
|
20
|
+
},
|
|
21
|
+
});
|
|
22
|
+
}
|
|
23
|
+
if (ctx.llmsTxt.valid) {
|
|
24
|
+
return createResult({ id: 'ai-llms-txt-exists', name: 'llms.txt File', category: 'ai-search', severity: 'info' }, 'pass', 'llms.txt found and valid');
|
|
25
|
+
}
|
|
26
|
+
return createResult({ id: 'ai-llms-txt-exists', name: 'llms.txt File', category: 'ai-search', severity: 'info' }, 'warn', 'llms.txt found but has issues', {
|
|
27
|
+
recommendation: 'Fix validation issues in llms.txt',
|
|
28
|
+
evidence: {
|
|
29
|
+
found: ctx.llmsTxt.issues?.map((i) => i.message) || [],
|
|
30
|
+
},
|
|
31
|
+
});
|
|
32
|
+
},
|
|
33
|
+
},
|
|
34
|
+
{
|
|
35
|
+
id: 'ai-llms-txt-structure',
|
|
36
|
+
name: 'llms.txt Structure',
|
|
37
|
+
category: 'ai-search',
|
|
38
|
+
severity: 'info',
|
|
39
|
+
description: 'llms.txt should have proper structure with site name and description',
|
|
40
|
+
check: (ctx) => {
|
|
41
|
+
if (!ctx.llmsTxt?.exists || !ctx.llmsTxt.parseResult) {
|
|
42
|
+
return createResult({ id: 'ai-llms-txt-structure', name: 'llms.txt Structure', category: 'ai-search', severity: 'info' }, 'info', 'Not applicable (llms.txt does not exist or parse data unavailable)', { recommendation: 'This rule validates llms.txt structure for AI comprehension' });
|
|
43
|
+
}
|
|
44
|
+
const { siteName, siteDescription, sections, links } = ctx.llmsTxt.parseResult;
|
|
45
|
+
const issues = [];
|
|
46
|
+
if (!siteName) {
|
|
47
|
+
issues.push('Missing site name (# heading)');
|
|
48
|
+
}
|
|
49
|
+
if (!siteDescription) {
|
|
50
|
+
issues.push('Missing site description (> blockquote)');
|
|
51
|
+
}
|
|
52
|
+
if (sections.length === 0) {
|
|
53
|
+
issues.push('No content sections (## headings)');
|
|
54
|
+
}
|
|
55
|
+
if (links.length === 0) {
|
|
56
|
+
issues.push('No links to content');
|
|
57
|
+
}
|
|
58
|
+
if (issues.length > 0) {
|
|
59
|
+
return createResult({ id: 'ai-llms-txt-structure', name: 'llms.txt Structure', category: 'ai-search', severity: 'info' }, 'info', `llms.txt structure can be improved`, {
|
|
60
|
+
recommendation: 'Add missing elements to llms.txt',
|
|
61
|
+
evidence: {
|
|
62
|
+
found: issues,
|
|
63
|
+
expected: 'Site name, description, sections, and links',
|
|
64
|
+
learnMore: 'https://llmstxt.org/',
|
|
65
|
+
},
|
|
66
|
+
});
|
|
67
|
+
}
|
|
68
|
+
return createResult({ id: 'ai-llms-txt-structure', name: 'llms.txt Structure', category: 'ai-search', severity: 'info' }, 'pass', 'llms.txt has good structure');
|
|
69
|
+
},
|
|
70
|
+
},
|
|
71
|
+
{
|
|
72
|
+
id: 'ai-content-structure',
|
|
73
|
+
name: 'AI-Friendly Content Structure',
|
|
74
|
+
category: 'ai-search',
|
|
75
|
+
severity: 'info',
|
|
76
|
+
description: 'Content should be well-structured for AI comprehension',
|
|
77
|
+
check: (ctx) => {
|
|
78
|
+
if (!ctx.headings?.structure) {
|
|
79
|
+
return createResult({ id: 'ai-content-structure', name: 'AI-Friendly Content Structure', category: 'ai-search', severity: 'info' }, 'info', 'Not applicable (heading structure data unavailable)', { recommendation: 'This rule checks for well-structured content for AI comprehension' });
|
|
80
|
+
}
|
|
81
|
+
const issues = [];
|
|
82
|
+
const h1Count = ctx.headings.h1Count || 0;
|
|
83
|
+
const hasH2 = ctx.headings.structure.some((h) => h.level === 2);
|
|
84
|
+
const hasH3 = ctx.headings.structure.some((h) => h.level === 3);
|
|
85
|
+
if (h1Count !== 1) {
|
|
86
|
+
issues.push(h1Count === 0 ? 'No H1 heading' : `Multiple H1 headings (${h1Count})`);
|
|
87
|
+
}
|
|
88
|
+
if (!hasH2 && ctx.wordCount && ctx.wordCount > 500) {
|
|
89
|
+
issues.push('Long content without H2 subheadings');
|
|
90
|
+
}
|
|
91
|
+
const questionHeadings = ctx.headings.structure.filter((h) => h.text.includes('?') || /^(what|how|why|when|where|who|which)/i.test(h.text));
|
|
92
|
+
if (issues.length > 0) {
|
|
93
|
+
return createResult({ id: 'ai-content-structure', name: 'AI-Friendly Content Structure', category: 'ai-search', severity: 'info' }, 'info', 'Content structure can be improved for AI', {
|
|
94
|
+
recommendation: 'Use clear heading hierarchy and question-based headings',
|
|
95
|
+
evidence: {
|
|
96
|
+
found: issues,
|
|
97
|
+
expected: 'Single H1, logical H2/H3 hierarchy, question headings',
|
|
98
|
+
},
|
|
99
|
+
});
|
|
100
|
+
}
|
|
101
|
+
return createResult({ id: 'ai-content-structure', name: 'AI-Friendly Content Structure', category: 'ai-search', severity: 'info' }, 'pass', 'Good content structure for AI comprehension');
|
|
102
|
+
},
|
|
103
|
+
},
|
|
104
|
+
{
|
|
105
|
+
id: 'ai-question-headings',
|
|
106
|
+
name: 'Question-Based Headings',
|
|
107
|
+
category: 'ai-search',
|
|
108
|
+
severity: 'info',
|
|
109
|
+
description: 'Question headings help with AI search featured snippets',
|
|
110
|
+
check: (ctx) => {
|
|
111
|
+
if (!ctx.headings?.structure) {
|
|
112
|
+
return createResult({ id: 'ai-question-headings', name: 'Question-Based Headings', category: 'ai-search', severity: 'info' }, 'info', 'Not applicable (heading structure data unavailable)', { recommendation: 'This rule checks for question-based headings that trigger featured snippets' });
|
|
113
|
+
}
|
|
114
|
+
const questionPatterns = [
|
|
115
|
+
/^what\s/i, /^how\s/i, /^why\s/i, /^when\s/i,
|
|
116
|
+
/^where\s/i, /^who\s/i, /^which\s/i, /^can\s/i,
|
|
117
|
+
/^does\s/i, /^is\s/i, /^are\s/i, /\?$/,
|
|
118
|
+
];
|
|
119
|
+
const questionHeadings = ctx.headings.structure.filter((h) => questionPatterns.some(p => p.test(h.text)));
|
|
120
|
+
if (questionHeadings.length > 0) {
|
|
121
|
+
return createResult({ id: 'ai-question-headings', name: 'Question-Based Headings', category: 'ai-search', severity: 'info' }, 'pass', `${questionHeadings.length} question-based heading(s) found`, {
|
|
122
|
+
value: questionHeadings.length,
|
|
123
|
+
evidence: {
|
|
124
|
+
found: questionHeadings.map((h) => h.text).slice(0, 5),
|
|
125
|
+
},
|
|
126
|
+
});
|
|
127
|
+
}
|
|
128
|
+
if (ctx.wordCount && ctx.wordCount > 500) {
|
|
129
|
+
return createResult({ id: 'ai-question-headings', name: 'Question-Based Headings', category: 'ai-search', severity: 'info' }, 'info', 'No question-based headings found', {
|
|
130
|
+
recommendation: 'Add question headings to improve AI search visibility',
|
|
131
|
+
evidence: {
|
|
132
|
+
example: 'Use headings like "What is...?", "How to...", "Why should...?"',
|
|
133
|
+
impact: 'Question headings often trigger featured snippets in AI search',
|
|
134
|
+
},
|
|
135
|
+
});
|
|
136
|
+
}
|
|
137
|
+
return createResult({ id: 'ai-question-headings', name: 'Question-Based Headings', category: 'ai-search', severity: 'info' }, 'pass', 'Content length does not require question headings');
|
|
138
|
+
},
|
|
139
|
+
},
|
|
140
|
+
{
|
|
141
|
+
id: 'ai-structured-data',
|
|
142
|
+
name: 'Structured Data for AI',
|
|
143
|
+
category: 'ai-search',
|
|
144
|
+
severity: 'info',
|
|
145
|
+
description: 'Schema.org structured data helps AI understand content',
|
|
146
|
+
check: (ctx) => {
|
|
147
|
+
if (!ctx.jsonLdTypes) {
|
|
148
|
+
return createResult({ id: 'ai-structured-data', name: 'Structured Data for AI', category: 'ai-search', severity: 'info' }, 'info', 'Not applicable (structured data unavailable)', { recommendation: 'This rule checks for Schema.org structured data for AI comprehension' });
|
|
149
|
+
}
|
|
150
|
+
const aiHelpfulTypes = [
|
|
151
|
+
'Article', 'BlogPosting', 'NewsArticle', 'TechArticle',
|
|
152
|
+
'FAQPage', 'HowTo', 'QAPage',
|
|
153
|
+
'Product', 'Review', 'Organization', 'Person',
|
|
154
|
+
'BreadcrumbList', 'WebPage', 'WebSite',
|
|
155
|
+
];
|
|
156
|
+
const foundTypes = ctx.jsonLdTypes;
|
|
157
|
+
const helpfulFound = foundTypes.filter((t) => aiHelpfulTypes.some(ht => t.includes(ht)));
|
|
158
|
+
if (helpfulFound.length > 0) {
|
|
159
|
+
return createResult({ id: 'ai-structured-data', name: 'Structured Data for AI', category: 'ai-search', severity: 'info' }, 'pass', `${helpfulFound.length} AI-helpful schema type(s) found`, {
|
|
160
|
+
value: helpfulFound.length,
|
|
161
|
+
evidence: {
|
|
162
|
+
found: helpfulFound,
|
|
163
|
+
},
|
|
164
|
+
});
|
|
165
|
+
}
|
|
166
|
+
if (foundTypes.length > 0) {
|
|
167
|
+
return createResult({ id: 'ai-structured-data', name: 'Structured Data for AI', category: 'ai-search', severity: 'info' }, 'info', 'Structured data found but could be more AI-friendly', {
|
|
168
|
+
value: foundTypes.length,
|
|
169
|
+
recommendation: 'Add Article, FAQPage, HowTo, or other content-rich schemas',
|
|
170
|
+
evidence: {
|
|
171
|
+
found: foundTypes,
|
|
172
|
+
expected: 'Article, FAQPage, HowTo, Product, Review, etc.',
|
|
173
|
+
},
|
|
174
|
+
});
|
|
175
|
+
}
|
|
176
|
+
return createResult({ id: 'ai-structured-data', name: 'Structured Data for AI', category: 'ai-search', severity: 'info' }, 'info', 'No structured data found', {
|
|
177
|
+
recommendation: 'Add Schema.org structured data for better AI understanding',
|
|
178
|
+
evidence: {
|
|
179
|
+
expected: 'JSON-LD with Article, FAQPage, HowTo, etc.',
|
|
180
|
+
impact: 'AI systems use structured data to understand content context',
|
|
181
|
+
},
|
|
182
|
+
});
|
|
183
|
+
},
|
|
184
|
+
},
|
|
185
|
+
{
|
|
186
|
+
id: 'ai-faq-schema',
|
|
187
|
+
name: 'FAQ Schema for AI',
|
|
188
|
+
category: 'ai-search',
|
|
189
|
+
severity: 'info',
|
|
190
|
+
description: 'FAQPage schema is highly valuable for AI search results',
|
|
191
|
+
check: (ctx) => {
|
|
192
|
+
if (!ctx.jsonLdTypes) {
|
|
193
|
+
return createResult({ id: 'ai-faq-schema', name: 'FAQ Schema for AI', category: 'ai-search', severity: 'info' }, 'info', 'Not applicable (structured data unavailable)', { recommendation: 'This rule checks for FAQ/HowTo schema for AI search rich results' });
|
|
194
|
+
}
|
|
195
|
+
const hasFaqSchema = ctx.jsonLdTypes.some((t) => t.includes('FAQPage'));
|
|
196
|
+
const hasHowToSchema = ctx.jsonLdTypes.some((t) => t.includes('HowTo'));
|
|
197
|
+
const hasQASchema = ctx.jsonLdTypes.some((t) => t.includes('QAPage'));
|
|
198
|
+
if (hasFaqSchema || hasHowToSchema || hasQASchema) {
|
|
199
|
+
const foundTypes = [];
|
|
200
|
+
if (hasFaqSchema)
|
|
201
|
+
foundTypes.push('FAQPage');
|
|
202
|
+
if (hasHowToSchema)
|
|
203
|
+
foundTypes.push('HowTo');
|
|
204
|
+
if (hasQASchema)
|
|
205
|
+
foundTypes.push('QAPage');
|
|
206
|
+
return createResult({ id: 'ai-faq-schema', name: 'FAQ Schema for AI', category: 'ai-search', severity: 'info' }, 'pass', 'FAQ/HowTo/QA schema found (great for AI search)', {
|
|
207
|
+
evidence: {
|
|
208
|
+
found: foundTypes,
|
|
209
|
+
},
|
|
210
|
+
});
|
|
211
|
+
}
|
|
212
|
+
const hasQuestionHeadings = ctx.headings?.structure?.some((h) => /\?$|^(how|what|why|when)/i.test(h.text));
|
|
213
|
+
if (hasQuestionHeadings) {
|
|
214
|
+
return createResult({ id: 'ai-faq-schema', name: 'FAQ Schema for AI', category: 'ai-search', severity: 'info' }, 'info', 'Content has Q&A structure but missing FAQ schema', {
|
|
215
|
+
recommendation: 'Add FAQPage schema for question-based content',
|
|
216
|
+
evidence: {
|
|
217
|
+
found: 'Question headings detected',
|
|
218
|
+
expected: 'FAQPage JSON-LD schema',
|
|
219
|
+
impact: 'FAQ schema enables rich results in AI-powered search',
|
|
220
|
+
},
|
|
221
|
+
});
|
|
222
|
+
}
|
|
223
|
+
return createResult({ id: 'ai-faq-schema', name: 'FAQ Schema for AI', category: 'ai-search', severity: 'info' }, 'pass', 'No Q&A structure detected requiring FAQ schema');
|
|
224
|
+
},
|
|
225
|
+
},
|
|
226
|
+
{
|
|
227
|
+
id: 'ai-content-depth',
|
|
228
|
+
name: 'Content Depth for AI',
|
|
229
|
+
category: 'ai-search',
|
|
230
|
+
severity: 'info',
|
|
231
|
+
description: 'AI systems prefer comprehensive, in-depth content',
|
|
232
|
+
check: (ctx) => {
|
|
233
|
+
if (ctx.wordCount === undefined) {
|
|
234
|
+
return createResult({ id: 'ai-content-depth', name: 'Content Depth for AI', category: 'ai-search', severity: 'info' }, 'info', 'Not applicable (word count data unavailable)', { recommendation: 'This rule checks content depth for AI search visibility' });
|
|
235
|
+
}
|
|
236
|
+
const minWords = 300;
|
|
237
|
+
const goodWords = 1000;
|
|
238
|
+
const excellentWords = 2000;
|
|
239
|
+
if (ctx.wordCount < minWords) {
|
|
240
|
+
return createResult({ id: 'ai-content-depth', name: 'Content Depth for AI', category: 'ai-search', severity: 'info' }, 'warn', `Thin content for AI search (${ctx.wordCount} words)`, {
|
|
241
|
+
value: ctx.wordCount,
|
|
242
|
+
recommendation: `Add more comprehensive content (min ${minWords} words)`,
|
|
243
|
+
evidence: {
|
|
244
|
+
found: ctx.wordCount,
|
|
245
|
+
expected: `${minWords}+ words for AI visibility`,
|
|
246
|
+
impact: 'AI systems may not surface thin content in results',
|
|
247
|
+
},
|
|
248
|
+
});
|
|
249
|
+
}
|
|
250
|
+
if (ctx.wordCount >= excellentWords) {
|
|
251
|
+
return createResult({ id: 'ai-content-depth', name: 'Content Depth for AI', category: 'ai-search', severity: 'info' }, 'pass', `Excellent content depth (${ctx.wordCount} words)`, { value: ctx.wordCount });
|
|
252
|
+
}
|
|
253
|
+
if (ctx.wordCount >= goodWords) {
|
|
254
|
+
return createResult({ id: 'ai-content-depth', name: 'Content Depth for AI', category: 'ai-search', severity: 'info' }, 'pass', `Good content depth (${ctx.wordCount} words)`, { value: ctx.wordCount });
|
|
255
|
+
}
|
|
256
|
+
return createResult({ id: 'ai-content-depth', name: 'Content Depth for AI', category: 'ai-search', severity: 'info' }, 'info', `Moderate content depth (${ctx.wordCount} words)`, {
|
|
257
|
+
value: ctx.wordCount,
|
|
258
|
+
recommendation: `Consider expanding to ${goodWords}+ words for better AI visibility`,
|
|
259
|
+
});
|
|
260
|
+
},
|
|
261
|
+
},
|
|
262
|
+
{
|
|
263
|
+
id: 'ai-content-freshness',
|
|
264
|
+
name: 'Content Freshness Signals',
|
|
265
|
+
category: 'ai-search',
|
|
266
|
+
severity: 'info',
|
|
267
|
+
description: 'AI systems value fresh, updated content',
|
|
268
|
+
check: (ctx) => {
|
|
269
|
+
if (!ctx.jsonLdTypes) {
|
|
270
|
+
return createResult({ id: 'ai-content-freshness', name: 'Content Freshness Signals', category: 'ai-search', severity: 'info' }, 'info', 'Not applicable (structured data unavailable)', { recommendation: 'This rule checks for content freshness signals in structured data' });
|
|
271
|
+
}
|
|
272
|
+
const hasDateSignals = ctx.jsonLdTypes.some((t) => t.includes('Article') || t.includes('BlogPosting') || t.includes('NewsArticle'));
|
|
273
|
+
if (hasDateSignals) {
|
|
274
|
+
return createResult({ id: 'ai-content-freshness', name: 'Content Freshness Signals', category: 'ai-search', severity: 'info' }, 'pass', 'Article schema with date signals found');
|
|
275
|
+
}
|
|
276
|
+
return createResult({ id: 'ai-content-freshness', name: 'Content Freshness Signals', category: 'ai-search', severity: 'info' }, 'info', 'No content freshness signals detected', {
|
|
277
|
+
recommendation: 'Add Article schema with datePublished and dateModified',
|
|
278
|
+
evidence: {
|
|
279
|
+
expected: 'Article/BlogPosting schema with date properties',
|
|
280
|
+
impact: 'AI systems prefer content with clear freshness indicators',
|
|
281
|
+
},
|
|
282
|
+
});
|
|
283
|
+
},
|
|
284
|
+
},
|
|
285
|
+
{
|
|
286
|
+
id: 'ai-robots-gpt-bot',
|
|
287
|
+
name: 'GPTBot Access',
|
|
288
|
+
category: 'ai-search',
|
|
289
|
+
severity: 'info',
|
|
290
|
+
description: 'Check if GPTBot (OpenAI) is allowed or blocked',
|
|
291
|
+
check: (ctx) => {
|
|
292
|
+
if (!ctx.robotsTxt?.parseResult) {
|
|
293
|
+
return createResult({ id: 'ai-robots-gpt-bot', name: 'GPTBot Access', category: 'ai-search', severity: 'info' }, 'info', 'Not applicable (robots.txt data unavailable)', { recommendation: 'This rule checks if GPTBot is allowed to crawl for ChatGPT training' });
|
|
294
|
+
}
|
|
295
|
+
const { userAgentBlocks } = ctx.robotsTxt.parseResult;
|
|
296
|
+
const gptBotBlock = userAgentBlocks.find((b) => b.userAgents.some(ua => ua.toLowerCase().includes('gptbot')));
|
|
297
|
+
if (gptBotBlock) {
|
|
298
|
+
const hasDisallow = gptBotBlock.rules.some((r) => r.type === 'disallow' && (r.path === '/' || r.path === ''));
|
|
299
|
+
if (hasDisallow) {
|
|
300
|
+
return createResult({ id: 'ai-robots-gpt-bot', name: 'GPTBot Access', category: 'ai-search', severity: 'info' }, 'info', 'GPTBot is blocked in robots.txt', {
|
|
301
|
+
recommendation: 'GPTBot blocked = content not used for ChatGPT training',
|
|
302
|
+
evidence: {
|
|
303
|
+
found: 'User-agent: GPTBot with Disallow',
|
|
304
|
+
impact: 'Your content will not appear in ChatGPT responses',
|
|
305
|
+
},
|
|
306
|
+
});
|
|
307
|
+
}
|
|
308
|
+
}
|
|
309
|
+
return createResult({ id: 'ai-robots-gpt-bot', name: 'GPTBot Access', category: 'ai-search', severity: 'info' }, 'pass', 'GPTBot is allowed (default)');
|
|
310
|
+
},
|
|
311
|
+
},
|
|
312
|
+
{
|
|
313
|
+
id: 'ai-robots-anthropic',
|
|
314
|
+
name: 'Anthropic Claude Access',
|
|
315
|
+
category: 'ai-search',
|
|
316
|
+
severity: 'info',
|
|
317
|
+
description: 'Check if Anthropic/Claude crawlers are allowed or blocked',
|
|
318
|
+
check: (ctx) => {
|
|
319
|
+
if (!ctx.robotsTxt?.parseResult) {
|
|
320
|
+
return createResult({ id: 'ai-robots-anthropic', name: 'Anthropic Claude Access', category: 'ai-search', severity: 'info' }, 'info', 'Not applicable (robots.txt data unavailable)', { recommendation: 'This rule checks if Anthropic/Claude crawlers are allowed' });
|
|
321
|
+
}
|
|
322
|
+
const { userAgentBlocks } = ctx.robotsTxt.parseResult;
|
|
323
|
+
const anthropicBlock = userAgentBlocks.find((b) => b.userAgents.some(ua => ua.toLowerCase().includes('anthropic') ||
|
|
324
|
+
ua.toLowerCase().includes('claude')));
|
|
325
|
+
if (anthropicBlock) {
|
|
326
|
+
const hasDisallow = anthropicBlock.rules.some((r) => r.type === 'disallow' && (r.path === '/' || r.path === ''));
|
|
327
|
+
if (hasDisallow) {
|
|
328
|
+
return createResult({ id: 'ai-robots-anthropic', name: 'Anthropic Claude Access', category: 'ai-search', severity: 'info' }, 'info', 'Anthropic/Claude crawler is blocked', {
|
|
329
|
+
recommendation: 'Consider implications of blocking AI training crawlers',
|
|
330
|
+
evidence: {
|
|
331
|
+
found: 'Anthropic/Claude blocked in robots.txt',
|
|
332
|
+
},
|
|
333
|
+
});
|
|
334
|
+
}
|
|
335
|
+
}
|
|
336
|
+
return createResult({ id: 'ai-robots-anthropic', name: 'Anthropic Claude Access', category: 'ai-search', severity: 'info' }, 'pass', 'Anthropic/Claude crawler is allowed (default)');
|
|
337
|
+
},
|
|
338
|
+
},
|
|
339
|
+
{
|
|
340
|
+
id: 'ai-content-last-modified',
|
|
341
|
+
name: 'Content Last Modified',
|
|
342
|
+
category: 'ai-search',
|
|
343
|
+
severity: 'info',
|
|
344
|
+
description: 'Content updated recently may perform better in AI search',
|
|
345
|
+
check: (ctx) => {
|
|
346
|
+
if (!ctx.lastModified) {
|
|
347
|
+
return createResult({ id: 'ai-content-last-modified', name: 'Content Last Modified', category: 'ai-search', severity: 'info' }, 'info', 'Not applicable (Last-Modified header unavailable)', { recommendation: 'This rule checks content freshness via Last-Modified header' });
|
|
348
|
+
}
|
|
349
|
+
const lastModifiedDate = new Date(ctx.lastModified);
|
|
350
|
+
const sixMonthsAgo = new Date();
|
|
351
|
+
sixMonthsAgo.setMonth(sixMonthsAgo.getMonth() - 6);
|
|
352
|
+
if (lastModifiedDate < sixMonthsAgo) {
|
|
353
|
+
const monthsOld = Math.floor((Date.now() - lastModifiedDate.getTime()) / (1000 * 60 * 60 * 24 * 30));
|
|
354
|
+
return createResult({ id: 'ai-content-last-modified', name: 'Content Last Modified', category: 'ai-search', severity: 'info' }, 'info', `Content last updated ${monthsOld} months ago`, {
|
|
355
|
+
value: ctx.lastModified,
|
|
356
|
+
recommendation: 'Consider updating content if it is time-sensitive',
|
|
357
|
+
evidence: {
|
|
358
|
+
found: `Last-Modified: ${ctx.lastModified}`,
|
|
359
|
+
expected: 'Content updated within last 6 months for time-sensitive topics',
|
|
360
|
+
impact: 'AI search engines may prefer recently updated content for some queries'
|
|
361
|
+
}
|
|
362
|
+
});
|
|
363
|
+
}
|
|
364
|
+
return createResult({ id: 'ai-content-last-modified', name: 'Content Last Modified', category: 'ai-search', severity: 'info' }, 'pass', 'Content has been updated recently');
|
|
365
|
+
},
|
|
366
|
+
},
|
|
367
|
+
{
|
|
368
|
+
id: 'ai-page-too-long',
|
|
369
|
+
name: 'Page Too Long for AI',
|
|
370
|
+
category: 'ai-search',
|
|
371
|
+
severity: 'info',
|
|
372
|
+
description: 'Very long pages may be truncated by AI search engines',
|
|
373
|
+
check: (ctx) => {
|
|
374
|
+
if (ctx.wordCount === undefined) {
|
|
375
|
+
return createResult({ id: 'ai-page-too-long', name: 'Page Too Long for AI', category: 'ai-search', severity: 'info' }, 'info', 'Not applicable (word count data unavailable)', { recommendation: 'This rule checks if page length may be too long for AI processing' });
|
|
376
|
+
}
|
|
377
|
+
if (ctx.wordCount > 10000) {
|
|
378
|
+
return createResult({ id: 'ai-page-too-long', name: 'Page Too Long for AI', category: 'ai-search', severity: 'info' }, 'info', `Page has ${ctx.wordCount.toLocaleString()} words (very long)`, {
|
|
379
|
+
value: ctx.wordCount,
|
|
380
|
+
recommendation: 'Consider splitting into multiple pages or summarizing key points',
|
|
381
|
+
evidence: {
|
|
382
|
+
found: `${ctx.wordCount.toLocaleString()} words`,
|
|
383
|
+
expected: 'Ideal: 1,000-3,000 words for comprehensive coverage',
|
|
384
|
+
impact: 'AI search engines may truncate or skip portions of very long content'
|
|
385
|
+
}
|
|
386
|
+
});
|
|
387
|
+
}
|
|
388
|
+
if (ctx.wordCount > 5000) {
|
|
389
|
+
return createResult({ id: 'ai-page-too-long', name: 'Page Too Long for AI', category: 'ai-search', severity: 'info' }, 'info', `Page has ${ctx.wordCount.toLocaleString()} words (long)`, {
|
|
390
|
+
value: ctx.wordCount,
|
|
391
|
+
recommendation: 'Ensure key information is near the beginning of the page',
|
|
392
|
+
evidence: {
|
|
393
|
+
found: `${ctx.wordCount.toLocaleString()} words`,
|
|
394
|
+
impact: 'Some AI systems may not process the entire page'
|
|
395
|
+
}
|
|
396
|
+
});
|
|
397
|
+
}
|
|
398
|
+
return createResult({ id: 'ai-page-too-long', name: 'Page Too Long for AI', category: 'ai-search', severity: 'info' }, 'pass', `Page length is optimal for AI processing (${ctx.wordCount.toLocaleString()} words)`);
|
|
399
|
+
},
|
|
400
|
+
},
|
|
401
|
+
{
|
|
402
|
+
id: 'semantic-html-ratio',
|
|
403
|
+
name: 'Semantic HTML Ratio',
|
|
404
|
+
category: 'ai-search',
|
|
405
|
+
severity: 'info',
|
|
406
|
+
description: 'Pages should use semantic HTML for better AI understanding',
|
|
407
|
+
check: (ctx) => {
|
|
408
|
+
if (ctx.semanticHtmlRatio === undefined) {
|
|
409
|
+
return createResult({ id: 'semantic-html-ratio', name: 'Semantic HTML Ratio', category: 'ai-search', severity: 'info' }, 'info', 'Not applicable (semantic HTML data unavailable)', { recommendation: 'This rule checks for semantic HTML usage for better AI understanding' });
|
|
410
|
+
}
|
|
411
|
+
if (ctx.semanticHtmlRatio < 0.1) {
|
|
412
|
+
return createResult({ id: 'semantic-html-ratio', name: 'Semantic HTML Ratio', category: 'ai-search', severity: 'info' }, 'warn', `Low semantic HTML usage (${(ctx.semanticHtmlRatio * 100).toFixed(1)}%)`, {
|
|
413
|
+
value: ctx.semanticHtmlRatio,
|
|
414
|
+
recommendation: 'Use semantic HTML tags (article, section, nav, header, footer, aside, main)',
|
|
415
|
+
evidence: {
|
|
416
|
+
found: `${(ctx.semanticHtmlRatio * 100).toFixed(1)}% semantic tags`,
|
|
417
|
+
expected: '>10% semantic HTML tags',
|
|
418
|
+
impact: 'Semantic HTML helps AI understand page structure and content hierarchy',
|
|
419
|
+
learnMore: 'https://web.dev/learn/html/semantic-html/'
|
|
420
|
+
}
|
|
421
|
+
});
|
|
422
|
+
}
|
|
423
|
+
if (ctx.semanticHtmlRatio < 0.2) {
|
|
424
|
+
return createResult({ id: 'semantic-html-ratio', name: 'Semantic HTML Ratio', category: 'ai-search', severity: 'info' }, 'info', `Moderate semantic HTML usage (${(ctx.semanticHtmlRatio * 100).toFixed(1)}%)`, {
|
|
425
|
+
value: ctx.semanticHtmlRatio,
|
|
426
|
+
recommendation: 'Consider adding more semantic structure',
|
|
427
|
+
evidence: {
|
|
428
|
+
found: `${(ctx.semanticHtmlRatio * 100).toFixed(1)}% semantic tags`,
|
|
429
|
+
expected: '>20% for optimal AI comprehension'
|
|
430
|
+
}
|
|
431
|
+
});
|
|
432
|
+
}
|
|
433
|
+
return createResult({ id: 'semantic-html-ratio', name: 'Semantic HTML Ratio', category: 'ai-search', severity: 'info' }, 'pass', `Good semantic HTML usage (${(ctx.semanticHtmlRatio * 100).toFixed(1)}%)`);
|
|
434
|
+
},
|
|
435
|
+
},
|
|
436
|
+
];
|