webpeel 0.19.4 → 0.20.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +2 -2
- package/dist/cache.d.ts +0 -1
- package/dist/cache.js +0 -1
- package/dist/cli/commands/auth.d.ts +5 -0
- package/dist/cli/commands/auth.js +476 -0
- package/dist/cli/commands/fetch.d.ts +6 -0
- package/dist/cli/commands/fetch.js +1015 -0
- package/dist/cli/commands/interact.d.ts +5 -0
- package/dist/cli/commands/interact.js +839 -0
- package/dist/cli/commands/jobs.d.ts +5 -0
- package/dist/cli/commands/jobs.js +997 -0
- package/dist/cli/commands/screenshot.d.ts +5 -0
- package/dist/cli/commands/screenshot.js +273 -0
- package/dist/cli/commands/search.d.ts +5 -0
- package/dist/cli/commands/search.js +524 -0
- package/dist/cli/utils.d.ts +84 -0
- package/dist/cli/utils.js +686 -0
- package/dist/cli-auth.d.ts +0 -1
- package/dist/cli-auth.js +0 -1
- package/dist/cli.d.ts +7 -6
- package/dist/cli.js +35 -4698
- package/dist/core/actions.d.ts +0 -1
- package/dist/core/actions.js +0 -1
- package/dist/core/agent.d.ts +0 -1
- package/dist/core/agent.js +9 -12
- package/dist/core/answer.d.ts +0 -1
- package/dist/core/answer.js +0 -1
- package/dist/core/application-tracker.d.ts +0 -1
- package/dist/core/application-tracker.js +0 -1
- package/dist/core/apply.d.ts +0 -1
- package/dist/core/apply.js +0 -1
- package/dist/core/auto-extract.d.ts +0 -1
- package/dist/core/auto-extract.js +0 -1
- package/dist/core/auto-interact.d.ts +0 -1
- package/dist/core/auto-interact.js +0 -1
- package/dist/core/bm25-filter.d.ts +0 -1
- package/dist/core/bm25-filter.js +0 -1
- package/dist/core/branding.d.ts +0 -1
- package/dist/core/branding.js +0 -1
- package/dist/core/browser-fetch.d.ts +0 -1
- package/dist/core/browser-fetch.js +17 -10
- package/dist/core/browser-pool.d.ts +0 -1
- package/dist/core/browser-pool.js +0 -1
- package/dist/core/budget.d.ts +0 -1
- package/dist/core/budget.js +0 -1
- package/dist/core/cache.d.ts +0 -1
- package/dist/core/cache.js +0 -1
- package/dist/core/cf-worker-proxy.d.ts +0 -1
- package/dist/core/cf-worker-proxy.js +0 -1
- package/dist/core/challenge-detection.d.ts +0 -1
- package/dist/core/challenge-detection.js +0 -1
- package/dist/core/change-tracking.d.ts +0 -1
- package/dist/core/change-tracking.js +0 -1
- package/dist/core/chunker.d.ts +0 -1
- package/dist/core/chunker.js +0 -1
- package/dist/core/chunking.d.ts +0 -1
- package/dist/core/chunking.js +0 -1
- package/dist/core/cloak-fetch.d.ts +0 -1
- package/dist/core/cloak-fetch.js +0 -1
- package/dist/core/content-pruner.d.ts +0 -1
- package/dist/core/content-pruner.js +0 -1
- package/dist/core/crawl-checkpoint.d.ts +0 -1
- package/dist/core/crawl-checkpoint.js +0 -1
- package/dist/core/crawler.d.ts +0 -1
- package/dist/core/crawler.js +6 -5
- package/dist/core/cycle-fetch.d.ts +0 -1
- package/dist/core/cycle-fetch.js +0 -1
- package/dist/core/deep-fetch.d.ts +0 -1
- package/dist/core/deep-fetch.js +0 -1
- package/dist/core/design-analysis.d.ts +0 -1
- package/dist/core/design-analysis.js +0 -1
- package/dist/core/design-compare.d.ts +0 -1
- package/dist/core/design-compare.js +0 -1
- package/dist/core/diff.d.ts +0 -1
- package/dist/core/diff.js +0 -1
- package/dist/core/dns-cache.d.ts +0 -1
- package/dist/core/dns-cache.js +0 -1
- package/dist/core/documents.d.ts +0 -1
- package/dist/core/documents.js +0 -1
- package/dist/core/domain-extractors.d.ts +0 -1
- package/dist/core/domain-extractors.js +0 -1
- package/dist/core/extract-inline.d.ts +0 -1
- package/dist/core/extract-inline.js +0 -1
- package/dist/core/extract-listings.d.ts +0 -1
- package/dist/core/extract-listings.js +0 -1
- package/dist/core/extract.d.ts +0 -1
- package/dist/core/extract.js +0 -1
- package/dist/core/fetcher.d.ts +0 -1
- package/dist/core/fetcher.js +0 -1
- package/dist/core/google-cache.d.ts +0 -1
- package/dist/core/google-cache.js +0 -1
- package/dist/core/hotel-search.d.ts +0 -1
- package/dist/core/hotel-search.js +0 -1
- package/dist/core/http-fetch.d.ts +0 -1
- package/dist/core/http-fetch.js +5 -7
- package/dist/core/human.d.ts +0 -1
- package/dist/core/human.js +0 -1
- package/dist/core/jobs.d.ts +0 -1
- package/dist/core/jobs.js +0 -1
- package/dist/core/json-ld.d.ts +0 -1
- package/dist/core/json-ld.js +0 -1
- package/dist/core/llm-extract.d.ts +0 -1
- package/dist/core/llm-extract.js +0 -1
- package/dist/core/logger.d.ts +17 -0
- package/dist/core/logger.js +44 -0
- package/dist/core/map.d.ts +0 -1
- package/dist/core/map.js +0 -1
- package/dist/core/markdown.d.ts +0 -1
- package/dist/core/markdown.js +0 -1
- package/dist/core/metadata.d.ts +0 -1
- package/dist/core/metadata.js +0 -1
- package/dist/core/paginate.d.ts +0 -1
- package/dist/core/paginate.js +0 -1
- package/dist/core/pdf.d.ts +0 -1
- package/dist/core/pdf.js +0 -1
- package/dist/core/peel-tls.d.ts +0 -1
- package/dist/core/peel-tls.js +0 -1
- package/dist/core/pipeline.d.ts +0 -1
- package/dist/core/pipeline.js +22 -25
- package/dist/core/profiles.d.ts +0 -1
- package/dist/core/profiles.js +0 -1
- package/dist/core/quick-answer.d.ts +0 -1
- package/dist/core/quick-answer.js +0 -1
- package/dist/core/rate-governor.d.ts +0 -1
- package/dist/core/rate-governor.js +0 -1
- package/dist/core/readability.d.ts +0 -1
- package/dist/core/readability.js +0 -1
- package/dist/core/research.d.ts +0 -1
- package/dist/core/research.js +0 -1
- package/dist/core/schema-extraction.d.ts +0 -1
- package/dist/core/schema-extraction.js +0 -1
- package/dist/core/schema-postprocess.d.ts +0 -1
- package/dist/core/schema-postprocess.js +0 -1
- package/dist/core/schema-templates.d.ts +0 -1
- package/dist/core/schema-templates.js +0 -1
- package/dist/core/screenshot.d.ts +0 -1
- package/dist/core/screenshot.js +0 -1
- package/dist/core/search-fallback.d.ts +0 -1
- package/dist/core/search-fallback.js +0 -1
- package/dist/core/search-provider.d.ts +0 -1
- package/dist/core/search-provider.js +18 -21
- package/dist/core/site-search.d.ts +0 -1
- package/dist/core/site-search.js +0 -1
- package/dist/core/sitemap.d.ts +0 -1
- package/dist/core/sitemap.js +0 -1
- package/dist/core/stealth-patches.d.ts +0 -1
- package/dist/core/stealth-patches.js +0 -1
- package/dist/core/stemmer.d.ts +0 -1
- package/dist/core/stemmer.js +0 -1
- package/dist/core/strategies.d.ts +6 -1
- package/dist/core/strategies.js +29 -41
- package/dist/core/strategy-hooks.d.ts +0 -1
- package/dist/core/strategy-hooks.js +0 -1
- package/dist/core/summarize.d.ts +0 -1
- package/dist/core/summarize.js +0 -1
- package/dist/core/synonyms.d.ts +0 -1
- package/dist/core/synonyms.js +0 -1
- package/dist/core/table-format.d.ts +0 -1
- package/dist/core/table-format.js +0 -1
- package/dist/core/timing.d.ts +0 -1
- package/dist/core/timing.js +0 -1
- package/dist/core/user-agents.d.ts +0 -1
- package/dist/core/user-agents.js +0 -1
- package/dist/core/watch-manager.d.ts +0 -1
- package/dist/core/watch-manager.js +0 -1
- package/dist/core/watch.d.ts +0 -1
- package/dist/core/watch.js +0 -1
- package/dist/core/youtube.d.ts +0 -1
- package/dist/core/youtube.js +0 -1
- package/dist/index.d.ts +8 -3
- package/dist/index.js +27 -3
- package/dist/integrations/index.d.ts +0 -1
- package/dist/integrations/index.js +0 -1
- package/dist/integrations/langchain.d.ts +0 -1
- package/dist/integrations/langchain.js +0 -1
- package/dist/integrations/llamaindex.d.ts +0 -1
- package/dist/integrations/llamaindex.js +0 -1
- package/dist/mcp/handlers/act.d.ts +5 -0
- package/dist/mcp/handlers/act.js +34 -0
- package/dist/mcp/handlers/definitions.d.ts +6 -0
- package/dist/mcp/handlers/definitions.js +266 -0
- package/dist/mcp/handlers/extract.d.ts +6 -0
- package/dist/mcp/handlers/extract.js +102 -0
- package/dist/mcp/handlers/fetch.d.ts +6 -0
- package/dist/mcp/handlers/fetch.js +98 -0
- package/dist/mcp/handlers/find.d.ts +5 -0
- package/dist/mcp/handlers/find.js +137 -0
- package/dist/mcp/handlers/index.d.ts +13 -0
- package/dist/mcp/handlers/index.js +61 -0
- package/dist/mcp/handlers/legacy.d.ts +25 -0
- package/dist/mcp/handlers/legacy.js +450 -0
- package/dist/mcp/handlers/meta.d.ts +6 -0
- package/dist/mcp/handlers/meta.js +31 -0
- package/dist/mcp/handlers/monitor.d.ts +5 -0
- package/dist/mcp/handlers/monitor.js +41 -0
- package/dist/mcp/handlers/read.d.ts +6 -0
- package/dist/mcp/handlers/read.js +63 -0
- package/dist/mcp/handlers/see.d.ts +5 -0
- package/dist/mcp/handlers/see.js +75 -0
- package/dist/mcp/handlers/types.d.ts +29 -0
- package/dist/mcp/handlers/types.js +28 -0
- package/dist/mcp/server.d.ts +3 -4
- package/dist/mcp/server.js +35 -1101
- package/dist/mcp/smart-router.d.ts +0 -1
- package/dist/mcp/smart-router.js +3 -1
- package/dist/types.d.ts +6 -1
- package/dist/types.js +0 -1
- package/package.json +3 -13
- package/dist/cache.d.ts.map +0 -1
- package/dist/cache.js.map +0 -1
- package/dist/cli-auth.d.ts.map +0 -1
- package/dist/cli-auth.js.map +0 -1
- package/dist/cli.bundle.cjs +0 -159248
- package/dist/cli.d.ts.map +0 -1
- package/dist/cli.js.map +0 -1
- package/dist/core/actions.d.ts.map +0 -1
- package/dist/core/actions.js.map +0 -1
- package/dist/core/agent.d.ts.map +0 -1
- package/dist/core/agent.js.map +0 -1
- package/dist/core/answer.d.ts.map +0 -1
- package/dist/core/answer.js.map +0 -1
- package/dist/core/application-tracker.d.ts.map +0 -1
- package/dist/core/application-tracker.js.map +0 -1
- package/dist/core/apply.d.ts.map +0 -1
- package/dist/core/apply.js.map +0 -1
- package/dist/core/auto-extract.d.ts.map +0 -1
- package/dist/core/auto-extract.js.map +0 -1
- package/dist/core/auto-interact.d.ts.map +0 -1
- package/dist/core/auto-interact.js.map +0 -1
- package/dist/core/bm25-filter.d.ts.map +0 -1
- package/dist/core/bm25-filter.js.map +0 -1
- package/dist/core/branding.d.ts.map +0 -1
- package/dist/core/branding.js.map +0 -1
- package/dist/core/browser-fetch.d.ts.map +0 -1
- package/dist/core/browser-fetch.js.map +0 -1
- package/dist/core/browser-pool.d.ts.map +0 -1
- package/dist/core/browser-pool.js.map +0 -1
- package/dist/core/budget.d.ts.map +0 -1
- package/dist/core/budget.js.map +0 -1
- package/dist/core/cache.d.ts.map +0 -1
- package/dist/core/cache.js.map +0 -1
- package/dist/core/cf-worker-proxy.d.ts.map +0 -1
- package/dist/core/cf-worker-proxy.js.map +0 -1
- package/dist/core/challenge-detection.d.ts.map +0 -1
- package/dist/core/challenge-detection.js.map +0 -1
- package/dist/core/change-tracking.d.ts.map +0 -1
- package/dist/core/change-tracking.js.map +0 -1
- package/dist/core/chunker.d.ts.map +0 -1
- package/dist/core/chunker.js.map +0 -1
- package/dist/core/chunking.d.ts.map +0 -1
- package/dist/core/chunking.js.map +0 -1
- package/dist/core/cloak-fetch.d.ts.map +0 -1
- package/dist/core/cloak-fetch.js.map +0 -1
- package/dist/core/content-pruner.d.ts.map +0 -1
- package/dist/core/content-pruner.js.map +0 -1
- package/dist/core/crawl-checkpoint.d.ts.map +0 -1
- package/dist/core/crawl-checkpoint.js.map +0 -1
- package/dist/core/crawler.d.ts.map +0 -1
- package/dist/core/crawler.js.map +0 -1
- package/dist/core/cycle-fetch.d.ts.map +0 -1
- package/dist/core/cycle-fetch.js.map +0 -1
- package/dist/core/deep-fetch.d.ts.map +0 -1
- package/dist/core/deep-fetch.js.map +0 -1
- package/dist/core/design-analysis.d.ts.map +0 -1
- package/dist/core/design-analysis.js.map +0 -1
- package/dist/core/design-compare.d.ts.map +0 -1
- package/dist/core/design-compare.js.map +0 -1
- package/dist/core/diff.d.ts.map +0 -1
- package/dist/core/diff.js.map +0 -1
- package/dist/core/dns-cache.d.ts.map +0 -1
- package/dist/core/dns-cache.js.map +0 -1
- package/dist/core/documents.d.ts.map +0 -1
- package/dist/core/documents.js.map +0 -1
- package/dist/core/domain-extractors.d.ts.map +0 -1
- package/dist/core/domain-extractors.js.map +0 -1
- package/dist/core/extract-inline.d.ts.map +0 -1
- package/dist/core/extract-inline.js.map +0 -1
- package/dist/core/extract-listings.d.ts.map +0 -1
- package/dist/core/extract-listings.js.map +0 -1
- package/dist/core/extract.d.ts.map +0 -1
- package/dist/core/extract.js.map +0 -1
- package/dist/core/fetcher.d.ts.map +0 -1
- package/dist/core/fetcher.js.map +0 -1
- package/dist/core/google-cache.d.ts.map +0 -1
- package/dist/core/google-cache.js.map +0 -1
- package/dist/core/hotel-search.d.ts.map +0 -1
- package/dist/core/hotel-search.js.map +0 -1
- package/dist/core/http-fetch.d.ts.map +0 -1
- package/dist/core/http-fetch.js.map +0 -1
- package/dist/core/human.d.ts.map +0 -1
- package/dist/core/human.js.map +0 -1
- package/dist/core/jobs.d.ts.map +0 -1
- package/dist/core/jobs.js.map +0 -1
- package/dist/core/json-ld.d.ts.map +0 -1
- package/dist/core/json-ld.js.map +0 -1
- package/dist/core/llm-extract.d.ts.map +0 -1
- package/dist/core/llm-extract.js.map +0 -1
- package/dist/core/map.d.ts.map +0 -1
- package/dist/core/map.js.map +0 -1
- package/dist/core/markdown.d.ts.map +0 -1
- package/dist/core/markdown.js.map +0 -1
- package/dist/core/metadata.d.ts.map +0 -1
- package/dist/core/metadata.js.map +0 -1
- package/dist/core/paginate.d.ts.map +0 -1
- package/dist/core/paginate.js.map +0 -1
- package/dist/core/pdf.d.ts.map +0 -1
- package/dist/core/pdf.js.map +0 -1
- package/dist/core/peel-tls.d.ts.map +0 -1
- package/dist/core/peel-tls.js.map +0 -1
- package/dist/core/pipeline.d.ts.map +0 -1
- package/dist/core/pipeline.js.map +0 -1
- package/dist/core/profiles.d.ts.map +0 -1
- package/dist/core/profiles.js.map +0 -1
- package/dist/core/quick-answer.d.ts.map +0 -1
- package/dist/core/quick-answer.js.map +0 -1
- package/dist/core/rate-governor.d.ts.map +0 -1
- package/dist/core/rate-governor.js.map +0 -1
- package/dist/core/readability.d.ts.map +0 -1
- package/dist/core/readability.js.map +0 -1
- package/dist/core/research.d.ts.map +0 -1
- package/dist/core/research.js.map +0 -1
- package/dist/core/schema-extraction.d.ts.map +0 -1
- package/dist/core/schema-extraction.js.map +0 -1
- package/dist/core/schema-postprocess.d.ts.map +0 -1
- package/dist/core/schema-postprocess.js.map +0 -1
- package/dist/core/schema-templates.d.ts.map +0 -1
- package/dist/core/schema-templates.js.map +0 -1
- package/dist/core/screenshot.d.ts.map +0 -1
- package/dist/core/screenshot.js.map +0 -1
- package/dist/core/search-fallback.d.ts.map +0 -1
- package/dist/core/search-fallback.js.map +0 -1
- package/dist/core/search-provider.d.ts.map +0 -1
- package/dist/core/search-provider.js.map +0 -1
- package/dist/core/site-search.d.ts.map +0 -1
- package/dist/core/site-search.js.map +0 -1
- package/dist/core/sitemap.d.ts.map +0 -1
- package/dist/core/sitemap.js.map +0 -1
- package/dist/core/stealth-patches.d.ts.map +0 -1
- package/dist/core/stealth-patches.js.map +0 -1
- package/dist/core/stemmer.d.ts.map +0 -1
- package/dist/core/stemmer.js.map +0 -1
- package/dist/core/strategies.d.ts.map +0 -1
- package/dist/core/strategies.js.map +0 -1
- package/dist/core/strategy-hooks.d.ts.map +0 -1
- package/dist/core/strategy-hooks.js.map +0 -1
- package/dist/core/summarize.d.ts.map +0 -1
- package/dist/core/summarize.js.map +0 -1
- package/dist/core/synonyms.d.ts.map +0 -1
- package/dist/core/synonyms.js.map +0 -1
- package/dist/core/table-format.d.ts.map +0 -1
- package/dist/core/table-format.js.map +0 -1
- package/dist/core/timing.d.ts.map +0 -1
- package/dist/core/timing.js.map +0 -1
- package/dist/core/user-agents.d.ts.map +0 -1
- package/dist/core/user-agents.js.map +0 -1
- package/dist/core/watch-manager.d.ts.map +0 -1
- package/dist/core/watch-manager.js.map +0 -1
- package/dist/core/watch.d.ts.map +0 -1
- package/dist/core/watch.js.map +0 -1
- package/dist/core/youtube.d.ts.map +0 -1
- package/dist/core/youtube.js.map +0 -1
- package/dist/index.d.ts.map +0 -1
- package/dist/index.js.map +0 -1
- package/dist/integrations/index.d.ts.map +0 -1
- package/dist/integrations/index.js.map +0 -1
- package/dist/integrations/langchain.d.ts.map +0 -1
- package/dist/integrations/langchain.js.map +0 -1
- package/dist/integrations/llamaindex.d.ts.map +0 -1
- package/dist/integrations/llamaindex.js.map +0 -1
- package/dist/mcp/server.d.ts.map +0 -1
- package/dist/mcp/server.js.map +0 -1
- package/dist/mcp/smart-router.d.ts.map +0 -1
- package/dist/mcp/smart-router.js.map +0 -1
- package/dist/server/app.d.ts +0 -15
- package/dist/server/app.d.ts.map +0 -1
- package/dist/server/app.js +0 -350
- package/dist/server/app.js.map +0 -1
- package/dist/server/auth-store.d.ts +0 -28
- package/dist/server/auth-store.d.ts.map +0 -1
- package/dist/server/auth-store.js +0 -89
- package/dist/server/auth-store.js.map +0 -1
- package/dist/server/email-service.d.ts +0 -22
- package/dist/server/email-service.d.ts.map +0 -1
- package/dist/server/email-service.js +0 -80
- package/dist/server/email-service.js.map +0 -1
- package/dist/server/job-queue.d.ts +0 -93
- package/dist/server/job-queue.d.ts.map +0 -1
- package/dist/server/job-queue.js +0 -146
- package/dist/server/job-queue.js.map +0 -1
- package/dist/server/logger.d.ts +0 -11
- package/dist/server/logger.d.ts.map +0 -1
- package/dist/server/logger.js +0 -38
- package/dist/server/logger.js.map +0 -1
- package/dist/server/middleware/auth.d.ts +0 -29
- package/dist/server/middleware/auth.d.ts.map +0 -1
- package/dist/server/middleware/auth.js +0 -222
- package/dist/server/middleware/auth.js.map +0 -1
- package/dist/server/middleware/rate-limit.d.ts +0 -25
- package/dist/server/middleware/rate-limit.d.ts.map +0 -1
- package/dist/server/middleware/rate-limit.js +0 -168
- package/dist/server/middleware/rate-limit.js.map +0 -1
- package/dist/server/middleware/url-validator.d.ts +0 -16
- package/dist/server/middleware/url-validator.d.ts.map +0 -1
- package/dist/server/middleware/url-validator.js +0 -187
- package/dist/server/middleware/url-validator.js.map +0 -1
- package/dist/server/openapi.yaml +0 -4944
- package/dist/server/pg-auth-store.d.ts +0 -133
- package/dist/server/pg-auth-store.d.ts.map +0 -1
- package/dist/server/pg-auth-store.js +0 -473
- package/dist/server/pg-auth-store.js.map +0 -1
- package/dist/server/pg-job-queue.d.ts +0 -60
- package/dist/server/pg-job-queue.d.ts.map +0 -1
- package/dist/server/pg-job-queue.js +0 -365
- package/dist/server/pg-job-queue.js.map +0 -1
- package/dist/server/premium/domain-intel.d.ts +0 -17
- package/dist/server/premium/domain-intel.d.ts.map +0 -1
- package/dist/server/premium/domain-intel.js +0 -134
- package/dist/server/premium/domain-intel.js.map +0 -1
- package/dist/server/premium/index.d.ts +0 -18
- package/dist/server/premium/index.d.ts.map +0 -1
- package/dist/server/premium/index.js +0 -36
- package/dist/server/premium/index.js.map +0 -1
- package/dist/server/premium/swr-cache.d.ts +0 -15
- package/dist/server/premium/swr-cache.d.ts.map +0 -1
- package/dist/server/premium/swr-cache.js +0 -35
- package/dist/server/premium/swr-cache.js.map +0 -1
- package/dist/server/routes/activity.d.ts +0 -7
- package/dist/server/routes/activity.d.ts.map +0 -1
- package/dist/server/routes/activity.js +0 -68
- package/dist/server/routes/activity.js.map +0 -1
- package/dist/server/routes/agent.d.ts +0 -16
- package/dist/server/routes/agent.d.ts.map +0 -1
- package/dist/server/routes/agent.js +0 -247
- package/dist/server/routes/agent.js.map +0 -1
- package/dist/server/routes/answer.d.ts +0 -6
- package/dist/server/routes/answer.d.ts.map +0 -1
- package/dist/server/routes/answer.js +0 -133
- package/dist/server/routes/answer.js.map +0 -1
- package/dist/server/routes/ask.d.ts +0 -23
- package/dist/server/routes/ask.d.ts.map +0 -1
- package/dist/server/routes/ask.js +0 -119
- package/dist/server/routes/ask.js.map +0 -1
- package/dist/server/routes/batch.d.ts +0 -7
- package/dist/server/routes/batch.d.ts.map +0 -1
- package/dist/server/routes/batch.js +0 -412
- package/dist/server/routes/batch.js.map +0 -1
- package/dist/server/routes/cli-usage.d.ts +0 -7
- package/dist/server/routes/cli-usage.d.ts.map +0 -1
- package/dist/server/routes/cli-usage.js +0 -121
- package/dist/server/routes/cli-usage.js.map +0 -1
- package/dist/server/routes/compat.d.ts +0 -24
- package/dist/server/routes/compat.d.ts.map +0 -1
- package/dist/server/routes/compat.js +0 -653
- package/dist/server/routes/compat.js.map +0 -1
- package/dist/server/routes/deep-fetch.d.ts +0 -9
- package/dist/server/routes/deep-fetch.d.ts.map +0 -1
- package/dist/server/routes/deep-fetch.js +0 -50
- package/dist/server/routes/deep-fetch.js.map +0 -1
- package/dist/server/routes/demo.d.ts +0 -25
- package/dist/server/routes/demo.d.ts.map +0 -1
- package/dist/server/routes/demo.js +0 -434
- package/dist/server/routes/demo.js.map +0 -1
- package/dist/server/routes/extract.d.ts +0 -9
- package/dist/server/routes/extract.d.ts.map +0 -1
- package/dist/server/routes/extract.js +0 -150
- package/dist/server/routes/extract.js.map +0 -1
- package/dist/server/routes/fetch.d.ts +0 -8
- package/dist/server/routes/fetch.d.ts.map +0 -1
- package/dist/server/routes/fetch.js +0 -988
- package/dist/server/routes/fetch.js.map +0 -1
- package/dist/server/routes/health.d.ts +0 -8
- package/dist/server/routes/health.d.ts.map +0 -1
- package/dist/server/routes/health.js +0 -20
- package/dist/server/routes/health.js.map +0 -1
- package/dist/server/routes/jobs.d.ts +0 -8
- package/dist/server/routes/jobs.d.ts.map +0 -1
- package/dist/server/routes/jobs.js +0 -487
- package/dist/server/routes/jobs.js.map +0 -1
- package/dist/server/routes/mcp.d.ts +0 -18
- package/dist/server/routes/mcp.d.ts.map +0 -1
- package/dist/server/routes/mcp.js +0 -1260
- package/dist/server/routes/mcp.js.map +0 -1
- package/dist/server/routes/oauth.d.ts +0 -10
- package/dist/server/routes/oauth.d.ts.map +0 -1
- package/dist/server/routes/oauth.js +0 -334
- package/dist/server/routes/oauth.js.map +0 -1
- package/dist/server/routes/quick-answer.d.ts +0 -9
- package/dist/server/routes/quick-answer.d.ts.map +0 -1
- package/dist/server/routes/quick-answer.js +0 -93
- package/dist/server/routes/quick-answer.js.map +0 -1
- package/dist/server/routes/screenshot.d.ts +0 -23
- package/dist/server/routes/screenshot.d.ts.map +0 -1
- package/dist/server/routes/screenshot.js +0 -819
- package/dist/server/routes/screenshot.js.map +0 -1
- package/dist/server/routes/search.d.ts +0 -7
- package/dist/server/routes/search.d.ts.map +0 -1
- package/dist/server/routes/search.js +0 -312
- package/dist/server/routes/search.js.map +0 -1
- package/dist/server/routes/session.d.ts +0 -16
- package/dist/server/routes/session.d.ts.map +0 -1
- package/dist/server/routes/session.js +0 -278
- package/dist/server/routes/session.js.map +0 -1
- package/dist/server/routes/stats.d.ts +0 -7
- package/dist/server/routes/stats.d.ts.map +0 -1
- package/dist/server/routes/stats.js +0 -65
- package/dist/server/routes/stats.js.map +0 -1
- package/dist/server/routes/stripe.d.ts +0 -16
- package/dist/server/routes/stripe.d.ts.map +0 -1
- package/dist/server/routes/stripe.js +0 -283
- package/dist/server/routes/stripe.js.map +0 -1
- package/dist/server/routes/users.d.ts +0 -9
- package/dist/server/routes/users.d.ts.map +0 -1
- package/dist/server/routes/users.js +0 -1211
- package/dist/server/routes/users.js.map +0 -1
- package/dist/server/routes/watch.d.ts +0 -16
- package/dist/server/routes/watch.d.ts.map +0 -1
- package/dist/server/routes/watch.js +0 -257
- package/dist/server/routes/watch.js.map +0 -1
- package/dist/server/routes/webhooks.d.ts +0 -16
- package/dist/server/routes/webhooks.d.ts.map +0 -1
- package/dist/server/routes/webhooks.js +0 -74
- package/dist/server/routes/webhooks.js.map +0 -1
- package/dist/server/routes/youtube.d.ts +0 -7
- package/dist/server/routes/youtube.d.ts.map +0 -1
- package/dist/server/routes/youtube.js +0 -93
- package/dist/server/routes/youtube.js.map +0 -1
- package/dist/server/sentry.d.ts +0 -14
- package/dist/server/sentry.d.ts.map +0 -1
- package/dist/server/sentry.js +0 -39
- package/dist/server/sentry.js.map +0 -1
- package/dist/server/types.d.ts +0 -16
- package/dist/server/types.d.ts.map +0 -1
- package/dist/server/types.js +0 -8
- package/dist/server/types.js.map +0 -1
- package/dist/server/utils/response.d.ts +0 -45
- package/dist/server/utils/response.d.ts.map +0 -1
- package/dist/server/utils/response.js +0 -70
- package/dist/server/utils/response.js.map +0 -1
- package/dist/server/utils/sse.d.ts +0 -23
- package/dist/server/utils/sse.d.ts.map +0 -1
- package/dist/server/utils/sse.js +0 -39
- package/dist/server/utils/sse.js.map +0 -1
- package/dist/types.d.ts.map +0 -1
- package/dist/types.js.map +0 -1
package/dist/mcp/server.js
CHANGED
|
@@ -1,1130 +1,61 @@
|
|
|
1
1
|
#!/usr/bin/env node
|
|
2
2
|
/**
|
|
3
|
-
* MCP Server for WebPeel
|
|
4
|
-
*
|
|
5
|
-
* All
|
|
3
|
+
* MCP Server for WebPeel — stdio transport.
|
|
4
|
+
* Thin wrapper: imports from the shared handler registry in ./handlers/.
|
|
5
|
+
* All tool logic lives in src/mcp/handlers/*.ts.
|
|
6
6
|
*/
|
|
7
7
|
import { Server } from '@modelcontextprotocol/sdk/server/index.js';
|
|
8
8
|
import { StdioServerTransport } from '@modelcontextprotocol/sdk/server/stdio.js';
|
|
9
9
|
import { CallToolRequestSchema, ListToolsRequestSchema, } from '@modelcontextprotocol/sdk/types.js';
|
|
10
|
-
import { peel, peelBatch } from '../index.js';
|
|
11
|
-
import { normalizeActions } from '../core/actions.js';
|
|
12
10
|
import { readFileSync } from 'fs';
|
|
13
11
|
import { join, dirname } from 'path';
|
|
14
12
|
import { fileURLToPath } from 'url';
|
|
15
|
-
import {
|
|
16
|
-
import {
|
|
17
|
-
import { extractInlineJson } from '../core/extract-inline.js';
|
|
18
|
-
import { quickAnswer } from '../core/quick-answer.js';
|
|
19
|
-
import { runAgent } from '../core/agent.js';
|
|
20
|
-
import { parseIntent } from './smart-router.js';
|
|
13
|
+
import { getHandler } from './handlers/index.js';
|
|
14
|
+
import { toolDefinitions } from './handlers/definitions.js';
|
|
21
15
|
// Read version from package.json
|
|
22
16
|
let pkgVersion = '0.3.1';
|
|
23
17
|
try {
|
|
24
|
-
const
|
|
18
|
+
const __filename = fileURLToPath(import.meta.url);
|
|
19
|
+
const __dirname = dirname(__filename);
|
|
25
20
|
const pkg = JSON.parse(readFileSync(join(__dirname, '..', '..', 'package.json'), 'utf-8'));
|
|
26
21
|
pkgVersion = pkg.version;
|
|
27
22
|
}
|
|
28
23
|
catch { /* fallback */ }
|
|
29
|
-
// ──
|
|
30
|
-
function extractColorsFromContent(content) {
|
|
31
|
-
const hexRegex = /#[0-9A-Fa-f]{6}|#[0-9A-Fa-f]{3}/g;
|
|
32
|
-
const matches = content.match(hexRegex);
|
|
33
|
-
return matches ? [...new Set(matches)].slice(0, 10) : [];
|
|
34
|
-
}
|
|
35
|
-
function extractFontsFromContent(content) {
|
|
36
|
-
const fontRegex = /font-family:\s*([^;}"'\n]+)/gi;
|
|
37
|
-
const fonts = [];
|
|
38
|
-
let match;
|
|
39
|
-
while ((match = fontRegex.exec(content)) !== null) {
|
|
40
|
-
fonts.push(match[1].trim());
|
|
41
|
-
}
|
|
42
|
-
return [...new Set(fonts)].slice(0, 5);
|
|
43
|
-
}
|
|
44
|
-
function safeJson(value) {
|
|
45
|
-
try {
|
|
46
|
-
return JSON.stringify(value, null, 2);
|
|
47
|
-
}
|
|
48
|
-
catch {
|
|
49
|
-
return JSON.stringify({ error: 'serialization_error', message: 'Failed to serialize result' });
|
|
50
|
-
}
|
|
51
|
-
}
|
|
52
|
-
function textResponse(text) {
|
|
53
|
-
return { content: [{ type: 'text', text }] };
|
|
54
|
-
}
|
|
55
|
-
function isYouTubeUrl(url) {
|
|
56
|
-
return /(?:youtube\.com|youtu\.be)/i.test(url);
|
|
57
|
-
}
|
|
58
|
-
function timeout(ms, label) {
|
|
59
|
-
return new Promise((_, reject) => setTimeout(() => reject(new Error(`${label} timed out after ${ms / 1000}s`)), ms));
|
|
60
|
-
}
|
|
61
|
-
// ── Tool definitions (7 public tools) ─────────────────────────────────────────
|
|
62
|
-
const publicTools = [
|
|
63
|
-
{
|
|
64
|
-
name: 'webpeel',
|
|
65
|
-
description: "Your complete web toolkit. Describe what you want in plain language. " +
|
|
66
|
-
"Examples: 'read https://stripe.com', 'screenshot bbc.com on mobile', " +
|
|
67
|
-
"'find best AI frameworks', 'extract prices from stripe.com/pricing', " +
|
|
68
|
-
"'watch stripe.com/pricing for changes'",
|
|
69
|
-
annotations: {
|
|
70
|
-
title: 'WebPeel Smart Web Tool',
|
|
71
|
-
readOnlyHint: true,
|
|
72
|
-
destructiveHint: false,
|
|
73
|
-
idempotentHint: true,
|
|
74
|
-
openWorldHint: true,
|
|
75
|
-
},
|
|
76
|
-
inputSchema: {
|
|
77
|
-
type: 'object',
|
|
78
|
-
properties: {
|
|
79
|
-
task: {
|
|
80
|
-
type: 'string',
|
|
81
|
-
description: 'Plain English description of what you want to do with the web.',
|
|
82
|
-
},
|
|
83
|
-
},
|
|
84
|
-
required: ['task'],
|
|
85
|
-
},
|
|
86
|
-
},
|
|
87
|
-
{
|
|
88
|
-
name: 'webpeel_read',
|
|
89
|
-
description: 'Read any URL and return clean markdown. Handles web pages, YouTube videos, and PDFs ' +
|
|
90
|
-
'automatically. Use question= for Q&A about the page, summary=true for a summary.',
|
|
91
|
-
annotations: {
|
|
92
|
-
title: 'Read Web Page',
|
|
93
|
-
readOnlyHint: true,
|
|
94
|
-
destructiveHint: false,
|
|
95
|
-
idempotentHint: true,
|
|
96
|
-
openWorldHint: true,
|
|
97
|
-
},
|
|
98
|
-
inputSchema: {
|
|
99
|
-
type: 'object',
|
|
100
|
-
properties: {
|
|
101
|
-
url: { type: 'string', description: 'URL to fetch' },
|
|
102
|
-
format: {
|
|
103
|
-
type: 'string',
|
|
104
|
-
enum: ['markdown', 'text', 'html'],
|
|
105
|
-
description: 'Output format (default: markdown)',
|
|
106
|
-
default: 'markdown',
|
|
107
|
-
},
|
|
108
|
-
render: {
|
|
109
|
-
type: 'boolean',
|
|
110
|
-
description: 'Force browser rendering for JS-heavy sites',
|
|
111
|
-
default: false,
|
|
112
|
-
},
|
|
113
|
-
question: {
|
|
114
|
-
type: 'string',
|
|
115
|
-
description: 'Ask a question about the page content (BM25, no LLM needed)',
|
|
116
|
-
},
|
|
117
|
-
summary: {
|
|
118
|
-
type: 'boolean',
|
|
119
|
-
description: 'Return a summary instead of full content',
|
|
120
|
-
default: false,
|
|
121
|
-
},
|
|
122
|
-
budget: {
|
|
123
|
-
type: 'number',
|
|
124
|
-
description: 'Smart token budget — distill content to N tokens',
|
|
125
|
-
},
|
|
126
|
-
readable: {
|
|
127
|
-
type: 'boolean',
|
|
128
|
-
description: 'Reader mode — extract only article content',
|
|
129
|
-
default: false,
|
|
130
|
-
},
|
|
131
|
-
},
|
|
132
|
-
required: ['url'],
|
|
133
|
-
},
|
|
134
|
-
},
|
|
135
|
-
{
|
|
136
|
-
name: 'webpeel_see',
|
|
137
|
-
description: "See any page visually. Returns a screenshot. Use mode='design' for design analysis, " +
|
|
138
|
-
"mode='compare' with compare_url for visual comparison.",
|
|
139
|
-
annotations: {
|
|
140
|
-
title: 'See Page Visually',
|
|
141
|
-
readOnlyHint: true,
|
|
142
|
-
destructiveHint: false,
|
|
143
|
-
idempotentHint: true,
|
|
144
|
-
openWorldHint: true,
|
|
145
|
-
},
|
|
146
|
-
inputSchema: {
|
|
147
|
-
type: 'object',
|
|
148
|
-
properties: {
|
|
149
|
-
url: { type: 'string', description: 'URL to screenshot' },
|
|
150
|
-
mode: {
|
|
151
|
-
type: 'string',
|
|
152
|
-
enum: ['screenshot', 'design', 'compare'],
|
|
153
|
-
description: "Mode: 'screenshot' (default), 'design' (analysis), 'compare' (visual diff)",
|
|
154
|
-
default: 'screenshot',
|
|
155
|
-
},
|
|
156
|
-
compare_url: {
|
|
157
|
-
type: 'string',
|
|
158
|
-
description: "Second URL to compare against (for mode='compare')",
|
|
159
|
-
},
|
|
160
|
-
viewport: {
|
|
161
|
-
description: "Viewport size: 'mobile' | 'tablet' | {width, height}",
|
|
162
|
-
oneOf: [
|
|
163
|
-
{ type: 'string', enum: ['mobile', 'tablet', 'desktop'] },
|
|
164
|
-
{
|
|
165
|
-
type: 'object',
|
|
166
|
-
properties: {
|
|
167
|
-
width: { type: 'number' },
|
|
168
|
-
height: { type: 'number' },
|
|
169
|
-
},
|
|
170
|
-
required: ['width', 'height'],
|
|
171
|
-
},
|
|
172
|
-
],
|
|
173
|
-
},
|
|
174
|
-
full_page: {
|
|
175
|
-
type: 'boolean',
|
|
176
|
-
description: 'Capture the full scrollable page',
|
|
177
|
-
default: false,
|
|
178
|
-
},
|
|
179
|
-
},
|
|
180
|
-
required: ['url'],
|
|
181
|
-
},
|
|
182
|
-
},
|
|
183
|
-
{
|
|
184
|
-
name: 'webpeel_find',
|
|
185
|
-
description: 'Find anything on the web. Pass a query to search, or a url to discover all pages on ' +
|
|
186
|
-
"that domain. Use depth='deep' for multi-source research.",
|
|
187
|
-
annotations: {
|
|
188
|
-
title: 'Find on the Web',
|
|
189
|
-
readOnlyHint: true,
|
|
190
|
-
destructiveHint: false,
|
|
191
|
-
idempotentHint: true,
|
|
192
|
-
openWorldHint: true,
|
|
193
|
-
},
|
|
194
|
-
inputSchema: {
|
|
195
|
-
type: 'object',
|
|
196
|
-
properties: {
|
|
197
|
-
query: { type: 'string', description: 'Search query' },
|
|
198
|
-
url: { type: 'string', description: 'Domain URL to map/discover all pages' },
|
|
199
|
-
depth: {
|
|
200
|
-
type: 'string',
|
|
201
|
-
enum: ['quick', 'deep'],
|
|
202
|
-
description: "Search depth: 'quick' = single search, 'deep' = multi-source research",
|
|
203
|
-
default: 'quick',
|
|
204
|
-
},
|
|
205
|
-
limit: {
|
|
206
|
-
type: 'number',
|
|
207
|
-
description: 'Max results to return (default: 5)',
|
|
208
|
-
default: 5,
|
|
209
|
-
},
|
|
210
|
-
},
|
|
211
|
-
},
|
|
212
|
-
},
|
|
213
|
-
{
|
|
214
|
-
name: 'webpeel_extract',
|
|
215
|
-
description: "Extract structured data from any URL. Pass fields=['price','title'] for specific data, " +
|
|
216
|
-
'or omit for auto-detection. Returns typed JSON.',
|
|
217
|
-
annotations: {
|
|
218
|
-
title: 'Extract Structured Data',
|
|
219
|
-
readOnlyHint: true,
|
|
220
|
-
destructiveHint: false,
|
|
221
|
-
idempotentHint: true,
|
|
222
|
-
openWorldHint: true,
|
|
223
|
-
},
|
|
224
|
-
inputSchema: {
|
|
225
|
-
type: 'object',
|
|
226
|
-
properties: {
|
|
227
|
-
url: { type: 'string', description: 'URL to extract from' },
|
|
228
|
-
schema: { type: 'object', description: 'JSON schema describing desired output structure' },
|
|
229
|
-
fields: {
|
|
230
|
-
type: 'array',
|
|
231
|
-
items: { type: 'string' },
|
|
232
|
-
description: "Specific fields to extract, e.g. ['price', 'title', 'description']",
|
|
233
|
-
},
|
|
234
|
-
format: {
|
|
235
|
-
type: 'string',
|
|
236
|
-
enum: ['json', 'markdown'],
|
|
237
|
-
description: 'Output format (default: json)',
|
|
238
|
-
default: 'json',
|
|
239
|
-
},
|
|
240
|
-
},
|
|
241
|
-
required: ['url'],
|
|
242
|
-
},
|
|
243
|
-
},
|
|
244
|
-
{
|
|
245
|
-
name: 'webpeel_monitor',
|
|
246
|
-
description: 'Watch a URL for changes. Returns diff on subsequent calls. ' +
|
|
247
|
-
'Add webhook= for persistent monitoring with notifications.',
|
|
248
|
-
annotations: {
|
|
249
|
-
title: 'Monitor URL for Changes',
|
|
250
|
-
readOnlyHint: false,
|
|
251
|
-
destructiveHint: false,
|
|
252
|
-
idempotentHint: false,
|
|
253
|
-
openWorldHint: true,
|
|
254
|
-
},
|
|
255
|
-
inputSchema: {
|
|
256
|
-
type: 'object',
|
|
257
|
-
properties: {
|
|
258
|
-
url: { type: 'string', description: 'URL to monitor' },
|
|
259
|
-
webhook: {
|
|
260
|
-
type: 'string',
|
|
261
|
-
description: 'Webhook URL to notify when content changes',
|
|
262
|
-
},
|
|
263
|
-
interval: {
|
|
264
|
-
type: 'string',
|
|
265
|
-
description: "Check interval, e.g. '1h', '30m', '1d'",
|
|
266
|
-
default: '1h',
|
|
267
|
-
},
|
|
268
|
-
selector: {
|
|
269
|
-
type: 'string',
|
|
270
|
-
description: 'CSS selector to monitor a specific part of the page',
|
|
271
|
-
},
|
|
272
|
-
},
|
|
273
|
-
required: ['url'],
|
|
274
|
-
},
|
|
275
|
-
},
|
|
276
|
-
{
|
|
277
|
-
name: 'webpeel_act',
|
|
278
|
-
description: 'Interact with a web page. Click buttons, fill forms, navigate. ' +
|
|
279
|
-
'Returns screenshot + extracted content after actions complete.',
|
|
280
|
-
annotations: {
|
|
281
|
-
title: 'Act on Web Page',
|
|
282
|
-
readOnlyHint: false,
|
|
283
|
-
destructiveHint: false,
|
|
284
|
-
idempotentHint: false,
|
|
285
|
-
openWorldHint: true,
|
|
286
|
-
},
|
|
287
|
-
inputSchema: {
|
|
288
|
-
type: 'object',
|
|
289
|
-
properties: {
|
|
290
|
-
url: { type: 'string', description: 'URL to interact with' },
|
|
291
|
-
actions: {
|
|
292
|
-
type: 'array',
|
|
293
|
-
description: 'Actions to perform, e.g. [{type:"click",selector:".btn"}, {type:"type",selector:"#q",value:"hello"}]',
|
|
294
|
-
items: {
|
|
295
|
-
type: 'object',
|
|
296
|
-
properties: {
|
|
297
|
-
type: {
|
|
298
|
-
type: 'string',
|
|
299
|
-
enum: ['click', 'type', 'fill', 'scroll', 'wait', 'press', 'hover', 'select'],
|
|
300
|
-
},
|
|
301
|
-
selector: { type: 'string' },
|
|
302
|
-
value: { type: 'string' },
|
|
303
|
-
key: { type: 'string' },
|
|
304
|
-
milliseconds: { type: 'number' },
|
|
305
|
-
},
|
|
306
|
-
required: ['type'],
|
|
307
|
-
},
|
|
308
|
-
},
|
|
309
|
-
extract_after: {
|
|
310
|
-
type: 'boolean',
|
|
311
|
-
description: 'Extract content after actions complete',
|
|
312
|
-
default: true,
|
|
313
|
-
},
|
|
314
|
-
screenshot_after: {
|
|
315
|
-
type: 'boolean',
|
|
316
|
-
description: 'Take screenshot after actions complete',
|
|
317
|
-
default: false,
|
|
318
|
-
},
|
|
319
|
-
},
|
|
320
|
-
required: ['url', 'actions'],
|
|
321
|
-
},
|
|
322
|
-
},
|
|
323
|
-
];
|
|
324
|
-
// ── Server setup ───────────────────────────────────────────────────────────────
|
|
24
|
+
// ── Server setup ───────────────────────────────────────────────────────────
|
|
325
25
|
const server = new Server({ name: 'webpeel', version: pkgVersion }, { capabilities: { tools: {} } });
|
|
326
|
-
// ──
|
|
327
|
-
|
|
328
|
-
|
|
329
|
-
|
|
330
|
-
|
|
331
|
-
async function handleRead(args) {
|
|
332
|
-
const url = args['url'];
|
|
333
|
-
if (!url || typeof url !== 'string')
|
|
334
|
-
throw new Error('url is required');
|
|
335
|
-
if (url.length > 2048)
|
|
336
|
-
throw new Error('URL too long (max 2048 characters)');
|
|
337
|
-
const format = args['format'] || 'markdown';
|
|
338
|
-
const render = args['render'] || false;
|
|
339
|
-
const question = args['question'];
|
|
340
|
-
const summary = args['summary'] || false;
|
|
341
|
-
const budgetArg = args['budget'];
|
|
342
|
-
const readable = args['readable'] || false;
|
|
343
|
-
// YouTube auto-detection
|
|
344
|
-
if (isYouTubeUrl(url)) {
|
|
345
|
-
const { getYouTubeTranscript } = await import('../core/youtube.js');
|
|
346
|
-
const language = args['language'] || 'en';
|
|
347
|
-
const transcript = await Promise.race([
|
|
348
|
-
getYouTubeTranscript(url, { language }),
|
|
349
|
-
timeout(60000, 'YouTube transcript'),
|
|
350
|
-
]);
|
|
351
|
-
return textResponse(safeJson(transcript));
|
|
352
|
-
}
|
|
353
|
-
// Build summary prompt if requested
|
|
354
|
-
const extractOpts = summary
|
|
355
|
-
? { prompt: 'Summarize this webpage in 2-3 concise sentences.' }
|
|
356
|
-
: undefined;
|
|
357
|
-
const options = {
|
|
358
|
-
render,
|
|
359
|
-
format: format,
|
|
360
|
-
question,
|
|
361
|
-
budget: budgetArg ?? 4000,
|
|
362
|
-
readable,
|
|
363
|
-
...(extractOpts ? { extract: extractOpts } : {}),
|
|
364
|
-
};
|
|
365
|
-
const result = await Promise.race([
|
|
366
|
-
peel(url, options),
|
|
367
|
-
timeout(60000, 'MCP read'),
|
|
368
|
-
]);
|
|
369
|
-
const out = {
|
|
370
|
-
url: result.url || url,
|
|
371
|
-
title: result.title || '',
|
|
372
|
-
tokens: result.tokens || 0,
|
|
373
|
-
content: result.content,
|
|
374
|
-
};
|
|
375
|
-
if (result.metadata)
|
|
376
|
-
out['metadata'] = result.metadata;
|
|
377
|
-
if (result.quickAnswer)
|
|
378
|
-
out['quickAnswer'] = result.quickAnswer;
|
|
379
|
-
if (result.extracted)
|
|
380
|
-
out['extracted'] = result.extracted;
|
|
381
|
-
if (result.images)
|
|
382
|
-
out['images'] = result.images;
|
|
383
|
-
return textResponse(safeJson(out));
|
|
384
|
-
}
|
|
385
|
-
/**
|
|
386
|
-
* webpeel_see: take a screenshot, optionally with design analysis or comparison.
|
|
387
|
-
*/
|
|
388
|
-
async function handleSee(args) {
|
|
389
|
-
const url = args['url'];
|
|
390
|
-
if (!url || typeof url !== 'string')
|
|
391
|
-
throw new Error('url is required');
|
|
392
|
-
if (url.length > 2048)
|
|
393
|
-
throw new Error('URL too long (max 2048 characters)');
|
|
394
|
-
const mode = args['mode'] || 'screenshot';
|
|
395
|
-
const compareUrl = args['compare_url'];
|
|
396
|
-
const fullPage = args['full_page'] || false;
|
|
397
|
-
const viewportArg = args['viewport'];
|
|
398
|
-
// Resolve viewport
|
|
399
|
-
let width = 1280;
|
|
400
|
-
let height = 720;
|
|
401
|
-
if (viewportArg && typeof viewportArg === 'object' && !Array.isArray(viewportArg)) {
|
|
402
|
-
const vp = viewportArg;
|
|
403
|
-
width = vp.width ?? 1280;
|
|
404
|
-
height = vp.height ?? 720;
|
|
405
|
-
}
|
|
406
|
-
else if (viewportArg === 'mobile') {
|
|
407
|
-
width = 390;
|
|
408
|
-
height = 844;
|
|
409
|
-
}
|
|
410
|
-
else if (viewportArg === 'tablet') {
|
|
411
|
-
width = 768;
|
|
412
|
-
height = 1024;
|
|
413
|
-
}
|
|
414
|
-
// Params from smart router viewport object
|
|
415
|
-
if (args['viewport'] && typeof args['viewport'] === 'object') {
|
|
416
|
-
const vp = args['viewport'];
|
|
417
|
-
if (vp.width)
|
|
418
|
-
width = vp.width;
|
|
419
|
-
if (vp.height)
|
|
420
|
-
height = vp.height;
|
|
421
|
-
}
|
|
422
|
-
if (mode === 'design') {
|
|
423
|
-
const { browserDesignAnalysis } = await import('../core/fetcher.js');
|
|
424
|
-
const result = await Promise.race([
|
|
425
|
-
browserDesignAnalysis(url, { width, height }),
|
|
426
|
-
timeout(90000, 'Design analysis'),
|
|
427
|
-
]);
|
|
428
|
-
return textResponse(safeJson({ url: result.finalUrl, mode: 'design', analysis: result.analysis }));
|
|
429
|
-
}
|
|
430
|
-
if (mode === 'compare' && compareUrl) {
|
|
431
|
-
const { browserDiff } = await import('../core/fetcher.js');
|
|
432
|
-
const diff = await Promise.race([
|
|
433
|
-
browserDiff(url, compareUrl, { width, height, fullPage }),
|
|
434
|
-
timeout(90000, 'Design compare'),
|
|
435
|
-
]);
|
|
436
|
-
return textResponse(safeJson({
|
|
437
|
-
url,
|
|
438
|
-
compare_url: compareUrl,
|
|
439
|
-
mode: 'compare',
|
|
440
|
-
diffPixels: diff.diffPixels,
|
|
441
|
-
totalPixels: diff.totalPixels,
|
|
442
|
-
diffPercent: diff.diffPercent,
|
|
443
|
-
screenshot: diff.diffBuffer.toString('base64'),
|
|
444
|
-
}));
|
|
445
|
-
}
|
|
446
|
-
// Default: screenshot
|
|
447
|
-
const { takeScreenshot } = await import('../core/screenshot.js');
|
|
448
|
-
const result = await Promise.race([
|
|
449
|
-
takeScreenshot(url, { fullPage, width, height, format: 'png' }),
|
|
450
|
-
timeout(60000, 'Screenshot'),
|
|
451
|
-
]);
|
|
452
|
-
return textResponse(safeJson({ url: result.url, mode: 'screenshot', screenshot: result.screenshot, format: result.format }));
|
|
453
|
-
}
|
|
454
|
-
/**
|
|
455
|
-
* webpeel_find: search the web, discover domain URLs, or do deep research.
|
|
456
|
-
*/
|
|
457
|
-
async function handleFind(args) {
|
|
458
|
-
const query = args['query'];
|
|
459
|
-
const url = args['url'];
|
|
460
|
-
const depth = args['depth'] || 'quick';
|
|
461
|
-
const limit = Math.min(Math.max(args['limit'] ?? 5, 1), 20);
|
|
462
|
-
// URL-based: map/discover pages on a domain
|
|
463
|
-
if (url && !query) {
|
|
464
|
-
const { mapDomain } = await import('../core/map.js');
|
|
465
|
-
const results = await Promise.race([
|
|
466
|
-
mapDomain(url, { maxUrls: limit * 100 }),
|
|
467
|
-
timeout(600000, 'Map domain'),
|
|
468
|
-
]);
|
|
469
|
-
return textResponse(safeJson(results));
|
|
470
|
-
}
|
|
471
|
-
if (!query)
|
|
472
|
-
throw new Error('Either query or url is required');
|
|
473
|
-
// Question-mode: if the query looks like a natural language question and depth
|
|
474
|
-
// isn't forced to 'deep', use the LLM-free BM25 Q&A path (search → fetch → BM25).
|
|
475
|
-
// This is the /v1/ask feature — no API key required, deterministic.
|
|
476
|
-
const isQuestion = /\?$/.test(query.trim()) ||
|
|
477
|
-
/^(what|how|when|where|why|who|which|can|does|is|are|do|did|will|would|could|should)\b/i.test(query.trim());
|
|
478
|
-
if (isQuestion && depth !== 'deep') {
|
|
479
|
-
const numSources = Math.min(limit, 5);
|
|
480
|
-
const { provider, apiKey } = getBestSearchProvider();
|
|
481
|
-
let searchResults;
|
|
482
|
-
try {
|
|
483
|
-
searchResults = (await Promise.race([
|
|
484
|
-
provider.searchWeb(query, { count: numSources, apiKey }),
|
|
485
|
-
timeout(30000, 'Ask search'),
|
|
486
|
-
]));
|
|
487
|
-
}
|
|
488
|
-
catch {
|
|
489
|
-
searchResults = [];
|
|
490
|
-
}
|
|
491
|
-
if (searchResults.length === 0) {
|
|
492
|
-
return textResponse(safeJson({ question: query, answer: null, confidence: 0, sources: [], method: 'bm25' }));
|
|
493
|
-
}
|
|
494
|
-
const fetched = await Promise.allSettled(searchResults.slice(0, numSources).map((r) => peel(r.url, { budget: 3000, format: 'markdown', timeout: 12000 }).then((result) => ({ result, searchResult: r }))));
|
|
495
|
-
const answers = fetched
|
|
496
|
-
.filter((f) => f.status === 'fulfilled')
|
|
497
|
-
.map((f) => {
|
|
498
|
-
const { result, searchResult } = f.value;
|
|
499
|
-
const qa = quickAnswer({ question: query, content: result.content || '', url: result.url || searchResult.url, maxPassages: 2 });
|
|
500
|
-
return {
|
|
501
|
-
answer: qa.answer,
|
|
502
|
-
confidence: qa.confidence,
|
|
503
|
-
source: { url: result.url || searchResult.url, title: result.title || searchResult.title, snippet: searchResult.snippet },
|
|
504
|
-
};
|
|
505
|
-
})
|
|
506
|
-
.sort((a, b) => b.confidence - a.confidence);
|
|
507
|
-
const best = answers[0];
|
|
508
|
-
return textResponse(safeJson({
|
|
509
|
-
question: query,
|
|
510
|
-
answer: best?.answer || null,
|
|
511
|
-
confidence: best?.confidence || 0,
|
|
512
|
-
sources: answers.map((a) => ({ ...a.source, confidence: a.confidence })),
|
|
513
|
-
method: 'bm25',
|
|
514
|
-
}));
|
|
515
|
-
}
|
|
516
|
-
// Deep research mode
|
|
517
|
-
if (depth === 'deep') {
|
|
518
|
-
const { provider, apiKey } = getBestSearchProvider();
|
|
519
|
-
const searchResults = await Promise.race([
|
|
520
|
-
provider.searchWeb(query, { count: limit, apiKey }),
|
|
521
|
-
timeout(30000, 'Search'),
|
|
522
|
-
]);
|
|
523
|
-
const results = Array.isArray(searchResults)
|
|
524
|
-
? searchResults
|
|
525
|
-
: searchResults.results ?? [];
|
|
526
|
-
const topN = results.slice(0, limit);
|
|
527
|
-
if (topN.length === 0) {
|
|
528
|
-
return textResponse(safeJson({ query, sources: [], content: '', totalTokens: 0 }));
|
|
529
|
-
}
|
|
530
|
-
const urls = topN.map((r) => r.url).filter(Boolean);
|
|
531
|
-
const pages = await Promise.race([
|
|
532
|
-
peelBatch(urls, { concurrency: 5, format: 'markdown' }),
|
|
533
|
-
timeout(120000, 'Batch fetch'),
|
|
534
|
-
]);
|
|
535
|
-
const sources = [];
|
|
536
|
-
const contentParts = [];
|
|
537
|
-
let totalTokens = 0;
|
|
538
|
-
for (let i = 0; i < pages.length; i++) {
|
|
539
|
-
const page = pages[i];
|
|
540
|
-
const sr = topN[i];
|
|
541
|
-
const pageUrl = urls[i];
|
|
542
|
-
const title = page.title || sr.title || pageUrl;
|
|
543
|
-
const relevanceScore = Math.round((1 - i / Math.max(pages.length, 1)) * 100) / 100;
|
|
544
|
-
sources.push({ url: pageUrl, title, relevanceScore });
|
|
545
|
-
if (page.content) {
|
|
546
|
-
contentParts.push(`## Source ${i + 1}: ${title}\n**URL:** ${pageUrl}\n\n${page.content}\n\n---\n`);
|
|
547
|
-
totalTokens += page.tokens || 0;
|
|
548
|
-
}
|
|
549
|
-
}
|
|
550
|
-
return textResponse(safeJson({ query, sources, content: contentParts.join('\n'), totalTokens }));
|
|
551
|
-
}
|
|
552
|
-
// Quick search (default)
|
|
553
|
-
const validProviders = ['duckduckgo', 'brave', 'stealth', 'google'];
|
|
554
|
-
const providerId = (args['provider'] && validProviders.includes(args['provider']))
|
|
555
|
-
? args['provider']
|
|
556
|
-
: 'duckduckgo';
|
|
557
|
-
const searchProvider = getSearchProvider(providerId);
|
|
558
|
-
const results = await Promise.race([
|
|
559
|
-
searchProvider.searchWeb(query, { count: limit }),
|
|
560
|
-
timeout(30000, 'Search'),
|
|
561
|
-
]);
|
|
562
|
-
return textResponse(safeJson(results));
|
|
563
|
-
}
|
|
564
|
-
/**
|
|
565
|
-
* webpeel_extract: extract structured data from a URL.
|
|
566
|
-
* Supports auto-detection, field lists, schema, and brand presets.
|
|
567
|
-
*/
|
|
568
|
-
async function handleExtract(args) {
|
|
569
|
-
const url = args['url'];
|
|
570
|
-
if (!url || typeof url !== 'string')
|
|
571
|
-
throw new Error('url is required');
|
|
572
|
-
if (url.length > 2048)
|
|
573
|
-
throw new Error('URL too long (max 2048 characters)');
|
|
574
|
-
const schema = args['schema'];
|
|
575
|
-
const fields = args['fields'];
|
|
576
|
-
const render = args['render'] || false;
|
|
577
|
-
// Brand preset: fields=['name','logo','colors','fonts','socials'] or brand flag
|
|
578
|
-
const isBrandPreset = args['_brand'] ||
|
|
579
|
-
(Array.isArray(fields) &&
|
|
580
|
-
['name', 'logo', 'colors', 'fonts', 'socials'].every((f) => fields.includes(f)));
|
|
581
|
-
if (isBrandPreset) {
|
|
582
|
-
const options = {
|
|
583
|
-
render,
|
|
584
|
-
extract: {
|
|
585
|
-
selectors: {
|
|
586
|
-
primaryColor: 'meta[name="theme-color"]',
|
|
587
|
-
title: 'title',
|
|
588
|
-
logo: 'img[class*="logo"], img[alt*="logo"]',
|
|
589
|
-
},
|
|
590
|
-
},
|
|
591
|
-
};
|
|
592
|
-
const result = await Promise.race([
|
|
593
|
-
peel(url, options),
|
|
594
|
-
timeout(60000, 'Brand extraction'),
|
|
595
|
-
]);
|
|
596
|
-
return textResponse(safeJson({
|
|
597
|
-
url: result.url,
|
|
598
|
-
title: result.title,
|
|
599
|
-
extracted: result.extracted,
|
|
600
|
-
metadata: result.metadata,
|
|
601
|
-
colors: extractColorsFromContent(result.content || ''),
|
|
602
|
-
fonts: extractFontsFromContent(result.content || ''),
|
|
603
|
-
}));
|
|
604
|
-
}
|
|
605
|
-
// Auto-extract when no schema provided
|
|
606
|
-
if (!schema && (!fields || fields.length === 0)) {
|
|
607
|
-
const htmlResult = await Promise.race([
|
|
608
|
-
peel(url, { format: 'html', render }),
|
|
609
|
-
timeout(60000, 'Auto-extract fetch'),
|
|
610
|
-
]);
|
|
611
|
-
const { autoExtract } = await import('../core/auto-extract.js');
|
|
612
|
-
const extracted = autoExtract(htmlResult.content || '', url);
|
|
613
|
-
return textResponse(safeJson({ url, pageType: extracted.type, structured: extracted }));
|
|
614
|
-
}
|
|
615
|
-
// Field-based extraction (CSS selectors from field names)
|
|
616
|
-
if (fields && fields.length > 0 && !schema) {
|
|
617
|
-
const selectors = {};
|
|
618
|
-
for (const field of fields) {
|
|
619
|
-
// Map common field names to CSS selectors
|
|
620
|
-
const fieldSelectorMap = {
|
|
621
|
-
price: '[class*="price"], [data-price]',
|
|
622
|
-
title: 'h1, title',
|
|
623
|
-
description: '[class*="description"], [class*="summary"]',
|
|
624
|
-
image: 'img[class*="main"], img[class*="hero"]',
|
|
625
|
-
name: 'h1, [class*="name"]',
|
|
626
|
-
logo: 'img[class*="logo"], img[alt*="logo"]',
|
|
627
|
-
colors: 'meta[name="theme-color"]',
|
|
628
|
-
fonts: 'link[rel="stylesheet"]',
|
|
629
|
-
socials: 'a[href*="twitter.com"], a[href*="linkedin.com"], a[href*="github.com"]',
|
|
630
|
-
};
|
|
631
|
-
selectors[field] = fieldSelectorMap[field] || `[class*="${field}"], [id*="${field}"]`;
|
|
632
|
-
}
|
|
633
|
-
const options = { render, extract: { selectors } };
|
|
634
|
-
const result = await Promise.race([
|
|
635
|
-
peel(url, options),
|
|
636
|
-
timeout(60000, 'Field extraction'),
|
|
637
|
-
]);
|
|
638
|
-
return textResponse(safeJson(result));
|
|
639
|
-
}
|
|
640
|
-
// Schema-based extraction
|
|
641
|
-
const options = {
|
|
642
|
-
render,
|
|
643
|
-
extract: { schema },
|
|
644
|
-
};
|
|
645
|
-
const result = await Promise.race([
|
|
646
|
-
peel(url, options),
|
|
647
|
-
timeout(60000, 'Schema extraction'),
|
|
648
|
-
]);
|
|
649
|
-
return textResponse(safeJson(result));
|
|
650
|
-
}
|
|
651
|
-
/**
|
|
652
|
-
* webpeel_monitor: watch a URL for changes, with optional webhook.
|
|
653
|
-
*/
|
|
654
|
-
async function handleMonitor(args) {
|
|
655
|
-
const url = args['url'];
|
|
656
|
-
if (!url || typeof url !== 'string')
|
|
657
|
-
throw new Error('url is required');
|
|
658
|
-
if (url.length > 2048)
|
|
659
|
-
throw new Error('URL too long (max 2048 characters)');
|
|
660
|
-
const webhook = args['webhook'];
|
|
661
|
-
const selector = args['selector'];
|
|
662
|
-
const render = args['render'] || false;
|
|
663
|
-
if (webhook) {
|
|
664
|
-
// Webhook-based persistent monitoring requires hosted API
|
|
665
|
-
return textResponse(safeJson({
|
|
666
|
-
message: 'Persistent webhook monitoring requires the hosted API (api.webpeel.dev). ' +
|
|
667
|
-
'Use webpeel_monitor without webhook= for one-time change detection.',
|
|
668
|
-
url,
|
|
669
|
-
webhook,
|
|
670
|
-
}));
|
|
671
|
-
}
|
|
672
|
-
// One-time change snapshot (change_track logic)
|
|
673
|
-
const options = { render: render || false, ...(selector ? { selector } : {}) };
|
|
674
|
-
const result = await Promise.race([
|
|
675
|
-
peel(url, options),
|
|
676
|
-
timeout(60000, 'Monitor'),
|
|
677
|
-
]);
|
|
678
|
-
return textResponse(safeJson({
|
|
679
|
-
url: result.url,
|
|
680
|
-
title: result.title,
|
|
681
|
-
fingerprint: result.fingerprint,
|
|
682
|
-
tokens: result.tokens,
|
|
683
|
-
contentType: result.contentType,
|
|
684
|
-
lastChecked: new Date().toISOString(),
|
|
685
|
-
}));
|
|
686
|
-
}
|
|
687
|
-
/**
|
|
688
|
-
* webpeel_act: perform browser actions on a page, then optionally extract content.
|
|
689
|
-
*/
|
|
690
|
-
async function handleAct(args) {
|
|
691
|
-
const url = args.url;
|
|
692
|
-
const actions = args.actions;
|
|
693
|
-
const extract = args.extract !== false; // default true
|
|
694
|
-
const screenshot = Boolean(args.screenshot);
|
|
695
|
-
if (!url)
|
|
696
|
-
return textResponse(safeJson({ error: 'url is required' }));
|
|
697
|
-
if (!actions?.length)
|
|
698
|
-
return textResponse(safeJson({ error: 'actions array is required' }));
|
|
699
|
-
// Reuse the full peel() pipeline — peel is already statically imported at module top
|
|
700
|
-
const result = await peel(url, {
|
|
701
|
-
render: true, // actions always require browser
|
|
702
|
-
actions,
|
|
703
|
-
screenshot,
|
|
704
|
-
format: 'markdown',
|
|
705
|
-
budget: 4000,
|
|
706
|
-
timeout: 60000,
|
|
707
|
-
});
|
|
708
|
-
return textResponse(safeJson({
|
|
709
|
-
url: result.url,
|
|
710
|
-
title: result.title,
|
|
711
|
-
content: extract ? result.content : undefined,
|
|
712
|
-
screenshot: result.screenshot,
|
|
713
|
-
method: result.method,
|
|
714
|
-
elapsed: result.elapsed,
|
|
715
|
-
}));
|
|
716
|
-
}
|
|
717
|
-
// ── Full webpeel_fetch handler (backward compat + power users) ─────────────────
|
|
718
|
-
async function handleFetch(args) {
|
|
719
|
-
const { url, render, stealth, wait, format, screenshot: ssFlag, screenshotFullPage, selector, exclude, includeTags, excludeTags, images, location, headers, actions: rawActions, autoScroll: autoScrollParam, maxTokens, extract, inlineExtract, llmProvider, llmApiKey, llmModel, question, budget: budgetArg, readable, } = args;
|
|
720
|
-
if (!url || typeof url !== 'string')
|
|
721
|
-
throw new Error('Invalid URL parameter');
|
|
722
|
-
if (url.length > 2048)
|
|
723
|
-
throw new Error('URL too long (max 2048 characters)');
|
|
724
|
-
const normalizedActions = rawActions ? normalizeActions(rawActions) : undefined;
|
|
725
|
-
const hasActions = normalizedActions && normalizedActions.length > 0;
|
|
726
|
-
const options = {
|
|
727
|
-
render: render || hasActions || !!autoScrollParam || false,
|
|
728
|
-
stealth: stealth || false,
|
|
729
|
-
wait: wait || 0,
|
|
730
|
-
format: format || 'markdown',
|
|
731
|
-
screenshot: ssFlag || false,
|
|
732
|
-
screenshotFullPage: screenshotFullPage || false,
|
|
733
|
-
selector,
|
|
734
|
-
exclude,
|
|
735
|
-
includeTags,
|
|
736
|
-
excludeTags,
|
|
737
|
-
images,
|
|
738
|
-
location: location ? { country: location } : undefined,
|
|
739
|
-
headers,
|
|
740
|
-
actions: normalizedActions,
|
|
741
|
-
autoScroll: autoScrollParam,
|
|
742
|
-
maxTokens,
|
|
743
|
-
extract,
|
|
744
|
-
readable: readable || false,
|
|
745
|
-
lite: args['lite'] || false,
|
|
746
|
-
question,
|
|
747
|
-
budget: args['lite'] ? undefined : (budgetArg ?? (maxTokens === undefined ? 4000 : undefined)),
|
|
748
|
-
};
|
|
749
|
-
const peeled = await Promise.race([
|
|
750
|
-
peel(url, options),
|
|
751
|
-
timeout(60000, 'MCP operation'),
|
|
752
|
-
]);
|
|
753
|
-
// For inline LLM extraction we need to attach extra fields; use a mutable wrapper
|
|
754
|
-
const result = peeled;
|
|
755
|
-
// Inline LLM extraction
|
|
756
|
-
if (inlineExtract && (inlineExtract.schema || inlineExtract.prompt) && llmApiKey && llmProvider) {
|
|
757
|
-
const validProviders = ['openai', 'anthropic', 'google'];
|
|
758
|
-
if (validProviders.includes(llmProvider)) {
|
|
759
|
-
const extractResult = await extractInlineJson(result.content, {
|
|
760
|
-
schema: inlineExtract.schema,
|
|
761
|
-
prompt: inlineExtract.prompt,
|
|
762
|
-
llmProvider: llmProvider,
|
|
763
|
-
llmApiKey,
|
|
764
|
-
llmModel,
|
|
765
|
-
});
|
|
766
|
-
result.json = extractResult.data;
|
|
767
|
-
result.extractTokensUsed = extractResult.tokensUsed;
|
|
768
|
-
}
|
|
769
|
-
}
|
|
770
|
-
const out = {
|
|
771
|
-
url: result.url || url,
|
|
772
|
-
title: result.title || result.metadata?.title || '',
|
|
773
|
-
tokens: result.tokens || 0,
|
|
774
|
-
content: result.content,
|
|
775
|
-
};
|
|
776
|
-
if (result.metadata)
|
|
777
|
-
out['metadata'] = result.metadata;
|
|
778
|
-
if (result.domainData)
|
|
779
|
-
out['domainData'] = result.domainData;
|
|
780
|
-
if (result.readability)
|
|
781
|
-
out['readability'] = { readingTime: result.readability.readingTime, wordCount: result.readability.wordCount };
|
|
782
|
-
if (result.quickAnswer)
|
|
783
|
-
out['quickAnswer'] = result.quickAnswer;
|
|
784
|
-
if (result.json)
|
|
785
|
-
out['json'] = result.json;
|
|
786
|
-
if (result.extracted)
|
|
787
|
-
out['extracted'] = result.extracted;
|
|
788
|
-
if (result.images?.length)
|
|
789
|
-
out['images'] = result.images;
|
|
790
|
-
if (result.screenshot)
|
|
791
|
-
out['screenshot'] = result.screenshot;
|
|
792
|
-
if (result.fingerprint)
|
|
793
|
-
out['fingerprint'] = result.fingerprint;
|
|
794
|
-
if (result.extractTokensUsed)
|
|
795
|
-
out['extractTokensUsed'] = result.extractTokensUsed;
|
|
796
|
-
if (result.quality !== undefined)
|
|
797
|
-
out['quality'] = result.quality;
|
|
798
|
-
if (result.timing)
|
|
799
|
-
out['timing'] = result.timing;
|
|
800
|
-
if (result.method)
|
|
801
|
-
out['method'] = result.method;
|
|
802
|
-
return textResponse(safeJson(out));
|
|
803
|
-
}
|
|
804
|
-
// ── ListToolsRequest — only the 7 public tools ─────────────────────────────────
|
|
805
|
-
server.setRequestHandler(ListToolsRequestSchema, async () => ({ tools: publicTools }));
|
|
806
|
-
// ── CallToolRequest — route to handlers ────────────────────────────────────────
|
|
26
|
+
// ── tools/list — return the 7 public tool definitions ─────────────────────
|
|
27
|
+
server.setRequestHandler(ListToolsRequestSchema, async () => ({
|
|
28
|
+
tools: toolDefinitions,
|
|
29
|
+
}));
|
|
30
|
+
// ── tools/call — dispatch to shared handler registry ──────────────────────
|
|
807
31
|
server.setRequestHandler(CallToolRequestSchema, async (request) => {
|
|
808
32
|
const { name, arguments: rawArgs } = request.params;
|
|
809
33
|
const args = (rawArgs || {});
|
|
810
34
|
try {
|
|
811
|
-
|
|
812
|
-
if (
|
|
813
|
-
|
|
814
|
-
|
|
815
|
-
|
|
816
|
-
|
|
817
|
-
const routedArgs = { ...parsed.params };
|
|
818
|
-
if (parsed.url)
|
|
819
|
-
routedArgs['url'] = parsed.url;
|
|
820
|
-
switch (parsed.intent) {
|
|
821
|
-
case 'read':
|
|
822
|
-
return handleRead(routedArgs);
|
|
823
|
-
case 'see':
|
|
824
|
-
return handleSee(routedArgs);
|
|
825
|
-
case 'find':
|
|
826
|
-
if (parsed.query)
|
|
827
|
-
routedArgs['query'] = parsed.query;
|
|
828
|
-
return handleFind(routedArgs);
|
|
829
|
-
case 'extract':
|
|
830
|
-
return handleExtract(routedArgs);
|
|
831
|
-
case 'monitor':
|
|
832
|
-
return handleMonitor(routedArgs);
|
|
833
|
-
case 'act':
|
|
834
|
-
return handleAct(routedArgs);
|
|
835
|
-
}
|
|
836
|
-
}
|
|
837
|
-
// ── New public tools ─────────────────────────────────────────────────────
|
|
838
|
-
if (name === 'webpeel_read')
|
|
839
|
-
return handleRead(args);
|
|
840
|
-
if (name === 'webpeel_see')
|
|
841
|
-
return handleSee(args);
|
|
842
|
-
if (name === 'webpeel_find')
|
|
843
|
-
return handleFind(args);
|
|
844
|
-
if (name === 'webpeel_extract')
|
|
845
|
-
return handleExtract(args);
|
|
846
|
-
if (name === 'webpeel_monitor')
|
|
847
|
-
return handleMonitor(args);
|
|
848
|
-
if (name === 'webpeel_act')
|
|
849
|
-
return handleAct(args);
|
|
850
|
-
// ── Backward compatibility aliases ───────────────────────────────────────
|
|
851
|
-
// webpeel_fetch → webpeel_read (full-featured)
|
|
852
|
-
if (name === 'webpeel_fetch')
|
|
853
|
-
return handleFetch(args);
|
|
854
|
-
// webpeel_youtube → webpeel_read (YouTube auto-detected)
|
|
855
|
-
if (name === 'webpeel_youtube') {
|
|
856
|
-
const { getYouTubeTranscript } = await import('../core/youtube.js');
|
|
857
|
-
const url = args['url'];
|
|
858
|
-
if (!url || typeof url !== 'string')
|
|
859
|
-
throw new Error('url is required');
|
|
860
|
-
const language = args['language'] || 'en';
|
|
861
|
-
const transcript = await Promise.race([
|
|
862
|
-
getYouTubeTranscript(url, { language }),
|
|
863
|
-
timeout(60000, 'YouTube transcript'),
|
|
864
|
-
]);
|
|
865
|
-
return textResponse(safeJson(transcript));
|
|
866
|
-
}
|
|
867
|
-
// webpeel_summarize → webpeel_read(summary=true)
|
|
868
|
-
if (name === 'webpeel_summarize') {
|
|
869
|
-
const { url, llmApiKey, prompt, llmModel, llmBaseUrl, render } = args;
|
|
870
|
-
if (!url)
|
|
871
|
-
throw new Error('url is required');
|
|
872
|
-
if (!llmApiKey)
|
|
873
|
-
throw new Error('llmApiKey is required');
|
|
874
|
-
const options = {
|
|
875
|
-
render: render || false,
|
|
876
|
-
extract: {
|
|
877
|
-
prompt: prompt || 'Summarize this webpage in 2-3 sentences.',
|
|
878
|
-
llmApiKey,
|
|
879
|
-
llmModel: llmModel || 'gpt-4o-mini',
|
|
880
|
-
llmBaseUrl: llmBaseUrl || 'https://api.openai.com/v1',
|
|
881
|
-
},
|
|
882
|
-
};
|
|
883
|
-
const result = await Promise.race([peel(url, options), timeout(60000, 'Summarize')]);
|
|
884
|
-
return textResponse(safeJson({ url: result.url, title: result.title, summary: result.extracted }));
|
|
885
|
-
}
|
|
886
|
-
// webpeel_answer → webpeel_read(question=...)
|
|
887
|
-
if (name === 'webpeel_answer') {
|
|
888
|
-
const { question: q, searchProvider: sp, searchApiKey: sak, llmProvider: lp, llmApiKey: lak, llmModel: lm, maxSources: ms } = args;
|
|
889
|
-
if (!q)
|
|
890
|
-
throw new Error('question is required');
|
|
891
|
-
if (!lp)
|
|
892
|
-
throw new Error('llmProvider is required');
|
|
893
|
-
if (!lak)
|
|
894
|
-
throw new Error('llmApiKey is required');
|
|
895
|
-
const validLlm = ['openai', 'anthropic', 'google'];
|
|
896
|
-
if (!validLlm.includes(lp))
|
|
897
|
-
throw new Error('Invalid llmProvider');
|
|
898
|
-
const validSp = ['duckduckgo', 'brave', 'stealth', 'google'];
|
|
899
|
-
const resolvedSp = (sp && validSp.includes(sp) ? sp : 'duckduckgo');
|
|
900
|
-
const result = await Promise.race([
|
|
901
|
-
answerQuestion({
|
|
902
|
-
question: q, searchProvider: resolvedSp, searchApiKey: sak,
|
|
903
|
-
llmProvider: lp, llmApiKey: lak, llmModel: lm,
|
|
904
|
-
maxSources: Math.min(Math.max(ms ?? 5, 1), 10), stream: false,
|
|
905
|
-
}),
|
|
906
|
-
timeout(180000, 'Answer'),
|
|
907
|
-
]);
|
|
908
|
-
return textResponse(safeJson(result));
|
|
909
|
-
}
|
|
910
|
-
// webpeel_quick_answer → webpeel_read(question=...)
|
|
911
|
-
if (name === 'webpeel_quick_answer') {
|
|
912
|
-
const { url, question: q, maxPassages: mpArg, render } = args;
|
|
913
|
-
if (!url)
|
|
914
|
-
throw new Error('url is required');
|
|
915
|
-
if (!q)
|
|
916
|
-
throw new Error('question is required');
|
|
917
|
-
const maxPassages = typeof mpArg === 'number' ? Math.min(Math.max(mpArg, 1), 10) : 3;
|
|
918
|
-
const peelResult = await Promise.race([
|
|
919
|
-
peel(url, { render: render || false, format: 'markdown', budget: 8000 }),
|
|
920
|
-
timeout(60000, 'Quick answer fetch'),
|
|
921
|
-
]);
|
|
922
|
-
const qa = quickAnswer({ question: q, content: peelResult.content || '', url: peelResult.url || url, maxPassages });
|
|
923
|
-
return textResponse(safeJson({
|
|
924
|
-
url: peelResult.url || url, title: peelResult.title,
|
|
925
|
-
question: qa.question, answer: qa.answer, confidence: qa.confidence,
|
|
926
|
-
passages: qa.passages, method: qa.method,
|
|
927
|
-
}));
|
|
928
|
-
}
|
|
929
|
-
// webpeel_screenshot → webpeel_see
|
|
930
|
-
if (name === 'webpeel_screenshot') {
|
|
931
|
-
const { takeScreenshot } = await import('../core/screenshot.js');
|
|
932
|
-
const { url, fullPage, width, height, format, quality, waitFor, stealth, actions } = args;
|
|
933
|
-
if (!url)
|
|
934
|
-
throw new Error('url is required');
|
|
935
|
-
const result = await Promise.race([
|
|
936
|
-
takeScreenshot(url, { fullPage: fullPage || false, width, height, format: format || 'png', quality, waitFor, stealth: stealth || false, actions }),
|
|
937
|
-
timeout(60000, 'Screenshot'),
|
|
938
|
-
]);
|
|
939
|
-
return textResponse(safeJson({ url: result.url, format: result.format, contentType: result.contentType, screenshot: result.screenshot }));
|
|
940
|
-
}
|
|
941
|
-
// webpeel_search → webpeel_find(quick)
|
|
942
|
-
if (name === 'webpeel_search') {
|
|
943
|
-
return handleFind({ ...args, depth: 'quick' });
|
|
944
|
-
}
|
|
945
|
-
// webpeel_research → webpeel_find(deep)
|
|
946
|
-
if (name === 'webpeel_research') {
|
|
947
|
-
const { query, maxSources, maxDepth, llmApiKey, llmModel, llmBaseUrl, outputFormat, timeout: resTimeout } = args;
|
|
948
|
-
const { research } = await import('../core/research.js');
|
|
949
|
-
const result = await research({
|
|
950
|
-
query, maxSources: maxSources ?? 5, maxDepth: maxDepth ?? 1,
|
|
951
|
-
apiKey: llmApiKey, model: llmModel, baseUrl: llmBaseUrl,
|
|
952
|
-
outputFormat: outputFormat ?? 'report', timeout: resTimeout ?? 60000,
|
|
953
|
-
});
|
|
954
|
-
return textResponse(safeJson({
|
|
955
|
-
report: result.report, sources: result.sources,
|
|
956
|
-
totalSourcesFound: result.totalSourcesFound, sourcesConsulted: result.sourcesConsulted,
|
|
957
|
-
elapsed: result.elapsed, tokensUsed: result.tokensUsed, cost: result.cost,
|
|
958
|
-
}));
|
|
959
|
-
}
|
|
960
|
-
// webpeel_deep_fetch → webpeel_find(depth='deep')
|
|
961
|
-
if (name === 'webpeel_deep_fetch') {
|
|
962
|
-
const { query, count: countArg, format: formatArg } = args;
|
|
963
|
-
if (!query)
|
|
964
|
-
throw new Error('query is required');
|
|
965
|
-
return handleFind({ query, depth: 'deep', limit: Math.min(Math.max(countArg ?? 5, 1), 10), format: formatArg });
|
|
966
|
-
}
|
|
967
|
-
// webpeel_map → webpeel_find(url=...)
|
|
968
|
-
if (name === 'webpeel_map') {
|
|
969
|
-
const { url, maxUrls, includePatterns, excludePatterns } = args;
|
|
970
|
-
if (!url)
|
|
971
|
-
throw new Error('url is required');
|
|
972
|
-
const { mapDomain } = await import('../core/map.js');
|
|
973
|
-
const results = await Promise.race([
|
|
974
|
-
mapDomain(url, { maxUrls, includePatterns, excludePatterns }),
|
|
975
|
-
timeout(600000, 'Map'),
|
|
976
|
-
]);
|
|
977
|
-
return textResponse(safeJson(results));
|
|
978
|
-
}
|
|
979
|
-
// webpeel_brand → webpeel_extract(fields=['name','logo','colors','fonts','socials'])
|
|
980
|
-
if (name === 'webpeel_brand') {
|
|
981
|
-
return handleExtract({ ...args, _brand: true });
|
|
982
|
-
}
|
|
983
|
-
// webpeel_auto_extract → webpeel_extract (auto mode)
|
|
984
|
-
if (name === 'webpeel_auto_extract') {
|
|
985
|
-
return handleExtract(args);
|
|
986
|
-
}
|
|
987
|
-
// webpeel_extract (legacy — same as new)
|
|
988
|
-
if (name === 'webpeel_extract') {
|
|
989
|
-
return handleExtract(args);
|
|
990
|
-
}
|
|
991
|
-
// webpeel_change_track → webpeel_monitor
|
|
992
|
-
if (name === 'webpeel_change_track')
|
|
993
|
-
return handleMonitor(args);
|
|
994
|
-
// webpeel_watch → webpeel_monitor
|
|
995
|
-
if (name === 'webpeel_watch') {
|
|
996
|
-
const action = args['action'];
|
|
997
|
-
if (!action || action === 'list' || action === 'check' || action === 'delete') {
|
|
998
|
-
return textResponse(safeJson({
|
|
999
|
-
message: 'URL watching requires the hosted API (api.webpeel.dev). Use webpeel_monitor for one-time change detection.',
|
|
1000
|
-
}));
|
|
1001
|
-
}
|
|
1002
|
-
if (action === 'create') {
|
|
1003
|
-
return handleMonitor({ url: args['url'], webhook: args['webhookUrl'] });
|
|
1004
|
-
}
|
|
1005
|
-
return textResponse(safeJson({ message: 'URL watching requires the hosted API (api.webpeel.dev).' }));
|
|
1006
|
-
}
|
|
1007
|
-
// webpeel_batch — developer tool, keep as-is
|
|
1008
|
-
if (name === 'webpeel_batch') {
|
|
1009
|
-
const { urls, concurrency, render, format, selector } = args;
|
|
1010
|
-
if (!urls || !Array.isArray(urls))
|
|
1011
|
-
throw new Error('urls must be an array');
|
|
1012
|
-
if (urls.length === 0)
|
|
1013
|
-
throw new Error('urls cannot be empty');
|
|
1014
|
-
if (urls.length > 50)
|
|
1015
|
-
throw new Error('Too many URLs (max 50)');
|
|
1016
|
-
const options = {
|
|
1017
|
-
concurrency: concurrency || 3, render: render || false,
|
|
1018
|
-
format: format || 'markdown', selector,
|
|
1019
|
-
};
|
|
1020
|
-
const results = await Promise.race([
|
|
1021
|
-
peelBatch(urls, options),
|
|
1022
|
-
timeout(300000, 'Batch'),
|
|
1023
|
-
]);
|
|
1024
|
-
return textResponse(safeJson(results));
|
|
1025
|
-
}
|
|
1026
|
-
// webpeel_crawl — developer tool, keep as-is
|
|
1027
|
-
if (name === 'webpeel_crawl') {
|
|
1028
|
-
const { crawl } = await import('../core/crawler.js');
|
|
1029
|
-
const { url, maxPages, maxDepth, allowedDomains, excludePatterns, respectRobotsTxt, rateLimitMs, sitemapFirst, render, stealth } = args;
|
|
1030
|
-
if (!url)
|
|
1031
|
-
throw new Error('url is required');
|
|
1032
|
-
const results = await Promise.race([
|
|
1033
|
-
crawl(url, { maxPages, maxDepth, allowedDomains, excludePatterns, respectRobotsTxt, rateLimitMs, sitemapFirst, render, stealth }),
|
|
1034
|
-
timeout(600000, 'Crawl'),
|
|
1035
|
-
]);
|
|
1036
|
-
return textResponse(safeJson(results));
|
|
1037
|
-
}
|
|
1038
|
-
// webpeel_hotels — deprecated
|
|
1039
|
-
if (name === 'webpeel_hotels') {
|
|
1040
|
-
return textResponse(safeJson({ message: 'This tool has been deprecated.' }));
|
|
1041
|
-
}
|
|
1042
|
-
// agent — deprecated
|
|
1043
|
-
if (name === 'agent') {
|
|
1044
|
-
// Keep backward compat: if llmApiKey provided, run agent
|
|
1045
|
-
const llmApiKey = args['llmApiKey'];
|
|
1046
|
-
if (llmApiKey) {
|
|
1047
|
-
const promptArg = args['prompt'];
|
|
1048
|
-
if (!promptArg)
|
|
1049
|
-
throw new Error('prompt is required');
|
|
1050
|
-
const result = await runAgent({
|
|
1051
|
-
prompt: promptArg,
|
|
1052
|
-
llmApiKey,
|
|
1053
|
-
urls: args['urls'],
|
|
1054
|
-
maxSources: args['maxResults'] || undefined,
|
|
1055
|
-
});
|
|
1056
|
-
return textResponse(safeJson(result));
|
|
1057
|
-
}
|
|
1058
|
-
// LLM-free mode
|
|
1059
|
-
const urlsArg = args['urls'] || [];
|
|
1060
|
-
const searchArg = args['search'];
|
|
1061
|
-
if (urlsArg.length === 0 && !searchArg) {
|
|
1062
|
-
return textResponse(safeJson({
|
|
1063
|
-
message: 'This tool has been deprecated. Use the webpeel tool instead.',
|
|
1064
|
-
}));
|
|
1065
|
-
}
|
|
1066
|
-
const promptArg = args['prompt'];
|
|
1067
|
-
const schema = args['schema'];
|
|
1068
|
-
const budgetArg = args['budget'] || 4000;
|
|
1069
|
-
const maxResults = Math.min(args['maxResults'] || 5, 20);
|
|
1070
|
-
const targetUrls = [...urlsArg];
|
|
1071
|
-
if (searchArg) {
|
|
1072
|
-
try {
|
|
1073
|
-
const { provider, apiKey } = getBestSearchProvider();
|
|
1074
|
-
const searchResults = await provider.searchWeb(searchArg, { count: Math.max(maxResults, 5), apiKey });
|
|
1075
|
-
for (const r of searchResults) {
|
|
1076
|
-
if (!targetUrls.includes(r.url))
|
|
1077
|
-
targetUrls.push(r.url);
|
|
1078
|
-
}
|
|
1079
|
-
}
|
|
1080
|
-
catch { /* continue with provided URLs */ }
|
|
1081
|
-
}
|
|
1082
|
-
const urlsToFetch = targetUrls.slice(0, maxResults);
|
|
1083
|
-
const agentResults = [];
|
|
1084
|
-
await Promise.all(urlsToFetch.map(async (u) => {
|
|
1085
|
-
try {
|
|
1086
|
-
const page = await peel(u, { budget: budgetArg, format: 'markdown' });
|
|
1087
|
-
const content = page.content || '';
|
|
1088
|
-
const title = page.title || u;
|
|
1089
|
-
let extracted = null;
|
|
1090
|
-
let confidence = 0;
|
|
1091
|
-
if (schema && Object.keys(schema).length > 0) {
|
|
1092
|
-
extracted = {};
|
|
1093
|
-
let total = 0;
|
|
1094
|
-
for (const [field] of Object.entries(schema)) {
|
|
1095
|
-
const q = promptArg ? `${promptArg} — specifically: what is the ${field}?` : `What is the ${field}?`;
|
|
1096
|
-
const qa = quickAnswer({ question: q, content, maxPassages: 1, url: u });
|
|
1097
|
-
extracted[field] = qa.answer || '';
|
|
1098
|
-
total += qa.confidence;
|
|
1099
|
-
}
|
|
1100
|
-
if ('source' in schema)
|
|
1101
|
-
extracted['source'] = u;
|
|
1102
|
-
confidence = Object.keys(schema).length > 0 ? total / Object.keys(schema).length : 0;
|
|
1103
|
-
}
|
|
1104
|
-
else if (promptArg) {
|
|
1105
|
-
const qa = quickAnswer({ question: promptArg, content, maxPassages: 3, url: u });
|
|
1106
|
-
confidence = qa.confidence;
|
|
1107
|
-
}
|
|
1108
|
-
agentResults.push({ url: u, title, extracted, content: content.slice(0, 500) + (content.length > 500 ? '…' : ''), confidence });
|
|
1109
|
-
}
|
|
1110
|
-
catch { /* skip failed URLs */ }
|
|
1111
|
-
}));
|
|
1112
|
-
return textResponse(safeJson({
|
|
1113
|
-
success: true,
|
|
1114
|
-
data: { results: agentResults, totalSources: agentResults.length },
|
|
1115
|
-
}));
|
|
1116
|
-
}
|
|
1117
|
-
throw new Error(`Unknown tool: ${name}`);
|
|
35
|
+
const handler = getHandler(name);
|
|
36
|
+
if (!handler)
|
|
37
|
+
throw new Error(`Unknown tool: ${name}`);
|
|
38
|
+
// Standalone has no accountId / pool context
|
|
39
|
+
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
|
40
|
+
return (await handler(args));
|
|
1118
41
|
}
|
|
1119
42
|
catch (error) {
|
|
1120
43
|
const err = error;
|
|
1121
44
|
return {
|
|
1122
|
-
content: [
|
|
45
|
+
content: [
|
|
46
|
+
{
|
|
47
|
+
type: 'text',
|
|
48
|
+
text: JSON.stringify({
|
|
49
|
+
error: err.name || 'Error',
|
|
50
|
+
message: err.message || 'Unknown error',
|
|
51
|
+
}),
|
|
52
|
+
},
|
|
53
|
+
],
|
|
1123
54
|
isError: true,
|
|
1124
55
|
};
|
|
1125
56
|
}
|
|
1126
57
|
});
|
|
1127
|
-
// ── Main
|
|
58
|
+
// ── Main — stdio or HTTP mode ──────────────────────────────────────────────
|
|
1128
59
|
async function main() {
|
|
1129
60
|
const isHttpMode = process.env['MCP_HTTP_MODE'] === 'true' ||
|
|
1130
61
|
process.env['HTTP_STREAMABLE_SERVER'] === 'true';
|
|
@@ -1143,9 +74,13 @@ async function main() {
|
|
|
1143
74
|
await transport.handleRequest(r, s, r.body);
|
|
1144
75
|
transport.close().catch(() => { });
|
|
1145
76
|
}
|
|
1146
|
-
catch
|
|
77
|
+
catch {
|
|
1147
78
|
if (!s.headersSent) {
|
|
1148
|
-
s.status(500).json({
|
|
79
|
+
s.status(500).json({
|
|
80
|
+
jsonrpc: '2.0',
|
|
81
|
+
error: { code: -32603, message: 'Internal error' },
|
|
82
|
+
id: null,
|
|
83
|
+
});
|
|
1149
84
|
}
|
|
1150
85
|
}
|
|
1151
86
|
});
|
|
@@ -1171,4 +106,3 @@ main().catch((error) => {
|
|
|
1171
106
|
process.stderr.write(`Fatal error: ${error}\n`);
|
|
1172
107
|
process.exit(1);
|
|
1173
108
|
});
|
|
1174
|
-
//# sourceMappingURL=server.js.map
|