webpeel 0.19.4 → 0.20.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +2 -2
- package/dist/cache.d.ts +0 -1
- package/dist/cache.js +0 -1
- package/dist/cli/commands/auth.d.ts +5 -0
- package/dist/cli/commands/auth.js +476 -0
- package/dist/cli/commands/fetch.d.ts +6 -0
- package/dist/cli/commands/fetch.js +1015 -0
- package/dist/cli/commands/interact.d.ts +5 -0
- package/dist/cli/commands/interact.js +839 -0
- package/dist/cli/commands/jobs.d.ts +5 -0
- package/dist/cli/commands/jobs.js +997 -0
- package/dist/cli/commands/screenshot.d.ts +5 -0
- package/dist/cli/commands/screenshot.js +273 -0
- package/dist/cli/commands/search.d.ts +5 -0
- package/dist/cli/commands/search.js +524 -0
- package/dist/cli/utils.d.ts +84 -0
- package/dist/cli/utils.js +686 -0
- package/dist/cli-auth.d.ts +0 -1
- package/dist/cli-auth.js +0 -1
- package/dist/cli.d.ts +7 -6
- package/dist/cli.js +35 -4698
- package/dist/core/actions.d.ts +0 -1
- package/dist/core/actions.js +0 -1
- package/dist/core/agent.d.ts +0 -1
- package/dist/core/agent.js +9 -12
- package/dist/core/answer.d.ts +0 -1
- package/dist/core/answer.js +0 -1
- package/dist/core/application-tracker.d.ts +0 -1
- package/dist/core/application-tracker.js +0 -1
- package/dist/core/apply.d.ts +0 -1
- package/dist/core/apply.js +0 -1
- package/dist/core/auto-extract.d.ts +0 -1
- package/dist/core/auto-extract.js +0 -1
- package/dist/core/auto-interact.d.ts +0 -1
- package/dist/core/auto-interact.js +0 -1
- package/dist/core/bm25-filter.d.ts +0 -1
- package/dist/core/bm25-filter.js +0 -1
- package/dist/core/branding.d.ts +0 -1
- package/dist/core/branding.js +0 -1
- package/dist/core/browser-fetch.d.ts +0 -1
- package/dist/core/browser-fetch.js +17 -10
- package/dist/core/browser-pool.d.ts +0 -1
- package/dist/core/browser-pool.js +0 -1
- package/dist/core/budget.d.ts +0 -1
- package/dist/core/budget.js +0 -1
- package/dist/core/cache.d.ts +0 -1
- package/dist/core/cache.js +0 -1
- package/dist/core/cf-worker-proxy.d.ts +0 -1
- package/dist/core/cf-worker-proxy.js +0 -1
- package/dist/core/challenge-detection.d.ts +0 -1
- package/dist/core/challenge-detection.js +0 -1
- package/dist/core/change-tracking.d.ts +0 -1
- package/dist/core/change-tracking.js +0 -1
- package/dist/core/chunker.d.ts +0 -1
- package/dist/core/chunker.js +0 -1
- package/dist/core/chunking.d.ts +0 -1
- package/dist/core/chunking.js +0 -1
- package/dist/core/cloak-fetch.d.ts +0 -1
- package/dist/core/cloak-fetch.js +0 -1
- package/dist/core/content-pruner.d.ts +0 -1
- package/dist/core/content-pruner.js +0 -1
- package/dist/core/crawl-checkpoint.d.ts +0 -1
- package/dist/core/crawl-checkpoint.js +0 -1
- package/dist/core/crawler.d.ts +0 -1
- package/dist/core/crawler.js +6 -5
- package/dist/core/cycle-fetch.d.ts +0 -1
- package/dist/core/cycle-fetch.js +0 -1
- package/dist/core/deep-fetch.d.ts +0 -1
- package/dist/core/deep-fetch.js +0 -1
- package/dist/core/design-analysis.d.ts +0 -1
- package/dist/core/design-analysis.js +0 -1
- package/dist/core/design-compare.d.ts +0 -1
- package/dist/core/design-compare.js +0 -1
- package/dist/core/diff.d.ts +0 -1
- package/dist/core/diff.js +0 -1
- package/dist/core/dns-cache.d.ts +0 -1
- package/dist/core/dns-cache.js +0 -1
- package/dist/core/documents.d.ts +0 -1
- package/dist/core/documents.js +0 -1
- package/dist/core/domain-extractors.d.ts +0 -1
- package/dist/core/domain-extractors.js +0 -1
- package/dist/core/extract-inline.d.ts +0 -1
- package/dist/core/extract-inline.js +0 -1
- package/dist/core/extract-listings.d.ts +0 -1
- package/dist/core/extract-listings.js +0 -1
- package/dist/core/extract.d.ts +0 -1
- package/dist/core/extract.js +0 -1
- package/dist/core/fetcher.d.ts +0 -1
- package/dist/core/fetcher.js +0 -1
- package/dist/core/google-cache.d.ts +0 -1
- package/dist/core/google-cache.js +0 -1
- package/dist/core/hotel-search.d.ts +0 -1
- package/dist/core/hotel-search.js +0 -1
- package/dist/core/http-fetch.d.ts +0 -1
- package/dist/core/http-fetch.js +5 -7
- package/dist/core/human.d.ts +0 -1
- package/dist/core/human.js +0 -1
- package/dist/core/jobs.d.ts +0 -1
- package/dist/core/jobs.js +0 -1
- package/dist/core/json-ld.d.ts +0 -1
- package/dist/core/json-ld.js +0 -1
- package/dist/core/llm-extract.d.ts +0 -1
- package/dist/core/llm-extract.js +0 -1
- package/dist/core/logger.d.ts +17 -0
- package/dist/core/logger.js +44 -0
- package/dist/core/map.d.ts +0 -1
- package/dist/core/map.js +0 -1
- package/dist/core/markdown.d.ts +0 -1
- package/dist/core/markdown.js +0 -1
- package/dist/core/metadata.d.ts +0 -1
- package/dist/core/metadata.js +0 -1
- package/dist/core/paginate.d.ts +0 -1
- package/dist/core/paginate.js +0 -1
- package/dist/core/pdf.d.ts +0 -1
- package/dist/core/pdf.js +0 -1
- package/dist/core/peel-tls.d.ts +0 -1
- package/dist/core/peel-tls.js +0 -1
- package/dist/core/pipeline.d.ts +0 -1
- package/dist/core/pipeline.js +22 -25
- package/dist/core/profiles.d.ts +0 -1
- package/dist/core/profiles.js +0 -1
- package/dist/core/quick-answer.d.ts +0 -1
- package/dist/core/quick-answer.js +0 -1
- package/dist/core/rate-governor.d.ts +0 -1
- package/dist/core/rate-governor.js +0 -1
- package/dist/core/readability.d.ts +0 -1
- package/dist/core/readability.js +0 -1
- package/dist/core/research.d.ts +0 -1
- package/dist/core/research.js +0 -1
- package/dist/core/schema-extraction.d.ts +0 -1
- package/dist/core/schema-extraction.js +0 -1
- package/dist/core/schema-postprocess.d.ts +0 -1
- package/dist/core/schema-postprocess.js +0 -1
- package/dist/core/schema-templates.d.ts +0 -1
- package/dist/core/schema-templates.js +0 -1
- package/dist/core/screenshot.d.ts +0 -1
- package/dist/core/screenshot.js +0 -1
- package/dist/core/search-fallback.d.ts +0 -1
- package/dist/core/search-fallback.js +0 -1
- package/dist/core/search-provider.d.ts +0 -1
- package/dist/core/search-provider.js +18 -21
- package/dist/core/site-search.d.ts +0 -1
- package/dist/core/site-search.js +0 -1
- package/dist/core/sitemap.d.ts +0 -1
- package/dist/core/sitemap.js +0 -1
- package/dist/core/stealth-patches.d.ts +0 -1
- package/dist/core/stealth-patches.js +0 -1
- package/dist/core/stemmer.d.ts +0 -1
- package/dist/core/stemmer.js +0 -1
- package/dist/core/strategies.d.ts +6 -1
- package/dist/core/strategies.js +29 -41
- package/dist/core/strategy-hooks.d.ts +0 -1
- package/dist/core/strategy-hooks.js +0 -1
- package/dist/core/summarize.d.ts +0 -1
- package/dist/core/summarize.js +0 -1
- package/dist/core/synonyms.d.ts +0 -1
- package/dist/core/synonyms.js +0 -1
- package/dist/core/table-format.d.ts +0 -1
- package/dist/core/table-format.js +0 -1
- package/dist/core/timing.d.ts +0 -1
- package/dist/core/timing.js +0 -1
- package/dist/core/user-agents.d.ts +0 -1
- package/dist/core/user-agents.js +0 -1
- package/dist/core/watch-manager.d.ts +0 -1
- package/dist/core/watch-manager.js +0 -1
- package/dist/core/watch.d.ts +0 -1
- package/dist/core/watch.js +0 -1
- package/dist/core/youtube.d.ts +0 -1
- package/dist/core/youtube.js +0 -1
- package/dist/index.d.ts +8 -3
- package/dist/index.js +27 -3
- package/dist/integrations/index.d.ts +0 -1
- package/dist/integrations/index.js +0 -1
- package/dist/integrations/langchain.d.ts +0 -1
- package/dist/integrations/langchain.js +0 -1
- package/dist/integrations/llamaindex.d.ts +0 -1
- package/dist/integrations/llamaindex.js +0 -1
- package/dist/mcp/handlers/act.d.ts +5 -0
- package/dist/mcp/handlers/act.js +34 -0
- package/dist/mcp/handlers/definitions.d.ts +6 -0
- package/dist/mcp/handlers/definitions.js +266 -0
- package/dist/mcp/handlers/extract.d.ts +6 -0
- package/dist/mcp/handlers/extract.js +102 -0
- package/dist/mcp/handlers/fetch.d.ts +6 -0
- package/dist/mcp/handlers/fetch.js +98 -0
- package/dist/mcp/handlers/find.d.ts +5 -0
- package/dist/mcp/handlers/find.js +137 -0
- package/dist/mcp/handlers/index.d.ts +13 -0
- package/dist/mcp/handlers/index.js +61 -0
- package/dist/mcp/handlers/legacy.d.ts +25 -0
- package/dist/mcp/handlers/legacy.js +450 -0
- package/dist/mcp/handlers/meta.d.ts +6 -0
- package/dist/mcp/handlers/meta.js +31 -0
- package/dist/mcp/handlers/monitor.d.ts +5 -0
- package/dist/mcp/handlers/monitor.js +41 -0
- package/dist/mcp/handlers/read.d.ts +6 -0
- package/dist/mcp/handlers/read.js +63 -0
- package/dist/mcp/handlers/see.d.ts +5 -0
- package/dist/mcp/handlers/see.js +75 -0
- package/dist/mcp/handlers/types.d.ts +29 -0
- package/dist/mcp/handlers/types.js +28 -0
- package/dist/mcp/server.d.ts +3 -4
- package/dist/mcp/server.js +35 -1101
- package/dist/mcp/smart-router.d.ts +0 -1
- package/dist/mcp/smart-router.js +3 -1
- package/dist/types.d.ts +6 -1
- package/dist/types.js +0 -1
- package/package.json +3 -13
- package/dist/cache.d.ts.map +0 -1
- package/dist/cache.js.map +0 -1
- package/dist/cli-auth.d.ts.map +0 -1
- package/dist/cli-auth.js.map +0 -1
- package/dist/cli.bundle.cjs +0 -159248
- package/dist/cli.d.ts.map +0 -1
- package/dist/cli.js.map +0 -1
- package/dist/core/actions.d.ts.map +0 -1
- package/dist/core/actions.js.map +0 -1
- package/dist/core/agent.d.ts.map +0 -1
- package/dist/core/agent.js.map +0 -1
- package/dist/core/answer.d.ts.map +0 -1
- package/dist/core/answer.js.map +0 -1
- package/dist/core/application-tracker.d.ts.map +0 -1
- package/dist/core/application-tracker.js.map +0 -1
- package/dist/core/apply.d.ts.map +0 -1
- package/dist/core/apply.js.map +0 -1
- package/dist/core/auto-extract.d.ts.map +0 -1
- package/dist/core/auto-extract.js.map +0 -1
- package/dist/core/auto-interact.d.ts.map +0 -1
- package/dist/core/auto-interact.js.map +0 -1
- package/dist/core/bm25-filter.d.ts.map +0 -1
- package/dist/core/bm25-filter.js.map +0 -1
- package/dist/core/branding.d.ts.map +0 -1
- package/dist/core/branding.js.map +0 -1
- package/dist/core/browser-fetch.d.ts.map +0 -1
- package/dist/core/browser-fetch.js.map +0 -1
- package/dist/core/browser-pool.d.ts.map +0 -1
- package/dist/core/browser-pool.js.map +0 -1
- package/dist/core/budget.d.ts.map +0 -1
- package/dist/core/budget.js.map +0 -1
- package/dist/core/cache.d.ts.map +0 -1
- package/dist/core/cache.js.map +0 -1
- package/dist/core/cf-worker-proxy.d.ts.map +0 -1
- package/dist/core/cf-worker-proxy.js.map +0 -1
- package/dist/core/challenge-detection.d.ts.map +0 -1
- package/dist/core/challenge-detection.js.map +0 -1
- package/dist/core/change-tracking.d.ts.map +0 -1
- package/dist/core/change-tracking.js.map +0 -1
- package/dist/core/chunker.d.ts.map +0 -1
- package/dist/core/chunker.js.map +0 -1
- package/dist/core/chunking.d.ts.map +0 -1
- package/dist/core/chunking.js.map +0 -1
- package/dist/core/cloak-fetch.d.ts.map +0 -1
- package/dist/core/cloak-fetch.js.map +0 -1
- package/dist/core/content-pruner.d.ts.map +0 -1
- package/dist/core/content-pruner.js.map +0 -1
- package/dist/core/crawl-checkpoint.d.ts.map +0 -1
- package/dist/core/crawl-checkpoint.js.map +0 -1
- package/dist/core/crawler.d.ts.map +0 -1
- package/dist/core/crawler.js.map +0 -1
- package/dist/core/cycle-fetch.d.ts.map +0 -1
- package/dist/core/cycle-fetch.js.map +0 -1
- package/dist/core/deep-fetch.d.ts.map +0 -1
- package/dist/core/deep-fetch.js.map +0 -1
- package/dist/core/design-analysis.d.ts.map +0 -1
- package/dist/core/design-analysis.js.map +0 -1
- package/dist/core/design-compare.d.ts.map +0 -1
- package/dist/core/design-compare.js.map +0 -1
- package/dist/core/diff.d.ts.map +0 -1
- package/dist/core/diff.js.map +0 -1
- package/dist/core/dns-cache.d.ts.map +0 -1
- package/dist/core/dns-cache.js.map +0 -1
- package/dist/core/documents.d.ts.map +0 -1
- package/dist/core/documents.js.map +0 -1
- package/dist/core/domain-extractors.d.ts.map +0 -1
- package/dist/core/domain-extractors.js.map +0 -1
- package/dist/core/extract-inline.d.ts.map +0 -1
- package/dist/core/extract-inline.js.map +0 -1
- package/dist/core/extract-listings.d.ts.map +0 -1
- package/dist/core/extract-listings.js.map +0 -1
- package/dist/core/extract.d.ts.map +0 -1
- package/dist/core/extract.js.map +0 -1
- package/dist/core/fetcher.d.ts.map +0 -1
- package/dist/core/fetcher.js.map +0 -1
- package/dist/core/google-cache.d.ts.map +0 -1
- package/dist/core/google-cache.js.map +0 -1
- package/dist/core/hotel-search.d.ts.map +0 -1
- package/dist/core/hotel-search.js.map +0 -1
- package/dist/core/http-fetch.d.ts.map +0 -1
- package/dist/core/http-fetch.js.map +0 -1
- package/dist/core/human.d.ts.map +0 -1
- package/dist/core/human.js.map +0 -1
- package/dist/core/jobs.d.ts.map +0 -1
- package/dist/core/jobs.js.map +0 -1
- package/dist/core/json-ld.d.ts.map +0 -1
- package/dist/core/json-ld.js.map +0 -1
- package/dist/core/llm-extract.d.ts.map +0 -1
- package/dist/core/llm-extract.js.map +0 -1
- package/dist/core/map.d.ts.map +0 -1
- package/dist/core/map.js.map +0 -1
- package/dist/core/markdown.d.ts.map +0 -1
- package/dist/core/markdown.js.map +0 -1
- package/dist/core/metadata.d.ts.map +0 -1
- package/dist/core/metadata.js.map +0 -1
- package/dist/core/paginate.d.ts.map +0 -1
- package/dist/core/paginate.js.map +0 -1
- package/dist/core/pdf.d.ts.map +0 -1
- package/dist/core/pdf.js.map +0 -1
- package/dist/core/peel-tls.d.ts.map +0 -1
- package/dist/core/peel-tls.js.map +0 -1
- package/dist/core/pipeline.d.ts.map +0 -1
- package/dist/core/pipeline.js.map +0 -1
- package/dist/core/profiles.d.ts.map +0 -1
- package/dist/core/profiles.js.map +0 -1
- package/dist/core/quick-answer.d.ts.map +0 -1
- package/dist/core/quick-answer.js.map +0 -1
- package/dist/core/rate-governor.d.ts.map +0 -1
- package/dist/core/rate-governor.js.map +0 -1
- package/dist/core/readability.d.ts.map +0 -1
- package/dist/core/readability.js.map +0 -1
- package/dist/core/research.d.ts.map +0 -1
- package/dist/core/research.js.map +0 -1
- package/dist/core/schema-extraction.d.ts.map +0 -1
- package/dist/core/schema-extraction.js.map +0 -1
- package/dist/core/schema-postprocess.d.ts.map +0 -1
- package/dist/core/schema-postprocess.js.map +0 -1
- package/dist/core/schema-templates.d.ts.map +0 -1
- package/dist/core/schema-templates.js.map +0 -1
- package/dist/core/screenshot.d.ts.map +0 -1
- package/dist/core/screenshot.js.map +0 -1
- package/dist/core/search-fallback.d.ts.map +0 -1
- package/dist/core/search-fallback.js.map +0 -1
- package/dist/core/search-provider.d.ts.map +0 -1
- package/dist/core/search-provider.js.map +0 -1
- package/dist/core/site-search.d.ts.map +0 -1
- package/dist/core/site-search.js.map +0 -1
- package/dist/core/sitemap.d.ts.map +0 -1
- package/dist/core/sitemap.js.map +0 -1
- package/dist/core/stealth-patches.d.ts.map +0 -1
- package/dist/core/stealth-patches.js.map +0 -1
- package/dist/core/stemmer.d.ts.map +0 -1
- package/dist/core/stemmer.js.map +0 -1
- package/dist/core/strategies.d.ts.map +0 -1
- package/dist/core/strategies.js.map +0 -1
- package/dist/core/strategy-hooks.d.ts.map +0 -1
- package/dist/core/strategy-hooks.js.map +0 -1
- package/dist/core/summarize.d.ts.map +0 -1
- package/dist/core/summarize.js.map +0 -1
- package/dist/core/synonyms.d.ts.map +0 -1
- package/dist/core/synonyms.js.map +0 -1
- package/dist/core/table-format.d.ts.map +0 -1
- package/dist/core/table-format.js.map +0 -1
- package/dist/core/timing.d.ts.map +0 -1
- package/dist/core/timing.js.map +0 -1
- package/dist/core/user-agents.d.ts.map +0 -1
- package/dist/core/user-agents.js.map +0 -1
- package/dist/core/watch-manager.d.ts.map +0 -1
- package/dist/core/watch-manager.js.map +0 -1
- package/dist/core/watch.d.ts.map +0 -1
- package/dist/core/watch.js.map +0 -1
- package/dist/core/youtube.d.ts.map +0 -1
- package/dist/core/youtube.js.map +0 -1
- package/dist/index.d.ts.map +0 -1
- package/dist/index.js.map +0 -1
- package/dist/integrations/index.d.ts.map +0 -1
- package/dist/integrations/index.js.map +0 -1
- package/dist/integrations/langchain.d.ts.map +0 -1
- package/dist/integrations/langchain.js.map +0 -1
- package/dist/integrations/llamaindex.d.ts.map +0 -1
- package/dist/integrations/llamaindex.js.map +0 -1
- package/dist/mcp/server.d.ts.map +0 -1
- package/dist/mcp/server.js.map +0 -1
- package/dist/mcp/smart-router.d.ts.map +0 -1
- package/dist/mcp/smart-router.js.map +0 -1
- package/dist/server/app.d.ts +0 -15
- package/dist/server/app.d.ts.map +0 -1
- package/dist/server/app.js +0 -350
- package/dist/server/app.js.map +0 -1
- package/dist/server/auth-store.d.ts +0 -28
- package/dist/server/auth-store.d.ts.map +0 -1
- package/dist/server/auth-store.js +0 -89
- package/dist/server/auth-store.js.map +0 -1
- package/dist/server/email-service.d.ts +0 -22
- package/dist/server/email-service.d.ts.map +0 -1
- package/dist/server/email-service.js +0 -80
- package/dist/server/email-service.js.map +0 -1
- package/dist/server/job-queue.d.ts +0 -93
- package/dist/server/job-queue.d.ts.map +0 -1
- package/dist/server/job-queue.js +0 -146
- package/dist/server/job-queue.js.map +0 -1
- package/dist/server/logger.d.ts +0 -11
- package/dist/server/logger.d.ts.map +0 -1
- package/dist/server/logger.js +0 -38
- package/dist/server/logger.js.map +0 -1
- package/dist/server/middleware/auth.d.ts +0 -29
- package/dist/server/middleware/auth.d.ts.map +0 -1
- package/dist/server/middleware/auth.js +0 -222
- package/dist/server/middleware/auth.js.map +0 -1
- package/dist/server/middleware/rate-limit.d.ts +0 -25
- package/dist/server/middleware/rate-limit.d.ts.map +0 -1
- package/dist/server/middleware/rate-limit.js +0 -168
- package/dist/server/middleware/rate-limit.js.map +0 -1
- package/dist/server/middleware/url-validator.d.ts +0 -16
- package/dist/server/middleware/url-validator.d.ts.map +0 -1
- package/dist/server/middleware/url-validator.js +0 -187
- package/dist/server/middleware/url-validator.js.map +0 -1
- package/dist/server/openapi.yaml +0 -4944
- package/dist/server/pg-auth-store.d.ts +0 -133
- package/dist/server/pg-auth-store.d.ts.map +0 -1
- package/dist/server/pg-auth-store.js +0 -473
- package/dist/server/pg-auth-store.js.map +0 -1
- package/dist/server/pg-job-queue.d.ts +0 -60
- package/dist/server/pg-job-queue.d.ts.map +0 -1
- package/dist/server/pg-job-queue.js +0 -365
- package/dist/server/pg-job-queue.js.map +0 -1
- package/dist/server/premium/domain-intel.d.ts +0 -17
- package/dist/server/premium/domain-intel.d.ts.map +0 -1
- package/dist/server/premium/domain-intel.js +0 -134
- package/dist/server/premium/domain-intel.js.map +0 -1
- package/dist/server/premium/index.d.ts +0 -18
- package/dist/server/premium/index.d.ts.map +0 -1
- package/dist/server/premium/index.js +0 -36
- package/dist/server/premium/index.js.map +0 -1
- package/dist/server/premium/swr-cache.d.ts +0 -15
- package/dist/server/premium/swr-cache.d.ts.map +0 -1
- package/dist/server/premium/swr-cache.js +0 -35
- package/dist/server/premium/swr-cache.js.map +0 -1
- package/dist/server/routes/activity.d.ts +0 -7
- package/dist/server/routes/activity.d.ts.map +0 -1
- package/dist/server/routes/activity.js +0 -68
- package/dist/server/routes/activity.js.map +0 -1
- package/dist/server/routes/agent.d.ts +0 -16
- package/dist/server/routes/agent.d.ts.map +0 -1
- package/dist/server/routes/agent.js +0 -247
- package/dist/server/routes/agent.js.map +0 -1
- package/dist/server/routes/answer.d.ts +0 -6
- package/dist/server/routes/answer.d.ts.map +0 -1
- package/dist/server/routes/answer.js +0 -133
- package/dist/server/routes/answer.js.map +0 -1
- package/dist/server/routes/ask.d.ts +0 -23
- package/dist/server/routes/ask.d.ts.map +0 -1
- package/dist/server/routes/ask.js +0 -119
- package/dist/server/routes/ask.js.map +0 -1
- package/dist/server/routes/batch.d.ts +0 -7
- package/dist/server/routes/batch.d.ts.map +0 -1
- package/dist/server/routes/batch.js +0 -412
- package/dist/server/routes/batch.js.map +0 -1
- package/dist/server/routes/cli-usage.d.ts +0 -7
- package/dist/server/routes/cli-usage.d.ts.map +0 -1
- package/dist/server/routes/cli-usage.js +0 -121
- package/dist/server/routes/cli-usage.js.map +0 -1
- package/dist/server/routes/compat.d.ts +0 -24
- package/dist/server/routes/compat.d.ts.map +0 -1
- package/dist/server/routes/compat.js +0 -653
- package/dist/server/routes/compat.js.map +0 -1
- package/dist/server/routes/deep-fetch.d.ts +0 -9
- package/dist/server/routes/deep-fetch.d.ts.map +0 -1
- package/dist/server/routes/deep-fetch.js +0 -50
- package/dist/server/routes/deep-fetch.js.map +0 -1
- package/dist/server/routes/demo.d.ts +0 -25
- package/dist/server/routes/demo.d.ts.map +0 -1
- package/dist/server/routes/demo.js +0 -434
- package/dist/server/routes/demo.js.map +0 -1
- package/dist/server/routes/extract.d.ts +0 -9
- package/dist/server/routes/extract.d.ts.map +0 -1
- package/dist/server/routes/extract.js +0 -150
- package/dist/server/routes/extract.js.map +0 -1
- package/dist/server/routes/fetch.d.ts +0 -8
- package/dist/server/routes/fetch.d.ts.map +0 -1
- package/dist/server/routes/fetch.js +0 -988
- package/dist/server/routes/fetch.js.map +0 -1
- package/dist/server/routes/health.d.ts +0 -8
- package/dist/server/routes/health.d.ts.map +0 -1
- package/dist/server/routes/health.js +0 -20
- package/dist/server/routes/health.js.map +0 -1
- package/dist/server/routes/jobs.d.ts +0 -8
- package/dist/server/routes/jobs.d.ts.map +0 -1
- package/dist/server/routes/jobs.js +0 -487
- package/dist/server/routes/jobs.js.map +0 -1
- package/dist/server/routes/mcp.d.ts +0 -18
- package/dist/server/routes/mcp.d.ts.map +0 -1
- package/dist/server/routes/mcp.js +0 -1260
- package/dist/server/routes/mcp.js.map +0 -1
- package/dist/server/routes/oauth.d.ts +0 -10
- package/dist/server/routes/oauth.d.ts.map +0 -1
- package/dist/server/routes/oauth.js +0 -334
- package/dist/server/routes/oauth.js.map +0 -1
- package/dist/server/routes/quick-answer.d.ts +0 -9
- package/dist/server/routes/quick-answer.d.ts.map +0 -1
- package/dist/server/routes/quick-answer.js +0 -93
- package/dist/server/routes/quick-answer.js.map +0 -1
- package/dist/server/routes/screenshot.d.ts +0 -23
- package/dist/server/routes/screenshot.d.ts.map +0 -1
- package/dist/server/routes/screenshot.js +0 -819
- package/dist/server/routes/screenshot.js.map +0 -1
- package/dist/server/routes/search.d.ts +0 -7
- package/dist/server/routes/search.d.ts.map +0 -1
- package/dist/server/routes/search.js +0 -312
- package/dist/server/routes/search.js.map +0 -1
- package/dist/server/routes/session.d.ts +0 -16
- package/dist/server/routes/session.d.ts.map +0 -1
- package/dist/server/routes/session.js +0 -278
- package/dist/server/routes/session.js.map +0 -1
- package/dist/server/routes/stats.d.ts +0 -7
- package/dist/server/routes/stats.d.ts.map +0 -1
- package/dist/server/routes/stats.js +0 -65
- package/dist/server/routes/stats.js.map +0 -1
- package/dist/server/routes/stripe.d.ts +0 -16
- package/dist/server/routes/stripe.d.ts.map +0 -1
- package/dist/server/routes/stripe.js +0 -283
- package/dist/server/routes/stripe.js.map +0 -1
- package/dist/server/routes/users.d.ts +0 -9
- package/dist/server/routes/users.d.ts.map +0 -1
- package/dist/server/routes/users.js +0 -1211
- package/dist/server/routes/users.js.map +0 -1
- package/dist/server/routes/watch.d.ts +0 -16
- package/dist/server/routes/watch.d.ts.map +0 -1
- package/dist/server/routes/watch.js +0 -257
- package/dist/server/routes/watch.js.map +0 -1
- package/dist/server/routes/webhooks.d.ts +0 -16
- package/dist/server/routes/webhooks.d.ts.map +0 -1
- package/dist/server/routes/webhooks.js +0 -74
- package/dist/server/routes/webhooks.js.map +0 -1
- package/dist/server/routes/youtube.d.ts +0 -7
- package/dist/server/routes/youtube.d.ts.map +0 -1
- package/dist/server/routes/youtube.js +0 -93
- package/dist/server/routes/youtube.js.map +0 -1
- package/dist/server/sentry.d.ts +0 -14
- package/dist/server/sentry.d.ts.map +0 -1
- package/dist/server/sentry.js +0 -39
- package/dist/server/sentry.js.map +0 -1
- package/dist/server/types.d.ts +0 -16
- package/dist/server/types.d.ts.map +0 -1
- package/dist/server/types.js +0 -8
- package/dist/server/types.js.map +0 -1
- package/dist/server/utils/response.d.ts +0 -45
- package/dist/server/utils/response.d.ts.map +0 -1
- package/dist/server/utils/response.js +0 -70
- package/dist/server/utils/response.js.map +0 -1
- package/dist/server/utils/sse.d.ts +0 -23
- package/dist/server/utils/sse.d.ts.map +0 -1
- package/dist/server/utils/sse.js +0 -39
- package/dist/server/utils/sse.js.map +0 -1
- package/dist/types.d.ts.map +0 -1
- package/dist/types.js.map +0 -1
|
@@ -1,119 +0,0 @@
|
|
|
1
|
-
/**
|
|
2
|
-
* GET /v1/ask?q=<question>&sources=3
|
|
3
|
-
* POST /v1/ask { "question": "...", "sources": 3 }
|
|
4
|
-
*
|
|
5
|
-
* LLM-free web Q&A: search → fetch top pages → BM25 → best answer
|
|
6
|
-
*
|
|
7
|
-
* Returns:
|
|
8
|
-
* {
|
|
9
|
-
* question: string,
|
|
10
|
-
* answer: string, // best passage from top sources
|
|
11
|
-
* confidence: number, // 0-1
|
|
12
|
-
* sources: [{url, title, snippet, confidence}],
|
|
13
|
-
* method: "bm25",
|
|
14
|
-
* elapsed: number // ms
|
|
15
|
-
* }
|
|
16
|
-
*
|
|
17
|
-
* No LLM key required — 100% deterministic BM25 ranking.
|
|
18
|
-
* Competitors: Tavily charges $50/mo and requires an API key.
|
|
19
|
-
* We do this with zero LLM cost, included in every plan.
|
|
20
|
-
*/
|
|
21
|
-
import { Router } from 'express';
|
|
22
|
-
import { peel } from '../../index.js';
|
|
23
|
-
import { quickAnswer } from '../../core/quick-answer.js';
|
|
24
|
-
import { getBestSearchProvider } from '../../core/search-provider.js';
|
|
25
|
-
export function createAskRouter() {
|
|
26
|
-
const router = Router();
|
|
27
|
-
async function handleAsk(question, numSources, req, res) {
|
|
28
|
-
const startMs = Date.now();
|
|
29
|
-
if (!question?.trim()) {
|
|
30
|
-
res.status(400).json({
|
|
31
|
-
error: 'missing_question',
|
|
32
|
-
message: 'Provide q= or question= parameter',
|
|
33
|
-
});
|
|
34
|
-
return;
|
|
35
|
-
}
|
|
36
|
-
// Auth check — global middleware sets req.auth
|
|
37
|
-
const authId = req.auth?.keyInfo?.accountId || req.user?.userId;
|
|
38
|
-
if (!authId) {
|
|
39
|
-
res.status(401).json({
|
|
40
|
-
error: 'authentication_required',
|
|
41
|
-
message: 'API key required. Get one at https://app.webpeel.dev/keys',
|
|
42
|
-
});
|
|
43
|
-
return;
|
|
44
|
-
}
|
|
45
|
-
const clampedSources = Math.min(Math.max(numSources, 1), 5);
|
|
46
|
-
// Step 1: search
|
|
47
|
-
const { provider, apiKey } = getBestSearchProvider();
|
|
48
|
-
let searchResults;
|
|
49
|
-
try {
|
|
50
|
-
searchResults = await provider.searchWeb(question.trim(), {
|
|
51
|
-
count: clampedSources,
|
|
52
|
-
apiKey,
|
|
53
|
-
});
|
|
54
|
-
}
|
|
55
|
-
catch {
|
|
56
|
-
searchResults = [];
|
|
57
|
-
}
|
|
58
|
-
if (!searchResults.length) {
|
|
59
|
-
res.json({
|
|
60
|
-
question,
|
|
61
|
-
answer: null,
|
|
62
|
-
confidence: 0,
|
|
63
|
-
sources: [],
|
|
64
|
-
method: 'bm25',
|
|
65
|
-
elapsed: Date.now() - startMs,
|
|
66
|
-
});
|
|
67
|
-
return;
|
|
68
|
-
}
|
|
69
|
-
// Step 2: fetch top sources in parallel (concurrency up to 5)
|
|
70
|
-
const fetched = await Promise.allSettled(searchResults.slice(0, clampedSources).map((r) => peel(r.url, { budget: 3000, format: 'markdown', timeout: 12000 })
|
|
71
|
-
.then((result) => ({ result, searchResult: r }))));
|
|
72
|
-
// Step 3: run quickAnswer on each, sort by confidence (desc)
|
|
73
|
-
const answers = fetched
|
|
74
|
-
.filter((f) => f.status === 'fulfilled')
|
|
75
|
-
.map((f) => {
|
|
76
|
-
const { result, searchResult } = f.value;
|
|
77
|
-
const qa = quickAnswer({
|
|
78
|
-
question,
|
|
79
|
-
content: result.content,
|
|
80
|
-
url: result.url,
|
|
81
|
-
maxPassages: 2,
|
|
82
|
-
});
|
|
83
|
-
return {
|
|
84
|
-
answer: qa.answer,
|
|
85
|
-
confidence: qa.confidence,
|
|
86
|
-
source: {
|
|
87
|
-
url: result.url,
|
|
88
|
-
title: result.title || searchResult.title,
|
|
89
|
-
snippet: searchResult.snippet,
|
|
90
|
-
},
|
|
91
|
-
};
|
|
92
|
-
})
|
|
93
|
-
.sort((a, b) => b.confidence - a.confidence);
|
|
94
|
-
const best = answers[0];
|
|
95
|
-
res.json({
|
|
96
|
-
question,
|
|
97
|
-
answer: best?.answer || null,
|
|
98
|
-
confidence: best?.confidence || 0,
|
|
99
|
-
sources: answers.map((a) => ({
|
|
100
|
-
...a.source,
|
|
101
|
-
confidence: a.confidence,
|
|
102
|
-
})),
|
|
103
|
-
method: 'bm25',
|
|
104
|
-
elapsed: Date.now() - startMs,
|
|
105
|
-
});
|
|
106
|
-
}
|
|
107
|
-
router.get('/v1/ask', async (req, res) => {
|
|
108
|
-
const question = req.query.q || req.query.question || '';
|
|
109
|
-
const sources = Math.min(parseInt(req.query.sources || '3', 10) || 3, 5);
|
|
110
|
-
await handleAsk(question, sources, req, res);
|
|
111
|
-
});
|
|
112
|
-
router.post('/v1/ask', async (req, res) => {
|
|
113
|
-
const question = req.body?.question || req.body?.q || '';
|
|
114
|
-
const sources = Math.min(parseInt(req.body?.sources ?? 3, 10) || 3, 5);
|
|
115
|
-
await handleAsk(question, sources, req, res);
|
|
116
|
-
});
|
|
117
|
-
return router;
|
|
118
|
-
}
|
|
119
|
-
//# sourceMappingURL=ask.js.map
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"ask.js","sourceRoot":"","sources":["../../../src/server/routes/ask.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;GAmBG;AAEH,OAAO,EAAE,MAAM,EAAqB,MAAM,SAAS,CAAC;AACpD,OAAO,EAAE,IAAI,EAAE,MAAM,gBAAgB,CAAC;AACtC,OAAO,EAAE,WAAW,EAAE,MAAM,4BAA4B,CAAC;AACzD,OAAO,EAAE,qBAAqB,EAAE,MAAM,+BAA+B,CAAC;AAEtE,MAAM,UAAU,eAAe;IAC7B,MAAM,MAAM,GAAG,MAAM,EAAE,CAAC;IAExB,KAAK,UAAU,SAAS,CAAC,QAAgB,EAAE,UAAkB,EAAE,GAAY,EAAE,GAAa;QACxF,MAAM,OAAO,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;QAE3B,IAAI,CAAC,QAAQ,EAAE,IAAI,EAAE,EAAE,CAAC;YACtB,GAAG,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC;gBACnB,KAAK,EAAE,kBAAkB;gBACzB,OAAO,EAAE,mCAAmC;aAC7C,CAAC,CAAC;YACH,OAAO;QACT,CAAC;QAED,+CAA+C;QAC/C,MAAM,MAAM,GAAI,GAAW,CAAC,IAAI,EAAE,OAAO,EAAE,SAAS,IAAK,GAAW,CAAC,IAAI,EAAE,MAAM,CAAC;QAClF,IAAI,CAAC,MAAM,EAAE,CAAC;YACZ,GAAG,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC;gBACnB,KAAK,EAAE,yBAAyB;gBAChC,OAAO,EAAE,2DAA2D;aACrE,CAAC,CAAC;YACH,OAAO;QACT,CAAC;QAED,MAAM,cAAc,GAAG,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,GAAG,CAAC,UAAU,EAAE,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;QAE5D,iBAAiB;QACjB,MAAM,EAAE,QAAQ,EAAE,MAAM,EAAE,GAAG,qBAAqB,EAAE,CAAC;QACrD,IAAI,aAAqE,CAAC;QAC1E,IAAI,CAAC;YACH,aAAa,GAAG,MAAM,QAAQ,CAAC,SAAS,CAAC,QAAQ,CAAC,IAAI,EAAE,EAAE;gBACxD,KAAK,EAAE,cAAc;gBACrB,MAAM;aACP,CAAC,CAAC;QACL,CAAC;QAAC,MAAM,CAAC;YACP,aAAa,GAAG,EAAE,CAAC;QACrB,CAAC;QAED,IAAI,CAAC,aAAa,CAAC,MAAM,EAAE,CAAC;YAC1B,GAAG,CAAC,IAAI,CAAC;gBACP,QAAQ;gBACR,MAAM,EAAE,IAAI;gBACZ,UAAU,EAAE,CAAC;gBACb,OAAO,EAAE,EAAE;gBACX,MAAM,EAAE,MAAM;gBACd,OAAO,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,OAAO;aAC9B,CAAC,CAAC;YACH,OAAO;QACT,CAAC;QAED,8DAA8D;QAC9D,MAAM,OAAO,GAAG,MAAM,OAAO,CAAC,UAAU,CACtC,aAAa,CAAC,KAAK,CAAC,CAAC,EAAE,cAAc,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAC/C,IAAI,CAAC,CAAC,CAAC,GAAG,EAAE,EAAE,MAAM,EAAE,IAAI,EAAE,MAAM,EAAE,UAAU,EAAE,OAAO,EAAE,KAAK,EAAE,CAAC;aAC9D,IAAI,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC,EAAE,MAAM,EAAE,YAAY,EAAE,CAAC,EAAE,CAAC,CAAC,CACnD,CACF,CAAC;QAEF,6DAA6D;QAC7D,MAAM,OAAO,GAAG,OAAO;aACpB,MAAM,CAAC,CAAC,CAAC,EAAoC,EAAE,CAAC,CAAC,CAAC,MAAM,KAAK,WAAW,CAAC;aACzE,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE;YACT,MAAM,EAAE,MAAM,EAAE,YAAY,EAAE,GAAG,CAAC,CAAC,KAGlC,CAAC;YACF,MAAM,EAAE,GAAG,WAAW,CAAC;gBACrB,QAAQ;gBACR,OAAO,EAAE,MAAM,CAAC,OAAO;gBACvB,GAAG,EAAE,MAAM,CAAC,GAAG;gBACf,WAAW,EAAE,CAAC;aACf,CAAC,CAAC;YACH,OAAO;gBACL,MAAM,EAAE,EAAE,CAAC,MAAM;gBACjB,UAAU,EAAE,EAAE,CAAC,UAAU;gBACzB,MAAM,EAAE;oBACN,GAAG,EAAE,MAAM,CAAC,GAAG;oBACf,KAAK,EAAE,MAAM,CAAC,KAAK,IAAI,YAAY,CAAC,KAAK;oBACzC,OAAO,EAAE,YAAY,CAAC,OAAO;iBAC9B;aACF,CAAC;QACJ,CAAC,CAAC;aACD,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,UAAU,GAAG,CAAC,CAAC,UAAU,CAAC,CAAC;QAE/C,MAAM,IAAI,GAAG,OAAO,CAAC,CAAC,CAAC,CAAC;QAExB,GAAG,CAAC,IAAI,CAAC;YACP,QAAQ;YACR,MAAM,EAAE,IAAI,EAAE,MAAM,IAAI,IAAI;YAC5B,UAAU,EAAE,IAAI,EAAE,UAAU,IAAI,CAAC;YACjC,OAAO,EAAE,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;gBAC3B,GAAG,CAAC,CAAC,MAAM;gBACX,UAAU,EAAE,CAAC,CAAC,UAAU;aACzB,CAAC,CAAC;YACH,MAAM,EAAE,MAAM;YACd,OAAO,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,OAAO;SAC9B,CAAC,CAAC;IACL,CAAC;IAED,MAAM,CAAC,GAAG,CAAC,SAAS,EAAE,KAAK,EAAE,GAAY,EAAE,GAAa,EAAE,EAAE;QAC1D,MAAM,QAAQ,GAAI,GAAG,CAAC,KAAK,CAAC,CAAY,IAAK,GAAG,CAAC,KAAK,CAAC,QAAmB,IAAI,EAAE,CAAC;QACjF,MAAM,OAAO,GAAG,IAAI,CAAC,GAAG,CAAC,QAAQ,CAAE,GAAG,CAAC,KAAK,CAAC,OAAkB,IAAI,GAAG,EAAE,EAAE,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC,CAAC;QACrF,MAAM,SAAS,CAAC,QAAQ,EAAE,OAAO,EAAE,GAAG,EAAE,GAAG,CAAC,CAAC;IAC/C,CAAC,CAAC,CAAC;IAEH,MAAM,CAAC,IAAI,CAAC,SAAS,EAAE,KAAK,EAAE,GAAY,EAAE,GAAa,EAAE,EAAE;QAC3D,MAAM,QAAQ,GAAI,GAAG,CAAC,IAAI,EAAE,QAAmB,IAAK,GAAG,CAAC,IAAI,EAAE,CAAY,IAAI,EAAE,CAAC;QACjF,MAAM,OAAO,GAAG,IAAI,CAAC,GAAG,CAAC,QAAQ,CAAC,GAAG,CAAC,IAAI,EAAE,OAAO,IAAI,CAAC,EAAE,EAAE,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC,CAAC;QACvE,MAAM,SAAS,CAAC,QAAQ,EAAE,OAAO,EAAE,GAAG,EAAE,GAAG,CAAC,CAAC;IAC/C,CAAC,CAAC,CAAC;IAEH,OAAO,MAAM,CAAC;AAChB,CAAC"}
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"batch.d.ts","sourceRoot":"","sources":["../../../src/server/routes/batch.ts"],"names":[],"mappings":"AAAA;;GAEG;AAEH,OAAO,EAAE,MAAM,EAAqB,MAAM,SAAS,CAAC;AAGpD,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,iBAAiB,CAAC;AAIjD,wBAAgB,iBAAiB,CAAC,QAAQ,EAAE,SAAS,GAAG,MAAM,CAgc7D"}
|
|
@@ -1,412 +0,0 @@
|
|
|
1
|
-
/**
|
|
2
|
-
* Batch scrape API - process multiple URLs concurrently
|
|
3
|
-
*/
|
|
4
|
-
import { Router } from 'express';
|
|
5
|
-
import { peel } from '../../index.js';
|
|
6
|
-
import { sendWebhook } from './webhooks.js';
|
|
7
|
-
import { initSSE, sendSSE, endSSE, wantsSSE } from '../utils/sse.js';
|
|
8
|
-
export function createBatchRouter(jobQueue) {
|
|
9
|
-
const router = Router();
|
|
10
|
-
/**
|
|
11
|
-
* POST /v1/batch/scrape - Submit batch of URLs
|
|
12
|
-
*/
|
|
13
|
-
router.post('/v1/batch/scrape', async (req, res) => {
|
|
14
|
-
try {
|
|
15
|
-
const { urls, formats, extract, maxTokens, webhook, concurrency } = req.body;
|
|
16
|
-
// Validate required parameters
|
|
17
|
-
if (!urls || !Array.isArray(urls) || urls.length === 0) {
|
|
18
|
-
res.status(400).json({
|
|
19
|
-
error: 'invalid_request',
|
|
20
|
-
message: 'Missing or invalid "urls" parameter (must be non-empty array)',
|
|
21
|
-
});
|
|
22
|
-
return;
|
|
23
|
-
}
|
|
24
|
-
// Limit batch size
|
|
25
|
-
if (urls.length > 100) {
|
|
26
|
-
res.status(400).json({
|
|
27
|
-
error: 'invalid_request',
|
|
28
|
-
message: 'Batch size too large (max 100 URLs)',
|
|
29
|
-
});
|
|
30
|
-
return;
|
|
31
|
-
}
|
|
32
|
-
// Validate URLs
|
|
33
|
-
for (const url of urls) {
|
|
34
|
-
if (typeof url !== 'string') {
|
|
35
|
-
res.status(400).json({
|
|
36
|
-
error: 'invalid_request',
|
|
37
|
-
message: 'All URLs must be strings',
|
|
38
|
-
});
|
|
39
|
-
return;
|
|
40
|
-
}
|
|
41
|
-
try {
|
|
42
|
-
new URL(url);
|
|
43
|
-
}
|
|
44
|
-
catch {
|
|
45
|
-
res.status(400).json({
|
|
46
|
-
error: 'invalid_url',
|
|
47
|
-
message: `Invalid URL format: ${url}`,
|
|
48
|
-
});
|
|
49
|
-
return;
|
|
50
|
-
}
|
|
51
|
-
}
|
|
52
|
-
const ownerId = req.auth?.keyInfo?.accountId;
|
|
53
|
-
// ── SSE streaming path ────────────────────────────────────────────────
|
|
54
|
-
if (wantsSSE(req)) {
|
|
55
|
-
const job = await jobQueue.createJob('batch', webhook, ownerId);
|
|
56
|
-
await jobQueue.updateJob(job.id, { total: urls.length });
|
|
57
|
-
// Set SSE headers (X-Request-Id already set by global middleware)
|
|
58
|
-
initSSE(res);
|
|
59
|
-
sendSSE(res, 'started', {
|
|
60
|
-
batchId: job.id,
|
|
61
|
-
totalUrls: urls.length,
|
|
62
|
-
});
|
|
63
|
-
let closed = false;
|
|
64
|
-
const heartbeat = setInterval(() => {
|
|
65
|
-
if (!closed)
|
|
66
|
-
res.write('event: ping\ndata: {}\n\n');
|
|
67
|
-
}, 15_000);
|
|
68
|
-
req.on('close', () => {
|
|
69
|
-
closed = true;
|
|
70
|
-
clearInterval(heartbeat);
|
|
71
|
-
});
|
|
72
|
-
const startTime = Date.now();
|
|
73
|
-
let completedCount = 0;
|
|
74
|
-
let failedCount = 0;
|
|
75
|
-
try {
|
|
76
|
-
jobQueue.updateJob(job.id, { status: 'processing' });
|
|
77
|
-
const peelOptions = {
|
|
78
|
-
format: formats?.[0] || 'markdown',
|
|
79
|
-
extract,
|
|
80
|
-
maxTokens,
|
|
81
|
-
};
|
|
82
|
-
// Process URLs with bounded concurrency (caller-specified, max 10)
|
|
83
|
-
const results = new Array(urls.length);
|
|
84
|
-
const maxConcurrent = Math.min(Math.max(parseInt(concurrency) || 5, 1), 10);
|
|
85
|
-
let activeCount = 0;
|
|
86
|
-
let urlIndex = 0;
|
|
87
|
-
const runBatch = async () => {
|
|
88
|
-
while (urlIndex < urls.length) {
|
|
89
|
-
// Check cancellation
|
|
90
|
-
const currentJob = await jobQueue.getJob(job.id);
|
|
91
|
-
if (currentJob?.status === 'cancelled')
|
|
92
|
-
break;
|
|
93
|
-
while (activeCount >= maxConcurrent) {
|
|
94
|
-
await new Promise(resolve => setTimeout(resolve, 50));
|
|
95
|
-
}
|
|
96
|
-
const url = urls[urlIndex];
|
|
97
|
-
const index = urlIndex;
|
|
98
|
-
urlIndex++;
|
|
99
|
-
activeCount++;
|
|
100
|
-
(async () => {
|
|
101
|
-
try {
|
|
102
|
-
const result = await peel(url, peelOptions);
|
|
103
|
-
results[index] = result;
|
|
104
|
-
completedCount++;
|
|
105
|
-
jobQueue.updateJob(job.id, {
|
|
106
|
-
completed: completedCount + failedCount,
|
|
107
|
-
creditsUsed: completedCount + failedCount,
|
|
108
|
-
});
|
|
109
|
-
if (!closed) {
|
|
110
|
-
sendSSE(res, 'result', {
|
|
111
|
-
url,
|
|
112
|
-
content: result.content,
|
|
113
|
-
metadata: result.metadata,
|
|
114
|
-
index,
|
|
115
|
-
});
|
|
116
|
-
}
|
|
117
|
-
}
|
|
118
|
-
catch (err) {
|
|
119
|
-
failedCount++;
|
|
120
|
-
results[index] = { url, error: err.message || 'Unknown error' };
|
|
121
|
-
jobQueue.updateJob(job.id, {
|
|
122
|
-
completed: completedCount + failedCount,
|
|
123
|
-
creditsUsed: completedCount + failedCount,
|
|
124
|
-
});
|
|
125
|
-
if (!closed) {
|
|
126
|
-
sendSSE(res, 'error', {
|
|
127
|
-
url,
|
|
128
|
-
error: 'FETCH_ERROR',
|
|
129
|
-
message: err.message || 'Unknown error',
|
|
130
|
-
index,
|
|
131
|
-
});
|
|
132
|
-
}
|
|
133
|
-
}
|
|
134
|
-
finally {
|
|
135
|
-
activeCount--;
|
|
136
|
-
}
|
|
137
|
-
})();
|
|
138
|
-
}
|
|
139
|
-
// Wait for all in-flight requests
|
|
140
|
-
while (activeCount > 0) {
|
|
141
|
-
await new Promise(resolve => setTimeout(resolve, 50));
|
|
142
|
-
}
|
|
143
|
-
};
|
|
144
|
-
await runBatch();
|
|
145
|
-
jobQueue.updateJob(job.id, {
|
|
146
|
-
status: 'completed',
|
|
147
|
-
data: results,
|
|
148
|
-
});
|
|
149
|
-
if (!closed) {
|
|
150
|
-
sendSSE(res, 'done', {
|
|
151
|
-
batchId: job.id,
|
|
152
|
-
completed: completedCount,
|
|
153
|
-
failed: failedCount,
|
|
154
|
-
duration: Date.now() - startTime,
|
|
155
|
-
});
|
|
156
|
-
}
|
|
157
|
-
}
|
|
158
|
-
catch (error) {
|
|
159
|
-
jobQueue.updateJob(job.id, {
|
|
160
|
-
status: 'failed',
|
|
161
|
-
error: error.message || 'Unknown error',
|
|
162
|
-
});
|
|
163
|
-
if (!closed) {
|
|
164
|
-
sendSSE(res, 'error', {
|
|
165
|
-
error: 'BATCH_FAILED',
|
|
166
|
-
message: error.message || 'Unknown error',
|
|
167
|
-
});
|
|
168
|
-
}
|
|
169
|
-
}
|
|
170
|
-
finally {
|
|
171
|
-
clearInterval(heartbeat);
|
|
172
|
-
if (!closed)
|
|
173
|
-
endSSE(res);
|
|
174
|
-
}
|
|
175
|
-
return;
|
|
176
|
-
}
|
|
177
|
-
// ── Regular async job path (backward compat) ─────────────────────────
|
|
178
|
-
const job = await jobQueue.createJob('batch', webhook, ownerId);
|
|
179
|
-
await jobQueue.updateJob(job.id, {
|
|
180
|
-
total: urls.length,
|
|
181
|
-
});
|
|
182
|
-
// Start batch processing in background
|
|
183
|
-
setImmediate(async () => {
|
|
184
|
-
try {
|
|
185
|
-
// Update job to processing
|
|
186
|
-
jobQueue.updateJob(job.id, { status: 'processing' });
|
|
187
|
-
// Send started webhook
|
|
188
|
-
if (webhook) {
|
|
189
|
-
await sendWebhook(webhook, 'started', {
|
|
190
|
-
jobId: job.id,
|
|
191
|
-
total: urls.length,
|
|
192
|
-
});
|
|
193
|
-
}
|
|
194
|
-
// Build peel options
|
|
195
|
-
const peelOptions = {
|
|
196
|
-
format: formats?.[0] || 'markdown',
|
|
197
|
-
extract,
|
|
198
|
-
maxTokens,
|
|
199
|
-
};
|
|
200
|
-
// Process URLs with semaphore (caller-specified, max 10 concurrent)
|
|
201
|
-
const results = [];
|
|
202
|
-
const maxConcurrent = Math.min(Math.max(parseInt(concurrency) || 5, 1), 10);
|
|
203
|
-
let activeCount = 0;
|
|
204
|
-
let urlIndex = 0;
|
|
205
|
-
const processBatch = async () => {
|
|
206
|
-
while (urlIndex < urls.length) {
|
|
207
|
-
// Check if job was cancelled
|
|
208
|
-
const currentJob = await jobQueue.getJob(job.id);
|
|
209
|
-
if (currentJob?.status === 'cancelled') {
|
|
210
|
-
return;
|
|
211
|
-
}
|
|
212
|
-
// Wait for available slot
|
|
213
|
-
while (activeCount >= maxConcurrent) {
|
|
214
|
-
await new Promise(resolve => setTimeout(resolve, 100));
|
|
215
|
-
}
|
|
216
|
-
const url = urls[urlIndex];
|
|
217
|
-
const index = urlIndex;
|
|
218
|
-
urlIndex++;
|
|
219
|
-
activeCount++;
|
|
220
|
-
// Process URL
|
|
221
|
-
(async () => {
|
|
222
|
-
try {
|
|
223
|
-
const result = await peel(url, peelOptions);
|
|
224
|
-
results[index] = result;
|
|
225
|
-
// Update progress
|
|
226
|
-
const completed = results.filter(r => r !== undefined).length;
|
|
227
|
-
jobQueue.updateJob(job.id, {
|
|
228
|
-
completed,
|
|
229
|
-
creditsUsed: completed,
|
|
230
|
-
});
|
|
231
|
-
// Send page webhook
|
|
232
|
-
if (webhook) {
|
|
233
|
-
sendWebhook(webhook, 'page', {
|
|
234
|
-
jobId: job.id,
|
|
235
|
-
url,
|
|
236
|
-
completed,
|
|
237
|
-
total: urls.length,
|
|
238
|
-
}).catch(() => { }); // Fire and forget
|
|
239
|
-
}
|
|
240
|
-
}
|
|
241
|
-
catch (error) {
|
|
242
|
-
// Store error as result
|
|
243
|
-
results[index] = {
|
|
244
|
-
url,
|
|
245
|
-
error: error.message || 'Unknown error',
|
|
246
|
-
};
|
|
247
|
-
// Update progress
|
|
248
|
-
const completed = results.filter(r => r !== undefined).length;
|
|
249
|
-
jobQueue.updateJob(job.id, {
|
|
250
|
-
completed,
|
|
251
|
-
creditsUsed: completed,
|
|
252
|
-
});
|
|
253
|
-
}
|
|
254
|
-
finally {
|
|
255
|
-
activeCount--;
|
|
256
|
-
}
|
|
257
|
-
})();
|
|
258
|
-
}
|
|
259
|
-
// Wait for all tasks to complete
|
|
260
|
-
while (activeCount > 0) {
|
|
261
|
-
await new Promise(resolve => setTimeout(resolve, 100));
|
|
262
|
-
}
|
|
263
|
-
};
|
|
264
|
-
await processBatch();
|
|
265
|
-
// Update job with results
|
|
266
|
-
jobQueue.updateJob(job.id, {
|
|
267
|
-
status: 'completed',
|
|
268
|
-
data: results,
|
|
269
|
-
});
|
|
270
|
-
// Send completed webhook
|
|
271
|
-
if (webhook) {
|
|
272
|
-
await sendWebhook(webhook, 'completed', {
|
|
273
|
-
jobId: job.id,
|
|
274
|
-
total: urls.length,
|
|
275
|
-
completed: results.length,
|
|
276
|
-
});
|
|
277
|
-
}
|
|
278
|
-
}
|
|
279
|
-
catch (error) {
|
|
280
|
-
// Update job with error
|
|
281
|
-
jobQueue.updateJob(job.id, {
|
|
282
|
-
status: 'failed',
|
|
283
|
-
error: error.message || 'Unknown error',
|
|
284
|
-
});
|
|
285
|
-
// Send failed webhook
|
|
286
|
-
if (webhook) {
|
|
287
|
-
await sendWebhook(webhook, 'failed', {
|
|
288
|
-
jobId: job.id,
|
|
289
|
-
error: error.message || 'Unknown error',
|
|
290
|
-
});
|
|
291
|
-
}
|
|
292
|
-
}
|
|
293
|
-
});
|
|
294
|
-
// Return job ID immediately
|
|
295
|
-
res.status(202).json({
|
|
296
|
-
success: true,
|
|
297
|
-
id: job.id,
|
|
298
|
-
url: `/v1/batch/scrape/${job.id}`,
|
|
299
|
-
});
|
|
300
|
-
}
|
|
301
|
-
catch (error) {
|
|
302
|
-
console.error('Batch scrape creation error:', error);
|
|
303
|
-
res.status(500).json({
|
|
304
|
-
error: 'internal_error',
|
|
305
|
-
message: 'Failed to create batch scrape job',
|
|
306
|
-
});
|
|
307
|
-
}
|
|
308
|
-
});
|
|
309
|
-
/**
|
|
310
|
-
* GET /v1/batch/scrape/:id - Get batch scrape status + results
|
|
311
|
-
*/
|
|
312
|
-
router.get('/v1/batch/scrape/:id', async (req, res) => {
|
|
313
|
-
try {
|
|
314
|
-
const id = req.params.id;
|
|
315
|
-
const job = await jobQueue.getJob(id);
|
|
316
|
-
if (!job) {
|
|
317
|
-
res.status(404).json({
|
|
318
|
-
error: 'not_found',
|
|
319
|
-
message: 'Job not found',
|
|
320
|
-
});
|
|
321
|
-
return;
|
|
322
|
-
}
|
|
323
|
-
// SECURITY: Verify the requester owns this job
|
|
324
|
-
const requestOwnerId = req.auth?.keyInfo?.accountId;
|
|
325
|
-
if (job.ownerId && requestOwnerId && job.ownerId !== requestOwnerId) {
|
|
326
|
-
res.status(404).json({
|
|
327
|
-
error: 'not_found',
|
|
328
|
-
message: 'Job not found',
|
|
329
|
-
});
|
|
330
|
-
return;
|
|
331
|
-
}
|
|
332
|
-
if (job.type !== 'batch') {
|
|
333
|
-
res.status(400).json({
|
|
334
|
-
error: 'invalid_request',
|
|
335
|
-
message: 'Job is not a batch scrape job',
|
|
336
|
-
});
|
|
337
|
-
return;
|
|
338
|
-
}
|
|
339
|
-
res.json({
|
|
340
|
-
success: true,
|
|
341
|
-
status: job.status,
|
|
342
|
-
progress: job.progress,
|
|
343
|
-
total: job.total,
|
|
344
|
-
completed: job.completed,
|
|
345
|
-
creditsUsed: job.creditsUsed,
|
|
346
|
-
data: job.data,
|
|
347
|
-
error: job.error,
|
|
348
|
-
expiresAt: job.expiresAt,
|
|
349
|
-
});
|
|
350
|
-
}
|
|
351
|
-
catch (error) {
|
|
352
|
-
console.error('Get batch scrape error:', error);
|
|
353
|
-
res.status(500).json({
|
|
354
|
-
error: 'internal_error',
|
|
355
|
-
message: 'Failed to retrieve job',
|
|
356
|
-
});
|
|
357
|
-
}
|
|
358
|
-
});
|
|
359
|
-
/**
|
|
360
|
-
* DELETE /v1/batch/scrape/:id - Cancel batch scrape job
|
|
361
|
-
*/
|
|
362
|
-
router.delete('/v1/batch/scrape/:id', async (req, res) => {
|
|
363
|
-
try {
|
|
364
|
-
const id = req.params.id;
|
|
365
|
-
const job = await jobQueue.getJob(id);
|
|
366
|
-
if (!job) {
|
|
367
|
-
res.status(404).json({
|
|
368
|
-
error: 'not_found',
|
|
369
|
-
message: 'Job not found',
|
|
370
|
-
});
|
|
371
|
-
return;
|
|
372
|
-
}
|
|
373
|
-
// SECURITY: Verify the requester owns this job
|
|
374
|
-
const requestOwnerId = req.auth?.keyInfo?.accountId;
|
|
375
|
-
if (job.ownerId && requestOwnerId && job.ownerId !== requestOwnerId) {
|
|
376
|
-
res.status(404).json({
|
|
377
|
-
error: 'not_found',
|
|
378
|
-
message: 'Job not found',
|
|
379
|
-
});
|
|
380
|
-
return;
|
|
381
|
-
}
|
|
382
|
-
if (job.type !== 'batch') {
|
|
383
|
-
res.status(400).json({
|
|
384
|
-
error: 'invalid_request',
|
|
385
|
-
message: 'Job is not a batch scrape job',
|
|
386
|
-
});
|
|
387
|
-
return;
|
|
388
|
-
}
|
|
389
|
-
const cancelled = await jobQueue.cancelJob(id);
|
|
390
|
-
if (!cancelled) {
|
|
391
|
-
res.status(400).json({
|
|
392
|
-
error: 'invalid_request',
|
|
393
|
-
message: 'Job cannot be cancelled (already completed or failed)',
|
|
394
|
-
});
|
|
395
|
-
return;
|
|
396
|
-
}
|
|
397
|
-
res.json({
|
|
398
|
-
success: true,
|
|
399
|
-
message: 'Job cancelled',
|
|
400
|
-
});
|
|
401
|
-
}
|
|
402
|
-
catch (error) {
|
|
403
|
-
console.error('Cancel batch scrape error:', error);
|
|
404
|
-
res.status(500).json({
|
|
405
|
-
error: 'internal_error',
|
|
406
|
-
message: 'Failed to cancel job',
|
|
407
|
-
});
|
|
408
|
-
}
|
|
409
|
-
});
|
|
410
|
-
return router;
|
|
411
|
-
}
|
|
412
|
-
//# sourceMappingURL=batch.js.map
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"batch.js","sourceRoot":"","sources":["../../../src/server/routes/batch.ts"],"names":[],"mappings":"AAAA;;GAEG;AAEH,OAAO,EAAE,MAAM,EAAqB,MAAM,SAAS,CAAC;AACpD,OAAO,EAAE,IAAI,EAAE,MAAM,gBAAgB,CAAC;AAGtC,OAAO,EAAE,WAAW,EAAE,MAAM,eAAe,CAAC;AAC5C,OAAO,EAAE,OAAO,EAAE,OAAO,EAAE,MAAM,EAAE,QAAQ,EAAE,MAAM,iBAAiB,CAAC;AAErE,MAAM,UAAU,iBAAiB,CAAC,QAAmB;IACnD,MAAM,MAAM,GAAG,MAAM,EAAE,CAAC;IAExB;;OAEG;IACH,MAAM,CAAC,IAAI,CAAC,kBAAkB,EAAE,KAAK,EAAE,GAAY,EAAE,GAAa,EAAE,EAAE;QACpE,IAAI,CAAC;YACH,MAAM,EAAE,IAAI,EAAE,OAAO,EAAE,OAAO,EAAE,SAAS,EAAE,OAAO,EAAE,WAAW,EAAE,GAAG,GAAG,CAAC,IAAI,CAAC;YAE7E,+BAA+B;YAC/B,IAAI,CAAC,IAAI,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,IAAI,CAAC,IAAI,IAAI,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;gBACvD,GAAG,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC;oBACnB,KAAK,EAAE,iBAAiB;oBACxB,OAAO,EAAE,+DAA+D;iBACzE,CAAC,CAAC;gBACH,OAAO;YACT,CAAC;YAED,mBAAmB;YACnB,IAAI,IAAI,CAAC,MAAM,GAAG,GAAG,EAAE,CAAC;gBACtB,GAAG,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC;oBACnB,KAAK,EAAE,iBAAiB;oBACxB,OAAO,EAAE,qCAAqC;iBAC/C,CAAC,CAAC;gBACH,OAAO;YACT,CAAC;YAED,gBAAgB;YAChB,KAAK,MAAM,GAAG,IAAI,IAAI,EAAE,CAAC;gBACvB,IAAI,OAAO,GAAG,KAAK,QAAQ,EAAE,CAAC;oBAC5B,GAAG,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC;wBACnB,KAAK,EAAE,iBAAiB;wBACxB,OAAO,EAAE,0BAA0B;qBACpC,CAAC,CAAC;oBACH,OAAO;gBACT,CAAC;gBAED,IAAI,CAAC;oBACH,IAAI,GAAG,CAAC,GAAG,CAAC,CAAC;gBACf,CAAC;gBAAC,MAAM,CAAC;oBACP,GAAG,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC;wBACnB,KAAK,EAAE,aAAa;wBACpB,OAAO,EAAE,uBAAuB,GAAG,EAAE;qBACtC,CAAC,CAAC;oBACH,OAAO;gBACT,CAAC;YACH,CAAC;YAED,MAAM,OAAO,GAAG,GAAG,CAAC,IAAI,EAAE,OAAO,EAAE,SAAS,CAAC;YAE7C,yEAAyE;YACzE,IAAI,QAAQ,CAAC,GAAG,CAAC,EAAE,CAAC;gBAClB,MAAM,GAAG,GAAG,MAAM,QAAQ,CAAC,SAAS,CAAC,OAAO,EAAE,OAAO,EAAE,OAAO,CAAC,CAAC;gBAChE,MAAM,QAAQ,CAAC,SAAS,CAAC,GAAG,CAAC,EAAE,EAAE,EAAE,KAAK,EAAE,IAAI,CAAC,MAAM,EAAE,CAAC,CAAC;gBAEzD,kEAAkE;gBAClE,OAAO,CAAC,GAAG,CAAC,CAAC;gBAEb,OAAO,CAAC,GAAG,EAAE,SAAS,EAAE;oBACtB,OAAO,EAAE,GAAG,CAAC,EAAE;oBACf,SAAS,EAAE,IAAI,CAAC,MAAM;iBACvB,CAAC,CAAC;gBAEH,IAAI,MAAM,GAAG,KAAK,CAAC;gBACnB,MAAM,SAAS,GAAG,WAAW,CAAC,GAAG,EAAE;oBACjC,IAAI,CAAC,MAAM;wBAAE,GAAG,CAAC,KAAK,CAAC,2BAA2B,CAAC,CAAC;gBACtD,CAAC,EAAE,MAAM,CAAC,CAAC;gBAEX,GAAG,CAAC,EAAE,CAAC,OAAO,EAAE,GAAG,EAAE;oBACnB,MAAM,GAAG,IAAI,CAAC;oBACd,aAAa,CAAC,SAAS,CAAC,CAAC;gBAC3B,CAAC,CAAC,CAAC;gBAEH,MAAM,SAAS,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;gBAC7B,IAAI,cAAc,GAAG,CAAC,CAAC;gBACvB,IAAI,WAAW,GAAG,CAAC,CAAC;gBAEpB,IAAI,CAAC;oBACH,QAAQ,CAAC,SAAS,CAAC,GAAG,CAAC,EAAE,EAAE,EAAE,MAAM,EAAE,YAAY,EAAE,CAAC,CAAC;oBAErD,MAAM,WAAW,GAAgB;wBAC/B,MAAM,EAAE,OAAO,EAAE,CAAC,CAAC,CAAC,IAAI,UAAU;wBAClC,OAAO;wBACP,SAAS;qBACV,CAAC;oBAEF,mEAAmE;oBACnE,MAAM,OAAO,GAAc,IAAI,KAAK,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;oBAClD,MAAM,aAAa,GAAG,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,GAAG,CAAC,QAAQ,CAAC,WAAW,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;oBAC5E,IAAI,WAAW,GAAG,CAAC,CAAC;oBACpB,IAAI,QAAQ,GAAG,CAAC,CAAC;oBAEjB,MAAM,QAAQ,GAAG,KAAK,IAAI,EAAE;wBAC1B,OAAO,QAAQ,GAAG,IAAI,CAAC,MAAM,EAAE,CAAC;4BAC9B,qBAAqB;4BACrB,MAAM,UAAU,GAAG,MAAM,QAAQ,CAAC,MAAM,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;4BACjD,IAAI,UAAU,EAAE,MAAM,KAAK,WAAW;gCAAE,MAAM;4BAE9C,OAAO,WAAW,IAAI,aAAa,EAAE,CAAC;gCACpC,MAAM,IAAI,OAAO,CAAO,OAAO,CAAC,EAAE,CAAC,UAAU,CAAC,OAAO,EAAE,EAAE,CAAC,CAAC,CAAC;4BAC9D,CAAC;4BAED,MAAM,GAAG,GAAG,IAAI,CAAC,QAAQ,CAAW,CAAC;4BACrC,MAAM,KAAK,GAAG,QAAQ,CAAC;4BACvB,QAAQ,EAAE,CAAC;4BACX,WAAW,EAAE,CAAC;4BAEd,CAAC,KAAK,IAAI,EAAE;gCACV,IAAI,CAAC;oCACH,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,GAAG,EAAE,WAAW,CAAC,CAAC;oCAC5C,OAAO,CAAC,KAAK,CAAC,GAAG,MAAM,CAAC;oCACxB,cAAc,EAAE,CAAC;oCAEjB,QAAQ,CAAC,SAAS,CAAC,GAAG,CAAC,EAAE,EAAE;wCACzB,SAAS,EAAE,cAAc,GAAG,WAAW;wCACvC,WAAW,EAAE,cAAc,GAAG,WAAW;qCAC1C,CAAC,CAAC;oCAEH,IAAI,CAAC,MAAM,EAAE,CAAC;wCACZ,OAAO,CAAC,GAAG,EAAE,QAAQ,EAAE;4CACrB,GAAG;4CACH,OAAO,EAAE,MAAM,CAAC,OAAO;4CACvB,QAAQ,EAAE,MAAM,CAAC,QAAQ;4CACzB,KAAK;yCACN,CAAC,CAAC;oCACL,CAAC;gCACH,CAAC;gCAAC,OAAO,GAAQ,EAAE,CAAC;oCAClB,WAAW,EAAE,CAAC;oCACd,OAAO,CAAC,KAAK,CAAC,GAAG,EAAE,GAAG,EAAE,KAAK,EAAE,GAAG,CAAC,OAAO,IAAI,eAAe,EAAE,CAAC;oCAEhE,QAAQ,CAAC,SAAS,CAAC,GAAG,CAAC,EAAE,EAAE;wCACzB,SAAS,EAAE,cAAc,GAAG,WAAW;wCACvC,WAAW,EAAE,cAAc,GAAG,WAAW;qCAC1C,CAAC,CAAC;oCAEH,IAAI,CAAC,MAAM,EAAE,CAAC;wCACZ,OAAO,CAAC,GAAG,EAAE,OAAO,EAAE;4CACpB,GAAG;4CACH,KAAK,EAAE,aAAa;4CACpB,OAAO,EAAE,GAAG,CAAC,OAAO,IAAI,eAAe;4CACvC,KAAK;yCACN,CAAC,CAAC;oCACL,CAAC;gCACH,CAAC;wCAAS,CAAC;oCACT,WAAW,EAAE,CAAC;gCAChB,CAAC;4BACH,CAAC,CAAC,EAAE,CAAC;wBACP,CAAC;wBAED,kCAAkC;wBAClC,OAAO,WAAW,GAAG,CAAC,EAAE,CAAC;4BACvB,MAAM,IAAI,OAAO,CAAO,OAAO,CAAC,EAAE,CAAC,UAAU,CAAC,OAAO,EAAE,EAAE,CAAC,CAAC,CAAC;wBAC9D,CAAC;oBACH,CAAC,CAAC;oBAEF,MAAM,QAAQ,EAAE,CAAC;oBAEjB,QAAQ,CAAC,SAAS,CAAC,GAAG,CAAC,EAAE,EAAE;wBACzB,MAAM,EAAE,WAAW;wBACnB,IAAI,EAAE,OAAO;qBACd,CAAC,CAAC;oBAEH,IAAI,CAAC,MAAM,EAAE,CAAC;wBACZ,OAAO,CAAC,GAAG,EAAE,MAAM,EAAE;4BACnB,OAAO,EAAE,GAAG,CAAC,EAAE;4BACf,SAAS,EAAE,cAAc;4BACzB,MAAM,EAAE,WAAW;4BACnB,QAAQ,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,SAAS;yBACjC,CAAC,CAAC;oBACL,CAAC;gBACH,CAAC;gBAAC,OAAO,KAAU,EAAE,CAAC;oBACpB,QAAQ,CAAC,SAAS,CAAC,GAAG,CAAC,EAAE,EAAE;wBACzB,MAAM,EAAE,QAAQ;wBAChB,KAAK,EAAE,KAAK,CAAC,OAAO,IAAI,eAAe;qBACxC,CAAC,CAAC;oBACH,IAAI,CAAC,MAAM,EAAE,CAAC;wBACZ,OAAO,CAAC,GAAG,EAAE,OAAO,EAAE;4BACpB,KAAK,EAAE,cAAc;4BACrB,OAAO,EAAE,KAAK,CAAC,OAAO,IAAI,eAAe;yBAC1C,CAAC,CAAC;oBACL,CAAC;gBACH,CAAC;wBAAS,CAAC;oBACT,aAAa,CAAC,SAAS,CAAC,CAAC;oBACzB,IAAI,CAAC,MAAM;wBAAE,MAAM,CAAC,GAAG,CAAC,CAAC;gBAC3B,CAAC;gBAED,OAAO;YACT,CAAC;YAED,wEAAwE;YACxE,MAAM,GAAG,GAAG,MAAM,QAAQ,CAAC,SAAS,CAAC,OAAO,EAAE,OAAO,EAAE,OAAO,CAAC,CAAC;YAChE,MAAM,QAAQ,CAAC,SAAS,CAAC,GAAG,CAAC,EAAE,EAAE;gBAC/B,KAAK,EAAE,IAAI,CAAC,MAAM;aACnB,CAAC,CAAC;YAEH,uCAAuC;YACvC,YAAY,CAAC,KAAK,IAAI,EAAE;gBACtB,IAAI,CAAC;oBACH,2BAA2B;oBAC3B,QAAQ,CAAC,SAAS,CAAC,GAAG,CAAC,EAAE,EAAE,EAAE,MAAM,EAAE,YAAY,EAAE,CAAC,CAAC;oBAErD,uBAAuB;oBACvB,IAAI,OAAO,EAAE,CAAC;wBACZ,MAAM,WAAW,CAAC,OAAO,EAAE,SAAS,EAAE;4BACpC,KAAK,EAAE,GAAG,CAAC,EAAE;4BACb,KAAK,EAAE,IAAI,CAAC,MAAM;yBACnB,CAAC,CAAC;oBACL,CAAC;oBAED,qBAAqB;oBACrB,MAAM,WAAW,GAAgB;wBAC/B,MAAM,EAAE,OAAO,EAAE,CAAC,CAAC,CAAC,IAAI,UAAU;wBAClC,OAAO;wBACP,SAAS;qBACV,CAAC;oBAEF,oEAAoE;oBACpE,MAAM,OAAO,GAAU,EAAE,CAAC;oBAC1B,MAAM,aAAa,GAAG,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,GAAG,CAAC,QAAQ,CAAC,WAAW,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;oBAC5E,IAAI,WAAW,GAAG,CAAC,CAAC;oBACpB,IAAI,QAAQ,GAAG,CAAC,CAAC;oBAEjB,MAAM,YAAY,GAAG,KAAK,IAAI,EAAE;wBAC9B,OAAO,QAAQ,GAAG,IAAI,CAAC,MAAM,EAAE,CAAC;4BAC9B,6BAA6B;4BAC7B,MAAM,UAAU,GAAG,MAAM,QAAQ,CAAC,MAAM,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;4BACjD,IAAI,UAAU,EAAE,MAAM,KAAK,WAAW,EAAE,CAAC;gCACvC,OAAO;4BACT,CAAC;4BAED,0BAA0B;4BAC1B,OAAO,WAAW,IAAI,aAAa,EAAE,CAAC;gCACpC,MAAM,IAAI,OAAO,CAAC,OAAO,CAAC,EAAE,CAAC,UAAU,CAAC,OAAO,EAAE,GAAG,CAAC,CAAC,CAAC;4BACzD,CAAC;4BAED,MAAM,GAAG,GAAG,IAAI,CAAC,QAAQ,CAAC,CAAC;4BAC3B,MAAM,KAAK,GAAG,QAAQ,CAAC;4BACvB,QAAQ,EAAE,CAAC;4BACX,WAAW,EAAE,CAAC;4BAEd,cAAc;4BACd,CAAC,KAAK,IAAI,EAAE;gCACV,IAAI,CAAC;oCACH,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,GAAG,EAAE,WAAW,CAAC,CAAC;oCAC5C,OAAO,CAAC,KAAK,CAAC,GAAG,MAAM,CAAC;oCAExB,kBAAkB;oCAClB,MAAM,SAAS,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,KAAK,SAAS,CAAC,CAAC,MAAM,CAAC;oCAC9D,QAAQ,CAAC,SAAS,CAAC,GAAG,CAAC,EAAE,EAAE;wCACzB,SAAS;wCACT,WAAW,EAAE,SAAS;qCACvB,CAAC,CAAC;oCAEH,oBAAoB;oCACpB,IAAI,OAAO,EAAE,CAAC;wCACZ,WAAW,CAAC,OAAO,EAAE,MAAM,EAAE;4CAC3B,KAAK,EAAE,GAAG,CAAC,EAAE;4CACb,GAAG;4CACH,SAAS;4CACT,KAAK,EAAE,IAAI,CAAC,MAAM;yCACnB,CAAC,CAAC,KAAK,CAAC,GAAG,EAAE,GAAE,CAAC,CAAC,CAAC,CAAC,kBAAkB;oCACxC,CAAC;gCACH,CAAC;gCAAC,OAAO,KAAU,EAAE,CAAC;oCACpB,wBAAwB;oCACxB,OAAO,CAAC,KAAK,CAAC,GAAG;wCACf,GAAG;wCACH,KAAK,EAAE,KAAK,CAAC,OAAO,IAAI,eAAe;qCACxC,CAAC;oCAEF,kBAAkB;oCAClB,MAAM,SAAS,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,KAAK,SAAS,CAAC,CAAC,MAAM,CAAC;oCAC9D,QAAQ,CAAC,SAAS,CAAC,GAAG,CAAC,EAAE,EAAE;wCACzB,SAAS;wCACT,WAAW,EAAE,SAAS;qCACvB,CAAC,CAAC;gCACL,CAAC;wCAAS,CAAC;oCACT,WAAW,EAAE,CAAC;gCAChB,CAAC;4BACH,CAAC,CAAC,EAAE,CAAC;wBACP,CAAC;wBAED,iCAAiC;wBACjC,OAAO,WAAW,GAAG,CAAC,EAAE,CAAC;4BACvB,MAAM,IAAI,OAAO,CAAC,OAAO,CAAC,EAAE,CAAC,UAAU,CAAC,OAAO,EAAE,GAAG,CAAC,CAAC,CAAC;wBACzD,CAAC;oBACH,CAAC,CAAC;oBAEF,MAAM,YAAY,EAAE,CAAC;oBAErB,0BAA0B;oBAC1B,QAAQ,CAAC,SAAS,CAAC,GAAG,CAAC,EAAE,EAAE;wBACzB,MAAM,EAAE,WAAW;wBACnB,IAAI,EAAE,OAAO;qBACd,CAAC,CAAC;oBAEH,yBAAyB;oBACzB,IAAI,OAAO,EAAE,CAAC;wBACZ,MAAM,WAAW,CAAC,OAAO,EAAE,WAAW,EAAE;4BACtC,KAAK,EAAE,GAAG,CAAC,EAAE;4BACb,KAAK,EAAE,IAAI,CAAC,MAAM;4BAClB,SAAS,EAAE,OAAO,CAAC,MAAM;yBAC1B,CAAC,CAAC;oBACL,CAAC;gBACH,CAAC;gBAAC,OAAO,KAAU,EAAE,CAAC;oBACpB,wBAAwB;oBACxB,QAAQ,CAAC,SAAS,CAAC,GAAG,CAAC,EAAE,EAAE;wBACzB,MAAM,EAAE,QAAQ;wBAChB,KAAK,EAAE,KAAK,CAAC,OAAO,IAAI,eAAe;qBACxC,CAAC,CAAC;oBAEH,sBAAsB;oBACtB,IAAI,OAAO,EAAE,CAAC;wBACZ,MAAM,WAAW,CAAC,OAAO,EAAE,QAAQ,EAAE;4BACnC,KAAK,EAAE,GAAG,CAAC,EAAE;4BACb,KAAK,EAAE,KAAK,CAAC,OAAO,IAAI,eAAe;yBACxC,CAAC,CAAC;oBACL,CAAC;gBACH,CAAC;YACH,CAAC,CAAC,CAAC;YAEH,4BAA4B;YAC5B,GAAG,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC;gBACnB,OAAO,EAAE,IAAI;gBACb,EAAE,EAAE,GAAG,CAAC,EAAE;gBACV,GAAG,EAAE,oBAAoB,GAAG,CAAC,EAAE,EAAE;aAClC,CAAC,CAAC;QACL,CAAC;QAAC,OAAO,KAAU,EAAE,CAAC;YACpB,OAAO,CAAC,KAAK,CAAC,8BAA8B,EAAE,KAAK,CAAC,CAAC;YACrD,GAAG,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC;gBACnB,KAAK,EAAE,gBAAgB;gBACvB,OAAO,EAAE,mCAAmC;aAC7C,CAAC,CAAC;QACL,CAAC;IACH,CAAC,CAAC,CAAC;IAEH;;OAEG;IACH,MAAM,CAAC,GAAG,CAAC,sBAAsB,EAAE,KAAK,EAAE,GAAY,EAAE,GAAa,EAAE,EAAE;QACvE,IAAI,CAAC;YACH,MAAM,EAAE,GAAG,GAAG,CAAC,MAAM,CAAC,EAAY,CAAC;YACnC,MAAM,GAAG,GAAG,MAAM,QAAQ,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC;YAEtC,IAAI,CAAC,GAAG,EAAE,CAAC;gBACT,GAAG,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC;oBACnB,KAAK,EAAE,WAAW;oBAClB,OAAO,EAAE,eAAe;iBACzB,CAAC,CAAC;gBACH,OAAO;YACT,CAAC;YAED,+CAA+C;YAC/C,MAAM,cAAc,GAAG,GAAG,CAAC,IAAI,EAAE,OAAO,EAAE,SAAS,CAAC;YACpD,IAAI,GAAG,CAAC,OAAO,IAAI,cAAc,IAAI,GAAG,CAAC,OAAO,KAAK,cAAc,EAAE,CAAC;gBACpE,GAAG,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC;oBACnB,KAAK,EAAE,WAAW;oBAClB,OAAO,EAAE,eAAe;iBACzB,CAAC,CAAC;gBACH,OAAO;YACT,CAAC;YAED,IAAI,GAAG,CAAC,IAAI,KAAK,OAAO,EAAE,CAAC;gBACzB,GAAG,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC;oBACnB,KAAK,EAAE,iBAAiB;oBACxB,OAAO,EAAE,+BAA+B;iBACzC,CAAC,CAAC;gBACH,OAAO;YACT,CAAC;YAED,GAAG,CAAC,IAAI,CAAC;gBACP,OAAO,EAAE,IAAI;gBACb,MAAM,EAAE,GAAG,CAAC,MAAM;gBAClB,QAAQ,EAAE,GAAG,CAAC,QAAQ;gBACtB,KAAK,EAAE,GAAG,CAAC,KAAK;gBAChB,SAAS,EAAE,GAAG,CAAC,SAAS;gBACxB,WAAW,EAAE,GAAG,CAAC,WAAW;gBAC5B,IAAI,EAAE,GAAG,CAAC,IAAI;gBACd,KAAK,EAAE,GAAG,CAAC,KAAK;gBAChB,SAAS,EAAE,GAAG,CAAC,SAAS;aACzB,CAAC,CAAC;QACL,CAAC;QAAC,OAAO,KAAU,EAAE,CAAC;YACpB,OAAO,CAAC,KAAK,CAAC,yBAAyB,EAAE,KAAK,CAAC,CAAC;YAChD,GAAG,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC;gBACnB,KAAK,EAAE,gBAAgB;gBACvB,OAAO,EAAE,wBAAwB;aAClC,CAAC,CAAC;QACL,CAAC;IACH,CAAC,CAAC,CAAC;IAEH;;OAEG;IACH,MAAM,CAAC,MAAM,CAAC,sBAAsB,EAAE,KAAK,EAAE,GAAY,EAAE,GAAa,EAAE,EAAE;QAC1E,IAAI,CAAC;YACH,MAAM,EAAE,GAAG,GAAG,CAAC,MAAM,CAAC,EAAY,CAAC;YACnC,MAAM,GAAG,GAAG,MAAM,QAAQ,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC;YAEtC,IAAI,CAAC,GAAG,EAAE,CAAC;gBACT,GAAG,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC;oBACnB,KAAK,EAAE,WAAW;oBAClB,OAAO,EAAE,eAAe;iBACzB,CAAC,CAAC;gBACH,OAAO;YACT,CAAC;YAED,+CAA+C;YAC/C,MAAM,cAAc,GAAG,GAAG,CAAC,IAAI,EAAE,OAAO,EAAE,SAAS,CAAC;YACpD,IAAI,GAAG,CAAC,OAAO,IAAI,cAAc,IAAI,GAAG,CAAC,OAAO,KAAK,cAAc,EAAE,CAAC;gBACpE,GAAG,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC;oBACnB,KAAK,EAAE,WAAW;oBAClB,OAAO,EAAE,eAAe;iBACzB,CAAC,CAAC;gBACH,OAAO;YACT,CAAC;YAED,IAAI,GAAG,CAAC,IAAI,KAAK,OAAO,EAAE,CAAC;gBACzB,GAAG,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC;oBACnB,KAAK,EAAE,iBAAiB;oBACxB,OAAO,EAAE,+BAA+B;iBACzC,CAAC,CAAC;gBACH,OAAO;YACT,CAAC;YAED,MAAM,SAAS,GAAG,MAAM,QAAQ,CAAC,SAAS,CAAC,EAAE,CAAC,CAAC;YAE/C,IAAI,CAAC,SAAS,EAAE,CAAC;gBACf,GAAG,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC;oBACnB,KAAK,EAAE,iBAAiB;oBACxB,OAAO,EAAE,uDAAuD;iBACjE,CAAC,CAAC;gBACH,OAAO;YACT,CAAC;YAED,GAAG,CAAC,IAAI,CAAC;gBACP,OAAO,EAAE,IAAI;gBACb,OAAO,EAAE,eAAe;aACzB,CAAC,CAAC;QACL,CAAC;QAAC,OAAO,KAAU,EAAE,CAAC;YACpB,OAAO,CAAC,KAAK,CAAC,4BAA4B,EAAE,KAAK,CAAC,CAAC;YACnD,GAAG,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC;gBACnB,KAAK,EAAE,gBAAgB;gBACvB,OAAO,EAAE,sBAAsB;aAChC,CAAC,CAAC;QACL,CAAC;IACH,CAAC,CAAC,CAAC;IAEH,OAAO,MAAM,CAAC;AAChB,CAAC"}
|
|
@@ -1,7 +0,0 @@
|
|
|
1
|
-
/**
|
|
2
|
-
* CLI Usage endpoint — works with API key auth (not JWT)
|
|
3
|
-
* Used by the `webpeel usage` command and pre-fetch usage checks
|
|
4
|
-
*/
|
|
5
|
-
import { Router } from 'express';
|
|
6
|
-
export declare function createCLIUsageRouter(): Router;
|
|
7
|
-
//# sourceMappingURL=cli-usage.d.ts.map
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"cli-usage.d.ts","sourceRoot":"","sources":["../../../src/server/routes/cli-usage.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAEH,OAAO,EAAE,MAAM,EAAqB,MAAM,SAAS,CAAC;AAKpD,wBAAgB,oBAAoB,IAAI,MAAM,CAwI7C"}
|