webpeel 0.19.4 → 0.20.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +2 -2
- package/dist/cache.d.ts +0 -1
- package/dist/cache.js +0 -1
- package/dist/cli/commands/auth.d.ts +5 -0
- package/dist/cli/commands/auth.js +476 -0
- package/dist/cli/commands/fetch.d.ts +6 -0
- package/dist/cli/commands/fetch.js +1015 -0
- package/dist/cli/commands/interact.d.ts +5 -0
- package/dist/cli/commands/interact.js +839 -0
- package/dist/cli/commands/jobs.d.ts +5 -0
- package/dist/cli/commands/jobs.js +997 -0
- package/dist/cli/commands/screenshot.d.ts +5 -0
- package/dist/cli/commands/screenshot.js +273 -0
- package/dist/cli/commands/search.d.ts +5 -0
- package/dist/cli/commands/search.js +524 -0
- package/dist/cli/utils.d.ts +84 -0
- package/dist/cli/utils.js +686 -0
- package/dist/cli-auth.d.ts +0 -1
- package/dist/cli-auth.js +0 -1
- package/dist/cli.d.ts +7 -6
- package/dist/cli.js +35 -4698
- package/dist/core/actions.d.ts +0 -1
- package/dist/core/actions.js +0 -1
- package/dist/core/agent.d.ts +0 -1
- package/dist/core/agent.js +9 -12
- package/dist/core/answer.d.ts +0 -1
- package/dist/core/answer.js +0 -1
- package/dist/core/application-tracker.d.ts +0 -1
- package/dist/core/application-tracker.js +0 -1
- package/dist/core/apply.d.ts +0 -1
- package/dist/core/apply.js +0 -1
- package/dist/core/auto-extract.d.ts +0 -1
- package/dist/core/auto-extract.js +0 -1
- package/dist/core/auto-interact.d.ts +0 -1
- package/dist/core/auto-interact.js +0 -1
- package/dist/core/bm25-filter.d.ts +0 -1
- package/dist/core/bm25-filter.js +0 -1
- package/dist/core/branding.d.ts +0 -1
- package/dist/core/branding.js +0 -1
- package/dist/core/browser-fetch.d.ts +0 -1
- package/dist/core/browser-fetch.js +17 -10
- package/dist/core/browser-pool.d.ts +0 -1
- package/dist/core/browser-pool.js +0 -1
- package/dist/core/budget.d.ts +0 -1
- package/dist/core/budget.js +0 -1
- package/dist/core/cache.d.ts +0 -1
- package/dist/core/cache.js +0 -1
- package/dist/core/cf-worker-proxy.d.ts +0 -1
- package/dist/core/cf-worker-proxy.js +0 -1
- package/dist/core/challenge-detection.d.ts +0 -1
- package/dist/core/challenge-detection.js +0 -1
- package/dist/core/change-tracking.d.ts +0 -1
- package/dist/core/change-tracking.js +0 -1
- package/dist/core/chunker.d.ts +0 -1
- package/dist/core/chunker.js +0 -1
- package/dist/core/chunking.d.ts +0 -1
- package/dist/core/chunking.js +0 -1
- package/dist/core/cloak-fetch.d.ts +0 -1
- package/dist/core/cloak-fetch.js +0 -1
- package/dist/core/content-pruner.d.ts +0 -1
- package/dist/core/content-pruner.js +0 -1
- package/dist/core/crawl-checkpoint.d.ts +0 -1
- package/dist/core/crawl-checkpoint.js +0 -1
- package/dist/core/crawler.d.ts +0 -1
- package/dist/core/crawler.js +6 -5
- package/dist/core/cycle-fetch.d.ts +0 -1
- package/dist/core/cycle-fetch.js +0 -1
- package/dist/core/deep-fetch.d.ts +0 -1
- package/dist/core/deep-fetch.js +0 -1
- package/dist/core/design-analysis.d.ts +0 -1
- package/dist/core/design-analysis.js +0 -1
- package/dist/core/design-compare.d.ts +0 -1
- package/dist/core/design-compare.js +0 -1
- package/dist/core/diff.d.ts +0 -1
- package/dist/core/diff.js +0 -1
- package/dist/core/dns-cache.d.ts +0 -1
- package/dist/core/dns-cache.js +0 -1
- package/dist/core/documents.d.ts +0 -1
- package/dist/core/documents.js +0 -1
- package/dist/core/domain-extractors.d.ts +0 -1
- package/dist/core/domain-extractors.js +0 -1
- package/dist/core/extract-inline.d.ts +0 -1
- package/dist/core/extract-inline.js +0 -1
- package/dist/core/extract-listings.d.ts +0 -1
- package/dist/core/extract-listings.js +0 -1
- package/dist/core/extract.d.ts +0 -1
- package/dist/core/extract.js +0 -1
- package/dist/core/fetcher.d.ts +0 -1
- package/dist/core/fetcher.js +0 -1
- package/dist/core/google-cache.d.ts +0 -1
- package/dist/core/google-cache.js +0 -1
- package/dist/core/hotel-search.d.ts +0 -1
- package/dist/core/hotel-search.js +0 -1
- package/dist/core/http-fetch.d.ts +0 -1
- package/dist/core/http-fetch.js +5 -7
- package/dist/core/human.d.ts +0 -1
- package/dist/core/human.js +0 -1
- package/dist/core/jobs.d.ts +0 -1
- package/dist/core/jobs.js +0 -1
- package/dist/core/json-ld.d.ts +0 -1
- package/dist/core/json-ld.js +0 -1
- package/dist/core/llm-extract.d.ts +0 -1
- package/dist/core/llm-extract.js +0 -1
- package/dist/core/logger.d.ts +17 -0
- package/dist/core/logger.js +44 -0
- package/dist/core/map.d.ts +0 -1
- package/dist/core/map.js +0 -1
- package/dist/core/markdown.d.ts +0 -1
- package/dist/core/markdown.js +0 -1
- package/dist/core/metadata.d.ts +0 -1
- package/dist/core/metadata.js +0 -1
- package/dist/core/paginate.d.ts +0 -1
- package/dist/core/paginate.js +0 -1
- package/dist/core/pdf.d.ts +0 -1
- package/dist/core/pdf.js +0 -1
- package/dist/core/peel-tls.d.ts +0 -1
- package/dist/core/peel-tls.js +0 -1
- package/dist/core/pipeline.d.ts +0 -1
- package/dist/core/pipeline.js +22 -25
- package/dist/core/profiles.d.ts +0 -1
- package/dist/core/profiles.js +0 -1
- package/dist/core/quick-answer.d.ts +0 -1
- package/dist/core/quick-answer.js +0 -1
- package/dist/core/rate-governor.d.ts +0 -1
- package/dist/core/rate-governor.js +0 -1
- package/dist/core/readability.d.ts +0 -1
- package/dist/core/readability.js +0 -1
- package/dist/core/research.d.ts +0 -1
- package/dist/core/research.js +0 -1
- package/dist/core/schema-extraction.d.ts +0 -1
- package/dist/core/schema-extraction.js +0 -1
- package/dist/core/schema-postprocess.d.ts +0 -1
- package/dist/core/schema-postprocess.js +0 -1
- package/dist/core/schema-templates.d.ts +0 -1
- package/dist/core/schema-templates.js +0 -1
- package/dist/core/screenshot.d.ts +0 -1
- package/dist/core/screenshot.js +0 -1
- package/dist/core/search-fallback.d.ts +0 -1
- package/dist/core/search-fallback.js +0 -1
- package/dist/core/search-provider.d.ts +0 -1
- package/dist/core/search-provider.js +18 -21
- package/dist/core/site-search.d.ts +0 -1
- package/dist/core/site-search.js +0 -1
- package/dist/core/sitemap.d.ts +0 -1
- package/dist/core/sitemap.js +0 -1
- package/dist/core/stealth-patches.d.ts +0 -1
- package/dist/core/stealth-patches.js +0 -1
- package/dist/core/stemmer.d.ts +0 -1
- package/dist/core/stemmer.js +0 -1
- package/dist/core/strategies.d.ts +6 -1
- package/dist/core/strategies.js +29 -41
- package/dist/core/strategy-hooks.d.ts +0 -1
- package/dist/core/strategy-hooks.js +0 -1
- package/dist/core/summarize.d.ts +0 -1
- package/dist/core/summarize.js +0 -1
- package/dist/core/synonyms.d.ts +0 -1
- package/dist/core/synonyms.js +0 -1
- package/dist/core/table-format.d.ts +0 -1
- package/dist/core/table-format.js +0 -1
- package/dist/core/timing.d.ts +0 -1
- package/dist/core/timing.js +0 -1
- package/dist/core/user-agents.d.ts +0 -1
- package/dist/core/user-agents.js +0 -1
- package/dist/core/watch-manager.d.ts +0 -1
- package/dist/core/watch-manager.js +0 -1
- package/dist/core/watch.d.ts +0 -1
- package/dist/core/watch.js +0 -1
- package/dist/core/youtube.d.ts +0 -1
- package/dist/core/youtube.js +0 -1
- package/dist/index.d.ts +8 -3
- package/dist/index.js +27 -3
- package/dist/integrations/index.d.ts +0 -1
- package/dist/integrations/index.js +0 -1
- package/dist/integrations/langchain.d.ts +0 -1
- package/dist/integrations/langchain.js +0 -1
- package/dist/integrations/llamaindex.d.ts +0 -1
- package/dist/integrations/llamaindex.js +0 -1
- package/dist/mcp/handlers/act.d.ts +5 -0
- package/dist/mcp/handlers/act.js +34 -0
- package/dist/mcp/handlers/definitions.d.ts +6 -0
- package/dist/mcp/handlers/definitions.js +266 -0
- package/dist/mcp/handlers/extract.d.ts +6 -0
- package/dist/mcp/handlers/extract.js +102 -0
- package/dist/mcp/handlers/fetch.d.ts +6 -0
- package/dist/mcp/handlers/fetch.js +98 -0
- package/dist/mcp/handlers/find.d.ts +5 -0
- package/dist/mcp/handlers/find.js +137 -0
- package/dist/mcp/handlers/index.d.ts +13 -0
- package/dist/mcp/handlers/index.js +61 -0
- package/dist/mcp/handlers/legacy.d.ts +25 -0
- package/dist/mcp/handlers/legacy.js +450 -0
- package/dist/mcp/handlers/meta.d.ts +6 -0
- package/dist/mcp/handlers/meta.js +31 -0
- package/dist/mcp/handlers/monitor.d.ts +5 -0
- package/dist/mcp/handlers/monitor.js +41 -0
- package/dist/mcp/handlers/read.d.ts +6 -0
- package/dist/mcp/handlers/read.js +63 -0
- package/dist/mcp/handlers/see.d.ts +5 -0
- package/dist/mcp/handlers/see.js +75 -0
- package/dist/mcp/handlers/types.d.ts +29 -0
- package/dist/mcp/handlers/types.js +28 -0
- package/dist/mcp/server.d.ts +3 -4
- package/dist/mcp/server.js +35 -1101
- package/dist/mcp/smart-router.d.ts +0 -1
- package/dist/mcp/smart-router.js +3 -1
- package/dist/types.d.ts +6 -1
- package/dist/types.js +0 -1
- package/package.json +3 -13
- package/dist/cache.d.ts.map +0 -1
- package/dist/cache.js.map +0 -1
- package/dist/cli-auth.d.ts.map +0 -1
- package/dist/cli-auth.js.map +0 -1
- package/dist/cli.bundle.cjs +0 -159248
- package/dist/cli.d.ts.map +0 -1
- package/dist/cli.js.map +0 -1
- package/dist/core/actions.d.ts.map +0 -1
- package/dist/core/actions.js.map +0 -1
- package/dist/core/agent.d.ts.map +0 -1
- package/dist/core/agent.js.map +0 -1
- package/dist/core/answer.d.ts.map +0 -1
- package/dist/core/answer.js.map +0 -1
- package/dist/core/application-tracker.d.ts.map +0 -1
- package/dist/core/application-tracker.js.map +0 -1
- package/dist/core/apply.d.ts.map +0 -1
- package/dist/core/apply.js.map +0 -1
- package/dist/core/auto-extract.d.ts.map +0 -1
- package/dist/core/auto-extract.js.map +0 -1
- package/dist/core/auto-interact.d.ts.map +0 -1
- package/dist/core/auto-interact.js.map +0 -1
- package/dist/core/bm25-filter.d.ts.map +0 -1
- package/dist/core/bm25-filter.js.map +0 -1
- package/dist/core/branding.d.ts.map +0 -1
- package/dist/core/branding.js.map +0 -1
- package/dist/core/browser-fetch.d.ts.map +0 -1
- package/dist/core/browser-fetch.js.map +0 -1
- package/dist/core/browser-pool.d.ts.map +0 -1
- package/dist/core/browser-pool.js.map +0 -1
- package/dist/core/budget.d.ts.map +0 -1
- package/dist/core/budget.js.map +0 -1
- package/dist/core/cache.d.ts.map +0 -1
- package/dist/core/cache.js.map +0 -1
- package/dist/core/cf-worker-proxy.d.ts.map +0 -1
- package/dist/core/cf-worker-proxy.js.map +0 -1
- package/dist/core/challenge-detection.d.ts.map +0 -1
- package/dist/core/challenge-detection.js.map +0 -1
- package/dist/core/change-tracking.d.ts.map +0 -1
- package/dist/core/change-tracking.js.map +0 -1
- package/dist/core/chunker.d.ts.map +0 -1
- package/dist/core/chunker.js.map +0 -1
- package/dist/core/chunking.d.ts.map +0 -1
- package/dist/core/chunking.js.map +0 -1
- package/dist/core/cloak-fetch.d.ts.map +0 -1
- package/dist/core/cloak-fetch.js.map +0 -1
- package/dist/core/content-pruner.d.ts.map +0 -1
- package/dist/core/content-pruner.js.map +0 -1
- package/dist/core/crawl-checkpoint.d.ts.map +0 -1
- package/dist/core/crawl-checkpoint.js.map +0 -1
- package/dist/core/crawler.d.ts.map +0 -1
- package/dist/core/crawler.js.map +0 -1
- package/dist/core/cycle-fetch.d.ts.map +0 -1
- package/dist/core/cycle-fetch.js.map +0 -1
- package/dist/core/deep-fetch.d.ts.map +0 -1
- package/dist/core/deep-fetch.js.map +0 -1
- package/dist/core/design-analysis.d.ts.map +0 -1
- package/dist/core/design-analysis.js.map +0 -1
- package/dist/core/design-compare.d.ts.map +0 -1
- package/dist/core/design-compare.js.map +0 -1
- package/dist/core/diff.d.ts.map +0 -1
- package/dist/core/diff.js.map +0 -1
- package/dist/core/dns-cache.d.ts.map +0 -1
- package/dist/core/dns-cache.js.map +0 -1
- package/dist/core/documents.d.ts.map +0 -1
- package/dist/core/documents.js.map +0 -1
- package/dist/core/domain-extractors.d.ts.map +0 -1
- package/dist/core/domain-extractors.js.map +0 -1
- package/dist/core/extract-inline.d.ts.map +0 -1
- package/dist/core/extract-inline.js.map +0 -1
- package/dist/core/extract-listings.d.ts.map +0 -1
- package/dist/core/extract-listings.js.map +0 -1
- package/dist/core/extract.d.ts.map +0 -1
- package/dist/core/extract.js.map +0 -1
- package/dist/core/fetcher.d.ts.map +0 -1
- package/dist/core/fetcher.js.map +0 -1
- package/dist/core/google-cache.d.ts.map +0 -1
- package/dist/core/google-cache.js.map +0 -1
- package/dist/core/hotel-search.d.ts.map +0 -1
- package/dist/core/hotel-search.js.map +0 -1
- package/dist/core/http-fetch.d.ts.map +0 -1
- package/dist/core/http-fetch.js.map +0 -1
- package/dist/core/human.d.ts.map +0 -1
- package/dist/core/human.js.map +0 -1
- package/dist/core/jobs.d.ts.map +0 -1
- package/dist/core/jobs.js.map +0 -1
- package/dist/core/json-ld.d.ts.map +0 -1
- package/dist/core/json-ld.js.map +0 -1
- package/dist/core/llm-extract.d.ts.map +0 -1
- package/dist/core/llm-extract.js.map +0 -1
- package/dist/core/map.d.ts.map +0 -1
- package/dist/core/map.js.map +0 -1
- package/dist/core/markdown.d.ts.map +0 -1
- package/dist/core/markdown.js.map +0 -1
- package/dist/core/metadata.d.ts.map +0 -1
- package/dist/core/metadata.js.map +0 -1
- package/dist/core/paginate.d.ts.map +0 -1
- package/dist/core/paginate.js.map +0 -1
- package/dist/core/pdf.d.ts.map +0 -1
- package/dist/core/pdf.js.map +0 -1
- package/dist/core/peel-tls.d.ts.map +0 -1
- package/dist/core/peel-tls.js.map +0 -1
- package/dist/core/pipeline.d.ts.map +0 -1
- package/dist/core/pipeline.js.map +0 -1
- package/dist/core/profiles.d.ts.map +0 -1
- package/dist/core/profiles.js.map +0 -1
- package/dist/core/quick-answer.d.ts.map +0 -1
- package/dist/core/quick-answer.js.map +0 -1
- package/dist/core/rate-governor.d.ts.map +0 -1
- package/dist/core/rate-governor.js.map +0 -1
- package/dist/core/readability.d.ts.map +0 -1
- package/dist/core/readability.js.map +0 -1
- package/dist/core/research.d.ts.map +0 -1
- package/dist/core/research.js.map +0 -1
- package/dist/core/schema-extraction.d.ts.map +0 -1
- package/dist/core/schema-extraction.js.map +0 -1
- package/dist/core/schema-postprocess.d.ts.map +0 -1
- package/dist/core/schema-postprocess.js.map +0 -1
- package/dist/core/schema-templates.d.ts.map +0 -1
- package/dist/core/schema-templates.js.map +0 -1
- package/dist/core/screenshot.d.ts.map +0 -1
- package/dist/core/screenshot.js.map +0 -1
- package/dist/core/search-fallback.d.ts.map +0 -1
- package/dist/core/search-fallback.js.map +0 -1
- package/dist/core/search-provider.d.ts.map +0 -1
- package/dist/core/search-provider.js.map +0 -1
- package/dist/core/site-search.d.ts.map +0 -1
- package/dist/core/site-search.js.map +0 -1
- package/dist/core/sitemap.d.ts.map +0 -1
- package/dist/core/sitemap.js.map +0 -1
- package/dist/core/stealth-patches.d.ts.map +0 -1
- package/dist/core/stealth-patches.js.map +0 -1
- package/dist/core/stemmer.d.ts.map +0 -1
- package/dist/core/stemmer.js.map +0 -1
- package/dist/core/strategies.d.ts.map +0 -1
- package/dist/core/strategies.js.map +0 -1
- package/dist/core/strategy-hooks.d.ts.map +0 -1
- package/dist/core/strategy-hooks.js.map +0 -1
- package/dist/core/summarize.d.ts.map +0 -1
- package/dist/core/summarize.js.map +0 -1
- package/dist/core/synonyms.d.ts.map +0 -1
- package/dist/core/synonyms.js.map +0 -1
- package/dist/core/table-format.d.ts.map +0 -1
- package/dist/core/table-format.js.map +0 -1
- package/dist/core/timing.d.ts.map +0 -1
- package/dist/core/timing.js.map +0 -1
- package/dist/core/user-agents.d.ts.map +0 -1
- package/dist/core/user-agents.js.map +0 -1
- package/dist/core/watch-manager.d.ts.map +0 -1
- package/dist/core/watch-manager.js.map +0 -1
- package/dist/core/watch.d.ts.map +0 -1
- package/dist/core/watch.js.map +0 -1
- package/dist/core/youtube.d.ts.map +0 -1
- package/dist/core/youtube.js.map +0 -1
- package/dist/index.d.ts.map +0 -1
- package/dist/index.js.map +0 -1
- package/dist/integrations/index.d.ts.map +0 -1
- package/dist/integrations/index.js.map +0 -1
- package/dist/integrations/langchain.d.ts.map +0 -1
- package/dist/integrations/langchain.js.map +0 -1
- package/dist/integrations/llamaindex.d.ts.map +0 -1
- package/dist/integrations/llamaindex.js.map +0 -1
- package/dist/mcp/server.d.ts.map +0 -1
- package/dist/mcp/server.js.map +0 -1
- package/dist/mcp/smart-router.d.ts.map +0 -1
- package/dist/mcp/smart-router.js.map +0 -1
- package/dist/server/app.d.ts +0 -15
- package/dist/server/app.d.ts.map +0 -1
- package/dist/server/app.js +0 -350
- package/dist/server/app.js.map +0 -1
- package/dist/server/auth-store.d.ts +0 -28
- package/dist/server/auth-store.d.ts.map +0 -1
- package/dist/server/auth-store.js +0 -89
- package/dist/server/auth-store.js.map +0 -1
- package/dist/server/email-service.d.ts +0 -22
- package/dist/server/email-service.d.ts.map +0 -1
- package/dist/server/email-service.js +0 -80
- package/dist/server/email-service.js.map +0 -1
- package/dist/server/job-queue.d.ts +0 -93
- package/dist/server/job-queue.d.ts.map +0 -1
- package/dist/server/job-queue.js +0 -146
- package/dist/server/job-queue.js.map +0 -1
- package/dist/server/logger.d.ts +0 -11
- package/dist/server/logger.d.ts.map +0 -1
- package/dist/server/logger.js +0 -38
- package/dist/server/logger.js.map +0 -1
- package/dist/server/middleware/auth.d.ts +0 -29
- package/dist/server/middleware/auth.d.ts.map +0 -1
- package/dist/server/middleware/auth.js +0 -222
- package/dist/server/middleware/auth.js.map +0 -1
- package/dist/server/middleware/rate-limit.d.ts +0 -25
- package/dist/server/middleware/rate-limit.d.ts.map +0 -1
- package/dist/server/middleware/rate-limit.js +0 -168
- package/dist/server/middleware/rate-limit.js.map +0 -1
- package/dist/server/middleware/url-validator.d.ts +0 -16
- package/dist/server/middleware/url-validator.d.ts.map +0 -1
- package/dist/server/middleware/url-validator.js +0 -187
- package/dist/server/middleware/url-validator.js.map +0 -1
- package/dist/server/openapi.yaml +0 -4944
- package/dist/server/pg-auth-store.d.ts +0 -133
- package/dist/server/pg-auth-store.d.ts.map +0 -1
- package/dist/server/pg-auth-store.js +0 -473
- package/dist/server/pg-auth-store.js.map +0 -1
- package/dist/server/pg-job-queue.d.ts +0 -60
- package/dist/server/pg-job-queue.d.ts.map +0 -1
- package/dist/server/pg-job-queue.js +0 -365
- package/dist/server/pg-job-queue.js.map +0 -1
- package/dist/server/premium/domain-intel.d.ts +0 -17
- package/dist/server/premium/domain-intel.d.ts.map +0 -1
- package/dist/server/premium/domain-intel.js +0 -134
- package/dist/server/premium/domain-intel.js.map +0 -1
- package/dist/server/premium/index.d.ts +0 -18
- package/dist/server/premium/index.d.ts.map +0 -1
- package/dist/server/premium/index.js +0 -36
- package/dist/server/premium/index.js.map +0 -1
- package/dist/server/premium/swr-cache.d.ts +0 -15
- package/dist/server/premium/swr-cache.d.ts.map +0 -1
- package/dist/server/premium/swr-cache.js +0 -35
- package/dist/server/premium/swr-cache.js.map +0 -1
- package/dist/server/routes/activity.d.ts +0 -7
- package/dist/server/routes/activity.d.ts.map +0 -1
- package/dist/server/routes/activity.js +0 -68
- package/dist/server/routes/activity.js.map +0 -1
- package/dist/server/routes/agent.d.ts +0 -16
- package/dist/server/routes/agent.d.ts.map +0 -1
- package/dist/server/routes/agent.js +0 -247
- package/dist/server/routes/agent.js.map +0 -1
- package/dist/server/routes/answer.d.ts +0 -6
- package/dist/server/routes/answer.d.ts.map +0 -1
- package/dist/server/routes/answer.js +0 -133
- package/dist/server/routes/answer.js.map +0 -1
- package/dist/server/routes/ask.d.ts +0 -23
- package/dist/server/routes/ask.d.ts.map +0 -1
- package/dist/server/routes/ask.js +0 -119
- package/dist/server/routes/ask.js.map +0 -1
- package/dist/server/routes/batch.d.ts +0 -7
- package/dist/server/routes/batch.d.ts.map +0 -1
- package/dist/server/routes/batch.js +0 -412
- package/dist/server/routes/batch.js.map +0 -1
- package/dist/server/routes/cli-usage.d.ts +0 -7
- package/dist/server/routes/cli-usage.d.ts.map +0 -1
- package/dist/server/routes/cli-usage.js +0 -121
- package/dist/server/routes/cli-usage.js.map +0 -1
- package/dist/server/routes/compat.d.ts +0 -24
- package/dist/server/routes/compat.d.ts.map +0 -1
- package/dist/server/routes/compat.js +0 -653
- package/dist/server/routes/compat.js.map +0 -1
- package/dist/server/routes/deep-fetch.d.ts +0 -9
- package/dist/server/routes/deep-fetch.d.ts.map +0 -1
- package/dist/server/routes/deep-fetch.js +0 -50
- package/dist/server/routes/deep-fetch.js.map +0 -1
- package/dist/server/routes/demo.d.ts +0 -25
- package/dist/server/routes/demo.d.ts.map +0 -1
- package/dist/server/routes/demo.js +0 -434
- package/dist/server/routes/demo.js.map +0 -1
- package/dist/server/routes/extract.d.ts +0 -9
- package/dist/server/routes/extract.d.ts.map +0 -1
- package/dist/server/routes/extract.js +0 -150
- package/dist/server/routes/extract.js.map +0 -1
- package/dist/server/routes/fetch.d.ts +0 -8
- package/dist/server/routes/fetch.d.ts.map +0 -1
- package/dist/server/routes/fetch.js +0 -988
- package/dist/server/routes/fetch.js.map +0 -1
- package/dist/server/routes/health.d.ts +0 -8
- package/dist/server/routes/health.d.ts.map +0 -1
- package/dist/server/routes/health.js +0 -20
- package/dist/server/routes/health.js.map +0 -1
- package/dist/server/routes/jobs.d.ts +0 -8
- package/dist/server/routes/jobs.d.ts.map +0 -1
- package/dist/server/routes/jobs.js +0 -487
- package/dist/server/routes/jobs.js.map +0 -1
- package/dist/server/routes/mcp.d.ts +0 -18
- package/dist/server/routes/mcp.d.ts.map +0 -1
- package/dist/server/routes/mcp.js +0 -1260
- package/dist/server/routes/mcp.js.map +0 -1
- package/dist/server/routes/oauth.d.ts +0 -10
- package/dist/server/routes/oauth.d.ts.map +0 -1
- package/dist/server/routes/oauth.js +0 -334
- package/dist/server/routes/oauth.js.map +0 -1
- package/dist/server/routes/quick-answer.d.ts +0 -9
- package/dist/server/routes/quick-answer.d.ts.map +0 -1
- package/dist/server/routes/quick-answer.js +0 -93
- package/dist/server/routes/quick-answer.js.map +0 -1
- package/dist/server/routes/screenshot.d.ts +0 -23
- package/dist/server/routes/screenshot.d.ts.map +0 -1
- package/dist/server/routes/screenshot.js +0 -819
- package/dist/server/routes/screenshot.js.map +0 -1
- package/dist/server/routes/search.d.ts +0 -7
- package/dist/server/routes/search.d.ts.map +0 -1
- package/dist/server/routes/search.js +0 -312
- package/dist/server/routes/search.js.map +0 -1
- package/dist/server/routes/session.d.ts +0 -16
- package/dist/server/routes/session.d.ts.map +0 -1
- package/dist/server/routes/session.js +0 -278
- package/dist/server/routes/session.js.map +0 -1
- package/dist/server/routes/stats.d.ts +0 -7
- package/dist/server/routes/stats.d.ts.map +0 -1
- package/dist/server/routes/stats.js +0 -65
- package/dist/server/routes/stats.js.map +0 -1
- package/dist/server/routes/stripe.d.ts +0 -16
- package/dist/server/routes/stripe.d.ts.map +0 -1
- package/dist/server/routes/stripe.js +0 -283
- package/dist/server/routes/stripe.js.map +0 -1
- package/dist/server/routes/users.d.ts +0 -9
- package/dist/server/routes/users.d.ts.map +0 -1
- package/dist/server/routes/users.js +0 -1211
- package/dist/server/routes/users.js.map +0 -1
- package/dist/server/routes/watch.d.ts +0 -16
- package/dist/server/routes/watch.d.ts.map +0 -1
- package/dist/server/routes/watch.js +0 -257
- package/dist/server/routes/watch.js.map +0 -1
- package/dist/server/routes/webhooks.d.ts +0 -16
- package/dist/server/routes/webhooks.d.ts.map +0 -1
- package/dist/server/routes/webhooks.js +0 -74
- package/dist/server/routes/webhooks.js.map +0 -1
- package/dist/server/routes/youtube.d.ts +0 -7
- package/dist/server/routes/youtube.d.ts.map +0 -1
- package/dist/server/routes/youtube.js +0 -93
- package/dist/server/routes/youtube.js.map +0 -1
- package/dist/server/sentry.d.ts +0 -14
- package/dist/server/sentry.d.ts.map +0 -1
- package/dist/server/sentry.js +0 -39
- package/dist/server/sentry.js.map +0 -1
- package/dist/server/types.d.ts +0 -16
- package/dist/server/types.d.ts.map +0 -1
- package/dist/server/types.js +0 -8
- package/dist/server/types.js.map +0 -1
- package/dist/server/utils/response.d.ts +0 -45
- package/dist/server/utils/response.d.ts.map +0 -1
- package/dist/server/utils/response.js +0 -70
- package/dist/server/utils/response.js.map +0 -1
- package/dist/server/utils/sse.d.ts +0 -23
- package/dist/server/utils/sse.d.ts.map +0 -1
- package/dist/server/utils/sse.js +0 -39
- package/dist/server/utils/sse.js.map +0 -1
- package/dist/types.d.ts.map +0 -1
- package/dist/types.js.map +0 -1
|
@@ -38,4 +38,3 @@ export interface FetchResult {
|
|
|
38
38
|
* SECURITY: Manual redirect handling with SSRF re-validation
|
|
39
39
|
*/
|
|
40
40
|
export declare function simpleFetch(url: string, userAgent?: string, timeoutMs?: number, customHeaders?: Record<string, string>, abortSignal?: AbortSignal, proxy?: string): Promise<FetchResult>;
|
|
41
|
-
//# sourceMappingURL=http-fetch.d.ts.map
|
package/dist/core/http-fetch.js
CHANGED
|
@@ -14,6 +14,8 @@ import { TimeoutError, BlockedError, NetworkError, WebPeelError } from '../types
|
|
|
14
14
|
import { getCached } from './cache.js';
|
|
15
15
|
import { cachedLookup, resolveAndCache, startDnsWarmup } from './dns-cache.js';
|
|
16
16
|
import { detectChallenge } from './challenge-detection.js';
|
|
17
|
+
import { createLogger } from './logger.js';
|
|
18
|
+
const log = createLogger('http');
|
|
17
19
|
// ── HTTP status text fallbacks (HTTP/2 omits reason phrases) ──────────────────
|
|
18
20
|
const HTTP_STATUS_TEXT = {
|
|
19
21
|
400: 'Bad Request',
|
|
@@ -81,8 +83,7 @@ function normalizeUrlForConditionalCache(url) {
|
|
|
81
83
|
}
|
|
82
84
|
catch (e) {
|
|
83
85
|
// Non-fatal: URL normalization failed, returning raw trimmed URL
|
|
84
|
-
|
|
85
|
-
console.debug('[webpeel]', 'URL normalization:', e instanceof Error ? e.message : e);
|
|
86
|
+
log.debug('URL normalization:', e instanceof Error ? e.message : e);
|
|
86
87
|
return url.trim();
|
|
87
88
|
}
|
|
88
89
|
}
|
|
@@ -409,8 +410,7 @@ export async function simpleFetch(url, userAgent, timeoutMs = 30000, customHeade
|
|
|
409
410
|
}
|
|
410
411
|
catch (e) {
|
|
411
412
|
// Ignore URL parsing errors here; validation handles invalid input below.
|
|
412
|
-
|
|
413
|
-
console.debug('[webpeel]', 'DNS prefetch (initial URL):', e instanceof Error ? e.message : e);
|
|
413
|
+
log.debug('DNS prefetch (initial URL):', e instanceof Error ? e.message : e);
|
|
414
414
|
}
|
|
415
415
|
while (redirectCount <= MAX_REDIRECTS) {
|
|
416
416
|
// Detect redirect loops
|
|
@@ -466,8 +466,7 @@ export async function simpleFetch(url, userAgent, timeoutMs = 30000, customHeade
|
|
|
466
466
|
}
|
|
467
467
|
catch (e) {
|
|
468
468
|
// Ignore URL parsing errors here; validation handles invalid input below.
|
|
469
|
-
|
|
470
|
-
console.debug('[webpeel]', 'DNS prefetch (redirect URL):', e instanceof Error ? e.message : e);
|
|
469
|
+
log.debug('DNS prefetch (redirect URL):', e instanceof Error ? e.message : e);
|
|
471
470
|
}
|
|
472
471
|
redirectCount++;
|
|
473
472
|
continue;
|
|
@@ -627,4 +626,3 @@ export async function simpleFetch(url, userAgent, timeoutMs = 30000, customHeade
|
|
|
627
626
|
}
|
|
628
627
|
throw new WebPeelError(`Too many redirects (max ${MAX_REDIRECTS})`);
|
|
629
628
|
}
|
|
630
|
-
//# sourceMappingURL=http-fetch.js.map
|
package/dist/core/human.d.ts
CHANGED
|
@@ -173,4 +173,3 @@ export declare function humanRead(page: Page, durationMs?: number): Promise<void
|
|
|
173
173
|
* @param durationMs How long to browse (default: 30000-60000 random)
|
|
174
174
|
*/
|
|
175
175
|
export declare function warmupBrowse(page: Page, durationMs?: number): Promise<void>;
|
|
176
|
-
//# sourceMappingURL=human.d.ts.map
|
package/dist/core/human.js
CHANGED
package/dist/core/jobs.d.ts
CHANGED
package/dist/core/jobs.js
CHANGED
package/dist/core/json-ld.d.ts
CHANGED
package/dist/core/json-ld.js
CHANGED
|
@@ -53,4 +53,3 @@ export declare function parseItems(text: string, _schema?: object): Array<Record
|
|
|
53
53
|
* Extract structured data from content using an LLM.
|
|
54
54
|
*/
|
|
55
55
|
export declare function extractWithLLM(options: LLMExtractionOptions): Promise<LLMExtractionResult>;
|
|
56
|
-
//# sourceMappingURL=llm-extract.d.ts.map
|
package/dist/core/llm-extract.js
CHANGED
|
@@ -0,0 +1,17 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Lightweight structured logger — no external dependencies.
|
|
3
|
+
*
|
|
4
|
+
* Levels: debug < info < warn < error < silent
|
|
5
|
+
*
|
|
6
|
+
* Respects WEBPEEL_LOG_LEVEL env var.
|
|
7
|
+
* Defaults: production → 'info', development → 'debug'.
|
|
8
|
+
*
|
|
9
|
+
* All output goes to stderr so stdout stays clean for data/JSON (CLI piping).
|
|
10
|
+
*/
|
|
11
|
+
export type LogLevel = 'debug' | 'info' | 'warn' | 'error' | 'silent';
|
|
12
|
+
export declare function createLogger(module: string): {
|
|
13
|
+
debug: (...args: any[]) => void;
|
|
14
|
+
info: (...args: any[]) => void;
|
|
15
|
+
warn: (...args: any[]) => void;
|
|
16
|
+
error: (...args: any[]) => void;
|
|
17
|
+
};
|
|
@@ -0,0 +1,44 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Lightweight structured logger — no external dependencies.
|
|
3
|
+
*
|
|
4
|
+
* Levels: debug < info < warn < error < silent
|
|
5
|
+
*
|
|
6
|
+
* Respects WEBPEEL_LOG_LEVEL env var.
|
|
7
|
+
* Defaults: production → 'info', development → 'debug'.
|
|
8
|
+
*
|
|
9
|
+
* All output goes to stderr so stdout stays clean for data/JSON (CLI piping).
|
|
10
|
+
*/
|
|
11
|
+
const LEVELS = {
|
|
12
|
+
debug: 0,
|
|
13
|
+
info: 1,
|
|
14
|
+
warn: 2,
|
|
15
|
+
error: 3,
|
|
16
|
+
silent: 4,
|
|
17
|
+
};
|
|
18
|
+
const currentLevel = () => {
|
|
19
|
+
const env = process.env.WEBPEEL_LOG_LEVEL?.toLowerCase();
|
|
20
|
+
if (env && env in LEVELS)
|
|
21
|
+
return env;
|
|
22
|
+
return process.env.NODE_ENV === 'production' ? 'info' : 'debug';
|
|
23
|
+
};
|
|
24
|
+
export function createLogger(module) {
|
|
25
|
+
const prefix = `[webpeel:${module}]`;
|
|
26
|
+
return {
|
|
27
|
+
debug: (...args) => {
|
|
28
|
+
if (LEVELS[currentLevel()] <= LEVELS.debug)
|
|
29
|
+
console.error(prefix, ...args);
|
|
30
|
+
},
|
|
31
|
+
info: (...args) => {
|
|
32
|
+
if (LEVELS[currentLevel()] <= LEVELS.info)
|
|
33
|
+
console.error(prefix, ...args);
|
|
34
|
+
},
|
|
35
|
+
warn: (...args) => {
|
|
36
|
+
if (LEVELS[currentLevel()] <= LEVELS.warn)
|
|
37
|
+
console.error(prefix, '[WARN]', ...args);
|
|
38
|
+
},
|
|
39
|
+
error: (...args) => {
|
|
40
|
+
if (LEVELS[currentLevel()] <= LEVELS.error)
|
|
41
|
+
console.error(prefix, '[ERROR]', ...args);
|
|
42
|
+
},
|
|
43
|
+
};
|
|
44
|
+
}
|
package/dist/core/map.d.ts
CHANGED
package/dist/core/map.js
CHANGED
package/dist/core/markdown.d.ts
CHANGED
|
@@ -90,4 +90,3 @@ export declare function cleanForAI(markdown: string): string;
|
|
|
90
90
|
* Removes empty links, orphaned image links, collapses excess newlines, strips trailing whitespace.
|
|
91
91
|
*/
|
|
92
92
|
export declare function cleanMarkdownNoise(content: string): string;
|
|
93
|
-
//# sourceMappingURL=markdown.d.ts.map
|
package/dist/core/markdown.js
CHANGED
package/dist/core/metadata.d.ts
CHANGED
package/dist/core/metadata.js
CHANGED
package/dist/core/paginate.d.ts
CHANGED
package/dist/core/paginate.js
CHANGED
package/dist/core/pdf.d.ts
CHANGED
package/dist/core/pdf.js
CHANGED
package/dist/core/peel-tls.d.ts
CHANGED
|
@@ -23,4 +23,3 @@ export declare function isPeelTLSAvailable(): boolean;
|
|
|
23
23
|
export declare function peelTLSFetch(url: string, options?: PeelTLSOptions): Promise<PeelTLSResult>;
|
|
24
24
|
/** Gracefully shut down the PeelTLS binary process. */
|
|
25
25
|
export declare function shutdownPeelTLS(): Promise<void>;
|
|
26
|
-
//# sourceMappingURL=peel-tls.d.ts.map
|
package/dist/core/peel-tls.js
CHANGED
package/dist/core/pipeline.d.ts
CHANGED
package/dist/core/pipeline.js
CHANGED
|
@@ -20,6 +20,8 @@ import { quickAnswer as runQuickAnswer } from './quick-answer.js';
|
|
|
20
20
|
import { Timer } from './timing.js';
|
|
21
21
|
import { chunkContent } from './chunker.js';
|
|
22
22
|
import { BlockedError } from '../types.js';
|
|
23
|
+
import { createLogger } from './logger.js';
|
|
24
|
+
const log = createLogger('pipeline');
|
|
23
25
|
/** Create the initial PipelineContext with defaults */
|
|
24
26
|
export function createContext(url, options) {
|
|
25
27
|
return {
|
|
@@ -263,8 +265,7 @@ export async function fetchContent(ctx) {
|
|
|
263
265
|
}
|
|
264
266
|
catch (e) {
|
|
265
267
|
// Domain API failed — fall through to normal fetch
|
|
266
|
-
|
|
267
|
-
console.debug('[webpeel]', 'domain API first-pass failed, falling back to fetch:', e instanceof Error ? e.message : e);
|
|
268
|
+
log.debug('domain API first-pass failed, falling back to fetch:', e instanceof Error ? e.message : e);
|
|
268
269
|
}
|
|
269
270
|
}
|
|
270
271
|
ctx.timer.mark('fetch');
|
|
@@ -295,6 +296,7 @@ export async function fetchContent(ctx) {
|
|
|
295
296
|
blockResources: ctx.options.blockResources,
|
|
296
297
|
cloaked: ctx.options.cloaked,
|
|
297
298
|
cycle: ctx.options.cycle,
|
|
299
|
+
noEscalate: ctx.options.noEscalate,
|
|
298
300
|
});
|
|
299
301
|
}
|
|
300
302
|
catch (fetchError) {
|
|
@@ -333,6 +335,7 @@ export async function fetchContent(ctx) {
|
|
|
333
335
|
// to get the title/snippet from the search engine's cached version.
|
|
334
336
|
if (fetchError instanceof BlockedError) {
|
|
335
337
|
try {
|
|
338
|
+
// @ts-ignore — proprietary module, gitignored
|
|
336
339
|
const { searchFallback } = await import('./search-fallback.js');
|
|
337
340
|
const searchResult = await searchFallback(ctx.url);
|
|
338
341
|
if (searchResult.cachedContent && searchResult.cachedContent.length > 50) {
|
|
@@ -371,8 +374,7 @@ export async function fetchContent(ctx) {
|
|
|
371
374
|
}
|
|
372
375
|
catch (e) {
|
|
373
376
|
// Non-fatal: auto-scroll failed, continuing with whatever HTML we have
|
|
374
|
-
|
|
375
|
-
console.debug('[webpeel]', 'auto-scroll failed:', e instanceof Error ? e.message : e);
|
|
377
|
+
log.debug('auto-scroll failed:', e instanceof Error ? e.message : e);
|
|
376
378
|
}
|
|
377
379
|
finally {
|
|
378
380
|
// Close page unless branding or design analysis also needs it
|
|
@@ -385,8 +387,7 @@ export async function fetchContent(ctx) {
|
|
|
385
387
|
}
|
|
386
388
|
catch (e) {
|
|
387
389
|
// Non-fatal: page/browser cleanup after auto-scroll
|
|
388
|
-
|
|
389
|
-
console.debug('[webpeel]', 'page/browser cleanup after auto-scroll:', e instanceof Error ? e.message : e);
|
|
390
|
+
log.debug('page/browser cleanup after auto-scroll:', e instanceof Error ? e.message : e);
|
|
390
391
|
}
|
|
391
392
|
fetchResult.page = undefined;
|
|
392
393
|
}
|
|
@@ -619,7 +620,7 @@ export async function parseContent(ctx) {
|
|
|
619
620
|
htmlForConvert = contentHtml.slice(0, minChars);
|
|
620
621
|
}
|
|
621
622
|
if (process.env.DEBUG) {
|
|
622
|
-
|
|
623
|
+
log.debug(`budget pre-truncate: ${contentHtml.length} → ${htmlForConvert.length} chars`);
|
|
623
624
|
}
|
|
624
625
|
}
|
|
625
626
|
}
|
|
@@ -659,7 +660,7 @@ export async function parseContent(ctx) {
|
|
|
659
660
|
// (common for listing/index pages, SPAs with shell-first layouts).
|
|
660
661
|
if (htmlForConvert !== contentHtml && convertedContent.length < 200 && contentHtml.length > 20000) {
|
|
661
662
|
if (process.env.DEBUG) {
|
|
662
|
-
|
|
663
|
+
log.debug(`budget pre-truncation produced thin content (${convertedContent.length} chars from ${htmlForConvert.length} HTML). Retrying with full HTML (${contentHtml.length} chars).`);
|
|
663
664
|
}
|
|
664
665
|
ctx.timer.mark('convert-retry');
|
|
665
666
|
let retryConverted;
|
|
@@ -701,8 +702,7 @@ export async function parseContent(ctx) {
|
|
|
701
702
|
}
|
|
702
703
|
catch (e) {
|
|
703
704
|
// Non-fatal: JSON parse failed, treating as malformed
|
|
704
|
-
|
|
705
|
-
console.debug('[webpeel]', 'JSON parse failed:', e instanceof Error ? e.message : e);
|
|
705
|
+
log.debug('JSON parse failed:', e instanceof Error ? e.message : e);
|
|
706
706
|
ctx.content = fetchResult.html;
|
|
707
707
|
ctx.title = 'JSON Response (malformed)';
|
|
708
708
|
}
|
|
@@ -734,8 +734,7 @@ export async function parseContent(ctx) {
|
|
|
734
734
|
}
|
|
735
735
|
catch (e) {
|
|
736
736
|
// Non-fatal: XML/RSS parse failed, using raw content
|
|
737
|
-
|
|
738
|
-
console.debug('[webpeel]', 'XML/RSS parse failed:', e instanceof Error ? e.message : e);
|
|
737
|
+
log.debug('XML/RSS parse failed:', e instanceof Error ? e.message : e);
|
|
739
738
|
ctx.content = fetchResult.html;
|
|
740
739
|
ctx.title = 'XML Document';
|
|
741
740
|
}
|
|
@@ -843,7 +842,7 @@ export async function postProcess(ctx) {
|
|
|
843
842
|
let budgetedContent = distillToBudget(ctx.content, options.budget, budgetFormat);
|
|
844
843
|
ctx.timer.end('budget');
|
|
845
844
|
if (process.env.DEBUG) {
|
|
846
|
-
|
|
845
|
+
log.debug(`budget result: ${originalContent.length} → ${budgetedContent.length} chars`);
|
|
847
846
|
}
|
|
848
847
|
// Safety net: if BM25 distillation stripped too much (< 10% of original)
|
|
849
848
|
// on a substantial page, fall back to simple head truncation.
|
|
@@ -860,7 +859,7 @@ export async function postProcess(ctx) {
|
|
|
860
859
|
ctx.budgetFallback = true;
|
|
861
860
|
ctx.warnings.push('Content was truncated to fit budget using head truncation (BM25 distillation produced insufficient content)');
|
|
862
861
|
if (process.env.DEBUG) {
|
|
863
|
-
|
|
862
|
+
log.debug(`budget distillation fallback: BM25 produced ${budgetedContent.length} chars (< 10% of ${originalContent.length}), using head truncation`);
|
|
864
863
|
}
|
|
865
864
|
}
|
|
866
865
|
ctx.content = budgetedContent;
|
|
@@ -879,8 +878,7 @@ export async function postProcess(ctx) {
|
|
|
879
878
|
}
|
|
880
879
|
catch (e) {
|
|
881
880
|
// Domain extraction failure is non-fatal; continue with normal content
|
|
882
|
-
|
|
883
|
-
console.debug('[webpeel]', 'domain extraction failed:', e instanceof Error ? e.message : e);
|
|
881
|
+
log.debug('domain extraction failed:', e instanceof Error ? e.message : e);
|
|
884
882
|
}
|
|
885
883
|
}
|
|
886
884
|
// === Challenge / bot-protection page detection ===
|
|
@@ -921,6 +919,7 @@ export async function postProcess(ctx) {
|
|
|
921
919
|
}
|
|
922
920
|
// Try search fallback for the real content
|
|
923
921
|
try {
|
|
922
|
+
// @ts-ignore — proprietary module, gitignored
|
|
924
923
|
const { searchFallback } = await import('./search-fallback.js');
|
|
925
924
|
const searchResult = await searchFallback(ctx.url);
|
|
926
925
|
if (searchResult.cachedContent && searchResult.cachedContent.length > 50) {
|
|
@@ -978,6 +977,7 @@ export async function postProcess(ctx) {
|
|
|
978
977
|
// Try 4: Search-as-proxy fallback (when page appears blocked)
|
|
979
978
|
// Search engines already crawled this page — use their cached snippet.
|
|
980
979
|
try {
|
|
980
|
+
// @ts-ignore — proprietary module, gitignored
|
|
981
981
|
const { searchFallback } = await import('./search-fallback.js');
|
|
982
982
|
const searchResult = await searchFallback(ctx.url);
|
|
983
983
|
if (searchResult.cachedContent && searchResult.cachedContent.length > 50) {
|
|
@@ -1013,7 +1013,7 @@ export async function finalize(ctx) {
|
|
|
1013
1013
|
ctx.brandingProfile = await extractBranding(fetchResult.page);
|
|
1014
1014
|
}
|
|
1015
1015
|
catch (error) {
|
|
1016
|
-
|
|
1016
|
+
log.error('Branding extraction failed:', error);
|
|
1017
1017
|
}
|
|
1018
1018
|
finally {
|
|
1019
1019
|
// Clean up the kept-open page and browser
|
|
@@ -1025,8 +1025,7 @@ export async function finalize(ctx) {
|
|
|
1025
1025
|
}
|
|
1026
1026
|
catch (e) {
|
|
1027
1027
|
// Non-fatal: page/browser cleanup after branding extraction
|
|
1028
|
-
|
|
1029
|
-
console.debug('[webpeel]', 'page/browser cleanup after branding:', e instanceof Error ? e.message : e);
|
|
1028
|
+
log.debug('page/browser cleanup after branding:', e instanceof Error ? e.message : e);
|
|
1030
1029
|
}
|
|
1031
1030
|
}
|
|
1032
1031
|
}
|
|
@@ -1037,7 +1036,7 @@ export async function finalize(ctx) {
|
|
|
1037
1036
|
ctx.designAnalysisResult = await extractDesignAnalysis(fetchResult.page);
|
|
1038
1037
|
}
|
|
1039
1038
|
catch (error) {
|
|
1040
|
-
|
|
1039
|
+
log.error('Design analysis extraction failed:', error);
|
|
1041
1040
|
}
|
|
1042
1041
|
finally {
|
|
1043
1042
|
if (!options.branding) {
|
|
@@ -1049,8 +1048,7 @@ export async function finalize(ctx) {
|
|
|
1049
1048
|
}
|
|
1050
1049
|
}
|
|
1051
1050
|
catch (e) {
|
|
1052
|
-
|
|
1053
|
-
console.debug('[webpeel]', 'page/browser cleanup after design analysis:', e instanceof Error ? e.message : e);
|
|
1051
|
+
log.debug('page/browser cleanup after design analysis:', e instanceof Error ? e.message : e);
|
|
1054
1052
|
}
|
|
1055
1053
|
}
|
|
1056
1054
|
}
|
|
@@ -1063,7 +1061,7 @@ export async function finalize(ctx) {
|
|
|
1063
1061
|
ctx.changeResult = await trackChange(fetchResult.url, ctx.content, fingerprint);
|
|
1064
1062
|
}
|
|
1065
1063
|
catch (error) {
|
|
1066
|
-
|
|
1064
|
+
log.error('Change tracking failed:', error);
|
|
1067
1065
|
}
|
|
1068
1066
|
}
|
|
1069
1067
|
// Generate AI summary if requested
|
|
@@ -1081,7 +1079,7 @@ export async function finalize(ctx) {
|
|
|
1081
1079
|
});
|
|
1082
1080
|
}
|
|
1083
1081
|
catch (error) {
|
|
1084
|
-
|
|
1082
|
+
log.error('Summary generation failed:', error);
|
|
1085
1083
|
}
|
|
1086
1084
|
}
|
|
1087
1085
|
}
|
|
@@ -1185,4 +1183,3 @@ export function buildResult(ctx) {
|
|
|
1185
1183
|
...(fetchResult.autoInteract !== undefined ? { autoInteract: fetchResult.autoInteract } : {}),
|
|
1186
1184
|
};
|
|
1187
1185
|
}
|
|
1188
|
-
//# sourceMappingURL=pipeline.js.map
|
package/dist/core/profiles.d.ts
CHANGED
|
@@ -45,4 +45,3 @@ export declare function deleteProfile(name: string): boolean;
|
|
|
45
45
|
* 3. On browser close or Ctrl+C, captures storage state and saves the profile
|
|
46
46
|
*/
|
|
47
47
|
export declare function createProfile(name: string, description?: string): Promise<void>;
|
|
48
|
-
//# sourceMappingURL=profiles.d.ts.map
|
package/dist/core/profiles.js
CHANGED
package/dist/core/readability.js
CHANGED
package/dist/core/research.d.ts
CHANGED
package/dist/core/research.js
CHANGED
|
@@ -64,4 +64,3 @@ export declare function findSchemaForUrl(url: string): ExtractionSchema | null;
|
|
|
64
64
|
* @returns Array of extracted items (may be empty)
|
|
65
65
|
*/
|
|
66
66
|
export declare function extractWithSchema(html: string, schema: ExtractionSchema, baseUrl?: string): ExtractedItem[];
|
|
67
|
-
//# sourceMappingURL=schema-extraction.d.ts.map
|
|
@@ -222,4 +222,3 @@ export interface DesignComparisonResult {
|
|
|
222
222
|
* parallel and diffing the result into a structured DesignComparison.
|
|
223
223
|
*/
|
|
224
224
|
export declare function takeDesignComparison(subjectUrl: string, referenceUrl: string, options?: DesignCompareOptions): Promise<DesignComparisonResult>;
|
|
225
|
-
//# sourceMappingURL=screenshot.d.ts.map
|
package/dist/core/screenshot.js
CHANGED