webpeel 0.19.4 → 0.20.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (544) hide show
  1. package/README.md +2 -2
  2. package/dist/cache.d.ts +0 -1
  3. package/dist/cache.js +0 -1
  4. package/dist/cli/commands/auth.d.ts +5 -0
  5. package/dist/cli/commands/auth.js +476 -0
  6. package/dist/cli/commands/fetch.d.ts +6 -0
  7. package/dist/cli/commands/fetch.js +1015 -0
  8. package/dist/cli/commands/interact.d.ts +5 -0
  9. package/dist/cli/commands/interact.js +839 -0
  10. package/dist/cli/commands/jobs.d.ts +5 -0
  11. package/dist/cli/commands/jobs.js +997 -0
  12. package/dist/cli/commands/screenshot.d.ts +5 -0
  13. package/dist/cli/commands/screenshot.js +273 -0
  14. package/dist/cli/commands/search.d.ts +5 -0
  15. package/dist/cli/commands/search.js +524 -0
  16. package/dist/cli/utils.d.ts +84 -0
  17. package/dist/cli/utils.js +686 -0
  18. package/dist/cli-auth.d.ts +0 -1
  19. package/dist/cli-auth.js +0 -1
  20. package/dist/cli.d.ts +7 -6
  21. package/dist/cli.js +35 -4698
  22. package/dist/core/actions.d.ts +0 -1
  23. package/dist/core/actions.js +0 -1
  24. package/dist/core/agent.d.ts +0 -1
  25. package/dist/core/agent.js +9 -12
  26. package/dist/core/answer.d.ts +0 -1
  27. package/dist/core/answer.js +0 -1
  28. package/dist/core/application-tracker.d.ts +0 -1
  29. package/dist/core/application-tracker.js +0 -1
  30. package/dist/core/apply.d.ts +0 -1
  31. package/dist/core/apply.js +0 -1
  32. package/dist/core/auto-extract.d.ts +0 -1
  33. package/dist/core/auto-extract.js +0 -1
  34. package/dist/core/auto-interact.d.ts +0 -1
  35. package/dist/core/auto-interact.js +0 -1
  36. package/dist/core/bm25-filter.d.ts +0 -1
  37. package/dist/core/bm25-filter.js +0 -1
  38. package/dist/core/branding.d.ts +0 -1
  39. package/dist/core/branding.js +0 -1
  40. package/dist/core/browser-fetch.d.ts +0 -1
  41. package/dist/core/browser-fetch.js +17 -10
  42. package/dist/core/browser-pool.d.ts +0 -1
  43. package/dist/core/browser-pool.js +0 -1
  44. package/dist/core/budget.d.ts +0 -1
  45. package/dist/core/budget.js +0 -1
  46. package/dist/core/cache.d.ts +0 -1
  47. package/dist/core/cache.js +0 -1
  48. package/dist/core/cf-worker-proxy.d.ts +0 -1
  49. package/dist/core/cf-worker-proxy.js +0 -1
  50. package/dist/core/challenge-detection.d.ts +0 -1
  51. package/dist/core/challenge-detection.js +0 -1
  52. package/dist/core/change-tracking.d.ts +0 -1
  53. package/dist/core/change-tracking.js +0 -1
  54. package/dist/core/chunker.d.ts +0 -1
  55. package/dist/core/chunker.js +0 -1
  56. package/dist/core/chunking.d.ts +0 -1
  57. package/dist/core/chunking.js +0 -1
  58. package/dist/core/cloak-fetch.d.ts +0 -1
  59. package/dist/core/cloak-fetch.js +0 -1
  60. package/dist/core/content-pruner.d.ts +0 -1
  61. package/dist/core/content-pruner.js +0 -1
  62. package/dist/core/crawl-checkpoint.d.ts +0 -1
  63. package/dist/core/crawl-checkpoint.js +0 -1
  64. package/dist/core/crawler.d.ts +0 -1
  65. package/dist/core/crawler.js +6 -5
  66. package/dist/core/cycle-fetch.d.ts +0 -1
  67. package/dist/core/cycle-fetch.js +0 -1
  68. package/dist/core/deep-fetch.d.ts +0 -1
  69. package/dist/core/deep-fetch.js +0 -1
  70. package/dist/core/design-analysis.d.ts +0 -1
  71. package/dist/core/design-analysis.js +0 -1
  72. package/dist/core/design-compare.d.ts +0 -1
  73. package/dist/core/design-compare.js +0 -1
  74. package/dist/core/diff.d.ts +0 -1
  75. package/dist/core/diff.js +0 -1
  76. package/dist/core/dns-cache.d.ts +0 -1
  77. package/dist/core/dns-cache.js +0 -1
  78. package/dist/core/documents.d.ts +0 -1
  79. package/dist/core/documents.js +0 -1
  80. package/dist/core/domain-extractors.d.ts +0 -1
  81. package/dist/core/domain-extractors.js +0 -1
  82. package/dist/core/extract-inline.d.ts +0 -1
  83. package/dist/core/extract-inline.js +0 -1
  84. package/dist/core/extract-listings.d.ts +0 -1
  85. package/dist/core/extract-listings.js +0 -1
  86. package/dist/core/extract.d.ts +0 -1
  87. package/dist/core/extract.js +0 -1
  88. package/dist/core/fetcher.d.ts +0 -1
  89. package/dist/core/fetcher.js +0 -1
  90. package/dist/core/google-cache.d.ts +0 -1
  91. package/dist/core/google-cache.js +0 -1
  92. package/dist/core/hotel-search.d.ts +0 -1
  93. package/dist/core/hotel-search.js +0 -1
  94. package/dist/core/http-fetch.d.ts +0 -1
  95. package/dist/core/http-fetch.js +5 -7
  96. package/dist/core/human.d.ts +0 -1
  97. package/dist/core/human.js +0 -1
  98. package/dist/core/jobs.d.ts +0 -1
  99. package/dist/core/jobs.js +0 -1
  100. package/dist/core/json-ld.d.ts +0 -1
  101. package/dist/core/json-ld.js +0 -1
  102. package/dist/core/llm-extract.d.ts +0 -1
  103. package/dist/core/llm-extract.js +0 -1
  104. package/dist/core/logger.d.ts +17 -0
  105. package/dist/core/logger.js +44 -0
  106. package/dist/core/map.d.ts +0 -1
  107. package/dist/core/map.js +0 -1
  108. package/dist/core/markdown.d.ts +0 -1
  109. package/dist/core/markdown.js +0 -1
  110. package/dist/core/metadata.d.ts +0 -1
  111. package/dist/core/metadata.js +0 -1
  112. package/dist/core/paginate.d.ts +0 -1
  113. package/dist/core/paginate.js +0 -1
  114. package/dist/core/pdf.d.ts +0 -1
  115. package/dist/core/pdf.js +0 -1
  116. package/dist/core/peel-tls.d.ts +0 -1
  117. package/dist/core/peel-tls.js +0 -1
  118. package/dist/core/pipeline.d.ts +0 -1
  119. package/dist/core/pipeline.js +22 -25
  120. package/dist/core/profiles.d.ts +0 -1
  121. package/dist/core/profiles.js +0 -1
  122. package/dist/core/quick-answer.d.ts +0 -1
  123. package/dist/core/quick-answer.js +0 -1
  124. package/dist/core/rate-governor.d.ts +0 -1
  125. package/dist/core/rate-governor.js +0 -1
  126. package/dist/core/readability.d.ts +0 -1
  127. package/dist/core/readability.js +0 -1
  128. package/dist/core/research.d.ts +0 -1
  129. package/dist/core/research.js +0 -1
  130. package/dist/core/schema-extraction.d.ts +0 -1
  131. package/dist/core/schema-extraction.js +0 -1
  132. package/dist/core/schema-postprocess.d.ts +0 -1
  133. package/dist/core/schema-postprocess.js +0 -1
  134. package/dist/core/schema-templates.d.ts +0 -1
  135. package/dist/core/schema-templates.js +0 -1
  136. package/dist/core/screenshot.d.ts +0 -1
  137. package/dist/core/screenshot.js +0 -1
  138. package/dist/core/search-fallback.d.ts +0 -1
  139. package/dist/core/search-fallback.js +0 -1
  140. package/dist/core/search-provider.d.ts +0 -1
  141. package/dist/core/search-provider.js +18 -21
  142. package/dist/core/site-search.d.ts +0 -1
  143. package/dist/core/site-search.js +0 -1
  144. package/dist/core/sitemap.d.ts +0 -1
  145. package/dist/core/sitemap.js +0 -1
  146. package/dist/core/stealth-patches.d.ts +0 -1
  147. package/dist/core/stealth-patches.js +0 -1
  148. package/dist/core/stemmer.d.ts +0 -1
  149. package/dist/core/stemmer.js +0 -1
  150. package/dist/core/strategies.d.ts +6 -1
  151. package/dist/core/strategies.js +29 -41
  152. package/dist/core/strategy-hooks.d.ts +0 -1
  153. package/dist/core/strategy-hooks.js +0 -1
  154. package/dist/core/summarize.d.ts +0 -1
  155. package/dist/core/summarize.js +0 -1
  156. package/dist/core/synonyms.d.ts +0 -1
  157. package/dist/core/synonyms.js +0 -1
  158. package/dist/core/table-format.d.ts +0 -1
  159. package/dist/core/table-format.js +0 -1
  160. package/dist/core/timing.d.ts +0 -1
  161. package/dist/core/timing.js +0 -1
  162. package/dist/core/user-agents.d.ts +0 -1
  163. package/dist/core/user-agents.js +0 -1
  164. package/dist/core/watch-manager.d.ts +0 -1
  165. package/dist/core/watch-manager.js +0 -1
  166. package/dist/core/watch.d.ts +0 -1
  167. package/dist/core/watch.js +0 -1
  168. package/dist/core/youtube.d.ts +0 -1
  169. package/dist/core/youtube.js +0 -1
  170. package/dist/index.d.ts +8 -3
  171. package/dist/index.js +27 -3
  172. package/dist/integrations/index.d.ts +0 -1
  173. package/dist/integrations/index.js +0 -1
  174. package/dist/integrations/langchain.d.ts +0 -1
  175. package/dist/integrations/langchain.js +0 -1
  176. package/dist/integrations/llamaindex.d.ts +0 -1
  177. package/dist/integrations/llamaindex.js +0 -1
  178. package/dist/mcp/handlers/act.d.ts +5 -0
  179. package/dist/mcp/handlers/act.js +34 -0
  180. package/dist/mcp/handlers/definitions.d.ts +6 -0
  181. package/dist/mcp/handlers/definitions.js +266 -0
  182. package/dist/mcp/handlers/extract.d.ts +6 -0
  183. package/dist/mcp/handlers/extract.js +102 -0
  184. package/dist/mcp/handlers/fetch.d.ts +6 -0
  185. package/dist/mcp/handlers/fetch.js +98 -0
  186. package/dist/mcp/handlers/find.d.ts +5 -0
  187. package/dist/mcp/handlers/find.js +137 -0
  188. package/dist/mcp/handlers/index.d.ts +13 -0
  189. package/dist/mcp/handlers/index.js +61 -0
  190. package/dist/mcp/handlers/legacy.d.ts +25 -0
  191. package/dist/mcp/handlers/legacy.js +450 -0
  192. package/dist/mcp/handlers/meta.d.ts +6 -0
  193. package/dist/mcp/handlers/meta.js +31 -0
  194. package/dist/mcp/handlers/monitor.d.ts +5 -0
  195. package/dist/mcp/handlers/monitor.js +41 -0
  196. package/dist/mcp/handlers/read.d.ts +6 -0
  197. package/dist/mcp/handlers/read.js +63 -0
  198. package/dist/mcp/handlers/see.d.ts +5 -0
  199. package/dist/mcp/handlers/see.js +75 -0
  200. package/dist/mcp/handlers/types.d.ts +29 -0
  201. package/dist/mcp/handlers/types.js +28 -0
  202. package/dist/mcp/server.d.ts +3 -4
  203. package/dist/mcp/server.js +35 -1101
  204. package/dist/mcp/smart-router.d.ts +0 -1
  205. package/dist/mcp/smart-router.js +3 -1
  206. package/dist/types.d.ts +6 -1
  207. package/dist/types.js +0 -1
  208. package/package.json +3 -13
  209. package/dist/cache.d.ts.map +0 -1
  210. package/dist/cache.js.map +0 -1
  211. package/dist/cli-auth.d.ts.map +0 -1
  212. package/dist/cli-auth.js.map +0 -1
  213. package/dist/cli.bundle.cjs +0 -159248
  214. package/dist/cli.d.ts.map +0 -1
  215. package/dist/cli.js.map +0 -1
  216. package/dist/core/actions.d.ts.map +0 -1
  217. package/dist/core/actions.js.map +0 -1
  218. package/dist/core/agent.d.ts.map +0 -1
  219. package/dist/core/agent.js.map +0 -1
  220. package/dist/core/answer.d.ts.map +0 -1
  221. package/dist/core/answer.js.map +0 -1
  222. package/dist/core/application-tracker.d.ts.map +0 -1
  223. package/dist/core/application-tracker.js.map +0 -1
  224. package/dist/core/apply.d.ts.map +0 -1
  225. package/dist/core/apply.js.map +0 -1
  226. package/dist/core/auto-extract.d.ts.map +0 -1
  227. package/dist/core/auto-extract.js.map +0 -1
  228. package/dist/core/auto-interact.d.ts.map +0 -1
  229. package/dist/core/auto-interact.js.map +0 -1
  230. package/dist/core/bm25-filter.d.ts.map +0 -1
  231. package/dist/core/bm25-filter.js.map +0 -1
  232. package/dist/core/branding.d.ts.map +0 -1
  233. package/dist/core/branding.js.map +0 -1
  234. package/dist/core/browser-fetch.d.ts.map +0 -1
  235. package/dist/core/browser-fetch.js.map +0 -1
  236. package/dist/core/browser-pool.d.ts.map +0 -1
  237. package/dist/core/browser-pool.js.map +0 -1
  238. package/dist/core/budget.d.ts.map +0 -1
  239. package/dist/core/budget.js.map +0 -1
  240. package/dist/core/cache.d.ts.map +0 -1
  241. package/dist/core/cache.js.map +0 -1
  242. package/dist/core/cf-worker-proxy.d.ts.map +0 -1
  243. package/dist/core/cf-worker-proxy.js.map +0 -1
  244. package/dist/core/challenge-detection.d.ts.map +0 -1
  245. package/dist/core/challenge-detection.js.map +0 -1
  246. package/dist/core/change-tracking.d.ts.map +0 -1
  247. package/dist/core/change-tracking.js.map +0 -1
  248. package/dist/core/chunker.d.ts.map +0 -1
  249. package/dist/core/chunker.js.map +0 -1
  250. package/dist/core/chunking.d.ts.map +0 -1
  251. package/dist/core/chunking.js.map +0 -1
  252. package/dist/core/cloak-fetch.d.ts.map +0 -1
  253. package/dist/core/cloak-fetch.js.map +0 -1
  254. package/dist/core/content-pruner.d.ts.map +0 -1
  255. package/dist/core/content-pruner.js.map +0 -1
  256. package/dist/core/crawl-checkpoint.d.ts.map +0 -1
  257. package/dist/core/crawl-checkpoint.js.map +0 -1
  258. package/dist/core/crawler.d.ts.map +0 -1
  259. package/dist/core/crawler.js.map +0 -1
  260. package/dist/core/cycle-fetch.d.ts.map +0 -1
  261. package/dist/core/cycle-fetch.js.map +0 -1
  262. package/dist/core/deep-fetch.d.ts.map +0 -1
  263. package/dist/core/deep-fetch.js.map +0 -1
  264. package/dist/core/design-analysis.d.ts.map +0 -1
  265. package/dist/core/design-analysis.js.map +0 -1
  266. package/dist/core/design-compare.d.ts.map +0 -1
  267. package/dist/core/design-compare.js.map +0 -1
  268. package/dist/core/diff.d.ts.map +0 -1
  269. package/dist/core/diff.js.map +0 -1
  270. package/dist/core/dns-cache.d.ts.map +0 -1
  271. package/dist/core/dns-cache.js.map +0 -1
  272. package/dist/core/documents.d.ts.map +0 -1
  273. package/dist/core/documents.js.map +0 -1
  274. package/dist/core/domain-extractors.d.ts.map +0 -1
  275. package/dist/core/domain-extractors.js.map +0 -1
  276. package/dist/core/extract-inline.d.ts.map +0 -1
  277. package/dist/core/extract-inline.js.map +0 -1
  278. package/dist/core/extract-listings.d.ts.map +0 -1
  279. package/dist/core/extract-listings.js.map +0 -1
  280. package/dist/core/extract.d.ts.map +0 -1
  281. package/dist/core/extract.js.map +0 -1
  282. package/dist/core/fetcher.d.ts.map +0 -1
  283. package/dist/core/fetcher.js.map +0 -1
  284. package/dist/core/google-cache.d.ts.map +0 -1
  285. package/dist/core/google-cache.js.map +0 -1
  286. package/dist/core/hotel-search.d.ts.map +0 -1
  287. package/dist/core/hotel-search.js.map +0 -1
  288. package/dist/core/http-fetch.d.ts.map +0 -1
  289. package/dist/core/http-fetch.js.map +0 -1
  290. package/dist/core/human.d.ts.map +0 -1
  291. package/dist/core/human.js.map +0 -1
  292. package/dist/core/jobs.d.ts.map +0 -1
  293. package/dist/core/jobs.js.map +0 -1
  294. package/dist/core/json-ld.d.ts.map +0 -1
  295. package/dist/core/json-ld.js.map +0 -1
  296. package/dist/core/llm-extract.d.ts.map +0 -1
  297. package/dist/core/llm-extract.js.map +0 -1
  298. package/dist/core/map.d.ts.map +0 -1
  299. package/dist/core/map.js.map +0 -1
  300. package/dist/core/markdown.d.ts.map +0 -1
  301. package/dist/core/markdown.js.map +0 -1
  302. package/dist/core/metadata.d.ts.map +0 -1
  303. package/dist/core/metadata.js.map +0 -1
  304. package/dist/core/paginate.d.ts.map +0 -1
  305. package/dist/core/paginate.js.map +0 -1
  306. package/dist/core/pdf.d.ts.map +0 -1
  307. package/dist/core/pdf.js.map +0 -1
  308. package/dist/core/peel-tls.d.ts.map +0 -1
  309. package/dist/core/peel-tls.js.map +0 -1
  310. package/dist/core/pipeline.d.ts.map +0 -1
  311. package/dist/core/pipeline.js.map +0 -1
  312. package/dist/core/profiles.d.ts.map +0 -1
  313. package/dist/core/profiles.js.map +0 -1
  314. package/dist/core/quick-answer.d.ts.map +0 -1
  315. package/dist/core/quick-answer.js.map +0 -1
  316. package/dist/core/rate-governor.d.ts.map +0 -1
  317. package/dist/core/rate-governor.js.map +0 -1
  318. package/dist/core/readability.d.ts.map +0 -1
  319. package/dist/core/readability.js.map +0 -1
  320. package/dist/core/research.d.ts.map +0 -1
  321. package/dist/core/research.js.map +0 -1
  322. package/dist/core/schema-extraction.d.ts.map +0 -1
  323. package/dist/core/schema-extraction.js.map +0 -1
  324. package/dist/core/schema-postprocess.d.ts.map +0 -1
  325. package/dist/core/schema-postprocess.js.map +0 -1
  326. package/dist/core/schema-templates.d.ts.map +0 -1
  327. package/dist/core/schema-templates.js.map +0 -1
  328. package/dist/core/screenshot.d.ts.map +0 -1
  329. package/dist/core/screenshot.js.map +0 -1
  330. package/dist/core/search-fallback.d.ts.map +0 -1
  331. package/dist/core/search-fallback.js.map +0 -1
  332. package/dist/core/search-provider.d.ts.map +0 -1
  333. package/dist/core/search-provider.js.map +0 -1
  334. package/dist/core/site-search.d.ts.map +0 -1
  335. package/dist/core/site-search.js.map +0 -1
  336. package/dist/core/sitemap.d.ts.map +0 -1
  337. package/dist/core/sitemap.js.map +0 -1
  338. package/dist/core/stealth-patches.d.ts.map +0 -1
  339. package/dist/core/stealth-patches.js.map +0 -1
  340. package/dist/core/stemmer.d.ts.map +0 -1
  341. package/dist/core/stemmer.js.map +0 -1
  342. package/dist/core/strategies.d.ts.map +0 -1
  343. package/dist/core/strategies.js.map +0 -1
  344. package/dist/core/strategy-hooks.d.ts.map +0 -1
  345. package/dist/core/strategy-hooks.js.map +0 -1
  346. package/dist/core/summarize.d.ts.map +0 -1
  347. package/dist/core/summarize.js.map +0 -1
  348. package/dist/core/synonyms.d.ts.map +0 -1
  349. package/dist/core/synonyms.js.map +0 -1
  350. package/dist/core/table-format.d.ts.map +0 -1
  351. package/dist/core/table-format.js.map +0 -1
  352. package/dist/core/timing.d.ts.map +0 -1
  353. package/dist/core/timing.js.map +0 -1
  354. package/dist/core/user-agents.d.ts.map +0 -1
  355. package/dist/core/user-agents.js.map +0 -1
  356. package/dist/core/watch-manager.d.ts.map +0 -1
  357. package/dist/core/watch-manager.js.map +0 -1
  358. package/dist/core/watch.d.ts.map +0 -1
  359. package/dist/core/watch.js.map +0 -1
  360. package/dist/core/youtube.d.ts.map +0 -1
  361. package/dist/core/youtube.js.map +0 -1
  362. package/dist/index.d.ts.map +0 -1
  363. package/dist/index.js.map +0 -1
  364. package/dist/integrations/index.d.ts.map +0 -1
  365. package/dist/integrations/index.js.map +0 -1
  366. package/dist/integrations/langchain.d.ts.map +0 -1
  367. package/dist/integrations/langchain.js.map +0 -1
  368. package/dist/integrations/llamaindex.d.ts.map +0 -1
  369. package/dist/integrations/llamaindex.js.map +0 -1
  370. package/dist/mcp/server.d.ts.map +0 -1
  371. package/dist/mcp/server.js.map +0 -1
  372. package/dist/mcp/smart-router.d.ts.map +0 -1
  373. package/dist/mcp/smart-router.js.map +0 -1
  374. package/dist/server/app.d.ts +0 -15
  375. package/dist/server/app.d.ts.map +0 -1
  376. package/dist/server/app.js +0 -350
  377. package/dist/server/app.js.map +0 -1
  378. package/dist/server/auth-store.d.ts +0 -28
  379. package/dist/server/auth-store.d.ts.map +0 -1
  380. package/dist/server/auth-store.js +0 -89
  381. package/dist/server/auth-store.js.map +0 -1
  382. package/dist/server/email-service.d.ts +0 -22
  383. package/dist/server/email-service.d.ts.map +0 -1
  384. package/dist/server/email-service.js +0 -80
  385. package/dist/server/email-service.js.map +0 -1
  386. package/dist/server/job-queue.d.ts +0 -93
  387. package/dist/server/job-queue.d.ts.map +0 -1
  388. package/dist/server/job-queue.js +0 -146
  389. package/dist/server/job-queue.js.map +0 -1
  390. package/dist/server/logger.d.ts +0 -11
  391. package/dist/server/logger.d.ts.map +0 -1
  392. package/dist/server/logger.js +0 -38
  393. package/dist/server/logger.js.map +0 -1
  394. package/dist/server/middleware/auth.d.ts +0 -29
  395. package/dist/server/middleware/auth.d.ts.map +0 -1
  396. package/dist/server/middleware/auth.js +0 -222
  397. package/dist/server/middleware/auth.js.map +0 -1
  398. package/dist/server/middleware/rate-limit.d.ts +0 -25
  399. package/dist/server/middleware/rate-limit.d.ts.map +0 -1
  400. package/dist/server/middleware/rate-limit.js +0 -168
  401. package/dist/server/middleware/rate-limit.js.map +0 -1
  402. package/dist/server/middleware/url-validator.d.ts +0 -16
  403. package/dist/server/middleware/url-validator.d.ts.map +0 -1
  404. package/dist/server/middleware/url-validator.js +0 -187
  405. package/dist/server/middleware/url-validator.js.map +0 -1
  406. package/dist/server/openapi.yaml +0 -4944
  407. package/dist/server/pg-auth-store.d.ts +0 -133
  408. package/dist/server/pg-auth-store.d.ts.map +0 -1
  409. package/dist/server/pg-auth-store.js +0 -473
  410. package/dist/server/pg-auth-store.js.map +0 -1
  411. package/dist/server/pg-job-queue.d.ts +0 -60
  412. package/dist/server/pg-job-queue.d.ts.map +0 -1
  413. package/dist/server/pg-job-queue.js +0 -365
  414. package/dist/server/pg-job-queue.js.map +0 -1
  415. package/dist/server/premium/domain-intel.d.ts +0 -17
  416. package/dist/server/premium/domain-intel.d.ts.map +0 -1
  417. package/dist/server/premium/domain-intel.js +0 -134
  418. package/dist/server/premium/domain-intel.js.map +0 -1
  419. package/dist/server/premium/index.d.ts +0 -18
  420. package/dist/server/premium/index.d.ts.map +0 -1
  421. package/dist/server/premium/index.js +0 -36
  422. package/dist/server/premium/index.js.map +0 -1
  423. package/dist/server/premium/swr-cache.d.ts +0 -15
  424. package/dist/server/premium/swr-cache.d.ts.map +0 -1
  425. package/dist/server/premium/swr-cache.js +0 -35
  426. package/dist/server/premium/swr-cache.js.map +0 -1
  427. package/dist/server/routes/activity.d.ts +0 -7
  428. package/dist/server/routes/activity.d.ts.map +0 -1
  429. package/dist/server/routes/activity.js +0 -68
  430. package/dist/server/routes/activity.js.map +0 -1
  431. package/dist/server/routes/agent.d.ts +0 -16
  432. package/dist/server/routes/agent.d.ts.map +0 -1
  433. package/dist/server/routes/agent.js +0 -247
  434. package/dist/server/routes/agent.js.map +0 -1
  435. package/dist/server/routes/answer.d.ts +0 -6
  436. package/dist/server/routes/answer.d.ts.map +0 -1
  437. package/dist/server/routes/answer.js +0 -133
  438. package/dist/server/routes/answer.js.map +0 -1
  439. package/dist/server/routes/ask.d.ts +0 -23
  440. package/dist/server/routes/ask.d.ts.map +0 -1
  441. package/dist/server/routes/ask.js +0 -119
  442. package/dist/server/routes/ask.js.map +0 -1
  443. package/dist/server/routes/batch.d.ts +0 -7
  444. package/dist/server/routes/batch.d.ts.map +0 -1
  445. package/dist/server/routes/batch.js +0 -412
  446. package/dist/server/routes/batch.js.map +0 -1
  447. package/dist/server/routes/cli-usage.d.ts +0 -7
  448. package/dist/server/routes/cli-usage.d.ts.map +0 -1
  449. package/dist/server/routes/cli-usage.js +0 -121
  450. package/dist/server/routes/cli-usage.js.map +0 -1
  451. package/dist/server/routes/compat.d.ts +0 -24
  452. package/dist/server/routes/compat.d.ts.map +0 -1
  453. package/dist/server/routes/compat.js +0 -653
  454. package/dist/server/routes/compat.js.map +0 -1
  455. package/dist/server/routes/deep-fetch.d.ts +0 -9
  456. package/dist/server/routes/deep-fetch.d.ts.map +0 -1
  457. package/dist/server/routes/deep-fetch.js +0 -50
  458. package/dist/server/routes/deep-fetch.js.map +0 -1
  459. package/dist/server/routes/demo.d.ts +0 -25
  460. package/dist/server/routes/demo.d.ts.map +0 -1
  461. package/dist/server/routes/demo.js +0 -434
  462. package/dist/server/routes/demo.js.map +0 -1
  463. package/dist/server/routes/extract.d.ts +0 -9
  464. package/dist/server/routes/extract.d.ts.map +0 -1
  465. package/dist/server/routes/extract.js +0 -150
  466. package/dist/server/routes/extract.js.map +0 -1
  467. package/dist/server/routes/fetch.d.ts +0 -8
  468. package/dist/server/routes/fetch.d.ts.map +0 -1
  469. package/dist/server/routes/fetch.js +0 -988
  470. package/dist/server/routes/fetch.js.map +0 -1
  471. package/dist/server/routes/health.d.ts +0 -8
  472. package/dist/server/routes/health.d.ts.map +0 -1
  473. package/dist/server/routes/health.js +0 -20
  474. package/dist/server/routes/health.js.map +0 -1
  475. package/dist/server/routes/jobs.d.ts +0 -8
  476. package/dist/server/routes/jobs.d.ts.map +0 -1
  477. package/dist/server/routes/jobs.js +0 -487
  478. package/dist/server/routes/jobs.js.map +0 -1
  479. package/dist/server/routes/mcp.d.ts +0 -18
  480. package/dist/server/routes/mcp.d.ts.map +0 -1
  481. package/dist/server/routes/mcp.js +0 -1260
  482. package/dist/server/routes/mcp.js.map +0 -1
  483. package/dist/server/routes/oauth.d.ts +0 -10
  484. package/dist/server/routes/oauth.d.ts.map +0 -1
  485. package/dist/server/routes/oauth.js +0 -334
  486. package/dist/server/routes/oauth.js.map +0 -1
  487. package/dist/server/routes/quick-answer.d.ts +0 -9
  488. package/dist/server/routes/quick-answer.d.ts.map +0 -1
  489. package/dist/server/routes/quick-answer.js +0 -93
  490. package/dist/server/routes/quick-answer.js.map +0 -1
  491. package/dist/server/routes/screenshot.d.ts +0 -23
  492. package/dist/server/routes/screenshot.d.ts.map +0 -1
  493. package/dist/server/routes/screenshot.js +0 -819
  494. package/dist/server/routes/screenshot.js.map +0 -1
  495. package/dist/server/routes/search.d.ts +0 -7
  496. package/dist/server/routes/search.d.ts.map +0 -1
  497. package/dist/server/routes/search.js +0 -312
  498. package/dist/server/routes/search.js.map +0 -1
  499. package/dist/server/routes/session.d.ts +0 -16
  500. package/dist/server/routes/session.d.ts.map +0 -1
  501. package/dist/server/routes/session.js +0 -278
  502. package/dist/server/routes/session.js.map +0 -1
  503. package/dist/server/routes/stats.d.ts +0 -7
  504. package/dist/server/routes/stats.d.ts.map +0 -1
  505. package/dist/server/routes/stats.js +0 -65
  506. package/dist/server/routes/stats.js.map +0 -1
  507. package/dist/server/routes/stripe.d.ts +0 -16
  508. package/dist/server/routes/stripe.d.ts.map +0 -1
  509. package/dist/server/routes/stripe.js +0 -283
  510. package/dist/server/routes/stripe.js.map +0 -1
  511. package/dist/server/routes/users.d.ts +0 -9
  512. package/dist/server/routes/users.d.ts.map +0 -1
  513. package/dist/server/routes/users.js +0 -1211
  514. package/dist/server/routes/users.js.map +0 -1
  515. package/dist/server/routes/watch.d.ts +0 -16
  516. package/dist/server/routes/watch.d.ts.map +0 -1
  517. package/dist/server/routes/watch.js +0 -257
  518. package/dist/server/routes/watch.js.map +0 -1
  519. package/dist/server/routes/webhooks.d.ts +0 -16
  520. package/dist/server/routes/webhooks.d.ts.map +0 -1
  521. package/dist/server/routes/webhooks.js +0 -74
  522. package/dist/server/routes/webhooks.js.map +0 -1
  523. package/dist/server/routes/youtube.d.ts +0 -7
  524. package/dist/server/routes/youtube.d.ts.map +0 -1
  525. package/dist/server/routes/youtube.js +0 -93
  526. package/dist/server/routes/youtube.js.map +0 -1
  527. package/dist/server/sentry.d.ts +0 -14
  528. package/dist/server/sentry.d.ts.map +0 -1
  529. package/dist/server/sentry.js +0 -39
  530. package/dist/server/sentry.js.map +0 -1
  531. package/dist/server/types.d.ts +0 -16
  532. package/dist/server/types.d.ts.map +0 -1
  533. package/dist/server/types.js +0 -8
  534. package/dist/server/types.js.map +0 -1
  535. package/dist/server/utils/response.d.ts +0 -45
  536. package/dist/server/utils/response.d.ts.map +0 -1
  537. package/dist/server/utils/response.js +0 -70
  538. package/dist/server/utils/response.js.map +0 -1
  539. package/dist/server/utils/sse.d.ts +0 -23
  540. package/dist/server/utils/sse.d.ts.map +0 -1
  541. package/dist/server/utils/sse.js +0 -39
  542. package/dist/server/utils/sse.js.map +0 -1
  543. package/dist/types.d.ts.map +0 -1
  544. package/dist/types.js.map +0 -1
@@ -1,1130 +1,61 @@
1
1
  #!/usr/bin/env node
2
2
  /**
3
- * MCP Server for WebPeel
4
- * 7 consolidated tools (1 meta + 6 specific) with full backward compatibility.
5
- * All 20 legacy tool names still work — they're just routed to the new handlers.
3
+ * MCP Server for WebPeel — stdio transport.
4
+ * Thin wrapper: imports from the shared handler registry in ./handlers/.
5
+ * All tool logic lives in src/mcp/handlers/*.ts.
6
6
  */
7
7
  import { Server } from '@modelcontextprotocol/sdk/server/index.js';
8
8
  import { StdioServerTransport } from '@modelcontextprotocol/sdk/server/stdio.js';
9
9
  import { CallToolRequestSchema, ListToolsRequestSchema, } from '@modelcontextprotocol/sdk/types.js';
10
- import { peel, peelBatch } from '../index.js';
11
- import { normalizeActions } from '../core/actions.js';
12
10
  import { readFileSync } from 'fs';
13
11
  import { join, dirname } from 'path';
14
12
  import { fileURLToPath } from 'url';
15
- import { getSearchProvider, getBestSearchProvider } from '../core/search-provider.js';
16
- import { answerQuestion } from '../core/answer.js';
17
- import { extractInlineJson } from '../core/extract-inline.js';
18
- import { quickAnswer } from '../core/quick-answer.js';
19
- import { runAgent } from '../core/agent.js';
20
- import { parseIntent } from './smart-router.js';
13
+ import { getHandler } from './handlers/index.js';
14
+ import { toolDefinitions } from './handlers/definitions.js';
21
15
  // Read version from package.json
22
16
  let pkgVersion = '0.3.1';
23
17
  try {
24
- const __dirname = dirname(fileURLToPath(import.meta.url));
18
+ const __filename = fileURLToPath(import.meta.url);
19
+ const __dirname = dirname(__filename);
25
20
  const pkg = JSON.parse(readFileSync(join(__dirname, '..', '..', 'package.json'), 'utf-8'));
26
21
  pkgVersion = pkg.version;
27
22
  }
28
23
  catch { /* fallback */ }
29
- // ── Helpers ────────────────────────────────────────────────────────────────────
30
- function extractColorsFromContent(content) {
31
- const hexRegex = /#[0-9A-Fa-f]{6}|#[0-9A-Fa-f]{3}/g;
32
- const matches = content.match(hexRegex);
33
- return matches ? [...new Set(matches)].slice(0, 10) : [];
34
- }
35
- function extractFontsFromContent(content) {
36
- const fontRegex = /font-family:\s*([^;}"'\n]+)/gi;
37
- const fonts = [];
38
- let match;
39
- while ((match = fontRegex.exec(content)) !== null) {
40
- fonts.push(match[1].trim());
41
- }
42
- return [...new Set(fonts)].slice(0, 5);
43
- }
44
- function safeJson(value) {
45
- try {
46
- return JSON.stringify(value, null, 2);
47
- }
48
- catch {
49
- return JSON.stringify({ error: 'serialization_error', message: 'Failed to serialize result' });
50
- }
51
- }
52
- function textResponse(text) {
53
- return { content: [{ type: 'text', text }] };
54
- }
55
- function isYouTubeUrl(url) {
56
- return /(?:youtube\.com|youtu\.be)/i.test(url);
57
- }
58
- function timeout(ms, label) {
59
- return new Promise((_, reject) => setTimeout(() => reject(new Error(`${label} timed out after ${ms / 1000}s`)), ms));
60
- }
61
- // ── Tool definitions (7 public tools) ─────────────────────────────────────────
62
- const publicTools = [
63
- {
64
- name: 'webpeel',
65
- description: "Your complete web toolkit. Describe what you want in plain language. " +
66
- "Examples: 'read https://stripe.com', 'screenshot bbc.com on mobile', " +
67
- "'find best AI frameworks', 'extract prices from stripe.com/pricing', " +
68
- "'watch stripe.com/pricing for changes'",
69
- annotations: {
70
- title: 'WebPeel Smart Web Tool',
71
- readOnlyHint: true,
72
- destructiveHint: false,
73
- idempotentHint: true,
74
- openWorldHint: true,
75
- },
76
- inputSchema: {
77
- type: 'object',
78
- properties: {
79
- task: {
80
- type: 'string',
81
- description: 'Plain English description of what you want to do with the web.',
82
- },
83
- },
84
- required: ['task'],
85
- },
86
- },
87
- {
88
- name: 'webpeel_read',
89
- description: 'Read any URL and return clean markdown. Handles web pages, YouTube videos, and PDFs ' +
90
- 'automatically. Use question= for Q&A about the page, summary=true for a summary.',
91
- annotations: {
92
- title: 'Read Web Page',
93
- readOnlyHint: true,
94
- destructiveHint: false,
95
- idempotentHint: true,
96
- openWorldHint: true,
97
- },
98
- inputSchema: {
99
- type: 'object',
100
- properties: {
101
- url: { type: 'string', description: 'URL to fetch' },
102
- format: {
103
- type: 'string',
104
- enum: ['markdown', 'text', 'html'],
105
- description: 'Output format (default: markdown)',
106
- default: 'markdown',
107
- },
108
- render: {
109
- type: 'boolean',
110
- description: 'Force browser rendering for JS-heavy sites',
111
- default: false,
112
- },
113
- question: {
114
- type: 'string',
115
- description: 'Ask a question about the page content (BM25, no LLM needed)',
116
- },
117
- summary: {
118
- type: 'boolean',
119
- description: 'Return a summary instead of full content',
120
- default: false,
121
- },
122
- budget: {
123
- type: 'number',
124
- description: 'Smart token budget — distill content to N tokens',
125
- },
126
- readable: {
127
- type: 'boolean',
128
- description: 'Reader mode — extract only article content',
129
- default: false,
130
- },
131
- },
132
- required: ['url'],
133
- },
134
- },
135
- {
136
- name: 'webpeel_see',
137
- description: "See any page visually. Returns a screenshot. Use mode='design' for design analysis, " +
138
- "mode='compare' with compare_url for visual comparison.",
139
- annotations: {
140
- title: 'See Page Visually',
141
- readOnlyHint: true,
142
- destructiveHint: false,
143
- idempotentHint: true,
144
- openWorldHint: true,
145
- },
146
- inputSchema: {
147
- type: 'object',
148
- properties: {
149
- url: { type: 'string', description: 'URL to screenshot' },
150
- mode: {
151
- type: 'string',
152
- enum: ['screenshot', 'design', 'compare'],
153
- description: "Mode: 'screenshot' (default), 'design' (analysis), 'compare' (visual diff)",
154
- default: 'screenshot',
155
- },
156
- compare_url: {
157
- type: 'string',
158
- description: "Second URL to compare against (for mode='compare')",
159
- },
160
- viewport: {
161
- description: "Viewport size: 'mobile' | 'tablet' | {width, height}",
162
- oneOf: [
163
- { type: 'string', enum: ['mobile', 'tablet', 'desktop'] },
164
- {
165
- type: 'object',
166
- properties: {
167
- width: { type: 'number' },
168
- height: { type: 'number' },
169
- },
170
- required: ['width', 'height'],
171
- },
172
- ],
173
- },
174
- full_page: {
175
- type: 'boolean',
176
- description: 'Capture the full scrollable page',
177
- default: false,
178
- },
179
- },
180
- required: ['url'],
181
- },
182
- },
183
- {
184
- name: 'webpeel_find',
185
- description: 'Find anything on the web. Pass a query to search, or a url to discover all pages on ' +
186
- "that domain. Use depth='deep' for multi-source research.",
187
- annotations: {
188
- title: 'Find on the Web',
189
- readOnlyHint: true,
190
- destructiveHint: false,
191
- idempotentHint: true,
192
- openWorldHint: true,
193
- },
194
- inputSchema: {
195
- type: 'object',
196
- properties: {
197
- query: { type: 'string', description: 'Search query' },
198
- url: { type: 'string', description: 'Domain URL to map/discover all pages' },
199
- depth: {
200
- type: 'string',
201
- enum: ['quick', 'deep'],
202
- description: "Search depth: 'quick' = single search, 'deep' = multi-source research",
203
- default: 'quick',
204
- },
205
- limit: {
206
- type: 'number',
207
- description: 'Max results to return (default: 5)',
208
- default: 5,
209
- },
210
- },
211
- },
212
- },
213
- {
214
- name: 'webpeel_extract',
215
- description: "Extract structured data from any URL. Pass fields=['price','title'] for specific data, " +
216
- 'or omit for auto-detection. Returns typed JSON.',
217
- annotations: {
218
- title: 'Extract Structured Data',
219
- readOnlyHint: true,
220
- destructiveHint: false,
221
- idempotentHint: true,
222
- openWorldHint: true,
223
- },
224
- inputSchema: {
225
- type: 'object',
226
- properties: {
227
- url: { type: 'string', description: 'URL to extract from' },
228
- schema: { type: 'object', description: 'JSON schema describing desired output structure' },
229
- fields: {
230
- type: 'array',
231
- items: { type: 'string' },
232
- description: "Specific fields to extract, e.g. ['price', 'title', 'description']",
233
- },
234
- format: {
235
- type: 'string',
236
- enum: ['json', 'markdown'],
237
- description: 'Output format (default: json)',
238
- default: 'json',
239
- },
240
- },
241
- required: ['url'],
242
- },
243
- },
244
- {
245
- name: 'webpeel_monitor',
246
- description: 'Watch a URL for changes. Returns diff on subsequent calls. ' +
247
- 'Add webhook= for persistent monitoring with notifications.',
248
- annotations: {
249
- title: 'Monitor URL for Changes',
250
- readOnlyHint: false,
251
- destructiveHint: false,
252
- idempotentHint: false,
253
- openWorldHint: true,
254
- },
255
- inputSchema: {
256
- type: 'object',
257
- properties: {
258
- url: { type: 'string', description: 'URL to monitor' },
259
- webhook: {
260
- type: 'string',
261
- description: 'Webhook URL to notify when content changes',
262
- },
263
- interval: {
264
- type: 'string',
265
- description: "Check interval, e.g. '1h', '30m', '1d'",
266
- default: '1h',
267
- },
268
- selector: {
269
- type: 'string',
270
- description: 'CSS selector to monitor a specific part of the page',
271
- },
272
- },
273
- required: ['url'],
274
- },
275
- },
276
- {
277
- name: 'webpeel_act',
278
- description: 'Interact with a web page. Click buttons, fill forms, navigate. ' +
279
- 'Returns screenshot + extracted content after actions complete.',
280
- annotations: {
281
- title: 'Act on Web Page',
282
- readOnlyHint: false,
283
- destructiveHint: false,
284
- idempotentHint: false,
285
- openWorldHint: true,
286
- },
287
- inputSchema: {
288
- type: 'object',
289
- properties: {
290
- url: { type: 'string', description: 'URL to interact with' },
291
- actions: {
292
- type: 'array',
293
- description: 'Actions to perform, e.g. [{type:"click",selector:".btn"}, {type:"type",selector:"#q",value:"hello"}]',
294
- items: {
295
- type: 'object',
296
- properties: {
297
- type: {
298
- type: 'string',
299
- enum: ['click', 'type', 'fill', 'scroll', 'wait', 'press', 'hover', 'select'],
300
- },
301
- selector: { type: 'string' },
302
- value: { type: 'string' },
303
- key: { type: 'string' },
304
- milliseconds: { type: 'number' },
305
- },
306
- required: ['type'],
307
- },
308
- },
309
- extract_after: {
310
- type: 'boolean',
311
- description: 'Extract content after actions complete',
312
- default: true,
313
- },
314
- screenshot_after: {
315
- type: 'boolean',
316
- description: 'Take screenshot after actions complete',
317
- default: false,
318
- },
319
- },
320
- required: ['url', 'actions'],
321
- },
322
- },
323
- ];
324
- // ── Server setup ───────────────────────────────────────────────────────────────
24
+ // ── Server setup ───────────────────────────────────────────────────────────
325
25
  const server = new Server({ name: 'webpeel', version: pkgVersion }, { capabilities: { tools: {} } });
326
- // ── Handlers ───────────────────────────────────────────────────────────────────
327
- /**
328
- * webpeel_read: fetch a URL as clean markdown.
329
- * Auto-detects YouTube URLs and extracts transcripts.
330
- */
331
- async function handleRead(args) {
332
- const url = args['url'];
333
- if (!url || typeof url !== 'string')
334
- throw new Error('url is required');
335
- if (url.length > 2048)
336
- throw new Error('URL too long (max 2048 characters)');
337
- const format = args['format'] || 'markdown';
338
- const render = args['render'] || false;
339
- const question = args['question'];
340
- const summary = args['summary'] || false;
341
- const budgetArg = args['budget'];
342
- const readable = args['readable'] || false;
343
- // YouTube auto-detection
344
- if (isYouTubeUrl(url)) {
345
- const { getYouTubeTranscript } = await import('../core/youtube.js');
346
- const language = args['language'] || 'en';
347
- const transcript = await Promise.race([
348
- getYouTubeTranscript(url, { language }),
349
- timeout(60000, 'YouTube transcript'),
350
- ]);
351
- return textResponse(safeJson(transcript));
352
- }
353
- // Build summary prompt if requested
354
- const extractOpts = summary
355
- ? { prompt: 'Summarize this webpage in 2-3 concise sentences.' }
356
- : undefined;
357
- const options = {
358
- render,
359
- format: format,
360
- question,
361
- budget: budgetArg ?? 4000,
362
- readable,
363
- ...(extractOpts ? { extract: extractOpts } : {}),
364
- };
365
- const result = await Promise.race([
366
- peel(url, options),
367
- timeout(60000, 'MCP read'),
368
- ]);
369
- const out = {
370
- url: result.url || url,
371
- title: result.title || '',
372
- tokens: result.tokens || 0,
373
- content: result.content,
374
- };
375
- if (result.metadata)
376
- out['metadata'] = result.metadata;
377
- if (result.quickAnswer)
378
- out['quickAnswer'] = result.quickAnswer;
379
- if (result.extracted)
380
- out['extracted'] = result.extracted;
381
- if (result.images)
382
- out['images'] = result.images;
383
- return textResponse(safeJson(out));
384
- }
385
- /**
386
- * webpeel_see: take a screenshot, optionally with design analysis or comparison.
387
- */
388
- async function handleSee(args) {
389
- const url = args['url'];
390
- if (!url || typeof url !== 'string')
391
- throw new Error('url is required');
392
- if (url.length > 2048)
393
- throw new Error('URL too long (max 2048 characters)');
394
- const mode = args['mode'] || 'screenshot';
395
- const compareUrl = args['compare_url'];
396
- const fullPage = args['full_page'] || false;
397
- const viewportArg = args['viewport'];
398
- // Resolve viewport
399
- let width = 1280;
400
- let height = 720;
401
- if (viewportArg && typeof viewportArg === 'object' && !Array.isArray(viewportArg)) {
402
- const vp = viewportArg;
403
- width = vp.width ?? 1280;
404
- height = vp.height ?? 720;
405
- }
406
- else if (viewportArg === 'mobile') {
407
- width = 390;
408
- height = 844;
409
- }
410
- else if (viewportArg === 'tablet') {
411
- width = 768;
412
- height = 1024;
413
- }
414
- // Params from smart router viewport object
415
- if (args['viewport'] && typeof args['viewport'] === 'object') {
416
- const vp = args['viewport'];
417
- if (vp.width)
418
- width = vp.width;
419
- if (vp.height)
420
- height = vp.height;
421
- }
422
- if (mode === 'design') {
423
- const { browserDesignAnalysis } = await import('../core/fetcher.js');
424
- const result = await Promise.race([
425
- browserDesignAnalysis(url, { width, height }),
426
- timeout(90000, 'Design analysis'),
427
- ]);
428
- return textResponse(safeJson({ url: result.finalUrl, mode: 'design', analysis: result.analysis }));
429
- }
430
- if (mode === 'compare' && compareUrl) {
431
- const { browserDiff } = await import('../core/fetcher.js');
432
- const diff = await Promise.race([
433
- browserDiff(url, compareUrl, { width, height, fullPage }),
434
- timeout(90000, 'Design compare'),
435
- ]);
436
- return textResponse(safeJson({
437
- url,
438
- compare_url: compareUrl,
439
- mode: 'compare',
440
- diffPixels: diff.diffPixels,
441
- totalPixels: diff.totalPixels,
442
- diffPercent: diff.diffPercent,
443
- screenshot: diff.diffBuffer.toString('base64'),
444
- }));
445
- }
446
- // Default: screenshot
447
- const { takeScreenshot } = await import('../core/screenshot.js');
448
- const result = await Promise.race([
449
- takeScreenshot(url, { fullPage, width, height, format: 'png' }),
450
- timeout(60000, 'Screenshot'),
451
- ]);
452
- return textResponse(safeJson({ url: result.url, mode: 'screenshot', screenshot: result.screenshot, format: result.format }));
453
- }
454
- /**
455
- * webpeel_find: search the web, discover domain URLs, or do deep research.
456
- */
457
- async function handleFind(args) {
458
- const query = args['query'];
459
- const url = args['url'];
460
- const depth = args['depth'] || 'quick';
461
- const limit = Math.min(Math.max(args['limit'] ?? 5, 1), 20);
462
- // URL-based: map/discover pages on a domain
463
- if (url && !query) {
464
- const { mapDomain } = await import('../core/map.js');
465
- const results = await Promise.race([
466
- mapDomain(url, { maxUrls: limit * 100 }),
467
- timeout(600000, 'Map domain'),
468
- ]);
469
- return textResponse(safeJson(results));
470
- }
471
- if (!query)
472
- throw new Error('Either query or url is required');
473
- // Question-mode: if the query looks like a natural language question and depth
474
- // isn't forced to 'deep', use the LLM-free BM25 Q&A path (search → fetch → BM25).
475
- // This is the /v1/ask feature — no API key required, deterministic.
476
- const isQuestion = /\?$/.test(query.trim()) ||
477
- /^(what|how|when|where|why|who|which|can|does|is|are|do|did|will|would|could|should)\b/i.test(query.trim());
478
- if (isQuestion && depth !== 'deep') {
479
- const numSources = Math.min(limit, 5);
480
- const { provider, apiKey } = getBestSearchProvider();
481
- let searchResults;
482
- try {
483
- searchResults = (await Promise.race([
484
- provider.searchWeb(query, { count: numSources, apiKey }),
485
- timeout(30000, 'Ask search'),
486
- ]));
487
- }
488
- catch {
489
- searchResults = [];
490
- }
491
- if (searchResults.length === 0) {
492
- return textResponse(safeJson({ question: query, answer: null, confidence: 0, sources: [], method: 'bm25' }));
493
- }
494
- const fetched = await Promise.allSettled(searchResults.slice(0, numSources).map((r) => peel(r.url, { budget: 3000, format: 'markdown', timeout: 12000 }).then((result) => ({ result, searchResult: r }))));
495
- const answers = fetched
496
- .filter((f) => f.status === 'fulfilled')
497
- .map((f) => {
498
- const { result, searchResult } = f.value;
499
- const qa = quickAnswer({ question: query, content: result.content || '', url: result.url || searchResult.url, maxPassages: 2 });
500
- return {
501
- answer: qa.answer,
502
- confidence: qa.confidence,
503
- source: { url: result.url || searchResult.url, title: result.title || searchResult.title, snippet: searchResult.snippet },
504
- };
505
- })
506
- .sort((a, b) => b.confidence - a.confidence);
507
- const best = answers[0];
508
- return textResponse(safeJson({
509
- question: query,
510
- answer: best?.answer || null,
511
- confidence: best?.confidence || 0,
512
- sources: answers.map((a) => ({ ...a.source, confidence: a.confidence })),
513
- method: 'bm25',
514
- }));
515
- }
516
- // Deep research mode
517
- if (depth === 'deep') {
518
- const { provider, apiKey } = getBestSearchProvider();
519
- const searchResults = await Promise.race([
520
- provider.searchWeb(query, { count: limit, apiKey }),
521
- timeout(30000, 'Search'),
522
- ]);
523
- const results = Array.isArray(searchResults)
524
- ? searchResults
525
- : searchResults.results ?? [];
526
- const topN = results.slice(0, limit);
527
- if (topN.length === 0) {
528
- return textResponse(safeJson({ query, sources: [], content: '', totalTokens: 0 }));
529
- }
530
- const urls = topN.map((r) => r.url).filter(Boolean);
531
- const pages = await Promise.race([
532
- peelBatch(urls, { concurrency: 5, format: 'markdown' }),
533
- timeout(120000, 'Batch fetch'),
534
- ]);
535
- const sources = [];
536
- const contentParts = [];
537
- let totalTokens = 0;
538
- for (let i = 0; i < pages.length; i++) {
539
- const page = pages[i];
540
- const sr = topN[i];
541
- const pageUrl = urls[i];
542
- const title = page.title || sr.title || pageUrl;
543
- const relevanceScore = Math.round((1 - i / Math.max(pages.length, 1)) * 100) / 100;
544
- sources.push({ url: pageUrl, title, relevanceScore });
545
- if (page.content) {
546
- contentParts.push(`## Source ${i + 1}: ${title}\n**URL:** ${pageUrl}\n\n${page.content}\n\n---\n`);
547
- totalTokens += page.tokens || 0;
548
- }
549
- }
550
- return textResponse(safeJson({ query, sources, content: contentParts.join('\n'), totalTokens }));
551
- }
552
- // Quick search (default)
553
- const validProviders = ['duckduckgo', 'brave', 'stealth', 'google'];
554
- const providerId = (args['provider'] && validProviders.includes(args['provider']))
555
- ? args['provider']
556
- : 'duckduckgo';
557
- const searchProvider = getSearchProvider(providerId);
558
- const results = await Promise.race([
559
- searchProvider.searchWeb(query, { count: limit }),
560
- timeout(30000, 'Search'),
561
- ]);
562
- return textResponse(safeJson(results));
563
- }
564
- /**
565
- * webpeel_extract: extract structured data from a URL.
566
- * Supports auto-detection, field lists, schema, and brand presets.
567
- */
568
- async function handleExtract(args) {
569
- const url = args['url'];
570
- if (!url || typeof url !== 'string')
571
- throw new Error('url is required');
572
- if (url.length > 2048)
573
- throw new Error('URL too long (max 2048 characters)');
574
- const schema = args['schema'];
575
- const fields = args['fields'];
576
- const render = args['render'] || false;
577
- // Brand preset: fields=['name','logo','colors','fonts','socials'] or brand flag
578
- const isBrandPreset = args['_brand'] ||
579
- (Array.isArray(fields) &&
580
- ['name', 'logo', 'colors', 'fonts', 'socials'].every((f) => fields.includes(f)));
581
- if (isBrandPreset) {
582
- const options = {
583
- render,
584
- extract: {
585
- selectors: {
586
- primaryColor: 'meta[name="theme-color"]',
587
- title: 'title',
588
- logo: 'img[class*="logo"], img[alt*="logo"]',
589
- },
590
- },
591
- };
592
- const result = await Promise.race([
593
- peel(url, options),
594
- timeout(60000, 'Brand extraction'),
595
- ]);
596
- return textResponse(safeJson({
597
- url: result.url,
598
- title: result.title,
599
- extracted: result.extracted,
600
- metadata: result.metadata,
601
- colors: extractColorsFromContent(result.content || ''),
602
- fonts: extractFontsFromContent(result.content || ''),
603
- }));
604
- }
605
- // Auto-extract when no schema provided
606
- if (!schema && (!fields || fields.length === 0)) {
607
- const htmlResult = await Promise.race([
608
- peel(url, { format: 'html', render }),
609
- timeout(60000, 'Auto-extract fetch'),
610
- ]);
611
- const { autoExtract } = await import('../core/auto-extract.js');
612
- const extracted = autoExtract(htmlResult.content || '', url);
613
- return textResponse(safeJson({ url, pageType: extracted.type, structured: extracted }));
614
- }
615
- // Field-based extraction (CSS selectors from field names)
616
- if (fields && fields.length > 0 && !schema) {
617
- const selectors = {};
618
- for (const field of fields) {
619
- // Map common field names to CSS selectors
620
- const fieldSelectorMap = {
621
- price: '[class*="price"], [data-price]',
622
- title: 'h1, title',
623
- description: '[class*="description"], [class*="summary"]',
624
- image: 'img[class*="main"], img[class*="hero"]',
625
- name: 'h1, [class*="name"]',
626
- logo: 'img[class*="logo"], img[alt*="logo"]',
627
- colors: 'meta[name="theme-color"]',
628
- fonts: 'link[rel="stylesheet"]',
629
- socials: 'a[href*="twitter.com"], a[href*="linkedin.com"], a[href*="github.com"]',
630
- };
631
- selectors[field] = fieldSelectorMap[field] || `[class*="${field}"], [id*="${field}"]`;
632
- }
633
- const options = { render, extract: { selectors } };
634
- const result = await Promise.race([
635
- peel(url, options),
636
- timeout(60000, 'Field extraction'),
637
- ]);
638
- return textResponse(safeJson(result));
639
- }
640
- // Schema-based extraction
641
- const options = {
642
- render,
643
- extract: { schema },
644
- };
645
- const result = await Promise.race([
646
- peel(url, options),
647
- timeout(60000, 'Schema extraction'),
648
- ]);
649
- return textResponse(safeJson(result));
650
- }
651
- /**
652
- * webpeel_monitor: watch a URL for changes, with optional webhook.
653
- */
654
- async function handleMonitor(args) {
655
- const url = args['url'];
656
- if (!url || typeof url !== 'string')
657
- throw new Error('url is required');
658
- if (url.length > 2048)
659
- throw new Error('URL too long (max 2048 characters)');
660
- const webhook = args['webhook'];
661
- const selector = args['selector'];
662
- const render = args['render'] || false;
663
- if (webhook) {
664
- // Webhook-based persistent monitoring requires hosted API
665
- return textResponse(safeJson({
666
- message: 'Persistent webhook monitoring requires the hosted API (api.webpeel.dev). ' +
667
- 'Use webpeel_monitor without webhook= for one-time change detection.',
668
- url,
669
- webhook,
670
- }));
671
- }
672
- // One-time change snapshot (change_track logic)
673
- const options = { render: render || false, ...(selector ? { selector } : {}) };
674
- const result = await Promise.race([
675
- peel(url, options),
676
- timeout(60000, 'Monitor'),
677
- ]);
678
- return textResponse(safeJson({
679
- url: result.url,
680
- title: result.title,
681
- fingerprint: result.fingerprint,
682
- tokens: result.tokens,
683
- contentType: result.contentType,
684
- lastChecked: new Date().toISOString(),
685
- }));
686
- }
687
- /**
688
- * webpeel_act: perform browser actions on a page, then optionally extract content.
689
- */
690
- async function handleAct(args) {
691
- const url = args.url;
692
- const actions = args.actions;
693
- const extract = args.extract !== false; // default true
694
- const screenshot = Boolean(args.screenshot);
695
- if (!url)
696
- return textResponse(safeJson({ error: 'url is required' }));
697
- if (!actions?.length)
698
- return textResponse(safeJson({ error: 'actions array is required' }));
699
- // Reuse the full peel() pipeline — peel is already statically imported at module top
700
- const result = await peel(url, {
701
- render: true, // actions always require browser
702
- actions,
703
- screenshot,
704
- format: 'markdown',
705
- budget: 4000,
706
- timeout: 60000,
707
- });
708
- return textResponse(safeJson({
709
- url: result.url,
710
- title: result.title,
711
- content: extract ? result.content : undefined,
712
- screenshot: result.screenshot,
713
- method: result.method,
714
- elapsed: result.elapsed,
715
- }));
716
- }
717
- // ── Full webpeel_fetch handler (backward compat + power users) ─────────────────
718
- async function handleFetch(args) {
719
- const { url, render, stealth, wait, format, screenshot: ssFlag, screenshotFullPage, selector, exclude, includeTags, excludeTags, images, location, headers, actions: rawActions, autoScroll: autoScrollParam, maxTokens, extract, inlineExtract, llmProvider, llmApiKey, llmModel, question, budget: budgetArg, readable, } = args;
720
- if (!url || typeof url !== 'string')
721
- throw new Error('Invalid URL parameter');
722
- if (url.length > 2048)
723
- throw new Error('URL too long (max 2048 characters)');
724
- const normalizedActions = rawActions ? normalizeActions(rawActions) : undefined;
725
- const hasActions = normalizedActions && normalizedActions.length > 0;
726
- const options = {
727
- render: render || hasActions || !!autoScrollParam || false,
728
- stealth: stealth || false,
729
- wait: wait || 0,
730
- format: format || 'markdown',
731
- screenshot: ssFlag || false,
732
- screenshotFullPage: screenshotFullPage || false,
733
- selector,
734
- exclude,
735
- includeTags,
736
- excludeTags,
737
- images,
738
- location: location ? { country: location } : undefined,
739
- headers,
740
- actions: normalizedActions,
741
- autoScroll: autoScrollParam,
742
- maxTokens,
743
- extract,
744
- readable: readable || false,
745
- lite: args['lite'] || false,
746
- question,
747
- budget: args['lite'] ? undefined : (budgetArg ?? (maxTokens === undefined ? 4000 : undefined)),
748
- };
749
- const peeled = await Promise.race([
750
- peel(url, options),
751
- timeout(60000, 'MCP operation'),
752
- ]);
753
- // For inline LLM extraction we need to attach extra fields; use a mutable wrapper
754
- const result = peeled;
755
- // Inline LLM extraction
756
- if (inlineExtract && (inlineExtract.schema || inlineExtract.prompt) && llmApiKey && llmProvider) {
757
- const validProviders = ['openai', 'anthropic', 'google'];
758
- if (validProviders.includes(llmProvider)) {
759
- const extractResult = await extractInlineJson(result.content, {
760
- schema: inlineExtract.schema,
761
- prompt: inlineExtract.prompt,
762
- llmProvider: llmProvider,
763
- llmApiKey,
764
- llmModel,
765
- });
766
- result.json = extractResult.data;
767
- result.extractTokensUsed = extractResult.tokensUsed;
768
- }
769
- }
770
- const out = {
771
- url: result.url || url,
772
- title: result.title || result.metadata?.title || '',
773
- tokens: result.tokens || 0,
774
- content: result.content,
775
- };
776
- if (result.metadata)
777
- out['metadata'] = result.metadata;
778
- if (result.domainData)
779
- out['domainData'] = result.domainData;
780
- if (result.readability)
781
- out['readability'] = { readingTime: result.readability.readingTime, wordCount: result.readability.wordCount };
782
- if (result.quickAnswer)
783
- out['quickAnswer'] = result.quickAnswer;
784
- if (result.json)
785
- out['json'] = result.json;
786
- if (result.extracted)
787
- out['extracted'] = result.extracted;
788
- if (result.images?.length)
789
- out['images'] = result.images;
790
- if (result.screenshot)
791
- out['screenshot'] = result.screenshot;
792
- if (result.fingerprint)
793
- out['fingerprint'] = result.fingerprint;
794
- if (result.extractTokensUsed)
795
- out['extractTokensUsed'] = result.extractTokensUsed;
796
- if (result.quality !== undefined)
797
- out['quality'] = result.quality;
798
- if (result.timing)
799
- out['timing'] = result.timing;
800
- if (result.method)
801
- out['method'] = result.method;
802
- return textResponse(safeJson(out));
803
- }
804
- // ── ListToolsRequest — only the 7 public tools ─────────────────────────────────
805
- server.setRequestHandler(ListToolsRequestSchema, async () => ({ tools: publicTools }));
806
- // ── CallToolRequest — route to handlers ────────────────────────────────────────
26
+ // ── tools/list — return the 7 public tool definitions ─────────────────────
27
+ server.setRequestHandler(ListToolsRequestSchema, async () => ({
28
+ tools: toolDefinitions,
29
+ }));
30
+ // ── tools/call — dispatch to shared handler registry ──────────────────────
807
31
  server.setRequestHandler(CallToolRequestSchema, async (request) => {
808
32
  const { name, arguments: rawArgs } = request.params;
809
33
  const args = (rawArgs || {});
810
34
  try {
811
- // ── Meta tool ────────────────────────────────────────────────────────────
812
- if (name === 'webpeel') {
813
- const task = args['task'];
814
- if (!task || typeof task !== 'string')
815
- throw new Error('task is required');
816
- const parsed = parseIntent(task);
817
- const routedArgs = { ...parsed.params };
818
- if (parsed.url)
819
- routedArgs['url'] = parsed.url;
820
- switch (parsed.intent) {
821
- case 'read':
822
- return handleRead(routedArgs);
823
- case 'see':
824
- return handleSee(routedArgs);
825
- case 'find':
826
- if (parsed.query)
827
- routedArgs['query'] = parsed.query;
828
- return handleFind(routedArgs);
829
- case 'extract':
830
- return handleExtract(routedArgs);
831
- case 'monitor':
832
- return handleMonitor(routedArgs);
833
- case 'act':
834
- return handleAct(routedArgs);
835
- }
836
- }
837
- // ── New public tools ─────────────────────────────────────────────────────
838
- if (name === 'webpeel_read')
839
- return handleRead(args);
840
- if (name === 'webpeel_see')
841
- return handleSee(args);
842
- if (name === 'webpeel_find')
843
- return handleFind(args);
844
- if (name === 'webpeel_extract')
845
- return handleExtract(args);
846
- if (name === 'webpeel_monitor')
847
- return handleMonitor(args);
848
- if (name === 'webpeel_act')
849
- return handleAct(args);
850
- // ── Backward compatibility aliases ───────────────────────────────────────
851
- // webpeel_fetch → webpeel_read (full-featured)
852
- if (name === 'webpeel_fetch')
853
- return handleFetch(args);
854
- // webpeel_youtube → webpeel_read (YouTube auto-detected)
855
- if (name === 'webpeel_youtube') {
856
- const { getYouTubeTranscript } = await import('../core/youtube.js');
857
- const url = args['url'];
858
- if (!url || typeof url !== 'string')
859
- throw new Error('url is required');
860
- const language = args['language'] || 'en';
861
- const transcript = await Promise.race([
862
- getYouTubeTranscript(url, { language }),
863
- timeout(60000, 'YouTube transcript'),
864
- ]);
865
- return textResponse(safeJson(transcript));
866
- }
867
- // webpeel_summarize → webpeel_read(summary=true)
868
- if (name === 'webpeel_summarize') {
869
- const { url, llmApiKey, prompt, llmModel, llmBaseUrl, render } = args;
870
- if (!url)
871
- throw new Error('url is required');
872
- if (!llmApiKey)
873
- throw new Error('llmApiKey is required');
874
- const options = {
875
- render: render || false,
876
- extract: {
877
- prompt: prompt || 'Summarize this webpage in 2-3 sentences.',
878
- llmApiKey,
879
- llmModel: llmModel || 'gpt-4o-mini',
880
- llmBaseUrl: llmBaseUrl || 'https://api.openai.com/v1',
881
- },
882
- };
883
- const result = await Promise.race([peel(url, options), timeout(60000, 'Summarize')]);
884
- return textResponse(safeJson({ url: result.url, title: result.title, summary: result.extracted }));
885
- }
886
- // webpeel_answer → webpeel_read(question=...)
887
- if (name === 'webpeel_answer') {
888
- const { question: q, searchProvider: sp, searchApiKey: sak, llmProvider: lp, llmApiKey: lak, llmModel: lm, maxSources: ms } = args;
889
- if (!q)
890
- throw new Error('question is required');
891
- if (!lp)
892
- throw new Error('llmProvider is required');
893
- if (!lak)
894
- throw new Error('llmApiKey is required');
895
- const validLlm = ['openai', 'anthropic', 'google'];
896
- if (!validLlm.includes(lp))
897
- throw new Error('Invalid llmProvider');
898
- const validSp = ['duckduckgo', 'brave', 'stealth', 'google'];
899
- const resolvedSp = (sp && validSp.includes(sp) ? sp : 'duckduckgo');
900
- const result = await Promise.race([
901
- answerQuestion({
902
- question: q, searchProvider: resolvedSp, searchApiKey: sak,
903
- llmProvider: lp, llmApiKey: lak, llmModel: lm,
904
- maxSources: Math.min(Math.max(ms ?? 5, 1), 10), stream: false,
905
- }),
906
- timeout(180000, 'Answer'),
907
- ]);
908
- return textResponse(safeJson(result));
909
- }
910
- // webpeel_quick_answer → webpeel_read(question=...)
911
- if (name === 'webpeel_quick_answer') {
912
- const { url, question: q, maxPassages: mpArg, render } = args;
913
- if (!url)
914
- throw new Error('url is required');
915
- if (!q)
916
- throw new Error('question is required');
917
- const maxPassages = typeof mpArg === 'number' ? Math.min(Math.max(mpArg, 1), 10) : 3;
918
- const peelResult = await Promise.race([
919
- peel(url, { render: render || false, format: 'markdown', budget: 8000 }),
920
- timeout(60000, 'Quick answer fetch'),
921
- ]);
922
- const qa = quickAnswer({ question: q, content: peelResult.content || '', url: peelResult.url || url, maxPassages });
923
- return textResponse(safeJson({
924
- url: peelResult.url || url, title: peelResult.title,
925
- question: qa.question, answer: qa.answer, confidence: qa.confidence,
926
- passages: qa.passages, method: qa.method,
927
- }));
928
- }
929
- // webpeel_screenshot → webpeel_see
930
- if (name === 'webpeel_screenshot') {
931
- const { takeScreenshot } = await import('../core/screenshot.js');
932
- const { url, fullPage, width, height, format, quality, waitFor, stealth, actions } = args;
933
- if (!url)
934
- throw new Error('url is required');
935
- const result = await Promise.race([
936
- takeScreenshot(url, { fullPage: fullPage || false, width, height, format: format || 'png', quality, waitFor, stealth: stealth || false, actions }),
937
- timeout(60000, 'Screenshot'),
938
- ]);
939
- return textResponse(safeJson({ url: result.url, format: result.format, contentType: result.contentType, screenshot: result.screenshot }));
940
- }
941
- // webpeel_search → webpeel_find(quick)
942
- if (name === 'webpeel_search') {
943
- return handleFind({ ...args, depth: 'quick' });
944
- }
945
- // webpeel_research → webpeel_find(deep)
946
- if (name === 'webpeel_research') {
947
- const { query, maxSources, maxDepth, llmApiKey, llmModel, llmBaseUrl, outputFormat, timeout: resTimeout } = args;
948
- const { research } = await import('../core/research.js');
949
- const result = await research({
950
- query, maxSources: maxSources ?? 5, maxDepth: maxDepth ?? 1,
951
- apiKey: llmApiKey, model: llmModel, baseUrl: llmBaseUrl,
952
- outputFormat: outputFormat ?? 'report', timeout: resTimeout ?? 60000,
953
- });
954
- return textResponse(safeJson({
955
- report: result.report, sources: result.sources,
956
- totalSourcesFound: result.totalSourcesFound, sourcesConsulted: result.sourcesConsulted,
957
- elapsed: result.elapsed, tokensUsed: result.tokensUsed, cost: result.cost,
958
- }));
959
- }
960
- // webpeel_deep_fetch → webpeel_find(depth='deep')
961
- if (name === 'webpeel_deep_fetch') {
962
- const { query, count: countArg, format: formatArg } = args;
963
- if (!query)
964
- throw new Error('query is required');
965
- return handleFind({ query, depth: 'deep', limit: Math.min(Math.max(countArg ?? 5, 1), 10), format: formatArg });
966
- }
967
- // webpeel_map → webpeel_find(url=...)
968
- if (name === 'webpeel_map') {
969
- const { url, maxUrls, includePatterns, excludePatterns } = args;
970
- if (!url)
971
- throw new Error('url is required');
972
- const { mapDomain } = await import('../core/map.js');
973
- const results = await Promise.race([
974
- mapDomain(url, { maxUrls, includePatterns, excludePatterns }),
975
- timeout(600000, 'Map'),
976
- ]);
977
- return textResponse(safeJson(results));
978
- }
979
- // webpeel_brand → webpeel_extract(fields=['name','logo','colors','fonts','socials'])
980
- if (name === 'webpeel_brand') {
981
- return handleExtract({ ...args, _brand: true });
982
- }
983
- // webpeel_auto_extract → webpeel_extract (auto mode)
984
- if (name === 'webpeel_auto_extract') {
985
- return handleExtract(args);
986
- }
987
- // webpeel_extract (legacy — same as new)
988
- if (name === 'webpeel_extract') {
989
- return handleExtract(args);
990
- }
991
- // webpeel_change_track → webpeel_monitor
992
- if (name === 'webpeel_change_track')
993
- return handleMonitor(args);
994
- // webpeel_watch → webpeel_monitor
995
- if (name === 'webpeel_watch') {
996
- const action = args['action'];
997
- if (!action || action === 'list' || action === 'check' || action === 'delete') {
998
- return textResponse(safeJson({
999
- message: 'URL watching requires the hosted API (api.webpeel.dev). Use webpeel_monitor for one-time change detection.',
1000
- }));
1001
- }
1002
- if (action === 'create') {
1003
- return handleMonitor({ url: args['url'], webhook: args['webhookUrl'] });
1004
- }
1005
- return textResponse(safeJson({ message: 'URL watching requires the hosted API (api.webpeel.dev).' }));
1006
- }
1007
- // webpeel_batch — developer tool, keep as-is
1008
- if (name === 'webpeel_batch') {
1009
- const { urls, concurrency, render, format, selector } = args;
1010
- if (!urls || !Array.isArray(urls))
1011
- throw new Error('urls must be an array');
1012
- if (urls.length === 0)
1013
- throw new Error('urls cannot be empty');
1014
- if (urls.length > 50)
1015
- throw new Error('Too many URLs (max 50)');
1016
- const options = {
1017
- concurrency: concurrency || 3, render: render || false,
1018
- format: format || 'markdown', selector,
1019
- };
1020
- const results = await Promise.race([
1021
- peelBatch(urls, options),
1022
- timeout(300000, 'Batch'),
1023
- ]);
1024
- return textResponse(safeJson(results));
1025
- }
1026
- // webpeel_crawl — developer tool, keep as-is
1027
- if (name === 'webpeel_crawl') {
1028
- const { crawl } = await import('../core/crawler.js');
1029
- const { url, maxPages, maxDepth, allowedDomains, excludePatterns, respectRobotsTxt, rateLimitMs, sitemapFirst, render, stealth } = args;
1030
- if (!url)
1031
- throw new Error('url is required');
1032
- const results = await Promise.race([
1033
- crawl(url, { maxPages, maxDepth, allowedDomains, excludePatterns, respectRobotsTxt, rateLimitMs, sitemapFirst, render, stealth }),
1034
- timeout(600000, 'Crawl'),
1035
- ]);
1036
- return textResponse(safeJson(results));
1037
- }
1038
- // webpeel_hotels — deprecated
1039
- if (name === 'webpeel_hotels') {
1040
- return textResponse(safeJson({ message: 'This tool has been deprecated.' }));
1041
- }
1042
- // agent — deprecated
1043
- if (name === 'agent') {
1044
- // Keep backward compat: if llmApiKey provided, run agent
1045
- const llmApiKey = args['llmApiKey'];
1046
- if (llmApiKey) {
1047
- const promptArg = args['prompt'];
1048
- if (!promptArg)
1049
- throw new Error('prompt is required');
1050
- const result = await runAgent({
1051
- prompt: promptArg,
1052
- llmApiKey,
1053
- urls: args['urls'],
1054
- maxSources: args['maxResults'] || undefined,
1055
- });
1056
- return textResponse(safeJson(result));
1057
- }
1058
- // LLM-free mode
1059
- const urlsArg = args['urls'] || [];
1060
- const searchArg = args['search'];
1061
- if (urlsArg.length === 0 && !searchArg) {
1062
- return textResponse(safeJson({
1063
- message: 'This tool has been deprecated. Use the webpeel tool instead.',
1064
- }));
1065
- }
1066
- const promptArg = args['prompt'];
1067
- const schema = args['schema'];
1068
- const budgetArg = args['budget'] || 4000;
1069
- const maxResults = Math.min(args['maxResults'] || 5, 20);
1070
- const targetUrls = [...urlsArg];
1071
- if (searchArg) {
1072
- try {
1073
- const { provider, apiKey } = getBestSearchProvider();
1074
- const searchResults = await provider.searchWeb(searchArg, { count: Math.max(maxResults, 5), apiKey });
1075
- for (const r of searchResults) {
1076
- if (!targetUrls.includes(r.url))
1077
- targetUrls.push(r.url);
1078
- }
1079
- }
1080
- catch { /* continue with provided URLs */ }
1081
- }
1082
- const urlsToFetch = targetUrls.slice(0, maxResults);
1083
- const agentResults = [];
1084
- await Promise.all(urlsToFetch.map(async (u) => {
1085
- try {
1086
- const page = await peel(u, { budget: budgetArg, format: 'markdown' });
1087
- const content = page.content || '';
1088
- const title = page.title || u;
1089
- let extracted = null;
1090
- let confidence = 0;
1091
- if (schema && Object.keys(schema).length > 0) {
1092
- extracted = {};
1093
- let total = 0;
1094
- for (const [field] of Object.entries(schema)) {
1095
- const q = promptArg ? `${promptArg} — specifically: what is the ${field}?` : `What is the ${field}?`;
1096
- const qa = quickAnswer({ question: q, content, maxPassages: 1, url: u });
1097
- extracted[field] = qa.answer || '';
1098
- total += qa.confidence;
1099
- }
1100
- if ('source' in schema)
1101
- extracted['source'] = u;
1102
- confidence = Object.keys(schema).length > 0 ? total / Object.keys(schema).length : 0;
1103
- }
1104
- else if (promptArg) {
1105
- const qa = quickAnswer({ question: promptArg, content, maxPassages: 3, url: u });
1106
- confidence = qa.confidence;
1107
- }
1108
- agentResults.push({ url: u, title, extracted, content: content.slice(0, 500) + (content.length > 500 ? '…' : ''), confidence });
1109
- }
1110
- catch { /* skip failed URLs */ }
1111
- }));
1112
- return textResponse(safeJson({
1113
- success: true,
1114
- data: { results: agentResults, totalSources: agentResults.length },
1115
- }));
1116
- }
1117
- throw new Error(`Unknown tool: ${name}`);
35
+ const handler = getHandler(name);
36
+ if (!handler)
37
+ throw new Error(`Unknown tool: ${name}`);
38
+ // Standalone has no accountId / pool context
39
+ // eslint-disable-next-line @typescript-eslint/no-explicit-any
40
+ return (await handler(args));
1118
41
  }
1119
42
  catch (error) {
1120
43
  const err = error;
1121
44
  return {
1122
- content: [{ type: 'text', text: safeJson({ error: err.name || 'Error', message: err.message || 'Unknown error' }) }],
45
+ content: [
46
+ {
47
+ type: 'text',
48
+ text: JSON.stringify({
49
+ error: err.name || 'Error',
50
+ message: err.message || 'Unknown error',
51
+ }),
52
+ },
53
+ ],
1123
54
  isError: true,
1124
55
  };
1125
56
  }
1126
57
  });
1127
- // ── Main ───────────────────────────────────────────────────────────────────────
58
+ // ── Main — stdio or HTTP mode ──────────────────────────────────────────────
1128
59
  async function main() {
1129
60
  const isHttpMode = process.env['MCP_HTTP_MODE'] === 'true' ||
1130
61
  process.env['HTTP_STREAMABLE_SERVER'] === 'true';
@@ -1143,9 +74,13 @@ async function main() {
1143
74
  await transport.handleRequest(r, s, r.body);
1144
75
  transport.close().catch(() => { });
1145
76
  }
1146
- catch (err) {
77
+ catch {
1147
78
  if (!s.headersSent) {
1148
- s.status(500).json({ jsonrpc: '2.0', error: { code: -32603, message: 'Internal error' }, id: null });
79
+ s.status(500).json({
80
+ jsonrpc: '2.0',
81
+ error: { code: -32603, message: 'Internal error' },
82
+ id: null,
83
+ });
1149
84
  }
1150
85
  }
1151
86
  });
@@ -1171,4 +106,3 @@ main().catch((error) => {
1171
106
  process.stderr.write(`Fatal error: ${error}\n`);
1172
107
  process.exit(1);
1173
108
  });
1174
- //# sourceMappingURL=server.js.map