webpeel 0.19.4 → 0.20.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (544) hide show
  1. package/README.md +2 -2
  2. package/dist/cache.d.ts +0 -1
  3. package/dist/cache.js +0 -1
  4. package/dist/cli/commands/auth.d.ts +5 -0
  5. package/dist/cli/commands/auth.js +476 -0
  6. package/dist/cli/commands/fetch.d.ts +6 -0
  7. package/dist/cli/commands/fetch.js +1015 -0
  8. package/dist/cli/commands/interact.d.ts +5 -0
  9. package/dist/cli/commands/interact.js +839 -0
  10. package/dist/cli/commands/jobs.d.ts +5 -0
  11. package/dist/cli/commands/jobs.js +997 -0
  12. package/dist/cli/commands/screenshot.d.ts +5 -0
  13. package/dist/cli/commands/screenshot.js +273 -0
  14. package/dist/cli/commands/search.d.ts +5 -0
  15. package/dist/cli/commands/search.js +524 -0
  16. package/dist/cli/utils.d.ts +84 -0
  17. package/dist/cli/utils.js +686 -0
  18. package/dist/cli-auth.d.ts +0 -1
  19. package/dist/cli-auth.js +0 -1
  20. package/dist/cli.d.ts +7 -6
  21. package/dist/cli.js +35 -4698
  22. package/dist/core/actions.d.ts +0 -1
  23. package/dist/core/actions.js +0 -1
  24. package/dist/core/agent.d.ts +0 -1
  25. package/dist/core/agent.js +9 -12
  26. package/dist/core/answer.d.ts +0 -1
  27. package/dist/core/answer.js +0 -1
  28. package/dist/core/application-tracker.d.ts +0 -1
  29. package/dist/core/application-tracker.js +0 -1
  30. package/dist/core/apply.d.ts +0 -1
  31. package/dist/core/apply.js +0 -1
  32. package/dist/core/auto-extract.d.ts +0 -1
  33. package/dist/core/auto-extract.js +0 -1
  34. package/dist/core/auto-interact.d.ts +0 -1
  35. package/dist/core/auto-interact.js +0 -1
  36. package/dist/core/bm25-filter.d.ts +0 -1
  37. package/dist/core/bm25-filter.js +0 -1
  38. package/dist/core/branding.d.ts +0 -1
  39. package/dist/core/branding.js +0 -1
  40. package/dist/core/browser-fetch.d.ts +0 -1
  41. package/dist/core/browser-fetch.js +17 -10
  42. package/dist/core/browser-pool.d.ts +0 -1
  43. package/dist/core/browser-pool.js +0 -1
  44. package/dist/core/budget.d.ts +0 -1
  45. package/dist/core/budget.js +0 -1
  46. package/dist/core/cache.d.ts +0 -1
  47. package/dist/core/cache.js +0 -1
  48. package/dist/core/cf-worker-proxy.d.ts +0 -1
  49. package/dist/core/cf-worker-proxy.js +0 -1
  50. package/dist/core/challenge-detection.d.ts +0 -1
  51. package/dist/core/challenge-detection.js +0 -1
  52. package/dist/core/change-tracking.d.ts +0 -1
  53. package/dist/core/change-tracking.js +0 -1
  54. package/dist/core/chunker.d.ts +0 -1
  55. package/dist/core/chunker.js +0 -1
  56. package/dist/core/chunking.d.ts +0 -1
  57. package/dist/core/chunking.js +0 -1
  58. package/dist/core/cloak-fetch.d.ts +0 -1
  59. package/dist/core/cloak-fetch.js +0 -1
  60. package/dist/core/content-pruner.d.ts +0 -1
  61. package/dist/core/content-pruner.js +0 -1
  62. package/dist/core/crawl-checkpoint.d.ts +0 -1
  63. package/dist/core/crawl-checkpoint.js +0 -1
  64. package/dist/core/crawler.d.ts +0 -1
  65. package/dist/core/crawler.js +6 -5
  66. package/dist/core/cycle-fetch.d.ts +0 -1
  67. package/dist/core/cycle-fetch.js +0 -1
  68. package/dist/core/deep-fetch.d.ts +0 -1
  69. package/dist/core/deep-fetch.js +0 -1
  70. package/dist/core/design-analysis.d.ts +0 -1
  71. package/dist/core/design-analysis.js +0 -1
  72. package/dist/core/design-compare.d.ts +0 -1
  73. package/dist/core/design-compare.js +0 -1
  74. package/dist/core/diff.d.ts +0 -1
  75. package/dist/core/diff.js +0 -1
  76. package/dist/core/dns-cache.d.ts +0 -1
  77. package/dist/core/dns-cache.js +0 -1
  78. package/dist/core/documents.d.ts +0 -1
  79. package/dist/core/documents.js +0 -1
  80. package/dist/core/domain-extractors.d.ts +0 -1
  81. package/dist/core/domain-extractors.js +0 -1
  82. package/dist/core/extract-inline.d.ts +0 -1
  83. package/dist/core/extract-inline.js +0 -1
  84. package/dist/core/extract-listings.d.ts +0 -1
  85. package/dist/core/extract-listings.js +0 -1
  86. package/dist/core/extract.d.ts +0 -1
  87. package/dist/core/extract.js +0 -1
  88. package/dist/core/fetcher.d.ts +0 -1
  89. package/dist/core/fetcher.js +0 -1
  90. package/dist/core/google-cache.d.ts +0 -1
  91. package/dist/core/google-cache.js +0 -1
  92. package/dist/core/hotel-search.d.ts +0 -1
  93. package/dist/core/hotel-search.js +0 -1
  94. package/dist/core/http-fetch.d.ts +0 -1
  95. package/dist/core/http-fetch.js +5 -7
  96. package/dist/core/human.d.ts +0 -1
  97. package/dist/core/human.js +0 -1
  98. package/dist/core/jobs.d.ts +0 -1
  99. package/dist/core/jobs.js +0 -1
  100. package/dist/core/json-ld.d.ts +0 -1
  101. package/dist/core/json-ld.js +0 -1
  102. package/dist/core/llm-extract.d.ts +0 -1
  103. package/dist/core/llm-extract.js +0 -1
  104. package/dist/core/logger.d.ts +17 -0
  105. package/dist/core/logger.js +44 -0
  106. package/dist/core/map.d.ts +0 -1
  107. package/dist/core/map.js +0 -1
  108. package/dist/core/markdown.d.ts +0 -1
  109. package/dist/core/markdown.js +0 -1
  110. package/dist/core/metadata.d.ts +0 -1
  111. package/dist/core/metadata.js +0 -1
  112. package/dist/core/paginate.d.ts +0 -1
  113. package/dist/core/paginate.js +0 -1
  114. package/dist/core/pdf.d.ts +0 -1
  115. package/dist/core/pdf.js +0 -1
  116. package/dist/core/peel-tls.d.ts +0 -1
  117. package/dist/core/peel-tls.js +0 -1
  118. package/dist/core/pipeline.d.ts +0 -1
  119. package/dist/core/pipeline.js +22 -25
  120. package/dist/core/profiles.d.ts +0 -1
  121. package/dist/core/profiles.js +0 -1
  122. package/dist/core/quick-answer.d.ts +0 -1
  123. package/dist/core/quick-answer.js +0 -1
  124. package/dist/core/rate-governor.d.ts +0 -1
  125. package/dist/core/rate-governor.js +0 -1
  126. package/dist/core/readability.d.ts +0 -1
  127. package/dist/core/readability.js +0 -1
  128. package/dist/core/research.d.ts +0 -1
  129. package/dist/core/research.js +0 -1
  130. package/dist/core/schema-extraction.d.ts +0 -1
  131. package/dist/core/schema-extraction.js +0 -1
  132. package/dist/core/schema-postprocess.d.ts +0 -1
  133. package/dist/core/schema-postprocess.js +0 -1
  134. package/dist/core/schema-templates.d.ts +0 -1
  135. package/dist/core/schema-templates.js +0 -1
  136. package/dist/core/screenshot.d.ts +0 -1
  137. package/dist/core/screenshot.js +0 -1
  138. package/dist/core/search-fallback.d.ts +0 -1
  139. package/dist/core/search-fallback.js +0 -1
  140. package/dist/core/search-provider.d.ts +0 -1
  141. package/dist/core/search-provider.js +18 -21
  142. package/dist/core/site-search.d.ts +0 -1
  143. package/dist/core/site-search.js +0 -1
  144. package/dist/core/sitemap.d.ts +0 -1
  145. package/dist/core/sitemap.js +0 -1
  146. package/dist/core/stealth-patches.d.ts +0 -1
  147. package/dist/core/stealth-patches.js +0 -1
  148. package/dist/core/stemmer.d.ts +0 -1
  149. package/dist/core/stemmer.js +0 -1
  150. package/dist/core/strategies.d.ts +6 -1
  151. package/dist/core/strategies.js +29 -41
  152. package/dist/core/strategy-hooks.d.ts +0 -1
  153. package/dist/core/strategy-hooks.js +0 -1
  154. package/dist/core/summarize.d.ts +0 -1
  155. package/dist/core/summarize.js +0 -1
  156. package/dist/core/synonyms.d.ts +0 -1
  157. package/dist/core/synonyms.js +0 -1
  158. package/dist/core/table-format.d.ts +0 -1
  159. package/dist/core/table-format.js +0 -1
  160. package/dist/core/timing.d.ts +0 -1
  161. package/dist/core/timing.js +0 -1
  162. package/dist/core/user-agents.d.ts +0 -1
  163. package/dist/core/user-agents.js +0 -1
  164. package/dist/core/watch-manager.d.ts +0 -1
  165. package/dist/core/watch-manager.js +0 -1
  166. package/dist/core/watch.d.ts +0 -1
  167. package/dist/core/watch.js +0 -1
  168. package/dist/core/youtube.d.ts +0 -1
  169. package/dist/core/youtube.js +0 -1
  170. package/dist/index.d.ts +8 -3
  171. package/dist/index.js +27 -3
  172. package/dist/integrations/index.d.ts +0 -1
  173. package/dist/integrations/index.js +0 -1
  174. package/dist/integrations/langchain.d.ts +0 -1
  175. package/dist/integrations/langchain.js +0 -1
  176. package/dist/integrations/llamaindex.d.ts +0 -1
  177. package/dist/integrations/llamaindex.js +0 -1
  178. package/dist/mcp/handlers/act.d.ts +5 -0
  179. package/dist/mcp/handlers/act.js +34 -0
  180. package/dist/mcp/handlers/definitions.d.ts +6 -0
  181. package/dist/mcp/handlers/definitions.js +266 -0
  182. package/dist/mcp/handlers/extract.d.ts +6 -0
  183. package/dist/mcp/handlers/extract.js +102 -0
  184. package/dist/mcp/handlers/fetch.d.ts +6 -0
  185. package/dist/mcp/handlers/fetch.js +98 -0
  186. package/dist/mcp/handlers/find.d.ts +5 -0
  187. package/dist/mcp/handlers/find.js +137 -0
  188. package/dist/mcp/handlers/index.d.ts +13 -0
  189. package/dist/mcp/handlers/index.js +61 -0
  190. package/dist/mcp/handlers/legacy.d.ts +25 -0
  191. package/dist/mcp/handlers/legacy.js +450 -0
  192. package/dist/mcp/handlers/meta.d.ts +6 -0
  193. package/dist/mcp/handlers/meta.js +31 -0
  194. package/dist/mcp/handlers/monitor.d.ts +5 -0
  195. package/dist/mcp/handlers/monitor.js +41 -0
  196. package/dist/mcp/handlers/read.d.ts +6 -0
  197. package/dist/mcp/handlers/read.js +63 -0
  198. package/dist/mcp/handlers/see.d.ts +5 -0
  199. package/dist/mcp/handlers/see.js +75 -0
  200. package/dist/mcp/handlers/types.d.ts +29 -0
  201. package/dist/mcp/handlers/types.js +28 -0
  202. package/dist/mcp/server.d.ts +3 -4
  203. package/dist/mcp/server.js +35 -1101
  204. package/dist/mcp/smart-router.d.ts +0 -1
  205. package/dist/mcp/smart-router.js +3 -1
  206. package/dist/types.d.ts +6 -1
  207. package/dist/types.js +0 -1
  208. package/package.json +3 -13
  209. package/dist/cache.d.ts.map +0 -1
  210. package/dist/cache.js.map +0 -1
  211. package/dist/cli-auth.d.ts.map +0 -1
  212. package/dist/cli-auth.js.map +0 -1
  213. package/dist/cli.bundle.cjs +0 -159248
  214. package/dist/cli.d.ts.map +0 -1
  215. package/dist/cli.js.map +0 -1
  216. package/dist/core/actions.d.ts.map +0 -1
  217. package/dist/core/actions.js.map +0 -1
  218. package/dist/core/agent.d.ts.map +0 -1
  219. package/dist/core/agent.js.map +0 -1
  220. package/dist/core/answer.d.ts.map +0 -1
  221. package/dist/core/answer.js.map +0 -1
  222. package/dist/core/application-tracker.d.ts.map +0 -1
  223. package/dist/core/application-tracker.js.map +0 -1
  224. package/dist/core/apply.d.ts.map +0 -1
  225. package/dist/core/apply.js.map +0 -1
  226. package/dist/core/auto-extract.d.ts.map +0 -1
  227. package/dist/core/auto-extract.js.map +0 -1
  228. package/dist/core/auto-interact.d.ts.map +0 -1
  229. package/dist/core/auto-interact.js.map +0 -1
  230. package/dist/core/bm25-filter.d.ts.map +0 -1
  231. package/dist/core/bm25-filter.js.map +0 -1
  232. package/dist/core/branding.d.ts.map +0 -1
  233. package/dist/core/branding.js.map +0 -1
  234. package/dist/core/browser-fetch.d.ts.map +0 -1
  235. package/dist/core/browser-fetch.js.map +0 -1
  236. package/dist/core/browser-pool.d.ts.map +0 -1
  237. package/dist/core/browser-pool.js.map +0 -1
  238. package/dist/core/budget.d.ts.map +0 -1
  239. package/dist/core/budget.js.map +0 -1
  240. package/dist/core/cache.d.ts.map +0 -1
  241. package/dist/core/cache.js.map +0 -1
  242. package/dist/core/cf-worker-proxy.d.ts.map +0 -1
  243. package/dist/core/cf-worker-proxy.js.map +0 -1
  244. package/dist/core/challenge-detection.d.ts.map +0 -1
  245. package/dist/core/challenge-detection.js.map +0 -1
  246. package/dist/core/change-tracking.d.ts.map +0 -1
  247. package/dist/core/change-tracking.js.map +0 -1
  248. package/dist/core/chunker.d.ts.map +0 -1
  249. package/dist/core/chunker.js.map +0 -1
  250. package/dist/core/chunking.d.ts.map +0 -1
  251. package/dist/core/chunking.js.map +0 -1
  252. package/dist/core/cloak-fetch.d.ts.map +0 -1
  253. package/dist/core/cloak-fetch.js.map +0 -1
  254. package/dist/core/content-pruner.d.ts.map +0 -1
  255. package/dist/core/content-pruner.js.map +0 -1
  256. package/dist/core/crawl-checkpoint.d.ts.map +0 -1
  257. package/dist/core/crawl-checkpoint.js.map +0 -1
  258. package/dist/core/crawler.d.ts.map +0 -1
  259. package/dist/core/crawler.js.map +0 -1
  260. package/dist/core/cycle-fetch.d.ts.map +0 -1
  261. package/dist/core/cycle-fetch.js.map +0 -1
  262. package/dist/core/deep-fetch.d.ts.map +0 -1
  263. package/dist/core/deep-fetch.js.map +0 -1
  264. package/dist/core/design-analysis.d.ts.map +0 -1
  265. package/dist/core/design-analysis.js.map +0 -1
  266. package/dist/core/design-compare.d.ts.map +0 -1
  267. package/dist/core/design-compare.js.map +0 -1
  268. package/dist/core/diff.d.ts.map +0 -1
  269. package/dist/core/diff.js.map +0 -1
  270. package/dist/core/dns-cache.d.ts.map +0 -1
  271. package/dist/core/dns-cache.js.map +0 -1
  272. package/dist/core/documents.d.ts.map +0 -1
  273. package/dist/core/documents.js.map +0 -1
  274. package/dist/core/domain-extractors.d.ts.map +0 -1
  275. package/dist/core/domain-extractors.js.map +0 -1
  276. package/dist/core/extract-inline.d.ts.map +0 -1
  277. package/dist/core/extract-inline.js.map +0 -1
  278. package/dist/core/extract-listings.d.ts.map +0 -1
  279. package/dist/core/extract-listings.js.map +0 -1
  280. package/dist/core/extract.d.ts.map +0 -1
  281. package/dist/core/extract.js.map +0 -1
  282. package/dist/core/fetcher.d.ts.map +0 -1
  283. package/dist/core/fetcher.js.map +0 -1
  284. package/dist/core/google-cache.d.ts.map +0 -1
  285. package/dist/core/google-cache.js.map +0 -1
  286. package/dist/core/hotel-search.d.ts.map +0 -1
  287. package/dist/core/hotel-search.js.map +0 -1
  288. package/dist/core/http-fetch.d.ts.map +0 -1
  289. package/dist/core/http-fetch.js.map +0 -1
  290. package/dist/core/human.d.ts.map +0 -1
  291. package/dist/core/human.js.map +0 -1
  292. package/dist/core/jobs.d.ts.map +0 -1
  293. package/dist/core/jobs.js.map +0 -1
  294. package/dist/core/json-ld.d.ts.map +0 -1
  295. package/dist/core/json-ld.js.map +0 -1
  296. package/dist/core/llm-extract.d.ts.map +0 -1
  297. package/dist/core/llm-extract.js.map +0 -1
  298. package/dist/core/map.d.ts.map +0 -1
  299. package/dist/core/map.js.map +0 -1
  300. package/dist/core/markdown.d.ts.map +0 -1
  301. package/dist/core/markdown.js.map +0 -1
  302. package/dist/core/metadata.d.ts.map +0 -1
  303. package/dist/core/metadata.js.map +0 -1
  304. package/dist/core/paginate.d.ts.map +0 -1
  305. package/dist/core/paginate.js.map +0 -1
  306. package/dist/core/pdf.d.ts.map +0 -1
  307. package/dist/core/pdf.js.map +0 -1
  308. package/dist/core/peel-tls.d.ts.map +0 -1
  309. package/dist/core/peel-tls.js.map +0 -1
  310. package/dist/core/pipeline.d.ts.map +0 -1
  311. package/dist/core/pipeline.js.map +0 -1
  312. package/dist/core/profiles.d.ts.map +0 -1
  313. package/dist/core/profiles.js.map +0 -1
  314. package/dist/core/quick-answer.d.ts.map +0 -1
  315. package/dist/core/quick-answer.js.map +0 -1
  316. package/dist/core/rate-governor.d.ts.map +0 -1
  317. package/dist/core/rate-governor.js.map +0 -1
  318. package/dist/core/readability.d.ts.map +0 -1
  319. package/dist/core/readability.js.map +0 -1
  320. package/dist/core/research.d.ts.map +0 -1
  321. package/dist/core/research.js.map +0 -1
  322. package/dist/core/schema-extraction.d.ts.map +0 -1
  323. package/dist/core/schema-extraction.js.map +0 -1
  324. package/dist/core/schema-postprocess.d.ts.map +0 -1
  325. package/dist/core/schema-postprocess.js.map +0 -1
  326. package/dist/core/schema-templates.d.ts.map +0 -1
  327. package/dist/core/schema-templates.js.map +0 -1
  328. package/dist/core/screenshot.d.ts.map +0 -1
  329. package/dist/core/screenshot.js.map +0 -1
  330. package/dist/core/search-fallback.d.ts.map +0 -1
  331. package/dist/core/search-fallback.js.map +0 -1
  332. package/dist/core/search-provider.d.ts.map +0 -1
  333. package/dist/core/search-provider.js.map +0 -1
  334. package/dist/core/site-search.d.ts.map +0 -1
  335. package/dist/core/site-search.js.map +0 -1
  336. package/dist/core/sitemap.d.ts.map +0 -1
  337. package/dist/core/sitemap.js.map +0 -1
  338. package/dist/core/stealth-patches.d.ts.map +0 -1
  339. package/dist/core/stealth-patches.js.map +0 -1
  340. package/dist/core/stemmer.d.ts.map +0 -1
  341. package/dist/core/stemmer.js.map +0 -1
  342. package/dist/core/strategies.d.ts.map +0 -1
  343. package/dist/core/strategies.js.map +0 -1
  344. package/dist/core/strategy-hooks.d.ts.map +0 -1
  345. package/dist/core/strategy-hooks.js.map +0 -1
  346. package/dist/core/summarize.d.ts.map +0 -1
  347. package/dist/core/summarize.js.map +0 -1
  348. package/dist/core/synonyms.d.ts.map +0 -1
  349. package/dist/core/synonyms.js.map +0 -1
  350. package/dist/core/table-format.d.ts.map +0 -1
  351. package/dist/core/table-format.js.map +0 -1
  352. package/dist/core/timing.d.ts.map +0 -1
  353. package/dist/core/timing.js.map +0 -1
  354. package/dist/core/user-agents.d.ts.map +0 -1
  355. package/dist/core/user-agents.js.map +0 -1
  356. package/dist/core/watch-manager.d.ts.map +0 -1
  357. package/dist/core/watch-manager.js.map +0 -1
  358. package/dist/core/watch.d.ts.map +0 -1
  359. package/dist/core/watch.js.map +0 -1
  360. package/dist/core/youtube.d.ts.map +0 -1
  361. package/dist/core/youtube.js.map +0 -1
  362. package/dist/index.d.ts.map +0 -1
  363. package/dist/index.js.map +0 -1
  364. package/dist/integrations/index.d.ts.map +0 -1
  365. package/dist/integrations/index.js.map +0 -1
  366. package/dist/integrations/langchain.d.ts.map +0 -1
  367. package/dist/integrations/langchain.js.map +0 -1
  368. package/dist/integrations/llamaindex.d.ts.map +0 -1
  369. package/dist/integrations/llamaindex.js.map +0 -1
  370. package/dist/mcp/server.d.ts.map +0 -1
  371. package/dist/mcp/server.js.map +0 -1
  372. package/dist/mcp/smart-router.d.ts.map +0 -1
  373. package/dist/mcp/smart-router.js.map +0 -1
  374. package/dist/server/app.d.ts +0 -15
  375. package/dist/server/app.d.ts.map +0 -1
  376. package/dist/server/app.js +0 -350
  377. package/dist/server/app.js.map +0 -1
  378. package/dist/server/auth-store.d.ts +0 -28
  379. package/dist/server/auth-store.d.ts.map +0 -1
  380. package/dist/server/auth-store.js +0 -89
  381. package/dist/server/auth-store.js.map +0 -1
  382. package/dist/server/email-service.d.ts +0 -22
  383. package/dist/server/email-service.d.ts.map +0 -1
  384. package/dist/server/email-service.js +0 -80
  385. package/dist/server/email-service.js.map +0 -1
  386. package/dist/server/job-queue.d.ts +0 -93
  387. package/dist/server/job-queue.d.ts.map +0 -1
  388. package/dist/server/job-queue.js +0 -146
  389. package/dist/server/job-queue.js.map +0 -1
  390. package/dist/server/logger.d.ts +0 -11
  391. package/dist/server/logger.d.ts.map +0 -1
  392. package/dist/server/logger.js +0 -38
  393. package/dist/server/logger.js.map +0 -1
  394. package/dist/server/middleware/auth.d.ts +0 -29
  395. package/dist/server/middleware/auth.d.ts.map +0 -1
  396. package/dist/server/middleware/auth.js +0 -222
  397. package/dist/server/middleware/auth.js.map +0 -1
  398. package/dist/server/middleware/rate-limit.d.ts +0 -25
  399. package/dist/server/middleware/rate-limit.d.ts.map +0 -1
  400. package/dist/server/middleware/rate-limit.js +0 -168
  401. package/dist/server/middleware/rate-limit.js.map +0 -1
  402. package/dist/server/middleware/url-validator.d.ts +0 -16
  403. package/dist/server/middleware/url-validator.d.ts.map +0 -1
  404. package/dist/server/middleware/url-validator.js +0 -187
  405. package/dist/server/middleware/url-validator.js.map +0 -1
  406. package/dist/server/openapi.yaml +0 -4944
  407. package/dist/server/pg-auth-store.d.ts +0 -133
  408. package/dist/server/pg-auth-store.d.ts.map +0 -1
  409. package/dist/server/pg-auth-store.js +0 -473
  410. package/dist/server/pg-auth-store.js.map +0 -1
  411. package/dist/server/pg-job-queue.d.ts +0 -60
  412. package/dist/server/pg-job-queue.d.ts.map +0 -1
  413. package/dist/server/pg-job-queue.js +0 -365
  414. package/dist/server/pg-job-queue.js.map +0 -1
  415. package/dist/server/premium/domain-intel.d.ts +0 -17
  416. package/dist/server/premium/domain-intel.d.ts.map +0 -1
  417. package/dist/server/premium/domain-intel.js +0 -134
  418. package/dist/server/premium/domain-intel.js.map +0 -1
  419. package/dist/server/premium/index.d.ts +0 -18
  420. package/dist/server/premium/index.d.ts.map +0 -1
  421. package/dist/server/premium/index.js +0 -36
  422. package/dist/server/premium/index.js.map +0 -1
  423. package/dist/server/premium/swr-cache.d.ts +0 -15
  424. package/dist/server/premium/swr-cache.d.ts.map +0 -1
  425. package/dist/server/premium/swr-cache.js +0 -35
  426. package/dist/server/premium/swr-cache.js.map +0 -1
  427. package/dist/server/routes/activity.d.ts +0 -7
  428. package/dist/server/routes/activity.d.ts.map +0 -1
  429. package/dist/server/routes/activity.js +0 -68
  430. package/dist/server/routes/activity.js.map +0 -1
  431. package/dist/server/routes/agent.d.ts +0 -16
  432. package/dist/server/routes/agent.d.ts.map +0 -1
  433. package/dist/server/routes/agent.js +0 -247
  434. package/dist/server/routes/agent.js.map +0 -1
  435. package/dist/server/routes/answer.d.ts +0 -6
  436. package/dist/server/routes/answer.d.ts.map +0 -1
  437. package/dist/server/routes/answer.js +0 -133
  438. package/dist/server/routes/answer.js.map +0 -1
  439. package/dist/server/routes/ask.d.ts +0 -23
  440. package/dist/server/routes/ask.d.ts.map +0 -1
  441. package/dist/server/routes/ask.js +0 -119
  442. package/dist/server/routes/ask.js.map +0 -1
  443. package/dist/server/routes/batch.d.ts +0 -7
  444. package/dist/server/routes/batch.d.ts.map +0 -1
  445. package/dist/server/routes/batch.js +0 -412
  446. package/dist/server/routes/batch.js.map +0 -1
  447. package/dist/server/routes/cli-usage.d.ts +0 -7
  448. package/dist/server/routes/cli-usage.d.ts.map +0 -1
  449. package/dist/server/routes/cli-usage.js +0 -121
  450. package/dist/server/routes/cli-usage.js.map +0 -1
  451. package/dist/server/routes/compat.d.ts +0 -24
  452. package/dist/server/routes/compat.d.ts.map +0 -1
  453. package/dist/server/routes/compat.js +0 -653
  454. package/dist/server/routes/compat.js.map +0 -1
  455. package/dist/server/routes/deep-fetch.d.ts +0 -9
  456. package/dist/server/routes/deep-fetch.d.ts.map +0 -1
  457. package/dist/server/routes/deep-fetch.js +0 -50
  458. package/dist/server/routes/deep-fetch.js.map +0 -1
  459. package/dist/server/routes/demo.d.ts +0 -25
  460. package/dist/server/routes/demo.d.ts.map +0 -1
  461. package/dist/server/routes/demo.js +0 -434
  462. package/dist/server/routes/demo.js.map +0 -1
  463. package/dist/server/routes/extract.d.ts +0 -9
  464. package/dist/server/routes/extract.d.ts.map +0 -1
  465. package/dist/server/routes/extract.js +0 -150
  466. package/dist/server/routes/extract.js.map +0 -1
  467. package/dist/server/routes/fetch.d.ts +0 -8
  468. package/dist/server/routes/fetch.d.ts.map +0 -1
  469. package/dist/server/routes/fetch.js +0 -988
  470. package/dist/server/routes/fetch.js.map +0 -1
  471. package/dist/server/routes/health.d.ts +0 -8
  472. package/dist/server/routes/health.d.ts.map +0 -1
  473. package/dist/server/routes/health.js +0 -20
  474. package/dist/server/routes/health.js.map +0 -1
  475. package/dist/server/routes/jobs.d.ts +0 -8
  476. package/dist/server/routes/jobs.d.ts.map +0 -1
  477. package/dist/server/routes/jobs.js +0 -487
  478. package/dist/server/routes/jobs.js.map +0 -1
  479. package/dist/server/routes/mcp.d.ts +0 -18
  480. package/dist/server/routes/mcp.d.ts.map +0 -1
  481. package/dist/server/routes/mcp.js +0 -1260
  482. package/dist/server/routes/mcp.js.map +0 -1
  483. package/dist/server/routes/oauth.d.ts +0 -10
  484. package/dist/server/routes/oauth.d.ts.map +0 -1
  485. package/dist/server/routes/oauth.js +0 -334
  486. package/dist/server/routes/oauth.js.map +0 -1
  487. package/dist/server/routes/quick-answer.d.ts +0 -9
  488. package/dist/server/routes/quick-answer.d.ts.map +0 -1
  489. package/dist/server/routes/quick-answer.js +0 -93
  490. package/dist/server/routes/quick-answer.js.map +0 -1
  491. package/dist/server/routes/screenshot.d.ts +0 -23
  492. package/dist/server/routes/screenshot.d.ts.map +0 -1
  493. package/dist/server/routes/screenshot.js +0 -819
  494. package/dist/server/routes/screenshot.js.map +0 -1
  495. package/dist/server/routes/search.d.ts +0 -7
  496. package/dist/server/routes/search.d.ts.map +0 -1
  497. package/dist/server/routes/search.js +0 -312
  498. package/dist/server/routes/search.js.map +0 -1
  499. package/dist/server/routes/session.d.ts +0 -16
  500. package/dist/server/routes/session.d.ts.map +0 -1
  501. package/dist/server/routes/session.js +0 -278
  502. package/dist/server/routes/session.js.map +0 -1
  503. package/dist/server/routes/stats.d.ts +0 -7
  504. package/dist/server/routes/stats.d.ts.map +0 -1
  505. package/dist/server/routes/stats.js +0 -65
  506. package/dist/server/routes/stats.js.map +0 -1
  507. package/dist/server/routes/stripe.d.ts +0 -16
  508. package/dist/server/routes/stripe.d.ts.map +0 -1
  509. package/dist/server/routes/stripe.js +0 -283
  510. package/dist/server/routes/stripe.js.map +0 -1
  511. package/dist/server/routes/users.d.ts +0 -9
  512. package/dist/server/routes/users.d.ts.map +0 -1
  513. package/dist/server/routes/users.js +0 -1211
  514. package/dist/server/routes/users.js.map +0 -1
  515. package/dist/server/routes/watch.d.ts +0 -16
  516. package/dist/server/routes/watch.d.ts.map +0 -1
  517. package/dist/server/routes/watch.js +0 -257
  518. package/dist/server/routes/watch.js.map +0 -1
  519. package/dist/server/routes/webhooks.d.ts +0 -16
  520. package/dist/server/routes/webhooks.d.ts.map +0 -1
  521. package/dist/server/routes/webhooks.js +0 -74
  522. package/dist/server/routes/webhooks.js.map +0 -1
  523. package/dist/server/routes/youtube.d.ts +0 -7
  524. package/dist/server/routes/youtube.d.ts.map +0 -1
  525. package/dist/server/routes/youtube.js +0 -93
  526. package/dist/server/routes/youtube.js.map +0 -1
  527. package/dist/server/sentry.d.ts +0 -14
  528. package/dist/server/sentry.d.ts.map +0 -1
  529. package/dist/server/sentry.js +0 -39
  530. package/dist/server/sentry.js.map +0 -1
  531. package/dist/server/types.d.ts +0 -16
  532. package/dist/server/types.d.ts.map +0 -1
  533. package/dist/server/types.js +0 -8
  534. package/dist/server/types.js.map +0 -1
  535. package/dist/server/utils/response.d.ts +0 -45
  536. package/dist/server/utils/response.d.ts.map +0 -1
  537. package/dist/server/utils/response.js +0 -70
  538. package/dist/server/utils/response.js.map +0 -1
  539. package/dist/server/utils/sse.d.ts +0 -23
  540. package/dist/server/utils/sse.d.ts.map +0 -1
  541. package/dist/server/utils/sse.js +0 -39
  542. package/dist/server/utils/sse.js.map +0 -1
  543. package/dist/types.d.ts.map +0 -1
  544. package/dist/types.js.map +0 -1
@@ -49,4 +49,3 @@ export declare function executeActions(page: Page, actions: PageAction[], screen
49
49
  type?: 'png' | 'jpeg';
50
50
  quality?: number;
51
51
  }): Promise<Buffer | undefined>;
52
- //# sourceMappingURL=actions.d.ts.map
@@ -306,4 +306,3 @@ export async function executeActions(page, actions, screenshotOptions) {
306
306
  }
307
307
  return lastScreenshot;
308
308
  }
309
- //# sourceMappingURL=actions.js.map
@@ -96,4 +96,3 @@ export type AgentStreamEvent = {
96
96
  * Run autonomous web research agent
97
97
  */
98
98
  export declare function runAgent(options: AgentOptions): Promise<AgentResult>;
99
- //# sourceMappingURL=agent.d.ts.map
@@ -11,6 +11,8 @@
11
11
  */
12
12
  import { load } from 'cheerio';
13
13
  import { peel } from '../index.js';
14
+ import { createLogger } from './logger.js';
15
+ const log = createLogger('agent');
14
16
  // ---------------------------------------------------------------------------
15
17
  // Helpers
16
18
  // ---------------------------------------------------------------------------
@@ -46,15 +48,14 @@ async function searchWeb(query, limit = 10) {
46
48
  results.push({ url: actualUrl, title, snippet: desc });
47
49
  }
48
50
  catch (e) {
49
- if (process.env.DEBUG)
50
- console.debug('[webpeel]', 'url decode failed:', e instanceof Error ? e.message : e);
51
+ log.debug('url decode failed:', e instanceof Error ? e.message : e);
51
52
  }
52
53
  }
53
54
  });
54
55
  return results.slice(0, limit);
55
56
  }
56
57
  catch (error) {
57
- console.error('Search failed:', error);
58
+ log.error('Search failed:', error);
58
59
  return [];
59
60
  }
60
61
  }
@@ -215,8 +216,7 @@ async function callLLMStreaming(messages, options, onChunk) {
215
216
  }
216
217
  }
217
218
  catch (e) {
218
- if (process.env.DEBUG)
219
- console.debug('[webpeel]', 'stream chunk parse failed:', e instanceof Error ? e.message : e);
219
+ log.debug('stream chunk parse failed:', e instanceof Error ? e.message : e);
220
220
  }
221
221
  }
222
222
  }
@@ -388,7 +388,7 @@ export async function runAgent(options) {
388
388
  reportProgress('visiting', `Fetched: ${result.title}`, url);
389
389
  }
390
390
  catch (error) {
391
- console.error(`Failed to fetch ${url}:`, error.message);
391
+ log.error(`Failed to fetch ${url}:`, error.message);
392
392
  }
393
393
  }
394
394
  // -----------------------------------------------------------------------
@@ -435,15 +435,13 @@ export async function runAgent(options) {
435
435
  sourcesDetailed.push({ url: result.url, title: result.title });
436
436
  }
437
437
  catch (e) {
438
- if (process.env.DEBUG)
439
- console.debug('[webpeel]', 'page fetch failed:', e instanceof Error ? e.message : e);
438
+ log.debug('page fetch failed:', e instanceof Error ? e.message : e);
440
439
  }
441
440
  }
442
441
  }
443
442
  }
444
443
  catch (e) {
445
- if (process.env.DEBUG)
446
- console.debug('[webpeel]', 'research batch failed:', e instanceof Error ? e.message : e);
444
+ log.debug('research batch failed:', e instanceof Error ? e.message : e);
447
445
  }
448
446
  }
449
447
  // -----------------------------------------------------------------------
@@ -546,7 +544,7 @@ export async function runAgent(options) {
546
544
  };
547
545
  }
548
546
  catch (error) {
549
- console.error('Agent error:', error);
547
+ log.error('Agent error:', error);
550
548
  return {
551
549
  success: false,
552
550
  data: { error: error.message || 'Unknown error occurred' },
@@ -558,4 +556,3 @@ export async function runAgent(options) {
558
556
  };
559
557
  }
560
558
  }
561
- //# sourceMappingURL=agent.js.map
@@ -40,4 +40,3 @@ export interface AnswerResponse {
40
40
  tokensUsed: TokensUsed;
41
41
  }
42
42
  export declare function answerQuestion(req: AnswerRequest): Promise<AnswerResponse>;
43
- //# sourceMappingURL=answer.d.ts.map
@@ -375,4 +375,3 @@ export async function answerQuestion(req) {
375
375
  tokensUsed,
376
376
  };
377
377
  }
378
- //# sourceMappingURL=answer.js.map
@@ -82,4 +82,3 @@ export declare class ApplicationTracker {
82
82
  private loadRecords;
83
83
  private save;
84
84
  }
85
- //# sourceMappingURL=application-tracker.d.ts.map
@@ -182,4 +182,3 @@ function normaliseUrl(rawUrl) {
182
182
  return rawUrl.toLowerCase().replace(/\/$/, '');
183
183
  }
184
184
  }
185
- //# sourceMappingURL=application-tracker.js.map
@@ -160,4 +160,3 @@ export declare function updateApplicationStatus(id: string, status: ApplicationR
160
160
  * ```
161
161
  */
162
162
  export declare function applyToJob(options: ApplyOptions): Promise<ApplyResult>;
163
- //# sourceMappingURL=apply.d.ts.map
@@ -814,4 +814,3 @@ export async function applyToJob(options) {
814
814
  result.elapsed = Date.now() - startTime;
815
815
  return result;
816
816
  }
817
- //# sourceMappingURL=apply.js.map
@@ -80,4 +80,3 @@ export { detectPageType as default };
80
80
  * Auto-extract structured data from a web page without an LLM API key.
81
81
  */
82
82
  export declare function autoExtract(html: string, url: string): AutoExtractResult;
83
- //# sourceMappingURL=auto-extract.d.ts.map
@@ -602,4 +602,3 @@ export function autoExtract(html, url) {
602
602
  }
603
603
  }
604
604
  }
605
- //# sourceMappingURL=auto-extract.js.map
@@ -21,4 +21,3 @@ export interface AutoInteractResult {
21
21
  * @returns Summary of what was dismissed
22
22
  */
23
23
  export declare function autoInteract(page: Page): Promise<AutoInteractResult>;
24
- //# sourceMappingURL=auto-interact.d.ts.map
@@ -244,4 +244,3 @@ export async function autoInteract(page) {
244
244
  }
245
245
  return result;
246
246
  }
247
- //# sourceMappingURL=auto-interact.js.map
@@ -64,4 +64,3 @@ export declare function computeRelevanceScore(content: string, query: string): n
64
64
  */
65
65
  export declare function filterByRelevance(content: string, options: BM25FilterOptions): BM25FilterResult;
66
66
  export {};
67
- //# sourceMappingURL=bm25-filter.d.ts.map
@@ -286,4 +286,3 @@ export function filterByRelevance(content, options) {
286
286
  reductionPercent,
287
287
  };
288
288
  }
289
- //# sourceMappingURL=bm25-filter.js.map
@@ -52,4 +52,3 @@ export interface BrandingProfile {
52
52
  * ```
53
53
  */
54
54
  export declare function extractBranding(page: Page): Promise<BrandingProfile>;
55
- //# sourceMappingURL=branding.d.ts.map
@@ -232,4 +232,3 @@ export async function extractBranding(page) {
232
232
  };
233
233
  }
234
234
  }
235
- //# sourceMappingURL=branding.js.map
@@ -311,4 +311,3 @@ export declare function browserDesignAnalysis(url: string, options?: {
311
311
  analysis: import('./design-analysis.js').DesignAnalysis;
312
312
  finalUrl: string;
313
313
  }>;
314
- //# sourceMappingURL=browser-fetch.d.ts.map
@@ -6,9 +6,20 @@ import { TimeoutError, BlockedError, NetworkError, WebPeelError } from '../types
6
6
  import { detectChallenge } from './challenge-detection.js';
7
7
  import { getRealisticUserAgent } from './user-agents.js';
8
8
  import { getRandomUserAgent, applyStealthScripts, takePooledPage, ensurePagePool, recyclePooledPage, getBrowser, getStealthBrowser, getProfileBrowser, PAGE_POOL_SIZE, MAX_CONCURRENT_PAGES, getPooledPagesCount, } from './browser-pool.js';
9
- import { applyStealthPatches, applyAcceptLanguageHeader } from './stealth-patches.js';
9
+ // Proprietary stealth module gitignored, loaded conditionally
10
+ let applyStealthPatches;
11
+ let applyAcceptLanguageHeader;
12
+ try {
13
+ // @ts-ignore — proprietary module, gitignored
14
+ const mod = await import('./stealth-patches.js');
15
+ applyStealthPatches = mod.applyStealthPatches;
16
+ applyAcceptLanguageHeader = mod.applyAcceptLanguageHeader;
17
+ }
18
+ catch { /* Not available in CI/open-source builds */ }
10
19
  import { validateUrl, validateUserAgent, createAbortError } from './http-fetch.js';
11
20
  import { autoInteract } from './auto-interact.js';
21
+ import { createLogger } from './logger.js';
22
+ const log = createLogger('browser');
12
23
  // ── Concurrency state (owned by this module) ─────────────────────────────────
13
24
  let activePagesCount = 0;
14
25
  // ── browserFetch ──────────────────────────────────────────────────────────────
@@ -121,8 +132,7 @@ export async function browserFetch(url, options = {}) {
121
132
  }
122
133
  catch (e) {
123
134
  // Fallback: use proxy string as-is
124
- if (process.env.DEBUG)
125
- console.debug('[webpeel]', 'proxy URL parse failed, using as-is:', e instanceof Error ? e.message : e);
135
+ log.debug('proxy URL parse failed, using as-is:', e instanceof Error ? e.message : e);
126
136
  playwrightProxy = { server: proxy };
127
137
  }
128
138
  // Create an isolated context with the proxy and optional storageState
@@ -154,8 +164,8 @@ export async function browserFetch(url, options = {}) {
154
164
  // Apply supplemental stealth patches (canvas noise, connection API, battery, etc.)
155
165
  // These go beyond what puppeteer-extra-plugin-stealth provides.
156
166
  if (stealth) {
157
- await applyStealthPatches(page);
158
- await applyAcceptLanguageHeader(page, 'en-US');
167
+ await applyStealthPatches?.(page);
168
+ await applyAcceptLanguageHeader?.(page, 'en-US');
159
169
  }
160
170
  usingPooledPage = false;
161
171
  }
@@ -281,8 +291,7 @@ export async function browserFetch(url, options = {}) {
281
291
  // Wait for a specific CSS selector if requested
282
292
  if (waitSelector) {
283
293
  await page.waitForSelector(waitSelector, { timeout: timeoutMs }).catch(() => {
284
- if (process.env.DEBUG)
285
- console.debug('[webpeel]', `waitSelector "${waitSelector}" not found within timeout`);
294
+ log.debug(`waitSelector "${waitSelector}" not found within timeout`);
286
295
  });
287
296
  throwIfAborted();
288
297
  }
@@ -812,8 +821,7 @@ export async function scrollAndWait(page, times = 3) {
812
821
  }
813
822
  catch (e) {
814
823
  // networkidle may never fire — fall back to a flat delay.
815
- if (process.env.DEBUG)
816
- console.debug('[webpeel]', 'networkidle timeout, falling back to flat delay:', e instanceof Error ? e.message : e);
824
+ log.debug('networkidle timeout, falling back to flat delay:', e instanceof Error ? e.message : e);
817
825
  await page.waitForTimeout(1000);
818
826
  }
819
827
  }
@@ -1458,4 +1466,3 @@ export async function browserDesignAnalysis(url, options = {}) {
1458
1466
  });
1459
1467
  return { analysis, finalUrl };
1460
1468
  }
1461
- //# sourceMappingURL=browser-fetch.js.map
@@ -67,4 +67,3 @@ export declare function cleanup(): Promise<void>;
67
67
  * @param profileDir Path to the profile directory used when launching
68
68
  */
69
69
  export declare function closeProfileBrowser(profileDir: string): Promise<void>;
70
- //# sourceMappingURL=browser-pool.d.ts.map
@@ -375,4 +375,3 @@ export async function closeProfileBrowser(profileDir) {
375
375
  await browser.close().catch(() => { });
376
376
  }
377
377
  }
378
- //# sourceMappingURL=browser-pool.js.map
@@ -40,4 +40,3 @@ export declare function budgetListings(totalItems: number, budget: number): {
40
40
  truncated: boolean;
41
41
  totalAvailable: number;
42
42
  };
43
- //# sourceMappingURL=budget.d.ts.map
@@ -322,4 +322,3 @@ function distillJson(content, budget) {
322
322
  return hardTruncate(content, budget);
323
323
  }
324
324
  }
325
- //# sourceMappingURL=budget.js.map
@@ -11,4 +11,3 @@ export declare function markRevalidating(url: string): boolean;
11
11
  export declare function setCached<T = unknown>(url: string, result: T): void;
12
12
  export declare function clearCache(): void;
13
13
  export declare function setCacheTTL(ms: number): void;
14
- //# sourceMappingURL=cache.d.ts.map
@@ -119,4 +119,3 @@ export function setCacheTTL(ms) {
119
119
  }
120
120
  cacheTTL = ms;
121
121
  }
122
- //# sourceMappingURL=cache.js.map
@@ -30,4 +30,3 @@ export declare function isCfWorkerAvailable(): boolean;
30
30
  * The Worker makes the request from Cloudflare's edge — clean IPs.
31
31
  */
32
32
  export declare function cfWorkerFetch(url: string, options?: CfWorkerProxyOptions): Promise<CfWorkerProxyResult>;
33
- //# sourceMappingURL=cf-worker-proxy.d.ts.map
@@ -85,4 +85,3 @@ export async function cfWorkerFetch(url, options) {
85
85
  edge: result.edge,
86
86
  };
87
87
  }
88
- //# sourceMappingURL=cf-worker-proxy.js.map
@@ -24,4 +24,3 @@ export interface ChallengeDetectionResult {
24
24
  * @param statusCode - HTTP status code (optional but improves accuracy).
25
25
  */
26
26
  export declare function detectChallenge(html: string, statusCode?: number): ChallengeDetectionResult;
27
- //# sourceMappingURL=challenge-detection.d.ts.map
@@ -466,4 +466,3 @@ export function detectChallenge(html, statusCode) {
466
466
  details: `Detected as ${best.type} (confidence ${best.score.toFixed(2)})`,
467
467
  };
468
468
  }
469
- //# sourceMappingURL=challenge-detection.js.map
@@ -73,4 +73,3 @@ export declare function trackChange(url: string, content: string, fingerprint: s
73
73
  * ```
74
74
  */
75
75
  export declare function clearSnapshots(urlPattern?: string): Promise<number>;
76
- //# sourceMappingURL=change-tracking.d.ts.map
@@ -274,4 +274,3 @@ export async function clearSnapshots(urlPattern) {
274
274
  return 0;
275
275
  }
276
276
  }
277
- //# sourceMappingURL=change-tracking.js.map
@@ -44,4 +44,3 @@ export interface ChunkResult {
44
44
  * Split content into RAG-ready chunks with metadata.
45
45
  */
46
46
  export declare function chunkContent(content: string, options?: ChunkOptions): ChunkResult;
47
- //# sourceMappingURL=chunker.d.ts.map
@@ -247,4 +247,3 @@ function splitByHeadings(content) {
247
247
  }
248
248
  return sections;
249
249
  }
250
- //# sourceMappingURL=chunker.js.map
@@ -40,4 +40,3 @@ export declare function estimateTokens(text: string): number;
40
40
  * Split content into chunks suitable for LLM processing.
41
41
  */
42
42
  export declare function chunkContent(content: string, options?: ChunkOptions): ChunkResult;
43
- //# sourceMappingURL=chunking.d.ts.map
@@ -179,4 +179,3 @@ function chunkParagraph(content, chunkSize) {
179
179
  }
180
180
  return result;
181
181
  }
182
- //# sourceMappingURL=chunking.js.map
@@ -40,4 +40,3 @@ export declare function isCloakBrowserAvailable(): boolean;
40
40
  * Fetch a URL using CloakBrowser's patched Chromium.
41
41
  */
42
42
  export declare function cloakFetch(options: CloakFetchOptions): Promise<CloakFetchResult>;
43
- //# sourceMappingURL=cloak-fetch.d.ts.map
@@ -146,4 +146,3 @@ export async function cloakFetch(options) {
146
146
  await browser.close().catch(() => { });
147
147
  }
148
148
  }
149
- //# sourceMappingURL=cloak-fetch.js.map
@@ -45,4 +45,3 @@ export declare function pruneContent(html: string, options?: PruneOptions): Prun
45
45
  * @returns Cleaned markdown string
46
46
  */
47
47
  export declare function pruneMarkdown(markdown: string): string;
48
- //# sourceMappingURL=content-pruner.d.ts.map
@@ -420,4 +420,3 @@ export function pruneMarkdown(markdown) {
420
420
  }
421
421
  return result.join('\n');
422
422
  }
423
- //# sourceMappingURL=content-pruner.js.map
@@ -52,4 +52,3 @@ export declare function listCheckpoints(): Array<{
52
52
  pending: number;
53
53
  lastCheckpoint: number;
54
54
  }>;
55
- //# sourceMappingURL=crawl-checkpoint.d.ts.map
@@ -102,4 +102,3 @@ export function listCheckpoints() {
102
102
  return [];
103
103
  }
104
104
  }
105
- //# sourceMappingURL=crawl-checkpoint.js.map
@@ -80,4 +80,3 @@ export interface CrawlResult {
80
80
  * ```
81
81
  */
82
82
  export declare function crawl(startUrl: string, options?: CrawlOptions): Promise<CrawlResult[]>;
83
- //# sourceMappingURL=crawler.d.ts.map
@@ -7,6 +7,8 @@ import { fetch as undiciFetch } from 'undici';
7
7
  import { createHash } from 'crypto';
8
8
  import { discoverSitemap } from './sitemap.js';
9
9
  import { generateJobId, loadCheckpoint, saveCheckpoint, deleteCheckpoint, } from './crawl-checkpoint.js';
10
+ import { createLogger } from './logger.js';
11
+ const log = createLogger('crawler');
10
12
  /** Safely compile a user-supplied regex pattern. Rejects patterns longer than 200 chars
11
13
  * and wraps compilation in a try-catch to prevent invalid regex crashes. */
12
14
  function safeRegex(pattern) {
@@ -140,7 +142,7 @@ export async function crawl(startUrl, options = {}) {
140
142
  robotsRules = await fetchRobotsTxt(startDomain);
141
143
  // Use crawl-delay from robots.txt if it's larger than our rate limit
142
144
  if (robotsRules.crawlDelay && robotsRules.crawlDelay > validatedRateLimit) {
143
- console.error(`[Crawler] Using Crawl-delay from robots.txt: ${robotsRules.crawlDelay}ms`);
145
+ log.info(`Using Crawl-delay from robots.txt: ${robotsRules.crawlDelay}ms`);
144
146
  }
145
147
  }
146
148
  const effectiveRateLimit = robotsRules.crawlDelay || validatedRateLimit;
@@ -155,7 +157,7 @@ export async function crawl(startUrl, options = {}) {
155
157
  // Load existing checkpoint if resume is requested
156
158
  const checkpoint = resume ? loadCheckpoint(jobId) : null;
157
159
  if (checkpoint) {
158
- console.error(`[Crawler] Resuming crawl from checkpoint: ${checkpoint.completed.size} pages already crawled`);
160
+ log.info(`Resuming crawl from checkpoint: ${checkpoint.completed.size} pages already crawled`);
159
161
  }
160
162
  // State tracking
161
163
  const results = [];
@@ -228,7 +230,7 @@ export async function crawl(startUrl, options = {}) {
228
230
  }
229
231
  // Check robots.txt
230
232
  if (respectRobotsTxt && !isAllowedByRobots(url, robotsRules)) {
231
- console.error(`[Crawler] Skipping ${url} (disallowed by robots.txt)`);
233
+ log.debug(`Skipping ${url} (disallowed by robots.txt)`);
232
234
  continue;
233
235
  }
234
236
  // Fetch the page
@@ -311,7 +313,7 @@ export async function crawl(startUrl, options = {}) {
311
313
  // Log error and continue
312
314
  failedCount++;
313
315
  const errorMessage = error instanceof Error ? error.message : 'Unknown error';
314
- console.error(`[Crawler] Failed to fetch ${url}: ${errorMessage}`);
316
+ log.error(`Failed to fetch ${url}: ${errorMessage}`);
315
317
  const errorResult = {
316
318
  url,
317
319
  title: '',
@@ -343,4 +345,3 @@ export async function crawl(startUrl, options = {}) {
343
345
  deleteCheckpoint(jobId);
344
346
  return results;
345
347
  }
346
- //# sourceMappingURL=crawler.js.map
@@ -24,4 +24,3 @@ export declare function isCycleTLSAvailable(): boolean;
24
24
  * Fetch a URL using CycleTLS with Chrome JA3 fingerprint spoofing.
25
25
  */
26
26
  export declare function cycleFetch(url: string, options?: CycleFetchOptions): Promise<CycleFetchResult>;
27
- //# sourceMappingURL=cycle-fetch.d.ts.map
@@ -96,4 +96,3 @@ export async function cycleFetch(url, options) {
96
96
  await cycleTLS.exit().catch(() => { });
97
97
  }
98
98
  }
99
- //# sourceMappingURL=cycle-fetch.js.map
@@ -72,4 +72,3 @@ export declare function extractComparedEntities(query: string): string[];
72
72
  */
73
73
  export declare function buildComparisonTable(content: string, entities: string[]): ComparisonData | undefined;
74
74
  export declare function deepFetch(options: DeepFetchOptions): Promise<DeepFetchResult>;
75
- //# sourceMappingURL=deep-fetch.d.ts.map
@@ -403,4 +403,3 @@ export async function deepFetch(options) {
403
403
  }
404
404
  return result;
405
405
  }
406
- //# sourceMappingURL=deep-fetch.js.map
@@ -68,4 +68,3 @@ export interface DesignAnalysis {
68
68
  * The page must already be navigated to the target URL.
69
69
  */
70
70
  export declare function extractDesignAnalysis(page: Page): Promise<DesignAnalysis>;
71
- //# sourceMappingURL=design-analysis.d.ts.map
@@ -488,4 +488,3 @@ export async function extractDesignAnalysis(page) {
488
488
  new Promise((_, reject) => setTimeout(() => reject(new Error('Design analysis timed out after 15s')), timeoutMs)),
489
489
  ]);
490
490
  }
491
- //# sourceMappingURL=design-analysis.js.map
@@ -36,4 +36,3 @@ export declare function buildDesignComparison(subjectUrl: string, referenceUrl:
36
36
  * both in parallel, then diffing the tokens into a structured DesignComparison.
37
37
  */
38
38
  export declare function compareDesigns(subjectPage: Page, referencePage: Page): Promise<DesignComparison>;
39
- //# sourceMappingURL=design-compare.d.ts.map
@@ -262,4 +262,3 @@ export async function compareDesigns(subjectPage, referencePage) {
262
262
  ]);
263
263
  return buildDesignComparison(subjectPage.url(), referencePage.url(), subjectAnalysis, referenceAnalysis);
264
264
  }
265
- //# sourceMappingURL=design-compare.js.map
@@ -59,4 +59,3 @@ export interface DiffChange {
59
59
  */
60
60
  export declare function diffUrl(url: string, options?: DiffOptions): Promise<DiffResult>;
61
61
  export { trackChange, getSnapshot } from './change-tracking.js';
62
- //# sourceMappingURL=diff.d.ts.map
package/dist/core/diff.js CHANGED
@@ -287,4 +287,3 @@ export async function diffUrl(url, options = {}) {
287
287
  }
288
288
  // ─── Re-export trackChange for CLI convenience ─────────────────────────────────
289
289
  export { trackChange, getSnapshot } from './change-tracking.js';
290
- //# sourceMappingURL=diff.js.map
@@ -18,4 +18,3 @@ export declare function cachedLookup(hostname: string, options: dns.LookupOption
18
18
  export declare function warmupDnsCache(domains?: string[]): Promise<void>;
19
19
  export declare function startDnsWarmup(): void;
20
20
  export declare function clearDnsCache(): void;
21
- //# sourceMappingURL=dns-cache.d.ts.map
@@ -181,4 +181,3 @@ export function startDnsWarmup() {
181
181
  export function clearDnsCache() {
182
182
  dnsCache.clear();
183
183
  }
184
- //# sourceMappingURL=dns-cache.js.map
@@ -21,4 +21,3 @@ export declare function extractDocumentToFormat(buffer: Buffer, options?: {
21
21
  contentType?: string;
22
22
  format?: DocumentFormat;
23
23
  }): Promise<DocumentExtractionResult>;
24
- //# sourceMappingURL=documents.d.ts.map
@@ -121,4 +121,3 @@ export async function extractDocumentToFormat(buffer, options = {}) {
121
121
  }
122
122
  throw new Error(`Unsupported document type: ${normalized || contentType || 'unknown'}`);
123
123
  }
124
- //# sourceMappingURL=documents.js.map
@@ -33,4 +33,3 @@ export declare function getDomainExtractor(url: string): DomainExtractor | null;
33
33
  * Returns null when no extractor matches or extraction fails.
34
34
  */
35
35
  export declare function extractDomainData(html: string, url: string): Promise<DomainExtractResult | null>;
36
- //# sourceMappingURL=domain-extractors.d.ts.map
@@ -2767,4 +2767,3 @@ Browse newsletters at:
2767
2767
  *WebPeel works best with individual Substack post URLs, not the root homepage.*`;
2768
2768
  return { domain: 'substack.com', type: 'homepage', structured, cleanContent };
2769
2769
  }
2770
- //# sourceMappingURL=domain-extractors.js.map
@@ -36,4 +36,3 @@ export interface InlineExtractResult {
36
36
  * @returns Extracted structured data + token usage
37
37
  */
38
38
  export declare function extractInlineJson(content: string, options: InlineExtractOptions): Promise<InlineExtractResult>;
39
- //# sourceMappingURL=extract-inline.d.ts.map
@@ -213,4 +213,3 @@ export async function extractInlineJson(content, options) {
213
213
  tokensUsed: result.usage,
214
214
  };
215
215
  }
216
- //# sourceMappingURL=extract-inline.js.map
@@ -36,4 +36,3 @@ export interface ListingItem {
36
36
  * ```
37
37
  */
38
38
  export declare function extractListings(html: string, url?: string): ListingItem[];
39
- //# sourceMappingURL=extract-listings.d.ts.map
@@ -459,4 +459,3 @@ export function extractListings(html, url) {
459
459
  }
460
460
  return items;
461
461
  }
462
- //# sourceMappingURL=extract-listings.js.map
@@ -7,4 +7,3 @@ import type { ExtractOptions } from '../types.js';
7
7
  */
8
8
  export declare function extractWithLLM(content: string, options: ExtractOptions): Promise<Record<string, any>>;
9
9
  export declare function extractStructured(html: string, options: ExtractOptions): Record<string, any>;
10
- //# sourceMappingURL=extract.d.ts.map
@@ -137,4 +137,3 @@ export function extractStructured(html, options) {
137
137
  }
138
138
  return result;
139
139
  }
140
- //# sourceMappingURL=extract.js.map
@@ -11,4 +11,3 @@ export { cleanup, warmup, closePool, closeProfileBrowser, playwrightLoaded } fro
11
11
  export { browserFetch, browserScreenshot, browserFilmstrip, browserAudit, browserAnimationCapture, browserViewports, browserDesignAudit, browserDiff, retryFetch, scrollAndWait, browserDesignAnalysis } from './browser-fetch.js';
12
12
  export type { DesignAuditResult } from './browser-fetch.js';
13
13
  export type { DesignAnalysis, EffectInstance } from './design-analysis.js';
14
- //# sourceMappingURL=fetcher.d.ts.map
@@ -10,4 +10,3 @@
10
10
  export { simpleFetch } from './http-fetch.js';
11
11
  export { cleanup, warmup, closePool, closeProfileBrowser, playwrightLoaded } from './browser-pool.js';
12
12
  export { browserFetch, browserScreenshot, browserFilmstrip, browserAudit, browserAnimationCapture, browserViewports, browserDesignAudit, browserDiff, retryFetch, scrollAndWait, browserDesignAnalysis } from './browser-fetch.js';
13
- //# sourceMappingURL=fetcher.js.map
@@ -27,4 +27,3 @@ export declare function fetchGoogleCache(url: string, options?: {
27
27
  * Google Cache is always available — no API key or special setup required.
28
28
  */
29
29
  export declare function isGoogleCacheAvailable(): boolean;
30
- //# sourceMappingURL=google-cache.d.ts.map
@@ -178,4 +178,3 @@ function findFirstCacheHr(html) {
178
178
  export function isGoogleCacheAvailable() {
179
179
  return true;
180
180
  }
181
- //# sourceMappingURL=google-cache.js.map
@@ -120,4 +120,3 @@ export declare function sortHotels(hotels: HotelResult[], sort: 'price' | 'ratin
120
120
  * Search multiple travel sites for hotels and return sorted, deduplicated results.
121
121
  */
122
122
  export declare function searchHotels(options: HotelSearchOptions): Promise<HotelSearchResult>;
123
- //# sourceMappingURL=hotel-search.d.ts.map