webpeel 0.19.4 → 0.20.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (544) hide show
  1. package/README.md +2 -2
  2. package/dist/cache.d.ts +0 -1
  3. package/dist/cache.js +0 -1
  4. package/dist/cli/commands/auth.d.ts +5 -0
  5. package/dist/cli/commands/auth.js +476 -0
  6. package/dist/cli/commands/fetch.d.ts +6 -0
  7. package/dist/cli/commands/fetch.js +1015 -0
  8. package/dist/cli/commands/interact.d.ts +5 -0
  9. package/dist/cli/commands/interact.js +839 -0
  10. package/dist/cli/commands/jobs.d.ts +5 -0
  11. package/dist/cli/commands/jobs.js +997 -0
  12. package/dist/cli/commands/screenshot.d.ts +5 -0
  13. package/dist/cli/commands/screenshot.js +273 -0
  14. package/dist/cli/commands/search.d.ts +5 -0
  15. package/dist/cli/commands/search.js +524 -0
  16. package/dist/cli/utils.d.ts +84 -0
  17. package/dist/cli/utils.js +686 -0
  18. package/dist/cli-auth.d.ts +0 -1
  19. package/dist/cli-auth.js +0 -1
  20. package/dist/cli.d.ts +7 -6
  21. package/dist/cli.js +35 -4698
  22. package/dist/core/actions.d.ts +0 -1
  23. package/dist/core/actions.js +0 -1
  24. package/dist/core/agent.d.ts +0 -1
  25. package/dist/core/agent.js +9 -12
  26. package/dist/core/answer.d.ts +0 -1
  27. package/dist/core/answer.js +0 -1
  28. package/dist/core/application-tracker.d.ts +0 -1
  29. package/dist/core/application-tracker.js +0 -1
  30. package/dist/core/apply.d.ts +0 -1
  31. package/dist/core/apply.js +0 -1
  32. package/dist/core/auto-extract.d.ts +0 -1
  33. package/dist/core/auto-extract.js +0 -1
  34. package/dist/core/auto-interact.d.ts +0 -1
  35. package/dist/core/auto-interact.js +0 -1
  36. package/dist/core/bm25-filter.d.ts +0 -1
  37. package/dist/core/bm25-filter.js +0 -1
  38. package/dist/core/branding.d.ts +0 -1
  39. package/dist/core/branding.js +0 -1
  40. package/dist/core/browser-fetch.d.ts +0 -1
  41. package/dist/core/browser-fetch.js +17 -10
  42. package/dist/core/browser-pool.d.ts +0 -1
  43. package/dist/core/browser-pool.js +0 -1
  44. package/dist/core/budget.d.ts +0 -1
  45. package/dist/core/budget.js +0 -1
  46. package/dist/core/cache.d.ts +0 -1
  47. package/dist/core/cache.js +0 -1
  48. package/dist/core/cf-worker-proxy.d.ts +0 -1
  49. package/dist/core/cf-worker-proxy.js +0 -1
  50. package/dist/core/challenge-detection.d.ts +0 -1
  51. package/dist/core/challenge-detection.js +0 -1
  52. package/dist/core/change-tracking.d.ts +0 -1
  53. package/dist/core/change-tracking.js +0 -1
  54. package/dist/core/chunker.d.ts +0 -1
  55. package/dist/core/chunker.js +0 -1
  56. package/dist/core/chunking.d.ts +0 -1
  57. package/dist/core/chunking.js +0 -1
  58. package/dist/core/cloak-fetch.d.ts +0 -1
  59. package/dist/core/cloak-fetch.js +0 -1
  60. package/dist/core/content-pruner.d.ts +0 -1
  61. package/dist/core/content-pruner.js +0 -1
  62. package/dist/core/crawl-checkpoint.d.ts +0 -1
  63. package/dist/core/crawl-checkpoint.js +0 -1
  64. package/dist/core/crawler.d.ts +0 -1
  65. package/dist/core/crawler.js +6 -5
  66. package/dist/core/cycle-fetch.d.ts +0 -1
  67. package/dist/core/cycle-fetch.js +0 -1
  68. package/dist/core/deep-fetch.d.ts +0 -1
  69. package/dist/core/deep-fetch.js +0 -1
  70. package/dist/core/design-analysis.d.ts +0 -1
  71. package/dist/core/design-analysis.js +0 -1
  72. package/dist/core/design-compare.d.ts +0 -1
  73. package/dist/core/design-compare.js +0 -1
  74. package/dist/core/diff.d.ts +0 -1
  75. package/dist/core/diff.js +0 -1
  76. package/dist/core/dns-cache.d.ts +0 -1
  77. package/dist/core/dns-cache.js +0 -1
  78. package/dist/core/documents.d.ts +0 -1
  79. package/dist/core/documents.js +0 -1
  80. package/dist/core/domain-extractors.d.ts +0 -1
  81. package/dist/core/domain-extractors.js +0 -1
  82. package/dist/core/extract-inline.d.ts +0 -1
  83. package/dist/core/extract-inline.js +0 -1
  84. package/dist/core/extract-listings.d.ts +0 -1
  85. package/dist/core/extract-listings.js +0 -1
  86. package/dist/core/extract.d.ts +0 -1
  87. package/dist/core/extract.js +0 -1
  88. package/dist/core/fetcher.d.ts +0 -1
  89. package/dist/core/fetcher.js +0 -1
  90. package/dist/core/google-cache.d.ts +0 -1
  91. package/dist/core/google-cache.js +0 -1
  92. package/dist/core/hotel-search.d.ts +0 -1
  93. package/dist/core/hotel-search.js +0 -1
  94. package/dist/core/http-fetch.d.ts +0 -1
  95. package/dist/core/http-fetch.js +5 -7
  96. package/dist/core/human.d.ts +0 -1
  97. package/dist/core/human.js +0 -1
  98. package/dist/core/jobs.d.ts +0 -1
  99. package/dist/core/jobs.js +0 -1
  100. package/dist/core/json-ld.d.ts +0 -1
  101. package/dist/core/json-ld.js +0 -1
  102. package/dist/core/llm-extract.d.ts +0 -1
  103. package/dist/core/llm-extract.js +0 -1
  104. package/dist/core/logger.d.ts +17 -0
  105. package/dist/core/logger.js +44 -0
  106. package/dist/core/map.d.ts +0 -1
  107. package/dist/core/map.js +0 -1
  108. package/dist/core/markdown.d.ts +0 -1
  109. package/dist/core/markdown.js +0 -1
  110. package/dist/core/metadata.d.ts +0 -1
  111. package/dist/core/metadata.js +0 -1
  112. package/dist/core/paginate.d.ts +0 -1
  113. package/dist/core/paginate.js +0 -1
  114. package/dist/core/pdf.d.ts +0 -1
  115. package/dist/core/pdf.js +0 -1
  116. package/dist/core/peel-tls.d.ts +0 -1
  117. package/dist/core/peel-tls.js +0 -1
  118. package/dist/core/pipeline.d.ts +0 -1
  119. package/dist/core/pipeline.js +22 -25
  120. package/dist/core/profiles.d.ts +0 -1
  121. package/dist/core/profiles.js +0 -1
  122. package/dist/core/quick-answer.d.ts +0 -1
  123. package/dist/core/quick-answer.js +0 -1
  124. package/dist/core/rate-governor.d.ts +0 -1
  125. package/dist/core/rate-governor.js +0 -1
  126. package/dist/core/readability.d.ts +0 -1
  127. package/dist/core/readability.js +0 -1
  128. package/dist/core/research.d.ts +0 -1
  129. package/dist/core/research.js +0 -1
  130. package/dist/core/schema-extraction.d.ts +0 -1
  131. package/dist/core/schema-extraction.js +0 -1
  132. package/dist/core/schema-postprocess.d.ts +0 -1
  133. package/dist/core/schema-postprocess.js +0 -1
  134. package/dist/core/schema-templates.d.ts +0 -1
  135. package/dist/core/schema-templates.js +0 -1
  136. package/dist/core/screenshot.d.ts +0 -1
  137. package/dist/core/screenshot.js +0 -1
  138. package/dist/core/search-fallback.d.ts +0 -1
  139. package/dist/core/search-fallback.js +0 -1
  140. package/dist/core/search-provider.d.ts +0 -1
  141. package/dist/core/search-provider.js +18 -21
  142. package/dist/core/site-search.d.ts +0 -1
  143. package/dist/core/site-search.js +0 -1
  144. package/dist/core/sitemap.d.ts +0 -1
  145. package/dist/core/sitemap.js +0 -1
  146. package/dist/core/stealth-patches.d.ts +0 -1
  147. package/dist/core/stealth-patches.js +0 -1
  148. package/dist/core/stemmer.d.ts +0 -1
  149. package/dist/core/stemmer.js +0 -1
  150. package/dist/core/strategies.d.ts +6 -1
  151. package/dist/core/strategies.js +29 -41
  152. package/dist/core/strategy-hooks.d.ts +0 -1
  153. package/dist/core/strategy-hooks.js +0 -1
  154. package/dist/core/summarize.d.ts +0 -1
  155. package/dist/core/summarize.js +0 -1
  156. package/dist/core/synonyms.d.ts +0 -1
  157. package/dist/core/synonyms.js +0 -1
  158. package/dist/core/table-format.d.ts +0 -1
  159. package/dist/core/table-format.js +0 -1
  160. package/dist/core/timing.d.ts +0 -1
  161. package/dist/core/timing.js +0 -1
  162. package/dist/core/user-agents.d.ts +0 -1
  163. package/dist/core/user-agents.js +0 -1
  164. package/dist/core/watch-manager.d.ts +0 -1
  165. package/dist/core/watch-manager.js +0 -1
  166. package/dist/core/watch.d.ts +0 -1
  167. package/dist/core/watch.js +0 -1
  168. package/dist/core/youtube.d.ts +0 -1
  169. package/dist/core/youtube.js +0 -1
  170. package/dist/index.d.ts +8 -3
  171. package/dist/index.js +27 -3
  172. package/dist/integrations/index.d.ts +0 -1
  173. package/dist/integrations/index.js +0 -1
  174. package/dist/integrations/langchain.d.ts +0 -1
  175. package/dist/integrations/langchain.js +0 -1
  176. package/dist/integrations/llamaindex.d.ts +0 -1
  177. package/dist/integrations/llamaindex.js +0 -1
  178. package/dist/mcp/handlers/act.d.ts +5 -0
  179. package/dist/mcp/handlers/act.js +34 -0
  180. package/dist/mcp/handlers/definitions.d.ts +6 -0
  181. package/dist/mcp/handlers/definitions.js +266 -0
  182. package/dist/mcp/handlers/extract.d.ts +6 -0
  183. package/dist/mcp/handlers/extract.js +102 -0
  184. package/dist/mcp/handlers/fetch.d.ts +6 -0
  185. package/dist/mcp/handlers/fetch.js +98 -0
  186. package/dist/mcp/handlers/find.d.ts +5 -0
  187. package/dist/mcp/handlers/find.js +137 -0
  188. package/dist/mcp/handlers/index.d.ts +13 -0
  189. package/dist/mcp/handlers/index.js +61 -0
  190. package/dist/mcp/handlers/legacy.d.ts +25 -0
  191. package/dist/mcp/handlers/legacy.js +450 -0
  192. package/dist/mcp/handlers/meta.d.ts +6 -0
  193. package/dist/mcp/handlers/meta.js +31 -0
  194. package/dist/mcp/handlers/monitor.d.ts +5 -0
  195. package/dist/mcp/handlers/monitor.js +41 -0
  196. package/dist/mcp/handlers/read.d.ts +6 -0
  197. package/dist/mcp/handlers/read.js +63 -0
  198. package/dist/mcp/handlers/see.d.ts +5 -0
  199. package/dist/mcp/handlers/see.js +75 -0
  200. package/dist/mcp/handlers/types.d.ts +29 -0
  201. package/dist/mcp/handlers/types.js +28 -0
  202. package/dist/mcp/server.d.ts +3 -4
  203. package/dist/mcp/server.js +35 -1101
  204. package/dist/mcp/smart-router.d.ts +0 -1
  205. package/dist/mcp/smart-router.js +3 -1
  206. package/dist/types.d.ts +6 -1
  207. package/dist/types.js +0 -1
  208. package/package.json +3 -13
  209. package/dist/cache.d.ts.map +0 -1
  210. package/dist/cache.js.map +0 -1
  211. package/dist/cli-auth.d.ts.map +0 -1
  212. package/dist/cli-auth.js.map +0 -1
  213. package/dist/cli.bundle.cjs +0 -159248
  214. package/dist/cli.d.ts.map +0 -1
  215. package/dist/cli.js.map +0 -1
  216. package/dist/core/actions.d.ts.map +0 -1
  217. package/dist/core/actions.js.map +0 -1
  218. package/dist/core/agent.d.ts.map +0 -1
  219. package/dist/core/agent.js.map +0 -1
  220. package/dist/core/answer.d.ts.map +0 -1
  221. package/dist/core/answer.js.map +0 -1
  222. package/dist/core/application-tracker.d.ts.map +0 -1
  223. package/dist/core/application-tracker.js.map +0 -1
  224. package/dist/core/apply.d.ts.map +0 -1
  225. package/dist/core/apply.js.map +0 -1
  226. package/dist/core/auto-extract.d.ts.map +0 -1
  227. package/dist/core/auto-extract.js.map +0 -1
  228. package/dist/core/auto-interact.d.ts.map +0 -1
  229. package/dist/core/auto-interact.js.map +0 -1
  230. package/dist/core/bm25-filter.d.ts.map +0 -1
  231. package/dist/core/bm25-filter.js.map +0 -1
  232. package/dist/core/branding.d.ts.map +0 -1
  233. package/dist/core/branding.js.map +0 -1
  234. package/dist/core/browser-fetch.d.ts.map +0 -1
  235. package/dist/core/browser-fetch.js.map +0 -1
  236. package/dist/core/browser-pool.d.ts.map +0 -1
  237. package/dist/core/browser-pool.js.map +0 -1
  238. package/dist/core/budget.d.ts.map +0 -1
  239. package/dist/core/budget.js.map +0 -1
  240. package/dist/core/cache.d.ts.map +0 -1
  241. package/dist/core/cache.js.map +0 -1
  242. package/dist/core/cf-worker-proxy.d.ts.map +0 -1
  243. package/dist/core/cf-worker-proxy.js.map +0 -1
  244. package/dist/core/challenge-detection.d.ts.map +0 -1
  245. package/dist/core/challenge-detection.js.map +0 -1
  246. package/dist/core/change-tracking.d.ts.map +0 -1
  247. package/dist/core/change-tracking.js.map +0 -1
  248. package/dist/core/chunker.d.ts.map +0 -1
  249. package/dist/core/chunker.js.map +0 -1
  250. package/dist/core/chunking.d.ts.map +0 -1
  251. package/dist/core/chunking.js.map +0 -1
  252. package/dist/core/cloak-fetch.d.ts.map +0 -1
  253. package/dist/core/cloak-fetch.js.map +0 -1
  254. package/dist/core/content-pruner.d.ts.map +0 -1
  255. package/dist/core/content-pruner.js.map +0 -1
  256. package/dist/core/crawl-checkpoint.d.ts.map +0 -1
  257. package/dist/core/crawl-checkpoint.js.map +0 -1
  258. package/dist/core/crawler.d.ts.map +0 -1
  259. package/dist/core/crawler.js.map +0 -1
  260. package/dist/core/cycle-fetch.d.ts.map +0 -1
  261. package/dist/core/cycle-fetch.js.map +0 -1
  262. package/dist/core/deep-fetch.d.ts.map +0 -1
  263. package/dist/core/deep-fetch.js.map +0 -1
  264. package/dist/core/design-analysis.d.ts.map +0 -1
  265. package/dist/core/design-analysis.js.map +0 -1
  266. package/dist/core/design-compare.d.ts.map +0 -1
  267. package/dist/core/design-compare.js.map +0 -1
  268. package/dist/core/diff.d.ts.map +0 -1
  269. package/dist/core/diff.js.map +0 -1
  270. package/dist/core/dns-cache.d.ts.map +0 -1
  271. package/dist/core/dns-cache.js.map +0 -1
  272. package/dist/core/documents.d.ts.map +0 -1
  273. package/dist/core/documents.js.map +0 -1
  274. package/dist/core/domain-extractors.d.ts.map +0 -1
  275. package/dist/core/domain-extractors.js.map +0 -1
  276. package/dist/core/extract-inline.d.ts.map +0 -1
  277. package/dist/core/extract-inline.js.map +0 -1
  278. package/dist/core/extract-listings.d.ts.map +0 -1
  279. package/dist/core/extract-listings.js.map +0 -1
  280. package/dist/core/extract.d.ts.map +0 -1
  281. package/dist/core/extract.js.map +0 -1
  282. package/dist/core/fetcher.d.ts.map +0 -1
  283. package/dist/core/fetcher.js.map +0 -1
  284. package/dist/core/google-cache.d.ts.map +0 -1
  285. package/dist/core/google-cache.js.map +0 -1
  286. package/dist/core/hotel-search.d.ts.map +0 -1
  287. package/dist/core/hotel-search.js.map +0 -1
  288. package/dist/core/http-fetch.d.ts.map +0 -1
  289. package/dist/core/http-fetch.js.map +0 -1
  290. package/dist/core/human.d.ts.map +0 -1
  291. package/dist/core/human.js.map +0 -1
  292. package/dist/core/jobs.d.ts.map +0 -1
  293. package/dist/core/jobs.js.map +0 -1
  294. package/dist/core/json-ld.d.ts.map +0 -1
  295. package/dist/core/json-ld.js.map +0 -1
  296. package/dist/core/llm-extract.d.ts.map +0 -1
  297. package/dist/core/llm-extract.js.map +0 -1
  298. package/dist/core/map.d.ts.map +0 -1
  299. package/dist/core/map.js.map +0 -1
  300. package/dist/core/markdown.d.ts.map +0 -1
  301. package/dist/core/markdown.js.map +0 -1
  302. package/dist/core/metadata.d.ts.map +0 -1
  303. package/dist/core/metadata.js.map +0 -1
  304. package/dist/core/paginate.d.ts.map +0 -1
  305. package/dist/core/paginate.js.map +0 -1
  306. package/dist/core/pdf.d.ts.map +0 -1
  307. package/dist/core/pdf.js.map +0 -1
  308. package/dist/core/peel-tls.d.ts.map +0 -1
  309. package/dist/core/peel-tls.js.map +0 -1
  310. package/dist/core/pipeline.d.ts.map +0 -1
  311. package/dist/core/pipeline.js.map +0 -1
  312. package/dist/core/profiles.d.ts.map +0 -1
  313. package/dist/core/profiles.js.map +0 -1
  314. package/dist/core/quick-answer.d.ts.map +0 -1
  315. package/dist/core/quick-answer.js.map +0 -1
  316. package/dist/core/rate-governor.d.ts.map +0 -1
  317. package/dist/core/rate-governor.js.map +0 -1
  318. package/dist/core/readability.d.ts.map +0 -1
  319. package/dist/core/readability.js.map +0 -1
  320. package/dist/core/research.d.ts.map +0 -1
  321. package/dist/core/research.js.map +0 -1
  322. package/dist/core/schema-extraction.d.ts.map +0 -1
  323. package/dist/core/schema-extraction.js.map +0 -1
  324. package/dist/core/schema-postprocess.d.ts.map +0 -1
  325. package/dist/core/schema-postprocess.js.map +0 -1
  326. package/dist/core/schema-templates.d.ts.map +0 -1
  327. package/dist/core/schema-templates.js.map +0 -1
  328. package/dist/core/screenshot.d.ts.map +0 -1
  329. package/dist/core/screenshot.js.map +0 -1
  330. package/dist/core/search-fallback.d.ts.map +0 -1
  331. package/dist/core/search-fallback.js.map +0 -1
  332. package/dist/core/search-provider.d.ts.map +0 -1
  333. package/dist/core/search-provider.js.map +0 -1
  334. package/dist/core/site-search.d.ts.map +0 -1
  335. package/dist/core/site-search.js.map +0 -1
  336. package/dist/core/sitemap.d.ts.map +0 -1
  337. package/dist/core/sitemap.js.map +0 -1
  338. package/dist/core/stealth-patches.d.ts.map +0 -1
  339. package/dist/core/stealth-patches.js.map +0 -1
  340. package/dist/core/stemmer.d.ts.map +0 -1
  341. package/dist/core/stemmer.js.map +0 -1
  342. package/dist/core/strategies.d.ts.map +0 -1
  343. package/dist/core/strategies.js.map +0 -1
  344. package/dist/core/strategy-hooks.d.ts.map +0 -1
  345. package/dist/core/strategy-hooks.js.map +0 -1
  346. package/dist/core/summarize.d.ts.map +0 -1
  347. package/dist/core/summarize.js.map +0 -1
  348. package/dist/core/synonyms.d.ts.map +0 -1
  349. package/dist/core/synonyms.js.map +0 -1
  350. package/dist/core/table-format.d.ts.map +0 -1
  351. package/dist/core/table-format.js.map +0 -1
  352. package/dist/core/timing.d.ts.map +0 -1
  353. package/dist/core/timing.js.map +0 -1
  354. package/dist/core/user-agents.d.ts.map +0 -1
  355. package/dist/core/user-agents.js.map +0 -1
  356. package/dist/core/watch-manager.d.ts.map +0 -1
  357. package/dist/core/watch-manager.js.map +0 -1
  358. package/dist/core/watch.d.ts.map +0 -1
  359. package/dist/core/watch.js.map +0 -1
  360. package/dist/core/youtube.d.ts.map +0 -1
  361. package/dist/core/youtube.js.map +0 -1
  362. package/dist/index.d.ts.map +0 -1
  363. package/dist/index.js.map +0 -1
  364. package/dist/integrations/index.d.ts.map +0 -1
  365. package/dist/integrations/index.js.map +0 -1
  366. package/dist/integrations/langchain.d.ts.map +0 -1
  367. package/dist/integrations/langchain.js.map +0 -1
  368. package/dist/integrations/llamaindex.d.ts.map +0 -1
  369. package/dist/integrations/llamaindex.js.map +0 -1
  370. package/dist/mcp/server.d.ts.map +0 -1
  371. package/dist/mcp/server.js.map +0 -1
  372. package/dist/mcp/smart-router.d.ts.map +0 -1
  373. package/dist/mcp/smart-router.js.map +0 -1
  374. package/dist/server/app.d.ts +0 -15
  375. package/dist/server/app.d.ts.map +0 -1
  376. package/dist/server/app.js +0 -350
  377. package/dist/server/app.js.map +0 -1
  378. package/dist/server/auth-store.d.ts +0 -28
  379. package/dist/server/auth-store.d.ts.map +0 -1
  380. package/dist/server/auth-store.js +0 -89
  381. package/dist/server/auth-store.js.map +0 -1
  382. package/dist/server/email-service.d.ts +0 -22
  383. package/dist/server/email-service.d.ts.map +0 -1
  384. package/dist/server/email-service.js +0 -80
  385. package/dist/server/email-service.js.map +0 -1
  386. package/dist/server/job-queue.d.ts +0 -93
  387. package/dist/server/job-queue.d.ts.map +0 -1
  388. package/dist/server/job-queue.js +0 -146
  389. package/dist/server/job-queue.js.map +0 -1
  390. package/dist/server/logger.d.ts +0 -11
  391. package/dist/server/logger.d.ts.map +0 -1
  392. package/dist/server/logger.js +0 -38
  393. package/dist/server/logger.js.map +0 -1
  394. package/dist/server/middleware/auth.d.ts +0 -29
  395. package/dist/server/middleware/auth.d.ts.map +0 -1
  396. package/dist/server/middleware/auth.js +0 -222
  397. package/dist/server/middleware/auth.js.map +0 -1
  398. package/dist/server/middleware/rate-limit.d.ts +0 -25
  399. package/dist/server/middleware/rate-limit.d.ts.map +0 -1
  400. package/dist/server/middleware/rate-limit.js +0 -168
  401. package/dist/server/middleware/rate-limit.js.map +0 -1
  402. package/dist/server/middleware/url-validator.d.ts +0 -16
  403. package/dist/server/middleware/url-validator.d.ts.map +0 -1
  404. package/dist/server/middleware/url-validator.js +0 -187
  405. package/dist/server/middleware/url-validator.js.map +0 -1
  406. package/dist/server/openapi.yaml +0 -4944
  407. package/dist/server/pg-auth-store.d.ts +0 -133
  408. package/dist/server/pg-auth-store.d.ts.map +0 -1
  409. package/dist/server/pg-auth-store.js +0 -473
  410. package/dist/server/pg-auth-store.js.map +0 -1
  411. package/dist/server/pg-job-queue.d.ts +0 -60
  412. package/dist/server/pg-job-queue.d.ts.map +0 -1
  413. package/dist/server/pg-job-queue.js +0 -365
  414. package/dist/server/pg-job-queue.js.map +0 -1
  415. package/dist/server/premium/domain-intel.d.ts +0 -17
  416. package/dist/server/premium/domain-intel.d.ts.map +0 -1
  417. package/dist/server/premium/domain-intel.js +0 -134
  418. package/dist/server/premium/domain-intel.js.map +0 -1
  419. package/dist/server/premium/index.d.ts +0 -18
  420. package/dist/server/premium/index.d.ts.map +0 -1
  421. package/dist/server/premium/index.js +0 -36
  422. package/dist/server/premium/index.js.map +0 -1
  423. package/dist/server/premium/swr-cache.d.ts +0 -15
  424. package/dist/server/premium/swr-cache.d.ts.map +0 -1
  425. package/dist/server/premium/swr-cache.js +0 -35
  426. package/dist/server/premium/swr-cache.js.map +0 -1
  427. package/dist/server/routes/activity.d.ts +0 -7
  428. package/dist/server/routes/activity.d.ts.map +0 -1
  429. package/dist/server/routes/activity.js +0 -68
  430. package/dist/server/routes/activity.js.map +0 -1
  431. package/dist/server/routes/agent.d.ts +0 -16
  432. package/dist/server/routes/agent.d.ts.map +0 -1
  433. package/dist/server/routes/agent.js +0 -247
  434. package/dist/server/routes/agent.js.map +0 -1
  435. package/dist/server/routes/answer.d.ts +0 -6
  436. package/dist/server/routes/answer.d.ts.map +0 -1
  437. package/dist/server/routes/answer.js +0 -133
  438. package/dist/server/routes/answer.js.map +0 -1
  439. package/dist/server/routes/ask.d.ts +0 -23
  440. package/dist/server/routes/ask.d.ts.map +0 -1
  441. package/dist/server/routes/ask.js +0 -119
  442. package/dist/server/routes/ask.js.map +0 -1
  443. package/dist/server/routes/batch.d.ts +0 -7
  444. package/dist/server/routes/batch.d.ts.map +0 -1
  445. package/dist/server/routes/batch.js +0 -412
  446. package/dist/server/routes/batch.js.map +0 -1
  447. package/dist/server/routes/cli-usage.d.ts +0 -7
  448. package/dist/server/routes/cli-usage.d.ts.map +0 -1
  449. package/dist/server/routes/cli-usage.js +0 -121
  450. package/dist/server/routes/cli-usage.js.map +0 -1
  451. package/dist/server/routes/compat.d.ts +0 -24
  452. package/dist/server/routes/compat.d.ts.map +0 -1
  453. package/dist/server/routes/compat.js +0 -653
  454. package/dist/server/routes/compat.js.map +0 -1
  455. package/dist/server/routes/deep-fetch.d.ts +0 -9
  456. package/dist/server/routes/deep-fetch.d.ts.map +0 -1
  457. package/dist/server/routes/deep-fetch.js +0 -50
  458. package/dist/server/routes/deep-fetch.js.map +0 -1
  459. package/dist/server/routes/demo.d.ts +0 -25
  460. package/dist/server/routes/demo.d.ts.map +0 -1
  461. package/dist/server/routes/demo.js +0 -434
  462. package/dist/server/routes/demo.js.map +0 -1
  463. package/dist/server/routes/extract.d.ts +0 -9
  464. package/dist/server/routes/extract.d.ts.map +0 -1
  465. package/dist/server/routes/extract.js +0 -150
  466. package/dist/server/routes/extract.js.map +0 -1
  467. package/dist/server/routes/fetch.d.ts +0 -8
  468. package/dist/server/routes/fetch.d.ts.map +0 -1
  469. package/dist/server/routes/fetch.js +0 -988
  470. package/dist/server/routes/fetch.js.map +0 -1
  471. package/dist/server/routes/health.d.ts +0 -8
  472. package/dist/server/routes/health.d.ts.map +0 -1
  473. package/dist/server/routes/health.js +0 -20
  474. package/dist/server/routes/health.js.map +0 -1
  475. package/dist/server/routes/jobs.d.ts +0 -8
  476. package/dist/server/routes/jobs.d.ts.map +0 -1
  477. package/dist/server/routes/jobs.js +0 -487
  478. package/dist/server/routes/jobs.js.map +0 -1
  479. package/dist/server/routes/mcp.d.ts +0 -18
  480. package/dist/server/routes/mcp.d.ts.map +0 -1
  481. package/dist/server/routes/mcp.js +0 -1260
  482. package/dist/server/routes/mcp.js.map +0 -1
  483. package/dist/server/routes/oauth.d.ts +0 -10
  484. package/dist/server/routes/oauth.d.ts.map +0 -1
  485. package/dist/server/routes/oauth.js +0 -334
  486. package/dist/server/routes/oauth.js.map +0 -1
  487. package/dist/server/routes/quick-answer.d.ts +0 -9
  488. package/dist/server/routes/quick-answer.d.ts.map +0 -1
  489. package/dist/server/routes/quick-answer.js +0 -93
  490. package/dist/server/routes/quick-answer.js.map +0 -1
  491. package/dist/server/routes/screenshot.d.ts +0 -23
  492. package/dist/server/routes/screenshot.d.ts.map +0 -1
  493. package/dist/server/routes/screenshot.js +0 -819
  494. package/dist/server/routes/screenshot.js.map +0 -1
  495. package/dist/server/routes/search.d.ts +0 -7
  496. package/dist/server/routes/search.d.ts.map +0 -1
  497. package/dist/server/routes/search.js +0 -312
  498. package/dist/server/routes/search.js.map +0 -1
  499. package/dist/server/routes/session.d.ts +0 -16
  500. package/dist/server/routes/session.d.ts.map +0 -1
  501. package/dist/server/routes/session.js +0 -278
  502. package/dist/server/routes/session.js.map +0 -1
  503. package/dist/server/routes/stats.d.ts +0 -7
  504. package/dist/server/routes/stats.d.ts.map +0 -1
  505. package/dist/server/routes/stats.js +0 -65
  506. package/dist/server/routes/stats.js.map +0 -1
  507. package/dist/server/routes/stripe.d.ts +0 -16
  508. package/dist/server/routes/stripe.d.ts.map +0 -1
  509. package/dist/server/routes/stripe.js +0 -283
  510. package/dist/server/routes/stripe.js.map +0 -1
  511. package/dist/server/routes/users.d.ts +0 -9
  512. package/dist/server/routes/users.d.ts.map +0 -1
  513. package/dist/server/routes/users.js +0 -1211
  514. package/dist/server/routes/users.js.map +0 -1
  515. package/dist/server/routes/watch.d.ts +0 -16
  516. package/dist/server/routes/watch.d.ts.map +0 -1
  517. package/dist/server/routes/watch.js +0 -257
  518. package/dist/server/routes/watch.js.map +0 -1
  519. package/dist/server/routes/webhooks.d.ts +0 -16
  520. package/dist/server/routes/webhooks.d.ts.map +0 -1
  521. package/dist/server/routes/webhooks.js +0 -74
  522. package/dist/server/routes/webhooks.js.map +0 -1
  523. package/dist/server/routes/youtube.d.ts +0 -7
  524. package/dist/server/routes/youtube.d.ts.map +0 -1
  525. package/dist/server/routes/youtube.js +0 -93
  526. package/dist/server/routes/youtube.js.map +0 -1
  527. package/dist/server/sentry.d.ts +0 -14
  528. package/dist/server/sentry.d.ts.map +0 -1
  529. package/dist/server/sentry.js +0 -39
  530. package/dist/server/sentry.js.map +0 -1
  531. package/dist/server/types.d.ts +0 -16
  532. package/dist/server/types.d.ts.map +0 -1
  533. package/dist/server/types.js +0 -8
  534. package/dist/server/types.js.map +0 -1
  535. package/dist/server/utils/response.d.ts +0 -45
  536. package/dist/server/utils/response.d.ts.map +0 -1
  537. package/dist/server/utils/response.js +0 -70
  538. package/dist/server/utils/response.js.map +0 -1
  539. package/dist/server/utils/sse.d.ts +0 -23
  540. package/dist/server/utils/sse.d.ts.map +0 -1
  541. package/dist/server/utils/sse.js +0 -39
  542. package/dist/server/utils/sse.js.map +0 -1
  543. package/dist/types.d.ts.map +0 -1
  544. package/dist/types.js.map +0 -1
@@ -380,4 +380,3 @@ export async function searchHotels(options) {
380
380
  elapsed,
381
381
  };
382
382
  }
383
- //# sourceMappingURL=hotel-search.js.map
@@ -38,4 +38,3 @@ export interface FetchResult {
38
38
  * SECURITY: Manual redirect handling with SSRF re-validation
39
39
  */
40
40
  export declare function simpleFetch(url: string, userAgent?: string, timeoutMs?: number, customHeaders?: Record<string, string>, abortSignal?: AbortSignal, proxy?: string): Promise<FetchResult>;
41
- //# sourceMappingURL=http-fetch.d.ts.map
@@ -14,6 +14,8 @@ import { TimeoutError, BlockedError, NetworkError, WebPeelError } from '../types
14
14
  import { getCached } from './cache.js';
15
15
  import { cachedLookup, resolveAndCache, startDnsWarmup } from './dns-cache.js';
16
16
  import { detectChallenge } from './challenge-detection.js';
17
+ import { createLogger } from './logger.js';
18
+ const log = createLogger('http');
17
19
  // ── HTTP status text fallbacks (HTTP/2 omits reason phrases) ──────────────────
18
20
  const HTTP_STATUS_TEXT = {
19
21
  400: 'Bad Request',
@@ -81,8 +83,7 @@ function normalizeUrlForConditionalCache(url) {
81
83
  }
82
84
  catch (e) {
83
85
  // Non-fatal: URL normalization failed, returning raw trimmed URL
84
- if (process.env.DEBUG)
85
- console.debug('[webpeel]', 'URL normalization:', e instanceof Error ? e.message : e);
86
+ log.debug('URL normalization:', e instanceof Error ? e.message : e);
86
87
  return url.trim();
87
88
  }
88
89
  }
@@ -409,8 +410,7 @@ export async function simpleFetch(url, userAgent, timeoutMs = 30000, customHeade
409
410
  }
410
411
  catch (e) {
411
412
  // Ignore URL parsing errors here; validation handles invalid input below.
412
- if (process.env.DEBUG)
413
- console.debug('[webpeel]', 'DNS prefetch (initial URL):', e instanceof Error ? e.message : e);
413
+ log.debug('DNS prefetch (initial URL):', e instanceof Error ? e.message : e);
414
414
  }
415
415
  while (redirectCount <= MAX_REDIRECTS) {
416
416
  // Detect redirect loops
@@ -466,8 +466,7 @@ export async function simpleFetch(url, userAgent, timeoutMs = 30000, customHeade
466
466
  }
467
467
  catch (e) {
468
468
  // Ignore URL parsing errors here; validation handles invalid input below.
469
- if (process.env.DEBUG)
470
- console.debug('[webpeel]', 'DNS prefetch (redirect URL):', e instanceof Error ? e.message : e);
469
+ log.debug('DNS prefetch (redirect URL):', e instanceof Error ? e.message : e);
471
470
  }
472
471
  redirectCount++;
473
472
  continue;
@@ -627,4 +626,3 @@ export async function simpleFetch(url, userAgent, timeoutMs = 30000, customHeade
627
626
  }
628
627
  throw new WebPeelError(`Too many redirects (max ${MAX_REDIRECTS})`);
629
628
  }
630
- //# sourceMappingURL=http-fetch.js.map
@@ -173,4 +173,3 @@ export declare function humanRead(page: Page, durationMs?: number): Promise<void
173
173
  * @param durationMs How long to browse (default: 30000-60000 random)
174
174
  */
175
175
  export declare function warmupBrowse(page: Page, durationMs?: number): Promise<void>;
176
- //# sourceMappingURL=human.d.ts.map
@@ -678,4 +678,3 @@ export async function warmupBrowse(page, durationMs) {
678
678
  site = 'upwork';
679
679
  await warmupSession(page, site, durationMs);
680
680
  }
681
- //# sourceMappingURL=human.js.map
@@ -73,4 +73,3 @@ export interface JobSearchResult {
73
73
  * browser instance is shared across the library).
74
74
  */
75
75
  export declare function searchJobs(options: JobSearchOptions): Promise<JobSearchResult>;
76
- //# sourceMappingURL=jobs.d.ts.map
package/dist/core/jobs.js CHANGED
@@ -632,4 +632,3 @@ export async function searchJobs(options) {
632
632
  timeTakenMs: Date.now() - startTime,
633
633
  };
634
634
  }
635
- //# sourceMappingURL=jobs.js.map
@@ -13,4 +13,3 @@ export interface JsonLdResult {
13
13
  data: any;
14
14
  }
15
15
  export declare function extractJsonLd(html: string): JsonLdResult | null;
16
- //# sourceMappingURL=json-ld.d.ts.map
@@ -615,4 +615,3 @@ function convertReview(item) {
615
615
  data: item,
616
616
  };
617
617
  }
618
- //# sourceMappingURL=json-ld.js.map
@@ -53,4 +53,3 @@ export declare function parseItems(text: string, _schema?: object): Array<Record
53
53
  * Extract structured data from content using an LLM.
54
54
  */
55
55
  export declare function extractWithLLM(options: LLMExtractionOptions): Promise<LLMExtractionResult>;
56
- //# sourceMappingURL=llm-extract.d.ts.map
@@ -261,4 +261,3 @@ export async function extractWithLLM(options) {
261
261
  cost,
262
262
  };
263
263
  }
264
- //# sourceMappingURL=llm-extract.js.map
@@ -0,0 +1,17 @@
1
+ /**
2
+ * Lightweight structured logger — no external dependencies.
3
+ *
4
+ * Levels: debug < info < warn < error < silent
5
+ *
6
+ * Respects WEBPEEL_LOG_LEVEL env var.
7
+ * Defaults: production → 'info', development → 'debug'.
8
+ *
9
+ * All output goes to stderr so stdout stays clean for data/JSON (CLI piping).
10
+ */
11
+ export type LogLevel = 'debug' | 'info' | 'warn' | 'error' | 'silent';
12
+ export declare function createLogger(module: string): {
13
+ debug: (...args: any[]) => void;
14
+ info: (...args: any[]) => void;
15
+ warn: (...args: any[]) => void;
16
+ error: (...args: any[]) => void;
17
+ };
@@ -0,0 +1,44 @@
1
+ /**
2
+ * Lightweight structured logger — no external dependencies.
3
+ *
4
+ * Levels: debug < info < warn < error < silent
5
+ *
6
+ * Respects WEBPEEL_LOG_LEVEL env var.
7
+ * Defaults: production → 'info', development → 'debug'.
8
+ *
9
+ * All output goes to stderr so stdout stays clean for data/JSON (CLI piping).
10
+ */
11
+ const LEVELS = {
12
+ debug: 0,
13
+ info: 1,
14
+ warn: 2,
15
+ error: 3,
16
+ silent: 4,
17
+ };
18
+ const currentLevel = () => {
19
+ const env = process.env.WEBPEEL_LOG_LEVEL?.toLowerCase();
20
+ if (env && env in LEVELS)
21
+ return env;
22
+ return process.env.NODE_ENV === 'production' ? 'info' : 'debug';
23
+ };
24
+ export function createLogger(module) {
25
+ const prefix = `[webpeel:${module}]`;
26
+ return {
27
+ debug: (...args) => {
28
+ if (LEVELS[currentLevel()] <= LEVELS.debug)
29
+ console.error(prefix, ...args);
30
+ },
31
+ info: (...args) => {
32
+ if (LEVELS[currentLevel()] <= LEVELS.info)
33
+ console.error(prefix, ...args);
34
+ },
35
+ warn: (...args) => {
36
+ if (LEVELS[currentLevel()] <= LEVELS.warn)
37
+ console.error(prefix, '[WARN]', ...args);
38
+ },
39
+ error: (...args) => {
40
+ if (LEVELS[currentLevel()] <= LEVELS.error)
41
+ console.error(prefix, '[ERROR]', ...args);
42
+ },
43
+ };
44
+ }
@@ -31,4 +31,3 @@ export interface MapResult {
31
31
  elapsed: number;
32
32
  }
33
33
  export declare function mapDomain(startUrl: string, options?: MapOptions): Promise<MapResult>;
34
- //# sourceMappingURL=map.d.ts.map
package/dist/core/map.js CHANGED
@@ -125,4 +125,3 @@ export async function mapDomain(startUrl, options = {}) {
125
125
  elapsed: Date.now() - startTime,
126
126
  };
127
127
  }
128
- //# sourceMappingURL=map.js.map
@@ -90,4 +90,3 @@ export declare function cleanForAI(markdown: string): string;
90
90
  * Removes empty links, orphaned image links, collapses excess newlines, strips trailing whitespace.
91
91
  */
92
92
  export declare function cleanMarkdownNoise(content: string): string;
93
- //# sourceMappingURL=markdown.d.ts.map
@@ -774,4 +774,3 @@ export function cleanMarkdownNoise(content) {
774
774
  result = result.replace(/\n{3,}/g, '\n\n').trim();
775
775
  return result;
776
776
  }
777
- //# sourceMappingURL=markdown.js.map
@@ -25,4 +25,3 @@ export declare function extractMetadata(html: string, _url: string): {
25
25
  title: string;
26
26
  metadata: PageMetadata;
27
27
  };
28
- //# sourceMappingURL=metadata.d.ts.map
@@ -394,4 +394,3 @@ export function extractMetadata(html, _url) {
394
394
  };
395
395
  return { title, metadata };
396
396
  }
397
- //# sourceMappingURL=metadata.js.map
@@ -29,4 +29,3 @@
29
29
  * ```
30
30
  */
31
31
  export declare function findNextPageUrl(html: string, currentUrl: string): string | null;
32
- //# sourceMappingURL=paginate.d.ts.map
@@ -104,4 +104,3 @@ function resolve(href, base) {
104
104
  return null;
105
105
  }
106
106
  }
107
- //# sourceMappingURL=paginate.js.map
@@ -6,4 +6,3 @@ export declare function extractPdf(buffer: Buffer): Promise<{
6
6
  metadata: Record<string, any>;
7
7
  pages: number;
8
8
  }>;
9
- //# sourceMappingURL=pdf.d.ts.map
package/dist/core/pdf.js CHANGED
@@ -23,4 +23,3 @@ export async function extractPdf(buffer) {
23
23
  throw new Error(`PDF parsing failed: ${error instanceof Error ? error.message : 'Unknown error'}. Install pdf-parse: npm install pdf-parse`);
24
24
  }
25
25
  }
26
- //# sourceMappingURL=pdf.js.map
@@ -23,4 +23,3 @@ export declare function isPeelTLSAvailable(): boolean;
23
23
  export declare function peelTLSFetch(url: string, options?: PeelTLSOptions): Promise<PeelTLSResult>;
24
24
  /** Gracefully shut down the PeelTLS binary process. */
25
25
  export declare function shutdownPeelTLS(): Promise<void>;
26
- //# sourceMappingURL=peel-tls.d.ts.map
@@ -218,4 +218,3 @@ process.on('SIGTERM', () => {
218
218
  binaryProcess?.kill('SIGTERM');
219
219
  process.exit(0);
220
220
  });
221
- //# sourceMappingURL=peel-tls.js.map
@@ -120,4 +120,3 @@ export declare function finalize(ctx: PipelineContext): Promise<void>;
120
120
  * Assemble the final PeelResult from the pipeline context.
121
121
  */
122
122
  export declare function buildResult(ctx: PipelineContext): PeelResult;
123
- //# sourceMappingURL=pipeline.d.ts.map
@@ -20,6 +20,8 @@ import { quickAnswer as runQuickAnswer } from './quick-answer.js';
20
20
  import { Timer } from './timing.js';
21
21
  import { chunkContent } from './chunker.js';
22
22
  import { BlockedError } from '../types.js';
23
+ import { createLogger } from './logger.js';
24
+ const log = createLogger('pipeline');
23
25
  /** Create the initial PipelineContext with defaults */
24
26
  export function createContext(url, options) {
25
27
  return {
@@ -263,8 +265,7 @@ export async function fetchContent(ctx) {
263
265
  }
264
266
  catch (e) {
265
267
  // Domain API failed — fall through to normal fetch
266
- if (process.env.DEBUG)
267
- console.debug('[webpeel]', 'domain API first-pass failed, falling back to fetch:', e instanceof Error ? e.message : e);
268
+ log.debug('domain API first-pass failed, falling back to fetch:', e instanceof Error ? e.message : e);
268
269
  }
269
270
  }
270
271
  ctx.timer.mark('fetch');
@@ -295,6 +296,7 @@ export async function fetchContent(ctx) {
295
296
  blockResources: ctx.options.blockResources,
296
297
  cloaked: ctx.options.cloaked,
297
298
  cycle: ctx.options.cycle,
299
+ noEscalate: ctx.options.noEscalate,
298
300
  });
299
301
  }
300
302
  catch (fetchError) {
@@ -333,6 +335,7 @@ export async function fetchContent(ctx) {
333
335
  // to get the title/snippet from the search engine's cached version.
334
336
  if (fetchError instanceof BlockedError) {
335
337
  try {
338
+ // @ts-ignore — proprietary module, gitignored
336
339
  const { searchFallback } = await import('./search-fallback.js');
337
340
  const searchResult = await searchFallback(ctx.url);
338
341
  if (searchResult.cachedContent && searchResult.cachedContent.length > 50) {
@@ -371,8 +374,7 @@ export async function fetchContent(ctx) {
371
374
  }
372
375
  catch (e) {
373
376
  // Non-fatal: auto-scroll failed, continuing with whatever HTML we have
374
- if (process.env.DEBUG)
375
- console.debug('[webpeel]', 'auto-scroll failed:', e instanceof Error ? e.message : e);
377
+ log.debug('auto-scroll failed:', e instanceof Error ? e.message : e);
376
378
  }
377
379
  finally {
378
380
  // Close page unless branding or design analysis also needs it
@@ -385,8 +387,7 @@ export async function fetchContent(ctx) {
385
387
  }
386
388
  catch (e) {
387
389
  // Non-fatal: page/browser cleanup after auto-scroll
388
- if (process.env.DEBUG)
389
- console.debug('[webpeel]', 'page/browser cleanup after auto-scroll:', e instanceof Error ? e.message : e);
390
+ log.debug('page/browser cleanup after auto-scroll:', e instanceof Error ? e.message : e);
390
391
  }
391
392
  fetchResult.page = undefined;
392
393
  }
@@ -619,7 +620,7 @@ export async function parseContent(ctx) {
619
620
  htmlForConvert = contentHtml.slice(0, minChars);
620
621
  }
621
622
  if (process.env.DEBUG) {
622
- console.debug('[webpeel]', `budget pre-truncate: ${contentHtml.length} → ${htmlForConvert.length} chars`);
623
+ log.debug(`budget pre-truncate: ${contentHtml.length} → ${htmlForConvert.length} chars`);
623
624
  }
624
625
  }
625
626
  }
@@ -659,7 +660,7 @@ export async function parseContent(ctx) {
659
660
  // (common for listing/index pages, SPAs with shell-first layouts).
660
661
  if (htmlForConvert !== contentHtml && convertedContent.length < 200 && contentHtml.length > 20000) {
661
662
  if (process.env.DEBUG) {
662
- console.debug('[webpeel]', `budget pre-truncation produced thin content (${convertedContent.length} chars from ${htmlForConvert.length} HTML). Retrying with full HTML (${contentHtml.length} chars).`);
663
+ log.debug(`budget pre-truncation produced thin content (${convertedContent.length} chars from ${htmlForConvert.length} HTML). Retrying with full HTML (${contentHtml.length} chars).`);
663
664
  }
664
665
  ctx.timer.mark('convert-retry');
665
666
  let retryConverted;
@@ -701,8 +702,7 @@ export async function parseContent(ctx) {
701
702
  }
702
703
  catch (e) {
703
704
  // Non-fatal: JSON parse failed, treating as malformed
704
- if (process.env.DEBUG)
705
- console.debug('[webpeel]', 'JSON parse failed:', e instanceof Error ? e.message : e);
705
+ log.debug('JSON parse failed:', e instanceof Error ? e.message : e);
706
706
  ctx.content = fetchResult.html;
707
707
  ctx.title = 'JSON Response (malformed)';
708
708
  }
@@ -734,8 +734,7 @@ export async function parseContent(ctx) {
734
734
  }
735
735
  catch (e) {
736
736
  // Non-fatal: XML/RSS parse failed, using raw content
737
- if (process.env.DEBUG)
738
- console.debug('[webpeel]', 'XML/RSS parse failed:', e instanceof Error ? e.message : e);
737
+ log.debug('XML/RSS parse failed:', e instanceof Error ? e.message : e);
739
738
  ctx.content = fetchResult.html;
740
739
  ctx.title = 'XML Document';
741
740
  }
@@ -843,7 +842,7 @@ export async function postProcess(ctx) {
843
842
  let budgetedContent = distillToBudget(ctx.content, options.budget, budgetFormat);
844
843
  ctx.timer.end('budget');
845
844
  if (process.env.DEBUG) {
846
- console.debug('[webpeel]', `budget result: ${originalContent.length} → ${budgetedContent.length} chars`);
845
+ log.debug(`budget result: ${originalContent.length} → ${budgetedContent.length} chars`);
847
846
  }
848
847
  // Safety net: if BM25 distillation stripped too much (< 10% of original)
849
848
  // on a substantial page, fall back to simple head truncation.
@@ -860,7 +859,7 @@ export async function postProcess(ctx) {
860
859
  ctx.budgetFallback = true;
861
860
  ctx.warnings.push('Content was truncated to fit budget using head truncation (BM25 distillation produced insufficient content)');
862
861
  if (process.env.DEBUG) {
863
- console.debug('[webpeel]', `budget distillation fallback: BM25 produced ${budgetedContent.length} chars (< 10% of ${originalContent.length}), using head truncation`);
862
+ log.debug(`budget distillation fallback: BM25 produced ${budgetedContent.length} chars (< 10% of ${originalContent.length}), using head truncation`);
864
863
  }
865
864
  }
866
865
  ctx.content = budgetedContent;
@@ -879,8 +878,7 @@ export async function postProcess(ctx) {
879
878
  }
880
879
  catch (e) {
881
880
  // Domain extraction failure is non-fatal; continue with normal content
882
- if (process.env.DEBUG)
883
- console.debug('[webpeel]', 'domain extraction failed:', e instanceof Error ? e.message : e);
881
+ log.debug('domain extraction failed:', e instanceof Error ? e.message : e);
884
882
  }
885
883
  }
886
884
  // === Challenge / bot-protection page detection ===
@@ -921,6 +919,7 @@ export async function postProcess(ctx) {
921
919
  }
922
920
  // Try search fallback for the real content
923
921
  try {
922
+ // @ts-ignore — proprietary module, gitignored
924
923
  const { searchFallback } = await import('./search-fallback.js');
925
924
  const searchResult = await searchFallback(ctx.url);
926
925
  if (searchResult.cachedContent && searchResult.cachedContent.length > 50) {
@@ -978,6 +977,7 @@ export async function postProcess(ctx) {
978
977
  // Try 4: Search-as-proxy fallback (when page appears blocked)
979
978
  // Search engines already crawled this page — use their cached snippet.
980
979
  try {
980
+ // @ts-ignore — proprietary module, gitignored
981
981
  const { searchFallback } = await import('./search-fallback.js');
982
982
  const searchResult = await searchFallback(ctx.url);
983
983
  if (searchResult.cachedContent && searchResult.cachedContent.length > 50) {
@@ -1013,7 +1013,7 @@ export async function finalize(ctx) {
1013
1013
  ctx.brandingProfile = await extractBranding(fetchResult.page);
1014
1014
  }
1015
1015
  catch (error) {
1016
- console.error('Branding extraction failed:', error);
1016
+ log.error('Branding extraction failed:', error);
1017
1017
  }
1018
1018
  finally {
1019
1019
  // Clean up the kept-open page and browser
@@ -1025,8 +1025,7 @@ export async function finalize(ctx) {
1025
1025
  }
1026
1026
  catch (e) {
1027
1027
  // Non-fatal: page/browser cleanup after branding extraction
1028
- if (process.env.DEBUG)
1029
- console.debug('[webpeel]', 'page/browser cleanup after branding:', e instanceof Error ? e.message : e);
1028
+ log.debug('page/browser cleanup after branding:', e instanceof Error ? e.message : e);
1030
1029
  }
1031
1030
  }
1032
1031
  }
@@ -1037,7 +1036,7 @@ export async function finalize(ctx) {
1037
1036
  ctx.designAnalysisResult = await extractDesignAnalysis(fetchResult.page);
1038
1037
  }
1039
1038
  catch (error) {
1040
- console.error('Design analysis extraction failed:', error);
1039
+ log.error('Design analysis extraction failed:', error);
1041
1040
  }
1042
1041
  finally {
1043
1042
  if (!options.branding) {
@@ -1049,8 +1048,7 @@ export async function finalize(ctx) {
1049
1048
  }
1050
1049
  }
1051
1050
  catch (e) {
1052
- if (process.env.DEBUG)
1053
- console.debug('[webpeel]', 'page/browser cleanup after design analysis:', e instanceof Error ? e.message : e);
1051
+ log.debug('page/browser cleanup after design analysis:', e instanceof Error ? e.message : e);
1054
1052
  }
1055
1053
  }
1056
1054
  }
@@ -1063,7 +1061,7 @@ export async function finalize(ctx) {
1063
1061
  ctx.changeResult = await trackChange(fetchResult.url, ctx.content, fingerprint);
1064
1062
  }
1065
1063
  catch (error) {
1066
- console.error('Change tracking failed:', error);
1064
+ log.error('Change tracking failed:', error);
1067
1065
  }
1068
1066
  }
1069
1067
  // Generate AI summary if requested
@@ -1081,7 +1079,7 @@ export async function finalize(ctx) {
1081
1079
  });
1082
1080
  }
1083
1081
  catch (error) {
1084
- console.error('Summary generation failed:', error);
1082
+ log.error('Summary generation failed:', error);
1085
1083
  }
1086
1084
  }
1087
1085
  }
@@ -1185,4 +1183,3 @@ export function buildResult(ctx) {
1185
1183
  ...(fetchResult.autoInteract !== undefined ? { autoInteract: fetchResult.autoInteract } : {}),
1186
1184
  };
1187
1185
  }
1188
- //# sourceMappingURL=pipeline.js.map
@@ -45,4 +45,3 @@ export declare function deleteProfile(name: string): boolean;
45
45
  * 3. On browser close or Ctrl+C, captures storage state and saves the profile
46
46
  */
47
47
  export declare function createProfile(name: string, description?: string): Promise<void>;
48
- //# sourceMappingURL=profiles.d.ts.map
@@ -213,4 +213,3 @@ export async function createProfile(name, description) {
213
213
  });
214
214
  });
215
215
  }
216
- //# sourceMappingURL=profiles.js.map
@@ -51,4 +51,3 @@ export interface QuickAnswerResult {
51
51
  * ```
52
52
  */
53
53
  export declare function quickAnswer(options: QuickAnswerOptions): QuickAnswerResult;
54
- //# sourceMappingURL=quick-answer.d.ts.map
@@ -831,4 +831,3 @@ export function quickAnswer(options) {
831
831
  method: 'bm25',
832
832
  };
833
833
  }
834
- //# sourceMappingURL=quick-answer.js.map
@@ -78,4 +78,3 @@ export declare class RateGovernor {
78
78
  }
79
79
  /** Human-readable duration string from milliseconds. */
80
80
  export declare function formatDuration(ms: number): string;
81
- //# sourceMappingURL=rate-governor.d.ts.map
@@ -236,4 +236,3 @@ export function formatDuration(ms) {
236
236
  const days = Math.floor(hours / 24);
237
237
  return `${days}d ${hours % 24}h`;
238
238
  }
239
- //# sourceMappingURL=rate-governor.js.map
@@ -55,4 +55,3 @@ export interface ReadabilityResult {
55
55
  * @param options - Extraction options
56
56
  */
57
57
  export declare function extractReadableContent(html: string, _url: string, options?: ReadabilityOptions): ReadabilityResult;
58
- //# sourceMappingURL=readability.d.ts.map
@@ -530,4 +530,3 @@ export function extractReadableContent(html, _url, options = {}) {
530
530
  language: meta.language,
531
531
  };
532
532
  }
533
- //# sourceMappingURL=readability.js.map
@@ -64,4 +64,3 @@ export interface ResearchResult {
64
64
  * Conduct autonomous multi-step web research on a topic.
65
65
  */
66
66
  export declare function research(options: ResearchOptions): Promise<ResearchResult>;
67
- //# sourceMappingURL=research.d.ts.map
@@ -268,4 +268,3 @@ Instructions:
268
268
  cost,
269
269
  };
270
270
  }
271
- //# sourceMappingURL=research.js.map
@@ -64,4 +64,3 @@ export declare function findSchemaForUrl(url: string): ExtractionSchema | null;
64
64
  * @returns Array of extracted items (may be empty)
65
65
  */
66
66
  export declare function extractWithSchema(html: string, schema: ExtractionSchema, baseUrl?: string): ExtractedItem[];
67
- //# sourceMappingURL=schema-extraction.d.ts.map
@@ -350,4 +350,3 @@ export function extractWithSchema(html, schema, baseUrl) {
350
350
  }
351
351
  return items;
352
352
  }
353
- //# sourceMappingURL=schema-extraction.js.map
@@ -30,4 +30,3 @@ export declare function smartExtractSchemaFields(content: string, templateFields
30
30
  pageUrl?: string;
31
31
  metadata?: Record<string, any>;
32
32
  }): Record<string, string>;
33
- //# sourceMappingURL=schema-postprocess.d.ts.map
@@ -467,4 +467,3 @@ export function smartExtractSchemaFields(content, templateFields, quickAnswerFn,
467
467
  }
468
468
  return extracted;
469
469
  }
470
- //# sourceMappingURL=schema-postprocess.js.map
@@ -17,4 +17,3 @@ export declare function getSchemaTemplate(nameOrJson: string): SchemaTemplate |
17
17
  * List all available schema template names.
18
18
  */
19
19
  export declare function listSchemaTemplates(): string[];
20
- //# sourceMappingURL=schema-templates.d.ts.map
@@ -128,4 +128,3 @@ export function getSchemaTemplate(nameOrJson) {
128
128
  export function listSchemaTemplates() {
129
129
  return Object.keys(SCHEMA_TEMPLATES);
130
130
  }
131
- //# sourceMappingURL=schema-templates.js.map
@@ -222,4 +222,3 @@ export interface DesignComparisonResult {
222
222
  * parallel and diffing the result into a structured DesignComparison.
223
223
  */
224
224
  export declare function takeDesignComparison(subjectUrl: string, referenceUrl: string, options?: DesignCompareOptions): Promise<DesignComparisonResult>;
225
- //# sourceMappingURL=screenshot.d.ts.map
@@ -205,4 +205,3 @@ export async function takeDesignComparison(subjectUrl, referenceUrl, options = {
205
205
  comparison,
206
206
  };
207
207
  }
208
- //# sourceMappingURL=screenshot.js.map
@@ -25,4 +25,3 @@ export interface SearchFallbackResult {
25
25
  * Returns the title, snippet, and any extracted product data.
26
26
  */
27
27
  export declare function searchFallback(url: string): Promise<SearchFallbackResult>;
28
- //# sourceMappingURL=search-fallback.d.ts.map
@@ -182,4 +182,3 @@ export async function searchFallback(url) {
182
182
  return emptyResult;
183
183
  }
184
184
  }
185
- //# sourceMappingURL=search-fallback.js.map
@@ -176,4 +176,3 @@ export declare function getBestSearchProvider(): {
176
176
  apiKey?: string;
177
177
  };
178
178
  export {};
179
- //# sourceMappingURL=search-provider.d.ts.map