webpeel 0.13.4 → 0.14.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (83) hide show
  1. package/README.md +120 -162
  2. package/dist/cli-auth.js +7 -7
  3. package/dist/cli-auth.js.map +1 -1
  4. package/dist/cli.js +197 -26
  5. package/dist/cli.js.map +1 -1
  6. package/dist/core/auto-extract.d.ts +83 -0
  7. package/dist/core/auto-extract.d.ts.map +1 -0
  8. package/dist/core/auto-extract.js +565 -0
  9. package/dist/core/auto-extract.js.map +1 -0
  10. package/dist/core/deep-fetch.d.ts +75 -0
  11. package/dist/core/deep-fetch.d.ts.map +1 -0
  12. package/dist/core/deep-fetch.js +406 -0
  13. package/dist/core/deep-fetch.js.map +1 -0
  14. package/dist/core/domain-extractors.d.ts +34 -0
  15. package/dist/core/domain-extractors.d.ts.map +1 -0
  16. package/dist/core/domain-extractors.js +654 -0
  17. package/dist/core/domain-extractors.js.map +1 -0
  18. package/dist/core/markdown.d.ts +8 -0
  19. package/dist/core/markdown.d.ts.map +1 -1
  20. package/dist/core/markdown.js +25 -0
  21. package/dist/core/markdown.js.map +1 -1
  22. package/dist/core/quick-answer.d.ts +28 -0
  23. package/dist/core/quick-answer.d.ts.map +1 -0
  24. package/dist/core/quick-answer.js +288 -0
  25. package/dist/core/quick-answer.js.map +1 -0
  26. package/dist/core/readability.d.ts +58 -0
  27. package/dist/core/readability.d.ts.map +1 -0
  28. package/dist/core/readability.js +496 -0
  29. package/dist/core/readability.js.map +1 -0
  30. package/dist/core/search-provider.d.ts.map +1 -1
  31. package/dist/core/search-provider.js +3 -6
  32. package/dist/core/search-provider.js.map +1 -1
  33. package/dist/core/strategies.d.ts.map +1 -1
  34. package/dist/core/strategies.js +70 -5
  35. package/dist/core/strategies.js.map +1 -1
  36. package/dist/core/watch-manager.d.ts +140 -0
  37. package/dist/core/watch-manager.d.ts.map +1 -0
  38. package/dist/core/watch-manager.js +348 -0
  39. package/dist/core/watch-manager.js.map +1 -0
  40. package/dist/core/youtube.d.ts +91 -0
  41. package/dist/core/youtube.d.ts.map +1 -0
  42. package/dist/core/youtube.js +380 -0
  43. package/dist/core/youtube.js.map +1 -0
  44. package/dist/index.d.ts +4 -0
  45. package/dist/index.d.ts.map +1 -1
  46. package/dist/index.js +103 -0
  47. package/dist/index.js.map +1 -1
  48. package/dist/mcp/server.js +58 -16
  49. package/dist/mcp/server.js.map +1 -1
  50. package/dist/server/app.d.ts.map +1 -1
  51. package/dist/server/app.js +19 -1
  52. package/dist/server/app.js.map +1 -1
  53. package/dist/server/routes/deep-fetch.d.ts +9 -0
  54. package/dist/server/routes/deep-fetch.d.ts.map +1 -0
  55. package/dist/server/routes/deep-fetch.js +38 -0
  56. package/dist/server/routes/deep-fetch.js.map +1 -0
  57. package/dist/server/routes/extract.d.ts.map +1 -1
  58. package/dist/server/routes/extract.js +11 -0
  59. package/dist/server/routes/extract.js.map +1 -1
  60. package/dist/server/routes/fetch.d.ts.map +1 -1
  61. package/dist/server/routes/fetch.js +45 -19
  62. package/dist/server/routes/fetch.js.map +1 -1
  63. package/dist/server/routes/mcp.d.ts +2 -1
  64. package/dist/server/routes/mcp.d.ts.map +1 -1
  65. package/dist/server/routes/mcp.js +307 -38
  66. package/dist/server/routes/mcp.js.map +1 -1
  67. package/dist/server/routes/quick-answer.d.ts +9 -0
  68. package/dist/server/routes/quick-answer.d.ts.map +1 -0
  69. package/dist/server/routes/quick-answer.js +84 -0
  70. package/dist/server/routes/quick-answer.js.map +1 -0
  71. package/dist/server/routes/watch.d.ts +16 -0
  72. package/dist/server/routes/watch.d.ts.map +1 -0
  73. package/dist/server/routes/watch.js +219 -0
  74. package/dist/server/routes/watch.js.map +1 -0
  75. package/dist/server/routes/youtube.d.ts +7 -0
  76. package/dist/server/routes/youtube.d.ts.map +1 -0
  77. package/dist/server/routes/youtube.js +87 -0
  78. package/dist/server/routes/youtube.js.map +1 -0
  79. package/dist/types.d.ts +18 -0
  80. package/dist/types.d.ts.map +1 -1
  81. package/dist/types.js.map +1 -1
  82. package/llms.txt +14 -5
  83. package/package.json +1 -1
@@ -0,0 +1 @@
1
+ {"version":3,"file":"auto-extract.js","sourceRoot":"","sources":["../../src/core/auto-extract.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;GAWG;AAEH,OAAO,EAAE,IAAI,EAAmB,MAAM,SAAS,CAAC;AAgFhD,8EAA8E;AAC9E,sBAAsB;AACtB,8EAA8E;AAE9E,MAAM,YAAY,GAAG,gBAAgB,CAAC;AACtC,MAAM,SAAS,GAAG,WAAW,CAAC;AAC9B,MAAM,mBAAmB,GAAG,8CAA8C,CAAC;AAC3E,MAAM,gBAAgB,GAAG,+CAA+C,CAAC;AACzE,MAAM,aAAa,GAAG,mDAAmD,CAAC;AAC1E,MAAM,aAAa,GACjB,yFAAyF,CAAC;AAE5F,qGAAqG;AACrG,SAAS,WAAW,CAAC,CAAa;IAChC,MAAM,IAAI,GAAG,CAAC,CAAC,MAAM,CAAC,CAAC,IAAI,EAAE,IAAI,EAAE,CAAC;IACpC,OAAO,IAAI,CAAC,OAAO,CAAC,UAAU,EAAE,GAAG,CAAC,CAAC,OAAO,CAAC,eAAe,EAAE,GAAG,CAAC,CAAC,OAAO,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC,IAAI,EAAE,CAAC;AACjG,CAAC;AAED,SAAS,MAAM,CAAC,GAAW,EAAE,GAAG,QAAkB;IAChD,IAAI,CAAC;QACH,MAAM,IAAI,GAAG,IAAI,GAAG,CAAC,GAAG,CAAC,CAAC,QAAQ,CAAC,WAAW,EAAE,CAAC;QACjD,OAAO,QAAQ,CAAC,IAAI,CAAC,CAAC,EAAE,EAAE,EAAE,CAAC,IAAI,CAAC,QAAQ,CAAC,EAAE,CAAC,CAAC,CAAC;IAClD,CAAC;IAAC,MAAM,CAAC;QACP,MAAM,KAAK,GAAG,GAAG,CAAC,WAAW,EAAE,CAAC;QAChC,OAAO,QAAQ,CAAC,IAAI,CAAC,CAAC,EAAE,EAAE,EAAE,CAAC,KAAK,CAAC,QAAQ,CAAC,EAAE,CAAC,CAAC,CAAC;IACnD,CAAC;AACH,CAAC;AAED;;;GAGG;AACH,MAAM,UAAU,cAAc,CAAC,IAAY,EAAE,GAAW;IACtD,MAAM,CAAC,GAAG,IAAI,CAAC,IAAI,CAAC,CAAC;IAErB,kBAAkB;IAClB,IAAI,MAAM,CAAC,GAAG,EAAE,UAAU,EAAE,QAAQ,EAAE,WAAW,EAAE,QAAQ,EAAE,UAAU,CAAC,EAAE,CAAC;QACzE,OAAO,SAAS,CAAC;IACnB,CAAC;IACD,MAAM,QAAQ,GAAG,WAAW,CAAC,CAAC,CAAC,CAAC;IAChC,MAAM,YAAY,GAAG,QAAQ,CAAC,KAAK,CAAC,iBAAiB,CAAC,IAAI,EAAE,CAAC;IAC7D,MAAM,gBAAgB,GAAG,QAAQ,CAAC,KAAK,CAAC,oCAAoC,CAAC,IAAI,EAAE,CAAC;IACpF,IAAI,YAAY,CAAC,MAAM,IAAI,CAAC,IAAI,gBAAgB,CAAC,MAAM,IAAI,CAAC,EAAE,CAAC;QAC7D,OAAO,SAAS,CAAC;IACnB,CAAC;IAED,kBAAkB;IAClB,IAAI,MAAM,CAAC,GAAG,EAAE,UAAU,EAAE,QAAQ,EAAE,QAAQ,EAAE,UAAU,EAAE,UAAU,CAAC,EAAE,CAAC;QACxE,MAAM,MAAM,GAAG,QAAQ,CAAC,KAAK,CAAC,aAAa,CAAC,IAAI,EAAE,CAAC;QACnD,IAAI,MAAM,CAAC,MAAM,GAAG,CAAC;YAAE,OAAO,SAAS,CAAC;IAC1C,CAAC;IACD,MAAM,MAAM,GAAG,QAAQ,CAAC,KAAK,CAAC,aAAa,CAAC,IAAI,EAAE,CAAC;IACnD,MAAM,MAAM,GAAG,QAAQ,CAAC,KAAK,CAAC,aAAa,CAAC,IAAI,EAAE,CAAC;IACnD,MAAM,WAAW,GAAG,CAAC,CAAC,wEAAwE,CAAC,CAAC,MAAM,CAAC;IACvG,IAAI,MAAM,CAAC,MAAM,GAAG,CAAC,IAAI,CAAC,MAAM,CAAC,MAAM,GAAG,CAAC,IAAI,WAAW,GAAG,CAAC,CAAC,EAAE,CAAC;QAChE,OAAO,SAAS,CAAC;IACnB,CAAC;IAED,kBAAkB;IAClB,MAAM,aAAa,GAAG,CAAC,CAAC,SAAS,CAAC,CAAC,MAAM,GAAG,CAAC,CAAC;IAC9C,MAAM,UAAU,GAAG,CAAC,CAAC,+BAA+B,CAAC,CAAC,MAAM,GAAG,CAAC,CAAC;IACjE,MAAM,aAAa,GACjB,CAAC,CAAC,qBAAqB,CAAC,CAAC,MAAM,GAAG,CAAC;QACnC,CAAC,CAAC,wCAAwC,CAAC,CAAC,MAAM,GAAG,CAAC,CAAC;IACzD,IAAI,aAAa,IAAI,CAAC,UAAU,IAAI,aAAa,CAAC,EAAE,CAAC;QACnD,OAAO,SAAS,CAAC;IACnB,CAAC;IACD,2DAA2D;IAC3D,MAAM,OAAO,GAAG,CAAC,CAAC,IAAI,CAAC,CAAC,MAAM,CAAC;IAC/B,MAAM,SAAS,GAAG,CAAC,CAAC,GAAG,CAAC,CAAC,MAAM,CAAC;IAChC,IAAI,OAAO,KAAK,CAAC,IAAI,SAAS,IAAI,CAAC,IAAI,UAAU,EAAE,CAAC;QAClD,OAAO,SAAS,CAAC;IACnB,CAAC;IAED,mBAAmB;IACnB,MAAM,QAAQ,GAAG,CAAC,CAAC,WAAW,CAAC,CAAC,IAAI,EAAE,CAAC;IACvC,MAAM,cAAc,GAAG,CAAC,QAAQ,CAAC,KAAK,CAAC,mBAAmB,CAAC,IAAI,EAAE,CAAC,CAAC,MAAM,CAAC;IAC1E,MAAM,WAAW,GAAG,CAAC,QAAQ,CAAC,KAAK,CAAC,gBAAgB,CAAC,IAAI,EAAE,CAAC,CAAC,MAAM,CAAC;IACpE,IAAI,cAAc,IAAI,CAAC,IAAI,WAAW,IAAI,CAAC,EAAE,CAAC;QAC5C,OAAO,UAAU,CAAC;IACpB,CAAC;IACD,wDAAwD;IACxD,MAAM,WAAW,GAAG,CAAC,CAAC,YAAY,CAAC,CAAC,IAAI,EAAE,CAAC;IAC3C,IACE,WAAW,CAAC,KAAK,CAAC,8CAA8C,CAAC;QACjE,cAAc,IAAI,CAAC,EACnB,CAAC;QACD,OAAO,UAAU,CAAC;IACpB,CAAC;IAED,mBAAmB;IACnB,+DAA+D;IAC/D,MAAM,0BAA0B,GAAG;QACjC,UAAU,EAAE,OAAO,EAAE,OAAO,EAAE,oBAAoB,EAAE,iBAAiB,EAAE,iBAAiB;KACzF,CAAC;IACF,KAAK,MAAM,GAAG,IAAI,0BAA0B,EAAE,CAAC;QAC7C,MAAM,KAAK,GAAG,CAAC,CAAC,GAAG,CAAC,CAAC;QACrB,IAAI,KAAK,CAAC,MAAM,IAAI,CAAC,EAAE,CAAC;YACtB,IAAI,SAAS,GAAG,CAAC,CAAC;YAClB,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,EAAE,EAAE;gBACnB,MAAM,IAAI,GAAG,CAAC,CAAC,EAAE,CAAC,CAAC,IAAI,EAAE,CAAC;gBAC1B,IAAI,YAAY,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,SAAS,CAAC,IAAI,CAAC,IAAI,CAAC;oBAAE,SAAS,EAAE,CAAC;YACnE,CAAC,CAAC,CAAC;YACH,IAAI,SAAS,IAAI,CAAC;gBAAE,OAAO,UAAU,CAAC;QACxC,CAAC;IACH,CAAC;IACD,qDAAqD;IACrD,MAAM,IAAI,GAAG,CAAC,CAAC,KAAK,CAAC,CAAC,MAAM,CAAC;IAC7B,IAAI,IAAI,IAAI,CAAC,IAAI,YAAY,CAAC,MAAM,IAAI,CAAC,EAAE,CAAC;QAC1C,OAAO,UAAU,CAAC;IACpB,CAAC;IAED,OAAO,SAAS,CAAC;AACnB,CAAC;AAED,8EAA8E;AAC9E,oBAAoB;AACpB,8EAA8E;AAE9E,SAAS,mBAAmB,CAAC,CAAa;IACxC,MAAM,KAAK,GAAkB,EAAE,CAAC;IAEhC,iEAAiE;IACjE,MAAM,kBAAkB,GAAG;QACzB,yBAAyB;QACzB,uBAAuB;QACvB,sBAAsB;QACtB,sBAAsB;QACtB,0BAA0B;QAC1B,iBAAiB;QACjB,yBAAyB;QACzB,6BAA6B;QAC7B,6BAA6B;QAC7B,OAAO;QACP,iBAAiB;KAClB,CAAC;IAEF,IAAI,UAAU,GAAkC,IAAI,CAAC;IACrD,KAAK,MAAM,GAAG,IAAI,kBAAkB,EAAE,CAAC;QACrC,MAAM,KAAK,GAAG,CAAC,CAAC,GAAG,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,EAAE,EAAE;YACpC,MAAM,IAAI,GAAG,CAAC,CAAC,EAAE,CAAC,CAAC,IAAI,EAAE,CAAC;YAC1B,OAAO,YAAY,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,SAAS,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QACzD,CAAC,CAAC,CAAC;QACH,IAAI,KAAK,CAAC,MAAM,IAAI,CAAC,EAAE,CAAC;YACtB,UAAU,GAAG,KAAK,CAAC;YACnB,MAAM;QACR,CAAC;IACH,CAAC;IAED,IAAI,CAAC,UAAU,IAAI,UAAU,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QAC3C,4DAA4D;QAC5D,OAAO,oBAAoB,CAAC,CAAC,CAAC,CAAC;IACjC,CAAC;IAED,UAAU,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,EAAE,EAAE;QACxB,IAAI,CAAC;YACH,MAAM,GAAG,GAAG,CAAC,CAAC,EAAE,CAAC,CAAC;YAClB,MAAM,IAAI,GAAG,GAAG,CAAC,IAAI,EAAE,CAAC,IAAI,EAAE,CAAC;YAE/B,qDAAqD;YACrD,MAAM,MAAM,GAAG,GAAG,CAAC,IAAI,CAAC,2DAA2D,CAAC,CAAC,KAAK,EAAE,CAAC;YAC7F,MAAM,IAAI,GAAG,MAAM,CAAC,IAAI,EAAE,CAAC,IAAI,EAAE,IAAI,MAAM,CAAC;YAE5C,gBAAgB;YAChB,MAAM,UAAU,GAAG,IAAI,CAAC,KAAK,CAAC,iCAAiC,CAAC,CAAC;YACjE,IAAI,CAAC,UAAU,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,IAAI,CAAC;gBAAE,OAAO,CAAC,4BAA4B;YAC9E,MAAM,KAAK,GAAG,SAAS,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,UAAU,EAAE,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC;YAErF,iBAAiB;YACjB,MAAM,WAAW,GAAG,IAAI,CAAC,KAAK,CAAC,uCAAuC,CAAC,CAAC;YACxE,MAAM,MAAM,GAAG,WAAW,CAAC,CAAC,CAAC,WAAW,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC;YAExD,8BAA8B;YAC9B,MAAM,QAAQ,GAAa,EAAE,CAAC;YAC9B,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,EAAE,EAAE;gBAC5B,MAAM,WAAW,GAAG,CAAC,CAAC,EAAE,CAAC,CAAC,IAAI,EAAE,CAAC,IAAI,EAAE,CAAC;gBACxC,IAAI,WAAW,IAAI,WAAW,CAAC,MAAM,GAAG,GAAG,EAAE,CAAC;oBAC5C,QAAQ,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC;gBAC7B,CAAC;YACH,CAAC,CAAC,CAAC;YAEH,qBAAqB;YACrB,MAAM,KAAK,GAAG,GAAG;iBACd,IAAI,CAAC,WAAW,CAAC;iBACjB,MAAM,CAAC,CAAC,CAAC,EAAE,GAAG,EAAE,EAAE,CACjB,oEAAoE,CAAC,IAAI,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,CAAC,CACzF;iBACA,KAAK,EAAE,CAAC;YACX,MAAM,GAAG,GAAG,KAAK,CAAC,IAAI,EAAE,CAAC,IAAI,EAAE,IAAI,SAAS,CAAC;YAE7C,IAAI,IAAI,IAAI,KAAK,EAAE,CAAC;gBAClB,KAAK,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,KAAK,EAAE,MAAM,EAAE,QAAQ,EAAE,GAAG,EAAE,CAAC,CAAC;YACrD,CAAC;QACH,CAAC;QAAC,MAAM,CAAC;YACP,qCAAqC;QACvC,CAAC;IACH,CAAC,CAAC,CAAC;IAEH,OAAO,gBAAgB,CAAC,KAAK,CAAC,CAAC;AACjC,CAAC;AAED,SAAS,oBAAoB,CAAC,CAAa;IACzC,wDAAwD;IACxD,MAAM,KAAK,GAAkB,EAAE,CAAC;IAChC,MAAM,QAAQ,GAAG,WAAW,CAAC,CAAC,CAAC,CAAC;IAEhC,MAAM,UAAU,GAAG,wEAAwE,CAAC;IAC5F,IAAI,KAAK,CAAC;IACV,MAAM,WAAW,GAAa,EAAE,CAAC;IACjC,OAAO,CAAC,KAAK,GAAG,UAAU,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC,KAAK,IAAI,EAAE,CAAC;QACpD,WAAW,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC;IAC7B,CAAC;IAED,+CAA+C;IAC/C,MAAM,YAAY,GAAG,CAAC,GAAG,IAAI,GAAG,CAAC,WAAW,CAAC,CAAC,CAAC;IAC/C,KAAK,MAAM,CAAC,IAAI,YAAY,EAAE,CAAC;QAC7B,KAAK,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,KAAK,EAAE,CAAC,EAAE,QAAQ,EAAE,EAAE,EAAE,CAAC,CAAC;IACvD,CAAC;IAED,OAAO,KAAK,CAAC;AACf,CAAC;AAED,SAAS,gBAAgB,CAAC,KAAoB;IAC5C,MAAM,IAAI,GAAG,IAAI,GAAG,EAAU,CAAC;IAC/B,OAAO,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE;QACxB,MAAM,GAAG,GAAG,GAAG,CAAC,CAAC,IAAI,IAAI,CAAC,CAAC,KAAK,EAAE,CAAC;QACnC,IAAI,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC;YAAE,OAAO,KAAK,CAAC;QAChC,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;QACd,OAAO,IAAI,CAAC;IACd,CAAC,CAAC,CAAC;AACL,CAAC;AAED,8EAA8E;AAC9E,qBAAqB;AACrB,8EAA8E;AAE9E,SAAS,eAAe,CAAC,CAAa,EAAE,OAAe;IACrD,MAAM,KAAK,GAAkB,EAAE,CAAC;IAChC,MAAM,MAAM,GAAG,CAAC,GAAG,EAAE;QACnB,IAAI,CAAC;YACH,OAAO,IAAI,GAAG,CAAC,OAAO,CAAC,CAAC,MAAM,CAAC;QACjC,CAAC;QAAC,MAAM,CAAC;YACP,OAAO,EAAE,CAAC;QACZ,CAAC;IACH,CAAC,CAAC,EAAE,CAAC;IAEL,MAAM,kBAAkB,GAAG;QACzB,oBAAoB;QACpB,iBAAiB;QACjB,iBAAiB;QACjB,IAAI;QACJ,SAAS;KACV,CAAC;IAEF,IAAI,UAAU,GAAkC,IAAI,CAAC;IACrD,KAAK,MAAM,GAAG,IAAI,kBAAkB,EAAE,CAAC;QACrC,MAAM,KAAK,GAAG,CAAC,CAAC,GAAG,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,EAAE,EAAE;YACpC,MAAM,IAAI,GAAG,CAAC,CAAC,EAAE,CAAC,CAAC,IAAI,EAAE,CAAC;YAC1B,OAAO,CAAC,YAAY,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,SAAS,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,IAAI,CAAC,CAAC,EAAE,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,MAAM,GAAG,CAAC,CAAC;QAC3F,CAAC,CAAC,CAAC;QACH,IAAI,KAAK,CAAC,MAAM,IAAI,CAAC,EAAE,CAAC;YACtB,UAAU,GAAG,KAAK,CAAC;YACnB,MAAM;QACR,CAAC;IACH,CAAC;IAED,IAAI,CAAC,UAAU,IAAI,UAAU,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,KAAK,CAAC;IAEzD,UAAU,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,EAAE,EAAE;QACxB,IAAI,CAAC;YACH,MAAM,GAAG,GAAG,CAAC,CAAC,EAAE,CAAC,CAAC;YAElB,OAAO;YACP,MAAM,MAAM,GAAG,GAAG,CAAC,IAAI,CAAC,oDAAoD,CAAC,CAAC,KAAK,EAAE,CAAC;YACtF,MAAM,IAAI,GAAG,MAAM,CAAC,IAAI,EAAE,CAAC,IAAI,EAAE,CAAC;YAClC,IAAI,CAAC,IAAI;gBAAE,OAAO;YAElB,QAAQ;YACR,MAAM,UAAU,GAAG,GAAG,CAAC,IAAI,EAAE,CAAC,KAAK,CAAC,2BAA2B,CAAC,CAAC;YACjE,MAAM,KAAK,GAAG,UAAU,CAAC,CAAC,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC,SAAS,CAAC;YAE5D,QAAQ;YACR,MAAM,KAAK,GAAG,GAAG,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,KAAK,EAAE,CAAC;YACtC,MAAM,MAAM,GAAG,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,KAAK,CAAC,IAAI,CAAC,UAAU,CAAC,IAAI,KAAK,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC;YACtF,MAAM,KAAK,GAAG,MAAM;gBAClB,CAAC,CAAC,MAAM,CAAC,UAAU,CAAC,MAAM,CAAC;oBACzB,CAAC,CAAC,MAAM;oBACR,CAAC,CAAC,GAAG,MAAM,GAAG,MAAM,CAAC,UAAU,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,GAAG,GAAG,MAAM,EAAE;gBAC5D,CAAC,CAAC,SAAS,CAAC;YAEd,MAAM;YACN,MAAM,MAAM,GAAG,GAAG,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,CAAC;YACrC,MAAM,IAAI,GAAG,MAAM,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;YACjC,MAAM,GAAG,GAAG,IAAI;gBACd,CAAC,CAAC,IAAI,CAAC,UAAU,CAAC,MAAM,CAAC;oBACvB,CAAC,CAAC,IAAI;oBACN,CAAC,CAAC,GAAG,MAAM,GAAG,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,GAAG,GAAG,IAAI,EAAE;gBACxD,CAAC,CAAC,SAAS,CAAC;YAEd,SAAS;YACT,MAAM,WAAW,GAAG,GAAG,CAAC,IAAI,EAAE,CAAC,KAAK,CAAC,kCAAkC,CAAC,CAAC;YACzE,MAAM,MAAM,GAAG,WAAW,CAAC,CAAC,CAAC,GAAG,WAAW,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,SAAS,CAAC;YAE/D,KAAK,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,KAAK,EAAE,KAAK,EAAE,GAAG,EAAE,MAAM,EAAE,CAAC,CAAC;QAClD,CAAC;QAAC,MAAM,CAAC;YACP,iBAAiB;QACnB,CAAC;IACH,CAAC,CAAC,CAAC;IAEH,OAAO,KAAK,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC,CAAC,CAAC,aAAa;AAC3C,CAAC;AAED,8EAA8E;AAC9E,oBAAoB;AACpB,8EAA8E;AAE9E,MAAM,cAAc,GAA2B;IAC7C,aAAa,EAAE,SAAS;IACxB,OAAO,EAAE,SAAS;IAClB,cAAc,EAAE,UAAU;IAC1B,YAAY,EAAE,QAAQ;IACtB,cAAc,EAAE,UAAU;IAC1B,eAAe,EAAE,WAAW;IAC5B,aAAa,EAAE,SAAS;IACxB,YAAY,EAAE,QAAQ;IACtB,YAAY,EAAE,SAAS;IACvB,aAAa,EAAE,SAAS;CACzB,CAAC;AAEF,MAAM,eAAe,GACnB,qIAAqI,CAAC;AAExI,SAAS,cAAc,CAAC,CAAa;IACnC,MAAM,QAAQ,GAAG,WAAW,CAAC,CAAC,CAAC,CAAC;IAEhC,SAAS;IACT,MAAM,YAAY,GAAG,QAAQ,CAAC,KAAK,CAAC,aAAa,CAAC,IAAI,EAAE,CAAC;IACzD,MAAM,MAAM,GAAG;QACb,GAAG,IAAI,GAAG,CAAC,YAAY,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,WAAW,EAAE,CAAC,CAAC;KACrD,CAAC;IAEF,SAAS;IACT,MAAM,YAAY,GAAG,QAAQ,CAAC,KAAK,CAAC,aAAa,CAAC,IAAI,EAAE,CAAC;IACzD,MAAM,MAAM,GAAG,CAAC,GAAG,IAAI,GAAG,CAAC,YAAY,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC,CAAC;IAE/D,YAAY;IACZ,MAAM,SAAS,GAAa,EAAE,CAAC;IAC/B,CAAC,CAAC,mDAAmD,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,EAAE,EAAE;QACpE,MAAM,IAAI,GAAG,CAAC,CAAE,EAAE,CAAC,CAAC,IAAI,EAAE,CAAC,OAAO,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC,IAAI,EAAE,CAAC;QACvD,IAAI,IAAI,CAAC,MAAM,GAAG,EAAE;YAAE,SAAS,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IAC7C,CAAC,CAAC,CAAC;IACH,mBAAmB;IACnB,MAAM,SAAS,GAAG,QAAQ,CAAC,KAAK,CAAC,eAAe,CAAC,CAAC;IAClD,IAAI,SAAS,EAAE,CAAC;QACd,MAAM,IAAI,GAAG,SAAS,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;QACjC,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,QAAQ,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;YAC9D,SAAS,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QACvB,CAAC;IACH,CAAC;IAED,eAAe;IACf,MAAM,MAAM,GAA2B,EAAE,CAAC;IAC1C,CAAC,CAAC,SAAS,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,EAAE,EAAE;QAC1B,MAAM,IAAI,GAAG,CAAC,CAAC,EAAE,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,IAAI,EAAE,CAAC;QACtC,KAAK,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,cAAc,CAAC,EAAE,CAAC;YAC3D,IAAI,IAAI,CAAC,QAAQ,CAAC,MAAM,CAAC,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,EAAE,CAAC;gBAC1C,MAAM,CAAC,GAAG,CAAC,GAAG,IAAI,CAAC;YACrB,CAAC;QACH,CAAC;IACH,CAAC,CAAC,CAAC;IAEH,OAAO,EAAE,IAAI,EAAE,SAAS,EAAE,MAAM,EAAE,MAAM,EAAE,SAAS,EAAE,MAAM,EAAE,CAAC;AAChE,CAAC;AAED,8EAA8E;AAC9E,oBAAoB;AACpB,8EAA8E;AAE9E,SAAS,cAAc,CAAC,CAAa;IACnC,QAAQ;IACR,MAAM,KAAK,GACT,CAAC,CAAC,IAAI,CAAC,CAAC,KAAK,EAAE,CAAC,IAAI,EAAE,CAAC,IAAI,EAAE;QAC7B,CAAC,CAAC,2BAA2B,CAAC,CAAC,IAAI,CAAC,SAAS,CAAC;QAC9C,CAAC,CAAC,OAAO,CAAC,CAAC,IAAI,EAAE,CAAC,IAAI,EAAE;QACxB,SAAS,CAAC;IAEZ,SAAS;IACT,MAAM,MAAM,GACV,CAAC,CAAC,qBAAqB,CAAC,CAAC,IAAI,CAAC,SAAS,CAAC;QACxC,CAAC,CAAC,qBAAqB,CAAC,CAAC,KAAK,EAAE,CAAC,IAAI,EAAE,CAAC,IAAI,EAAE;QAC9C,CAAC,CAAC,mBAAmB,CAAC,CAAC,KAAK,EAAE,CAAC,IAAI,EAAE,CAAC,IAAI,EAAE;QAC5C,CAAC,CAAC,gBAAgB,CAAC,CAAC,KAAK,EAAE,CAAC,IAAI,EAAE,CAAC,IAAI,EAAE;QACzC,SAAS,CAAC;IAEZ,OAAO;IACP,MAAM,IAAI,GACR,CAAC,CAAC,gBAAgB,CAAC,CAAC,KAAK,EAAE,CAAC,IAAI,CAAC,UAAU,CAAC;QAC5C,CAAC,CAAC,eAAe,CAAC,CAAC,KAAK,EAAE,CAAC,IAAI,CAAC,UAAU,CAAC;QAC3C,CAAC,CAAC,mBAAmB,CAAC,CAAC,IAAI,CAAC,SAAS,CAAC;QACtC,CAAC,CAAC,yCAAyC,CAAC,CAAC,IAAI,CAAC,SAAS,CAAC;QAC5D,CAAC,CAAC,MAAM,CAAC,CAAC,KAAK,EAAE,CAAC,IAAI,EAAE,CAAC,IAAI,EAAE;QAC/B,SAAS,CAAC;IAEZ,eAAe;IACf,MAAM,aAAa,GAAG,CAAC,CAAC,oEAAoE,CAAC,CAAC,KAAK,EAAE,CAAC;IACtG,MAAM,WAAW,GAAG,aAAa,CAAC,MAAM,CAAC,CAAC,CAAC,aAAa,CAAC,IAAI,EAAE,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC,mBAAmB,CAAC,CAAC,CAAC,CAAC;IAEhG,iDAAiD;IACjD,MAAM,SAAS,GAAG,CAAC,CAAC,SAAS,CAAC,CAAC,KAAK,EAAE,CAAC;IACvC,MAAM,SAAS,GAAG,SAAS,CAAC,MAAM,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,KAAK,EAAE,CAAC;IACnE,MAAM,SAAS,GACb,SAAS,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,CAAC,IAAI,EAAE,CAAC,IAAI,EAAE;QACzC,CAAC,CAAC,0BAA0B,CAAC,CAAC,IAAI,CAAC,SAAS,CAAC;QAC7C,CAAC,CAAC,iCAAiC,CAAC,CAAC,IAAI,CAAC,SAAS,CAAC;QACpD,EAAE,CAAC;IACL,MAAM,OAAO,GAAG,SAAS,CAAC,CAAC,CAAC,qBAAqB,CAAC,SAAS,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC;IAE5E,sCAAsC;IACtC,MAAM,QAAQ,GAAqB,EAAE,CAAC;IACtC,MAAM,QAAQ,GAAG,SAAS,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;IAC1C,QAAQ,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,EAAE,EAAE;QACtB,MAAM,OAAO,GAAG,CAAC,CAAC,EAAE,CAAC,CAAC,IAAI,EAAE,CAAC,IAAI,EAAE,CAAC;QACpC,IAAI,CAAC,OAAO;YAAE,OAAO;QACrB,0DAA0D;QAC1D,MAAM,YAAY,GAAa,EAAE,CAAC;QAClC,IAAI,OAAO,GAAG,CAAC,CAAC,EAAE,CAAC,CAAC,IAAI,EAAE,CAAC;QAC3B,OAAO,OAAO,CAAC,MAAM,IAAI,CAAC,OAAO,CAAC,EAAE,CAAC,QAAQ,CAAC,EAAE,CAAC;YAC/C,MAAM,IAAI,GAAG,OAAO,CAAC,IAAI,EAAE,CAAC,IAAI,EAAE,CAAC;YACnC,IAAI,IAAI;gBAAE,YAAY,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;YAClC,OAAO,GAAG,OAAO,CAAC,IAAI,EAAE,CAAC;QAC3B,CAAC;QACD,IAAI,YAAY,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YAC5B,QAAQ,CAAC,IAAI,CAAC,EAAE,OAAO,EAAE,OAAO,EAAE,YAAY,CAAC,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;QAC9D,CAAC;IACH,CAAC,CAAC,CAAC;IAEH,OAAO,EAAE,IAAI,EAAE,SAAS,EAAE,KAAK,EAAE,MAAM,EAAE,IAAI,EAAE,WAAW,EAAE,OAAO,EAAE,QAAQ,EAAE,CAAC;AAClF,CAAC;AAED,SAAS,qBAAqB,CAAC,IAAY,EAAE,KAAa;IACxD,MAAM,WAAW,GAAG,YAAY,CAAC;IACjC,IAAI,KAAK,CAAC;IACV,IAAI,SAAS,GAAG,CAAC,CAAC;IAClB,IAAI,aAAa,GAAG,CAAC,CAAC;IACtB,OAAO,CAAC,KAAK,GAAG,WAAW,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,KAAK,IAAI,EAAE,CAAC;QACjD,SAAS,GAAG,KAAK,CAAC,KAAK,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC;QAC1C,aAAa,EAAE,CAAC;QAChB,IAAI,aAAa,IAAI,KAAK;YAAE,MAAM;IACpC,CAAC;IACD,OAAO,aAAa,GAAG,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,SAAS,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC,CAAC,IAAI,EAAE,CAAC;AACzF,CAAC;AAED,SAAS,mBAAmB,CAAC,CAAa;IACxC,MAAM,cAAc,GAAG,GAAG,CAAC;IAC3B,MAAM,IAAI,GAAG,CAAC,CAAC,yCAAyC,CAAC,CAAC,KAAK,EAAE,CAAC,IAAI,EAAE,CAAC;IACzE,MAAM,SAAS,GAAG,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,MAAM,CAAC;IAC3D,MAAM,OAAO,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,IAAI,CAAC,SAAS,GAAG,cAAc,CAAC,CAAC,CAAC;IACnE,OAAO,GAAG,OAAO,MAAM,CAAC;AAC1B,CAAC;AAED,8EAA8E;AAC9E,qBAAqB;AACrB,8EAA8E;AAE9E,MAAM,YAAY,GAAG,CAAC,KAAK,EAAE,MAAM,EAAE,KAAK,EAAE,OAAO,EAAE,QAAQ,EAAE,MAAM,EAAE,SAAS,CAAC,CAAC;AAElF,SAAS,cAAc,CAAC,CAAa,EAAE,GAAW;IAChD,MAAM,SAAS,GAAkB,EAAE,CAAC;IAEpC,0CAA0C;IAC1C,IAAI,OAA2B,CAAC;IAChC,MAAM,QAAQ,GAAG,WAAW,CAAC,CAAC,CAAC,CAAC;IAChC,MAAM,YAAY,GAAG,QAAQ,CAAC,KAAK,CAAC,gCAAgC,CAAC,CAAC;IACtE,IAAI,YAAY,EAAE,CAAC;QACjB,OAAO,GAAG,YAAY,CAAC,CAAC,CAAC,CAAC;IAC5B,CAAC;SAAM,CAAC;QACN,IAAI,CAAC;YACH,MAAM,MAAM,GAAG,IAAI,GAAG,CAAC,GAAG,CAAC,CAAC;YAC5B,OAAO,GAAG,GAAG,MAAM,CAAC,QAAQ,SAAS,MAAM,CAAC,QAAQ,EAAE,CAAC;QACzD,CAAC;QAAC,MAAM,CAAC;YACP,OAAO,GAAG,SAAS,CAAC;QACtB,CAAC;IACH,CAAC;IAED,gEAAgE;IAChE,CAAC,CAAC,WAAW,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,EAAE,EAAE;QAC5B,MAAM,IAAI,GAAG,CAAC,CAAC,EAAE,CAAC,CAAC,IAAI,EAAE,CAAC,IAAI,EAAE,CAAC;QACjC,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;QAC/B,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;YACzB,MAAM,OAAO,GAAG,IAAI,CAAC,IAAI,EAAE,CAAC;YAC5B,KAAK,MAAM,MAAM,IAAI,YAAY,EAAE,CAAC;gBAClC,IAAI,OAAO,CAAC,UAAU,CAAC,MAAM,GAAG,GAAG,CAAC,IAAI,OAAO,CAAC,UAAU,CAAC,MAAM,GAAG,IAAI,CAAC,EAAE,CAAC;oBAC1E,MAAM,IAAI,GAAG,OAAO,CAAC,KAAK,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,IAAI,EAAE,CAAC;oBACjD,sCAAsC;oBACtC,MAAM,SAAS,GAAG,IAAI,CAAC,KAAK,CAAC,+BAA+B,CAAC,CAAC;oBAC9D,IAAI,SAAS,EAAE,CAAC;wBACd,IAAI,IAAI,GAAG,SAAS,CAAC,CAAC,CAAC,CAAC;wBACxB,8CAA8C;wBAC9C,IAAI,OAAO,IAAI,IAAI,CAAC,UAAU,CAAC,OAAO,CAAC,EAAE,CAAC;4BACxC,IAAI,GAAG,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC;wBACpC,CAAC;wBACD,qBAAqB;wBACrB,IAAI,GAAG,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC;wBAE1B,4EAA4E;wBAC5E,MAAM,WAAW,GAAG,kBAAkB,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,IAAI,SAAS,CAAC;wBAE3D,SAAS,CAAC,IAAI,CAAC,EAAE,MAAM,EAAE,IAAI,EAAE,WAAW,EAAE,CAAC,CAAC;oBAChD,CAAC;gBACH,CAAC;YACH,CAAC;QACH,CAAC;IACH,CAAC,CAAC,CAAC;IAEH,oEAAoE;IACpE,CAAC,CAAC,2DAA2D,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,EAAE,EAAE;QAC5E,MAAM,UAAU,GAAG,CAAC,CAAC,EAAE,CAAC,CAAC,IAAI,EAAE,CAAC,IAAI,EAAE,CAAC,WAAW,EAAE,CAAC;QACrD,IAAI,CAAC,YAAY,CAAC,QAAQ,CAAC,UAAU,CAAC;YAAE,OAAO;QAE/C,iCAAiC;QACjC,MAAM,QAAQ,GAAG;YACf,CAAC,CAAC,EAAE,CAAC,CAAC,IAAI,CAAC,8DAA8D,CAAC;YAC1E,CAAC,CAAC,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,KAAK,EAAE;SACpC,CAAC;QACF,KAAK,MAAM,OAAO,IAAI,QAAQ,EAAE,CAAC;YAC/B,IAAI,OAAO,CAAC,MAAM,EAAE,CAAC;gBACnB,MAAM,IAAI,GAAG,OAAO,CAAC,IAAI,EAAE,CAAC,IAAI,EAAE,CAAC;gBACnC,IAAI,gBAAgB,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC;oBAChC,SAAS,CAAC,IAAI,CAAC,EAAE,MAAM,EAAE,UAAU,EAAE,IAAI,EAAE,CAAC,CAAC;oBAC7C,MAAM;gBACR,CAAC;YACH,CAAC;QACH,CAAC;IACH,CAAC,CAAC,CAAC;IAEH,6BAA6B;IAC7B,MAAM,IAAI,GAAG,IAAI,GAAG,EAAU,CAAC;IAC/B,MAAM,MAAM,GAAG,SAAS,CAAC,MAAM,CAAC,CAAC,EAAE,EAAE,EAAE;QACrC,MAAM,GAAG,GAAG,GAAG,EAAE,CAAC,MAAM,IAAI,EAAE,CAAC,IAAI,EAAE,CAAC;QACtC,IAAI,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC;YAAE,OAAO,KAAK,CAAC;QAChC,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;QACd,OAAO,IAAI,CAAC;IACd,CAAC,CAAC,CAAC;IAEH,OAAO,EAAE,IAAI,EAAE,UAAU,EAAE,OAAO,EAAE,SAAS,EAAE,MAAM,EAAE,CAAC;AAC1D,CAAC;AAED,SAAS,kBAAkB,CAAC,GAA2B;IACrD,kEAAkE;IAClE,IAAI,OAAO,GAAG,GAAG,CAAC,IAAI,EAAE,CAAC;IACzB,IAAI,KAAK,GAAG,CAAC,CAAC;IACd,OAAO,KAAK,GAAG,CAAC,EAAE,CAAC;QACjB,IAAI,OAAO,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YACzB,MAAM,MAAM,GAAG,GAAG,CAAC,MAAM,EAAE,CAAC;YAC5B,IAAI,CAAC,MAAM,CAAC,MAAM;gBAAE,MAAM;YAC1B,OAAO,GAAG,MAAM,CAAC,IAAI,EAAE,CAAC;QAC1B,CAAC;aAAM,IAAI,OAAO,CAAC,EAAE,CAAC,mBAAmB,CAAC,EAAE,CAAC;YAC3C,OAAO,OAAO,CAAC,IAAI,EAAE,CAAC,IAAI,EAAE,CAAC;QAC/B,CAAC;aAAM,CAAC;YACN,OAAO,GAAG,OAAO,CAAC,IAAI,EAAE,CAAC;QAC3B,CAAC;QACD,KAAK,EAAE,CAAC;IACV,CAAC;IACD,OAAO,IAAI,CAAC;AACd,CAAC;AAED,8EAA8E;AAC9E,oBAAoB;AACpB,8EAA8E;AAE9E;;GAEG;AACH,OAAO,EAAE,cAAc,IAAI,OAAO,EAAE,CAAC;AAErC;;GAEG;AACH,MAAM,UAAU,WAAW,CAAC,IAAY,EAAE,GAAW;IACnD,MAAM,IAAI,GAAG,cAAc,CAAC,IAAI,EAAE,GAAG,CAAC,CAAC;IACvC,MAAM,CAAC,GAAG,IAAI,CAAC,IAAI,CAAC,CAAC;IAErB,IAAI,CAAC;QACH,QAAQ,IAAI,EAAE,CAAC;YACb,KAAK,SAAS;gBACZ,OAAO,EAAE,IAAI,EAAE,SAAS,EAAE,KAAK,EAAE,mBAAmB,CAAC,CAAC,CAAC,EAAE,CAAC;YAE5D,KAAK,UAAU;gBACb,OAAO,EAAE,IAAI,EAAE,UAAU,EAAE,KAAK,EAAE,eAAe,CAAC,CAAC,EAAE,GAAG,CAAC,EAAE,CAAC;YAE9D,KAAK,SAAS;gBACZ,OAAO,cAAc,CAAC,CAAC,CAAC,CAAC;YAE3B,KAAK,SAAS;gBACZ,OAAO,cAAc,CAAC,CAAC,CAAC,CAAC;YAE3B,KAAK,UAAU;gBACb,OAAO,cAAc,CAAC,CAAC,EAAE,GAAG,CAAC,CAAC;YAEhC;gBACE,OAAO,EAAE,IAAI,EAAE,SAAS,EAAE,CAAC;QAC/B,CAAC;IACH,CAAC;IAAC,MAAM,CAAC;QACP,mDAAmD;QACnD,QAAQ,IAAI,EAAE,CAAC;YACb,KAAK,SAAS;gBACZ,OAAO,EAAE,IAAI,EAAE,SAAS,EAAE,KAAK,EAAE,EAAE,EAAE,CAAC;YACxC,KAAK,UAAU;gBACb,OAAO,EAAE,IAAI,EAAE,UAAU,EAAE,KAAK,EAAE,EAAE,EAAE,CAAC;YACzC,KAAK,SAAS;gBACZ,OAAO,EAAE,IAAI,EAAE,SAAS,EAAE,MAAM,EAAE,EAAE,EAAE,MAAM,EAAE,EAAE,EAAE,SAAS,EAAE,EAAE,EAAE,MAAM,EAAE,EAAE,EAAE,CAAC;YAChF,KAAK,SAAS;gBACZ,OAAO,EAAE,IAAI,EAAE,SAAS,EAAE,QAAQ,EAAE,EAAE,EAAE,CAAC;YAC3C,KAAK,UAAU;gBACb,OAAO,EAAE,IAAI,EAAE,UAAU,EAAE,SAAS,EAAE,EAAE,EAAE,CAAC;YAC7C;gBACE,OAAO,EAAE,IAAI,EAAE,SAAS,EAAE,CAAC;QAC/B,CAAC;IACH,CAAC;AACH,CAAC"}
@@ -0,0 +1,75 @@
1
+ /**
2
+ * Deep Fetch — Web Intelligence Module
3
+ *
4
+ * Transforms "search + fetch" into "search + fetch + synthesize + structure".
5
+ * No LLM required — pure heuristic signal extraction, BM25 relevance scoring,
6
+ * deduplication, entity/number extraction, and comparison detection.
7
+ */
8
+ export interface DeepFetchOptions {
9
+ query: string;
10
+ /** Number of sources to fetch (default: 5, max: 10) */
11
+ count?: number;
12
+ /** Output format (default: 'merged') */
13
+ format?: 'merged' | 'structured' | 'comparison';
14
+ /** Approximate max characters in merged output (default: 32000 ≈ 8k tokens) */
15
+ maxChars?: number;
16
+ /** Minimum BM25 relevance score (0-1) to include a source (default: 0.05) */
17
+ relevanceThreshold?: number;
18
+ }
19
+ export interface SourceResult {
20
+ url: string;
21
+ title: string;
22
+ relevanceScore: number;
23
+ keyPoints: string[];
24
+ fetchedAt: string;
25
+ }
26
+ export interface StructuredData {
27
+ facts: string[];
28
+ entities: string[];
29
+ dates: string[];
30
+ numbers: Record<string, string>;
31
+ }
32
+ export interface ComparisonData {
33
+ columns: string[];
34
+ rows: Record<string, Record<string, string>>;
35
+ }
36
+ export interface DeepFetchResult {
37
+ query: string;
38
+ format: string;
39
+ sources: SourceResult[];
40
+ merged: string;
41
+ structured?: StructuredData;
42
+ comparison?: ComparisonData;
43
+ elapsed: number;
44
+ }
45
+ /**
46
+ * Extract up to `maxPoints` key points from content, ranked by signal value.
47
+ */
48
+ export declare function extractKeyPoints(content: string, query: string, maxPoints?: number): string[];
49
+ /**
50
+ * Deduplicate a list of sentences.
51
+ * When two sentences are >60% similar, keep the longer (more detailed) one.
52
+ */
53
+ export declare function deduplicateSentences(sentences: string[], threshold?: number): string[];
54
+ /** Extract numbers, prices, percentages from text. */
55
+ export declare function extractNumbers(text: string): Record<string, string>;
56
+ /** Extract dates from text. */
57
+ export declare function extractDates(text: string): string[];
58
+ /**
59
+ * Extract named entities (proper nouns) that appear in at least 2 sources.
60
+ * Simple heuristic: capitalized words/phrases not at sentence start.
61
+ */
62
+ export declare function extractEntities(texts: string[]): string[];
63
+ /** Detect if the query is a comparison query. */
64
+ export declare function isComparisonQuery(query: string): boolean;
65
+ /**
66
+ * Extract the entities being compared from a query.
67
+ * Handles patterns like "A vs B", "compare A and B", "A or B".
68
+ */
69
+ export declare function extractComparedEntities(query: string): string[];
70
+ /**
71
+ * Build a comparison table from merged content and entity names.
72
+ */
73
+ export declare function buildComparisonTable(content: string, entities: string[]): ComparisonData | undefined;
74
+ export declare function deepFetch(options: DeepFetchOptions): Promise<DeepFetchResult>;
75
+ //# sourceMappingURL=deep-fetch.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"deep-fetch.d.ts","sourceRoot":"","sources":["../../src/core/deep-fetch.ts"],"names":[],"mappings":"AAAA;;;;;;GAMG;AAUH,MAAM,WAAW,gBAAgB;IAC/B,KAAK,EAAE,MAAM,CAAC;IACd,uDAAuD;IACvD,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,wCAAwC;IACxC,MAAM,CAAC,EAAE,QAAQ,GAAG,YAAY,GAAG,YAAY,CAAC;IAChD,+EAA+E;IAC/E,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,6EAA6E;IAC7E,kBAAkB,CAAC,EAAE,MAAM,CAAC;CAC7B;AAED,MAAM,WAAW,YAAY;IAC3B,GAAG,EAAE,MAAM,CAAC;IACZ,KAAK,EAAE,MAAM,CAAC;IACd,cAAc,EAAE,MAAM,CAAC;IACvB,SAAS,EAAE,MAAM,EAAE,CAAC;IACpB,SAAS,EAAE,MAAM,CAAC;CACnB;AAED,MAAM,WAAW,cAAc;IAC7B,KAAK,EAAE,MAAM,EAAE,CAAC;IAChB,QAAQ,EAAE,MAAM,EAAE,CAAC;IACnB,KAAK,EAAE,MAAM,EAAE,CAAC;IAChB,OAAO,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;CACjC;AAED,MAAM,WAAW,cAAc;IAC7B,OAAO,EAAE,MAAM,EAAE,CAAC;IAClB,IAAI,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC,CAAC;CAC9C;AAED,MAAM,WAAW,eAAe;IAC9B,KAAK,EAAE,MAAM,CAAC;IACd,MAAM,EAAE,MAAM,CAAC;IACf,OAAO,EAAE,YAAY,EAAE,CAAC;IACxB,MAAM,EAAE,MAAM,CAAC;IACf,UAAU,CAAC,EAAE,cAAc,CAAC;IAC5B,UAAU,CAAC,EAAE,cAAc,CAAC;IAC5B,OAAO,EAAE,MAAM,CAAC;CACjB;AA+ED;;GAEG;AACH,wBAAgB,gBAAgB,CAAC,OAAO,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,SAAS,SAAI,GAAG,MAAM,EAAE,CAgBxF;AAwBD;;;GAGG;AACH,wBAAgB,oBAAoB,CAAC,SAAS,EAAE,MAAM,EAAE,EAAE,SAAS,SAAM,GAAG,MAAM,EAAE,CAsBnF;AAWD,sDAAsD;AACtD,wBAAgB,cAAc,CAAC,IAAI,EAAE,MAAM,GAAG,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAyBnE;AAED,+BAA+B;AAC/B,wBAAgB,YAAY,CAAC,IAAI,EAAE,MAAM,GAAG,MAAM,EAAE,CAGnD;AAED;;;GAGG;AACH,wBAAgB,eAAe,CAAC,KAAK,EAAE,MAAM,EAAE,GAAG,MAAM,EAAE,CA8BzD;AAQD,iDAAiD;AACjD,wBAAgB,iBAAiB,CAAC,KAAK,EAAE,MAAM,GAAG,OAAO,CAGxD;AAED;;;GAGG;AACH,wBAAgB,uBAAuB,CAAC,KAAK,EAAE,MAAM,GAAG,MAAM,EAAE,CAmB/D;AAWD;;GAEG;AACH,wBAAgB,oBAAoB,CAAC,OAAO,EAAE,MAAM,EAAE,QAAQ,EAAE,MAAM,EAAE,GAAG,cAAc,GAAG,SAAS,CA4BpG;AAuDD,wBAAsB,SAAS,CAAC,OAAO,EAAE,gBAAgB,GAAG,OAAO,CAAC,eAAe,CAAC,CAgInF"}
@@ -0,0 +1,406 @@
1
+ /**
2
+ * Deep Fetch — Web Intelligence Module
3
+ *
4
+ * Transforms "search + fetch" into "search + fetch + synthesize + structure".
5
+ * No LLM required — pure heuristic signal extraction, BM25 relevance scoring,
6
+ * deduplication, entity/number extraction, and comparison detection.
7
+ */
8
+ import { peelBatch } from '../index.js';
9
+ import { computeRelevanceScore } from './bm25-filter.js';
10
+ import { getBestSearchProvider } from './search-provider.js';
11
+ // ---------------------------------------------------------------------------
12
+ // Key-point extraction
13
+ // ---------------------------------------------------------------------------
14
+ const SIGNAL_WORDS = new Set([
15
+ 'announced', 'launched', 'released', 'costs', 'requires', 'supports',
16
+ 'offers', 'provides', 'includes', 'features', 'enables', 'allows',
17
+ 'introduces', 'reveals', 'claims', 'states', 'reports', 'shows',
18
+ 'found', 'discovered', 'improved', 'updated', 'deprecated', 'removed',
19
+ 'increased', 'decreased', 'grew', 'declined', 'reached', 'exceeded',
20
+ ]);
21
+ /** Split text into sentences (rough but fast — no NLP library needed). */
22
+ function splitSentences(text) {
23
+ // Strip markdown formatting
24
+ const plain = text
25
+ .replace(/```[\s\S]*?```/g, ' ')
26
+ .replace(/`[^`]+`/g, ' ')
27
+ .replace(/!\[.*?\]\(.*?\)/g, ' ')
28
+ .replace(/\[([^\]]*)\]\([^)]*\)/g, '$1')
29
+ .replace(/^#{1,6}\s+/gm, '')
30
+ .replace(/[*_~`>|\\]/g, ' ')
31
+ .replace(/\s+/g, ' ')
32
+ .trim();
33
+ // Split on sentence-ending punctuation followed by space + capital
34
+ return plain
35
+ .split(/(?<=[.!?])\s+(?=[A-Z"'])/)
36
+ .map(s => s.trim())
37
+ .filter(s => s.length > 20 && s.length < 500);
38
+ }
39
+ /** Extract first sentence of each paragraph (topic sentences). */
40
+ function topicSentences(content) {
41
+ const paragraphs = content.split(/\n{2,}/);
42
+ const result = [];
43
+ for (const para of paragraphs) {
44
+ const sentences = splitSentences(para);
45
+ if (sentences.length > 0) {
46
+ result.push(sentences[0]);
47
+ }
48
+ }
49
+ return result;
50
+ }
51
+ /** Score a sentence for "key point" worthiness. Higher = more useful. */
52
+ function sentenceScore(sentence, queryTerms) {
53
+ const lower = sentence.toLowerCase();
54
+ const words = lower.split(/\s+/);
55
+ let score = 0;
56
+ // Query term overlap
57
+ let queryHits = 0;
58
+ for (const term of queryTerms) {
59
+ if (lower.includes(term))
60
+ queryHits++;
61
+ }
62
+ score += (queryHits / Math.max(queryTerms.size, 1)) * 3;
63
+ // Numbers / statistics
64
+ const numberMatches = sentence.match(/\b\d[\d,.]*\b|\$[\d,.]+|[\d.]+%/g);
65
+ if (numberMatches)
66
+ score += Math.min(numberMatches.length * 0.5, 2);
67
+ // Signal words
68
+ for (const word of words) {
69
+ if (SIGNAL_WORDS.has(word)) {
70
+ score += 1;
71
+ break;
72
+ }
73
+ }
74
+ // Prefer medium-length sentences
75
+ if (sentence.length > 60 && sentence.length < 300)
76
+ score += 0.5;
77
+ return score;
78
+ }
79
+ /**
80
+ * Extract up to `maxPoints` key points from content, ranked by signal value.
81
+ */
82
+ export function extractKeyPoints(content, query, maxPoints = 5) {
83
+ const queryTerms = new Set(query.toLowerCase().split(/\s+/).filter(t => t.length > 2));
84
+ const allSentences = splitSentences(content);
85
+ const topics = topicSentences(content);
86
+ // Combine and deduplicate
87
+ const candidates = [...new Set([...allSentences, ...topics])];
88
+ // Score and sort
89
+ const scored = candidates.map(s => ({ s, score: sentenceScore(s, queryTerms) }));
90
+ scored.sort((a, b) => b.score - a.score);
91
+ return scored.slice(0, maxPoints).map(x => x.s);
92
+ }
93
+ // ---------------------------------------------------------------------------
94
+ // Deduplication
95
+ // ---------------------------------------------------------------------------
96
+ /** Normalize a sentence for comparison. */
97
+ function normalize(s) {
98
+ return s.toLowerCase().replace(/[^\w\s]/g, '').replace(/\s+/g, ' ').trim();
99
+ }
100
+ /** Jaccard similarity on word sets (fast approximation). */
101
+ function similarity(a, b) {
102
+ const wa = new Set(normalize(a).split(' ').filter(w => w.length > 2));
103
+ const wb = new Set(normalize(b).split(' ').filter(w => w.length > 2));
104
+ if (wa.size === 0 && wb.size === 0)
105
+ return 1;
106
+ let intersection = 0;
107
+ for (const w of wa) {
108
+ if (wb.has(w))
109
+ intersection++;
110
+ }
111
+ const union = wa.size + wb.size - intersection;
112
+ return union === 0 ? 0 : intersection / union;
113
+ }
114
+ /**
115
+ * Deduplicate a list of sentences.
116
+ * When two sentences are >60% similar, keep the longer (more detailed) one.
117
+ */
118
+ export function deduplicateSentences(sentences, threshold = 0.6) {
119
+ const kept = [];
120
+ for (const candidate of sentences) {
121
+ let dominated = false;
122
+ for (let i = 0; i < kept.length; i++) {
123
+ const sim = similarity(candidate, kept[i]);
124
+ if (sim >= threshold) {
125
+ // Keep the longer one
126
+ if (candidate.length > kept[i].length) {
127
+ kept[i] = candidate;
128
+ }
129
+ dominated = true;
130
+ break;
131
+ }
132
+ }
133
+ if (!dominated) {
134
+ kept.push(candidate);
135
+ }
136
+ }
137
+ return kept;
138
+ }
139
+ // ---------------------------------------------------------------------------
140
+ // Number / entity extraction
141
+ // ---------------------------------------------------------------------------
142
+ const PRICE_RE = /(?:\$|€|£|¥)\s?[\d,]+(?:\.\d+)?(?:\s?(?:\/mo(?:nth)?|\/yr|\/year|\/user|\/month))?/gi;
143
+ const PERCENT_RE = /\d+(?:\.\d+)?\s?%/g;
144
+ const COUNT_RE = /\d+(?:\.\d+)?\s?(?:million|billion|thousand|M|B|K)\s?\+?(?:\s?(?:users?|customers?|downloads?|installs?))?/gi;
145
+ const DATE_RE = /(?:January|February|March|April|May|June|July|August|September|October|November|December)\s+\d{1,2},?\s+\d{4}|\d{4}-\d{2}-\d{2}|Q[1-4]\s+\d{4}/g;
146
+ /** Extract numbers, prices, percentages from text. */
147
+ export function extractNumbers(text) {
148
+ const result = {};
149
+ const prices = text.match(PRICE_RE);
150
+ if (prices) {
151
+ prices.slice(0, 5).forEach((p, i) => {
152
+ result[`price_${i + 1}`] = p.trim();
153
+ });
154
+ }
155
+ const percents = text.match(PERCENT_RE);
156
+ if (percents) {
157
+ percents.slice(0, 5).forEach((p, i) => {
158
+ result[`percent_${i + 1}`] = p.trim();
159
+ });
160
+ }
161
+ const counts = text.match(COUNT_RE);
162
+ if (counts) {
163
+ counts.slice(0, 5).forEach((c, i) => {
164
+ result[`count_${i + 1}`] = c.trim();
165
+ });
166
+ }
167
+ return result;
168
+ }
169
+ /** Extract dates from text. */
170
+ export function extractDates(text) {
171
+ const matches = text.match(DATE_RE) ?? [];
172
+ return [...new Set(matches)].slice(0, 10);
173
+ }
174
+ /**
175
+ * Extract named entities (proper nouns) that appear in at least 2 sources.
176
+ * Simple heuristic: capitalized words/phrases not at sentence start.
177
+ */
178
+ export function extractEntities(texts) {
179
+ // Collect capitalized words/phrases from each source.
180
+ // Matches: standard proper nouns (New York), CamelCase brands (PayPal, GitHub),
181
+ // and ALL-CAPS acronyms (AI, API) with length >= 2.
182
+ const ENTITY_RE = /\b([A-Z][A-Za-z0-9]*(?:[A-Z][a-z0-9]+)*(?:\s+[A-Z][A-Za-z0-9]*(?:[A-Z][a-z0-9]+)*)*)\b/g;
183
+ const termFreq = new Map();
184
+ for (const text of texts) {
185
+ const seen = new Set();
186
+ let m;
187
+ ENTITY_RE.lastIndex = 0;
188
+ while ((m = ENTITY_RE.exec(text)) !== null) {
189
+ const term = m[1];
190
+ if (term.length < 3)
191
+ continue;
192
+ // Skip common sentence starters / stopwords
193
+ if (/^(The|A|An|In|On|At|To|For|Of|And|Or|But|This|That|These|Those|It|He|She|They|We|You|I)$/.test(term))
194
+ continue;
195
+ if (!seen.has(term)) {
196
+ seen.add(term);
197
+ termFreq.set(term, (termFreq.get(term) ?? 0) + 1);
198
+ }
199
+ }
200
+ }
201
+ // Only return entities that appear in 2+ sources
202
+ return [...termFreq.entries()]
203
+ .filter(([, freq]) => freq >= 2)
204
+ .sort((a, b) => b[1] - a[1])
205
+ .slice(0, 20)
206
+ .map(([term]) => term);
207
+ }
208
+ // ---------------------------------------------------------------------------
209
+ // Comparison mode
210
+ // ---------------------------------------------------------------------------
211
+ const COMPARISON_TRIGGERS = ['vs', 'versus', 'compare', 'comparison', 'difference', 'differences', 'alternative', 'alternatives'];
212
+ /** Detect if the query is a comparison query. */
213
+ export function isComparisonQuery(query) {
214
+ const lower = query.toLowerCase();
215
+ return COMPARISON_TRIGGERS.some(t => lower.includes(t));
216
+ }
217
+ /**
218
+ * Extract the entities being compared from a query.
219
+ * Handles patterns like "A vs B", "compare A and B", "A or B".
220
+ */
221
+ export function extractComparedEntities(query) {
222
+ const vsMatch = query.match(/(.+?)\s+(?:vs\.?|versus)\s+(.+)/i);
223
+ if (vsMatch) {
224
+ return [vsMatch[1].trim(), vsMatch[2].trim()].map(e => e.replace(/^(?:compare|difference|between)\s+/i, '').trim());
225
+ }
226
+ const compareMatch = query.match(/compare\s+(.+?)\s+(?:and|to|with|vs)\s+(.+)/i);
227
+ if (compareMatch) {
228
+ return [compareMatch[1].trim(), compareMatch[2].trim()];
229
+ }
230
+ const diffMatch = query.match(/difference(?:s)?\s+between\s+(.+?)\s+and\s+(.+)/i);
231
+ if (diffMatch) {
232
+ return [diffMatch[1].trim(), diffMatch[2].trim()];
233
+ }
234
+ return [];
235
+ }
236
+ const COMPARISON_ATTRIBUTES = [
237
+ { name: 'price', patterns: [/(?:price|cost|pricing|fee|subscription)[:\s]+([^.\n]+)/i, /(\$[\d,.]+(?:\/\w+)?)/] },
238
+ { name: 'features', patterns: [/(?:features?|capabilities|supports?|includes?)[:\s]+([^.\n]+)/i] },
239
+ { name: 'pros', patterns: [/(?:pros?|advantages?|benefits?|strengths?)[:\s]+([^.\n]+)/i] },
240
+ { name: 'cons', patterns: [/(?:cons?|disadvantages?|drawbacks?|weaknesses?|limitations?)[:\s]+([^.\n]+)/i] },
241
+ { name: 'platform', patterns: [/(?:platform|works?\s+(?:on|with)|available\s+(?:on|for))[:\s]+([^.\n]+)/i] },
242
+ { name: 'rating', patterns: [/(?:rating|score|stars?)[:\s]+([^.\n]+)/i, /(\d+(?:\.\d+)?\s*\/\s*\d+\s*(?:stars?))/i] },
243
+ ];
244
+ /**
245
+ * Build a comparison table from merged content and entity names.
246
+ */
247
+ export function buildComparisonTable(content, entities) {
248
+ if (entities.length < 2)
249
+ return undefined;
250
+ const columns = COMPARISON_ATTRIBUTES.map(a => a.name);
251
+ const rows = {};
252
+ for (const entity of entities) {
253
+ rows[entity] = {};
254
+ // Find paragraphs mentioning this entity
255
+ const lines = content.split(/\n+/);
256
+ const relevant = lines.filter(l => l.toLowerCase().includes(entity.toLowerCase()));
257
+ const entityText = relevant.join(' ');
258
+ for (const attr of COMPARISON_ATTRIBUTES) {
259
+ for (const pattern of attr.patterns) {
260
+ const m = entityText.match(pattern);
261
+ if (m && m[1]) {
262
+ rows[entity][attr.name] = m[1].trim().slice(0, 120);
263
+ break;
264
+ }
265
+ }
266
+ if (!rows[entity][attr.name]) {
267
+ rows[entity][attr.name] = 'N/A';
268
+ }
269
+ }
270
+ }
271
+ return { columns, rows };
272
+ }
273
+ // ---------------------------------------------------------------------------
274
+ // Smart content merging
275
+ // ---------------------------------------------------------------------------
276
+ /**
277
+ * Merge content from multiple sources intelligently:
278
+ * - Sort by relevance (most relevant first)
279
+ * - Add source attribution
280
+ * - Truncate to maxChars
281
+ */
282
+ function mergeContent(pages, maxChars) {
283
+ // Sort most-relevant first
284
+ const sorted = [...pages].sort((a, b) => b.relevanceScore - a.relevanceScore);
285
+ const parts = [];
286
+ let totalChars = 0;
287
+ for (const page of sorted) {
288
+ if (!page.content)
289
+ continue;
290
+ const header = `## [${page.title}](${page.url})\n\n`;
291
+ const body = page.content;
292
+ const section = `${header}${body}\n\n---\n\n`;
293
+ if (totalChars + section.length > maxChars) {
294
+ // Add truncated version
295
+ const remaining = maxChars - totalChars - header.length - 20;
296
+ if (remaining > 200) {
297
+ parts.push(`${header}${body.slice(0, remaining)}...\n\n---\n\n`);
298
+ }
299
+ break;
300
+ }
301
+ parts.push(section);
302
+ totalChars += section.length;
303
+ }
304
+ return parts.join('');
305
+ }
306
+ // ---------------------------------------------------------------------------
307
+ // Main deepFetch function
308
+ // ---------------------------------------------------------------------------
309
+ function timeout(ms, label) {
310
+ return new Promise((_, reject) => setTimeout(() => reject(new Error(`${label} timed out after ${ms}ms`)), ms));
311
+ }
312
+ export async function deepFetch(options) {
313
+ const { query, count = 5, format = 'merged', maxChars = 32000, relevanceThreshold = 0.05, } = options;
314
+ const startTime = Date.now();
315
+ const safeCount = Math.min(Math.max(count, 1), 10);
316
+ // ── Step 1: Search ────────────────────────────────────────────────────────
317
+ const { provider, apiKey } = getBestSearchProvider();
318
+ const searchResults = await Promise.race([
319
+ provider.searchWeb(query, { count: safeCount + 2, apiKey }), // fetch extras, filter low-relevance
320
+ timeout(30000, 'Search'),
321
+ ]);
322
+ const topResults = (Array.isArray(searchResults) ? searchResults : searchResults?.results ?? [])
323
+ .slice(0, safeCount + 2);
324
+ if (topResults.length === 0) {
325
+ return {
326
+ query,
327
+ format,
328
+ sources: [],
329
+ merged: '',
330
+ elapsed: Date.now() - startTime,
331
+ };
332
+ }
333
+ // ── Step 2: Fetch all URLs in parallel ────────────────────────────────────
334
+ const urls = topResults.map((r) => r.url).filter(Boolean);
335
+ const pages = await Promise.race([
336
+ peelBatch(urls, { concurrency: 5, format: 'markdown' }),
337
+ timeout(120000, 'Batch fetch'),
338
+ ]);
339
+ const scoredPages = [];
340
+ for (let i = 0; i < pages.length; i++) {
341
+ const page = pages[i];
342
+ const searchResult = topResults[i];
343
+ const url = urls[i];
344
+ if (!page || page.error)
345
+ continue;
346
+ const content = page.content || '';
347
+ const title = page.title || searchResult?.title || url;
348
+ const relevanceScore = computeRelevanceScore(content, query);
349
+ if (relevanceScore < relevanceThreshold && scoredPages.length >= 2) {
350
+ // Skip low-relevance if we already have enough
351
+ continue;
352
+ }
353
+ const keyPoints = extractKeyPoints(content, query, 5);
354
+ scoredPages.push({
355
+ url,
356
+ title,
357
+ content,
358
+ relevanceScore,
359
+ keyPoints,
360
+ fetchedAt: new Date().toISOString(),
361
+ });
362
+ if (scoredPages.length >= safeCount)
363
+ break;
364
+ }
365
+ // ── Step 4: Merge content ─────────────────────────────────────────────────
366
+ const mergedContent = mergeContent(scoredPages, maxChars);
367
+ // ── Step 5: Build sources list ────────────────────────────────────────────
368
+ const sources = scoredPages.map(p => ({
369
+ url: p.url,
370
+ title: p.title,
371
+ relevanceScore: Math.round(p.relevanceScore * 1000) / 1000,
372
+ keyPoints: p.keyPoints,
373
+ fetchedAt: p.fetchedAt,
374
+ }));
375
+ const result = {
376
+ query,
377
+ format,
378
+ sources,
379
+ merged: mergedContent,
380
+ elapsed: Date.now() - startTime,
381
+ };
382
+ // ── Step 6: Structured extraction (optional) ──────────────────────────────
383
+ if (format === 'structured' || format === 'comparison') {
384
+ const allTexts = scoredPages.map(p => p.content);
385
+ const allFacts = scoredPages.flatMap(p => p.keyPoints);
386
+ const deduplicatedFacts = deduplicateSentences(allFacts);
387
+ const entities = extractEntities(allTexts);
388
+ const dates = extractDates(mergedContent);
389
+ const numbers = extractNumbers(mergedContent);
390
+ result.structured = {
391
+ facts: deduplicatedFacts,
392
+ entities,
393
+ dates,
394
+ numbers,
395
+ };
396
+ }
397
+ // ── Step 7: Comparison table (optional) ───────────────────────────────────
398
+ if (format === 'comparison' || isComparisonQuery(query)) {
399
+ const comparedEntities = extractComparedEntities(query);
400
+ if (comparedEntities.length >= 2) {
401
+ result.comparison = buildComparisonTable(mergedContent, comparedEntities);
402
+ }
403
+ }
404
+ return result;
405
+ }
406
+ //# sourceMappingURL=deep-fetch.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"deep-fetch.js","sourceRoot":"","sources":["../../src/core/deep-fetch.ts"],"names":[],"mappings":"AAAA;;;;;;GAMG;AAEH,OAAO,EAAE,SAAS,EAAE,MAAM,aAAa,CAAC;AACxC,OAAO,EAAE,qBAAqB,EAAE,MAAM,kBAAkB,CAAC;AACzD,OAAO,EAAE,qBAAqB,EAAE,MAAM,sBAAsB,CAAC;AAgD7D,8EAA8E;AAC9E,uBAAuB;AACvB,8EAA8E;AAE9E,MAAM,YAAY,GAAG,IAAI,GAAG,CAAC;IAC3B,WAAW,EAAE,UAAU,EAAE,UAAU,EAAE,OAAO,EAAE,UAAU,EAAE,UAAU;IACpE,QAAQ,EAAE,UAAU,EAAE,UAAU,EAAE,UAAU,EAAE,SAAS,EAAE,QAAQ;IACjE,YAAY,EAAE,SAAS,EAAE,QAAQ,EAAE,QAAQ,EAAE,SAAS,EAAE,OAAO;IAC/D,OAAO,EAAE,YAAY,EAAE,UAAU,EAAE,SAAS,EAAE,YAAY,EAAE,SAAS;IACrE,WAAW,EAAE,WAAW,EAAE,MAAM,EAAE,UAAU,EAAE,SAAS,EAAE,UAAU;CACpE,CAAC,CAAC;AAEH,0EAA0E;AAC1E,SAAS,cAAc,CAAC,IAAY;IAClC,4BAA4B;IAC5B,MAAM,KAAK,GAAG,IAAI;SACf,OAAO,CAAC,iBAAiB,EAAE,GAAG,CAAC;SAC/B,OAAO,CAAC,UAAU,EAAE,GAAG,CAAC;SACxB,OAAO,CAAC,kBAAkB,EAAE,GAAG,CAAC;SAChC,OAAO,CAAC,wBAAwB,EAAE,IAAI,CAAC;SACvC,OAAO,CAAC,cAAc,EAAE,EAAE,CAAC;SAC3B,OAAO,CAAC,aAAa,EAAE,GAAG,CAAC;SAC3B,OAAO,CAAC,MAAM,EAAE,GAAG,CAAC;SACpB,IAAI,EAAE,CAAC;IAEV,mEAAmE;IACnE,OAAO,KAAK;SACT,KAAK,CAAC,0BAA0B,CAAC;SACjC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;SAClB,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,MAAM,GAAG,EAAE,IAAI,CAAC,CAAC,MAAM,GAAG,GAAG,CAAC,CAAC;AAClD,CAAC;AAED,kEAAkE;AAClE,SAAS,cAAc,CAAC,OAAe;IACrC,MAAM,UAAU,GAAG,OAAO,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAC;IAC3C,MAAM,MAAM,GAAa,EAAE,CAAC;IAC5B,KAAK,MAAM,IAAI,IAAI,UAAU,EAAE,CAAC;QAC9B,MAAM,SAAS,GAAG,cAAc,CAAC,IAAI,CAAC,CAAC;QACvC,IAAI,SAAS,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YACzB,MAAM,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,CAAC;QAC5B,CAAC;IACH,CAAC;IACD,OAAO,MAAM,CAAC;AAChB,CAAC;AAED,yEAAyE;AACzE,SAAS,aAAa,CAAC,QAAgB,EAAE,UAAuB;IAC9D,MAAM,KAAK,GAAG,QAAQ,CAAC,WAAW,EAAE,CAAC;IACrC,MAAM,KAAK,GAAG,KAAK,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;IAEjC,IAAI,KAAK,GAAG,CAAC,CAAC;IAEd,qBAAqB;IACrB,IAAI,SAAS,GAAG,CAAC,CAAC;IAClB,KAAK,MAAM,IAAI,IAAI,UAAU,EAAE,CAAC;QAC9B,IAAI,KAAK,CAAC,QAAQ,CAAC,IAAI,CAAC;YAAE,SAAS,EAAE,CAAC;IACxC,CAAC;IACD,KAAK,IAAI,CAAC,SAAS,GAAG,IAAI,CAAC,GAAG,CAAC,UAAU,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC;IAExD,uBAAuB;IACvB,MAAM,aAAa,GAAG,QAAQ,CAAC,KAAK,CAAC,kCAAkC,CAAC,CAAC;IACzE,IAAI,aAAa;QAAE,KAAK,IAAI,IAAI,CAAC,GAAG,CAAC,aAAa,CAAC,MAAM,GAAG,GAAG,EAAE,CAAC,CAAC,CAAC;IAEpE,eAAe;IACf,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;QACzB,IAAI,YAAY,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC;YAC3B,KAAK,IAAI,CAAC,CAAC;YACX,MAAM;QACR,CAAC;IACH,CAAC;IAED,iCAAiC;IACjC,IAAI,QAAQ,CAAC,MAAM,GAAG,EAAE,IAAI,QAAQ,CAAC,MAAM,GAAG,GAAG;QAAE,KAAK,IAAI,GAAG,CAAC;IAEhE,OAAO,KAAK,CAAC;AACf,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,gBAAgB,CAAC,OAAe,EAAE,KAAa,EAAE,SAAS,GAAG,CAAC;IAC5E,MAAM,UAAU,GAAG,IAAI,GAAG,CACxB,KAAK,CAAC,WAAW,EAAE,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,MAAM,GAAG,CAAC,CAAC,CAC3D,CAAC;IAEF,MAAM,YAAY,GAAG,cAAc,CAAC,OAAO,CAAC,CAAC;IAC7C,MAAM,MAAM,GAAG,cAAc,CAAC,OAAO,CAAC,CAAC;IAEvC,0BAA0B;IAC1B,MAAM,UAAU,GAAG,CAAC,GAAG,IAAI,GAAG,CAAC,CAAC,GAAG,YAAY,EAAE,GAAG,MAAM,CAAC,CAAC,CAAC,CAAC;IAE9D,iBAAiB;IACjB,MAAM,MAAM,GAAG,UAAU,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,EAAE,CAAC,EAAE,KAAK,EAAE,aAAa,CAAC,CAAC,EAAE,UAAU,CAAC,EAAE,CAAC,CAAC,CAAC;IACjF,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,GAAG,CAAC,CAAC,KAAK,CAAC,CAAC;IAEzC,OAAO,MAAM,CAAC,KAAK,CAAC,CAAC,EAAE,SAAS,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;AAClD,CAAC;AAED,8EAA8E;AAC9E,gBAAgB;AAChB,8EAA8E;AAE9E,2CAA2C;AAC3C,SAAS,SAAS,CAAC,CAAS;IAC1B,OAAO,CAAC,CAAC,WAAW,EAAE,CAAC,OAAO,CAAC,UAAU,EAAE,EAAE,CAAC,CAAC,OAAO,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC,IAAI,EAAE,CAAC;AAC7E,CAAC;AAED,4DAA4D;AAC5D,SAAS,UAAU,CAAC,CAAS,EAAE,CAAS;IACtC,MAAM,EAAE,GAAG,IAAI,GAAG,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC;IACtE,MAAM,EAAE,GAAG,IAAI,GAAG,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC;IACtE,IAAI,EAAE,CAAC,IAAI,KAAK,CAAC,IAAI,EAAE,CAAC,IAAI,KAAK,CAAC;QAAE,OAAO,CAAC,CAAC;IAC7C,IAAI,YAAY,GAAG,CAAC,CAAC;IACrB,KAAK,MAAM,CAAC,IAAI,EAAE,EAAE,CAAC;QACnB,IAAI,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC;YAAE,YAAY,EAAE,CAAC;IAChC,CAAC;IACD,MAAM,KAAK,GAAG,EAAE,CAAC,IAAI,GAAG,EAAE,CAAC,IAAI,GAAG,YAAY,CAAC;IAC/C,OAAO,KAAK,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,YAAY,GAAG,KAAK,CAAC;AAChD,CAAC;AAED;;;GAGG;AACH,MAAM,UAAU,oBAAoB,CAAC,SAAmB,EAAE,SAAS,GAAG,GAAG;IACvE,MAAM,IAAI,GAAa,EAAE,CAAC;IAE1B,KAAK,MAAM,SAAS,IAAI,SAAS,EAAE,CAAC;QAClC,IAAI,SAAS,GAAG,KAAK,CAAC;QACtB,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,IAAI,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;YACrC,MAAM,GAAG,GAAG,UAAU,CAAC,SAAS,EAAE,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC;YAC3C,IAAI,GAAG,IAAI,SAAS,EAAE,CAAC;gBACrB,sBAAsB;gBACtB,IAAI,SAAS,CAAC,MAAM,GAAG,IAAI,CAAC,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC;oBACtC,IAAI,CAAC,CAAC,CAAC,GAAG,SAAS,CAAC;gBACtB,CAAC;gBACD,SAAS,GAAG,IAAI,CAAC;gBACjB,MAAM;YACR,CAAC;QACH,CAAC;QACD,IAAI,CAAC,SAAS,EAAE,CAAC;YACf,IAAI,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;QACvB,CAAC;IACH,CAAC;IAED,OAAO,IAAI,CAAC;AACd,CAAC;AAED,8EAA8E;AAC9E,6BAA6B;AAC7B,8EAA8E;AAE9E,MAAM,QAAQ,GAAG,sFAAsF,CAAC;AACxG,MAAM,UAAU,GAAG,oBAAoB,CAAC;AACxC,MAAM,QAAQ,GAAG,8GAA8G,CAAC;AAChI,MAAM,OAAO,GAAG,iJAAiJ,CAAC;AAElK,sDAAsD;AACtD,MAAM,UAAU,cAAc,CAAC,IAAY;IACzC,MAAM,MAAM,GAA2B,EAAE,CAAC;IAE1C,MAAM,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAC;IACpC,IAAI,MAAM,EAAE,CAAC;QACX,MAAM,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE;YAClC,MAAM,CAAC,SAAS,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,CAAC;QACtC,CAAC,CAAC,CAAC;IACL,CAAC;IAED,MAAM,QAAQ,GAAG,IAAI,CAAC,KAAK,CAAC,UAAU,CAAC,CAAC;IACxC,IAAI,QAAQ,EAAE,CAAC;QACb,QAAQ,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE;YACpC,MAAM,CAAC,WAAW,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,CAAC;QACxC,CAAC,CAAC,CAAC;IACL,CAAC;IAED,MAAM,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAC;IACpC,IAAI,MAAM,EAAE,CAAC;QACX,MAAM,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE;YAClC,MAAM,CAAC,SAAS,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,CAAC;QACtC,CAAC,CAAC,CAAC;IACL,CAAC;IAED,OAAO,MAAM,CAAC;AAChB,CAAC;AAED,+BAA+B;AAC/B,MAAM,UAAU,YAAY,CAAC,IAAY;IACvC,MAAM,OAAO,GAAG,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,IAAI,EAAE,CAAC;IAC1C,OAAO,CAAC,GAAG,IAAI,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;AAC5C,CAAC;AAED;;;GAGG;AACH,MAAM,UAAU,eAAe,CAAC,KAAe;IAC7C,sDAAsD;IACtD,gFAAgF;IAChF,oDAAoD;IACpD,MAAM,SAAS,GAAG,yFAAyF,CAAC;IAE5G,MAAM,QAAQ,GAAG,IAAI,GAAG,EAAkB,CAAC;IAE3C,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;QACzB,MAAM,IAAI,GAAG,IAAI,GAAG,EAAU,CAAC;QAC/B,IAAI,CAAyB,CAAC;QAC9B,SAAS,CAAC,SAAS,GAAG,CAAC,CAAC;QACxB,OAAO,CAAC,CAAC,GAAG,SAAS,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,KAAK,IAAI,EAAE,CAAC;YAC3C,MAAM,IAAI,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC;YAClB,IAAI,IAAI,CAAC,MAAM,GAAG,CAAC;gBAAE,SAAS;YAC9B,4CAA4C;YAC5C,IAAI,0FAA0F,CAAC,IAAI,CAAC,IAAI,CAAC;gBAAE,SAAS;YACpH,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC;gBACpB,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;gBACf,QAAQ,CAAC,GAAG,CAAC,IAAI,EAAE,CAAC,QAAQ,CAAC,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;YACpD,CAAC;QACH,CAAC;IACH,CAAC;IAED,iDAAiD;IACjD,OAAO,CAAC,GAAG,QAAQ,CAAC,OAAO,EAAE,CAAC;SAC3B,MAAM,CAAC,CAAC,CAAC,EAAE,IAAI,CAAC,EAAE,EAAE,CAAC,IAAI,IAAI,CAAC,CAAC;SAC/B,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC;SAC3B,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC;SACZ,GAAG,CAAC,CAAC,CAAC,IAAI,CAAC,EAAE,EAAE,CAAC,IAAI,CAAC,CAAC;AAC3B,CAAC;AAED,8EAA8E;AAC9E,kBAAkB;AAClB,8EAA8E;AAE9E,MAAM,mBAAmB,GAAG,CAAC,IAAI,EAAE,QAAQ,EAAE,SAAS,EAAE,YAAY,EAAE,YAAY,EAAE,aAAa,EAAE,aAAa,EAAE,cAAc,CAAC,CAAC;AAElI,iDAAiD;AACjD,MAAM,UAAU,iBAAiB,CAAC,KAAa;IAC7C,MAAM,KAAK,GAAG,KAAK,CAAC,WAAW,EAAE,CAAC;IAClC,OAAO,mBAAmB,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,CAAC;AAC1D,CAAC;AAED;;;GAGG;AACH,MAAM,UAAU,uBAAuB,CAAC,KAAa;IACnD,MAAM,OAAO,GAAG,KAAK,CAAC,KAAK,CAAC,kCAAkC,CAAC,CAAC;IAChE,IAAI,OAAO,EAAE,CAAC;QACZ,OAAO,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,EAAE,OAAO,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CACpD,CAAC,CAAC,OAAO,CAAC,qCAAqC,EAAE,EAAE,CAAC,CAAC,IAAI,EAAE,CAC5D,CAAC;IACJ,CAAC;IAED,MAAM,YAAY,GAAG,KAAK,CAAC,KAAK,CAAC,8CAA8C,CAAC,CAAC;IACjF,IAAI,YAAY,EAAE,CAAC;QACjB,OAAO,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,EAAE,YAAY,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC;IAC1D,CAAC;IAED,MAAM,SAAS,GAAG,KAAK,CAAC,KAAK,CAAC,kDAAkD,CAAC,CAAC;IAClF,IAAI,SAAS,EAAE,CAAC;QACd,OAAO,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,EAAE,SAAS,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC;IACpD,CAAC;IAED,OAAO,EAAE,CAAC;AACZ,CAAC;AAED,MAAM,qBAAqB,GAAG;IAC5B,EAAE,IAAI,EAAE,OAAO,EAAE,QAAQ,EAAE,CAAC,yDAAyD,EAAE,uBAAuB,CAAC,EAAE;IACjH,EAAE,IAAI,EAAE,UAAU,EAAE,QAAQ,EAAE,CAAC,gEAAgE,CAAC,EAAE;IAClG,EAAE,IAAI,EAAE,MAAM,EAAE,QAAQ,EAAE,CAAC,4DAA4D,CAAC,EAAE;IAC1F,EAAE,IAAI,EAAE,MAAM,EAAE,QAAQ,EAAE,CAAC,8EAA8E,CAAC,EAAE;IAC5G,EAAE,IAAI,EAAE,UAAU,EAAE,QAAQ,EAAE,CAAC,0EAA0E,CAAC,EAAE;IAC5G,EAAE,IAAI,EAAE,QAAQ,EAAE,QAAQ,EAAE,CAAC,yCAAyC,EAAE,0CAA0C,CAAC,EAAE;CACtH,CAAC;AAEF;;GAEG;AACH,MAAM,UAAU,oBAAoB,CAAC,OAAe,EAAE,QAAkB;IACtE,IAAI,QAAQ,CAAC,MAAM,GAAG,CAAC;QAAE,OAAO,SAAS,CAAC;IAE1C,MAAM,OAAO,GAAG,qBAAqB,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC;IACvD,MAAM,IAAI,GAA2C,EAAE,CAAC;IAExD,KAAK,MAAM,MAAM,IAAI,QAAQ,EAAE,CAAC;QAC9B,IAAI,CAAC,MAAM,CAAC,GAAG,EAAE,CAAC;QAClB,yCAAyC;QACzC,MAAM,KAAK,GAAG,OAAO,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;QACnC,MAAM,QAAQ,GAAG,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,WAAW,EAAE,CAAC,QAAQ,CAAC,MAAM,CAAC,WAAW,EAAE,CAAC,CAAC,CAAC;QACnF,MAAM,UAAU,GAAG,QAAQ,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;QAEtC,KAAK,MAAM,IAAI,IAAI,qBAAqB,EAAE,CAAC;YACzC,KAAK,MAAM,OAAO,IAAI,IAAI,CAAC,QAAQ,EAAE,CAAC;gBACpC,MAAM,CAAC,GAAG,UAAU,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC;gBACpC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;oBACd,IAAI,CAAC,MAAM,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC,CAAC;oBACpD,MAAM;gBACR,CAAC;YACH,CAAC;YACD,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC;gBAC7B,IAAI,CAAC,MAAM,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,KAAK,CAAC;YAClC,CAAC;QACH,CAAC;IACH,CAAC;IAED,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE,CAAC;AAC3B,CAAC;AAED,8EAA8E;AAC9E,wBAAwB;AACxB,8EAA8E;AAE9E;;;;;GAKG;AACH,SAAS,YAAY,CACnB,KAAqF,EACrF,QAAgB;IAEhB,2BAA2B;IAC3B,MAAM,MAAM,GAAG,CAAC,GAAG,KAAK,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,cAAc,GAAG,CAAC,CAAC,cAAc,CAAC,CAAC;IAE9E,MAAM,KAAK,GAAa,EAAE,CAAC;IAC3B,IAAI,UAAU,GAAG,CAAC,CAAC;IAEnB,KAAK,MAAM,IAAI,IAAI,MAAM,EAAE,CAAC;QAC1B,IAAI,CAAC,IAAI,CAAC,OAAO;YAAE,SAAS;QAE5B,MAAM,MAAM,GAAG,OAAO,IAAI,CAAC,KAAK,KAAK,IAAI,CAAC,GAAG,OAAO,CAAC;QACrD,MAAM,IAAI,GAAG,IAAI,CAAC,OAAO,CAAC;QAC1B,MAAM,OAAO,GAAG,GAAG,MAAM,GAAG,IAAI,aAAa,CAAC;QAE9C,IAAI,UAAU,GAAG,OAAO,CAAC,MAAM,GAAG,QAAQ,EAAE,CAAC;YAC3C,wBAAwB;YACxB,MAAM,SAAS,GAAG,QAAQ,GAAG,UAAU,GAAG,MAAM,CAAC,MAAM,GAAG,EAAE,CAAC;YAC7D,IAAI,SAAS,GAAG,GAAG,EAAE,CAAC;gBACpB,KAAK,CAAC,IAAI,CAAC,GAAG,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,SAAS,CAAC,gBAAgB,CAAC,CAAC;YACnE,CAAC;YACD,MAAM;QACR,CAAC;QAED,KAAK,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;QACpB,UAAU,IAAI,OAAO,CAAC,MAAM,CAAC;IAC/B,CAAC;IAED,OAAO,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;AACxB,CAAC;AAED,8EAA8E;AAC9E,0BAA0B;AAC1B,8EAA8E;AAE9E,SAAS,OAAO,CAAI,EAAU,EAAE,KAAa;IAC3C,OAAO,IAAI,OAAO,CAAC,CAAC,CAAC,EAAE,MAAM,EAAE,EAAE,CAC/B,UAAU,CAAC,GAAG,EAAE,CAAC,MAAM,CAAC,IAAI,KAAK,CAAC,GAAG,KAAK,oBAAoB,EAAE,IAAI,CAAC,CAAC,EAAE,EAAE,CAAC,CAC5E,CAAC;AACJ,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,SAAS,CAAC,OAAyB;IACvD,MAAM,EACJ,KAAK,EACL,KAAK,GAAG,CAAC,EACT,MAAM,GAAG,QAAQ,EACjB,QAAQ,GAAG,KAAK,EAChB,kBAAkB,GAAG,IAAI,GAC1B,GAAG,OAAO,CAAC;IAEZ,MAAM,SAAS,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;IAC7B,MAAM,SAAS,GAAG,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,GAAG,CAAC,KAAK,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;IAEnD,6EAA6E;IAC7E,MAAM,EAAE,QAAQ,EAAE,MAAM,EAAE,GAAG,qBAAqB,EAAE,CAAC;IACrD,MAAM,aAAa,GAAG,MAAM,OAAO,CAAC,IAAI,CAAC;QACvC,QAAQ,CAAC,SAAS,CAAC,KAAK,EAAE,EAAE,KAAK,EAAE,SAAS,GAAG,CAAC,EAAE,MAAM,EAAE,CAAC,EAAE,qCAAqC;QAClG,OAAO,CAAQ,KAAK,EAAE,QAAQ,CAAC;KAChC,CAAC,CAAC;IAEH,MAAM,UAAU,GAAG,CAAC,KAAK,CAAC,OAAO,CAAC,aAAa,CAAC,CAAC,CAAC,CAAC,aAAa,CAAC,CAAC,CAAE,aAAqB,EAAE,OAAO,IAAI,EAAE,CAAC;SACtG,KAAK,CAAC,CAAC,EAAE,SAAS,GAAG,CAAC,CAAC,CAAC;IAE3B,IAAI,UAAU,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QAC5B,OAAO;YACL,KAAK;YACL,MAAM;YACN,OAAO,EAAE,EAAE;YACX,MAAM,EAAE,EAAE;YACV,OAAO,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,SAAS;SAChC,CAAC;IACJ,CAAC;IAED,6EAA6E;IAC7E,MAAM,IAAI,GAAG,UAAU,CAAC,GAAG,CAAC,CAAC,CAAM,EAAE,EAAE,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC;IAC/D,MAAM,KAAK,GAAG,MAAM,OAAO,CAAC,IAAI,CAAC;QAC/B,SAAS,CAAC,IAAI,EAAE,EAAE,WAAW,EAAE,CAAC,EAAE,MAAM,EAAE,UAAU,EAAE,CAAC;QACvD,OAAO,CAAQ,MAAM,EAAE,aAAa,CAAC;KACtC,CAAC,CAAC;IAYH,MAAM,WAAW,GAAe,EAAE,CAAC;IAEnC,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,KAAK,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;QACtC,MAAM,IAAI,GAAG,KAAK,CAAC,CAAC,CAAQ,CAAC;QAC7B,MAAM,YAAY,GAAG,UAAU,CAAC,CAAC,CAAQ,CAAC;QAC1C,MAAM,GAAG,GAAG,IAAI,CAAC,CAAC,CAAC,CAAC;QAEpB,IAAI,CAAC,IAAI,IAAI,IAAI,CAAC,KAAK;YAAE,SAAS;QAElC,MAAM,OAAO,GAAW,IAAI,CAAC,OAAO,IAAI,EAAE,CAAC;QAC3C,MAAM,KAAK,GAAW,IAAI,CAAC,KAAK,IAAI,YAAY,EAAE,KAAK,IAAI,GAAG,CAAC;QAE/D,MAAM,cAAc,GAAG,qBAAqB,CAAC,OAAO,EAAE,KAAK,CAAC,CAAC;QAE7D,IAAI,cAAc,GAAG,kBAAkB,IAAI,WAAW,CAAC,MAAM,IAAI,CAAC,EAAE,CAAC;YACnE,+CAA+C;YAC/C,SAAS;QACX,CAAC;QAED,MAAM,SAAS,GAAG,gBAAgB,CAAC,OAAO,EAAE,KAAK,EAAE,CAAC,CAAC,CAAC;QAEtD,WAAW,CAAC,IAAI,CAAC;YACf,GAAG;YACH,KAAK;YACL,OAAO;YACP,cAAc;YACd,SAAS;YACT,SAAS,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;SACpC,CAAC,CAAC;QAEH,IAAI,WAAW,CAAC,MAAM,IAAI,SAAS;YAAE,MAAM;IAC7C,CAAC;IAED,6EAA6E;IAC7E,MAAM,aAAa,GAAG,YAAY,CAAC,WAAW,EAAE,QAAQ,CAAC,CAAC;IAE1D,6EAA6E;IAC7E,MAAM,OAAO,GAAmB,WAAW,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC;QACpD,GAAG,EAAE,CAAC,CAAC,GAAG;QACV,KAAK,EAAE,CAAC,CAAC,KAAK;QACd,cAAc,EAAE,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,cAAc,GAAG,IAAI,CAAC,GAAG,IAAI;QAC1D,SAAS,EAAE,CAAC,CAAC,SAAS;QACtB,SAAS,EAAE,CAAC,CAAC,SAAS;KACvB,CAAC,CAAC,CAAC;IAEJ,MAAM,MAAM,GAAoB;QAC9B,KAAK;QACL,MAAM;QACN,OAAO;QACP,MAAM,EAAE,aAAa;QACrB,OAAO,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,SAAS;KAChC,CAAC;IAEF,6EAA6E;IAC7E,IAAI,MAAM,KAAK,YAAY,IAAI,MAAM,KAAK,YAAY,EAAE,CAAC;QACvD,MAAM,QAAQ,GAAG,WAAW,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC;QACjD,MAAM,QAAQ,GAAG,WAAW,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC;QACvD,MAAM,iBAAiB,GAAG,oBAAoB,CAAC,QAAQ,CAAC,CAAC;QACzD,MAAM,QAAQ,GAAG,eAAe,CAAC,QAAQ,CAAC,CAAC;QAC3C,MAAM,KAAK,GAAG,YAAY,CAAC,aAAa,CAAC,CAAC;QAC1C,MAAM,OAAO,GAAG,cAAc,CAAC,aAAa,CAAC,CAAC;QAE9C,MAAM,CAAC,UAAU,GAAG;YAClB,KAAK,EAAE,iBAAiB;YACxB,QAAQ;YACR,KAAK;YACL,OAAO;SACR,CAAC;IACJ,CAAC;IAED,6EAA6E;IAC7E,IAAI,MAAM,KAAK,YAAY,IAAI,iBAAiB,CAAC,KAAK,CAAC,EAAE,CAAC;QACxD,MAAM,gBAAgB,GAAG,uBAAuB,CAAC,KAAK,CAAC,CAAC;QACxD,IAAI,gBAAgB,CAAC,MAAM,IAAI,CAAC,EAAE,CAAC;YACjC,MAAM,CAAC,UAAU,GAAG,oBAAoB,CAAC,aAAa,EAAE,gBAAgB,CAAC,CAAC;QAC5E,CAAC;IACH,CAAC;IAED,OAAO,MAAM,CAAC;AAChB,CAAC"}
@@ -0,0 +1,34 @@
1
+ /**
2
+ * Domain-aware structured extractors for WebPeel.
3
+ *
4
+ * When peel() fetches a URL that matches a known domain, the relevant
5
+ * extractor fires and returns clean structured data + a markdown summary.
6
+ *
7
+ * Supported domains:
8
+ * - twitter.com / x.com — tweets, threads, profiles
9
+ * - reddit.com — posts with comments (via JSON API)
10
+ * - github.com — repos, issues, PRs, users (via GitHub API)
11
+ * - news.ycombinator.com — stories with comments (via HN Firebase API)
12
+ */
13
+ export interface DomainExtractResult {
14
+ /** Canonical domain name (e.g. 'twitter.com') */
15
+ domain: string;
16
+ /** Page type within the domain (e.g. 'tweet', 'thread', 'repo', 'issue') */
17
+ type: string;
18
+ /** Domain-specific structured data */
19
+ structured: Record<string, any>;
20
+ /** Clean markdown representation of the content */
21
+ cleanContent: string;
22
+ }
23
+ /** An extractor receives the raw HTML and original URL, may make API calls. */
24
+ export type DomainExtractor = (html: string, url: string) => Promise<DomainExtractResult | null>;
25
+ /**
26
+ * Returns the domain extractor for a URL, or null if none matches.
27
+ */
28
+ export declare function getDomainExtractor(url: string): DomainExtractor | null;
29
+ /**
30
+ * Convenience: run the extractor for the URL (if one exists).
31
+ * Returns null when no extractor matches or extraction fails.
32
+ */
33
+ export declare function extractDomainData(html: string, url: string): Promise<DomainExtractResult | null>;
34
+ //# sourceMappingURL=domain-extractors.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"domain-extractors.d.ts","sourceRoot":"","sources":["../../src/core/domain-extractors.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;GAWG;AAQH,MAAM,WAAW,mBAAmB;IAClC,iDAAiD;IACjD,MAAM,EAAE,MAAM,CAAC;IACf,4EAA4E;IAC5E,IAAI,EAAE,MAAM,CAAC;IACb,sCAAsC;IACtC,UAAU,EAAE,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC;IAChC,mDAAmD;IACnD,YAAY,EAAE,MAAM,CAAC;CACtB;AAED,+EAA+E;AAC/E,MAAM,MAAM,eAAe,GAAG,CAC5B,IAAI,EAAE,MAAM,EACZ,GAAG,EAAE,MAAM,KACR,OAAO,CAAC,mBAAmB,GAAG,IAAI,CAAC,CAAC;AAgBzC;;GAEG;AACH,wBAAgB,kBAAkB,CAAC,GAAG,EAAE,MAAM,GAAG,eAAe,GAAG,IAAI,CAWtE;AAED;;;GAGG;AACH,wBAAsB,iBAAiB,CACrC,IAAI,EAAE,MAAM,EACZ,GAAG,EAAE,MAAM,GACV,OAAO,CAAC,mBAAmB,GAAG,IAAI,CAAC,CAQrC"}