webpeel 0.13.4 → 0.14.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +120 -162
- package/dist/cli-auth.js +7 -7
- package/dist/cli-auth.js.map +1 -1
- package/dist/cli.js +197 -26
- package/dist/cli.js.map +1 -1
- package/dist/core/auto-extract.d.ts +83 -0
- package/dist/core/auto-extract.d.ts.map +1 -0
- package/dist/core/auto-extract.js +565 -0
- package/dist/core/auto-extract.js.map +1 -0
- package/dist/core/deep-fetch.d.ts +75 -0
- package/dist/core/deep-fetch.d.ts.map +1 -0
- package/dist/core/deep-fetch.js +406 -0
- package/dist/core/deep-fetch.js.map +1 -0
- package/dist/core/domain-extractors.d.ts +34 -0
- package/dist/core/domain-extractors.d.ts.map +1 -0
- package/dist/core/domain-extractors.js +654 -0
- package/dist/core/domain-extractors.js.map +1 -0
- package/dist/core/markdown.d.ts +8 -0
- package/dist/core/markdown.d.ts.map +1 -1
- package/dist/core/markdown.js +25 -0
- package/dist/core/markdown.js.map +1 -1
- package/dist/core/quick-answer.d.ts +28 -0
- package/dist/core/quick-answer.d.ts.map +1 -0
- package/dist/core/quick-answer.js +288 -0
- package/dist/core/quick-answer.js.map +1 -0
- package/dist/core/readability.d.ts +58 -0
- package/dist/core/readability.d.ts.map +1 -0
- package/dist/core/readability.js +496 -0
- package/dist/core/readability.js.map +1 -0
- package/dist/core/search-provider.d.ts.map +1 -1
- package/dist/core/search-provider.js +3 -6
- package/dist/core/search-provider.js.map +1 -1
- package/dist/core/strategies.d.ts.map +1 -1
- package/dist/core/strategies.js +70 -5
- package/dist/core/strategies.js.map +1 -1
- package/dist/core/watch-manager.d.ts +140 -0
- package/dist/core/watch-manager.d.ts.map +1 -0
- package/dist/core/watch-manager.js +348 -0
- package/dist/core/watch-manager.js.map +1 -0
- package/dist/core/youtube.d.ts +91 -0
- package/dist/core/youtube.d.ts.map +1 -0
- package/dist/core/youtube.js +380 -0
- package/dist/core/youtube.js.map +1 -0
- package/dist/index.d.ts +4 -0
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +103 -0
- package/dist/index.js.map +1 -1
- package/dist/mcp/server.js +58 -16
- package/dist/mcp/server.js.map +1 -1
- package/dist/server/app.d.ts.map +1 -1
- package/dist/server/app.js +19 -1
- package/dist/server/app.js.map +1 -1
- package/dist/server/routes/deep-fetch.d.ts +9 -0
- package/dist/server/routes/deep-fetch.d.ts.map +1 -0
- package/dist/server/routes/deep-fetch.js +38 -0
- package/dist/server/routes/deep-fetch.js.map +1 -0
- package/dist/server/routes/extract.d.ts.map +1 -1
- package/dist/server/routes/extract.js +11 -0
- package/dist/server/routes/extract.js.map +1 -1
- package/dist/server/routes/fetch.d.ts.map +1 -1
- package/dist/server/routes/fetch.js +45 -19
- package/dist/server/routes/fetch.js.map +1 -1
- package/dist/server/routes/mcp.d.ts +2 -1
- package/dist/server/routes/mcp.d.ts.map +1 -1
- package/dist/server/routes/mcp.js +307 -38
- package/dist/server/routes/mcp.js.map +1 -1
- package/dist/server/routes/quick-answer.d.ts +9 -0
- package/dist/server/routes/quick-answer.d.ts.map +1 -0
- package/dist/server/routes/quick-answer.js +84 -0
- package/dist/server/routes/quick-answer.js.map +1 -0
- package/dist/server/routes/watch.d.ts +16 -0
- package/dist/server/routes/watch.d.ts.map +1 -0
- package/dist/server/routes/watch.js +219 -0
- package/dist/server/routes/watch.js.map +1 -0
- package/dist/server/routes/youtube.d.ts +7 -0
- package/dist/server/routes/youtube.d.ts.map +1 -0
- package/dist/server/routes/youtube.js +87 -0
- package/dist/server/routes/youtube.js.map +1 -0
- package/dist/types.d.ts +18 -0
- package/dist/types.d.ts.map +1 -1
- package/dist/types.js.map +1 -1
- package/llms.txt +14 -5
- package/package.json +1 -1
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"auto-extract.js","sourceRoot":"","sources":["../../src/core/auto-extract.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;GAWG;AAEH,OAAO,EAAE,IAAI,EAAmB,MAAM,SAAS,CAAC;AAgFhD,8EAA8E;AAC9E,sBAAsB;AACtB,8EAA8E;AAE9E,MAAM,YAAY,GAAG,gBAAgB,CAAC;AACtC,MAAM,SAAS,GAAG,WAAW,CAAC;AAC9B,MAAM,mBAAmB,GAAG,8CAA8C,CAAC;AAC3E,MAAM,gBAAgB,GAAG,+CAA+C,CAAC;AACzE,MAAM,aAAa,GAAG,mDAAmD,CAAC;AAC1E,MAAM,aAAa,GACjB,yFAAyF,CAAC;AAE5F,qGAAqG;AACrG,SAAS,WAAW,CAAC,CAAa;IAChC,MAAM,IAAI,GAAG,CAAC,CAAC,MAAM,CAAC,CAAC,IAAI,EAAE,IAAI,EAAE,CAAC;IACpC,OAAO,IAAI,CAAC,OAAO,CAAC,UAAU,EAAE,GAAG,CAAC,CAAC,OAAO,CAAC,eAAe,EAAE,GAAG,CAAC,CAAC,OAAO,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC,IAAI,EAAE,CAAC;AACjG,CAAC;AAED,SAAS,MAAM,CAAC,GAAW,EAAE,GAAG,QAAkB;IAChD,IAAI,CAAC;QACH,MAAM,IAAI,GAAG,IAAI,GAAG,CAAC,GAAG,CAAC,CAAC,QAAQ,CAAC,WAAW,EAAE,CAAC;QACjD,OAAO,QAAQ,CAAC,IAAI,CAAC,CAAC,EAAE,EAAE,EAAE,CAAC,IAAI,CAAC,QAAQ,CAAC,EAAE,CAAC,CAAC,CAAC;IAClD,CAAC;IAAC,MAAM,CAAC;QACP,MAAM,KAAK,GAAG,GAAG,CAAC,WAAW,EAAE,CAAC;QAChC,OAAO,QAAQ,CAAC,IAAI,CAAC,CAAC,EAAE,EAAE,EAAE,CAAC,KAAK,CAAC,QAAQ,CAAC,EAAE,CAAC,CAAC,CAAC;IACnD,CAAC;AACH,CAAC;AAED;;;GAGG;AACH,MAAM,UAAU,cAAc,CAAC,IAAY,EAAE,GAAW;IACtD,MAAM,CAAC,GAAG,IAAI,CAAC,IAAI,CAAC,CAAC;IAErB,kBAAkB;IAClB,IAAI,MAAM,CAAC,GAAG,EAAE,UAAU,EAAE,QAAQ,EAAE,WAAW,EAAE,QAAQ,EAAE,UAAU,CAAC,EAAE,CAAC;QACzE,OAAO,SAAS,CAAC;IACnB,CAAC;IACD,MAAM,QAAQ,GAAG,WAAW,CAAC,CAAC,CAAC,CAAC;IAChC,MAAM,YAAY,GAAG,QAAQ,CAAC,KAAK,CAAC,iBAAiB,CAAC,IAAI,EAAE,CAAC;IAC7D,MAAM,gBAAgB,GAAG,QAAQ,CAAC,KAAK,CAAC,oCAAoC,CAAC,IAAI,EAAE,CAAC;IACpF,IAAI,YAAY,CAAC,MAAM,IAAI,CAAC,IAAI,gBAAgB,CAAC,MAAM,IAAI,CAAC,EAAE,CAAC;QAC7D,OAAO,SAAS,CAAC;IACnB,CAAC;IAED,kBAAkB;IAClB,IAAI,MAAM,CAAC,GAAG,EAAE,UAAU,EAAE,QAAQ,EAAE,QAAQ,EAAE,UAAU,EAAE,UAAU,CAAC,EAAE,CAAC;QACxE,MAAM,MAAM,GAAG,QAAQ,CAAC,KAAK,CAAC,aAAa,CAAC,IAAI,EAAE,CAAC;QACnD,IAAI,MAAM,CAAC,MAAM,GAAG,CAAC;YAAE,OAAO,SAAS,CAAC;IAC1C,CAAC;IACD,MAAM,MAAM,GAAG,QAAQ,CAAC,KAAK,CAAC,aAAa,CAAC,IAAI,EAAE,CAAC;IACnD,MAAM,MAAM,GAAG,QAAQ,CAAC,KAAK,CAAC,aAAa,CAAC,IAAI,EAAE,CAAC;IACnD,MAAM,WAAW,GAAG,CAAC,CAAC,wEAAwE,CAAC,CAAC,MAAM,CAAC;IACvG,IAAI,MAAM,CAAC,MAAM,GAAG,CAAC,IAAI,CAAC,MAAM,CAAC,MAAM,GAAG,CAAC,IAAI,WAAW,GAAG,CAAC,CAAC,EAAE,CAAC;QAChE,OAAO,SAAS,CAAC;IACnB,CAAC;IAED,kBAAkB;IAClB,MAAM,aAAa,GAAG,CAAC,CAAC,SAAS,CAAC,CAAC,MAAM,GAAG,CAAC,CAAC;IAC9C,MAAM,UAAU,GAAG,CAAC,CAAC,+BAA+B,CAAC,CAAC,MAAM,GAAG,CAAC,CAAC;IACjE,MAAM,aAAa,GACjB,CAAC,CAAC,qBAAqB,CAAC,CAAC,MAAM,GAAG,CAAC;QACnC,CAAC,CAAC,wCAAwC,CAAC,CAAC,MAAM,GAAG,CAAC,CAAC;IACzD,IAAI,aAAa,IAAI,CAAC,UAAU,IAAI,aAAa,CAAC,EAAE,CAAC;QACnD,OAAO,SAAS,CAAC;IACnB,CAAC;IACD,2DAA2D;IAC3D,MAAM,OAAO,GAAG,CAAC,CAAC,IAAI,CAAC,CAAC,MAAM,CAAC;IAC/B,MAAM,SAAS,GAAG,CAAC,CAAC,GAAG,CAAC,CAAC,MAAM,CAAC;IAChC,IAAI,OAAO,KAAK,CAAC,IAAI,SAAS,IAAI,CAAC,IAAI,UAAU,EAAE,CAAC;QAClD,OAAO,SAAS,CAAC;IACnB,CAAC;IAED,mBAAmB;IACnB,MAAM,QAAQ,GAAG,CAAC,CAAC,WAAW,CAAC,CAAC,IAAI,EAAE,CAAC;IACvC,MAAM,cAAc,GAAG,CAAC,QAAQ,CAAC,KAAK,CAAC,mBAAmB,CAAC,IAAI,EAAE,CAAC,CAAC,MAAM,CAAC;IAC1E,MAAM,WAAW,GAAG,CAAC,QAAQ,CAAC,KAAK,CAAC,gBAAgB,CAAC,IAAI,EAAE,CAAC,CAAC,MAAM,CAAC;IACpE,IAAI,cAAc,IAAI,CAAC,IAAI,WAAW,IAAI,CAAC,EAAE,CAAC;QAC5C,OAAO,UAAU,CAAC;IACpB,CAAC;IACD,wDAAwD;IACxD,MAAM,WAAW,GAAG,CAAC,CAAC,YAAY,CAAC,CAAC,IAAI,EAAE,CAAC;IAC3C,IACE,WAAW,CAAC,KAAK,CAAC,8CAA8C,CAAC;QACjE,cAAc,IAAI,CAAC,EACnB,CAAC;QACD,OAAO,UAAU,CAAC;IACpB,CAAC;IAED,mBAAmB;IACnB,+DAA+D;IAC/D,MAAM,0BAA0B,GAAG;QACjC,UAAU,EAAE,OAAO,EAAE,OAAO,EAAE,oBAAoB,EAAE,iBAAiB,EAAE,iBAAiB;KACzF,CAAC;IACF,KAAK,MAAM,GAAG,IAAI,0BAA0B,EAAE,CAAC;QAC7C,MAAM,KAAK,GAAG,CAAC,CAAC,GAAG,CAAC,CAAC;QACrB,IAAI,KAAK,CAAC,MAAM,IAAI,CAAC,EAAE,CAAC;YACtB,IAAI,SAAS,GAAG,CAAC,CAAC;YAClB,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,EAAE,EAAE;gBACnB,MAAM,IAAI,GAAG,CAAC,CAAC,EAAE,CAAC,CAAC,IAAI,EAAE,CAAC;gBAC1B,IAAI,YAAY,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,SAAS,CAAC,IAAI,CAAC,IAAI,CAAC;oBAAE,SAAS,EAAE,CAAC;YACnE,CAAC,CAAC,CAAC;YACH,IAAI,SAAS,IAAI,CAAC;gBAAE,OAAO,UAAU,CAAC;QACxC,CAAC;IACH,CAAC;IACD,qDAAqD;IACrD,MAAM,IAAI,GAAG,CAAC,CAAC,KAAK,CAAC,CAAC,MAAM,CAAC;IAC7B,IAAI,IAAI,IAAI,CAAC,IAAI,YAAY,CAAC,MAAM,IAAI,CAAC,EAAE,CAAC;QAC1C,OAAO,UAAU,CAAC;IACpB,CAAC;IAED,OAAO,SAAS,CAAC;AACnB,CAAC;AAED,8EAA8E;AAC9E,oBAAoB;AACpB,8EAA8E;AAE9E,SAAS,mBAAmB,CAAC,CAAa;IACxC,MAAM,KAAK,GAAkB,EAAE,CAAC;IAEhC,iEAAiE;IACjE,MAAM,kBAAkB,GAAG;QACzB,yBAAyB;QACzB,uBAAuB;QACvB,sBAAsB;QACtB,sBAAsB;QACtB,0BAA0B;QAC1B,iBAAiB;QACjB,yBAAyB;QACzB,6BAA6B;QAC7B,6BAA6B;QAC7B,OAAO;QACP,iBAAiB;KAClB,CAAC;IAEF,IAAI,UAAU,GAAkC,IAAI,CAAC;IACrD,KAAK,MAAM,GAAG,IAAI,kBAAkB,EAAE,CAAC;QACrC,MAAM,KAAK,GAAG,CAAC,CAAC,GAAG,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,EAAE,EAAE;YACpC,MAAM,IAAI,GAAG,CAAC,CAAC,EAAE,CAAC,CAAC,IAAI,EAAE,CAAC;YAC1B,OAAO,YAAY,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,SAAS,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QACzD,CAAC,CAAC,CAAC;QACH,IAAI,KAAK,CAAC,MAAM,IAAI,CAAC,EAAE,CAAC;YACtB,UAAU,GAAG,KAAK,CAAC;YACnB,MAAM;QACR,CAAC;IACH,CAAC;IAED,IAAI,CAAC,UAAU,IAAI,UAAU,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QAC3C,4DAA4D;QAC5D,OAAO,oBAAoB,CAAC,CAAC,CAAC,CAAC;IACjC,CAAC;IAED,UAAU,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,EAAE,EAAE;QACxB,IAAI,CAAC;YACH,MAAM,GAAG,GAAG,CAAC,CAAC,EAAE,CAAC,CAAC;YAClB,MAAM,IAAI,GAAG,GAAG,CAAC,IAAI,EAAE,CAAC,IAAI,EAAE,CAAC;YAE/B,qDAAqD;YACrD,MAAM,MAAM,GAAG,GAAG,CAAC,IAAI,CAAC,2DAA2D,CAAC,CAAC,KAAK,EAAE,CAAC;YAC7F,MAAM,IAAI,GAAG,MAAM,CAAC,IAAI,EAAE,CAAC,IAAI,EAAE,IAAI,MAAM,CAAC;YAE5C,gBAAgB;YAChB,MAAM,UAAU,GAAG,IAAI,CAAC,KAAK,CAAC,iCAAiC,CAAC,CAAC;YACjE,IAAI,CAAC,UAAU,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,IAAI,CAAC;gBAAE,OAAO,CAAC,4BAA4B;YAC9E,MAAM,KAAK,GAAG,SAAS,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,UAAU,EAAE,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC;YAErF,iBAAiB;YACjB,MAAM,WAAW,GAAG,IAAI,CAAC,KAAK,CAAC,uCAAuC,CAAC,CAAC;YACxE,MAAM,MAAM,GAAG,WAAW,CAAC,CAAC,CAAC,WAAW,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC;YAExD,8BAA8B;YAC9B,MAAM,QAAQ,GAAa,EAAE,CAAC;YAC9B,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,EAAE,EAAE;gBAC5B,MAAM,WAAW,GAAG,CAAC,CAAC,EAAE,CAAC,CAAC,IAAI,EAAE,CAAC,IAAI,EAAE,CAAC;gBACxC,IAAI,WAAW,IAAI,WAAW,CAAC,MAAM,GAAG,GAAG,EAAE,CAAC;oBAC5C,QAAQ,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC;gBAC7B,CAAC;YACH,CAAC,CAAC,CAAC;YAEH,qBAAqB;YACrB,MAAM,KAAK,GAAG,GAAG;iBACd,IAAI,CAAC,WAAW,CAAC;iBACjB,MAAM,CAAC,CAAC,CAAC,EAAE,GAAG,EAAE,EAAE,CACjB,oEAAoE,CAAC,IAAI,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,CAAC,CACzF;iBACA,KAAK,EAAE,CAAC;YACX,MAAM,GAAG,GAAG,KAAK,CAAC,IAAI,EAAE,CAAC,IAAI,EAAE,IAAI,SAAS,CAAC;YAE7C,IAAI,IAAI,IAAI,KAAK,EAAE,CAAC;gBAClB,KAAK,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,KAAK,EAAE,MAAM,EAAE,QAAQ,EAAE,GAAG,EAAE,CAAC,CAAC;YACrD,CAAC;QACH,CAAC;QAAC,MAAM,CAAC;YACP,qCAAqC;QACvC,CAAC;IACH,CAAC,CAAC,CAAC;IAEH,OAAO,gBAAgB,CAAC,KAAK,CAAC,CAAC;AACjC,CAAC;AAED,SAAS,oBAAoB,CAAC,CAAa;IACzC,wDAAwD;IACxD,MAAM,KAAK,GAAkB,EAAE,CAAC;IAChC,MAAM,QAAQ,GAAG,WAAW,CAAC,CAAC,CAAC,CAAC;IAEhC,MAAM,UAAU,GAAG,wEAAwE,CAAC;IAC5F,IAAI,KAAK,CAAC;IACV,MAAM,WAAW,GAAa,EAAE,CAAC;IACjC,OAAO,CAAC,KAAK,GAAG,UAAU,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC,KAAK,IAAI,EAAE,CAAC;QACpD,WAAW,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC;IAC7B,CAAC;IAED,+CAA+C;IAC/C,MAAM,YAAY,GAAG,CAAC,GAAG,IAAI,GAAG,CAAC,WAAW,CAAC,CAAC,CAAC;IAC/C,KAAK,MAAM,CAAC,IAAI,YAAY,EAAE,CAAC;QAC7B,KAAK,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,KAAK,EAAE,CAAC,EAAE,QAAQ,EAAE,EAAE,EAAE,CAAC,CAAC;IACvD,CAAC;IAED,OAAO,KAAK,CAAC;AACf,CAAC;AAED,SAAS,gBAAgB,CAAC,KAAoB;IAC5C,MAAM,IAAI,GAAG,IAAI,GAAG,EAAU,CAAC;IAC/B,OAAO,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE;QACxB,MAAM,GAAG,GAAG,GAAG,CAAC,CAAC,IAAI,IAAI,CAAC,CAAC,KAAK,EAAE,CAAC;QACnC,IAAI,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC;YAAE,OAAO,KAAK,CAAC;QAChC,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;QACd,OAAO,IAAI,CAAC;IACd,CAAC,CAAC,CAAC;AACL,CAAC;AAED,8EAA8E;AAC9E,qBAAqB;AACrB,8EAA8E;AAE9E,SAAS,eAAe,CAAC,CAAa,EAAE,OAAe;IACrD,MAAM,KAAK,GAAkB,EAAE,CAAC;IAChC,MAAM,MAAM,GAAG,CAAC,GAAG,EAAE;QACnB,IAAI,CAAC;YACH,OAAO,IAAI,GAAG,CAAC,OAAO,CAAC,CAAC,MAAM,CAAC;QACjC,CAAC;QAAC,MAAM,CAAC;YACP,OAAO,EAAE,CAAC;QACZ,CAAC;IACH,CAAC,CAAC,EAAE,CAAC;IAEL,MAAM,kBAAkB,GAAG;QACzB,oBAAoB;QACpB,iBAAiB;QACjB,iBAAiB;QACjB,IAAI;QACJ,SAAS;KACV,CAAC;IAEF,IAAI,UAAU,GAAkC,IAAI,CAAC;IACrD,KAAK,MAAM,GAAG,IAAI,kBAAkB,EAAE,CAAC;QACrC,MAAM,KAAK,GAAG,CAAC,CAAC,GAAG,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,EAAE,EAAE;YACpC,MAAM,IAAI,GAAG,CAAC,CAAC,EAAE,CAAC,CAAC,IAAI,EAAE,CAAC;YAC1B,OAAO,CAAC,YAAY,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,SAAS,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,IAAI,CAAC,CAAC,EAAE,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,MAAM,GAAG,CAAC,CAAC;QAC3F,CAAC,CAAC,CAAC;QACH,IAAI,KAAK,CAAC,MAAM,IAAI,CAAC,EAAE,CAAC;YACtB,UAAU,GAAG,KAAK,CAAC;YACnB,MAAM;QACR,CAAC;IACH,CAAC;IAED,IAAI,CAAC,UAAU,IAAI,UAAU,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,KAAK,CAAC;IAEzD,UAAU,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,EAAE,EAAE;QACxB,IAAI,CAAC;YACH,MAAM,GAAG,GAAG,CAAC,CAAC,EAAE,CAAC,CAAC;YAElB,OAAO;YACP,MAAM,MAAM,GAAG,GAAG,CAAC,IAAI,CAAC,oDAAoD,CAAC,CAAC,KAAK,EAAE,CAAC;YACtF,MAAM,IAAI,GAAG,MAAM,CAAC,IAAI,EAAE,CAAC,IAAI,EAAE,CAAC;YAClC,IAAI,CAAC,IAAI;gBAAE,OAAO;YAElB,QAAQ;YACR,MAAM,UAAU,GAAG,GAAG,CAAC,IAAI,EAAE,CAAC,KAAK,CAAC,2BAA2B,CAAC,CAAC;YACjE,MAAM,KAAK,GAAG,UAAU,CAAC,CAAC,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC,SAAS,CAAC;YAE5D,QAAQ;YACR,MAAM,KAAK,GAAG,GAAG,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,KAAK,EAAE,CAAC;YACtC,MAAM,MAAM,GAAG,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,KAAK,CAAC,IAAI,CAAC,UAAU,CAAC,IAAI,KAAK,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC;YACtF,MAAM,KAAK,GAAG,MAAM;gBAClB,CAAC,CAAC,MAAM,CAAC,UAAU,CAAC,MAAM,CAAC;oBACzB,CAAC,CAAC,MAAM;oBACR,CAAC,CAAC,GAAG,MAAM,GAAG,MAAM,CAAC,UAAU,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,GAAG,GAAG,MAAM,EAAE;gBAC5D,CAAC,CAAC,SAAS,CAAC;YAEd,MAAM;YACN,MAAM,MAAM,GAAG,GAAG,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,CAAC;YACrC,MAAM,IAAI,GAAG,MAAM,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;YACjC,MAAM,GAAG,GAAG,IAAI;gBACd,CAAC,CAAC,IAAI,CAAC,UAAU,CAAC,MAAM,CAAC;oBACvB,CAAC,CAAC,IAAI;oBACN,CAAC,CAAC,GAAG,MAAM,GAAG,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,GAAG,GAAG,IAAI,EAAE;gBACxD,CAAC,CAAC,SAAS,CAAC;YAEd,SAAS;YACT,MAAM,WAAW,GAAG,GAAG,CAAC,IAAI,EAAE,CAAC,KAAK,CAAC,kCAAkC,CAAC,CAAC;YACzE,MAAM,MAAM,GAAG,WAAW,CAAC,CAAC,CAAC,GAAG,WAAW,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,SAAS,CAAC;YAE/D,KAAK,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,KAAK,EAAE,KAAK,EAAE,GAAG,EAAE,MAAM,EAAE,CAAC,CAAC;QAClD,CAAC;QAAC,MAAM,CAAC;YACP,iBAAiB;QACnB,CAAC;IACH,CAAC,CAAC,CAAC;IAEH,OAAO,KAAK,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC,CAAC,CAAC,aAAa;AAC3C,CAAC;AAED,8EAA8E;AAC9E,oBAAoB;AACpB,8EAA8E;AAE9E,MAAM,cAAc,GAA2B;IAC7C,aAAa,EAAE,SAAS;IACxB,OAAO,EAAE,SAAS;IAClB,cAAc,EAAE,UAAU;IAC1B,YAAY,EAAE,QAAQ;IACtB,cAAc,EAAE,UAAU;IAC1B,eAAe,EAAE,WAAW;IAC5B,aAAa,EAAE,SAAS;IACxB,YAAY,EAAE,QAAQ;IACtB,YAAY,EAAE,SAAS;IACvB,aAAa,EAAE,SAAS;CACzB,CAAC;AAEF,MAAM,eAAe,GACnB,qIAAqI,CAAC;AAExI,SAAS,cAAc,CAAC,CAAa;IACnC,MAAM,QAAQ,GAAG,WAAW,CAAC,CAAC,CAAC,CAAC;IAEhC,SAAS;IACT,MAAM,YAAY,GAAG,QAAQ,CAAC,KAAK,CAAC,aAAa,CAAC,IAAI,EAAE,CAAC;IACzD,MAAM,MAAM,GAAG;QACb,GAAG,IAAI,GAAG,CAAC,YAAY,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,WAAW,EAAE,CAAC,CAAC;KACrD,CAAC;IAEF,SAAS;IACT,MAAM,YAAY,GAAG,QAAQ,CAAC,KAAK,CAAC,aAAa,CAAC,IAAI,EAAE,CAAC;IACzD,MAAM,MAAM,GAAG,CAAC,GAAG,IAAI,GAAG,CAAC,YAAY,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC,CAAC;IAE/D,YAAY;IACZ,MAAM,SAAS,GAAa,EAAE,CAAC;IAC/B,CAAC,CAAC,mDAAmD,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,EAAE,EAAE;QACpE,MAAM,IAAI,GAAG,CAAC,CAAE,EAAE,CAAC,CAAC,IAAI,EAAE,CAAC,OAAO,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC,IAAI,EAAE,CAAC;QACvD,IAAI,IAAI,CAAC,MAAM,GAAG,EAAE;YAAE,SAAS,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IAC7C,CAAC,CAAC,CAAC;IACH,mBAAmB;IACnB,MAAM,SAAS,GAAG,QAAQ,CAAC,KAAK,CAAC,eAAe,CAAC,CAAC;IAClD,IAAI,SAAS,EAAE,CAAC;QACd,MAAM,IAAI,GAAG,SAAS,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;QACjC,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,QAAQ,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;YAC9D,SAAS,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QACvB,CAAC;IACH,CAAC;IAED,eAAe;IACf,MAAM,MAAM,GAA2B,EAAE,CAAC;IAC1C,CAAC,CAAC,SAAS,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,EAAE,EAAE;QAC1B,MAAM,IAAI,GAAG,CAAC,CAAC,EAAE,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,IAAI,EAAE,CAAC;QACtC,KAAK,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,cAAc,CAAC,EAAE,CAAC;YAC3D,IAAI,IAAI,CAAC,QAAQ,CAAC,MAAM,CAAC,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,EAAE,CAAC;gBAC1C,MAAM,CAAC,GAAG,CAAC,GAAG,IAAI,CAAC;YACrB,CAAC;QACH,CAAC;IACH,CAAC,CAAC,CAAC;IAEH,OAAO,EAAE,IAAI,EAAE,SAAS,EAAE,MAAM,EAAE,MAAM,EAAE,SAAS,EAAE,MAAM,EAAE,CAAC;AAChE,CAAC;AAED,8EAA8E;AAC9E,oBAAoB;AACpB,8EAA8E;AAE9E,SAAS,cAAc,CAAC,CAAa;IACnC,QAAQ;IACR,MAAM,KAAK,GACT,CAAC,CAAC,IAAI,CAAC,CAAC,KAAK,EAAE,CAAC,IAAI,EAAE,CAAC,IAAI,EAAE;QAC7B,CAAC,CAAC,2BAA2B,CAAC,CAAC,IAAI,CAAC,SAAS,CAAC;QAC9C,CAAC,CAAC,OAAO,CAAC,CAAC,IAAI,EAAE,CAAC,IAAI,EAAE;QACxB,SAAS,CAAC;IAEZ,SAAS;IACT,MAAM,MAAM,GACV,CAAC,CAAC,qBAAqB,CAAC,CAAC,IAAI,CAAC,SAAS,CAAC;QACxC,CAAC,CAAC,qBAAqB,CAAC,CAAC,KAAK,EAAE,CAAC,IAAI,EAAE,CAAC,IAAI,EAAE;QAC9C,CAAC,CAAC,mBAAmB,CAAC,CAAC,KAAK,EAAE,CAAC,IAAI,EAAE,CAAC,IAAI,EAAE;QAC5C,CAAC,CAAC,gBAAgB,CAAC,CAAC,KAAK,EAAE,CAAC,IAAI,EAAE,CAAC,IAAI,EAAE;QACzC,SAAS,CAAC;IAEZ,OAAO;IACP,MAAM,IAAI,GACR,CAAC,CAAC,gBAAgB,CAAC,CAAC,KAAK,EAAE,CAAC,IAAI,CAAC,UAAU,CAAC;QAC5C,CAAC,CAAC,eAAe,CAAC,CAAC,KAAK,EAAE,CAAC,IAAI,CAAC,UAAU,CAAC;QAC3C,CAAC,CAAC,mBAAmB,CAAC,CAAC,IAAI,CAAC,SAAS,CAAC;QACtC,CAAC,CAAC,yCAAyC,CAAC,CAAC,IAAI,CAAC,SAAS,CAAC;QAC5D,CAAC,CAAC,MAAM,CAAC,CAAC,KAAK,EAAE,CAAC,IAAI,EAAE,CAAC,IAAI,EAAE;QAC/B,SAAS,CAAC;IAEZ,eAAe;IACf,MAAM,aAAa,GAAG,CAAC,CAAC,oEAAoE,CAAC,CAAC,KAAK,EAAE,CAAC;IACtG,MAAM,WAAW,GAAG,aAAa,CAAC,MAAM,CAAC,CAAC,CAAC,aAAa,CAAC,IAAI,EAAE,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC,mBAAmB,CAAC,CAAC,CAAC,CAAC;IAEhG,iDAAiD;IACjD,MAAM,SAAS,GAAG,CAAC,CAAC,SAAS,CAAC,CAAC,KAAK,EAAE,CAAC;IACvC,MAAM,SAAS,GAAG,SAAS,CAAC,MAAM,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,KAAK,EAAE,CAAC;IACnE,MAAM,SAAS,GACb,SAAS,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,CAAC,IAAI,EAAE,CAAC,IAAI,EAAE;QACzC,CAAC,CAAC,0BAA0B,CAAC,CAAC,IAAI,CAAC,SAAS,CAAC;QAC7C,CAAC,CAAC,iCAAiC,CAAC,CAAC,IAAI,CAAC,SAAS,CAAC;QACpD,EAAE,CAAC;IACL,MAAM,OAAO,GAAG,SAAS,CAAC,CAAC,CAAC,qBAAqB,CAAC,SAAS,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC;IAE5E,sCAAsC;IACtC,MAAM,QAAQ,GAAqB,EAAE,CAAC;IACtC,MAAM,QAAQ,GAAG,SAAS,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;IAC1C,QAAQ,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,EAAE,EAAE;QACtB,MAAM,OAAO,GAAG,CAAC,CAAC,EAAE,CAAC,CAAC,IAAI,EAAE,CAAC,IAAI,EAAE,CAAC;QACpC,IAAI,CAAC,OAAO;YAAE,OAAO;QACrB,0DAA0D;QAC1D,MAAM,YAAY,GAAa,EAAE,CAAC;QAClC,IAAI,OAAO,GAAG,CAAC,CAAC,EAAE,CAAC,CAAC,IAAI,EAAE,CAAC;QAC3B,OAAO,OAAO,CAAC,MAAM,IAAI,CAAC,OAAO,CAAC,EAAE,CAAC,QAAQ,CAAC,EAAE,CAAC;YAC/C,MAAM,IAAI,GAAG,OAAO,CAAC,IAAI,EAAE,CAAC,IAAI,EAAE,CAAC;YACnC,IAAI,IAAI;gBAAE,YAAY,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;YAClC,OAAO,GAAG,OAAO,CAAC,IAAI,EAAE,CAAC;QAC3B,CAAC;QACD,IAAI,YAAY,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YAC5B,QAAQ,CAAC,IAAI,CAAC,EAAE,OAAO,EAAE,OAAO,EAAE,YAAY,CAAC,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;QAC9D,CAAC;IACH,CAAC,CAAC,CAAC;IAEH,OAAO,EAAE,IAAI,EAAE,SAAS,EAAE,KAAK,EAAE,MAAM,EAAE,IAAI,EAAE,WAAW,EAAE,OAAO,EAAE,QAAQ,EAAE,CAAC;AAClF,CAAC;AAED,SAAS,qBAAqB,CAAC,IAAY,EAAE,KAAa;IACxD,MAAM,WAAW,GAAG,YAAY,CAAC;IACjC,IAAI,KAAK,CAAC;IACV,IAAI,SAAS,GAAG,CAAC,CAAC;IAClB,IAAI,aAAa,GAAG,CAAC,CAAC;IACtB,OAAO,CAAC,KAAK,GAAG,WAAW,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,KAAK,IAAI,EAAE,CAAC;QACjD,SAAS,GAAG,KAAK,CAAC,KAAK,GAAG,KAAK,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC;QAC1C,aAAa,EAAE,CAAC;QAChB,IAAI,aAAa,IAAI,KAAK;YAAE,MAAM;IACpC,CAAC;IACD,OAAO,aAAa,GAAG,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,SAAS,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC,CAAC,IAAI,EAAE,CAAC;AACzF,CAAC;AAED,SAAS,mBAAmB,CAAC,CAAa;IACxC,MAAM,cAAc,GAAG,GAAG,CAAC;IAC3B,MAAM,IAAI,GAAG,CAAC,CAAC,yCAAyC,CAAC,CAAC,KAAK,EAAE,CAAC,IAAI,EAAE,CAAC;IACzE,MAAM,SAAS,GAAG,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,MAAM,CAAC;IAC3D,MAAM,OAAO,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,IAAI,CAAC,SAAS,GAAG,cAAc,CAAC,CAAC,CAAC;IACnE,OAAO,GAAG,OAAO,MAAM,CAAC;AAC1B,CAAC;AAED,8EAA8E;AAC9E,qBAAqB;AACrB,8EAA8E;AAE9E,MAAM,YAAY,GAAG,CAAC,KAAK,EAAE,MAAM,EAAE,KAAK,EAAE,OAAO,EAAE,QAAQ,EAAE,MAAM,EAAE,SAAS,CAAC,CAAC;AAElF,SAAS,cAAc,CAAC,CAAa,EAAE,GAAW;IAChD,MAAM,SAAS,GAAkB,EAAE,CAAC;IAEpC,0CAA0C;IAC1C,IAAI,OAA2B,CAAC;IAChC,MAAM,QAAQ,GAAG,WAAW,CAAC,CAAC,CAAC,CAAC;IAChC,MAAM,YAAY,GAAG,QAAQ,CAAC,KAAK,CAAC,gCAAgC,CAAC,CAAC;IACtE,IAAI,YAAY,EAAE,CAAC;QACjB,OAAO,GAAG,YAAY,CAAC,CAAC,CAAC,CAAC;IAC5B,CAAC;SAAM,CAAC;QACN,IAAI,CAAC;YACH,MAAM,MAAM,GAAG,IAAI,GAAG,CAAC,GAAG,CAAC,CAAC;YAC5B,OAAO,GAAG,GAAG,MAAM,CAAC,QAAQ,SAAS,MAAM,CAAC,QAAQ,EAAE,CAAC;QACzD,CAAC;QAAC,MAAM,CAAC;YACP,OAAO,GAAG,SAAS,CAAC;QACtB,CAAC;IACH,CAAC;IAED,gEAAgE;IAChE,CAAC,CAAC,WAAW,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,EAAE,EAAE;QAC5B,MAAM,IAAI,GAAG,CAAC,CAAC,EAAE,CAAC,CAAC,IAAI,EAAE,CAAC,IAAI,EAAE,CAAC;QACjC,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;QAC/B,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;YACzB,MAAM,OAAO,GAAG,IAAI,CAAC,IAAI,EAAE,CAAC;YAC5B,KAAK,MAAM,MAAM,IAAI,YAAY,EAAE,CAAC;gBAClC,IAAI,OAAO,CAAC,UAAU,CAAC,MAAM,GAAG,GAAG,CAAC,IAAI,OAAO,CAAC,UAAU,CAAC,MAAM,GAAG,IAAI,CAAC,EAAE,CAAC;oBAC1E,MAAM,IAAI,GAAG,OAAO,CAAC,KAAK,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,IAAI,EAAE,CAAC;oBACjD,sCAAsC;oBACtC,MAAM,SAAS,GAAG,IAAI,CAAC,KAAK,CAAC,+BAA+B,CAAC,CAAC;oBAC9D,IAAI,SAAS,EAAE,CAAC;wBACd,IAAI,IAAI,GAAG,SAAS,CAAC,CAAC,CAAC,CAAC;wBACxB,8CAA8C;wBAC9C,IAAI,OAAO,IAAI,IAAI,CAAC,UAAU,CAAC,OAAO,CAAC,EAAE,CAAC;4BACxC,IAAI,GAAG,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC;wBACpC,CAAC;wBACD,qBAAqB;wBACrB,IAAI,GAAG,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC;wBAE1B,4EAA4E;wBAC5E,MAAM,WAAW,GAAG,kBAAkB,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,IAAI,SAAS,CAAC;wBAE3D,SAAS,CAAC,IAAI,CAAC,EAAE,MAAM,EAAE,IAAI,EAAE,WAAW,EAAE,CAAC,CAAC;oBAChD,CAAC;gBACH,CAAC;YACH,CAAC;QACH,CAAC;IACH,CAAC,CAAC,CAAC;IAEH,oEAAoE;IACpE,CAAC,CAAC,2DAA2D,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,EAAE,EAAE;QAC5E,MAAM,UAAU,GAAG,CAAC,CAAC,EAAE,CAAC,CAAC,IAAI,EAAE,CAAC,IAAI,EAAE,CAAC,WAAW,EAAE,CAAC;QACrD,IAAI,CAAC,YAAY,CAAC,QAAQ,CAAC,UAAU,CAAC;YAAE,OAAO;QAE/C,iCAAiC;QACjC,MAAM,QAAQ,GAAG;YACf,CAAC,CAAC,EAAE,CAAC,CAAC,IAAI,CAAC,8DAA8D,CAAC;YAC1E,CAAC,CAAC,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,KAAK,EAAE;SACpC,CAAC;QACF,KAAK,MAAM,OAAO,IAAI,QAAQ,EAAE,CAAC;YAC/B,IAAI,OAAO,CAAC,MAAM,EAAE,CAAC;gBACnB,MAAM,IAAI,GAAG,OAAO,CAAC,IAAI,EAAE,CAAC,IAAI,EAAE,CAAC;gBACnC,IAAI,gBAAgB,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC;oBAChC,SAAS,CAAC,IAAI,CAAC,EAAE,MAAM,EAAE,UAAU,EAAE,IAAI,EAAE,CAAC,CAAC;oBAC7C,MAAM;gBACR,CAAC;YACH,CAAC;QACH,CAAC;IACH,CAAC,CAAC,CAAC;IAEH,6BAA6B;IAC7B,MAAM,IAAI,GAAG,IAAI,GAAG,EAAU,CAAC;IAC/B,MAAM,MAAM,GAAG,SAAS,CAAC,MAAM,CAAC,CAAC,EAAE,EAAE,EAAE;QACrC,MAAM,GAAG,GAAG,GAAG,EAAE,CAAC,MAAM,IAAI,EAAE,CAAC,IAAI,EAAE,CAAC;QACtC,IAAI,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC;YAAE,OAAO,KAAK,CAAC;QAChC,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;QACd,OAAO,IAAI,CAAC;IACd,CAAC,CAAC,CAAC;IAEH,OAAO,EAAE,IAAI,EAAE,UAAU,EAAE,OAAO,EAAE,SAAS,EAAE,MAAM,EAAE,CAAC;AAC1D,CAAC;AAED,SAAS,kBAAkB,CAAC,GAA2B;IACrD,kEAAkE;IAClE,IAAI,OAAO,GAAG,GAAG,CAAC,IAAI,EAAE,CAAC;IACzB,IAAI,KAAK,GAAG,CAAC,CAAC;IACd,OAAO,KAAK,GAAG,CAAC,EAAE,CAAC;QACjB,IAAI,OAAO,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YACzB,MAAM,MAAM,GAAG,GAAG,CAAC,MAAM,EAAE,CAAC;YAC5B,IAAI,CAAC,MAAM,CAAC,MAAM;gBAAE,MAAM;YAC1B,OAAO,GAAG,MAAM,CAAC,IAAI,EAAE,CAAC;QAC1B,CAAC;aAAM,IAAI,OAAO,CAAC,EAAE,CAAC,mBAAmB,CAAC,EAAE,CAAC;YAC3C,OAAO,OAAO,CAAC,IAAI,EAAE,CAAC,IAAI,EAAE,CAAC;QAC/B,CAAC;aAAM,CAAC;YACN,OAAO,GAAG,OAAO,CAAC,IAAI,EAAE,CAAC;QAC3B,CAAC;QACD,KAAK,EAAE,CAAC;IACV,CAAC;IACD,OAAO,IAAI,CAAC;AACd,CAAC;AAED,8EAA8E;AAC9E,oBAAoB;AACpB,8EAA8E;AAE9E;;GAEG;AACH,OAAO,EAAE,cAAc,IAAI,OAAO,EAAE,CAAC;AAErC;;GAEG;AACH,MAAM,UAAU,WAAW,CAAC,IAAY,EAAE,GAAW;IACnD,MAAM,IAAI,GAAG,cAAc,CAAC,IAAI,EAAE,GAAG,CAAC,CAAC;IACvC,MAAM,CAAC,GAAG,IAAI,CAAC,IAAI,CAAC,CAAC;IAErB,IAAI,CAAC;QACH,QAAQ,IAAI,EAAE,CAAC;YACb,KAAK,SAAS;gBACZ,OAAO,EAAE,IAAI,EAAE,SAAS,EAAE,KAAK,EAAE,mBAAmB,CAAC,CAAC,CAAC,EAAE,CAAC;YAE5D,KAAK,UAAU;gBACb,OAAO,EAAE,IAAI,EAAE,UAAU,EAAE,KAAK,EAAE,eAAe,CAAC,CAAC,EAAE,GAAG,CAAC,EAAE,CAAC;YAE9D,KAAK,SAAS;gBACZ,OAAO,cAAc,CAAC,CAAC,CAAC,CAAC;YAE3B,KAAK,SAAS;gBACZ,OAAO,cAAc,CAAC,CAAC,CAAC,CAAC;YAE3B,KAAK,UAAU;gBACb,OAAO,cAAc,CAAC,CAAC,EAAE,GAAG,CAAC,CAAC;YAEhC;gBACE,OAAO,EAAE,IAAI,EAAE,SAAS,EAAE,CAAC;QAC/B,CAAC;IACH,CAAC;IAAC,MAAM,CAAC;QACP,mDAAmD;QACnD,QAAQ,IAAI,EAAE,CAAC;YACb,KAAK,SAAS;gBACZ,OAAO,EAAE,IAAI,EAAE,SAAS,EAAE,KAAK,EAAE,EAAE,EAAE,CAAC;YACxC,KAAK,UAAU;gBACb,OAAO,EAAE,IAAI,EAAE,UAAU,EAAE,KAAK,EAAE,EAAE,EAAE,CAAC;YACzC,KAAK,SAAS;gBACZ,OAAO,EAAE,IAAI,EAAE,SAAS,EAAE,MAAM,EAAE,EAAE,EAAE,MAAM,EAAE,EAAE,EAAE,SAAS,EAAE,EAAE,EAAE,MAAM,EAAE,EAAE,EAAE,CAAC;YAChF,KAAK,SAAS;gBACZ,OAAO,EAAE,IAAI,EAAE,SAAS,EAAE,QAAQ,EAAE,EAAE,EAAE,CAAC;YAC3C,KAAK,UAAU;gBACb,OAAO,EAAE,IAAI,EAAE,UAAU,EAAE,SAAS,EAAE,EAAE,EAAE,CAAC;YAC7C;gBACE,OAAO,EAAE,IAAI,EAAE,SAAS,EAAE,CAAC;QAC/B,CAAC;IACH,CAAC;AACH,CAAC"}
|
|
@@ -0,0 +1,75 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Deep Fetch — Web Intelligence Module
|
|
3
|
+
*
|
|
4
|
+
* Transforms "search + fetch" into "search + fetch + synthesize + structure".
|
|
5
|
+
* No LLM required — pure heuristic signal extraction, BM25 relevance scoring,
|
|
6
|
+
* deduplication, entity/number extraction, and comparison detection.
|
|
7
|
+
*/
|
|
8
|
+
export interface DeepFetchOptions {
|
|
9
|
+
query: string;
|
|
10
|
+
/** Number of sources to fetch (default: 5, max: 10) */
|
|
11
|
+
count?: number;
|
|
12
|
+
/** Output format (default: 'merged') */
|
|
13
|
+
format?: 'merged' | 'structured' | 'comparison';
|
|
14
|
+
/** Approximate max characters in merged output (default: 32000 ≈ 8k tokens) */
|
|
15
|
+
maxChars?: number;
|
|
16
|
+
/** Minimum BM25 relevance score (0-1) to include a source (default: 0.05) */
|
|
17
|
+
relevanceThreshold?: number;
|
|
18
|
+
}
|
|
19
|
+
export interface SourceResult {
|
|
20
|
+
url: string;
|
|
21
|
+
title: string;
|
|
22
|
+
relevanceScore: number;
|
|
23
|
+
keyPoints: string[];
|
|
24
|
+
fetchedAt: string;
|
|
25
|
+
}
|
|
26
|
+
export interface StructuredData {
|
|
27
|
+
facts: string[];
|
|
28
|
+
entities: string[];
|
|
29
|
+
dates: string[];
|
|
30
|
+
numbers: Record<string, string>;
|
|
31
|
+
}
|
|
32
|
+
export interface ComparisonData {
|
|
33
|
+
columns: string[];
|
|
34
|
+
rows: Record<string, Record<string, string>>;
|
|
35
|
+
}
|
|
36
|
+
export interface DeepFetchResult {
|
|
37
|
+
query: string;
|
|
38
|
+
format: string;
|
|
39
|
+
sources: SourceResult[];
|
|
40
|
+
merged: string;
|
|
41
|
+
structured?: StructuredData;
|
|
42
|
+
comparison?: ComparisonData;
|
|
43
|
+
elapsed: number;
|
|
44
|
+
}
|
|
45
|
+
/**
|
|
46
|
+
* Extract up to `maxPoints` key points from content, ranked by signal value.
|
|
47
|
+
*/
|
|
48
|
+
export declare function extractKeyPoints(content: string, query: string, maxPoints?: number): string[];
|
|
49
|
+
/**
|
|
50
|
+
* Deduplicate a list of sentences.
|
|
51
|
+
* When two sentences are >60% similar, keep the longer (more detailed) one.
|
|
52
|
+
*/
|
|
53
|
+
export declare function deduplicateSentences(sentences: string[], threshold?: number): string[];
|
|
54
|
+
/** Extract numbers, prices, percentages from text. */
|
|
55
|
+
export declare function extractNumbers(text: string): Record<string, string>;
|
|
56
|
+
/** Extract dates from text. */
|
|
57
|
+
export declare function extractDates(text: string): string[];
|
|
58
|
+
/**
|
|
59
|
+
* Extract named entities (proper nouns) that appear in at least 2 sources.
|
|
60
|
+
* Simple heuristic: capitalized words/phrases not at sentence start.
|
|
61
|
+
*/
|
|
62
|
+
export declare function extractEntities(texts: string[]): string[];
|
|
63
|
+
/** Detect if the query is a comparison query. */
|
|
64
|
+
export declare function isComparisonQuery(query: string): boolean;
|
|
65
|
+
/**
|
|
66
|
+
* Extract the entities being compared from a query.
|
|
67
|
+
* Handles patterns like "A vs B", "compare A and B", "A or B".
|
|
68
|
+
*/
|
|
69
|
+
export declare function extractComparedEntities(query: string): string[];
|
|
70
|
+
/**
|
|
71
|
+
* Build a comparison table from merged content and entity names.
|
|
72
|
+
*/
|
|
73
|
+
export declare function buildComparisonTable(content: string, entities: string[]): ComparisonData | undefined;
|
|
74
|
+
export declare function deepFetch(options: DeepFetchOptions): Promise<DeepFetchResult>;
|
|
75
|
+
//# sourceMappingURL=deep-fetch.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"deep-fetch.d.ts","sourceRoot":"","sources":["../../src/core/deep-fetch.ts"],"names":[],"mappings":"AAAA;;;;;;GAMG;AAUH,MAAM,WAAW,gBAAgB;IAC/B,KAAK,EAAE,MAAM,CAAC;IACd,uDAAuD;IACvD,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,wCAAwC;IACxC,MAAM,CAAC,EAAE,QAAQ,GAAG,YAAY,GAAG,YAAY,CAAC;IAChD,+EAA+E;IAC/E,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,6EAA6E;IAC7E,kBAAkB,CAAC,EAAE,MAAM,CAAC;CAC7B;AAED,MAAM,WAAW,YAAY;IAC3B,GAAG,EAAE,MAAM,CAAC;IACZ,KAAK,EAAE,MAAM,CAAC;IACd,cAAc,EAAE,MAAM,CAAC;IACvB,SAAS,EAAE,MAAM,EAAE,CAAC;IACpB,SAAS,EAAE,MAAM,CAAC;CACnB;AAED,MAAM,WAAW,cAAc;IAC7B,KAAK,EAAE,MAAM,EAAE,CAAC;IAChB,QAAQ,EAAE,MAAM,EAAE,CAAC;IACnB,KAAK,EAAE,MAAM,EAAE,CAAC;IAChB,OAAO,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;CACjC;AAED,MAAM,WAAW,cAAc;IAC7B,OAAO,EAAE,MAAM,EAAE,CAAC;IAClB,IAAI,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC,CAAC;CAC9C;AAED,MAAM,WAAW,eAAe;IAC9B,KAAK,EAAE,MAAM,CAAC;IACd,MAAM,EAAE,MAAM,CAAC;IACf,OAAO,EAAE,YAAY,EAAE,CAAC;IACxB,MAAM,EAAE,MAAM,CAAC;IACf,UAAU,CAAC,EAAE,cAAc,CAAC;IAC5B,UAAU,CAAC,EAAE,cAAc,CAAC;IAC5B,OAAO,EAAE,MAAM,CAAC;CACjB;AA+ED;;GAEG;AACH,wBAAgB,gBAAgB,CAAC,OAAO,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,SAAS,SAAI,GAAG,MAAM,EAAE,CAgBxF;AAwBD;;;GAGG;AACH,wBAAgB,oBAAoB,CAAC,SAAS,EAAE,MAAM,EAAE,EAAE,SAAS,SAAM,GAAG,MAAM,EAAE,CAsBnF;AAWD,sDAAsD;AACtD,wBAAgB,cAAc,CAAC,IAAI,EAAE,MAAM,GAAG,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAyBnE;AAED,+BAA+B;AAC/B,wBAAgB,YAAY,CAAC,IAAI,EAAE,MAAM,GAAG,MAAM,EAAE,CAGnD;AAED;;;GAGG;AACH,wBAAgB,eAAe,CAAC,KAAK,EAAE,MAAM,EAAE,GAAG,MAAM,EAAE,CA8BzD;AAQD,iDAAiD;AACjD,wBAAgB,iBAAiB,CAAC,KAAK,EAAE,MAAM,GAAG,OAAO,CAGxD;AAED;;;GAGG;AACH,wBAAgB,uBAAuB,CAAC,KAAK,EAAE,MAAM,GAAG,MAAM,EAAE,CAmB/D;AAWD;;GAEG;AACH,wBAAgB,oBAAoB,CAAC,OAAO,EAAE,MAAM,EAAE,QAAQ,EAAE,MAAM,EAAE,GAAG,cAAc,GAAG,SAAS,CA4BpG;AAuDD,wBAAsB,SAAS,CAAC,OAAO,EAAE,gBAAgB,GAAG,OAAO,CAAC,eAAe,CAAC,CAgInF"}
|
|
@@ -0,0 +1,406 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Deep Fetch — Web Intelligence Module
|
|
3
|
+
*
|
|
4
|
+
* Transforms "search + fetch" into "search + fetch + synthesize + structure".
|
|
5
|
+
* No LLM required — pure heuristic signal extraction, BM25 relevance scoring,
|
|
6
|
+
* deduplication, entity/number extraction, and comparison detection.
|
|
7
|
+
*/
|
|
8
|
+
import { peelBatch } from '../index.js';
|
|
9
|
+
import { computeRelevanceScore } from './bm25-filter.js';
|
|
10
|
+
import { getBestSearchProvider } from './search-provider.js';
|
|
11
|
+
// ---------------------------------------------------------------------------
|
|
12
|
+
// Key-point extraction
|
|
13
|
+
// ---------------------------------------------------------------------------
|
|
14
|
+
const SIGNAL_WORDS = new Set([
|
|
15
|
+
'announced', 'launched', 'released', 'costs', 'requires', 'supports',
|
|
16
|
+
'offers', 'provides', 'includes', 'features', 'enables', 'allows',
|
|
17
|
+
'introduces', 'reveals', 'claims', 'states', 'reports', 'shows',
|
|
18
|
+
'found', 'discovered', 'improved', 'updated', 'deprecated', 'removed',
|
|
19
|
+
'increased', 'decreased', 'grew', 'declined', 'reached', 'exceeded',
|
|
20
|
+
]);
|
|
21
|
+
/** Split text into sentences (rough but fast — no NLP library needed). */
|
|
22
|
+
function splitSentences(text) {
|
|
23
|
+
// Strip markdown formatting
|
|
24
|
+
const plain = text
|
|
25
|
+
.replace(/```[\s\S]*?```/g, ' ')
|
|
26
|
+
.replace(/`[^`]+`/g, ' ')
|
|
27
|
+
.replace(/!\[.*?\]\(.*?\)/g, ' ')
|
|
28
|
+
.replace(/\[([^\]]*)\]\([^)]*\)/g, '$1')
|
|
29
|
+
.replace(/^#{1,6}\s+/gm, '')
|
|
30
|
+
.replace(/[*_~`>|\\]/g, ' ')
|
|
31
|
+
.replace(/\s+/g, ' ')
|
|
32
|
+
.trim();
|
|
33
|
+
// Split on sentence-ending punctuation followed by space + capital
|
|
34
|
+
return plain
|
|
35
|
+
.split(/(?<=[.!?])\s+(?=[A-Z"'])/)
|
|
36
|
+
.map(s => s.trim())
|
|
37
|
+
.filter(s => s.length > 20 && s.length < 500);
|
|
38
|
+
}
|
|
39
|
+
/** Extract first sentence of each paragraph (topic sentences). */
|
|
40
|
+
function topicSentences(content) {
|
|
41
|
+
const paragraphs = content.split(/\n{2,}/);
|
|
42
|
+
const result = [];
|
|
43
|
+
for (const para of paragraphs) {
|
|
44
|
+
const sentences = splitSentences(para);
|
|
45
|
+
if (sentences.length > 0) {
|
|
46
|
+
result.push(sentences[0]);
|
|
47
|
+
}
|
|
48
|
+
}
|
|
49
|
+
return result;
|
|
50
|
+
}
|
|
51
|
+
/** Score a sentence for "key point" worthiness. Higher = more useful. */
|
|
52
|
+
function sentenceScore(sentence, queryTerms) {
|
|
53
|
+
const lower = sentence.toLowerCase();
|
|
54
|
+
const words = lower.split(/\s+/);
|
|
55
|
+
let score = 0;
|
|
56
|
+
// Query term overlap
|
|
57
|
+
let queryHits = 0;
|
|
58
|
+
for (const term of queryTerms) {
|
|
59
|
+
if (lower.includes(term))
|
|
60
|
+
queryHits++;
|
|
61
|
+
}
|
|
62
|
+
score += (queryHits / Math.max(queryTerms.size, 1)) * 3;
|
|
63
|
+
// Numbers / statistics
|
|
64
|
+
const numberMatches = sentence.match(/\b\d[\d,.]*\b|\$[\d,.]+|[\d.]+%/g);
|
|
65
|
+
if (numberMatches)
|
|
66
|
+
score += Math.min(numberMatches.length * 0.5, 2);
|
|
67
|
+
// Signal words
|
|
68
|
+
for (const word of words) {
|
|
69
|
+
if (SIGNAL_WORDS.has(word)) {
|
|
70
|
+
score += 1;
|
|
71
|
+
break;
|
|
72
|
+
}
|
|
73
|
+
}
|
|
74
|
+
// Prefer medium-length sentences
|
|
75
|
+
if (sentence.length > 60 && sentence.length < 300)
|
|
76
|
+
score += 0.5;
|
|
77
|
+
return score;
|
|
78
|
+
}
|
|
79
|
+
/**
|
|
80
|
+
* Extract up to `maxPoints` key points from content, ranked by signal value.
|
|
81
|
+
*/
|
|
82
|
+
export function extractKeyPoints(content, query, maxPoints = 5) {
|
|
83
|
+
const queryTerms = new Set(query.toLowerCase().split(/\s+/).filter(t => t.length > 2));
|
|
84
|
+
const allSentences = splitSentences(content);
|
|
85
|
+
const topics = topicSentences(content);
|
|
86
|
+
// Combine and deduplicate
|
|
87
|
+
const candidates = [...new Set([...allSentences, ...topics])];
|
|
88
|
+
// Score and sort
|
|
89
|
+
const scored = candidates.map(s => ({ s, score: sentenceScore(s, queryTerms) }));
|
|
90
|
+
scored.sort((a, b) => b.score - a.score);
|
|
91
|
+
return scored.slice(0, maxPoints).map(x => x.s);
|
|
92
|
+
}
|
|
93
|
+
// ---------------------------------------------------------------------------
|
|
94
|
+
// Deduplication
|
|
95
|
+
// ---------------------------------------------------------------------------
|
|
96
|
+
/** Normalize a sentence for comparison. */
|
|
97
|
+
function normalize(s) {
|
|
98
|
+
return s.toLowerCase().replace(/[^\w\s]/g, '').replace(/\s+/g, ' ').trim();
|
|
99
|
+
}
|
|
100
|
+
/** Jaccard similarity on word sets (fast approximation). */
|
|
101
|
+
function similarity(a, b) {
|
|
102
|
+
const wa = new Set(normalize(a).split(' ').filter(w => w.length > 2));
|
|
103
|
+
const wb = new Set(normalize(b).split(' ').filter(w => w.length > 2));
|
|
104
|
+
if (wa.size === 0 && wb.size === 0)
|
|
105
|
+
return 1;
|
|
106
|
+
let intersection = 0;
|
|
107
|
+
for (const w of wa) {
|
|
108
|
+
if (wb.has(w))
|
|
109
|
+
intersection++;
|
|
110
|
+
}
|
|
111
|
+
const union = wa.size + wb.size - intersection;
|
|
112
|
+
return union === 0 ? 0 : intersection / union;
|
|
113
|
+
}
|
|
114
|
+
/**
|
|
115
|
+
* Deduplicate a list of sentences.
|
|
116
|
+
* When two sentences are >60% similar, keep the longer (more detailed) one.
|
|
117
|
+
*/
|
|
118
|
+
export function deduplicateSentences(sentences, threshold = 0.6) {
|
|
119
|
+
const kept = [];
|
|
120
|
+
for (const candidate of sentences) {
|
|
121
|
+
let dominated = false;
|
|
122
|
+
for (let i = 0; i < kept.length; i++) {
|
|
123
|
+
const sim = similarity(candidate, kept[i]);
|
|
124
|
+
if (sim >= threshold) {
|
|
125
|
+
// Keep the longer one
|
|
126
|
+
if (candidate.length > kept[i].length) {
|
|
127
|
+
kept[i] = candidate;
|
|
128
|
+
}
|
|
129
|
+
dominated = true;
|
|
130
|
+
break;
|
|
131
|
+
}
|
|
132
|
+
}
|
|
133
|
+
if (!dominated) {
|
|
134
|
+
kept.push(candidate);
|
|
135
|
+
}
|
|
136
|
+
}
|
|
137
|
+
return kept;
|
|
138
|
+
}
|
|
139
|
+
// ---------------------------------------------------------------------------
|
|
140
|
+
// Number / entity extraction
|
|
141
|
+
// ---------------------------------------------------------------------------
|
|
142
|
+
const PRICE_RE = /(?:\$|€|£|¥)\s?[\d,]+(?:\.\d+)?(?:\s?(?:\/mo(?:nth)?|\/yr|\/year|\/user|\/month))?/gi;
|
|
143
|
+
const PERCENT_RE = /\d+(?:\.\d+)?\s?%/g;
|
|
144
|
+
const COUNT_RE = /\d+(?:\.\d+)?\s?(?:million|billion|thousand|M|B|K)\s?\+?(?:\s?(?:users?|customers?|downloads?|installs?))?/gi;
|
|
145
|
+
const DATE_RE = /(?:January|February|March|April|May|June|July|August|September|October|November|December)\s+\d{1,2},?\s+\d{4}|\d{4}-\d{2}-\d{2}|Q[1-4]\s+\d{4}/g;
|
|
146
|
+
/** Extract numbers, prices, percentages from text. */
|
|
147
|
+
export function extractNumbers(text) {
|
|
148
|
+
const result = {};
|
|
149
|
+
const prices = text.match(PRICE_RE);
|
|
150
|
+
if (prices) {
|
|
151
|
+
prices.slice(0, 5).forEach((p, i) => {
|
|
152
|
+
result[`price_${i + 1}`] = p.trim();
|
|
153
|
+
});
|
|
154
|
+
}
|
|
155
|
+
const percents = text.match(PERCENT_RE);
|
|
156
|
+
if (percents) {
|
|
157
|
+
percents.slice(0, 5).forEach((p, i) => {
|
|
158
|
+
result[`percent_${i + 1}`] = p.trim();
|
|
159
|
+
});
|
|
160
|
+
}
|
|
161
|
+
const counts = text.match(COUNT_RE);
|
|
162
|
+
if (counts) {
|
|
163
|
+
counts.slice(0, 5).forEach((c, i) => {
|
|
164
|
+
result[`count_${i + 1}`] = c.trim();
|
|
165
|
+
});
|
|
166
|
+
}
|
|
167
|
+
return result;
|
|
168
|
+
}
|
|
169
|
+
/** Extract dates from text. */
|
|
170
|
+
export function extractDates(text) {
|
|
171
|
+
const matches = text.match(DATE_RE) ?? [];
|
|
172
|
+
return [...new Set(matches)].slice(0, 10);
|
|
173
|
+
}
|
|
174
|
+
/**
|
|
175
|
+
* Extract named entities (proper nouns) that appear in at least 2 sources.
|
|
176
|
+
* Simple heuristic: capitalized words/phrases not at sentence start.
|
|
177
|
+
*/
|
|
178
|
+
export function extractEntities(texts) {
|
|
179
|
+
// Collect capitalized words/phrases from each source.
|
|
180
|
+
// Matches: standard proper nouns (New York), CamelCase brands (PayPal, GitHub),
|
|
181
|
+
// and ALL-CAPS acronyms (AI, API) with length >= 2.
|
|
182
|
+
const ENTITY_RE = /\b([A-Z][A-Za-z0-9]*(?:[A-Z][a-z0-9]+)*(?:\s+[A-Z][A-Za-z0-9]*(?:[A-Z][a-z0-9]+)*)*)\b/g;
|
|
183
|
+
const termFreq = new Map();
|
|
184
|
+
for (const text of texts) {
|
|
185
|
+
const seen = new Set();
|
|
186
|
+
let m;
|
|
187
|
+
ENTITY_RE.lastIndex = 0;
|
|
188
|
+
while ((m = ENTITY_RE.exec(text)) !== null) {
|
|
189
|
+
const term = m[1];
|
|
190
|
+
if (term.length < 3)
|
|
191
|
+
continue;
|
|
192
|
+
// Skip common sentence starters / stopwords
|
|
193
|
+
if (/^(The|A|An|In|On|At|To|For|Of|And|Or|But|This|That|These|Those|It|He|She|They|We|You|I)$/.test(term))
|
|
194
|
+
continue;
|
|
195
|
+
if (!seen.has(term)) {
|
|
196
|
+
seen.add(term);
|
|
197
|
+
termFreq.set(term, (termFreq.get(term) ?? 0) + 1);
|
|
198
|
+
}
|
|
199
|
+
}
|
|
200
|
+
}
|
|
201
|
+
// Only return entities that appear in 2+ sources
|
|
202
|
+
return [...termFreq.entries()]
|
|
203
|
+
.filter(([, freq]) => freq >= 2)
|
|
204
|
+
.sort((a, b) => b[1] - a[1])
|
|
205
|
+
.slice(0, 20)
|
|
206
|
+
.map(([term]) => term);
|
|
207
|
+
}
|
|
208
|
+
// ---------------------------------------------------------------------------
|
|
209
|
+
// Comparison mode
|
|
210
|
+
// ---------------------------------------------------------------------------
|
|
211
|
+
const COMPARISON_TRIGGERS = ['vs', 'versus', 'compare', 'comparison', 'difference', 'differences', 'alternative', 'alternatives'];
|
|
212
|
+
/** Detect if the query is a comparison query. */
|
|
213
|
+
export function isComparisonQuery(query) {
|
|
214
|
+
const lower = query.toLowerCase();
|
|
215
|
+
return COMPARISON_TRIGGERS.some(t => lower.includes(t));
|
|
216
|
+
}
|
|
217
|
+
/**
|
|
218
|
+
* Extract the entities being compared from a query.
|
|
219
|
+
* Handles patterns like "A vs B", "compare A and B", "A or B".
|
|
220
|
+
*/
|
|
221
|
+
export function extractComparedEntities(query) {
|
|
222
|
+
const vsMatch = query.match(/(.+?)\s+(?:vs\.?|versus)\s+(.+)/i);
|
|
223
|
+
if (vsMatch) {
|
|
224
|
+
return [vsMatch[1].trim(), vsMatch[2].trim()].map(e => e.replace(/^(?:compare|difference|between)\s+/i, '').trim());
|
|
225
|
+
}
|
|
226
|
+
const compareMatch = query.match(/compare\s+(.+?)\s+(?:and|to|with|vs)\s+(.+)/i);
|
|
227
|
+
if (compareMatch) {
|
|
228
|
+
return [compareMatch[1].trim(), compareMatch[2].trim()];
|
|
229
|
+
}
|
|
230
|
+
const diffMatch = query.match(/difference(?:s)?\s+between\s+(.+?)\s+and\s+(.+)/i);
|
|
231
|
+
if (diffMatch) {
|
|
232
|
+
return [diffMatch[1].trim(), diffMatch[2].trim()];
|
|
233
|
+
}
|
|
234
|
+
return [];
|
|
235
|
+
}
|
|
236
|
+
const COMPARISON_ATTRIBUTES = [
|
|
237
|
+
{ name: 'price', patterns: [/(?:price|cost|pricing|fee|subscription)[:\s]+([^.\n]+)/i, /(\$[\d,.]+(?:\/\w+)?)/] },
|
|
238
|
+
{ name: 'features', patterns: [/(?:features?|capabilities|supports?|includes?)[:\s]+([^.\n]+)/i] },
|
|
239
|
+
{ name: 'pros', patterns: [/(?:pros?|advantages?|benefits?|strengths?)[:\s]+([^.\n]+)/i] },
|
|
240
|
+
{ name: 'cons', patterns: [/(?:cons?|disadvantages?|drawbacks?|weaknesses?|limitations?)[:\s]+([^.\n]+)/i] },
|
|
241
|
+
{ name: 'platform', patterns: [/(?:platform|works?\s+(?:on|with)|available\s+(?:on|for))[:\s]+([^.\n]+)/i] },
|
|
242
|
+
{ name: 'rating', patterns: [/(?:rating|score|stars?)[:\s]+([^.\n]+)/i, /(\d+(?:\.\d+)?\s*\/\s*\d+\s*(?:stars?))/i] },
|
|
243
|
+
];
|
|
244
|
+
/**
|
|
245
|
+
* Build a comparison table from merged content and entity names.
|
|
246
|
+
*/
|
|
247
|
+
export function buildComparisonTable(content, entities) {
|
|
248
|
+
if (entities.length < 2)
|
|
249
|
+
return undefined;
|
|
250
|
+
const columns = COMPARISON_ATTRIBUTES.map(a => a.name);
|
|
251
|
+
const rows = {};
|
|
252
|
+
for (const entity of entities) {
|
|
253
|
+
rows[entity] = {};
|
|
254
|
+
// Find paragraphs mentioning this entity
|
|
255
|
+
const lines = content.split(/\n+/);
|
|
256
|
+
const relevant = lines.filter(l => l.toLowerCase().includes(entity.toLowerCase()));
|
|
257
|
+
const entityText = relevant.join(' ');
|
|
258
|
+
for (const attr of COMPARISON_ATTRIBUTES) {
|
|
259
|
+
for (const pattern of attr.patterns) {
|
|
260
|
+
const m = entityText.match(pattern);
|
|
261
|
+
if (m && m[1]) {
|
|
262
|
+
rows[entity][attr.name] = m[1].trim().slice(0, 120);
|
|
263
|
+
break;
|
|
264
|
+
}
|
|
265
|
+
}
|
|
266
|
+
if (!rows[entity][attr.name]) {
|
|
267
|
+
rows[entity][attr.name] = 'N/A';
|
|
268
|
+
}
|
|
269
|
+
}
|
|
270
|
+
}
|
|
271
|
+
return { columns, rows };
|
|
272
|
+
}
|
|
273
|
+
// ---------------------------------------------------------------------------
|
|
274
|
+
// Smart content merging
|
|
275
|
+
// ---------------------------------------------------------------------------
|
|
276
|
+
/**
|
|
277
|
+
* Merge content from multiple sources intelligently:
|
|
278
|
+
* - Sort by relevance (most relevant first)
|
|
279
|
+
* - Add source attribution
|
|
280
|
+
* - Truncate to maxChars
|
|
281
|
+
*/
|
|
282
|
+
function mergeContent(pages, maxChars) {
|
|
283
|
+
// Sort most-relevant first
|
|
284
|
+
const sorted = [...pages].sort((a, b) => b.relevanceScore - a.relevanceScore);
|
|
285
|
+
const parts = [];
|
|
286
|
+
let totalChars = 0;
|
|
287
|
+
for (const page of sorted) {
|
|
288
|
+
if (!page.content)
|
|
289
|
+
continue;
|
|
290
|
+
const header = `## [${page.title}](${page.url})\n\n`;
|
|
291
|
+
const body = page.content;
|
|
292
|
+
const section = `${header}${body}\n\n---\n\n`;
|
|
293
|
+
if (totalChars + section.length > maxChars) {
|
|
294
|
+
// Add truncated version
|
|
295
|
+
const remaining = maxChars - totalChars - header.length - 20;
|
|
296
|
+
if (remaining > 200) {
|
|
297
|
+
parts.push(`${header}${body.slice(0, remaining)}...\n\n---\n\n`);
|
|
298
|
+
}
|
|
299
|
+
break;
|
|
300
|
+
}
|
|
301
|
+
parts.push(section);
|
|
302
|
+
totalChars += section.length;
|
|
303
|
+
}
|
|
304
|
+
return parts.join('');
|
|
305
|
+
}
|
|
306
|
+
// ---------------------------------------------------------------------------
|
|
307
|
+
// Main deepFetch function
|
|
308
|
+
// ---------------------------------------------------------------------------
|
|
309
|
+
function timeout(ms, label) {
|
|
310
|
+
return new Promise((_, reject) => setTimeout(() => reject(new Error(`${label} timed out after ${ms}ms`)), ms));
|
|
311
|
+
}
|
|
312
|
+
export async function deepFetch(options) {
|
|
313
|
+
const { query, count = 5, format = 'merged', maxChars = 32000, relevanceThreshold = 0.05, } = options;
|
|
314
|
+
const startTime = Date.now();
|
|
315
|
+
const safeCount = Math.min(Math.max(count, 1), 10);
|
|
316
|
+
// ── Step 1: Search ────────────────────────────────────────────────────────
|
|
317
|
+
const { provider, apiKey } = getBestSearchProvider();
|
|
318
|
+
const searchResults = await Promise.race([
|
|
319
|
+
provider.searchWeb(query, { count: safeCount + 2, apiKey }), // fetch extras, filter low-relevance
|
|
320
|
+
timeout(30000, 'Search'),
|
|
321
|
+
]);
|
|
322
|
+
const topResults = (Array.isArray(searchResults) ? searchResults : searchResults?.results ?? [])
|
|
323
|
+
.slice(0, safeCount + 2);
|
|
324
|
+
if (topResults.length === 0) {
|
|
325
|
+
return {
|
|
326
|
+
query,
|
|
327
|
+
format,
|
|
328
|
+
sources: [],
|
|
329
|
+
merged: '',
|
|
330
|
+
elapsed: Date.now() - startTime,
|
|
331
|
+
};
|
|
332
|
+
}
|
|
333
|
+
// ── Step 2: Fetch all URLs in parallel ────────────────────────────────────
|
|
334
|
+
const urls = topResults.map((r) => r.url).filter(Boolean);
|
|
335
|
+
const pages = await Promise.race([
|
|
336
|
+
peelBatch(urls, { concurrency: 5, format: 'markdown' }),
|
|
337
|
+
timeout(120000, 'Batch fetch'),
|
|
338
|
+
]);
|
|
339
|
+
const scoredPages = [];
|
|
340
|
+
for (let i = 0; i < pages.length; i++) {
|
|
341
|
+
const page = pages[i];
|
|
342
|
+
const searchResult = topResults[i];
|
|
343
|
+
const url = urls[i];
|
|
344
|
+
if (!page || page.error)
|
|
345
|
+
continue;
|
|
346
|
+
const content = page.content || '';
|
|
347
|
+
const title = page.title || searchResult?.title || url;
|
|
348
|
+
const relevanceScore = computeRelevanceScore(content, query);
|
|
349
|
+
if (relevanceScore < relevanceThreshold && scoredPages.length >= 2) {
|
|
350
|
+
// Skip low-relevance if we already have enough
|
|
351
|
+
continue;
|
|
352
|
+
}
|
|
353
|
+
const keyPoints = extractKeyPoints(content, query, 5);
|
|
354
|
+
scoredPages.push({
|
|
355
|
+
url,
|
|
356
|
+
title,
|
|
357
|
+
content,
|
|
358
|
+
relevanceScore,
|
|
359
|
+
keyPoints,
|
|
360
|
+
fetchedAt: new Date().toISOString(),
|
|
361
|
+
});
|
|
362
|
+
if (scoredPages.length >= safeCount)
|
|
363
|
+
break;
|
|
364
|
+
}
|
|
365
|
+
// ── Step 4: Merge content ─────────────────────────────────────────────────
|
|
366
|
+
const mergedContent = mergeContent(scoredPages, maxChars);
|
|
367
|
+
// ── Step 5: Build sources list ────────────────────────────────────────────
|
|
368
|
+
const sources = scoredPages.map(p => ({
|
|
369
|
+
url: p.url,
|
|
370
|
+
title: p.title,
|
|
371
|
+
relevanceScore: Math.round(p.relevanceScore * 1000) / 1000,
|
|
372
|
+
keyPoints: p.keyPoints,
|
|
373
|
+
fetchedAt: p.fetchedAt,
|
|
374
|
+
}));
|
|
375
|
+
const result = {
|
|
376
|
+
query,
|
|
377
|
+
format,
|
|
378
|
+
sources,
|
|
379
|
+
merged: mergedContent,
|
|
380
|
+
elapsed: Date.now() - startTime,
|
|
381
|
+
};
|
|
382
|
+
// ── Step 6: Structured extraction (optional) ──────────────────────────────
|
|
383
|
+
if (format === 'structured' || format === 'comparison') {
|
|
384
|
+
const allTexts = scoredPages.map(p => p.content);
|
|
385
|
+
const allFacts = scoredPages.flatMap(p => p.keyPoints);
|
|
386
|
+
const deduplicatedFacts = deduplicateSentences(allFacts);
|
|
387
|
+
const entities = extractEntities(allTexts);
|
|
388
|
+
const dates = extractDates(mergedContent);
|
|
389
|
+
const numbers = extractNumbers(mergedContent);
|
|
390
|
+
result.structured = {
|
|
391
|
+
facts: deduplicatedFacts,
|
|
392
|
+
entities,
|
|
393
|
+
dates,
|
|
394
|
+
numbers,
|
|
395
|
+
};
|
|
396
|
+
}
|
|
397
|
+
// ── Step 7: Comparison table (optional) ───────────────────────────────────
|
|
398
|
+
if (format === 'comparison' || isComparisonQuery(query)) {
|
|
399
|
+
const comparedEntities = extractComparedEntities(query);
|
|
400
|
+
if (comparedEntities.length >= 2) {
|
|
401
|
+
result.comparison = buildComparisonTable(mergedContent, comparedEntities);
|
|
402
|
+
}
|
|
403
|
+
}
|
|
404
|
+
return result;
|
|
405
|
+
}
|
|
406
|
+
//# sourceMappingURL=deep-fetch.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"deep-fetch.js","sourceRoot":"","sources":["../../src/core/deep-fetch.ts"],"names":[],"mappings":"AAAA;;;;;;GAMG;AAEH,OAAO,EAAE,SAAS,EAAE,MAAM,aAAa,CAAC;AACxC,OAAO,EAAE,qBAAqB,EAAE,MAAM,kBAAkB,CAAC;AACzD,OAAO,EAAE,qBAAqB,EAAE,MAAM,sBAAsB,CAAC;AAgD7D,8EAA8E;AAC9E,uBAAuB;AACvB,8EAA8E;AAE9E,MAAM,YAAY,GAAG,IAAI,GAAG,CAAC;IAC3B,WAAW,EAAE,UAAU,EAAE,UAAU,EAAE,OAAO,EAAE,UAAU,EAAE,UAAU;IACpE,QAAQ,EAAE,UAAU,EAAE,UAAU,EAAE,UAAU,EAAE,SAAS,EAAE,QAAQ;IACjE,YAAY,EAAE,SAAS,EAAE,QAAQ,EAAE,QAAQ,EAAE,SAAS,EAAE,OAAO;IAC/D,OAAO,EAAE,YAAY,EAAE,UAAU,EAAE,SAAS,EAAE,YAAY,EAAE,SAAS;IACrE,WAAW,EAAE,WAAW,EAAE,MAAM,EAAE,UAAU,EAAE,SAAS,EAAE,UAAU;CACpE,CAAC,CAAC;AAEH,0EAA0E;AAC1E,SAAS,cAAc,CAAC,IAAY;IAClC,4BAA4B;IAC5B,MAAM,KAAK,GAAG,IAAI;SACf,OAAO,CAAC,iBAAiB,EAAE,GAAG,CAAC;SAC/B,OAAO,CAAC,UAAU,EAAE,GAAG,CAAC;SACxB,OAAO,CAAC,kBAAkB,EAAE,GAAG,CAAC;SAChC,OAAO,CAAC,wBAAwB,EAAE,IAAI,CAAC;SACvC,OAAO,CAAC,cAAc,EAAE,EAAE,CAAC;SAC3B,OAAO,CAAC,aAAa,EAAE,GAAG,CAAC;SAC3B,OAAO,CAAC,MAAM,EAAE,GAAG,CAAC;SACpB,IAAI,EAAE,CAAC;IAEV,mEAAmE;IACnE,OAAO,KAAK;SACT,KAAK,CAAC,0BAA0B,CAAC;SACjC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;SAClB,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,MAAM,GAAG,EAAE,IAAI,CAAC,CAAC,MAAM,GAAG,GAAG,CAAC,CAAC;AAClD,CAAC;AAED,kEAAkE;AAClE,SAAS,cAAc,CAAC,OAAe;IACrC,MAAM,UAAU,GAAG,OAAO,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAC;IAC3C,MAAM,MAAM,GAAa,EAAE,CAAC;IAC5B,KAAK,MAAM,IAAI,IAAI,UAAU,EAAE,CAAC;QAC9B,MAAM,SAAS,GAAG,cAAc,CAAC,IAAI,CAAC,CAAC;QACvC,IAAI,SAAS,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YACzB,MAAM,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,CAAC;QAC5B,CAAC;IACH,CAAC;IACD,OAAO,MAAM,CAAC;AAChB,CAAC;AAED,yEAAyE;AACzE,SAAS,aAAa,CAAC,QAAgB,EAAE,UAAuB;IAC9D,MAAM,KAAK,GAAG,QAAQ,CAAC,WAAW,EAAE,CAAC;IACrC,MAAM,KAAK,GAAG,KAAK,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;IAEjC,IAAI,KAAK,GAAG,CAAC,CAAC;IAEd,qBAAqB;IACrB,IAAI,SAAS,GAAG,CAAC,CAAC;IAClB,KAAK,MAAM,IAAI,IAAI,UAAU,EAAE,CAAC;QAC9B,IAAI,KAAK,CAAC,QAAQ,CAAC,IAAI,CAAC;YAAE,SAAS,EAAE,CAAC;IACxC,CAAC;IACD,KAAK,IAAI,CAAC,SAAS,GAAG,IAAI,CAAC,GAAG,CAAC,UAAU,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC;IAExD,uBAAuB;IACvB,MAAM,aAAa,GAAG,QAAQ,CAAC,KAAK,CAAC,kCAAkC,CAAC,CAAC;IACzE,IAAI,aAAa;QAAE,KAAK,IAAI,IAAI,CAAC,GAAG,CAAC,aAAa,CAAC,MAAM,GAAG,GAAG,EAAE,CAAC,CAAC,CAAC;IAEpE,eAAe;IACf,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;QACzB,IAAI,YAAY,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC;YAC3B,KAAK,IAAI,CAAC,CAAC;YACX,MAAM;QACR,CAAC;IACH,CAAC;IAED,iCAAiC;IACjC,IAAI,QAAQ,CAAC,MAAM,GAAG,EAAE,IAAI,QAAQ,CAAC,MAAM,GAAG,GAAG;QAAE,KAAK,IAAI,GAAG,CAAC;IAEhE,OAAO,KAAK,CAAC;AACf,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,gBAAgB,CAAC,OAAe,EAAE,KAAa,EAAE,SAAS,GAAG,CAAC;IAC5E,MAAM,UAAU,GAAG,IAAI,GAAG,CACxB,KAAK,CAAC,WAAW,EAAE,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,MAAM,GAAG,CAAC,CAAC,CAC3D,CAAC;IAEF,MAAM,YAAY,GAAG,cAAc,CAAC,OAAO,CAAC,CAAC;IAC7C,MAAM,MAAM,GAAG,cAAc,CAAC,OAAO,CAAC,CAAC;IAEvC,0BAA0B;IAC1B,MAAM,UAAU,GAAG,CAAC,GAAG,IAAI,GAAG,CAAC,CAAC,GAAG,YAAY,EAAE,GAAG,MAAM,CAAC,CAAC,CAAC,CAAC;IAE9D,iBAAiB;IACjB,MAAM,MAAM,GAAG,UAAU,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,EAAE,CAAC,EAAE,KAAK,EAAE,aAAa,CAAC,CAAC,EAAE,UAAU,CAAC,EAAE,CAAC,CAAC,CAAC;IACjF,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,GAAG,CAAC,CAAC,KAAK,CAAC,CAAC;IAEzC,OAAO,MAAM,CAAC,KAAK,CAAC,CAAC,EAAE,SAAS,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;AAClD,CAAC;AAED,8EAA8E;AAC9E,gBAAgB;AAChB,8EAA8E;AAE9E,2CAA2C;AAC3C,SAAS,SAAS,CAAC,CAAS;IAC1B,OAAO,CAAC,CAAC,WAAW,EAAE,CAAC,OAAO,CAAC,UAAU,EAAE,EAAE,CAAC,CAAC,OAAO,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC,IAAI,EAAE,CAAC;AAC7E,CAAC;AAED,4DAA4D;AAC5D,SAAS,UAAU,CAAC,CAAS,EAAE,CAAS;IACtC,MAAM,EAAE,GAAG,IAAI,GAAG,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC;IACtE,MAAM,EAAE,GAAG,IAAI,GAAG,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC;IACtE,IAAI,EAAE,CAAC,IAAI,KAAK,CAAC,IAAI,EAAE,CAAC,IAAI,KAAK,CAAC;QAAE,OAAO,CAAC,CAAC;IAC7C,IAAI,YAAY,GAAG,CAAC,CAAC;IACrB,KAAK,MAAM,CAAC,IAAI,EAAE,EAAE,CAAC;QACnB,IAAI,EAAE,CAAC,GAAG,CAAC,CAAC,CAAC;YAAE,YAAY,EAAE,CAAC;IAChC,CAAC;IACD,MAAM,KAAK,GAAG,EAAE,CAAC,IAAI,GAAG,EAAE,CAAC,IAAI,GAAG,YAAY,CAAC;IAC/C,OAAO,KAAK,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,YAAY,GAAG,KAAK,CAAC;AAChD,CAAC;AAED;;;GAGG;AACH,MAAM,UAAU,oBAAoB,CAAC,SAAmB,EAAE,SAAS,GAAG,GAAG;IACvE,MAAM,IAAI,GAAa,EAAE,CAAC;IAE1B,KAAK,MAAM,SAAS,IAAI,SAAS,EAAE,CAAC;QAClC,IAAI,SAAS,GAAG,KAAK,CAAC;QACtB,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,IAAI,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;YACrC,MAAM,GAAG,GAAG,UAAU,CAAC,SAAS,EAAE,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC;YAC3C,IAAI,GAAG,IAAI,SAAS,EAAE,CAAC;gBACrB,sBAAsB;gBACtB,IAAI,SAAS,CAAC,MAAM,GAAG,IAAI,CAAC,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC;oBACtC,IAAI,CAAC,CAAC,CAAC,GAAG,SAAS,CAAC;gBACtB,CAAC;gBACD,SAAS,GAAG,IAAI,CAAC;gBACjB,MAAM;YACR,CAAC;QACH,CAAC;QACD,IAAI,CAAC,SAAS,EAAE,CAAC;YACf,IAAI,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;QACvB,CAAC;IACH,CAAC;IAED,OAAO,IAAI,CAAC;AACd,CAAC;AAED,8EAA8E;AAC9E,6BAA6B;AAC7B,8EAA8E;AAE9E,MAAM,QAAQ,GAAG,sFAAsF,CAAC;AACxG,MAAM,UAAU,GAAG,oBAAoB,CAAC;AACxC,MAAM,QAAQ,GAAG,8GAA8G,CAAC;AAChI,MAAM,OAAO,GAAG,iJAAiJ,CAAC;AAElK,sDAAsD;AACtD,MAAM,UAAU,cAAc,CAAC,IAAY;IACzC,MAAM,MAAM,GAA2B,EAAE,CAAC;IAE1C,MAAM,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAC;IACpC,IAAI,MAAM,EAAE,CAAC;QACX,MAAM,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE;YAClC,MAAM,CAAC,SAAS,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,CAAC;QACtC,CAAC,CAAC,CAAC;IACL,CAAC;IAED,MAAM,QAAQ,GAAG,IAAI,CAAC,KAAK,CAAC,UAAU,CAAC,CAAC;IACxC,IAAI,QAAQ,EAAE,CAAC;QACb,QAAQ,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE;YACpC,MAAM,CAAC,WAAW,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,CAAC;QACxC,CAAC,CAAC,CAAC;IACL,CAAC;IAED,MAAM,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAC;IACpC,IAAI,MAAM,EAAE,CAAC;QACX,MAAM,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE;YAClC,MAAM,CAAC,SAAS,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,CAAC;QACtC,CAAC,CAAC,CAAC;IACL,CAAC;IAED,OAAO,MAAM,CAAC;AAChB,CAAC;AAED,+BAA+B;AAC/B,MAAM,UAAU,YAAY,CAAC,IAAY;IACvC,MAAM,OAAO,GAAG,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,IAAI,EAAE,CAAC;IAC1C,OAAO,CAAC,GAAG,IAAI,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;AAC5C,CAAC;AAED;;;GAGG;AACH,MAAM,UAAU,eAAe,CAAC,KAAe;IAC7C,sDAAsD;IACtD,gFAAgF;IAChF,oDAAoD;IACpD,MAAM,SAAS,GAAG,yFAAyF,CAAC;IAE5G,MAAM,QAAQ,GAAG,IAAI,GAAG,EAAkB,CAAC;IAE3C,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;QACzB,MAAM,IAAI,GAAG,IAAI,GAAG,EAAU,CAAC;QAC/B,IAAI,CAAyB,CAAC;QAC9B,SAAS,CAAC,SAAS,GAAG,CAAC,CAAC;QACxB,OAAO,CAAC,CAAC,GAAG,SAAS,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,KAAK,IAAI,EAAE,CAAC;YAC3C,MAAM,IAAI,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC;YAClB,IAAI,IAAI,CAAC,MAAM,GAAG,CAAC;gBAAE,SAAS;YAC9B,4CAA4C;YAC5C,IAAI,0FAA0F,CAAC,IAAI,CAAC,IAAI,CAAC;gBAAE,SAAS;YACpH,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC;gBACpB,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;gBACf,QAAQ,CAAC,GAAG,CAAC,IAAI,EAAE,CAAC,QAAQ,CAAC,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;YACpD,CAAC;QACH,CAAC;IACH,CAAC;IAED,iDAAiD;IACjD,OAAO,CAAC,GAAG,QAAQ,CAAC,OAAO,EAAE,CAAC;SAC3B,MAAM,CAAC,CAAC,CAAC,EAAE,IAAI,CAAC,EAAE,EAAE,CAAC,IAAI,IAAI,CAAC,CAAC;SAC/B,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC;SAC3B,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC;SACZ,GAAG,CAAC,CAAC,CAAC,IAAI,CAAC,EAAE,EAAE,CAAC,IAAI,CAAC,CAAC;AAC3B,CAAC;AAED,8EAA8E;AAC9E,kBAAkB;AAClB,8EAA8E;AAE9E,MAAM,mBAAmB,GAAG,CAAC,IAAI,EAAE,QAAQ,EAAE,SAAS,EAAE,YAAY,EAAE,YAAY,EAAE,aAAa,EAAE,aAAa,EAAE,cAAc,CAAC,CAAC;AAElI,iDAAiD;AACjD,MAAM,UAAU,iBAAiB,CAAC,KAAa;IAC7C,MAAM,KAAK,GAAG,KAAK,CAAC,WAAW,EAAE,CAAC;IAClC,OAAO,mBAAmB,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,CAAC;AAC1D,CAAC;AAED;;;GAGG;AACH,MAAM,UAAU,uBAAuB,CAAC,KAAa;IACnD,MAAM,OAAO,GAAG,KAAK,CAAC,KAAK,CAAC,kCAAkC,CAAC,CAAC;IAChE,IAAI,OAAO,EAAE,CAAC;QACZ,OAAO,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,EAAE,OAAO,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CACpD,CAAC,CAAC,OAAO,CAAC,qCAAqC,EAAE,EAAE,CAAC,CAAC,IAAI,EAAE,CAC5D,CAAC;IACJ,CAAC;IAED,MAAM,YAAY,GAAG,KAAK,CAAC,KAAK,CAAC,8CAA8C,CAAC,CAAC;IACjF,IAAI,YAAY,EAAE,CAAC;QACjB,OAAO,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,EAAE,YAAY,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC;IAC1D,CAAC;IAED,MAAM,SAAS,GAAG,KAAK,CAAC,KAAK,CAAC,kDAAkD,CAAC,CAAC;IAClF,IAAI,SAAS,EAAE,CAAC;QACd,OAAO,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,EAAE,SAAS,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC;IACpD,CAAC;IAED,OAAO,EAAE,CAAC;AACZ,CAAC;AAED,MAAM,qBAAqB,GAAG;IAC5B,EAAE,IAAI,EAAE,OAAO,EAAE,QAAQ,EAAE,CAAC,yDAAyD,EAAE,uBAAuB,CAAC,EAAE;IACjH,EAAE,IAAI,EAAE,UAAU,EAAE,QAAQ,EAAE,CAAC,gEAAgE,CAAC,EAAE;IAClG,EAAE,IAAI,EAAE,MAAM,EAAE,QAAQ,EAAE,CAAC,4DAA4D,CAAC,EAAE;IAC1F,EAAE,IAAI,EAAE,MAAM,EAAE,QAAQ,EAAE,CAAC,8EAA8E,CAAC,EAAE;IAC5G,EAAE,IAAI,EAAE,UAAU,EAAE,QAAQ,EAAE,CAAC,0EAA0E,CAAC,EAAE;IAC5G,EAAE,IAAI,EAAE,QAAQ,EAAE,QAAQ,EAAE,CAAC,yCAAyC,EAAE,0CAA0C,CAAC,EAAE;CACtH,CAAC;AAEF;;GAEG;AACH,MAAM,UAAU,oBAAoB,CAAC,OAAe,EAAE,QAAkB;IACtE,IAAI,QAAQ,CAAC,MAAM,GAAG,CAAC;QAAE,OAAO,SAAS,CAAC;IAE1C,MAAM,OAAO,GAAG,qBAAqB,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC;IACvD,MAAM,IAAI,GAA2C,EAAE,CAAC;IAExD,KAAK,MAAM,MAAM,IAAI,QAAQ,EAAE,CAAC;QAC9B,IAAI,CAAC,MAAM,CAAC,GAAG,EAAE,CAAC;QAClB,yCAAyC;QACzC,MAAM,KAAK,GAAG,OAAO,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;QACnC,MAAM,QAAQ,GAAG,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,WAAW,EAAE,CAAC,QAAQ,CAAC,MAAM,CAAC,WAAW,EAAE,CAAC,CAAC,CAAC;QACnF,MAAM,UAAU,GAAG,QAAQ,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;QAEtC,KAAK,MAAM,IAAI,IAAI,qBAAqB,EAAE,CAAC;YACzC,KAAK,MAAM,OAAO,IAAI,IAAI,CAAC,QAAQ,EAAE,CAAC;gBACpC,MAAM,CAAC,GAAG,UAAU,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC;gBACpC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;oBACd,IAAI,CAAC,MAAM,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC,CAAC;oBACpD,MAAM;gBACR,CAAC;YACH,CAAC;YACD,IAAI,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC;gBAC7B,IAAI,CAAC,MAAM,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,KAAK,CAAC;YAClC,CAAC;QACH,CAAC;IACH,CAAC;IAED,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE,CAAC;AAC3B,CAAC;AAED,8EAA8E;AAC9E,wBAAwB;AACxB,8EAA8E;AAE9E;;;;;GAKG;AACH,SAAS,YAAY,CACnB,KAAqF,EACrF,QAAgB;IAEhB,2BAA2B;IAC3B,MAAM,MAAM,GAAG,CAAC,GAAG,KAAK,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,cAAc,GAAG,CAAC,CAAC,cAAc,CAAC,CAAC;IAE9E,MAAM,KAAK,GAAa,EAAE,CAAC;IAC3B,IAAI,UAAU,GAAG,CAAC,CAAC;IAEnB,KAAK,MAAM,IAAI,IAAI,MAAM,EAAE,CAAC;QAC1B,IAAI,CAAC,IAAI,CAAC,OAAO;YAAE,SAAS;QAE5B,MAAM,MAAM,GAAG,OAAO,IAAI,CAAC,KAAK,KAAK,IAAI,CAAC,GAAG,OAAO,CAAC;QACrD,MAAM,IAAI,GAAG,IAAI,CAAC,OAAO,CAAC;QAC1B,MAAM,OAAO,GAAG,GAAG,MAAM,GAAG,IAAI,aAAa,CAAC;QAE9C,IAAI,UAAU,GAAG,OAAO,CAAC,MAAM,GAAG,QAAQ,EAAE,CAAC;YAC3C,wBAAwB;YACxB,MAAM,SAAS,GAAG,QAAQ,GAAG,UAAU,GAAG,MAAM,CAAC,MAAM,GAAG,EAAE,CAAC;YAC7D,IAAI,SAAS,GAAG,GAAG,EAAE,CAAC;gBACpB,KAAK,CAAC,IAAI,CAAC,GAAG,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,SAAS,CAAC,gBAAgB,CAAC,CAAC;YACnE,CAAC;YACD,MAAM;QACR,CAAC;QAED,KAAK,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;QACpB,UAAU,IAAI,OAAO,CAAC,MAAM,CAAC;IAC/B,CAAC;IAED,OAAO,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;AACxB,CAAC;AAED,8EAA8E;AAC9E,0BAA0B;AAC1B,8EAA8E;AAE9E,SAAS,OAAO,CAAI,EAAU,EAAE,KAAa;IAC3C,OAAO,IAAI,OAAO,CAAC,CAAC,CAAC,EAAE,MAAM,EAAE,EAAE,CAC/B,UAAU,CAAC,GAAG,EAAE,CAAC,MAAM,CAAC,IAAI,KAAK,CAAC,GAAG,KAAK,oBAAoB,EAAE,IAAI,CAAC,CAAC,EAAE,EAAE,CAAC,CAC5E,CAAC;AACJ,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,SAAS,CAAC,OAAyB;IACvD,MAAM,EACJ,KAAK,EACL,KAAK,GAAG,CAAC,EACT,MAAM,GAAG,QAAQ,EACjB,QAAQ,GAAG,KAAK,EAChB,kBAAkB,GAAG,IAAI,GAC1B,GAAG,OAAO,CAAC;IAEZ,MAAM,SAAS,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;IAC7B,MAAM,SAAS,GAAG,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,GAAG,CAAC,KAAK,EAAE,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;IAEnD,6EAA6E;IAC7E,MAAM,EAAE,QAAQ,EAAE,MAAM,EAAE,GAAG,qBAAqB,EAAE,CAAC;IACrD,MAAM,aAAa,GAAG,MAAM,OAAO,CAAC,IAAI,CAAC;QACvC,QAAQ,CAAC,SAAS,CAAC,KAAK,EAAE,EAAE,KAAK,EAAE,SAAS,GAAG,CAAC,EAAE,MAAM,EAAE,CAAC,EAAE,qCAAqC;QAClG,OAAO,CAAQ,KAAK,EAAE,QAAQ,CAAC;KAChC,CAAC,CAAC;IAEH,MAAM,UAAU,GAAG,CAAC,KAAK,CAAC,OAAO,CAAC,aAAa,CAAC,CAAC,CAAC,CAAC,aAAa,CAAC,CAAC,CAAE,aAAqB,EAAE,OAAO,IAAI,EAAE,CAAC;SACtG,KAAK,CAAC,CAAC,EAAE,SAAS,GAAG,CAAC,CAAC,CAAC;IAE3B,IAAI,UAAU,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QAC5B,OAAO;YACL,KAAK;YACL,MAAM;YACN,OAAO,EAAE,EAAE;YACX,MAAM,EAAE,EAAE;YACV,OAAO,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,SAAS;SAChC,CAAC;IACJ,CAAC;IAED,6EAA6E;IAC7E,MAAM,IAAI,GAAG,UAAU,CAAC,GAAG,CAAC,CAAC,CAAM,EAAE,EAAE,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC;IAC/D,MAAM,KAAK,GAAG,MAAM,OAAO,CAAC,IAAI,CAAC;QAC/B,SAAS,CAAC,IAAI,EAAE,EAAE,WAAW,EAAE,CAAC,EAAE,MAAM,EAAE,UAAU,EAAE,CAAC;QACvD,OAAO,CAAQ,MAAM,EAAE,aAAa,CAAC;KACtC,CAAC,CAAC;IAYH,MAAM,WAAW,GAAe,EAAE,CAAC;IAEnC,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,KAAK,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;QACtC,MAAM,IAAI,GAAG,KAAK,CAAC,CAAC,CAAQ,CAAC;QAC7B,MAAM,YAAY,GAAG,UAAU,CAAC,CAAC,CAAQ,CAAC;QAC1C,MAAM,GAAG,GAAG,IAAI,CAAC,CAAC,CAAC,CAAC;QAEpB,IAAI,CAAC,IAAI,IAAI,IAAI,CAAC,KAAK;YAAE,SAAS;QAElC,MAAM,OAAO,GAAW,IAAI,CAAC,OAAO,IAAI,EAAE,CAAC;QAC3C,MAAM,KAAK,GAAW,IAAI,CAAC,KAAK,IAAI,YAAY,EAAE,KAAK,IAAI,GAAG,CAAC;QAE/D,MAAM,cAAc,GAAG,qBAAqB,CAAC,OAAO,EAAE,KAAK,CAAC,CAAC;QAE7D,IAAI,cAAc,GAAG,kBAAkB,IAAI,WAAW,CAAC,MAAM,IAAI,CAAC,EAAE,CAAC;YACnE,+CAA+C;YAC/C,SAAS;QACX,CAAC;QAED,MAAM,SAAS,GAAG,gBAAgB,CAAC,OAAO,EAAE,KAAK,EAAE,CAAC,CAAC,CAAC;QAEtD,WAAW,CAAC,IAAI,CAAC;YACf,GAAG;YACH,KAAK;YACL,OAAO;YACP,cAAc;YACd,SAAS;YACT,SAAS,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;SACpC,CAAC,CAAC;QAEH,IAAI,WAAW,CAAC,MAAM,IAAI,SAAS;YAAE,MAAM;IAC7C,CAAC;IAED,6EAA6E;IAC7E,MAAM,aAAa,GAAG,YAAY,CAAC,WAAW,EAAE,QAAQ,CAAC,CAAC;IAE1D,6EAA6E;IAC7E,MAAM,OAAO,GAAmB,WAAW,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC;QACpD,GAAG,EAAE,CAAC,CAAC,GAAG;QACV,KAAK,EAAE,CAAC,CAAC,KAAK;QACd,cAAc,EAAE,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,cAAc,GAAG,IAAI,CAAC,GAAG,IAAI;QAC1D,SAAS,EAAE,CAAC,CAAC,SAAS;QACtB,SAAS,EAAE,CAAC,CAAC,SAAS;KACvB,CAAC,CAAC,CAAC;IAEJ,MAAM,MAAM,GAAoB;QAC9B,KAAK;QACL,MAAM;QACN,OAAO;QACP,MAAM,EAAE,aAAa;QACrB,OAAO,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,SAAS;KAChC,CAAC;IAEF,6EAA6E;IAC7E,IAAI,MAAM,KAAK,YAAY,IAAI,MAAM,KAAK,YAAY,EAAE,CAAC;QACvD,MAAM,QAAQ,GAAG,WAAW,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC;QACjD,MAAM,QAAQ,GAAG,WAAW,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC;QACvD,MAAM,iBAAiB,GAAG,oBAAoB,CAAC,QAAQ,CAAC,CAAC;QACzD,MAAM,QAAQ,GAAG,eAAe,CAAC,QAAQ,CAAC,CAAC;QAC3C,MAAM,KAAK,GAAG,YAAY,CAAC,aAAa,CAAC,CAAC;QAC1C,MAAM,OAAO,GAAG,cAAc,CAAC,aAAa,CAAC,CAAC;QAE9C,MAAM,CAAC,UAAU,GAAG;YAClB,KAAK,EAAE,iBAAiB;YACxB,QAAQ;YACR,KAAK;YACL,OAAO;SACR,CAAC;IACJ,CAAC;IAED,6EAA6E;IAC7E,IAAI,MAAM,KAAK,YAAY,IAAI,iBAAiB,CAAC,KAAK,CAAC,EAAE,CAAC;QACxD,MAAM,gBAAgB,GAAG,uBAAuB,CAAC,KAAK,CAAC,CAAC;QACxD,IAAI,gBAAgB,CAAC,MAAM,IAAI,CAAC,EAAE,CAAC;YACjC,MAAM,CAAC,UAAU,GAAG,oBAAoB,CAAC,aAAa,EAAE,gBAAgB,CAAC,CAAC;QAC5E,CAAC;IACH,CAAC;IAED,OAAO,MAAM,CAAC;AAChB,CAAC"}
|
|
@@ -0,0 +1,34 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Domain-aware structured extractors for WebPeel.
|
|
3
|
+
*
|
|
4
|
+
* When peel() fetches a URL that matches a known domain, the relevant
|
|
5
|
+
* extractor fires and returns clean structured data + a markdown summary.
|
|
6
|
+
*
|
|
7
|
+
* Supported domains:
|
|
8
|
+
* - twitter.com / x.com — tweets, threads, profiles
|
|
9
|
+
* - reddit.com — posts with comments (via JSON API)
|
|
10
|
+
* - github.com — repos, issues, PRs, users (via GitHub API)
|
|
11
|
+
* - news.ycombinator.com — stories with comments (via HN Firebase API)
|
|
12
|
+
*/
|
|
13
|
+
export interface DomainExtractResult {
|
|
14
|
+
/** Canonical domain name (e.g. 'twitter.com') */
|
|
15
|
+
domain: string;
|
|
16
|
+
/** Page type within the domain (e.g. 'tweet', 'thread', 'repo', 'issue') */
|
|
17
|
+
type: string;
|
|
18
|
+
/** Domain-specific structured data */
|
|
19
|
+
structured: Record<string, any>;
|
|
20
|
+
/** Clean markdown representation of the content */
|
|
21
|
+
cleanContent: string;
|
|
22
|
+
}
|
|
23
|
+
/** An extractor receives the raw HTML and original URL, may make API calls. */
|
|
24
|
+
export type DomainExtractor = (html: string, url: string) => Promise<DomainExtractResult | null>;
|
|
25
|
+
/**
|
|
26
|
+
* Returns the domain extractor for a URL, or null if none matches.
|
|
27
|
+
*/
|
|
28
|
+
export declare function getDomainExtractor(url: string): DomainExtractor | null;
|
|
29
|
+
/**
|
|
30
|
+
* Convenience: run the extractor for the URL (if one exists).
|
|
31
|
+
* Returns null when no extractor matches or extraction fails.
|
|
32
|
+
*/
|
|
33
|
+
export declare function extractDomainData(html: string, url: string): Promise<DomainExtractResult | null>;
|
|
34
|
+
//# sourceMappingURL=domain-extractors.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"domain-extractors.d.ts","sourceRoot":"","sources":["../../src/core/domain-extractors.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;GAWG;AAQH,MAAM,WAAW,mBAAmB;IAClC,iDAAiD;IACjD,MAAM,EAAE,MAAM,CAAC;IACf,4EAA4E;IAC5E,IAAI,EAAE,MAAM,CAAC;IACb,sCAAsC;IACtC,UAAU,EAAE,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC;IAChC,mDAAmD;IACnD,YAAY,EAAE,MAAM,CAAC;CACtB;AAED,+EAA+E;AAC/E,MAAM,MAAM,eAAe,GAAG,CAC5B,IAAI,EAAE,MAAM,EACZ,GAAG,EAAE,MAAM,KACR,OAAO,CAAC,mBAAmB,GAAG,IAAI,CAAC,CAAC;AAgBzC;;GAEG;AACH,wBAAgB,kBAAkB,CAAC,GAAG,EAAE,MAAM,GAAG,eAAe,GAAG,IAAI,CAWtE;AAED;;;GAGG;AACH,wBAAsB,iBAAiB,CACrC,IAAI,EAAE,MAAM,EACZ,GAAG,EAAE,MAAM,GACV,OAAO,CAAC,mBAAmB,GAAG,IAAI,CAAC,CAQrC"}
|