webpeel 0.1.2 → 0.3.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -1 +1 @@
1
- {"version":3,"file":"fetcher.js","sourceRoot":"","sources":["../../src/core/fetcher.ts"],"names":[],"mappings":"AAAA;;GAEG;AAEH,OAAO,EAAE,QAAQ,EAA2B,MAAM,YAAY,CAAC;AAC/D,OAAO,EAAE,YAAY,EAAE,YAAY,EAAE,YAAY,EAAE,YAAY,EAAE,MAAM,aAAa,CAAC;AAErF,MAAM,WAAW,GAAG;IAClB,uHAAuH;IACvH,iHAAiH;IACjH,uHAAuH;IACvH,kFAAkF;IAClF,uGAAuG;CACxG,CAAC;AAEF,SAAS,kBAAkB;IACzB,OAAO,WAAW,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,MAAM,EAAE,GAAG,WAAW,CAAC,MAAM,CAAC,CAAC,CAAC;AACrE,CAAC;AAED;;;GAGG;AACH,SAAS,WAAW,CAAC,SAAiB;IACpC,eAAe;IACf,IAAI,SAAS,CAAC,MAAM,GAAG,IAAI,EAAE,CAAC;QAC5B,MAAM,IAAI,YAAY,CAAC,oCAAoC,CAAC,CAAC;IAC/D,CAAC;IAED,uDAAuD;IACvD,IAAI,iBAAiB,CAAC,IAAI,CAAC,SAAS,CAAC,EAAE,CAAC;QACtC,MAAM,IAAI,YAAY,CAAC,yCAAyC,CAAC,CAAC;IACpE,CAAC;IAED,IAAI,GAAQ,CAAC;IACb,IAAI,CAAC;QACH,GAAG,GAAG,IAAI,GAAG,CAAC,SAAS,CAAC,CAAC;IAC3B,CAAC;IAAC,MAAM,CAAC;QACP,MAAM,IAAI,YAAY,CAAC,oBAAoB,CAAC,CAAC;IAC/C,CAAC;IAED,qBAAqB;IACrB,IAAI,CAAC,CAAC,OAAO,EAAE,QAAQ,CAAC,CAAC,QAAQ,CAAC,GAAG,CAAC,QAAQ,CAAC,EAAE,CAAC;QAChD,MAAM,IAAI,YAAY,CAAC,2CAA2C,CAAC,CAAC;IACtE,CAAC;IAED,iCAAiC;IACjC,IAAI,CAAC,GAAG,CAAC,QAAQ,EAAE,CAAC;QAClB,MAAM,IAAI,YAAY,CAAC,kBAAkB,CAAC,CAAC;IAC7C,CAAC;IAED,MAAM,QAAQ,GAAG,GAAG,CAAC,QAAQ,CAAC,WAAW,EAAE,CAAC;IAE5C,2BAA2B;IAC3B,MAAM,iBAAiB,GAAG,CAAC,WAAW,EAAE,SAAS,CAAC,CAAC;IACnD,IAAI,iBAAiB,CAAC,IAAI,CAAC,OAAO,CAAC,EAAE,CAAC,QAAQ,KAAK,OAAO,IAAI,QAAQ,CAAC,QAAQ,CAAC,GAAG,GAAG,OAAO,CAAC,CAAC,EAAE,CAAC;QAChG,MAAM,IAAI,YAAY,CAAC,oCAAoC,CAAC,CAAC;IAC/D,CAAC;IAED,iFAAiF;IACjF,MAAM,QAAQ,GAAG,oBAAoB,CAAC,QAAQ,CAAC,CAAC;IAChD,IAAI,QAAQ,EAAE,CAAC;QACb,mBAAmB,CAAC,QAAQ,CAAC,CAAC;IAChC,CAAC;IAED,0CAA0C;IAC1C,IAAI,QAAQ,CAAC,QAAQ,CAAC,GAAG,CAAC,EAAE,CAAC;QAC3B,mBAAmB,CAAC,QAAQ,CAAC,CAAC;IAChC,CAAC;AACH,CAAC;AAED;;;GAGG;AACH,SAAS,oBAAoB,CAAC,QAAgB;IAC5C,6BAA6B;IAC7B,MAAM,OAAO,GAAG,QAAQ,CAAC,OAAO,CAAC,UAAU,EAAE,EAAE,CAAC,CAAC;IAEjD,wCAAwC;IACxC,MAAM,WAAW,GAAG,8CAA8C,CAAC;IACnE,MAAM,WAAW,GAAG,OAAO,CAAC,KAAK,CAAC,WAAW,CAAC,CAAC;IAC/C,IAAI,WAAW,EAAE,CAAC;QAChB,MAAM,MAAM,GAAG,WAAW,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC;QAChD,IAAI,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,GAAG,CAAC,EAAE,CAAC;YAC1C,OAAO,MAAM,CAAC;QAChB,CAAC;QACD,MAAM,IAAI,YAAY,CAAC,sBAAsB,CAAC,CAAC;IACjD,CAAC;IAED,2BAA2B;IAC3B,IAAI,kBAAkB,CAAC,IAAI,CAAC,OAAO,CAAC,EAAE,CAAC;QACrC,MAAM,GAAG,GAAG,QAAQ,CAAC,OAAO,EAAE,EAAE,CAAC,CAAC;QAClC,OAAO;YACL,CAAC,GAAG,KAAK,EAAE,CAAC,GAAG,IAAI;YACnB,CAAC,GAAG,KAAK,EAAE,CAAC,GAAG,IAAI;YACnB,CAAC,GAAG,KAAK,CAAC,CAAC,GAAG,IAAI;YAClB,GAAG,GAAG,IAAI;SACX,CAAC;IACJ,CAAC;IAED,wDAAwD;IACxD,IAAI,SAAS,CAAC,IAAI,CAAC,OAAO,CAAC,EAAE,CAAC;QAC5B,0BAA0B;QAC1B,IAAI,WAAW,CAAC,IAAI,CAAC,OAAO,CAAC,EAAE,CAAC;YAC9B,MAAM,GAAG,GAAG,QAAQ,CAAC,OAAO,EAAE,CAAC,CAAC,CAAC;YACjC,IAAI,GAAG,IAAI,UAAU,EAAE,CAAC;gBACtB,OAAO;oBACL,CAAC,GAAG,KAAK,EAAE,CAAC,GAAG,IAAI;oBACnB,CAAC,GAAG,KAAK,EAAE,CAAC,GAAG,IAAI;oBACnB,CAAC,GAAG,KAAK,CAAC,CAAC,GAAG,IAAI;oBAClB,GAAG,GAAG,IAAI;iBACX,CAAC;YACJ,CAAC;QACH,CAAC;QACD,kCAAkC;QAClC,MAAM,KAAK,GAAG,OAAO,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;QACjC,IAAI,KAAK,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YACvB,MAAM,MAAM,GAAG,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,QAAQ,CAAC,CAAC,EAAE,SAAS,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;YACvE,IAAI,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,GAAG,CAAC,EAAE,CAAC;gBAC1C,OAAO,MAAM,CAAC;YAChB,CAAC;QACH,CAAC;IACH,CAAC;IAED,+BAA+B;IAC/B,IAAI,OAAO,CAAC,IAAI,CAAC,OAAO,CAAC,EAAE,CAAC;QAC1B,MAAM,GAAG,GAAG,QAAQ,CAAC,OAAO,EAAE,EAAE,CAAC,CAAC;QAClC,IAAI,GAAG,IAAI,UAAU,EAAE,CAAC;YACtB,OAAO;gBACL,CAAC,GAAG,KAAK,EAAE,CAAC,GAAG,IAAI;gBACnB,CAAC,GAAG,KAAK,EAAE,CAAC,GAAG,IAAI;gBACnB,CAAC,GAAG,KAAK,CAAC,CAAC,GAAG,IAAI;gBAClB,GAAG,GAAG,IAAI;aACX,CAAC;QACJ,CAAC;IACH,CAAC;IAED,OAAO,IAAI,CAAC;AACd,CAAC;AAED;;GAEG;AACH,SAAS,mBAAmB,CAAC,MAAgB;IAC3C,MAAM,CAAC,CAAC,EAAE,CAAC,EAAE,CAAC,EAAE,CAAC,CAAC,GAAG,MAAM,CAAC;IAE5B,wBAAwB;IACxB,IAAI,CAAC,KAAK,GAAG,EAAE,CAAC;QACd,MAAM,IAAI,YAAY,CAAC,6CAA6C,CAAC,CAAC;IACxE,CAAC;IAED,sBAAsB;IACtB,IAAI,CAAC,KAAK,EAAE,EAAE,CAAC;QACb,MAAM,IAAI,YAAY,CAAC,+CAA+C,CAAC,CAAC;IAC1E,CAAC;IAED,yBAAyB;IACzB,IAAI,CAAC,KAAK,GAAG,IAAI,CAAC,IAAI,EAAE,IAAI,CAAC,IAAI,EAAE,EAAE,CAAC;QACpC,MAAM,IAAI,YAAY,CAAC,+CAA+C,CAAC,CAAC;IAC1E,CAAC;IAED,0BAA0B;IAC1B,IAAI,CAAC,KAAK,GAAG,IAAI,CAAC,KAAK,GAAG,EAAE,CAAC;QAC3B,MAAM,IAAI,YAAY,CAAC,+CAA+C,CAAC,CAAC;IAC1E,CAAC;IAED,6BAA6B;IAC7B,IAAI,CAAC,KAAK,GAAG,IAAI,CAAC,KAAK,GAAG,EAAE,CAAC;QAC3B,MAAM,IAAI,YAAY,CAAC,+CAA+C,CAAC,CAAC;IAC1E,CAAC;IAED,6BAA6B;IAC7B,IAAI,CAAC,KAAK,GAAG,IAAI,CAAC,KAAK,GAAG,IAAI,CAAC,KAAK,GAAG,IAAI,CAAC,KAAK,GAAG,EAAE,CAAC;QACrD,MAAM,IAAI,YAAY,CAAC,4CAA4C,CAAC,CAAC;IACvE,CAAC;IAED,0BAA0B;IAC1B,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC;QACZ,MAAM,IAAI,YAAY,CAAC,mDAAmD,CAAC,CAAC;IAC9E,CAAC;AACH,CAAC;AAED;;GAEG;AACH,SAAS,mBAAmB,CAAC,QAAgB;IAC3C,kBAAkB;IAClB,MAAM,IAAI,GAAG,QAAQ,CAAC,OAAO,CAAC,UAAU,EAAE,EAAE,CAAC,CAAC,WAAW,EAAE,CAAC;IAE5D,gBAAgB;IAChB,IAAI,IAAI,KAAK,KAAK,IAAI,IAAI,KAAK,iBAAiB,EAAE,CAAC;QACjD,MAAM,IAAI,YAAY,CAAC,6CAA6C,CAAC,CAAC;IACxE,CAAC;IAED,2DAA2D;IAC3D,IAAI,IAAI,CAAC,UAAU,CAAC,SAAS,CAAC,EAAE,CAAC;QAC/B,wBAAwB;QACxB,MAAM,QAAQ,GAAG,IAAI,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC;QAEnC,iEAAiE;QACjE,IAAI,QAAQ,CAAC,QAAQ,CAAC,GAAG,CAAC,EAAE,CAAC;YAC3B,oBAAoB;YACpB,MAAM,KAAK,GAAG,QAAQ,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;YAClC,IAAI,KAAK,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;gBACvB,MAAM,MAAM,GAAG,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,QAAQ,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC;gBAC/C,IAAI,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,GAAG,CAAC,EAAE,CAAC;oBACvD,mBAAmB,CAAC,MAAM,CAAC,CAAC;gBAC9B,CAAC;YACH,CAAC;QACH,CAAC;aAAM,CAAC;YACN,gDAAgD;YAChD,MAAM,MAAM,GAAG,QAAQ,CAAC,OAAO,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC;YAC1C,IAAI,iBAAiB,CAAC,IAAI,CAAC,MAAM,CAAC,EAAE,CAAC;gBACnC,MAAM,GAAG,GAAG,QAAQ,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC;gBACjC,MAAM,MAAM,GAAG;oBACb,CAAC,GAAG,KAAK,EAAE,CAAC,GAAG,IAAI;oBACnB,CAAC,GAAG,KAAK,EAAE,CAAC,GAAG,IAAI;oBACnB,CAAC,GAAG,KAAK,CAAC,CAAC,GAAG,IAAI;oBAClB,GAAG,GAAG,IAAI;iBACX,CAAC;gBACF,mBAAmB,CAAC,MAAM,CAAC,CAAC;YAC9B,CAAC;QACH,CAAC;QACD,MAAM,IAAI,YAAY,CAAC,qDAAqD,CAAC,CAAC;IAChF,CAAC;IAED,sDAAsD;IACtD,IAAI,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC,IAAI,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC,EAAE,CAAC;QACnD,MAAM,IAAI,YAAY,CAAC,sDAAsD,CAAC,CAAC;IACjF,CAAC;IAED,wBAAwB;IACxB,IAAI,IAAI,CAAC,UAAU,CAAC,KAAK,CAAC,IAAI,IAAI,CAAC,UAAU,CAAC,KAAK,CAAC;QAChD,IAAI,CAAC,UAAU,CAAC,KAAK,CAAC,IAAI,IAAI,CAAC,UAAU,CAAC,KAAK,CAAC,EAAE,CAAC;QACrD,MAAM,IAAI,YAAY,CAAC,oDAAoD,CAAC,CAAC;IAC/E,CAAC;AACH,CAAC;AAED;;GAEG;AACH,SAAS,iBAAiB,CAAC,SAAiB;IAC1C,IAAI,SAAS,CAAC,MAAM,GAAG,GAAG,EAAE,CAAC;QAC3B,MAAM,IAAI,YAAY,CAAC,0CAA0C,CAAC,CAAC;IACrE,CAAC;IACD,wCAAwC;IACxC,IAAI,CAAC,gBAAgB,CAAC,IAAI,CAAC,SAAS,CAAC,EAAE,CAAC;QACtC,MAAM,IAAI,YAAY,CAAC,wCAAwC,CAAC,CAAC;IACnE,CAAC;IACD,OAAO,SAAS,CAAC;AACnB,CAAC;AAQD;;;;GAIG;AACH,MAAM,CAAC,KAAK,UAAU,WAAW,CAC/B,GAAW,EACX,SAAkB,EAClB,YAAoB,KAAK;IAEzB,yCAAyC;IACzC,WAAW,CAAC,GAAG,CAAC,CAAC;IAEjB,kCAAkC;IAClC,MAAM,kBAAkB,GAAG,SAAS,CAAC,CAAC,CAAC,iBAAiB,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,kBAAkB,EAAE,CAAC;IAE3F,MAAM,aAAa,GAAG,EAAE,CAAC;IACzB,IAAI,aAAa,GAAG,CAAC,CAAC;IACtB,IAAI,UAAU,GAAG,GAAG,CAAC;IACrB,MAAM,QAAQ,GAAG,IAAI,GAAG,EAAU,CAAC;IAEnC,OAAO,aAAa,IAAI,aAAa,EAAE,CAAC;QACtC,wBAAwB;QACxB,IAAI,QAAQ,CAAC,GAAG,CAAC,UAAU,CAAC,EAAE,CAAC;YAC7B,MAAM,IAAI,YAAY,CAAC,wBAAwB,CAAC,CAAC;QACnD,CAAC;QACD,QAAQ,CAAC,GAAG,CAAC,UAAU,CAAC,CAAC;QAEzB,+BAA+B;QAC/B,WAAW,CAAC,UAAU,CAAC,CAAC;QAExB,MAAM,UAAU,GAAG,IAAI,eAAe,EAAE,CAAC;QACzC,MAAM,KAAK,GAAG,UAAU,CAAC,GAAG,EAAE,CAAC,UAAU,CAAC,KAAK,EAAE,EAAE,SAAS,CAAC,CAAC;QAE9D,IAAI,CAAC;YACH,MAAM,QAAQ,GAAG,MAAM,KAAK,CAAC,UAAU,EAAE;gBACvC,OAAO,EAAE;oBACP,YAAY,EAAE,kBAAkB;oBAChC,QAAQ,EAAE,4EAA4E;oBACtF,iBAAiB,EAAE,gBAAgB;oBACnC,iBAAiB,EAAE,mBAAmB;oBACtC,KAAK,EAAE,GAAG;oBACV,YAAY,EAAE,YAAY;oBAC1B,2BAA2B,EAAE,GAAG;iBACjC;gBACD,MAAM,EAAE,UAAU,CAAC,MAAM;gBACzB,QAAQ,EAAE,QAAQ,EAAE,qCAAqC;aAC1D,CAAC,CAAC;YAEH,YAAY,CAAC,KAAK,CAAC,CAAC;YAEpB,4BAA4B;YAC5B,IAAI,QAAQ,CAAC,MAAM,IAAI,GAAG,IAAI,QAAQ,CAAC,MAAM,GAAG,GAAG,EAAE,CAAC;gBACpD,MAAM,QAAQ,GAAG,QAAQ,CAAC,OAAO,CAAC,GAAG,CAAC,UAAU,CAAC,CAAC;gBAClD,IAAI,CAAC,QAAQ,EAAE,CAAC;oBACd,MAAM,IAAI,YAAY,CAAC,2CAA2C,CAAC,CAAC;gBACtE,CAAC;gBAED,wBAAwB;gBACxB,UAAU,GAAG,IAAI,GAAG,CAAC,QAAQ,EAAE,UAAU,CAAC,CAAC,IAAI,CAAC;gBAChD,aAAa,EAAE,CAAC;gBAChB,SAAS;YACX,CAAC;YAED,IAAI,CAAC,QAAQ,CAAC,EAAE,EAAE,CAAC;gBACjB,IAAI,QAAQ,CAAC,MAAM,KAAK,GAAG,IAAI,QAAQ,CAAC,MAAM,KAAK,GAAG,EAAE,CAAC;oBACvD,MAAM,IAAI,YAAY,CACpB,QAAQ,QAAQ,CAAC,MAAM,iEAAiE,CACzF,CAAC;gBACJ,CAAC;gBACD,MAAM,IAAI,YAAY,CAAC,QAAQ,QAAQ,CAAC,MAAM,KAAK,QAAQ,CAAC,UAAU,EAAE,CAAC,CAAC;YAC5E,CAAC;YAED,kCAAkC;YAClC,MAAM,WAAW,GAAG,QAAQ,CAAC,OAAO,CAAC,GAAG,CAAC,cAAc,CAAC,IAAI,EAAE,CAAC;YAC/D,IAAI,CAAC,WAAW,CAAC,QAAQ,CAAC,WAAW,CAAC,IAAI,CAAC,WAAW,CAAC,QAAQ,CAAC,uBAAuB,CAAC,EAAE,CAAC;gBACzF,MAAM,IAAI,YAAY,CAAC,mDAAmD,CAAC,CAAC;YAC9E,CAAC;YAED,wEAAwE;YACxE,MAAM,MAAM,GAAiB,EAAE,CAAC;YAChC,IAAI,SAAS,GAAG,CAAC,CAAC;YAClB,MAAM,QAAQ,GAAG,EAAE,GAAG,IAAI,GAAG,IAAI,CAAC,CAAC,OAAO;YAE1C,MAAM,MAAM,GAAG,QAAQ,CAAC,IAAI,EAAE,SAAS,EAAE,CAAC;YAC1C,IAAI,CAAC,MAAM,EAAE,CAAC;gBACZ,MAAM,IAAI,YAAY,CAAC,+BAA+B,CAAC,CAAC;YAC1D,CAAC;YAED,IAAI,CAAC;gBACH,OAAO,IAAI,EAAE,CAAC;oBACZ,MAAM,EAAE,IAAI,EAAE,KAAK,EAAE,GAAG,MAAM,MAAM,CAAC,IAAI,EAAE,CAAC;oBAC5C,IAAI,IAAI;wBAAE,MAAM;oBAEhB,SAAS,IAAI,KAAK,CAAC,MAAM,CAAC;oBAC1B,IAAI,SAAS,GAAG,QAAQ,EAAE,CAAC;wBACzB,MAAM,CAAC,MAAM,EAAE,CAAC;wBAChB,MAAM,IAAI,YAAY,CAAC,+BAA+B,CAAC,CAAC;oBAC1D,CAAC;oBAED,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;gBACrB,CAAC;YACH,CAAC;oBAAS,CAAC;gBACT,MAAM,CAAC,WAAW,EAAE,CAAC;YACvB,CAAC;YAED,iBAAiB;YACjB,MAAM,QAAQ,GAAG,IAAI,UAAU,CAAC,SAAS,CAAC,CAAC;YAC3C,IAAI,MAAM,GAAG,CAAC,CAAC;YACf,KAAK,MAAM,KAAK,IAAI,MAAM,EAAE,CAAC;gBAC3B,QAAQ,CAAC,GAAG,CAAC,KAAK,EAAE,MAAM,CAAC,CAAC;gBAC5B,MAAM,IAAI,KAAK,CAAC,MAAM,CAAC;YACzB,CAAC;YAED,MAAM,IAAI,GAAG,IAAI,WAAW,EAAE,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC;YAEhD,IAAI,CAAC,IAAI,IAAI,IAAI,CAAC,MAAM,GAAG,GAAG,EAAE,CAAC;gBAC/B,MAAM,IAAI,YAAY,CAAC,oEAAoE,CAAC,CAAC;YAC/F,CAAC;YAED,iCAAiC;YACjC,IAAI,IAAI,CAAC,QAAQ,CAAC,yBAAyB,CAAC,IAAI,IAAI,CAAC,QAAQ,CAAC,kBAAkB,CAAC,EAAE,CAAC;gBAClF,MAAM,IAAI,YAAY,CAAC,+DAA+D,CAAC,CAAC;YAC1F,CAAC;YAED,OAAO;gBACL,IAAI;gBACJ,GAAG,EAAE,UAAU;gBACf,UAAU,EAAE,QAAQ,CAAC,MAAM;aAC5B,CAAC;QACJ,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,YAAY,CAAC,KAAK,CAAC,CAAC;YAEpB,IAAI,KAAK,YAAY,YAAY,IAAI,KAAK,YAAY,YAAY,IAAI,KAAK,YAAY,YAAY,EAAE,CAAC;gBACpG,MAAM,KAAK,CAAC;YACd,CAAC;YAED,IAAI,KAAK,YAAY,KAAK,IAAI,KAAK,CAAC,IAAI,KAAK,YAAY,EAAE,CAAC;gBAC1D,MAAM,IAAI,YAAY,CAAC,2BAA2B,SAAS,IAAI,CAAC,CAAC;YACnE,CAAC;YAED,MAAM,IAAI,YAAY,CAAC,oBAAoB,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,eAAe,EAAE,CAAC,CAAC;QACzG,CAAC;IACH,CAAC;IAED,MAAM,IAAI,YAAY,CAAC,2BAA2B,aAAa,GAAG,CAAC,CAAC;AACtE,CAAC;AAED,IAAI,aAAa,GAAmB,IAAI,CAAC;AACzC,IAAI,gBAAgB,GAAG,CAAC,CAAC;AACzB,MAAM,oBAAoB,GAAG,CAAC,CAAC;AAE/B,KAAK,UAAU,UAAU;IACvB,4DAA4D;IAC5D,IAAI,aAAa,EAAE,CAAC;QAClB,IAAI,CAAC;YACH,IAAI,aAAa,CAAC,WAAW,EAAE,EAAE,CAAC;gBAChC,OAAO,aAAa,CAAC;YACvB,CAAC;QACH,CAAC;QAAC,MAAM,CAAC;YACP,4BAA4B;YAC5B,aAAa,GAAG,IAAI,CAAC;QACvB,CAAC;IACH,CAAC;IAED,aAAa,GAAG,MAAM,QAAQ,CAAC,MAAM,CAAC,EAAE,QAAQ,EAAE,IAAI,EAAE,CAAC,CAAC;IAC1D,OAAO,aAAa,CAAC;AACvB,CAAC;AAED;;;GAGG;AACH,MAAM,CAAC,KAAK,UAAU,YAAY,CAChC,GAAW,EACX,UAII,EAAE;IAEN,yCAAyC;IACzC,WAAW,CAAC,GAAG,CAAC,CAAC;IAEjB,MAAM,EAAE,SAAS,EAAE,MAAM,GAAG,CAAC,EAAE,SAAS,GAAG,KAAK,EAAE,GAAG,OAAO,CAAC;IAE7D,kCAAkC;IAClC,MAAM,kBAAkB,GAAG,SAAS,CAAC,CAAC,CAAC,iBAAiB,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,kBAAkB,EAAE,CAAC;IAE3F,qBAAqB;IACrB,IAAI,MAAM,GAAG,CAAC,IAAI,MAAM,GAAG,KAAK,EAAE,CAAC;QACjC,MAAM,IAAI,YAAY,CAAC,yCAAyC,CAAC,CAAC;IACpE,CAAC;IAED,wDAAwD;IACxD,MAAM,cAAc,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;IAClC,MAAM,gBAAgB,GAAG,KAAK,CAAC,CAAC,qBAAqB;IAErD,OAAO,gBAAgB,IAAI,oBAAoB,EAAE,CAAC;QAChD,IAAI,IAAI,CAAC,GAAG,EAAE,GAAG,cAAc,GAAG,gBAAgB,EAAE,CAAC;YACnD,MAAM,IAAI,YAAY,CAAC,2DAA2D,CAAC,CAAC;QACtF,CAAC;QACD,MAAM,IAAI,OAAO,CAAC,OAAO,CAAC,EAAE,CAAC,UAAU,CAAC,OAAO,EAAE,GAAG,CAAC,CAAC,CAAC;IACzD,CAAC;IAED,gBAAgB,EAAE,CAAC;IACnB,IAAI,IAAI,GAAgB,IAAI,CAAC;IAE7B,IAAI,CAAC;QACH,MAAM,OAAO,GAAG,MAAM,UAAU,EAAE,CAAC;QACnC,IAAI,GAAG,MAAM,OAAO,CAAC,OAAO,CAAC;YAC3B,SAAS,EAAE,kBAAkB;SAC9B,CAAC,CAAC;QAEH,2DAA2D;QAC3D,MAAM,IAAI,CAAC,KAAK,CAAC,MAAM,EAAE,CAAC,KAAK,EAAE,EAAE;YACjC,MAAM,YAAY,GAAG,KAAK,CAAC,OAAO,EAAE,CAAC,YAAY,EAAE,CAAC;YACpD,IAAI,CAAC,OAAO,EAAE,MAAM,EAAE,OAAO,EAAE,YAAY,CAAC,CAAC,QAAQ,CAAC,YAAY,CAAC,EAAE,CAAC;gBACpE,KAAK,CAAC,KAAK,EAAE,CAAC;YAChB,CAAC;iBAAM,CAAC;gBACN,KAAK,CAAC,QAAQ,EAAE,CAAC;YACnB,CAAC;QACH,CAAC,CAAC,CAAC;QAEH,6CAA6C;QAC7C,MAAM,YAAY,GAAG,CAAC,KAAK,IAAI,EAAE;YAC/B,MAAM,IAAK,CAAC,IAAI,CAAC,GAAG,EAAE;gBACpB,SAAS,EAAE,kBAAkB;gBAC7B,OAAO,EAAE,SAAS;aACnB,CAAC,CAAC;YAEH,8DAA8D;YAC9D,IAAI,MAAM,GAAG,CAAC,EAAE,CAAC;gBACf,MAAM,IAAK,CAAC,cAAc,CAAC,MAAM,CAAC,CAAC;YACrC,CAAC;YAED,MAAM,IAAI,GAAG,MAAM,IAAK,CAAC,OAAO,EAAE,CAAC;YACnC,MAAM,QAAQ,GAAG,IAAK,CAAC,GAAG,EAAE,CAAC;YAE7B,OAAO,EAAE,IAAI,EAAE,QAAQ,EAAE,CAAC;QAC5B,CAAC,CAAC,EAAE,CAAC;QAEL,MAAM,cAAc,GAAG,IAAI,OAAO,CAAQ,CAAC,CAAC,EAAE,MAAM,EAAE,EAAE;YACtD,UAAU,CAAC,GAAG,EAAE,CAAC,MAAM,CAAC,IAAI,YAAY,CAAC,6BAA6B,SAAS,IAAI,CAAC,CAAC,EAAE,SAAS,CAAC,CAAC;QACpG,CAAC,CAAC,CAAC;QAEH,MAAM,EAAE,IAAI,EAAE,QAAQ,EAAE,GAAG,MAAM,OAAO,CAAC,IAAI,CAAC,CAAC,YAAY,EAAE,cAAc,CAAC,CAAC,CAAC;QAE9E,4BAA4B;QAC5B,IAAI,IAAI,CAAC,MAAM,GAAG,EAAE,GAAG,IAAI,GAAG,IAAI,EAAE,CAAC,CAAC,aAAa;YACjD,MAAM,IAAI,YAAY,CAAC,+BAA+B,CAAC,CAAC;QAC1D,CAAC;QAED,IAAI,CAAC,IAAI,IAAI,IAAI,CAAC,MAAM,GAAG,GAAG,EAAE,CAAC;YAC/B,MAAM,IAAI,YAAY,CAAC,oDAAoD,CAAC,CAAC;QAC/E,CAAC;QAED,OAAO;YACL,IAAI;YACJ,GAAG,EAAE,QAAQ;SACd,CAAC;IACJ,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,IAAI,KAAK,YAAY,YAAY,IAAI,KAAK,YAAY,YAAY,IAAI,KAAK,YAAY,YAAY,EAAE,CAAC;YACpG,MAAM,KAAK,CAAC;QACd,CAAC;QAED,IAAI,KAAK,YAAY,KAAK,IAAI,KAAK,CAAC,OAAO,CAAC,QAAQ,CAAC,SAAS,CAAC,EAAE,CAAC;YAChE,MAAM,IAAI,YAAY,CAAC,8BAA8B,CAAC,CAAC;QACzD,CAAC;QAED,MAAM,IAAI,YAAY,CACpB,yBAAyB,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,eAAe,EAAE,CACpF,CAAC;IACJ,CAAC;YAAS,CAAC;QACT,oDAAoD;QACpD,IAAI,IAAI,EAAE,CAAC;YACT,MAAM,IAAI,CAAC,KAAK,EAAE,CAAC,KAAK,CAAC,GAAG,EAAE,GAAE,CAAC,CAAC,CAAC;QACrC,CAAC;QACD,gBAAgB,EAAE,CAAC;IACrB,CAAC;AACH,CAAC;AAED;;GAEG;AACH,MAAM,CAAC,KAAK,UAAU,UAAU,CAC9B,EAAoB,EACpB,cAAsB,CAAC,EACvB,cAAsB,IAAI;IAE1B,IAAI,SAAS,GAAiB,IAAI,CAAC;IAEnC,KAAK,IAAI,OAAO,GAAG,CAAC,EAAE,OAAO,IAAI,WAAW,EAAE,OAAO,EAAE,EAAE,CAAC;QACxD,IAAI,CAAC;YACH,OAAO,MAAM,EAAE,EAAE,CAAC;QACpB,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,SAAS,GAAG,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,IAAI,KAAK,CAAC,eAAe,CAAC,CAAC;YAExE,4CAA4C;YAC5C,IAAI,KAAK,YAAY,YAAY,IAAI,KAAK,YAAY,YAAY,EAAE,CAAC;gBACnE,MAAM,KAAK,CAAC;YACd,CAAC;YAED,IAAI,OAAO,GAAG,WAAW,EAAE,CAAC;gBAC1B,MAAM,KAAK,GAAG,WAAW,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,OAAO,GAAG,CAAC,CAAC,CAAC;gBACrD,MAAM,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,UAAU,CAAC,OAAO,EAAE,KAAK,CAAC,CAAC,CAAC;YAC7D,CAAC;QACH,CAAC;IACH,CAAC;IAED,MAAM,SAAS,IAAI,IAAI,YAAY,CAAC,cAAc,CAAC,CAAC;AACtD,CAAC;AAED;;GAEG;AACH,MAAM,CAAC,KAAK,UAAU,OAAO;IAC3B,IAAI,aAAa,EAAE,CAAC;QAClB,MAAM,aAAa,CAAC,KAAK,EAAE,CAAC;QAC5B,aAAa,GAAG,IAAI,CAAC;IACvB,CAAC;AACH,CAAC"}
1
+ {"version":3,"file":"fetcher.js","sourceRoot":"","sources":["../../src/core/fetcher.ts"],"names":[],"mappings":"AAAA;;GAEG;AAEH,OAAO,EAAE,QAAQ,EAA2B,MAAM,YAAY,CAAC;AAC/D,OAAO,EAAE,QAAQ,IAAI,eAAe,EAAE,MAAM,kBAAkB,CAAC;AAC/D,OAAO,aAAa,MAAM,gCAAgC,CAAC;AAC3D,OAAO,EAAE,YAAY,EAAE,YAAY,EAAE,YAAY,EAAE,YAAY,EAAE,MAAM,aAAa,CAAC;AAErF,yCAAyC;AACzC,eAAe,CAAC,GAAG,CAAC,aAAa,EAAE,CAAC,CAAC;AAErC,MAAM,WAAW,GAAG;IAClB,uHAAuH;IACvH,iHAAiH;IACjH,uHAAuH;IACvH,kFAAkF;IAClF,uGAAuG;CACxG,CAAC;AAEF,SAAS,kBAAkB;IACzB,OAAO,WAAW,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,MAAM,EAAE,GAAG,WAAW,CAAC,MAAM,CAAC,CAAC,CAAC;AACrE,CAAC;AAED;;;GAGG;AACH,SAAS,WAAW,CAAC,SAAiB;IACpC,eAAe;IACf,IAAI,SAAS,CAAC,MAAM,GAAG,IAAI,EAAE,CAAC;QAC5B,MAAM,IAAI,YAAY,CAAC,oCAAoC,CAAC,CAAC;IAC/D,CAAC;IAED,uDAAuD;IACvD,IAAI,iBAAiB,CAAC,IAAI,CAAC,SAAS,CAAC,EAAE,CAAC;QACtC,MAAM,IAAI,YAAY,CAAC,yCAAyC,CAAC,CAAC;IACpE,CAAC;IAED,IAAI,GAAQ,CAAC;IACb,IAAI,CAAC;QACH,GAAG,GAAG,IAAI,GAAG,CAAC,SAAS,CAAC,CAAC;IAC3B,CAAC;IAAC,MAAM,CAAC;QACP,MAAM,IAAI,YAAY,CAAC,oBAAoB,CAAC,CAAC;IAC/C,CAAC;IAED,qBAAqB;IACrB,IAAI,CAAC,CAAC,OAAO,EAAE,QAAQ,CAAC,CAAC,QAAQ,CAAC,GAAG,CAAC,QAAQ,CAAC,EAAE,CAAC;QAChD,MAAM,IAAI,YAAY,CAAC,2CAA2C,CAAC,CAAC;IACtE,CAAC;IAED,iCAAiC;IACjC,IAAI,CAAC,GAAG,CAAC,QAAQ,EAAE,CAAC;QAClB,MAAM,IAAI,YAAY,CAAC,kBAAkB,CAAC,CAAC;IAC7C,CAAC;IAED,MAAM,QAAQ,GAAG,GAAG,CAAC,QAAQ,CAAC,WAAW,EAAE,CAAC;IAE5C,2BAA2B;IAC3B,MAAM,iBAAiB,GAAG,CAAC,WAAW,EAAE,SAAS,CAAC,CAAC;IACnD,IAAI,iBAAiB,CAAC,IAAI,CAAC,OAAO,CAAC,EAAE,CAAC,QAAQ,KAAK,OAAO,IAAI,QAAQ,CAAC,QAAQ,CAAC,GAAG,GAAG,OAAO,CAAC,CAAC,EAAE,CAAC;QAChG,MAAM,IAAI,YAAY,CAAC,oCAAoC,CAAC,CAAC;IAC/D,CAAC;IAED,iFAAiF;IACjF,MAAM,QAAQ,GAAG,oBAAoB,CAAC,QAAQ,CAAC,CAAC;IAChD,IAAI,QAAQ,EAAE,CAAC;QACb,mBAAmB,CAAC,QAAQ,CAAC,CAAC;IAChC,CAAC;IAED,0CAA0C;IAC1C,IAAI,QAAQ,CAAC,QAAQ,CAAC,GAAG,CAAC,EAAE,CAAC;QAC3B,mBAAmB,CAAC,QAAQ,CAAC,CAAC;IAChC,CAAC;AACH,CAAC;AAED;;;GAGG;AACH,SAAS,oBAAoB,CAAC,QAAgB;IAC5C,6BAA6B;IAC7B,MAAM,OAAO,GAAG,QAAQ,CAAC,OAAO,CAAC,UAAU,EAAE,EAAE,CAAC,CAAC;IAEjD,wCAAwC;IACxC,MAAM,WAAW,GAAG,8CAA8C,CAAC;IACnE,MAAM,WAAW,GAAG,OAAO,CAAC,KAAK,CAAC,WAAW,CAAC,CAAC;IAC/C,IAAI,WAAW,EAAE,CAAC;QAChB,MAAM,MAAM,GAAG,WAAW,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC;QAChD,IAAI,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,GAAG,CAAC,EAAE,CAAC;YAC1C,OAAO,MAAM,CAAC;QAChB,CAAC;QACD,MAAM,IAAI,YAAY,CAAC,sBAAsB,CAAC,CAAC;IACjD,CAAC;IAED,2BAA2B;IAC3B,IAAI,kBAAkB,CAAC,IAAI,CAAC,OAAO,CAAC,EAAE,CAAC;QACrC,MAAM,GAAG,GAAG,QAAQ,CAAC,OAAO,EAAE,EAAE,CAAC,CAAC;QAClC,OAAO;YACL,CAAC,GAAG,KAAK,EAAE,CAAC,GAAG,IAAI;YACnB,CAAC,GAAG,KAAK,EAAE,CAAC,GAAG,IAAI;YACnB,CAAC,GAAG,KAAK,CAAC,CAAC,GAAG,IAAI;YAClB,GAAG,GAAG,IAAI;SACX,CAAC;IACJ,CAAC;IAED,wDAAwD;IACxD,IAAI,SAAS,CAAC,IAAI,CAAC,OAAO,CAAC,EAAE,CAAC;QAC5B,0BAA0B;QAC1B,IAAI,WAAW,CAAC,IAAI,CAAC,OAAO,CAAC,EAAE,CAAC;YAC9B,MAAM,GAAG,GAAG,QAAQ,CAAC,OAAO,EAAE,CAAC,CAAC,CAAC;YACjC,IAAI,GAAG,IAAI,UAAU,EAAE,CAAC;gBACtB,OAAO;oBACL,CAAC,GAAG,KAAK,EAAE,CAAC,GAAG,IAAI;oBACnB,CAAC,GAAG,KAAK,EAAE,CAAC,GAAG,IAAI;oBACnB,CAAC,GAAG,KAAK,CAAC,CAAC,GAAG,IAAI;oBAClB,GAAG,GAAG,IAAI;iBACX,CAAC;YACJ,CAAC;QACH,CAAC;QACD,kCAAkC;QAClC,MAAM,KAAK,GAAG,OAAO,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;QACjC,IAAI,KAAK,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YACvB,MAAM,MAAM,GAAG,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,QAAQ,CAAC,CAAC,EAAE,SAAS,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;YACvE,IAAI,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,GAAG,CAAC,EAAE,CAAC;gBAC1C,OAAO,MAAM,CAAC;YAChB,CAAC;QACH,CAAC;IACH,CAAC;IAED,+BAA+B;IAC/B,IAAI,OAAO,CAAC,IAAI,CAAC,OAAO,CAAC,EAAE,CAAC;QAC1B,MAAM,GAAG,GAAG,QAAQ,CAAC,OAAO,EAAE,EAAE,CAAC,CAAC;QAClC,IAAI,GAAG,IAAI,UAAU,EAAE,CAAC;YACtB,OAAO;gBACL,CAAC,GAAG,KAAK,EAAE,CAAC,GAAG,IAAI;gBACnB,CAAC,GAAG,KAAK,EAAE,CAAC,GAAG,IAAI;gBACnB,CAAC,GAAG,KAAK,CAAC,CAAC,GAAG,IAAI;gBAClB,GAAG,GAAG,IAAI;aACX,CAAC;QACJ,CAAC;IACH,CAAC;IAED,OAAO,IAAI,CAAC;AACd,CAAC;AAED;;GAEG;AACH,SAAS,mBAAmB,CAAC,MAAgB;IAC3C,MAAM,CAAC,CAAC,EAAE,CAAC,EAAE,CAAC,EAAE,CAAC,CAAC,GAAG,MAAM,CAAC;IAE5B,wBAAwB;IACxB,IAAI,CAAC,KAAK,GAAG,EAAE,CAAC;QACd,MAAM,IAAI,YAAY,CAAC,6CAA6C,CAAC,CAAC;IACxE,CAAC;IAED,sBAAsB;IACtB,IAAI,CAAC,KAAK,EAAE,EAAE,CAAC;QACb,MAAM,IAAI,YAAY,CAAC,+CAA+C,CAAC,CAAC;IAC1E,CAAC;IAED,yBAAyB;IACzB,IAAI,CAAC,KAAK,GAAG,IAAI,CAAC,IAAI,EAAE,IAAI,CAAC,IAAI,EAAE,EAAE,CAAC;QACpC,MAAM,IAAI,YAAY,CAAC,+CAA+C,CAAC,CAAC;IAC1E,CAAC;IAED,0BAA0B;IAC1B,IAAI,CAAC,KAAK,GAAG,IAAI,CAAC,KAAK,GAAG,EAAE,CAAC;QAC3B,MAAM,IAAI,YAAY,CAAC,+CAA+C,CAAC,CAAC;IAC1E,CAAC;IAED,6BAA6B;IAC7B,IAAI,CAAC,KAAK,GAAG,IAAI,CAAC,KAAK,GAAG,EAAE,CAAC;QAC3B,MAAM,IAAI,YAAY,CAAC,+CAA+C,CAAC,CAAC;IAC1E,CAAC;IAED,6BAA6B;IAC7B,IAAI,CAAC,KAAK,GAAG,IAAI,CAAC,KAAK,GAAG,IAAI,CAAC,KAAK,GAAG,IAAI,CAAC,KAAK,GAAG,EAAE,CAAC;QACrD,MAAM,IAAI,YAAY,CAAC,4CAA4C,CAAC,CAAC;IACvE,CAAC;IAED,0BAA0B;IAC1B,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC;QACZ,MAAM,IAAI,YAAY,CAAC,mDAAmD,CAAC,CAAC;IAC9E,CAAC;AACH,CAAC;AAED;;GAEG;AACH,SAAS,mBAAmB,CAAC,QAAgB;IAC3C,kBAAkB;IAClB,MAAM,IAAI,GAAG,QAAQ,CAAC,OAAO,CAAC,UAAU,EAAE,EAAE,CAAC,CAAC,WAAW,EAAE,CAAC;IAE5D,gBAAgB;IAChB,IAAI,IAAI,KAAK,KAAK,IAAI,IAAI,KAAK,iBAAiB,EAAE,CAAC;QACjD,MAAM,IAAI,YAAY,CAAC,6CAA6C,CAAC,CAAC;IACxE,CAAC;IAED,2DAA2D;IAC3D,IAAI,IAAI,CAAC,UAAU,CAAC,SAAS,CAAC,EAAE,CAAC;QAC/B,wBAAwB;QACxB,MAAM,QAAQ,GAAG,IAAI,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC;QAEnC,iEAAiE;QACjE,IAAI,QAAQ,CAAC,QAAQ,CAAC,GAAG,CAAC,EAAE,CAAC;YAC3B,oBAAoB;YACpB,MAAM,KAAK,GAAG,QAAQ,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;YAClC,IAAI,KAAK,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;gBACvB,MAAM,MAAM,GAAG,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,QAAQ,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC;gBAC/C,IAAI,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,GAAG,CAAC,EAAE,CAAC;oBACvD,mBAAmB,CAAC,MAAM,CAAC,CAAC;gBAC9B,CAAC;YACH,CAAC;QACH,CAAC;aAAM,CAAC;YACN,gDAAgD;YAChD,MAAM,MAAM,GAAG,QAAQ,CAAC,OAAO,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC;YAC1C,IAAI,iBAAiB,CAAC,IAAI,CAAC,MAAM,CAAC,EAAE,CAAC;gBACnC,MAAM,GAAG,GAAG,QAAQ,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC;gBACjC,MAAM,MAAM,GAAG;oBACb,CAAC,GAAG,KAAK,EAAE,CAAC,GAAG,IAAI;oBACnB,CAAC,GAAG,KAAK,EAAE,CAAC,GAAG,IAAI;oBACnB,CAAC,GAAG,KAAK,CAAC,CAAC,GAAG,IAAI;oBAClB,GAAG,GAAG,IAAI;iBACX,CAAC;gBACF,mBAAmB,CAAC,MAAM,CAAC,CAAC;YAC9B,CAAC;QACH,CAAC;QACD,MAAM,IAAI,YAAY,CAAC,qDAAqD,CAAC,CAAC;IAChF,CAAC;IAED,sDAAsD;IACtD,IAAI,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC,IAAI,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC,EAAE,CAAC;QACnD,MAAM,IAAI,YAAY,CAAC,sDAAsD,CAAC,CAAC;IACjF,CAAC;IAED,wBAAwB;IACxB,IAAI,IAAI,CAAC,UAAU,CAAC,KAAK,CAAC,IAAI,IAAI,CAAC,UAAU,CAAC,KAAK,CAAC;QAChD,IAAI,CAAC,UAAU,CAAC,KAAK,CAAC,IAAI,IAAI,CAAC,UAAU,CAAC,KAAK,CAAC,EAAE,CAAC;QACrD,MAAM,IAAI,YAAY,CAAC,oDAAoD,CAAC,CAAC;IAC/E,CAAC;AACH,CAAC;AAED;;GAEG;AACH,SAAS,iBAAiB,CAAC,SAAiB;IAC1C,IAAI,SAAS,CAAC,MAAM,GAAG,GAAG,EAAE,CAAC;QAC3B,MAAM,IAAI,YAAY,CAAC,0CAA0C,CAAC,CAAC;IACrE,CAAC;IACD,wCAAwC;IACxC,IAAI,CAAC,gBAAgB,CAAC,IAAI,CAAC,SAAS,CAAC,EAAE,CAAC;QACtC,MAAM,IAAI,YAAY,CAAC,wCAAwC,CAAC,CAAC;IACnE,CAAC;IACD,OAAO,SAAS,CAAC;AACnB,CAAC;AAUD;;;;GAIG;AACH,MAAM,CAAC,KAAK,UAAU,WAAW,CAC/B,GAAW,EACX,SAAkB,EAClB,YAAoB,KAAK,EACzB,aAAsC;IAEtC,yCAAyC;IACzC,WAAW,CAAC,GAAG,CAAC,CAAC;IAEjB,kCAAkC;IAClC,MAAM,kBAAkB,GAAG,SAAS,CAAC,CAAC,CAAC,iBAAiB,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,kBAAkB,EAAE,CAAC;IAE3F,2EAA2E;IAC3E,MAAM,cAAc,GAA2B;QAC7C,YAAY,EAAE,kBAAkB;QAChC,QAAQ,EAAE,4EAA4E;QACtF,iBAAiB,EAAE,gBAAgB;QACnC,iBAAiB,EAAE,mBAAmB;QACtC,KAAK,EAAE,GAAG;QACV,YAAY,EAAE,YAAY;QAC1B,2BAA2B,EAAE,GAAG;KACjC,CAAC;IAEF,MAAM,aAAa,GAAG,EAAE,GAAG,cAAc,EAAE,CAAC;IAE5C,IAAI,aAAa,EAAE,CAAC;QAClB,KAAK,MAAM,CAAC,GAAG,EAAE,KAAK,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,aAAa,CAAC,EAAE,CAAC;YACzD,uCAAuC;YACvC,IAAI,GAAG,CAAC,WAAW,EAAE,KAAK,MAAM,EAAE,CAAC;gBACjC,MAAM,IAAI,YAAY,CAAC,mCAAmC,CAAC,CAAC;YAC9D,CAAC;YACD,aAAa,CAAC,GAAG,CAAC,GAAG,KAAK,CAAC;QAC7B,CAAC;IACH,CAAC;IAED,MAAM,aAAa,GAAG,EAAE,CAAC;IACzB,IAAI,aAAa,GAAG,CAAC,CAAC;IACtB,IAAI,UAAU,GAAG,GAAG,CAAC;IACrB,MAAM,QAAQ,GAAG,IAAI,GAAG,EAAU,CAAC;IAEnC,OAAO,aAAa,IAAI,aAAa,EAAE,CAAC;QACtC,wBAAwB;QACxB,IAAI,QAAQ,CAAC,GAAG,CAAC,UAAU,CAAC,EAAE,CAAC;YAC7B,MAAM,IAAI,YAAY,CAAC,wBAAwB,CAAC,CAAC;QACnD,CAAC;QACD,QAAQ,CAAC,GAAG,CAAC,UAAU,CAAC,CAAC;QAEzB,+BAA+B;QAC/B,WAAW,CAAC,UAAU,CAAC,CAAC;QAExB,MAAM,UAAU,GAAG,IAAI,eAAe,EAAE,CAAC;QACzC,MAAM,KAAK,GAAG,UAAU,CAAC,GAAG,EAAE,CAAC,UAAU,CAAC,KAAK,EAAE,EAAE,SAAS,CAAC,CAAC;QAE9D,IAAI,CAAC;YACH,MAAM,QAAQ,GAAG,MAAM,KAAK,CAAC,UAAU,EAAE;gBACvC,OAAO,EAAE,aAAa;gBACtB,MAAM,EAAE,UAAU,CAAC,MAAM;gBACzB,QAAQ,EAAE,QAAQ,EAAE,qCAAqC;aAC1D,CAAC,CAAC;YAEH,YAAY,CAAC,KAAK,CAAC,CAAC;YAEpB,4BAA4B;YAC5B,IAAI,QAAQ,CAAC,MAAM,IAAI,GAAG,IAAI,QAAQ,CAAC,MAAM,GAAG,GAAG,EAAE,CAAC;gBACpD,MAAM,QAAQ,GAAG,QAAQ,CAAC,OAAO,CAAC,GAAG,CAAC,UAAU,CAAC,CAAC;gBAClD,IAAI,CAAC,QAAQ,EAAE,CAAC;oBACd,MAAM,IAAI,YAAY,CAAC,2CAA2C,CAAC,CAAC;gBACtE,CAAC;gBAED,wBAAwB;gBACxB,UAAU,GAAG,IAAI,GAAG,CAAC,QAAQ,EAAE,UAAU,CAAC,CAAC,IAAI,CAAC;gBAChD,aAAa,EAAE,CAAC;gBAChB,SAAS;YACX,CAAC;YAED,IAAI,CAAC,QAAQ,CAAC,EAAE,EAAE,CAAC;gBACjB,IAAI,QAAQ,CAAC,MAAM,KAAK,GAAG,IAAI,QAAQ,CAAC,MAAM,KAAK,GAAG,EAAE,CAAC;oBACvD,MAAM,IAAI,YAAY,CACpB,QAAQ,QAAQ,CAAC,MAAM,iEAAiE,CACzF,CAAC;gBACJ,CAAC;gBACD,MAAM,IAAI,YAAY,CAAC,QAAQ,QAAQ,CAAC,MAAM,KAAK,QAAQ,CAAC,UAAU,EAAE,CAAC,CAAC;YAC5E,CAAC;YAED,kCAAkC;YAClC,MAAM,WAAW,GAAG,QAAQ,CAAC,OAAO,CAAC,GAAG,CAAC,cAAc,CAAC,IAAI,EAAE,CAAC;YAC/D,IAAI,CAAC,WAAW,CAAC,QAAQ,CAAC,WAAW,CAAC;gBAClC,CAAC,WAAW,CAAC,QAAQ,CAAC,uBAAuB,CAAC;gBAC9C,CAAC,WAAW,CAAC,QAAQ,CAAC,iBAAiB,CAAC,EAAE,CAAC;gBAC7C,MAAM,IAAI,YAAY,CAAC,6BAA6B,WAAW,wBAAwB,CAAC,CAAC;YAC3F,CAAC;YAED,wEAAwE;YACxE,MAAM,MAAM,GAAiB,EAAE,CAAC;YAChC,IAAI,SAAS,GAAG,CAAC,CAAC;YAClB,MAAM,QAAQ,GAAG,EAAE,GAAG,IAAI,GAAG,IAAI,CAAC,CAAC,OAAO;YAE1C,MAAM,MAAM,GAAG,QAAQ,CAAC,IAAI,EAAE,SAAS,EAAE,CAAC;YAC1C,IAAI,CAAC,MAAM,EAAE,CAAC;gBACZ,MAAM,IAAI,YAAY,CAAC,+BAA+B,CAAC,CAAC;YAC1D,CAAC;YAED,IAAI,CAAC;gBACH,OAAO,IAAI,EAAE,CAAC;oBACZ,MAAM,EAAE,IAAI,EAAE,KAAK,EAAE,GAAG,MAAM,MAAM,CAAC,IAAI,EAAE,CAAC;oBAC5C,IAAI,IAAI;wBAAE,MAAM;oBAEhB,SAAS,IAAI,KAAK,CAAC,MAAM,CAAC;oBAC1B,IAAI,SAAS,GAAG,QAAQ,EAAE,CAAC;wBACzB,MAAM,CAAC,MAAM,EAAE,CAAC;wBAChB,MAAM,IAAI,YAAY,CAAC,+BAA+B,CAAC,CAAC;oBAC1D,CAAC;oBAED,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;gBACrB,CAAC;YACH,CAAC;oBAAS,CAAC;gBACT,MAAM,CAAC,WAAW,EAAE,CAAC;YACvB,CAAC;YAED,iBAAiB;YACjB,MAAM,QAAQ,GAAG,IAAI,UAAU,CAAC,SAAS,CAAC,CAAC;YAC3C,IAAI,MAAM,GAAG,CAAC,CAAC;YACf,KAAK,MAAM,KAAK,IAAI,MAAM,EAAE,CAAC;gBAC3B,QAAQ,CAAC,GAAG,CAAC,KAAK,EAAE,MAAM,CAAC,CAAC;gBAC5B,MAAM,IAAI,KAAK,CAAC,MAAM,CAAC;YACzB,CAAC;YAED,MAAM,IAAI,GAAG,IAAI,WAAW,EAAE,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC;YAEhD,IAAI,CAAC,IAAI,IAAI,IAAI,CAAC,MAAM,GAAG,GAAG,EAAE,CAAC;gBAC/B,MAAM,IAAI,YAAY,CAAC,oEAAoE,CAAC,CAAC;YAC/F,CAAC;YAED,iCAAiC;YACjC,IAAI,IAAI,CAAC,QAAQ,CAAC,yBAAyB,CAAC,IAAI,IAAI,CAAC,QAAQ,CAAC,kBAAkB,CAAC,EAAE,CAAC;gBAClF,MAAM,IAAI,YAAY,CAAC,+DAA+D,CAAC,CAAC;YAC1F,CAAC;YAED,OAAO;gBACL,IAAI;gBACJ,GAAG,EAAE,UAAU;gBACf,UAAU,EAAE,QAAQ,CAAC,MAAM;gBAC3B,WAAW;aACZ,CAAC;QACJ,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,YAAY,CAAC,KAAK,CAAC,CAAC;YAEpB,IAAI,KAAK,YAAY,YAAY,IAAI,KAAK,YAAY,YAAY,IAAI,KAAK,YAAY,YAAY,EAAE,CAAC;gBACpG,MAAM,KAAK,CAAC;YACd,CAAC;YAED,IAAI,KAAK,YAAY,KAAK,IAAI,KAAK,CAAC,IAAI,KAAK,YAAY,EAAE,CAAC;gBAC1D,MAAM,IAAI,YAAY,CAAC,2BAA2B,SAAS,IAAI,CAAC,CAAC;YACnE,CAAC;YAED,MAAM,IAAI,YAAY,CAAC,oBAAoB,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,eAAe,EAAE,CAAC,CAAC;QACzG,CAAC;IACH,CAAC;IAED,MAAM,IAAI,YAAY,CAAC,2BAA2B,aAAa,GAAG,CAAC,CAAC;AACtE,CAAC;AAED,IAAI,aAAa,GAAmB,IAAI,CAAC;AACzC,IAAI,oBAAoB,GAAmB,IAAI,CAAC;AAChD,IAAI,gBAAgB,GAAG,CAAC,CAAC;AACzB,MAAM,oBAAoB,GAAG,CAAC,CAAC;AAE/B,KAAK,UAAU,UAAU;IACvB,4DAA4D;IAC5D,IAAI,aAAa,EAAE,CAAC;QAClB,IAAI,CAAC;YACH,IAAI,aAAa,CAAC,WAAW,EAAE,EAAE,CAAC;gBAChC,OAAO,aAAa,CAAC;YACvB,CAAC;QACH,CAAC;QAAC,MAAM,CAAC;YACP,4BAA4B;YAC5B,aAAa,GAAG,IAAI,CAAC;QACvB,CAAC;IACH,CAAC;IAED,aAAa,GAAG,MAAM,QAAQ,CAAC,MAAM,CAAC,EAAE,QAAQ,EAAE,IAAI,EAAE,CAAC,CAAC;IAC1D,OAAO,aAAa,CAAC;AACvB,CAAC;AAED,KAAK,UAAU,iBAAiB;IAC9B,oEAAoE;IACpE,IAAI,oBAAoB,EAAE,CAAC;QACzB,IAAI,CAAC;YACH,IAAI,oBAAoB,CAAC,WAAW,EAAE,EAAE,CAAC;gBACvC,OAAO,oBAAoB,CAAC;YAC9B,CAAC;QACH,CAAC;QAAC,MAAM,CAAC;YACP,4BAA4B;YAC5B,oBAAoB,GAAG,IAAI,CAAC;QAC9B,CAAC;IACH,CAAC;IAED,oBAAoB,GAAG,MAAM,eAAe,CAAC,MAAM,CAAC,EAAE,QAAQ,EAAE,IAAI,EAAE,CAAC,CAAC;IACxE,OAAO,oBAAoB,CAAC;AAC9B,CAAC;AAED;;;GAGG;AACH,MAAM,CAAC,KAAK,UAAU,YAAY,CAChC,GAAW,EACX,UASI,EAAE;IAEN,yCAAyC;IACzC,WAAW,CAAC,GAAG,CAAC,CAAC;IAEjB,MAAM,EACJ,SAAS,EACT,MAAM,GAAG,CAAC,EACV,SAAS,GAAG,KAAK,EACjB,UAAU,GAAG,KAAK,EAClB,kBAAkB,GAAG,KAAK,EAC1B,OAAO,EACP,OAAO,EACP,OAAO,GAAG,KAAK,EAChB,GAAG,OAAO,CAAC;IAEZ,kCAAkC;IAClC,MAAM,kBAAkB,GAAG,SAAS,CAAC,CAAC,CAAC,iBAAiB,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,kBAAkB,EAAE,CAAC;IAE3F,qBAAqB;IACrB,IAAI,MAAM,GAAG,CAAC,IAAI,MAAM,GAAG,KAAK,EAAE,CAAC;QACjC,MAAM,IAAI,YAAY,CAAC,yCAAyC,CAAC,CAAC;IACpE,CAAC;IAED,gDAAgD;IAChD,IAAI,OAAO,EAAE,CAAC;QACZ,KAAK,MAAM,CAAC,GAAG,EAAE,KAAK,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,OAAO,CAAC,EAAE,CAAC;YACnD,6BAA6B;YAC7B,IAAI,GAAG,CAAC,WAAW,EAAE,KAAK,MAAM,EAAE,CAAC;gBACjC,MAAM,IAAI,YAAY,CAAC,mCAAmC,CAAC,CAAC;YAC9D,CAAC;YACD,IAAI,OAAO,KAAK,KAAK,QAAQ,IAAI,KAAK,CAAC,MAAM,GAAG,GAAG,EAAE,CAAC;gBACpD,MAAM,IAAI,YAAY,CAAC,sBAAsB,CAAC,CAAC;YACjD,CAAC;QACH,CAAC;IACH,CAAC;IAED,wDAAwD;IACxD,MAAM,cAAc,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;IAClC,MAAM,gBAAgB,GAAG,KAAK,CAAC,CAAC,qBAAqB;IAErD,OAAO,gBAAgB,IAAI,oBAAoB,EAAE,CAAC;QAChD,IAAI,IAAI,CAAC,GAAG,EAAE,GAAG,cAAc,GAAG,gBAAgB,EAAE,CAAC;YACnD,MAAM,IAAI,YAAY,CAAC,2DAA2D,CAAC,CAAC;QACtF,CAAC;QACD,MAAM,IAAI,OAAO,CAAC,OAAO,CAAC,EAAE,CAAC,UAAU,CAAC,OAAO,EAAE,GAAG,CAAC,CAAC,CAAC;IACzD,CAAC;IAED,gBAAgB,EAAE,CAAC;IACnB,IAAI,IAAI,GAAgB,IAAI,CAAC;IAE7B,IAAI,CAAC;QACH,MAAM,OAAO,GAAG,OAAO,CAAC,CAAC,CAAC,MAAM,iBAAiB,EAAE,CAAC,CAAC,CAAC,MAAM,UAAU,EAAE,CAAC;QACzE,IAAI,GAAG,MAAM,OAAO,CAAC,OAAO,CAAC;YAC3B,SAAS,EAAE,kBAAkB;SAC9B,CAAC,CAAC;QAEH,iCAAiC;QACjC,IAAI,OAAO,IAAI,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YAC/C,MAAM,IAAI,CAAC,mBAAmB,CAAC,OAAO,CAAC,CAAC;QAC1C,CAAC;QAED,0BAA0B;QAC1B,IAAI,OAAO,IAAI,OAAO,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YAClC,MAAM,aAAa,GAAG,OAAO,CAAC,GAAG,CAAC,MAAM,CAAC,EAAE;gBACzC,MAAM,CAAC,SAAS,CAAC,GAAG,MAAM,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,CAAC;gBACzD,MAAM,CAAC,IAAI,EAAE,KAAK,CAAC,GAAG,SAAS,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;gBAE3C,IAAI,CAAC,IAAI,IAAI,KAAK,KAAK,SAAS,EAAE,CAAC;oBACjC,MAAM,IAAI,YAAY,CAAC,0BAA0B,MAAM,EAAE,CAAC,CAAC;gBAC7D,CAAC;gBAED,OAAO;oBACL,IAAI,EAAE,IAAI,CAAC,IAAI,EAAE;oBACjB,KAAK,EAAE,KAAK,CAAC,IAAI,EAAE;oBACnB,GAAG;iBACJ,CAAC;YACJ,CAAC,CAAC,CAAC;YAEH,MAAM,IAAI,CAAC,OAAO,EAAE,CAAC,UAAU,CAAC,aAAa,CAAC,CAAC;QACjD,CAAC;QAED,4FAA4F;QAC5F,IAAI,CAAC,UAAU,EAAE,CAAC;YAChB,MAAM,IAAI,CAAC,KAAK,CAAC,MAAM,EAAE,CAAC,KAAK,EAAE,EAAE;gBACjC,MAAM,YAAY,GAAG,KAAK,CAAC,OAAO,EAAE,CAAC,YAAY,EAAE,CAAC;gBACpD,IAAI,CAAC,OAAO,EAAE,MAAM,EAAE,OAAO,EAAE,YAAY,CAAC,CAAC,QAAQ,CAAC,YAAY,CAAC,EAAE,CAAC;oBACpE,KAAK,CAAC,KAAK,EAAE,CAAC;gBAChB,CAAC;qBAAM,CAAC;oBACN,KAAK,CAAC,QAAQ,EAAE,CAAC;gBACnB,CAAC;YACH,CAAC,CAAC,CAAC;QACL,CAAC;aAAM,CAAC;YACN,uCAAuC;YACvC,MAAM,IAAI,CAAC,KAAK,CAAC,MAAM,EAAE,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,QAAQ,EAAE,CAAC,CAAC;QACxD,CAAC;QAED,6CAA6C;QAC7C,MAAM,YAAY,GAAG,CAAC,KAAK,IAAI,EAAE;YAC/B,MAAM,IAAK,CAAC,IAAI,CAAC,GAAG,EAAE;gBACpB,SAAS,EAAE,kBAAkB;gBAC7B,OAAO,EAAE,SAAS;aACnB,CAAC,CAAC;YAEH,8DAA8D;YAC9D,IAAI,MAAM,GAAG,CAAC,EAAE,CAAC;gBACf,MAAM,IAAK,CAAC,cAAc,CAAC,MAAM,CAAC,CAAC;YACrC,CAAC;YAED,MAAM,IAAI,GAAG,MAAM,IAAK,CAAC,OAAO,EAAE,CAAC;YACnC,MAAM,QAAQ,GAAG,IAAK,CAAC,GAAG,EAAE,CAAC;YAE7B,OAAO,EAAE,IAAI,EAAE,QAAQ,EAAE,CAAC;QAC5B,CAAC,CAAC,EAAE,CAAC;QAEL,MAAM,cAAc,GAAG,IAAI,OAAO,CAAQ,CAAC,CAAC,EAAE,MAAM,EAAE,EAAE;YACtD,UAAU,CAAC,GAAG,EAAE,CAAC,MAAM,CAAC,IAAI,YAAY,CAAC,6BAA6B,SAAS,IAAI,CAAC,CAAC,EAAE,SAAS,CAAC,CAAC;QACpG,CAAC,CAAC,CAAC;QAEH,MAAM,EAAE,IAAI,EAAE,QAAQ,EAAE,GAAG,MAAM,OAAO,CAAC,IAAI,CAAC,CAAC,YAAY,EAAE,cAAc,CAAC,CAAC,CAAC;QAE9E,4BAA4B;QAC5B,IAAI,IAAI,CAAC,MAAM,GAAG,EAAE,GAAG,IAAI,GAAG,IAAI,EAAE,CAAC,CAAC,aAAa;YACjD,MAAM,IAAI,YAAY,CAAC,+BAA+B,CAAC,CAAC;QAC1D,CAAC;QAED,IAAI,CAAC,IAAI,IAAI,IAAI,CAAC,MAAM,GAAG,GAAG,EAAE,CAAC;YAC/B,MAAM,IAAI,YAAY,CAAC,oDAAoD,CAAC,CAAC;QAC/E,CAAC;QAED,kCAAkC;QAClC,IAAI,gBAAoC,CAAC;QACzC,IAAI,UAAU,EAAE,CAAC;YACf,gBAAgB,GAAG,MAAM,IAAK,CAAC,UAAU,CAAC;gBACxC,QAAQ,EAAE,kBAAkB;gBAC5B,IAAI,EAAE,KAAK;aACZ,CAAC,CAAC;QACL,CAAC;QAED,OAAO;YACL,IAAI;YACJ,GAAG,EAAE,QAAQ;YACb,UAAU,EAAE,gBAAgB;SAC7B,CAAC;IACJ,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,IAAI,KAAK,YAAY,YAAY,IAAI,KAAK,YAAY,YAAY,IAAI,KAAK,YAAY,YAAY,EAAE,CAAC;YACpG,MAAM,KAAK,CAAC;QACd,CAAC;QAED,IAAI,KAAK,YAAY,KAAK,IAAI,KAAK,CAAC,OAAO,CAAC,QAAQ,CAAC,SAAS,CAAC,EAAE,CAAC;YAChE,MAAM,IAAI,YAAY,CAAC,8BAA8B,CAAC,CAAC;QACzD,CAAC;QAED,MAAM,IAAI,YAAY,CACpB,yBAAyB,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,eAAe,EAAE,CACpF,CAAC;IACJ,CAAC;YAAS,CAAC;QACT,oDAAoD;QACpD,IAAI,IAAI,EAAE,CAAC;YACT,MAAM,IAAI,CAAC,KAAK,EAAE,CAAC,KAAK,CAAC,GAAG,EAAE,GAAE,CAAC,CAAC,CAAC;QACrC,CAAC;QACD,gBAAgB,EAAE,CAAC;IACrB,CAAC;AACH,CAAC;AAED;;GAEG;AACH,MAAM,CAAC,KAAK,UAAU,UAAU,CAC9B,EAAoB,EACpB,cAAsB,CAAC,EACvB,cAAsB,IAAI;IAE1B,IAAI,SAAS,GAAiB,IAAI,CAAC;IAEnC,KAAK,IAAI,OAAO,GAAG,CAAC,EAAE,OAAO,IAAI,WAAW,EAAE,OAAO,EAAE,EAAE,CAAC;QACxD,IAAI,CAAC;YACH,OAAO,MAAM,EAAE,EAAE,CAAC;QACpB,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,SAAS,GAAG,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,IAAI,KAAK,CAAC,eAAe,CAAC,CAAC;YAExE,4CAA4C;YAC5C,IAAI,KAAK,YAAY,YAAY,IAAI,KAAK,YAAY,YAAY,EAAE,CAAC;gBACnE,MAAM,KAAK,CAAC;YACd,CAAC;YAED,IAAI,OAAO,GAAG,WAAW,EAAE,CAAC;gBAC1B,MAAM,KAAK,GAAG,WAAW,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,OAAO,GAAG,CAAC,CAAC,CAAC;gBACrD,MAAM,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,UAAU,CAAC,OAAO,EAAE,KAAK,CAAC,CAAC,CAAC;YAC7D,CAAC;QACH,CAAC;IACH,CAAC;IAED,MAAM,SAAS,IAAI,IAAI,YAAY,CAAC,cAAc,CAAC,CAAC;AACtD,CAAC;AAED;;GAEG;AACH,MAAM,CAAC,KAAK,UAAU,OAAO;IAC3B,IAAI,aAAa,EAAE,CAAC;QAClB,MAAM,aAAa,CAAC,KAAK,EAAE,CAAC;QAC5B,aAAa,GAAG,IAAI,CAAC;IACvB,CAAC;IACD,IAAI,oBAAoB,EAAE,CAAC;QACzB,MAAM,oBAAoB,CAAC,KAAK,EAAE,CAAC;QACnC,oBAAoB,GAAG,IAAI,CAAC;IAC9B,CAAC;AACH,CAAC"}
@@ -1,6 +1,11 @@
1
1
  /**
2
2
  * HTML to Markdown conversion with smart cleanup
3
3
  */
4
+ /**
5
+ * Extract content matching a CSS selector
6
+ * Returns filtered HTML or full HTML if selector matches nothing
7
+ */
8
+ export declare function selectContent(html: string, selector: string, exclude?: string[]): string;
4
9
  /**
5
10
  * Convert HTML to clean, readable Markdown
6
11
  */
@@ -1 +1 @@
1
- {"version":3,"file":"markdown.d.ts","sourceRoot":"","sources":["../../src/core/markdown.ts"],"names":[],"mappings":"AAAA;;GAEG;AA8DH;;GAEG;AACH,wBAAgB,cAAc,CAAC,IAAI,EAAE,MAAM,GAAG,MAAM,CA4DnD;AAED;;GAEG;AACH,wBAAgB,UAAU,CAAC,IAAI,EAAE,MAAM,GAAG,MAAM,CAuB/C;AAED;;;GAGG;AACH,wBAAgB,cAAc,CAAC,IAAI,EAAE,MAAM,GAAG,MAAM,CAEnD"}
1
+ {"version":3,"file":"markdown.d.ts","sourceRoot":"","sources":["../../src/core/markdown.ts"],"names":[],"mappings":"AAAA;;GAEG;AAuCH;;;GAGG;AACH,wBAAgB,aAAa,CAAC,IAAI,EAAE,MAAM,EAAE,QAAQ,EAAE,MAAM,EAAE,OAAO,CAAC,EAAE,MAAM,EAAE,GAAG,MAAM,CAiBxF;AAkCD;;GAEG;AACH,wBAAgB,cAAc,CAAC,IAAI,EAAE,MAAM,GAAG,MAAM,CA4DnD;AAED;;GAEG;AACH,wBAAgB,UAAU,CAAC,IAAI,EAAE,MAAM,GAAG,MAAM,CAuB/C;AAED;;;GAGG;AACH,wBAAgB,cAAc,CAAC,IAAI,EAAE,MAAM,GAAG,MAAM,CAEnD"}
@@ -4,29 +4,57 @@
4
4
  import TurndownService from 'turndown';
5
5
  import * as cheerio from 'cheerio';
6
6
  const JUNK_SELECTORS = [
7
- 'script',
8
- 'style',
9
- 'nav',
10
- 'footer',
11
- 'header.site-header',
12
- 'aside',
13
- '.sidebar',
14
- '.advertisement',
15
- '.ad',
16
- '.cookie-banner',
17
- '.cookie-notice',
18
- '.newsletter-signup',
19
- '.social-share',
20
- '.related-posts',
21
- '.comments',
22
- '#comments',
23
- '.cookie-consent',
24
- '[class*="cookie"]',
25
- '[id*="cookie"]',
26
- '[class*="banner"]',
27
- '[class*="popup"]',
28
- '[class*="modal"]',
7
+ // Scripts, styles, metadata
8
+ 'script', 'style', 'noscript', 'iframe', 'link[rel="stylesheet"]',
9
+ // Navigation & structure
10
+ 'nav', 'header', 'footer', 'aside',
11
+ '[role="navigation"]', '[role="banner"]', '[role="contentinfo"]',
12
+ '[role="complementary"]', '[role="search"]',
13
+ '.sidebar', '.topbar', '.top-bar', '.site-nav', '.main-nav',
14
+ '.breadcrumb', '.breadcrumbs', '[class*="breadcrumb"]',
15
+ '.pagination', '[class*="pagination"]',
16
+ // Ads & tracking
17
+ '.advertisement', '.ad', '[class*="ad-"]', '[id*="ad-"]',
18
+ '[class*="advert"]', '[class*="sponsor"]', '[class*="promo"]',
19
+ // Cookie & consent
20
+ '.cookie-banner', '.cookie-notice', '.cookie-consent',
21
+ '[class*="cookie"]', '[id*="cookie"]',
22
+ '[class*="consent"]', '[class*="gdpr"]',
23
+ // Popups, modals, banners
24
+ '[class*="banner"]', '[class*="popup"]', '[class*="modal"]',
25
+ '[class*="overlay"]', '[class*="notification-bar"]',
26
+ // Social & sharing
27
+ '.social-share', '[class*="share"]', '[class*="social"]',
28
+ // Newsletter & CTA
29
+ '.newsletter-signup', '[class*="newsletter"]', '[class*="subscribe"]',
30
+ '[class*="cta"]', '[class*="call-to-action"]', '[class*="signup"]',
31
+ // Related content
32
+ '.related-posts', '[class*="related"]', '[class*="recommended"]',
33
+ '[class*="you-may-also"]', '[class*="more-stories"]',
34
+ // Comments
35
+ '.comments', '#comments', '[class*="comment"]',
36
+ // SVG decorations (icons, decorative elements)
37
+ 'svg:not(img svg)',
29
38
  ];
39
+ /**
40
+ * Extract content matching a CSS selector
41
+ * Returns filtered HTML or full HTML if selector matches nothing
42
+ */
43
+ export function selectContent(html, selector, exclude) {
44
+ const $ = cheerio.load(html);
45
+ // Apply excludes first
46
+ if (exclude?.length) {
47
+ exclude.forEach(sel => $(sel).remove());
48
+ }
49
+ // Select matching elements
50
+ const selected = $(selector);
51
+ if (selected.length === 0) {
52
+ // Fallback to full page if selector matches nothing
53
+ return html;
54
+ }
55
+ // Return the HTML of all matched elements
56
+ return selected.map((_, el) => $.html(el)).get().join('\n');
57
+ }
30
58
  /**
31
59
  * Clean HTML before conversion
32
60
  * Remove navigation, ads, cookie banners, and other junk
@@ -1 +1 @@
1
- {"version":3,"file":"markdown.js","sourceRoot":"","sources":["../../src/core/markdown.ts"],"names":[],"mappings":"AAAA;;GAEG;AAEH,OAAO,eAAe,MAAM,UAAU,CAAC;AACvC,OAAO,KAAK,OAAO,MAAM,SAAS,CAAC;AAEnC,MAAM,cAAc,GAAG;IACrB,QAAQ;IACR,OAAO;IACP,KAAK;IACL,QAAQ;IACR,oBAAoB;IACpB,OAAO;IACP,UAAU;IACV,gBAAgB;IAChB,KAAK;IACL,gBAAgB;IAChB,gBAAgB;IAChB,oBAAoB;IACpB,eAAe;IACf,gBAAgB;IAChB,WAAW;IACX,WAAW;IACX,iBAAiB;IACjB,mBAAmB;IACnB,gBAAgB;IAChB,mBAAmB;IACnB,kBAAkB;IAClB,kBAAkB;CACnB,CAAC;AAEF;;;GAGG;AACH,SAAS,SAAS,CAAC,IAAY;IAC7B,2CAA2C;IAC3C,IAAI,IAAI,CAAC,MAAM,GAAG,EAAE,GAAG,IAAI,GAAG,IAAI,EAAE,CAAC,CAAC,OAAO;QAC3C,MAAM,IAAI,KAAK,CAAC,sCAAsC,CAAC,CAAC;IAC1D,CAAC;IAED,MAAM,CAAC,GAAG,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IAE7B,uBAAuB;IACvB,cAAc,CAAC,OAAO,CAAC,CAAC,QAAQ,EAAE,EAAE;QAClC,CAAC,CAAC,QAAQ,CAAC,CAAC,MAAM,EAAE,CAAC;IACvB,CAAC,CAAC,CAAC;IAEH,mCAAmC;IACnC,CAAC,CAAC,oBAAoB,CAAC,CAAC,MAAM,EAAE,CAAC;IAEjC,uCAAuC;IACvC,CAAC,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,IAAI,EAAE,EAAE;QACtB,MAAM,KAAK,GAAG,CAAC,CAAC,IAAI,CAAC,CAAC;QACtB,MAAM,IAAI,GAAG,KAAK,CAAC,IAAI,EAAE,CAAC,IAAI,EAAE,CAAC;QACjC,IAAI,CAAC,IAAI,IAAI,KAAK,CAAC,QAAQ,EAAE,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YAC3C,KAAK,CAAC,MAAM,EAAE,CAAC;QACjB,CAAC;IACH,CAAC,CAAC,CAAC;IAEH,OAAO,CAAC,CAAC,IAAI,EAAE,CAAC;AAClB,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,cAAc,CAAC,IAAY;IACzC,MAAM,WAAW,GAAG,SAAS,CAAC,IAAI,CAAC,CAAC;IAEpC,MAAM,QAAQ,GAAG,IAAI,eAAe,CAAC;QACnC,YAAY,EAAE,KAAK;QACnB,cAAc,EAAE,QAAQ;QACxB,gBAAgB,EAAE,GAAG;QACrB,WAAW,EAAE,GAAG;QAChB,eAAe,EAAE,IAAI;KACtB,CAAC,CAAC;IAEH,kBAAkB;IAClB,QAAQ,CAAC,IAAI,CAAC,CAAC,OAAO,EAAE,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE,IAAI,EAAE,IAAI,CAAC,CAAC,CAAC;IAE7D,kDAAkD;IAClD,QAAQ,CAAC,OAAO,CAAC,QAAQ,EAAE;QACzB,MAAM,EAAE,KAAK;QACb,WAAW,EAAE,CAAC,QAAQ,EAAE,IAAI,EAAE,EAAE;YAC9B,MAAM,GAAG,GAAI,IAAY,CAAC,GAAG,CAAC;YAC9B,MAAM,GAAG,GAAI,IAAY,CAAC,GAAG,CAAC;YAC9B,IAAI,GAAG,EAAE,CAAC;gBACR,OAAO,KAAK,GAAG,KAAK,GAAG,GAAG,CAAC;YAC7B,CAAC;YACD,OAAO,EAAE,CAAC;QACZ,CAAC;KACF,CAAC,CAAC;IAEH,oCAAoC;IACpC,QAAQ,CAAC,OAAO,CAAC,YAAY,EAAE;QAC7B,MAAM,EAAE,CAAC,IAAI,EAAE,EAAE;YACf,OAAO,IAAI,CAAC,QAAQ,KAAK,KAAK,IAAI,IAAI,CAAC,UAAU,EAAE,QAAQ,KAAK,MAAM,CAAC;QACzE,CAAC;QACD,WAAW,EAAE,CAAC,QAAQ,EAAE,IAAI,EAAE,EAAE;YAC9B,MAAM,QAAQ,GAAG,IAAI,CAAC,UAAiB,CAAC;YACxC,MAAM,SAAS,GAAG,QAAQ,CAAC,YAAY,CAAC,OAAO,CAAC,IAAI,EAAE,CAAC;YACvD,MAAM,QAAQ,GAAG,SAAS,CAAC,KAAK,CAAC,gBAAgB,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;YAC9D,OAAO,SAAS,GAAG,QAAQ,GAAG,IAAI,GAAG,QAAQ,CAAC,WAAW,GAAG,WAAW,CAAC;QAC1E,CAAC;KACF,CAAC,CAAC;IAEH,IAAI,QAAQ,GAAG,QAAQ,CAAC,QAAQ,CAAC,WAAW,CAAC,CAAC;IAE9C,kEAAkE;IAClE,IAAI,QAAQ,CAAC,MAAM,GAAG,IAAI,GAAG,IAAI,EAAE,CAAC,CAAC,yBAAyB;QAC5D,QAAQ,GAAG,QAAQ,CAAC,KAAK,CAAC,CAAC,EAAE,IAAI,GAAG,IAAI,CAAC,CAAC;IAC5C,CAAC;IAED,8DAA8D;IAC9D,QAAQ,GAAG,QAAQ,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,IAAI,EAAE,CAAC,EAAE,GAAG,EAAE,EAAE;QAC3D,IAAI,CAAC,KAAK,CAAC;YAAE,OAAO,IAAI,CAAC;QACzB,MAAM,SAAS,GAAG,GAAG,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,EAAE,KAAK,EAAE,CAAC;QAC3C,MAAM,SAAS,GAAG,IAAI,CAAC,IAAI,EAAE,KAAK,EAAE,CAAC;QACrC,IAAI,SAAS,IAAI,SAAS;YAAE,OAAO,GAAG,CAAC;QACvC,OAAO,GAAG,GAAG,IAAI,GAAG,IAAI,CAAC;IAC3B,CAAC,EAAE,EAAE,CAAC,CAAC;IAEP,qCAAqC;IACrC,QAAQ,GAAG,QAAQ,CAAC,IAAI,EAAE,CAAC;IAE3B,OAAO,QAAQ,CAAC;AAClB,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,UAAU,CAAC,IAAY;IACrC,MAAM,WAAW,GAAG,SAAS,CAAC,IAAI,CAAC,CAAC;IACpC,MAAM,CAAC,GAAG,OAAO,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC;IAEpC,8CAA8C;IAC9C,IAAI,IAAI,GAAG,EAAE,CAAC;IACd,CAAC,CAAC,+BAA+B,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,IAAI,EAAE,EAAE;QAClD,MAAM,OAAO,GAAG,CAAC,CAAC,IAAI,CAAC,CAAC,IAAI,EAAE,CAAC,IAAI,EAAE,CAAC;QACtC,IAAI,OAAO,EAAE,CAAC;YACZ,IAAI,IAAI,OAAO,GAAG,MAAM,CAAC;QAC3B,CAAC;IACH,CAAC,CAAC,CAAC;IAEH,yDAAyD;IACzD,IAAI,CAAC,IAAI,CAAC,IAAI,EAAE,EAAE,CAAC;QACjB,IAAI,GAAG,CAAC,CAAC,MAAM,CAAC,CAAC,IAAI,EAAE,CAAC;IAC1B,CAAC;IAED,gCAAgC;IAChC,IAAI,GAAG,IAAI,CAAC,OAAO,CAAC,SAAS,EAAE,MAAM,CAAC,CAAC;IACvC,IAAI,GAAG,IAAI,CAAC,OAAO,CAAC,SAAS,EAAE,GAAG,CAAC,CAAC;IAEpC,OAAO,IAAI,CAAC,IAAI,EAAE,CAAC;AACrB,CAAC;AAED;;;GAGG;AACH,MAAM,UAAU,cAAc,CAAC,IAAY;IACzC,OAAO,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;AACpC,CAAC"}
1
+ {"version":3,"file":"markdown.js","sourceRoot":"","sources":["../../src/core/markdown.ts"],"names":[],"mappings":"AAAA;;GAEG;AAEH,OAAO,eAAe,MAAM,UAAU,CAAC;AACvC,OAAO,KAAK,OAAO,MAAM,SAAS,CAAC;AAEnC,MAAM,cAAc,GAAG;IACrB,4BAA4B;IAC5B,QAAQ,EAAE,OAAO,EAAE,UAAU,EAAE,QAAQ,EAAE,wBAAwB;IACjE,yBAAyB;IACzB,KAAK,EAAE,QAAQ,EAAE,QAAQ,EAAE,OAAO;IAClC,qBAAqB,EAAE,iBAAiB,EAAE,sBAAsB;IAChE,wBAAwB,EAAE,iBAAiB;IAC3C,UAAU,EAAE,SAAS,EAAE,UAAU,EAAE,WAAW,EAAE,WAAW;IAC3D,aAAa,EAAE,cAAc,EAAE,uBAAuB;IACtD,aAAa,EAAE,uBAAuB;IACtC,iBAAiB;IACjB,gBAAgB,EAAE,KAAK,EAAE,gBAAgB,EAAE,aAAa;IACxD,mBAAmB,EAAE,oBAAoB,EAAE,kBAAkB;IAC7D,mBAAmB;IACnB,gBAAgB,EAAE,gBAAgB,EAAE,iBAAiB;IACrD,mBAAmB,EAAE,gBAAgB;IACrC,oBAAoB,EAAE,iBAAiB;IACvC,0BAA0B;IAC1B,mBAAmB,EAAE,kBAAkB,EAAE,kBAAkB;IAC3D,oBAAoB,EAAE,6BAA6B;IACnD,mBAAmB;IACnB,eAAe,EAAE,kBAAkB,EAAE,mBAAmB;IACxD,mBAAmB;IACnB,oBAAoB,EAAE,uBAAuB,EAAE,sBAAsB;IACrE,gBAAgB,EAAE,2BAA2B,EAAE,mBAAmB;IAClE,kBAAkB;IAClB,gBAAgB,EAAE,oBAAoB,EAAE,wBAAwB;IAChE,yBAAyB,EAAE,yBAAyB;IACpD,WAAW;IACX,WAAW,EAAE,WAAW,EAAE,oBAAoB;IAC9C,+CAA+C;IAC/C,kBAAkB;CACnB,CAAC;AAEF;;;GAGG;AACH,MAAM,UAAU,aAAa,CAAC,IAAY,EAAE,QAAgB,EAAE,OAAkB;IAC9E,MAAM,CAAC,GAAG,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IAE7B,uBAAuB;IACvB,IAAI,OAAO,EAAE,MAAM,EAAE,CAAC;QACpB,OAAO,CAAC,OAAO,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,MAAM,EAAE,CAAC,CAAC;IAC1C,CAAC;IAED,2BAA2B;IAC3B,MAAM,QAAQ,GAAG,CAAC,CAAC,QAAQ,CAAC,CAAC;IAC7B,IAAI,QAAQ,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QAC1B,oDAAoD;QACpD,OAAO,IAAI,CAAC;IACd,CAAC;IAED,0CAA0C;IAC1C,OAAO,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC,CAAC,GAAG,EAAE,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;AAC9D,CAAC;AAED;;;GAGG;AACH,SAAS,SAAS,CAAC,IAAY;IAC7B,2CAA2C;IAC3C,IAAI,IAAI,CAAC,MAAM,GAAG,EAAE,GAAG,IAAI,GAAG,IAAI,EAAE,CAAC,CAAC,OAAO;QAC3C,MAAM,IAAI,KAAK,CAAC,sCAAsC,CAAC,CAAC;IAC1D,CAAC;IAED,MAAM,CAAC,GAAG,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IAE7B,uBAAuB;IACvB,cAAc,CAAC,OAAO,CAAC,CAAC,QAAQ,EAAE,EAAE;QAClC,CAAC,CAAC,QAAQ,CAAC,CAAC,MAAM,EAAE,CAAC;IACvB,CAAC,CAAC,CAAC;IAEH,mCAAmC;IACnC,CAAC,CAAC,oBAAoB,CAAC,CAAC,MAAM,EAAE,CAAC;IAEjC,uCAAuC;IACvC,CAAC,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,IAAI,EAAE,EAAE;QACtB,MAAM,KAAK,GAAG,CAAC,CAAC,IAAI,CAAC,CAAC;QACtB,MAAM,IAAI,GAAG,KAAK,CAAC,IAAI,EAAE,CAAC,IAAI,EAAE,CAAC;QACjC,IAAI,CAAC,IAAI,IAAI,KAAK,CAAC,QAAQ,EAAE,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YAC3C,KAAK,CAAC,MAAM,EAAE,CAAC;QACjB,CAAC;IACH,CAAC,CAAC,CAAC;IAEH,OAAO,CAAC,CAAC,IAAI,EAAE,CAAC;AAClB,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,cAAc,CAAC,IAAY;IACzC,MAAM,WAAW,GAAG,SAAS,CAAC,IAAI,CAAC,CAAC;IAEpC,MAAM,QAAQ,GAAG,IAAI,eAAe,CAAC;QACnC,YAAY,EAAE,KAAK;QACnB,cAAc,EAAE,QAAQ;QACxB,gBAAgB,EAAE,GAAG;QACrB,WAAW,EAAE,GAAG;QAChB,eAAe,EAAE,IAAI;KACtB,CAAC,CAAC;IAEH,kBAAkB;IAClB,QAAQ,CAAC,IAAI,CAAC,CAAC,OAAO,EAAE,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE,IAAI,EAAE,IAAI,CAAC,CAAC,CAAC;IAE7D,kDAAkD;IAClD,QAAQ,CAAC,OAAO,CAAC,QAAQ,EAAE;QACzB,MAAM,EAAE,KAAK;QACb,WAAW,EAAE,CAAC,QAAQ,EAAE,IAAI,EAAE,EAAE;YAC9B,MAAM,GAAG,GAAI,IAAY,CAAC,GAAG,CAAC;YAC9B,MAAM,GAAG,GAAI,IAAY,CAAC,GAAG,CAAC;YAC9B,IAAI,GAAG,EAAE,CAAC;gBACR,OAAO,KAAK,GAAG,KAAK,GAAG,GAAG,CAAC;YAC7B,CAAC;YACD,OAAO,EAAE,CAAC;QACZ,CAAC;KACF,CAAC,CAAC;IAEH,oCAAoC;IACpC,QAAQ,CAAC,OAAO,CAAC,YAAY,EAAE;QAC7B,MAAM,EAAE,CAAC,IAAI,EAAE,EAAE;YACf,OAAO,IAAI,CAAC,QAAQ,KAAK,KAAK,IAAI,IAAI,CAAC,UAAU,EAAE,QAAQ,KAAK,MAAM,CAAC;QACzE,CAAC;QACD,WAAW,EAAE,CAAC,QAAQ,EAAE,IAAI,EAAE,EAAE;YAC9B,MAAM,QAAQ,GAAG,IAAI,CAAC,UAAiB,CAAC;YACxC,MAAM,SAAS,GAAG,QAAQ,CAAC,YAAY,CAAC,OAAO,CAAC,IAAI,EAAE,CAAC;YACvD,MAAM,QAAQ,GAAG,SAAS,CAAC,KAAK,CAAC,gBAAgB,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;YAC9D,OAAO,SAAS,GAAG,QAAQ,GAAG,IAAI,GAAG,QAAQ,CAAC,WAAW,GAAG,WAAW,CAAC;QAC1E,CAAC;KACF,CAAC,CAAC;IAEH,IAAI,QAAQ,GAAG,QAAQ,CAAC,QAAQ,CAAC,WAAW,CAAC,CAAC;IAE9C,kEAAkE;IAClE,IAAI,QAAQ,CAAC,MAAM,GAAG,IAAI,GAAG,IAAI,EAAE,CAAC,CAAC,yBAAyB;QAC5D,QAAQ,GAAG,QAAQ,CAAC,KAAK,CAAC,CAAC,EAAE,IAAI,GAAG,IAAI,CAAC,CAAC;IAC5C,CAAC;IAED,8DAA8D;IAC9D,QAAQ,GAAG,QAAQ,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,IAAI,EAAE,CAAC,EAAE,GAAG,EAAE,EAAE;QAC3D,IAAI,CAAC,KAAK,CAAC;YAAE,OAAO,IAAI,CAAC;QACzB,MAAM,SAAS,GAAG,GAAG,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,IAAI,EAAE,KAAK,EAAE,CAAC;QAC3C,MAAM,SAAS,GAAG,IAAI,CAAC,IAAI,EAAE,KAAK,EAAE,CAAC;QACrC,IAAI,SAAS,IAAI,SAAS;YAAE,OAAO,GAAG,CAAC;QACvC,OAAO,GAAG,GAAG,IAAI,GAAG,IAAI,CAAC;IAC3B,CAAC,EAAE,EAAE,CAAC,CAAC;IAEP,qCAAqC;IACrC,QAAQ,GAAG,QAAQ,CAAC,IAAI,EAAE,CAAC;IAE3B,OAAO,QAAQ,CAAC;AAClB,CAAC;AAED;;GAEG;AACH,MAAM,UAAU,UAAU,CAAC,IAAY;IACrC,MAAM,WAAW,GAAG,SAAS,CAAC,IAAI,CAAC,CAAC;IACpC,MAAM,CAAC,GAAG,OAAO,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC;IAEpC,8CAA8C;IAC9C,IAAI,IAAI,GAAG,EAAE,CAAC;IACd,CAAC,CAAC,+BAA+B,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,IAAI,EAAE,EAAE;QAClD,MAAM,OAAO,GAAG,CAAC,CAAC,IAAI,CAAC,CAAC,IAAI,EAAE,CAAC,IAAI,EAAE,CAAC;QACtC,IAAI,OAAO,EAAE,CAAC;YACZ,IAAI,IAAI,OAAO,GAAG,MAAM,CAAC;QAC3B,CAAC;IACH,CAAC,CAAC,CAAC;IAEH,yDAAyD;IACzD,IAAI,CAAC,IAAI,CAAC,IAAI,EAAE,EAAE,CAAC;QACjB,IAAI,GAAG,CAAC,CAAC,MAAM,CAAC,CAAC,IAAI,EAAE,CAAC;IAC1B,CAAC;IAED,gCAAgC;IAChC,IAAI,GAAG,IAAI,CAAC,OAAO,CAAC,SAAS,EAAE,MAAM,CAAC,CAAC;IACvC,IAAI,GAAG,IAAI,CAAC,OAAO,CAAC,SAAS,EAAE,GAAG,CAAC,CAAC;IAEpC,OAAO,IAAI,CAAC,IAAI,EAAE,CAAC;AACrB,CAAC;AAED;;;GAGG;AACH,MAAM,UAAU,cAAc,CAAC,IAAY;IACzC,OAAO,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;AACpC,CAAC"}
@@ -5,16 +5,26 @@ import { type FetchResult } from './fetcher.js';
5
5
  export interface StrategyOptions {
6
6
  /** Force browser mode (skip simple fetch) */
7
7
  forceBrowser?: boolean;
8
+ /** Use stealth mode to bypass bot detection */
9
+ stealth?: boolean;
8
10
  /** Wait time after page load in browser mode (ms) */
9
11
  waitMs?: number;
10
12
  /** Custom user agent */
11
13
  userAgent?: string;
12
14
  /** Request timeout (ms) */
13
15
  timeoutMs?: number;
16
+ /** Capture a screenshot of the page */
17
+ screenshot?: boolean;
18
+ /** Full-page screenshot (default: viewport only) */
19
+ screenshotFullPage?: boolean;
20
+ /** Custom HTTP headers to send */
21
+ headers?: Record<string, string>;
22
+ /** Cookies to set (key=value pairs) */
23
+ cookies?: string[];
14
24
  }
15
25
  export interface StrategyResult extends FetchResult {
16
- /** Which strategy succeeded: 'simple' | 'browser' */
17
- method: 'simple' | 'browser';
26
+ /** Which strategy succeeded: 'simple' | 'browser' | 'stealth' */
27
+ method: 'simple' | 'browser' | 'stealth';
18
28
  }
19
29
  /**
20
30
  * Smart fetch with automatic escalation
@@ -22,7 +32,8 @@ export interface StrategyResult extends FetchResult {
22
32
  * Strategy:
23
33
  * 1. Try simple HTTP fetch first (fast, ~200ms)
24
34
  * 2. If blocked (403, 503, Cloudflare, empty body) → try browser
25
- * 3. If browser encounters Cloudflare challengewait 5s and retry
35
+ * 3. If browser gets blocked (403, CAPTCHA) try stealth mode
36
+ * 4. If stealth mode is explicitly requested → skip to stealth
26
37
  *
27
38
  * Returns the result along with which method worked
28
39
  */
@@ -1 +1 @@
1
- {"version":3,"file":"strategies.d.ts","sourceRoot":"","sources":["../../src/core/strategies.ts"],"names":[],"mappings":"AAAA;;GAEG;AAEH,OAAO,EAAyC,KAAK,WAAW,EAAE,MAAM,cAAc,CAAC;AAGvF,MAAM,WAAW,eAAe;IAC9B,6CAA6C;IAC7C,YAAY,CAAC,EAAE,OAAO,CAAC;IACvB,qDAAqD;IACrD,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,wBAAwB;IACxB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,2BAA2B;IAC3B,SAAS,CAAC,EAAE,MAAM,CAAC;CACpB;AAED,MAAM,WAAW,cAAe,SAAQ,WAAW;IACjD,qDAAqD;IACrD,MAAM,EAAE,QAAQ,GAAG,SAAS,CAAC;CAC9B;AAED;;;;;;;;;GASG;AACH,wBAAsB,UAAU,CAAC,GAAG,EAAE,MAAM,EAAE,OAAO,GAAE,eAAoB,GAAG,OAAO,CAAC,cAAc,CAAC,CAuDpG"}
1
+ {"version":3,"file":"strategies.d.ts","sourceRoot":"","sources":["../../src/core/strategies.ts"],"names":[],"mappings":"AAAA;;GAEG;AAEH,OAAO,EAAyC,KAAK,WAAW,EAAE,MAAM,cAAc,CAAC;AAGvF,MAAM,WAAW,eAAe;IAC9B,6CAA6C;IAC7C,YAAY,CAAC,EAAE,OAAO,CAAC;IACvB,+CAA+C;IAC/C,OAAO,CAAC,EAAE,OAAO,CAAC;IAClB,qDAAqD;IACrD,MAAM,CAAC,EAAE,MAAM,CAAC;IAChB,wBAAwB;IACxB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,2BAA2B;IAC3B,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,uCAAuC;IACvC,UAAU,CAAC,EAAE,OAAO,CAAC;IACrB,oDAAoD;IACpD,kBAAkB,CAAC,EAAE,OAAO,CAAC;IAC7B,kCAAkC;IAClC,OAAO,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IACjC,uCAAuC;IACvC,OAAO,CAAC,EAAE,MAAM,EAAE,CAAC;CACpB;AAED,MAAM,WAAW,cAAe,SAAQ,WAAW;IACjD,iEAAiE;IACjE,MAAM,EAAE,QAAQ,GAAG,SAAS,GAAG,SAAS,CAAC;CAC1C;AAED;;;;;;;;;;GAUG;AACH,wBAAsB,UAAU,CAAC,GAAG,EAAE,MAAM,EAAE,OAAO,GAAE,eAAoB,GAAG,OAAO,CAAC,cAAc,CAAC,CAqGpG"}
@@ -9,16 +9,19 @@ import { BlockedError, NetworkError } from '../types.js';
9
9
  * Strategy:
10
10
  * 1. Try simple HTTP fetch first (fast, ~200ms)
11
11
  * 2. If blocked (403, 503, Cloudflare, empty body) → try browser
12
- * 3. If browser encounters Cloudflare challengewait 5s and retry
12
+ * 3. If browser gets blocked (403, CAPTCHA) try stealth mode
13
+ * 4. If stealth mode is explicitly requested → skip to stealth
13
14
  *
14
15
  * Returns the result along with which method worked
15
16
  */
16
17
  export async function smartFetch(url, options = {}) {
17
- const { forceBrowser = false, waitMs = 0, userAgent, timeoutMs = 30000 } = options;
18
- // Strategy 1: Simple fetch (unless browser is forced)
19
- if (!forceBrowser) {
18
+ const { forceBrowser = false, stealth = false, waitMs = 0, userAgent, timeoutMs = 30000, screenshot = false, screenshotFullPage = false, headers, cookies } = options;
19
+ // If stealth is requested, force browser mode (stealth requires browser)
20
+ const shouldUseBrowser = forceBrowser || screenshot || stealth;
21
+ // Strategy 1: Simple fetch (unless browser is forced or screenshot is requested)
22
+ if (!shouldUseBrowser) {
20
23
  try {
21
- const result = await retryFetch(() => simpleFetch(url, userAgent, timeoutMs), 3);
24
+ const result = await retryFetch(() => simpleFetch(url, userAgent, timeoutMs, headers), 3);
22
25
  return {
23
26
  ...result,
24
27
  method: 'simple',
@@ -35,19 +38,47 @@ export async function smartFetch(url, options = {}) {
35
38
  }
36
39
  }
37
40
  }
38
- // Strategy 2: Browser fetch
41
+ // Strategy 2: Browser fetch (with or without stealth)
39
42
  try {
40
43
  const result = await browserFetch(url, {
41
44
  userAgent,
42
45
  waitMs,
43
46
  timeoutMs,
47
+ screenshot,
48
+ screenshotFullPage,
49
+ headers,
50
+ cookies,
51
+ stealth,
44
52
  });
45
53
  return {
46
54
  ...result,
47
- method: 'browser',
55
+ method: stealth ? 'stealth' : 'browser',
48
56
  };
49
57
  }
50
58
  catch (error) {
59
+ // Strategy 3: If browser gets blocked, try stealth mode as fallback (unless already using stealth)
60
+ if (!stealth && error instanceof BlockedError) {
61
+ try {
62
+ const result = await browserFetch(url, {
63
+ userAgent,
64
+ waitMs,
65
+ timeoutMs,
66
+ screenshot,
67
+ screenshotFullPage,
68
+ headers,
69
+ cookies,
70
+ stealth: true, // Escalate to stealth mode
71
+ });
72
+ return {
73
+ ...result,
74
+ method: 'stealth',
75
+ };
76
+ }
77
+ catch (stealthError) {
78
+ // If stealth also fails, throw the original error
79
+ throw stealthError;
80
+ }
81
+ }
51
82
  // If browser encounters Cloudflare, retry with extra wait time
52
83
  if (error instanceof NetworkError &&
53
84
  error.message.toLowerCase().includes('cloudflare')) {
@@ -55,10 +86,15 @@ export async function smartFetch(url, options = {}) {
55
86
  userAgent,
56
87
  waitMs: 5000, // Wait 5s for Cloudflare challenge
57
88
  timeoutMs,
89
+ screenshot,
90
+ screenshotFullPage,
91
+ headers,
92
+ cookies,
93
+ stealth, // Keep stealth setting
58
94
  });
59
95
  return {
60
96
  ...result,
61
- method: 'browser',
97
+ method: stealth ? 'stealth' : 'browser',
62
98
  };
63
99
  }
64
100
  throw error;
@@ -1 +1 @@
1
- {"version":3,"file":"strategies.js","sourceRoot":"","sources":["../../src/core/strategies.ts"],"names":[],"mappings":"AAAA;;GAEG;AAEH,OAAO,EAAE,WAAW,EAAE,YAAY,EAAE,UAAU,EAAoB,MAAM,cAAc,CAAC;AACvF,OAAO,EAAE,YAAY,EAAE,YAAY,EAAE,MAAM,aAAa,CAAC;AAkBzD;;;;;;;;;GASG;AACH,MAAM,CAAC,KAAK,UAAU,UAAU,CAAC,GAAW,EAAE,UAA2B,EAAE;IACzE,MAAM,EAAE,YAAY,GAAG,KAAK,EAAE,MAAM,GAAG,CAAC,EAAE,SAAS,EAAE,SAAS,GAAG,KAAK,EAAE,GAAG,OAAO,CAAC;IAEnF,sDAAsD;IACtD,IAAI,CAAC,YAAY,EAAE,CAAC;QAClB,IAAI,CAAC;YACH,MAAM,MAAM,GAAG,MAAM,UAAU,CAC7B,GAAG,EAAE,CAAC,WAAW,CAAC,GAAG,EAAE,SAAS,EAAE,SAAS,CAAC,EAC5C,CAAC,CACF,CAAC;YACF,OAAO;gBACL,GAAG,MAAM;gBACT,MAAM,EAAE,QAAQ;aACjB,CAAC;QACJ,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,8CAA8C;YAC9C,IAAI,KAAK,YAAY,YAAY,EAAE,CAAC;gBAClC,mCAAmC;YACrC,CAAC;iBAAM,CAAC;gBACN,kDAAkD;gBAClD,MAAM,KAAK,CAAC;YACd,CAAC;QACH,CAAC;IACH,CAAC;IAED,4BAA4B;IAC5B,IAAI,CAAC;QACH,MAAM,MAAM,GAAG,MAAM,YAAY,CAAC,GAAG,EAAE;YACrC,SAAS;YACT,MAAM;YACN,SAAS;SACV,CAAC,CAAC;QACH,OAAO;YACL,GAAG,MAAM;YACT,MAAM,EAAE,SAAS;SAClB,CAAC;IACJ,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,+DAA+D;QAC/D,IACE,KAAK,YAAY,YAAY;YAC7B,KAAK,CAAC,OAAO,CAAC,WAAW,EAAE,CAAC,QAAQ,CAAC,YAAY,CAAC,EAClD,CAAC;YACD,MAAM,MAAM,GAAG,MAAM,YAAY,CAAC,GAAG,EAAE;gBACrC,SAAS;gBACT,MAAM,EAAE,IAAI,EAAE,mCAAmC;gBACjD,SAAS;aACV,CAAC,CAAC;YACH,OAAO;gBACL,GAAG,MAAM;gBACT,MAAM,EAAE,SAAS;aAClB,CAAC;QACJ,CAAC;QAED,MAAM,KAAK,CAAC;IACd,CAAC;AACH,CAAC"}
1
+ {"version":3,"file":"strategies.js","sourceRoot":"","sources":["../../src/core/strategies.ts"],"names":[],"mappings":"AAAA;;GAEG;AAEH,OAAO,EAAE,WAAW,EAAE,YAAY,EAAE,UAAU,EAAoB,MAAM,cAAc,CAAC;AACvF,OAAO,EAAE,YAAY,EAAE,YAAY,EAAE,MAAM,aAAa,CAAC;AA4BzD;;;;;;;;;;GAUG;AACH,MAAM,CAAC,KAAK,UAAU,UAAU,CAAC,GAAW,EAAE,UAA2B,EAAE;IACzE,MAAM,EACJ,YAAY,GAAG,KAAK,EACpB,OAAO,GAAG,KAAK,EACf,MAAM,GAAG,CAAC,EACV,SAAS,EACT,SAAS,GAAG,KAAK,EACjB,UAAU,GAAG,KAAK,EAClB,kBAAkB,GAAG,KAAK,EAC1B,OAAO,EACP,OAAO,EACR,GAAG,OAAO,CAAC;IAEZ,yEAAyE;IACzE,MAAM,gBAAgB,GAAG,YAAY,IAAI,UAAU,IAAI,OAAO,CAAC;IAE/D,iFAAiF;IACjF,IAAI,CAAC,gBAAgB,EAAE,CAAC;QACtB,IAAI,CAAC;YACH,MAAM,MAAM,GAAG,MAAM,UAAU,CAC7B,GAAG,EAAE,CAAC,WAAW,CAAC,GAAG,EAAE,SAAS,EAAE,SAAS,EAAE,OAAO,CAAC,EACrD,CAAC,CACF,CAAC;YACF,OAAO;gBACL,GAAG,MAAM;gBACT,MAAM,EAAE,QAAQ;aACjB,CAAC;QACJ,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,8CAA8C;YAC9C,IAAI,KAAK,YAAY,YAAY,EAAE,CAAC;gBAClC,mCAAmC;YACrC,CAAC;iBAAM,CAAC;gBACN,kDAAkD;gBAClD,MAAM,KAAK,CAAC;YACd,CAAC;QACH,CAAC;IACH,CAAC;IAED,sDAAsD;IACtD,IAAI,CAAC;QACH,MAAM,MAAM,GAAG,MAAM,YAAY,CAAC,GAAG,EAAE;YACrC,SAAS;YACT,MAAM;YACN,SAAS;YACT,UAAU;YACV,kBAAkB;YAClB,OAAO;YACP,OAAO;YACP,OAAO;SACR,CAAC,CAAC;QACH,OAAO;YACL,GAAG,MAAM;YACT,MAAM,EAAE,OAAO,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,SAAS;SACxC,CAAC;IACJ,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,mGAAmG;QACnG,IAAI,CAAC,OAAO,IAAI,KAAK,YAAY,YAAY,EAAE,CAAC;YAC9C,IAAI,CAAC;gBACH,MAAM,MAAM,GAAG,MAAM,YAAY,CAAC,GAAG,EAAE;oBACrC,SAAS;oBACT,MAAM;oBACN,SAAS;oBACT,UAAU;oBACV,kBAAkB;oBAClB,OAAO;oBACP,OAAO;oBACP,OAAO,EAAE,IAAI,EAAE,2BAA2B;iBAC3C,CAAC,CAAC;gBACH,OAAO;oBACL,GAAG,MAAM;oBACT,MAAM,EAAE,SAAS;iBAClB,CAAC;YACJ,CAAC;YAAC,OAAO,YAAY,EAAE,CAAC;gBACtB,kDAAkD;gBAClD,MAAM,YAAY,CAAC;YACrB,CAAC;QACH,CAAC;QAED,+DAA+D;QAC/D,IACE,KAAK,YAAY,YAAY;YAC7B,KAAK,CAAC,OAAO,CAAC,WAAW,EAAE,CAAC,QAAQ,CAAC,YAAY,CAAC,EAClD,CAAC;YACD,MAAM,MAAM,GAAG,MAAM,YAAY,CAAC,GAAG,EAAE;gBACrC,SAAS;gBACT,MAAM,EAAE,IAAI,EAAE,mCAAmC;gBACjD,SAAS;gBACT,UAAU;gBACV,kBAAkB;gBAClB,OAAO;gBACP,OAAO;gBACP,OAAO,EAAE,uBAAuB;aACjC,CAAC,CAAC;YACH,OAAO;gBACL,GAAG,MAAM;gBACT,MAAM,EAAE,OAAO,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,SAAS;aACxC,CAAC;QACJ,CAAC;QAED,MAAM,KAAK,CAAC;IACd,CAAC;AACH,CAAC"}
package/dist/index.d.ts CHANGED
@@ -6,6 +6,7 @@
6
6
  import { cleanup } from './core/fetcher.js';
7
7
  import type { PeelOptions, PeelResult } from './types.js';
8
8
  export * from './types.js';
9
+ export { crawl, type CrawlOptions, type CrawlResult } from './core/crawler.js';
9
10
  /**
10
11
  * Fetch and extract content from a URL
11
12
  *
@@ -23,6 +24,27 @@ export * from './types.js';
23
24
  * ```
24
25
  */
25
26
  export declare function peel(url: string, options?: PeelOptions): Promise<PeelResult>;
27
+ /**
28
+ * Fetch multiple URLs in batch with concurrency control
29
+ *
30
+ * @param urls - Array of URLs to fetch
31
+ * @param options - Fetch options (including concurrency)
32
+ * @returns Array of results or errors
33
+ *
34
+ * @example
35
+ * ```typescript
36
+ * import { peelBatch } from 'webpeel';
37
+ *
38
+ * const urls = ['https://example.com', 'https://example.org'];
39
+ * const results = await peelBatch(urls, { concurrency: 3 });
40
+ * ```
41
+ */
42
+ export declare function peelBatch(urls: string[], options?: PeelOptions & {
43
+ concurrency?: number;
44
+ }): Promise<(PeelResult | {
45
+ url: string;
46
+ error: string;
47
+ })[]>;
26
48
  /**
27
49
  * Clean up any browser resources
28
50
  * Call this when you're done using WebPeel
@@ -1 +1 @@
1
- {"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAKH,OAAO,EAAE,OAAO,EAAE,MAAM,mBAAmB,CAAC;AAC5C,OAAO,KAAK,EAAE,WAAW,EAAE,UAAU,EAAE,MAAM,YAAY,CAAC;AAE1D,cAAc,YAAY,CAAC;AAE3B;;;;;;;;;;;;;;;GAeG;AACH,wBAAsB,IAAI,CAAC,GAAG,EAAE,MAAM,EAAE,OAAO,GAAE,WAAgB,GAAG,OAAO,CAAC,UAAU,CAAC,CA4DtF;AAED;;;GAGG;AACH,OAAO,EAAE,OAAO,EAAE,CAAC"}
1
+ {"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAKH,OAAO,EAAE,OAAO,EAAE,MAAM,mBAAmB,CAAC;AAC5C,OAAO,KAAK,EAAE,WAAW,EAAE,UAAU,EAAE,MAAM,YAAY,CAAC;AAE1D,cAAc,YAAY,CAAC;AAC3B,OAAO,EAAE,KAAK,EAAE,KAAK,YAAY,EAAE,KAAK,WAAW,EAAE,MAAM,mBAAmB,CAAC;AAE/E;;;;;;;;;;;;;;;GAeG;AACH,wBAAsB,IAAI,CAAC,GAAG,EAAE,MAAM,EAAE,OAAO,GAAE,WAAgB,GAAG,OAAO,CAAC,UAAU,CAAC,CAkGtF;AAED;;;;;;;;;;;;;;GAcG;AACH,wBAAsB,SAAS,CAC7B,IAAI,EAAE,MAAM,EAAE,EACd,OAAO,GAAE,WAAW,GAAG;IAAE,WAAW,CAAC,EAAE,MAAM,CAAA;CAAO,GACnD,OAAO,CAAC,CAAC,UAAU,GAAG;IAAE,GAAG,EAAE,MAAM,CAAC;IAAC,KAAK,EAAE,MAAM,CAAA;CAAE,CAAC,EAAE,CAAC,CAwB1D;AAED;;;GAGG;AACH,OAAO,EAAE,OAAO,EAAE,CAAC"}
package/dist/index.js CHANGED
@@ -4,10 +4,11 @@
4
4
  * Main library export
5
5
  */
6
6
  import { smartFetch } from './core/strategies.js';
7
- import { htmlToMarkdown, htmlToText, estimateTokens } from './core/markdown.js';
7
+ import { htmlToMarkdown, htmlToText, estimateTokens, selectContent } from './core/markdown.js';
8
8
  import { extractMetadata, extractLinks } from './core/metadata.js';
9
9
  import { cleanup } from './core/fetcher.js';
10
10
  export * from './types.js';
11
+ export { crawl } from './core/crawler.js';
11
12
  /**
12
13
  * Fetch and extract content from a URL
13
14
  *
@@ -26,36 +27,61 @@ export * from './types.js';
26
27
  */
27
28
  export async function peel(url, options = {}) {
28
29
  const startTime = Date.now();
29
- const { render = false, wait = 0, format = 'markdown', timeout = 30000, userAgent, } = options;
30
+ let { render = false, stealth = false, wait = 0, format = 'markdown', timeout = 30000, userAgent, screenshot = false, screenshotFullPage = false, selector, exclude, headers, cookies, } = options;
31
+ // Detect PDF URLs and force browser rendering
32
+ const isPdf = url.toLowerCase().endsWith('.pdf');
33
+ if (isPdf) {
34
+ render = true;
35
+ }
36
+ // If screenshot is requested, force render mode
37
+ if (screenshot) {
38
+ render = true;
39
+ }
40
+ // If stealth is requested, force render mode
41
+ if (stealth) {
42
+ render = true;
43
+ }
30
44
  try {
31
45
  // Fetch the page
32
46
  const fetchResult = await smartFetch(url, {
33
47
  forceBrowser: render,
48
+ stealth,
34
49
  waitMs: wait,
35
50
  userAgent,
36
51
  timeoutMs: timeout,
52
+ screenshot,
53
+ screenshotFullPage,
54
+ headers,
55
+ cookies,
37
56
  });
57
+ // Apply selector filtering if requested
58
+ let html = fetchResult.html;
59
+ if (selector) {
60
+ html = selectContent(html, selector, exclude);
61
+ }
38
62
  // Extract metadata and title
39
- const { title, metadata } = extractMetadata(fetchResult.html, fetchResult.url);
63
+ const { title, metadata } = extractMetadata(html, fetchResult.url);
40
64
  // Extract links
41
- const links = extractLinks(fetchResult.html, fetchResult.url);
65
+ const links = extractLinks(html, fetchResult.url);
42
66
  // Convert content to requested format
43
67
  let content;
44
68
  switch (format) {
45
69
  case 'html':
46
- content = fetchResult.html;
70
+ content = html;
47
71
  break;
48
72
  case 'text':
49
- content = htmlToText(fetchResult.html);
73
+ content = htmlToText(html);
50
74
  break;
51
75
  case 'markdown':
52
76
  default:
53
- content = htmlToMarkdown(fetchResult.html);
77
+ content = htmlToMarkdown(html);
54
78
  break;
55
79
  }
56
80
  // Calculate elapsed time and token estimate
57
81
  const elapsed = Date.now() - startTime;
58
82
  const tokens = estimateTokens(content);
83
+ // Convert screenshot buffer to base64 if present
84
+ const screenshotBase64 = fetchResult.screenshot?.toString('base64');
59
85
  return {
60
86
  url: fetchResult.url,
61
87
  title,
@@ -65,6 +91,7 @@ export async function peel(url, options = {}) {
65
91
  tokens,
66
92
  method: fetchResult.method,
67
93
  elapsed,
94
+ screenshot: screenshotBase64,
68
95
  };
69
96
  }
70
97
  catch (error) {
@@ -73,6 +100,42 @@ export async function peel(url, options = {}) {
73
100
  throw error;
74
101
  }
75
102
  }
103
+ /**
104
+ * Fetch multiple URLs in batch with concurrency control
105
+ *
106
+ * @param urls - Array of URLs to fetch
107
+ * @param options - Fetch options (including concurrency)
108
+ * @returns Array of results or errors
109
+ *
110
+ * @example
111
+ * ```typescript
112
+ * import { peelBatch } from 'webpeel';
113
+ *
114
+ * const urls = ['https://example.com', 'https://example.org'];
115
+ * const results = await peelBatch(urls, { concurrency: 3 });
116
+ * ```
117
+ */
118
+ export async function peelBatch(urls, options = {}) {
119
+ const { concurrency = 3, ...peelOpts } = options;
120
+ const results = [];
121
+ // Process in batches
122
+ for (let i = 0; i < urls.length; i += concurrency) {
123
+ const batch = urls.slice(i, i + concurrency);
124
+ const batchResults = await Promise.allSettled(batch.map(url => peel(url, peelOpts)));
125
+ batchResults.forEach((result, j) => {
126
+ if (result.status === 'fulfilled') {
127
+ results.push(result.value);
128
+ }
129
+ else {
130
+ results.push({
131
+ url: batch[j],
132
+ error: result.reason?.message || 'Unknown error'
133
+ });
134
+ }
135
+ });
136
+ }
137
+ return results;
138
+ }
76
139
  /**
77
140
  * Clean up any browser resources
78
141
  * Call this when you're done using WebPeel
package/dist/index.js.map CHANGED
@@ -1 +1 @@
1
- {"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAEH,OAAO,EAAE,UAAU,EAAE,MAAM,sBAAsB,CAAC;AAClD,OAAO,EAAE,cAAc,EAAE,UAAU,EAAE,cAAc,EAAE,MAAM,oBAAoB,CAAC;AAChF,OAAO,EAAE,eAAe,EAAE,YAAY,EAAE,MAAM,oBAAoB,CAAC;AACnE,OAAO,EAAE,OAAO,EAAE,MAAM,mBAAmB,CAAC;AAG5C,cAAc,YAAY,CAAC;AAE3B;;;;;;;;;;;;;;;GAeG;AACH,MAAM,CAAC,KAAK,UAAU,IAAI,CAAC,GAAW,EAAE,UAAuB,EAAE;IAC/D,MAAM,SAAS,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;IAE7B,MAAM,EACJ,MAAM,GAAG,KAAK,EACd,IAAI,GAAG,CAAC,EACR,MAAM,GAAG,UAAU,EACnB,OAAO,GAAG,KAAK,EACf,SAAS,GACV,GAAG,OAAO,CAAC;IAEZ,IAAI,CAAC;QACH,iBAAiB;QACjB,MAAM,WAAW,GAAG,MAAM,UAAU,CAAC,GAAG,EAAE;YACxC,YAAY,EAAE,MAAM;YACpB,MAAM,EAAE,IAAI;YACZ,SAAS;YACT,SAAS,EAAE,OAAO;SACnB,CAAC,CAAC;QAEH,6BAA6B;QAC7B,MAAM,EAAE,KAAK,EAAE,QAAQ,EAAE,GAAG,eAAe,CAAC,WAAW,CAAC,IAAI,EAAE,WAAW,CAAC,GAAG,CAAC,CAAC;QAE/E,gBAAgB;QAChB,MAAM,KAAK,GAAG,YAAY,CAAC,WAAW,CAAC,IAAI,EAAE,WAAW,CAAC,GAAG,CAAC,CAAC;QAE9D,sCAAsC;QACtC,IAAI,OAAe,CAAC;QACpB,QAAQ,MAAM,EAAE,CAAC;YACf,KAAK,MAAM;gBACT,OAAO,GAAG,WAAW,CAAC,IAAI,CAAC;gBAC3B,MAAM;YACR,KAAK,MAAM;gBACT,OAAO,GAAG,UAAU,CAAC,WAAW,CAAC,IAAI,CAAC,CAAC;gBACvC,MAAM;YACR,KAAK,UAAU,CAAC;YAChB;gBACE,OAAO,GAAG,cAAc,CAAC,WAAW,CAAC,IAAI,CAAC,CAAC;gBAC3C,MAAM;QACV,CAAC;QAED,4CAA4C;QAC5C,MAAM,OAAO,GAAG,IAAI,CAAC,GAAG,EAAE,GAAG,SAAS,CAAC;QACvC,MAAM,MAAM,GAAG,cAAc,CAAC,OAAO,CAAC,CAAC;QAEvC,OAAO;YACL,GAAG,EAAE,WAAW,CAAC,GAAG;YACpB,KAAK;YACL,OAAO;YACP,QAAQ;YACR,KAAK;YACL,MAAM;YACN,MAAM,EAAE,WAAW,CAAC,MAAM;YAC1B,OAAO;SACR,CAAC;IACJ,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,sCAAsC;QACtC,MAAM,OAAO,EAAE,CAAC;QAChB,MAAM,KAAK,CAAC;IACd,CAAC;AACH,CAAC;AAED;;;GAGG;AACH,OAAO,EAAE,OAAO,EAAE,CAAC"}
1
+ {"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAEH,OAAO,EAAE,UAAU,EAAE,MAAM,sBAAsB,CAAC;AAClD,OAAO,EAAE,cAAc,EAAE,UAAU,EAAE,cAAc,EAAE,aAAa,EAAE,MAAM,oBAAoB,CAAC;AAC/F,OAAO,EAAE,eAAe,EAAE,YAAY,EAAE,MAAM,oBAAoB,CAAC;AACnE,OAAO,EAAE,OAAO,EAAE,MAAM,mBAAmB,CAAC;AAG5C,cAAc,YAAY,CAAC;AAC3B,OAAO,EAAE,KAAK,EAAuC,MAAM,mBAAmB,CAAC;AAE/E;;;;;;;;;;;;;;;GAeG;AACH,MAAM,CAAC,KAAK,UAAU,IAAI,CAAC,GAAW,EAAE,UAAuB,EAAE;IAC/D,MAAM,SAAS,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;IAE7B,IAAI,EACF,MAAM,GAAG,KAAK,EACd,OAAO,GAAG,KAAK,EACf,IAAI,GAAG,CAAC,EACR,MAAM,GAAG,UAAU,EACnB,OAAO,GAAG,KAAK,EACf,SAAS,EACT,UAAU,GAAG,KAAK,EAClB,kBAAkB,GAAG,KAAK,EAC1B,QAAQ,EACR,OAAO,EACP,OAAO,EACP,OAAO,GACR,GAAG,OAAO,CAAC;IAEZ,8CAA8C;IAC9C,MAAM,KAAK,GAAG,GAAG,CAAC,WAAW,EAAE,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC;IACjD,IAAI,KAAK,EAAE,CAAC;QACV,MAAM,GAAG,IAAI,CAAC;IAChB,CAAC;IAED,gDAAgD;IAChD,IAAI,UAAU,EAAE,CAAC;QACf,MAAM,GAAG,IAAI,CAAC;IAChB,CAAC;IAED,6CAA6C;IAC7C,IAAI,OAAO,EAAE,CAAC;QACZ,MAAM,GAAG,IAAI,CAAC;IAChB,CAAC;IAED,IAAI,CAAC;QACH,iBAAiB;QACjB,MAAM,WAAW,GAAG,MAAM,UAAU,CAAC,GAAG,EAAE;YACxC,YAAY,EAAE,MAAM;YACpB,OAAO;YACP,MAAM,EAAE,IAAI;YACZ,SAAS;YACT,SAAS,EAAE,OAAO;YAClB,UAAU;YACV,kBAAkB;YAClB,OAAO;YACP,OAAO;SACR,CAAC,CAAC;QAEH,wCAAwC;QACxC,IAAI,IAAI,GAAG,WAAW,CAAC,IAAI,CAAC;QAC5B,IAAI,QAAQ,EAAE,CAAC;YACb,IAAI,GAAG,aAAa,CAAC,IAAI,EAAE,QAAQ,EAAE,OAAO,CAAC,CAAC;QAChD,CAAC;QAED,6BAA6B;QAC7B,MAAM,EAAE,KAAK,EAAE,QAAQ,EAAE,GAAG,eAAe,CAAC,IAAI,EAAE,WAAW,CAAC,GAAG,CAAC,CAAC;QAEnE,gBAAgB;QAChB,MAAM,KAAK,GAAG,YAAY,CAAC,IAAI,EAAE,WAAW,CAAC,GAAG,CAAC,CAAC;QAElD,sCAAsC;QACtC,IAAI,OAAe,CAAC;QACpB,QAAQ,MAAM,EAAE,CAAC;YACf,KAAK,MAAM;gBACT,OAAO,GAAG,IAAI,CAAC;gBACf,MAAM;YACR,KAAK,MAAM;gBACT,OAAO,GAAG,UAAU,CAAC,IAAI,CAAC,CAAC;gBAC3B,MAAM;YACR,KAAK,UAAU,CAAC;YAChB;gBACE,OAAO,GAAG,cAAc,CAAC,IAAI,CAAC,CAAC;gBAC/B,MAAM;QACV,CAAC;QAED,4CAA4C;QAC5C,MAAM,OAAO,GAAG,IAAI,CAAC,GAAG,EAAE,GAAG,SAAS,CAAC;QACvC,MAAM,MAAM,GAAG,cAAc,CAAC,OAAO,CAAC,CAAC;QAEvC,iDAAiD;QACjD,MAAM,gBAAgB,GAAG,WAAW,CAAC,UAAU,EAAE,QAAQ,CAAC,QAAQ,CAAC,CAAC;QAEpE,OAAO;YACL,GAAG,EAAE,WAAW,CAAC,GAAG;YACpB,KAAK;YACL,OAAO;YACP,QAAQ;YACR,KAAK;YACL,MAAM;YACN,MAAM,EAAE,WAAW,CAAC,MAAM;YAC1B,OAAO;YACP,UAAU,EAAE,gBAAgB;SAC7B,CAAC;IACJ,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,sCAAsC;QACtC,MAAM,OAAO,EAAE,CAAC;QAChB,MAAM,KAAK,CAAC;IACd,CAAC;AACH,CAAC;AAED;;;;;;;;;;;;;;GAcG;AACH,MAAM,CAAC,KAAK,UAAU,SAAS,CAC7B,IAAc,EACd,UAAkD,EAAE;IAEpD,MAAM,EAAE,WAAW,GAAG,CAAC,EAAE,GAAG,QAAQ,EAAE,GAAG,OAAO,CAAC;IACjD,MAAM,OAAO,GAAoD,EAAE,CAAC;IAEpE,qBAAqB;IACrB,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,IAAI,CAAC,MAAM,EAAE,CAAC,IAAI,WAAW,EAAE,CAAC;QAClD,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,GAAG,WAAW,CAAC,CAAC;QAC7C,MAAM,YAAY,GAAG,MAAM,OAAO,CAAC,UAAU,CAC3C,KAAK,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE,CAAC,IAAI,CAAC,GAAG,EAAE,QAAQ,CAAC,CAAC,CACtC,CAAC;QAEF,YAAY,CAAC,OAAO,CAAC,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE;YACjC,IAAI,MAAM,CAAC,MAAM,KAAK,WAAW,EAAE,CAAC;gBAClC,OAAO,CAAC,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;YAC7B,CAAC;iBAAM,CAAC;gBACN,OAAO,CAAC,IAAI,CAAC;oBACX,GAAG,EAAE,KAAK,CAAC,CAAC,CAAC;oBACb,KAAK,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,IAAI,eAAe;iBACjD,CAAC,CAAC;YACL,CAAC;QACH,CAAC,CAAC,CAAC;IACL,CAAC;IAED,OAAO,OAAO,CAAC;AACjB,CAAC;AAED;;;GAGG;AACH,OAAO,EAAE,OAAO,EAAE,CAAC"}