website-to-markdown 1.0.0 → 1.1.0

package/dist/fetch-page.js.map

@@ -1 +1 @@
-{"version":3,"file":"fetch-page.js","sourceRoot":"","sources":["../src/fetch-page.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,eAAe,EAAE,MAAM,QAAQ,CAAA;AACxC,OAAO,EAEN,0BAA0B,EAC1B,sBAAsB,EACtB,MAAM,aAAa,CAAA;AACpB,OAAO,EACN,mBAAmB,EAEnB,uBAAuB,EACvB,MAAM,mBAAmB,CAAA;AAC1B,OAAO,EAAE,oBAAoB,EAAE,oBAAoB,EAAE,MAAM,qBAAqB,CAAA;AAChF,OAAO,EACN,kBAAkB,EAClB,eAAe,EACf,aAAa,EACb,MAAM,sBAAsB,CAAA;AAC7B,OAAO,EAAE,SAAS,EAAE,MAAM,WAAW,CAAA;AACrC,OAAO,EAAE,mBAAmB,EAAE,MAAM,oBAAoB,CAAA;AACxD,OAAO,EAA+B,gBAAgB,EAAE,MAAM,YAAY,CAAA;AAC1E,OAAO,EAAE,kBAAkB,EAAe,YAAY,EAAE,MAAM,aAAa,CAAA;AAC3E,OAAO,EACN,UAAU,EACV,SAAS,EACT,YAAY,EACZ,qBAAqB,EACrB,gBAAgB,EAChB,iBAAiB,EACjB,MAAM,oBAAoB,CAAA;AAC3B,OAAO,EAAE,4BAA4B,EAAE,MAAM,mBAAmB,CAAA;AAChE,OAAO,EAAE,aAAa,EAAE,mBAAmB,EAAE,MAAM,gBAAgB,CAAA;AACnE,OAAO,EAAE,sBAAsB,EAAE,MAAM,uBAAuB,CAAA;AAC9D,OAAO,EAAE,kBAAkB,EAAE,MAAM,iBAAiB,CAAA;AACpD,OAAO,EAAE,iBAAiB,EAAE,MAAM,iBAAiB,CAAA;AAEnD,MAAM,gBAAgB,GAAG,MAAM,CAAA;AAC/B,+EAA+E;AAC/E,MAAM,cAAc,GAAG,CAAC,GAAG,IAAI,GAAG,IAAI,CAAA;AACtC,MAAM,kBAAkB,GAAG,CAAC,GAAG,IAAI,GAAG,IAAI,CAAA;AAC1C,MAAM,aAAa,GAAG,EAAE,GAAG,IAAI,GAAG,IAAI,CAAA;AACtC,MAAM,aAAa,GAAG,EAAE,CAAA;AAExB,0FAA0F;AAC1F,MAAM,CAAC,MAAM,kBAAkB,GAAG,OAAO,CAAA;AAiFzC;;;;GAIG;AACH,SAAS,SAAS,CACjB,IAAc,EACd,OAAiB,EACjB,MAAc;IAEd,0EAA0E;IAC1E,4EAA4E;IAC5E,2EAA2E;IAC3E,yEAAyE;IACzE,2EAA2E;IAC3E,0BAA0B;IAC1B,MAAM,KAAK,GAAG,CAAC,IAA+B,EAAE,EAAE,CACjD,IAAI,EAAE,UAAU,CAAC,QAAQ,EAAE,EAAE,CAAC,CAAC,SAAS,CAAC,KAAK,CAAC,CAAA;IAChD,MAAM,IAAI,GAAG;QACZ,GAAG,OAAO;QACV,QAAQ,EAAE,KAAK,CAAC,OAAO,CAAC,QAAQ,CAAC;QACjC,IAAI,EAAE,KAAK,CAAC,OAAO,CAAC,IAAI,CAAC;KACzB,CAAA;IACD,MAAM,WAAW,GAAG,gBAAgB,CAAC,IAAI,CAAC,QAAQ,IAAI,EAAE,CAAC,CAAA;IACzD,IAAI,QAAQ,GAAG,WAAW,CAAC,QAAQ,CAAA;IACnC,IAAI,QAAQ,CAAC,MAAM,GAAG,kBAAkB,EAAE,CAAC;QAC1C,MAAM,CAAC,IAAI,CAAC,2BAA2B,EAAE;YACxC,GAAG,EAAE,IAAI,CAAC,QAAQ;YAClB,KAAK,EAAE,QAAQ,CAAC,MAAM;SACtB,CAAC,CAAA;QACF,QAAQ,GAAG,QAAQ,CAAC,KAAK,CAAC,CAAC,EAAE,kBAAkB,CAAC,CAAA;IACjD,CAAC;IAED,MAAM,QAAQ,GAAG,IAAI,CAAC,IAAI;QACzB,CAAC,CAAC,mBAAmB,CAAC,IAAI,CAAC,IAAI,EAAE,IAAI,CAAC,QAAQ,CAAC;QAC/C,CAAC,CAAC,IAAI,CAAA;IACP,MAAM,IAAI,GAAG,YAAY,CAAC,QAAQ,EAAE,IAAI,CAAC,QAAQ,CAAC,CAAA;IAClD,sEAAsE;IACtE,wEAAwE;IACxE,0DAA0D;IAC1D,MAAM,KAAK,GAAG,IAAI,CAAC,IAAI;QACtB,CAAC,CAAC,CAAC,GAAG,IAAI,GAAG,CAAC,CAAC,GAAG,IAAI,CAAC,KAAK,EAAE,GAAG,SAAS,CAAC,IAAI,CAAC,IAAI,EAAE,IAAI,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC;QACvE,CAAC,CAAC,IAAI,CAAC,KAAK,CAAA;IAEb,OAAO;QACN,GAAG,EAAE,IAAI,CAAC,GAAG;QACb,QAAQ,EAAE,IAAI,CAAC,QAAQ;QACvB,MAAM,EAAE,IAAI,CAAC,MAAM;QACnB,OAAO,EAAE,IAAI,CAAC,OAAO;QACrB,MAAM,EAAE,IAAI,CAAC,MAAM;QACnB,WAAW,EAAE,IAAI,CAAC,WAAW;QAC7B,IAAI,EAAE,IAAI,CAAC,IAAI;QACf,YAAY,EAAE,IAAI,CAAC,YAAY;QAC/B,QAAQ;QACR,KAAK,EAAE,QAAQ,EAAE,KAAK,IAAI,WAAW,CAAC,KAAK,IAAI,iBAAiB,CAAC,QAAQ,CAAC;QAC1E,IAAI,EAAE,QAAQ,EAAE,IAAI,IAAI,IAAI;QAC5B,SAAS,EAAE,QAAQ,EAAE,SAAS,IAAI,IAAI;QACtC,UAAU,EAAE,QAAQ,EAAE,UAAU,IAAI,EAAE;QACtC,kBAAkB,EAAE,QAAQ,EAAE,kBAAkB,IAAI,EAAE;QACtD,eAAe,EACd,QAAQ,EAAE,eAAe,IAAI,WAAW,CAAC,WAAW,IAAI,IAAI;QAC7D,MAAM,EAAE,QAAQ,EAAE,MAAM,IAAI,EAAE;QAC9B,KAAK;QACL,MAAM,EAAE;YACP,GAAG,IAAI,GAAG,CAAC;gBACV,GAAG,IAAI,CAAC,MAAM;gBACd,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,UAAU,CAAC,IAAI,CAAC,IAAI,EAAE,IAAI,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;aAC1D,CAAC;SACF;QACD,YAAY,EACX,WAAW,CAAC,QAAQ,CAAC,MAAM,GAAG,kBAAkB;YAC/C,CAAC,CAAC,gBAAgB;YAClB,CAAC,CAAC,CAAC,IAAI,CAAC,YAAY,IAAI,IAAI,CAAC;QAC/B,UAAU,EAAE,IAAI,CAAC,KAAK,CAAC,WAAW,CAAC,GAAG,EAAE,GAAG,IAAI,CAAC,SAAS,CAAC;QAC1D,QAAQ,EAAE,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,IAAI;KAChD,CAAA;AACF,CAAC;AAED;;;;GAIG;AACH,SAAS,iBAAiB,CAAC,GAAW;IACrC,IAAI,MAAW,CAAA;IACf,IAAI,CAAC;QACJ,MAAM,GAAG,IAAI,GAAG,CAAC,GAAG,CAAC,CAAA;IACtB,CAAC;IAAC,MAAM,CAAC;QACR,OAAO,IAAI,CAAA;IACZ,CAAC;IACD,IAAI,MAAM,CAAC,QAAQ,KAAK,QAAQ;QAAE,OAAO,MAAM,CAAA;IAC/C,IAAI,MAAM,CAAC,QAAQ,KAAK,OAAO;QAAE,OAAO,IAAI,CAAA;IAC5C,IAAI,mBAAmB,CAAC,MAAM,CAAC,QAAQ,CAAC,EAAE,CAAC;QAC1C,OAAO,OAAO,CAAC,GAAG,CAAC,QAAQ,KAAK,YAAY,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,MAAM,CAAA;IAC7D,CAAC;IACD,MAAM,CAAC,QAAQ,GAAG,QAAQ,CAAA;IAC1B,IAAI,MAAM,CAAC,IAAI,KAAK,IAAI;QAAE,MAAM,CAAC,IAAI,GAAG,EAAE,CAAA;IAC1C,OAAO,MAAM,CAAA;AACd,CAAC;AAED;;;GAGG;AACH,SAAS,cAAc,CAAC,GAAQ;IAC/B,MAAM,IAAI,GAAG,GAAG,CAAC,QAAQ,CAAA;IACzB,IAAI,IAAI,CAAC,UAAU,CAAC,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC;QAAE,OAAO,IAAI,CAAA;IAC/D,IAAI,mBAAmB,CAAC,IAAI,CAAC,IAAI,UAAU,CAAC,IAAI,CAAC,IAAI,CAAC;QAAE,OAAO,IAAI,CAAA;IACnE,MAAM,IAAI,GAAG,IAAI,GAAG,CAAC,GAAG,CAAC,IAAI,CAAC,CAAA;IAC9B,IAAI,CAAC,QAAQ,GAAG,OAAO,IAAI,EAAE,CAAA;IAC7B,OAAO,IAAI,CAAA;AACZ,CAAC;AAED;;;;;;;GAOG;AACH,MAAM,sBAAsB,GAAG,CAAC,CAAA;AAChC,MAAM,sBAAsB,GAAG,MAAM,CAAA;AAErC,uEAAuE;AACvE,SAAS,eAAe,CAAC,QAAkB,EAAE,KAAa;IACzD,MAAM,OAAO,GAAG,eAAe,CAAC,QAAQ,CAAC,OAAO,CAAC,GAAG,CAAC,aAAa,CAAC,CAAC,CAAA;IACpE,OAAO,IAAI,CAAC,GAAG,CAAC,sBAAsB,EAAE,OAAO,IAAI,KAAK,GAAG,CAAC,IAAI,CAAC,KAAK,GAAG,CAAC,CAAC,CAAC,CAAA;AAC7E,CAAC;AAED;;;;;GAKG;AACH,MAAM,kBAAkB,GAAG,CAAC,CAAA;AAE5B,SAAS,QAAQ,CAAC,IAAY;IAC7B,OAAO,SAAS,CAAC,GAAG,kBAAkB,KAAK,IAAI,EAAE,CAAC,CAAA;AACnD,CAAC;AAED,iFAAiF;AACjF,SAAS,QAAQ,CAAC,GAAQ,EAAE,OAAkC;IAC7D,IAAI,CAAC,OAAO,IAAI,GAAG,CAAC,YAAY,CAAC,GAAG,CAAC,SAAS,CAAC;QAAE,OAAO,GAAG,CAAA;IAC3D,MAAM,MAAM,GAAG,IAAI,GAAG,CAAC,GAAG,CAAC,IAAI,CAAC,CAAA;IAChC,MAAM,CAAC,YAAY,CAAC,GAAG,CAAC,SAAS,EAAE,OAAO,CAAC,CAAA;IAC3C,OAAO,MAAM,CAAA;AACd,CAAC;AAED,gFAAgF;AAChF,SAAS,aAAa,CAAC,WAAmB,EAAE,UAAe;IAC1D,IAAI,CAAC,WAAW;QAAE,OAAO,UAAU,CAAA;IACnC,IAAI,CAAC;QACJ,OAAO,IAAI,GAAG,CAAC,WAAW,CAAC,CAAA;IAC5B,CAAC;IAAC,MAAM,CAAC;QACR,OAAO,UAAU,CAAA;IAClB,CAAC;AACF,CAAC;AAED,SAAS,YAAY,CAAC,KAAc;IACnC,MAAM,IAAI,GACT,OAAO,KAAK,KAAK,QAAQ,IAAI,KAAK,IAAI,MAAM,IAAI,KAAK;QACpD,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC;QACpB,CAAC,CAAC,EAAE,CAAA;IACN,OAAO,IAAI,KAAK,YAAY,IAAI,IAAI,KAAK,cAAc,CAAA;AACxD,CAAC;AAeD,mFAAmF;AACnF,SAAS,WAAW,CAAC,QAAkB;IACtC,KAAK,QAAQ,CAAC,IAAI,EAAE,MAAM,EAAE,CAAC,KAAK,CAAC,GAAG,EAAE,GAAE,CAAC,CAAC,CAAA;AAC7C,CAAC;AAED;;;;GAIG;AACH,KAAK,UAAU,cAAc,CAC5B,QAAkB,EAClB,QAAa,EACb,MAAc;IAEd,MAAM,MAAM,GAAG,QAAQ,CAAC,MAAM,CAAA;IAC9B,IAAI,MAAM,GAAG,GAAG,IAAI,MAAM,IAAI,GAAG,EAAE,CAAC;QACnC,WAAW,CAAC,QAAQ,CAAC,CAAA;QACrB,OAAO,EAAE,IAAI,EAAE,MAAM,EAAE,CAAA;IACxB,CAAC;IAED,MAAM,WAAW,GAAG,QAAQ,CAAC,OAAO,CAAC,GAAG,CAAC,cAAc,CAAC,CAAA;IACxD,MAAM,IAAI,GAAG,mBAAmB,CAAC,WAAW,CAAC,CAAA;IAC7C,IAAI,IAAI,KAAK,OAAO,EAAE,CAAC;QACtB,WAAW,CAAC,QAAQ,CAAC,CAAA;QACrB,OAAO,EAAE,IAAI,EAAE,MAAM,EAAE,CAAA;IACxB,CAAC;IACD,IAAI,IAAI,KAAK,UAAU,EAAE,CAAC;QACzB,MAAM,IAAI,GAAG,uBAAuB,CAAC,WAAW,CAAC,CAAA;QACjD,IAAI,CAAC,IAAI,EAAE,CAAC;YACX,WAAW,CAAC,QAAQ,CAAC,CAAA;YACrB,OAAO,EAAE,IAAI,EAAE,MAAM,EAAE,CAAA;QACxB,CAAC;QACD,MAAM,KAAK,GAAG,IAAI,CAAC,SAAS,KAAK,KAAK,CAAC,CAAC,CAAC,aAAa,CAAC,CAAC,CAAC,kBAAkB,CAAA;QAC3E,MAAM,QAAQ,GAAG,MAAM,CAAC,QAAQ,CAAC,OAAO,CAAC,GAAG,CAAC,gBAAgB,CAAC,CAAC,CAAA;QAC/D,IAAI,MAAM,CAAC,QAAQ,CAAC,QAAQ,CAAC,IAAI,QAAQ,GAAG,KAAK,EAAE,CAAC;YACnD,WAAW,CAAC,QAAQ,CAAC,CAAA;YACrB,MAAM,CAAC,IAAI,CAAC,oCAAoC,EAAE;gBACjD,GAAG,EAAE,QAAQ,CAAC,IAAI;gBAClB,KAAK,EAAE,QAAQ;aACf,CAAC,CAAA;YACF,OAAO,EAAE,IAAI,EAAE,MAAM,EAAE,YAAY,EAAE,qBAAqB,EAAE,CAAA;QAC7D,CAAC;QACD,MAAM,KAAK,GAAG,MAAM,eAAe,CAAC,QAAQ,EAAE,KAAK,CAAC,CAAA;QACpD,OAAO,KAAK;YACX,CAAC,CAAC,EAAE,IAAI,EAAE,OAAO,EAAE,KAAK,EAAE;YAC1B,CAAC,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,YAAY,EAAE,qBAAqB,EAAE,CAAA;IACzD,CAAC;IACD,OAAO,EAAE,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,MAAM,QAAQ,CAAC,IAAI,EAAE,EAAE,CAAA;AACrD,CAAC;AAED,SAAS,gBAAgB,CAAC,QAAgB,EAAE,SAAiB;IAC5D,MAAM,OAAO,GAAG,QAAQ,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,EAAE,KAAK,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,IAAI,UAAU,CAAA;IACzE,MAAM,IAAI,GAAG,OAAO,CAAC,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC,CAAA;IACjC,OAAO,IAAI,CAAC,WAAW,EAAE,CAAC,QAAQ,CAAC,IAAI,SAAS,EAAE,CAAC;QAClD,CAAC,CAAC,IAAI;QACN,CAAC,CAAC,GAAG,IAAI,IAAI,SAAS,EAAE,CAAA;AAC1B,CAAC;AAED,gFAAgF;AAChF,KAAK,UAAU,eAAe,CAC7B,QAAkB,EAClB,QAAgB;IAEhB,MAAM,MAAM,GAAG,QAAQ,CAAC,IAAI,EAAE,SAAS,EAAE,CAAA;IACzC,IAAI,CAAC,MAAM;QAAE,OAAO,IAAI,WAAW,CAAC,CAAC,CAAC,CAAA;IACtC,MAAM,MAAM,GAAiB,EAAE,CAAA;IAC/B,IAAI,IAAI,GAAG,CAAC,CAAA;IACZ,OAAO,IAAI,EAAE,CAAC;QACb,MAAM,IAAI,GAAG,MAAM,MAAM,CAAC,IAAI,EAAE,CAAA;QAChC,IAAI,IAAI,CAAC,IAAI;YAAE,MAAK;QACpB,IAAI,IAAI,IAAI,CAAC,KAAK,CAAC,UAAU,CAAA;QAC7B,IAAI,IAAI,GAAG,QAAQ,EAAE,CAAC;YACrB,MAAM,MAAM,CAAC,MAAM,EAAE,CAAA;YACrB,OAAO,IAAI,CAAA;QACZ,CAAC;QACD,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,CAAA;IACxB,CAAC;IACD,MAAM,KAAK,GAAG,IAAI,UAAU,CAAC,IAAI,CAAC,CAAA;IAClC,IAAI,MAAM,GAAG,CAAC,CAAA;IACd,KAAK,MAAM,KAAK,IAAI,MAAM,EAAE,CAAC;QAC5B,KAAK,CAAC,GAAG,CAAC,KAAK,EAAE,MAAM,CAAC,CAAA;QACxB,MAAM,IAAI,KAAK,CAAC,UAAU,CAAA;IAC3B,CAAC;IACD,OAAO,KAAK,CAAC,MAAM,CAAA;AACpB,CAAC;AAED;;;;;;;;;;;;;;;GAeG;AACH,MAAM,kBAAkB,GAAG;IAC1B,UAAU;IACV,oBAAoB;IACpB,0BAA0B;IAC1B,+BAA+B;IAC/B,gCAAgC;CAChC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;AAEZ,SAAS,eAAe,CAAC,IAAY;IACpC,OAAO,IAAI,YAAY,EAAE;SACvB,EAAE,CAAC,kBAAkB,EAAE;QACvB,OAAO,CAAC,OAAO;YACd,OAAO,CAAC,MAAM,EAAE,CAAA;QACjB,CAAC;KACD,CAAC;SACD,SAAS,CAAC,IAAI,CAAC,CAAA;AAClB,CAAC;AAED,+EAA+E;AAC/E,MAAM,iBAAiB,GAAG,IAAI,GAAG,CAAC,CAAC,GAAG,EAAE,GAAG,CAAC,CAAC,CAAA;AAE7C,4EAA4E;AAC5E,SAAS,aAAa,CAAC,EAAU,EAAE,MAA+B;IACjE,MAAM,OAAO,GAAG,WAAW,CAAC,OAAO,CAAC,EAAE,CAAC,CAAA;IACvC,OAAO,MAAM,CAAC,CAAC,CAAC,WAAW,CAAC,GAAG,CAAC,CAAC,OAAO,EAAE,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC,OAAO,CAAA;AAC7D,CAAC;AAmDD;;;GAGG;AACH,MAAM,UAAU,uBAAuB,CAAC,OAAiC;IACxE,MAAM,MAAM,GAAG,OAAO,CAAC,MAAM,IAAI,YAAY,CAAA;IAC7C,MAAM,WAAW,GAAG,OAAO,CAAC,WAAW,IAAI,4BAA4B,CAAA;IACvE,MAAM,MAAM,GAAG,sBAAsB,CAAC;QACrC,WAAW,EAAE,OAAO,CAAC,UAAU;QAC/B,WAAW;QACX,UAAU,EAAE,OAAO,CAAC,UAAU;QAC9B,MAAM;KACN,CAAC,CAAA;IACF,MAAM,UAAU,GAAG,gBAAgB,CAAC,WAAW,CAAC,CAAA;IAChD,MAAM,SAAS,GACd,OAAO,CAAC,aAAa;QACrB,CAAC,CAAC,QAAgB,EAAE,EAAE,CAAC,aAAa,CAAC,QAAQ,EAAE,MAAM,CAAC,CAAC,CAAA;IAExD,KAAK,UAAU,aAAa,CAC3B,GAAQ,EACR,OAA+B,EAC/B,MAAgC;QAEhC,IAAI,MAAM,EAAE,OAAO;YAAE,OAAO,EAAE,IAAI,EAAE,QAAQ,EAAE,SAAS,EAAE,KAAK,EAAE,CAAA;QAChE,IAAI,CAAC,CAAC,MAAM,SAAS,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC,EAAE,CAAC;YACtC,OAAO,EAAE,IAAI,EAAE,SAAS,EAAE,CAAA;QAC3B,CAAC;QACD,IAAI,CAAC;YACJ,wEAAwE;YACxE,0EAA0E;YAC1E,sEAAsE;YACtE,wEAAwE;YACxE,kDAAkD;YAClD,oEAAoE;YACpE,oEAAoE;YACpE,MAAM,IAAI,GAAG,MAAM,UAAU,CAAC,aAAa,CAAC,GAAG,CAAC,QAAQ,EAAE,KAAK,IAAI,EAAE;gBACpE,MAAM,QAAQ,GAAG,MAAM,KAAK,CAAC,GAAG,EAAE;oBACjC,QAAQ,EAAE,QAAQ;oBAClB,MAAM,EAAE,aAAa,CAAC,gBAAgB,EAAE,MAAM,CAAC;oBAC/C,OAAO;iBACP,CAAC,CAAA;gBACF,MAAM,QAAQ,GAAG,aAAa,CAAC,QAAQ,CAAC,GAAG,EAAE,GAAG,CAAC,CAAA;gBACjD,OAAO;oBACN,QAAQ;oBACR,QAAQ;oBACR,IAAI,EAAE,MAAM,cAAc,CAAC,QAAQ,EAAE,QAAQ,EAAE,MAAM,CAAC;iBACtD,CAAA;YACF,CAAC,CAAC,CAAA;YACF,OAAO,EAAE,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,CAAA;QAC9B,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YAChB,MAAM,CAAC,IAAI,CAAC,+BAA+B,EAAE;gBAC5C,GAAG,EAAE,GAAG,CAAC,IAAI;gBACb,GAAG,kBAAkB,CAAC,KAAK,CAAC;aAC5B,CAAC,CAAA;YACF,wEAAwE;YACxE,qDAAqD;YACrD,OAAO,EAAE,IAAI,EAAE,QAAQ,EAAE,SAAS,EAAE,CAAC,YAAY,CAAC,KAAK,CAAC,EAAE,CAAA;QAC3D,CAAC;IACF,CAAC;IAED;;;;;;;;OAQG;IACH,KAAK,UAAU,oBAAoB,CAAC,KAOnC;QACA,MAAM,EAAE,KAAK,EAAE,GAAG,KAAK,CAAA;QACvB,IAAI,CAAC,KAAK,CAAC,UAAU;YAAE,OAAO,EAAE,OAAO,EAAE,aAAa,EAAE,MAAM,EAAE,IAAI,EAAE,CAAA;QACtE,IAAI,KAAK,CAAC,MAAM,EAAE,OAAO;YAAE,OAAO,EAAE,OAAO,EAAE,WAAW,EAAE,MAAM,EAAE,IAAI,EAAE,CAAA;QACxE,IAAI,oBAAoB,GAAG,KAAK,CAAA;QAChC,IAAI,QAAQ,GAAkB,IAAI,CAAA;QAClC,IAAI,KAAK,CAAC,UAAU,IAAI,CAAC,KAAK,CAAC,iBAAiB,IAAI,kBAAkB,CAAC,EAAE,CAAC;YACzE,IAAI,CAAC;gBACJ,QAAQ,GAAG,MAAM,MAAM,CAAC,wBAAwB,CAAC;oBAChD,IAAI,EAAE,KAAK,CAAC,SAAS,KAAK,MAAM,CAAC,CAAC,CAAC,iBAAiB,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,KAAK;oBACnE,QAAQ,EAAE,KAAK,CAAC,QAAQ;oBACxB,QAAQ,EAAE,KAAK,CAAC,QAAQ;oBACxB,MAAM,EAAE,KAAK,CAAC,MAAM;iBACpB,CAAC,CAAA;YACH,CAAC;YAAC,OAAO,KAAK,EAAE,CAAC;gBAChB,IAAI,CAAC,CAAC,KAAK,YAAY,0BAA0B,CAAC;oBAAE,MAAM,KAAK,CAAA;gBAC/D,oBAAoB,GAAG,IAAI,CAAA;YAC5B,CAAC;QACF,CAAC;QACD,MAAM,IAAI,GAAG,qBAAqB,CAAC,QAAQ,IAAI,EAAE,CAAC,CAAA;QAClD,IAAI,CAAC,eAAe,CAAC,IAAI,CAAC;YACzB,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE,MAAM,EAAE,YAAY,EAAE,QAAQ,EAAE,IAAI,EAAE,CAAA;QAC/D,IAAI,KAAK,CAAC,SAAS,KAAK,KAAK,EAAE,CAAC;YAC/B,OAAO;gBACN,OAAO,EAAE,WAAW;gBACpB,MAAM,EAAE,IAAI;gBACZ,YAAY,EAAE,oBAAoB,CAAC,CAAC,CAAC,yBAAyB,CAAC,CAAC,CAAC,IAAI;aACrE,CAAA;QACF,CAAC;QACD,IAAI,KAAK,CAAC,MAAM,EAAE,OAAO;YAAE,OAAO,EAAE,OAAO,EAAE,WAAW,EAAE,MAAM,EAAE,IAAI,EAAE,CAAA;QACxE,IAAI,CAAC;YACJ,IAAI,CAAC,OAAO,CAAC,QAAQ;gBAAE,MAAM,IAAI,KAAK,CAAC,0BAA0B,CAAC,CAAA;YAClE,MAAM,MAAM,GAAG,MAAM,OAAO,CAAC,QAAQ,CAAC;gBACrC,KAAK;gBACL,QAAQ,EAAE,KAAK,CAAC,QAAQ;gBACxB,QAAQ,EAAE,KAAK,CAAC,QAAQ;gBACxB,QAAQ,EAAE,aAAa;gBACvB,MAAM,EAAE,KAAK,CAAC,MAAM;aACpB,CAAC,CAAA;YACF,OAAO;gBACN,OAAO,EAAE,IAAI;gBACb,MAAM,EAAE,iBAAiB;gBACzB,QAAQ,EAAE,MAAM,CAAC,QAAQ;gBACzB,2EAA2E;gBAC3E,0EAA0E;gBAC1E,YAAY,EACX,MAAM,CAAC,QAAQ,KAAK,IAAI,IAAI,MAAM,CAAC,QAAQ,IAAI,aAAa;oBAC3D,CAAC,CAAC,2BAA2B;oBAC7B,CAAC,CAAC,IAAI;aACR,CAAA;QACF,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YAChB,MAAM,CAAC,IAAI,CAAC,kCAAkC,EAAE;gBAC/C,QAAQ,EAAE,KAAK,CAAC,QAAQ;gBACxB,GAAG,kBAAkB,CAAC,KAAK,CAAC;aAC5B,CAAC,CAAA;YACF,OAAO;gBACN,OAAO,EAAE,WAAW;gBACpB,MAAM,EAAE,IAAI;gBACZ,YAAY,EAAE,oBAAoB;oBACjC,CAAC,CAAC,yBAAyB;oBAC3B,CAAC,CAAC,uBAAuB;aAC1B,CAAA;QACF,CAAC;IACF,CAAC;IAED,KAAK,UAAU,eAAe,CAAC,KAK9B;QACA,MAAM,IAAI,GAAG,uBAAuB,CAAC,KAAK,CAAC,WAAW,CAAC,CAAA;QACvD,IAAI,CAAC,IAAI,IAAI,KAAK,CAAC,IAAI,CAAC,IAAI,KAAK,OAAO,EAAE,CAAC;YAC1C,OAAO;gBACN,OAAO,EAAE,aAAa;gBACtB,MAAM,EAAE,IAAI;gBACZ,YAAY,EACX,KAAK,CAAC,IAAI,CAAC,IAAI,KAAK,MAAM,CAAC,CAAC,CAAC,KAAK,CAAC,IAAI,CAAC,YAAY,CAAC,CAAC,CAAC,IAAI;aAC5D,CAAA;QACF,CAAC;QACD,OAAO,oBAAoB,CAAC;YAC3B,KAAK,EAAE,KAAK,CAAC,IAAI,CAAC,KAAK;YACvB,QAAQ,EAAE,gBAAgB,CAAC,KAAK,CAAC,QAAQ,CAAC,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC;YAC/D,QAAQ,EAAE,IAAI,CAAC,QAAQ;YACvB,SAAS,EAAE,IAAI,CAAC,SAAS;YACzB,MAAM,EAAE,KAAK,CAAC,MAAM;SACpB,CAAC,CAAA;IACH,CAAC;IAED,KAAK,UAAU,cAAc,CAAC,KAQ7B;QACA,MAAM,IAAI,GAAG,KAAK,CAAC,IAAI,CAAA;QACvB,sEAAsE;QACtE,4EAA4E;QAC5E,sEAAsE;QACtE,MAAM,MAAM,GACX,KAAK,CAAC,IAAI,KAAK,MAAM;YACrB,CAAC,KAAK,CAAC,IAAI,KAAK,UAAU,IAAI,aAAa,CAAC,IAAI,CAAC,CAAC,CAAA;QAEnD,IAAI,CAAC,MAAM,IAAI,CAAC,eAAe,CAAC,IAAI,CAAC,EAAE,CAAC;YACvC,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE,MAAM,EAAE,qBAAqB,EAAE,QAAQ,EAAE,IAAI,EAAE,CAAA;QACxE,CAAC;QAED,MAAM,IAAI,GAAG,MAAM,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,CAAA;QACjC,yEAAyE;QACzE,yEAAyE;QACzE,IAAI,IAAI,IAAI,KAAK,CAAC,QAAQ,IAAI,QAAQ,CAAC,IAAI,CAAC,KAAK,KAAK,CAAC,QAAQ,EAAE,CAAC;YACjE,OAAO,EAAE,OAAO,EAAE,WAAW,EAAE,MAAM,EAAE,cAAc,EAAE,CAAA;QACxD,CAAC;QACD,IAAI,IAAI,IAAI,MAAM,CAAC,UAAU,CAAC,IAAI,CAAC,GAAG,cAAc,EAAE,CAAC;YACtD,mEAAmE;YACnE,yEAAyE;YACzE,kEAAkE;YAClE,MAAM,CAAC,IAAI,CAAC,gCAAgC,EAAE;gBAC7C,GAAG,EAAE,KAAK,CAAC,QAAQ,CAAC,IAAI;gBACxB,KAAK,EAAE,MAAM,CAAC,UAAU,CAAC,IAAI,CAAC;aAC9B,CAAC,CAAA;YACF,OAAO,EAAE,OAAO,EAAE,aAAa,EAAE,MAAM,EAAE,IAAI,EAAE,IAAI,EAAE,CAAA;QACtD,CAAC;QAED,IAAI,KAAK,GAAG,KAAK,CAAA;QACjB,IAAI,oBAAoB,GAAG,KAAK,CAAA;QAChC,IAAI,IAAI,EAAE,CAAC;YACV,MAAM,OAAO,GAAG,eAAe,CAAC,IAAI,CAAC,CAAA;YACrC,IAAI,SAAS,GAAkB,IAAI,CAAA;YACnC,IAAI,CAAC;gBACJ,SAAS,GAAG,MAAM,MAAM,CAAC,wBAAwB,CAAC;oBACjD,IAAI,EAAE,OAAO;oBACb,yEAAyE;oBACzE,QAAQ,EAAE,KAAK,CAAC,QAAQ,CAAC,QAAQ;oBACjC,MAAM,EAAE,KAAK,CAAC,MAAM;iBACpB,CAAC,CAAA;YACH,CAAC;YAAC,OAAO,KAAK,EAAE,CAAC;gBAChB,IAAI,CAAC,CAAC,KAAK,YAAY,0BAA0B,CAAC;oBAAE,MAAM,KAAK,CAAA;gBAC/D,oBAAoB,GAAG,IAAI,CAAA;YAC5B,CAAC;YACD,IAAI,SAAS,IAAI,CAAC,eAAe,CAAC,SAAS,CAAC,EAAE,CAAC;gBAC9C,KAAK,GAAG,kBAAkB,CAAC,OAAO,EAAE,SAAS,CAAC,CAAA;gBAC9C,IAAI,CAAC,KAAK;oBACT,OAAO;wBACN,OAAO,EAAE,IAAI;wBACb,MAAM,EAAE,YAAY;wBACpB,QAAQ,EAAE,SAAS;wBACnB,IAAI;qBACJ,CAAA;YACH,CAAC;QACF,CAAC;QAED,4EAA4E;QAC5E,yCAAyC;QACzC,IAAI,KAAK,CAAC,MAAM,IAAI,CAAC,KAAK,CAAC,MAAM,EAAE,OAAO,EAAE,CAAC;YAC5C,IAAI,QAAQ,GAAkB,IAAI,CAAA;YAClC,IAAI,CAAC;gBACJ,QAAQ,GAAG,MAAM,MAAM,CAAC,mBAAmB,CAC1C,KAAK,CAAC,QAAQ,CAAC,IAAI,EACnB,KAAK,CAAC,MAAM,CACZ,CAAA;YACF,CAAC;YAAC,OAAO,KAAK,EAAE,CAAC;gBAChB,IAAI,CAAC,CAAC,KAAK,YAAY,0BAA0B,CAAC;oBAAE,MAAM,KAAK,CAAA;gBAC/D,oBAAoB,GAAG,IAAI,CAAA;YAC5B,CAAC;YACD,IACC,QAAQ;gBACR,CAAC,eAAe,CAAC,QAAQ,CAAC;gBAC1B,CAAC,CAAC,IAAI,IAAI,CAAC,kBAAkB,CAAC,IAAI,EAAE,QAAQ,CAAC,CAAC,EAC7C,CAAC;gBACF,OAAO;oBACN,OAAO,EAAE,IAAI;oBACb,MAAM,EAAE,mBAAmB;oBAC3B,QAAQ,EAAE,QAAQ;oBAClB,IAAI;iBACJ,CAAA;YACF,CAAC;QACF,CAAC;QACD,OAAO;YACN,OAAO,EAAE,WAAW;YACpB,MAAM,EAAE,IAAI;YACZ,IAAI;YACJ,YAAY,EAAE,KAAK;gBAClB,CAAC,CAAC,mBAAmB;gBACrB,CAAC,CAAC,oBAAoB;oBACrB,CAAC,CAAC,yBAAyB;oBAC3B,CAAC,CAAC,IAAI;SACR,CAAA;IACF,CAAC;IAED,+EAA+E;IAC/E,KAAK,UAAU,YAAY,CAC1B,KAIC;QAED,MAAM,EAAE,QAAQ,EAAE,IAAI,EAAE,GAAG,KAAK,CAAA;QAChC,MAAM,MAAM,GAAG,QAAQ,CAAC,MAAM,CAAA;QAC9B,MAAM,WAAW,GAAG,QAAQ,CAAC,OAAO,CAAC,GAAG,CAAC,cAAc,CAAC,CAAA;QAExD,IAAI,MAAM,KAAK,GAAG;YAAE,OAAO,EAAE,OAAO,EAAE,WAAW,EAAE,MAAM,EAAE,cAAc,EAAE,CAAA;QAC3E,IAAI,MAAM,KAAK,GAAG,IAAI,MAAM,KAAK,GAAG,EAAE,CAAC;YACtC,OAAO,EAAE,OAAO,EAAE,MAAM,EAAE,MAAM,EAAE,IAAI,EAAE,CAAA;QACzC,CAAC;QACD,yEAAyE;QACzE,sCAAsC;QACtC,IAAI,MAAM,IAAI,GAAG,IAAI,MAAM,KAAK,GAAG;YAClC,OAAO,EAAE,OAAO,EAAE,WAAW,EAAE,MAAM,EAAE,IAAI,EAAE,CAAA;QAC9C,IAAI,MAAM,IAAI,GAAG,EAAE,CAAC;YACnB,IAAI,CAAC,KAAK,CAAC,MAAM,IAAI,CAAC,iBAAiB,CAAC,GAAG,CAAC,MAAM,CAAC,EAAE,CAAC;gBACrD,OAAO,EAAE,OAAO,EAAE,SAAS,EAAE,MAAM,EAAE,IAAI,EAAE,CAAA;YAC5C,CAAC;YACD,MAAM,QAAQ,GAAG,MAAM,MAAM,CAAC,mBAAmB,CAChD,KAAK,CAAC,QAAQ,CAAC,IAAI,EACnB,KAAK,CAAC,MAAM,CACZ,CAAA;YACD,OAAO,QAAQ;gBACd,CAAC,CAAC,EAAE,OAAO,EAAE,IAAI,EAAE,MAAM,EAAE,mBAAmB,EAAE,QAAQ,EAAE,QAAQ,EAAE;gBACpE,CAAC,CAAC,EAAE,OAAO,EAAE,SAAS,EAAE,MAAM,EAAE,IAAI,EAAE,CAAA;QACxC,CAAC;QACD,IAAI,MAAM,GAAG,GAAG,IAAI,MAAM,IAAI,GAAG,EAAE,CAAC;YACnC,OAAO,EAAE,OAAO,EAAE,WAAW,EAAE,MAAM,EAAE,IAAI,EAAE,CAAA;QAC9C,CAAC;QAED,MAAM,IAAI,GAAG,mBAAmB,CAAC,WAAW,CAAC,CAAA;QAC7C,IAAI,IAAI,KAAK,OAAO;YAAE,OAAO,EAAE,OAAO,EAAE,aAAa,EAAE,MAAM,EAAE,IAAI,EAAE,CAAA;QACrE,IAAI,IAAI,KAAK,UAAU,EAAE,CAAC;YACzB,OAAO,eAAe,CAAC;gBACtB,IAAI;gBACJ,WAAW;gBACX,QAAQ,EAAE,KAAK,CAAC,QAAQ;gBACxB,MAAM,EAAE,KAAK,CAAC,MAAM;aACpB,CAAC,CAAA;QACH,CAAC;QACD,IAAI,IAAI,CAAC,IAAI,KAAK,MAAM;YAAE,OAAO,EAAE,OAAO,EAAE,aAAa,EAAE,MAAM,EAAE,IAAI,EAAE,CAAA;QACzE,OAAO,cAAc,CAAC;YACrB,IAAI,EAAE,IAAI,CAAC,IAAI;YACf,IAAI;YACJ,QAAQ,EAAE,KAAK,CAAC,QAAQ;YACxB,MAAM,EAAE,KAAK,CAAC,MAAM;YACpB,QAAQ,EAAE,KAAK,CAAC,QAAQ;YACxB,MAAM,EAAE,KAAK,CAAC,MAAM;SACpB,CAAC,CAAA;IACH,CAAC;IAED;;;;;;;;;OASG;IACH,KAAK,UAAU,mBAAmB,CACjC,KAAqB;QAErB,IAAI,CAAC,oBAAoB,EAAE;YAAE,OAAO,aAAa,CAAC,KAAK,CAAC,CAAA;QAExD,4EAA4E;QAC5E,sEAAsE;QACtE,MAAM,IAAI,GAAG,MAAM,oBAAoB,CACtC,QAAQ,KAAK,CAAC,GAAG,IAAI,KAAK,CAAC,OAAO,IAAI,EAAE,IAAI,KAAK,CAAC,MAAM,IAAI,IAAI,EAAE,EAClE,GAAG,EAAE,CACJ,aAAa,CAAC;YACb,GAAG,EAAE,KAAK,CAAC,GAAG;YACd,OAAO,EAAE,KAAK,CAAC,OAAO;YACtB,MAAM,EAAE,KAAK,CAAC,MAAM;YACpB,MAAM,EAAE,KAAK,CAAC,MAAM;SACpB,CAAC,CACH,CAAA;QACD,MAAM,SAAS,GACd,IAAI,CAAC,OAAO,KAAK,IAAI;YACrB,CAAC,CAAC,KAAK,CAAC,QAAQ,IAAI,IAAI,CAAC,QAAQ,KAAK,KAAK,CAAC,QAAQ,CAAC;gBACpD,CAAC,KAAK,CAAC,IAAI,IAAI,IAAI,CAAC,IAAI,KAAK,KAAK,CAAC,IAAI,CAAC,CAAC,CAAA;QAC3C,IAAI,CAAC,SAAS;YAAE,OAAO,IAAI,CAAA;QAE3B,OAAO;YACN,GAAG,IAAI;YACP,OAAO,EAAE,WAAW;YACpB,MAAM,EAAE,cAAc;YACtB,QAAQ,EAAE,EAAE;YACZ,KAAK,EAAE,IAAI;YACX,IAAI,EAAE,IAAI;YACV,SAAS,EAAE,IAAI;YACf,UAAU,EAAE,EAAE;YACd,kBAAkB,EAAE,EAAE;YACtB,eAAe,EAAE,IAAI;YACrB,MAAM,EAAE,EAAE;YACV,KAAK,EAAE,EAAE;YACT,MAAM,EAAE,EAAE;YACV,YAAY,EAAE,IAAI;YAClB,QAAQ,EAAE,IAAI;SACd,CAAA;IACF,CAAC;IAED,KAAK,UAAU,aAAa,CAAC,KAAqB;QACjD,MAAM,MAAM,GAAG,KAAK,CAAC,MAAM,IAAI,sBAAsB,EAAE,CAAA;QACvD,MAAM,SAAS,GAAG,WAAW,CAAC,GAAG,EAAE,CAAA;QACnC,MAAM,MAAM,GAAG,KAAK,CAAC,MAAM,IAAI,IAAI,CAAA;QAEnC,MAAM,QAAQ,GAAG,iBAAiB,CAAC,KAAK,CAAC,GAAG,CAAC,CAAA;QAC7C,IAAI,CAAC,QAAQ,EAAE,CAAC;YACf,OAAO,SAAS,CACf;gBACC,GAAG,EAAE,KAAK,CAAC,GAAG;gBACd,QAAQ,EAAE,KAAK,CAAC,GAAG;gBACnB,MAAM,EAAE,CAAC;gBACT,WAAW,EAAE,IAAI;gBACjB,IAAI,EAAE,IAAI;gBACV,YAAY,EAAE,IAAI;gBAClB,SAAS;aACT,EACD,EAAE,OAAO,EAAE,aAAa,EAAE,MAAM,EAAE,IAAI,EAAE,EACxC,MAAM,CACN,CAAA;QACF,CAAC;QAED,MAAM,OAAO,GAA2B;YACvC,MAAM,EAAE,2CAA2C;YACnD,iBAAiB,EAAE,GAAG;YACtB,YAAY,EAAE,kBAAkB;SAChC,CAAA;QACD,IAAI,KAAK,CAAC,IAAI;YAAE,OAAO,CAAC,eAAe,CAAC,GAAG,KAAK,CAAC,IAAI,CAAA;QACrD,IAAI,KAAK,CAAC,YAAY;YAAE,OAAO,CAAC,mBAAmB,CAAC,GAAG,KAAK,CAAC,YAAY,CAAA;QAEzE,MAAM,OAAO,GAAG,CAAC,GAAQ,EAAE,EAAE,CAC5B,aAAa,CAAC,QAAQ,CAAC,GAAG,EAAE,KAAK,CAAC,OAAO,CAAC,EAAE,OAAO,EAAE,MAAM,CAAC,CAAA;QAE7D,IAAI,UAAU,GAAG,QAAQ,CAAA;QACzB,IAAI,OAAO,GAAkB,MAAM,OAAO,CAAC,UAAU,CAAC,CAAA;QAEtD,IAAI,OAAO,CAAC,IAAI,KAAK,QAAQ,IAAI,OAAO,CAAC,SAAS,EAAE,CAAC;YACpD,MAAM,QAAQ,GAAG,cAAc,CAAC,UAAU,CAAC,CAAA;YAC3C,IAAI,QAAQ,EAAE,CAAC;gBACd,MAAM,KAAK,GAAG,MAAM,OAAO,CAAC,QAAQ,CAAC,CAAA;gBACrC,IAAI,KAAK,CAAC,IAAI,KAAK,MAAM,EAAE,CAAC;oBAC3B,UAAU,GAAG,QAAQ,CAAA;oBACrB,OAAO,GAAG,KAAK,CAAA;gBAChB,CAAC;YACF,CAAC;QACF,CAAC;QAED,KACC,IAAI,KAAK,GAAG,CAAC,EACb,KAAK,IAAI,sBAAsB;YAC/B,OAAO,CAAC,IAAI,KAAK,MAAM;YACvB,OAAO,CAAC,IAAI,CAAC,QAAQ,CAAC,MAAM,KAAK,GAAG,EACpC,KAAK,EAAE,EACN,CAAC;YACF,MAAM,MAAM,GAAG,eAAe,CAAC,OAAO,CAAC,IAAI,CAAC,QAAQ,EAAE,KAAK,CAAC,CAAA;YAC5D,IAAI,MAAM,UAAU,CAAC,YAAY,CAAC,UAAU,CAAC,QAAQ,EAAE,MAAM,CAAC,EAAE,CAAC;gBAChE,MAAM,CAAC,IAAI,CAAC,mCAAmC,EAAE;oBAChD,IAAI,EAAE,UAAU,CAAC,QAAQ;oBACzB,WAAW,EAAE,OAAO,CAAC,IAAI,CAAC,QAAQ,CAAC,OAAO,CAAC,GAAG,CAAC,aAAa,CAAC,IAAI,EAAE;oBACnE,OAAO,EAAE,MAAM;iBACf,CAAC,CAAA;YACH,CAAC;YACD,OAAO,GAAG,MAAM,OAAO,CAAC,UAAU,CAAC,CAAA;QACpC,CAAC;QAED,IAAI,OAAO,CAAC,IAAI,KAAK,MAAM,EAAE,CAAC;YAC7B,OAAO,SAAS,CACf;gBACC,GAAG,EAAE,UAAU,CAAC,IAAI;gBACpB,QAAQ,EAAE,UAAU,CAAC,IAAI;gBACzB,MAAM,EAAE,CAAC;gBACT,WAAW,EAAE,IAAI;gBACjB,IAAI,EAAE,IAAI;gBACV,YAAY,EAAE,IAAI;gBAClB,SAAS;aACT;YACD,qEAAqE;YACrE;gBACC,OAAO,EAAE,OAAO,CAAC,IAAI,KAAK,SAAS,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,WAAW;gBAC7D,MAAM,EAAE,IAAI;aACZ,EACD,MAAM,CACN,CAAA;QACF,CAAC;QAED,MAAM,EAAE,QAAQ,EAAE,QAAQ,EAAE,GAAG,OAAO,CAAC,IAAI,CAAA;QAC3C,MAAM,WAAW,GAAG,CAAC,MAAiC,EAAE,EAAE,CACzD,QAAQ,CAAC,MAAM,KAAK,GAAG,CAAC,CAAC,CAAC,CAAC,MAAM,IAAI,IAAI,CAAC,CAAC,CAAC,CAAC,IAAI,CAAA;QAClD,MAAM,IAAI,GAAG,MAAM,YAAY,CAAC;YAC/B,GAAG,OAAO,CAAC,IAAI;YACf,MAAM;YACN,QAAQ,EAAE,KAAK,CAAC,QAAQ;YACxB,MAAM;SACN,CAAC,CAAA;QAEF,MAAM,IAAI,GAAG,SAAS,CACrB;YACC,GAAG,EAAE,UAAU,CAAC,IAAI;YACpB,QAAQ,EAAE,QAAQ,CAAC,IAAI;YACvB,MAAM,EAAE,QAAQ,CAAC,MAAM;YACvB,WAAW,EAAE,QAAQ,CAAC,OAAO,CAAC,GAAG,CAAC,cAAc,CAAC;YACjD,uEAAuE;YACvE,wEAAwE;YACxE,2DAA2D;YAC3D,IAAI,EAAE,QAAQ,CAAC,OAAO,CAAC,GAAG,CAAC,MAAM,CAAC,IAAI,WAAW,CAAC,KAAK,CAAC,IAAI,CAAC;YAC7D,YAAY,EACX,QAAQ,CAAC,OAAO,CAAC,GAAG,CAAC,eAAe,CAAC;gBACrC,WAAW,CAAC,KAAK,CAAC,YAAY,CAAC;YAChC,SAAS;SACT,EACD,IAAI,EACJ,MAAM,CACN,CAAA;QAED,MAAM,CAAC,IAAI,CAAC,2BAA2B,EAAE;YACxC,GAAG,EAAE,IAAI,CAAC,QAAQ;YAClB,MAAM,EAAE,IAAI,CAAC,MAAM;YACnB,OAAO,EAAE,IAAI,CAAC,OAAO;YACrB,MAAM,EAAE,IAAI,CAAC,MAAM,IAAI,EAAE;YACzB,KAAK,EAAE,IAAI,CAAC,QAAQ,CAAC,MAAM;YAC3B,WAAW,EAAE,IAAI,CAAC,UAAU;SAC5B,CAAC,CAAA;QACF,OAAO,IAAI,CAAA;IACZ,CAAC;IAED,gDAAgD;IAChD,KAAK,UAAU,aAAa,CAAC,SAAiB;QAC7C,MAAM,IAAI,GAAG,MAAM,mBAAmB,CAAC,EAAE,GAAG,EAAE,SAAS,EAAE,CAAC,CAAA;QAC1D,MAAM,EAAE,QAAQ,EAAE,MAAM,EAAE,GAAG,IAAI,CAAA;QACjC,IAAI,IAAI,CAAC,OAAO,KAAK,IAAI,IAAI,CAAC,QAAQ,IAAI,CAAC,MAAM,EAAE,CAAC;YACnD,MAAM,IAAI,KAAK,CACd,sCAAsC,IAAI,CAAC,OAAO,YAAY,IAAI,CAAC,MAAM,MAAM,SAAS,EAAE,CAC1F,CAAA;QACF,CAAC;QACD,OAAO,EAAE,QAAQ,EAAE,MAAM,EAAE,CAAA;IAC5B,CAAC;IAED,OAAO;QACN,mBAAmB;QACnB,oBAAoB;QACpB,aAAa;QACb,wBAAwB,EAAE,MAAM,CAAC,wBAAwB;QACzD,mBAAmB,EAAE,MAAM,CAAC,mBAAmB;KAC/C,CAAA;AACF,CAAC","sourcesContent":["import type { ApiError } from 'hanzio/api-wrapper'\nimport { parseRetryAfter } from 'hanzio'\nimport {\n\ttype CloudflareCredentials,\n\tCloudflareRateLimitedError,\n\tcreateCloudflareClient\n} from './client.js'\nimport {\n\tclassifyContentType,\n\ttype ContentKind,\n\tconvertibleDocumentType\n} from './content-type.js'\nimport { currentCrawlSnapshot, throughCrawlSnapshot } from './crawl-snapshot.js'\nimport {\n\tisApplicationShell,\n\tisBlankMarkdown,\n\tlooksLikeHtml\n} from './detect-markdown.js'\nimport { sha256Hex } from './hash.js'\nimport { extractHtmlMetadata } from './html-metadata.js'\nimport { type CreateConcurrencyLimit, createHostLimits } from './limit.js'\nimport { errorLogAttributes, type Logger, silentLogger } from './logger.js'\nimport {\n\thtmlImages,\n\thtmlLinks,\n\tmarkdownRefs,\n\tstripDocumentPreamble,\n\tstripFrontMatter,\n\ttitleFromMarkdown\n} from './markdown-refs.js'\nimport { createMemoryConcurrencyLimit } from './memory-limit.js'\nimport { isHostAllowed, isLocalhostHostname } from './net-guard.js'\nimport { currentOperationSignal } from './operation-signal.js'\nimport { BROWSER_USER_AGENT } from './user-agent.js'\nimport { preserveXlsxZeros } from './xlsx-zeros.js'\n\nconst FETCH_TIMEOUT_MS = 30_000\n/** Browser Rendering once returned a 6 MB markdown page from a 6 MB upload. */\nconst MAX_HTML_BYTES = 2 * 1024 * 1024\nconst MAX_DOCUMENT_BYTES = 2 * 1024 * 1024\nconst MAX_PDF_BYTES = 12 * 1024 * 1024\nconst MAX_PDF_PAGES = 60\n\n/** Stored markdown ceiling; a page over this is a listing or an upload, not knowledge. */\nexport const MAX_MARKDOWN_CHARS = 300_000\n\nexport type FetchOutcome =\n\t| 'ok'\n\t| 'unchanged'\n\t| 'gone'\n\t| 'blocked'\n\t| 'transient'\n\t| 'unsupported'\nexport type FetchSource =\n\t| 'content-negotiation'\n\t| 'workers-ai'\n\t| 'browser-rendering'\n\t| 'not-modified'\n\t| 'firecrawl-parse'\n\nexport type FetchedPage = {\n\t/** The URL we requested, after http->https upgrade and www fallback. */\n\turl: string\n\t/** After redirects; may differ in host or path. */\n\tfinalUrl: string\n\t/** HTTP status of the origin response, or 0 when the request never completed. */\n\tstatus: number\n\toutcome: FetchOutcome\n\tsource: FetchSource | null\n\tcontentType: string | null\n\tetag: string | null\n\tlastModified: string | null\n\t/** Empty string unless outcome === 'ok'. Front-matter stripped, capped. */\n\tmarkdown: string\n\t/** <title> when an HTML tier read the page, else the first markdown H1. */\n\ttitle: string | null\n\t/** ISO 639-1 from <html lang>, lowercased, else null. */\n\tlang: string | null\n\t/** <link rel=\"canonical\">, absolute. */\n\tcanonical: string | null\n\t/** `<link rel=\"alternate\" hreflang>` targets, absolute, `x-default` excluded. Optional: many callers build a page from parts. */\n\talternates?: string[]\n\t/** `alternates` by language: URL -> ISO 639-1 (`da` of `da-DK`). */\n\talternateLanguages?: Record<string, string>\n\t/** <meta name=\"description\"> or the og: equivalent. */\n\tmetaDescription: string | null\n\t/** Parsed application/ld+json blocks (any shape; the profile layer reads LocalBusiness/OpeningHoursSpecification). */\n\tjsonLd: unknown[]\n\t/**\n\t * Absolute, deduped link targets: the markdown's, unioned with every\n\t * `<a href>` when an HTML tier read the page. Conversion loses navigation,\n\t * and a link lost here is a page discovery can never reach.\n\t */\n\tlinks: string[]\n\t/** Absolute, deduped image URLs found in the markdown. */\n\timages: string[]\n\tdurationMs: number\n\tcaptureIssue?: string | null | undefined\n\t/**\n\t * Hash of the HTML the origin served, when an origin tier read HTML. Given\n\t * back on the next read (`fetchPageAsMarkdown({ htmlHash })`), identical HTML\n\t * is answered as `unchanged` without converting it again.\n\t */\n\thtmlHash?: string | null | undefined\n}\n\ntype PageBase = {\n\turl: string\n\tfinalUrl: string\n\tstatus: number\n\tcontentType: string | null\n\tetag: string | null\n\tlastModified: string | null\n\tstartedAt: number\n}\n\nexport type PageBody = {\n\tcaptureIssue?: string | null | undefined\n\toutcome: FetchOutcome\n\tsource: FetchSource | null\n\tmarkdown?: string | null | undefined\n\t/** The HTML an origin tier read, when one did - the only source of metadata. */\n\thtml?: string | null | undefined\n}\n\n/**\n * The one place markdown, front-matter and HTML metadata are reconciled: the\n * HTML `<title>` wins, the Workers AI front-matter is the fallback for pages we\n * only ever saw as markdown, and links/images are resolved against `finalUrl`.\n */\nfunction buildPage(\n\tbase: PageBase,\n\trawBody: PageBody,\n\tlogger: Logger\n): FetchedPage {\n\t// A NUL byte is never text, and Postgres refuses a string that holds one:\n\t// a single page with one (a Wix site's conversion) failed the whole import.\n\t// Composed Unicode, because every comparison downstream assumes it: a site\n\t// that wrote \"Gård\" as \"a\" plus a combining ring gave the fact guard the\n\t// name \"Broges Ga\", which no draft could contain, and the document shipped\n\t// as its verbatim source.\n\tconst clean = (text: string | null | undefined) =>\n\t\ttext?.replaceAll('\\u0000', '').normalize('NFC')\n\tconst body = {\n\t\t...rawBody,\n\t\tmarkdown: clean(rawBody.markdown),\n\t\thtml: clean(rawBody.html)\n\t}\n\tconst frontMatter = stripFrontMatter(body.markdown ?? '')\n\tlet markdown = frontMatter.markdown\n\tif (markdown.length > MAX_MARKDOWN_CHARS) {\n\t\tlogger.warn('url-to-markdown.truncated', {\n\t\t\turl: base.finalUrl,\n\t\t\tchars: markdown.length\n\t\t})\n\t\tmarkdown = markdown.slice(0, MAX_MARKDOWN_CHARS)\n\t}\n\n\tconst metadata = body.html\n\t\t? extractHtmlMetadata(body.html, base.finalUrl)\n\t\t: null\n\tconst refs = markdownRefs(markdown, base.finalUrl)\n\t// The markdown is the document; the HTML is the map. Conversion drops\n\t// navigation (image-wrapped anchors, folded menus), so when we hold the\n\t// HTML its anchors decide what discovery can still reach.\n\tconst links = body.html\n\t\t? [...new Set([...refs.links, ...htmlLinks(body.html, base.finalUrl)])]\n\t\t: refs.links\n\n\treturn {\n\t\turl: base.url,\n\t\tfinalUrl: base.finalUrl,\n\t\tstatus: base.status,\n\t\toutcome: body.outcome,\n\t\tsource: body.source,\n\t\tcontentType: base.contentType,\n\t\tetag: base.etag,\n\t\tlastModified: base.lastModified,\n\t\tmarkdown,\n\t\ttitle: metadata?.title ?? frontMatter.title ?? titleFromMarkdown(markdown),\n\t\tlang: metadata?.lang ?? null,\n\t\tcanonical: metadata?.canonical ?? null,\n\t\talternates: metadata?.alternates ?? [],\n\t\talternateLanguages: metadata?.alternateLanguages ?? {},\n\t\tmetaDescription:\n\t\t\tmetadata?.metaDescription ?? frontMatter.description ?? null,\n\t\tjsonLd: metadata?.jsonLd ?? [],\n\t\tlinks,\n\t\timages: [\n\t\t\t...new Set([\n\t\t\t\t...refs.images,\n\t\t\t\t...(body.html ? htmlImages(body.html, base.finalUrl) : [])\n\t\t\t])\n\t\t],\n\t\tcaptureIssue:\n\t\t\tfrontMatter.markdown.length > MAX_MARKDOWN_CHARS\n\t\t\t\t? 'markdown_limit'\n\t\t\t\t: (body.captureIssue ?? null),\n\t\tdurationMs: Math.round(performance.now() - base.startedAt),\n\t\thtmlHash: body.html ? htmlHash(body.html) : null\n\t}\n}\n\n/**\n * `http://` is upgraded rather than rejected: two carpenter sites in the probe\n * link internally with `http://` and the https-only guard threw away most of\n * their pages. Localhost http stays reachable outside production.\n */\nfunction prepareRequestUrl(raw: string) {\n\tlet parsed: URL\n\ttry {\n\t\tparsed = new URL(raw)\n\t} catch {\n\t\treturn null\n\t}\n\tif (parsed.protocol === 'https:') return parsed\n\tif (parsed.protocol !== 'http:') return null\n\tif (isLocalhostHostname(parsed.hostname)) {\n\t\treturn process.env.NODE_ENV === 'production' ? null : parsed\n\t}\n\tparsed.protocol = 'https:'\n\tif (parsed.port === '80') parsed.port = ''\n\treturn parsed\n}\n\n/**\n * `apoteket.dk` answers no TLS handshake at the apex while `www.apoteket.dk`\n * serves the whole site, so a connection failure gets one www retry.\n */\nfunction wwwFallbackUrl(url: URL) {\n\tconst host = url.hostname\n\tif (host.startsWith('www.') || !host.includes('.')) return null\n\tif (isLocalhostHostname(host) || /^[\\d.]+$/.test(host)) return null\n\tconst next = new URL(url.href)\n\tnext.hostname = `www.${host}`\n\treturn next\n}\n\n/**\n * A 429 is an origin asking us to slow down, not a bot wall. Shopify answered\n * 117 of Good Habit's 244 reads with it; sent on to Browser Rendering they\n * tripped Cloudflare's own limit (79 times, 25-30 s each) and 7 came back.\n * So the host is narrowed and paused (`coolDownHost`), the URL is asked again,\n * and a page still refused after that is `transient`: its stored copy stands\n * and the next run reads it.\n */\nconst MAX_RATE_LIMIT_RETRIES = 3\nconst MAX_RATE_LIMIT_WAIT_MS = 30_000\n\n/** The origin's `Retry-After` when it sent one, else 1 s, 2 s, 4 s. */\nfunction rateLimitWaitMs(response: Response, retry: number) {\n\tconst askedMs = parseRetryAfter(response.headers.get('retry-after'))\n\treturn Math.min(MAX_RATE_LIMIT_WAIT_MS, askedMs ?? 1_000 * 2 ** (retry - 1))\n}\n\n/**\n * Bump whenever conversion changes what the same HTML becomes (`withoutHiddenUi`,\n * the Workers AI call, the fallback tiers): the stored hash then no longer\n * matches, and every page is converted again once instead of keeping the\n * markdown the old conversion made.\n */\nconst CONVERSION_VERSION = 1\n\nfunction htmlHash(html: string) {\n\treturn sha256Hex(`${CONVERSION_VERSION}\\n${html}`)\n}\n\n/** The request URL with the market pinned; the URL itself when there is none. */\nfunction inMarket(url: URL, country: string | null | undefined) {\n\tif (!country || url.searchParams.has('country')) return url\n\tconst pinned = new URL(url.href)\n\tpinned.searchParams.set('country', country)\n\treturn pinned\n}\n\n/** `Response.url` is empty on a synthesised response and after some proxies. */\nfunction parseFinalUrl(responseUrl: string, requestUrl: URL) {\n\tif (!responseUrl) return requestUrl\n\ttry {\n\t\treturn new URL(responseUrl)\n\t} catch {\n\t\treturn requestUrl\n\t}\n}\n\nfunction isAbortError(error: unknown) {\n\tconst name =\n\t\ttypeof error === 'object' && error && 'name' in error\n\t\t\t? String(error.name)\n\t\t\t: ''\n\treturn name === 'AbortError' || name === 'TimeoutError'\n}\n\n/** What the origin gave us, already downloaded under the host's politeness slot. */\ntype OriginBody =\n\t| { kind: 'none'; captureIssue?: string }\n\t| { kind: 'text'; text: string }\n\t| { kind: 'bytes'; bytes: ArrayBuffer }\n\ntype OriginRead = { response: Response; finalUrl: URL; body: OriginBody }\n\ntype OriginAttempt =\n\t| { kind: 'read'; read: OriginRead }\n\t| { kind: 'refused' }\n\t| { kind: 'failed'; retryable: boolean }\n\n/** Reading the body is optional; an unsupported type is dropped at the headers. */\nfunction discardBody(response: Response) {\n\tvoid response.body?.cancel().catch(() => {})\n}\n\n/**\n * Decides from the headers alone whether the body is worth downloading, and\n * downloads it when it is. Everything downstream works from bytes already in\n * hand, so nothing past this point needs the origin's slot.\n */\nasync function readOriginBody(\n\tresponse: Response,\n\tfinalUrl: URL,\n\tlogger: Logger\n): Promise<OriginBody> {\n\tconst status = response.status\n\tif (status < 200 || status >= 300) {\n\t\tdiscardBody(response)\n\t\treturn { kind: 'none' }\n\t}\n\n\tconst contentType = response.headers.get('content-type')\n\tconst kind = classifyContentType(contentType)\n\tif (kind === 'asset') {\n\t\tdiscardBody(response)\n\t\treturn { kind: 'none' }\n\t}\n\tif (kind === 'document') {\n\t\tconst type = convertibleDocumentType(contentType)\n\t\tif (!type) {\n\t\t\tdiscardBody(response)\n\t\t\treturn { kind: 'none' }\n\t\t}\n\t\tconst limit = type.extension === 'pdf' ? MAX_PDF_BYTES : MAX_DOCUMENT_BYTES\n\t\tconst declared = Number(response.headers.get('content-length'))\n\t\tif (Number.isFinite(declared) && declared > limit) {\n\t\t\tdiscardBody(response)\n\t\t\tlogger.info('url-to-markdown.document_oversized', {\n\t\t\t\turl: finalUrl.href,\n\t\t\t\tbytes: declared\n\t\t\t})\n\t\t\treturn { kind: 'none', captureIssue: 'document_byte_limit' }\n\t\t}\n\t\tconst bytes = await readBoundedBody(response, limit)\n\t\treturn bytes\n\t\t\t? { kind: 'bytes', bytes }\n\t\t\t: { kind: 'none', captureIssue: 'document_byte_limit' }\n\t}\n\treturn { kind: 'text', text: await response.text() }\n}\n\nfunction documentFilename(finalUrl: string, extension: string) {\n\tconst segment = finalUrl.split(/[?#]/)[0]?.split('/').pop() || 'document'\n\tconst name = segment.slice(0, 80)\n\treturn name.toLowerCase().endsWith(`.${extension}`)\n\t\t? name\n\t\t: `${name}.${extension}`\n}\n\n/** Streaming limits bound chunked bodies as well as declared Content-Length. */\nasync function readBoundedBody(\n\tresponse: Response,\n\tmaxBytes: number\n): Promise<ArrayBuffer | null> {\n\tconst reader = response.body?.getReader()\n\tif (!reader) return new ArrayBuffer(0)\n\tconst chunks: Uint8Array[] = []\n\tlet size = 0\n\twhile (true) {\n\t\tconst next = await reader.read()\n\t\tif (next.done) break\n\t\tsize += next.value.byteLength\n\t\tif (size > maxBytes) {\n\t\t\tawait reader.cancel()\n\t\t\treturn null\n\t\t}\n\t\tchunks.push(next.value)\n\t}\n\tconst bytes = new Uint8Array(size)\n\tlet offset = 0\n\tfor (const chunk of chunks) {\n\t\tbytes.set(chunk, offset)\n\t\toffset += chunk.byteLength\n\t}\n\treturn bytes.buffer\n}\n\n/**\n * Markup a visitor never sees until they open it: `<template>` content and\n * closed `<dialog>`s. A Shopify theme ships its cart drawer (\"Your cart is\n * empty\") and search overlay (a product list with prices) in every page this\n * way; converted, they reached 70 of one store's 89 pages and a price table\n * in seven of its documents (Good Habit audit §3.4). An open dialog is shown\n * and stays. A contact or newsletter form (Shopify posts both to `/contact`)\n * is labels and a button, and \"Telefon Kommentar Send us a note\" read as the\n * name `Kommentar Send` the fact guard then demanded of the document.\n *\n * Plus the form fields a visitor cannot see: an anti-spam honeypot inside a\n * `display:none` block renamed its label (\"Navn\" / \"Url\") on every request,\n * so a carpenter's seven pages hashed as changed and five documents were\n * rewritten every night. Only labels go - a collapsed FAQ answer is hidden the\n * same way and must stay.\n */\nconst HIDDEN_UI_SELECTOR = [\n\t'template',\n\t'dialog:not([open])',\n\t'form[action*=\"/contact\"]',\n\t'[style*=\"display:none\"] label',\n\t'[style*=\"display: none\"] label'\n].join(', ')\n\nfunction withoutHiddenUi(html: string) {\n\treturn new HTMLRewriter()\n\t\t.on(HIDDEN_UI_SELECTOR, {\n\t\t\telement(element) {\n\t\t\t\telement.remove()\n\t\t\t}\n\t\t})\n\t\t.transform(html)\n}\n\n/** A real browser passes these; every other 4xx has nothing for it to find. */\nconst BOT_WALL_STATUSES = new Set([401, 403])\n\n/** `AbortSignal.timeout` and the caller's signal, whichever fires first. */\nfunction timeoutSignal(ms: number, signal: AbortSignal | undefined) {\n\tconst timeout = AbortSignal.timeout(ms)\n\treturn signal ? AbortSignal.any([timeout, signal]) : timeout\n}\n\nexport type FetchPageInput = {\n\turl: string\n\t/** Stored validators from the last run; a 304 returns outcome 'unchanged'. */\n\tetag?: string | null | undefined\n\tlastModified?: string | null | undefined\n\t/** Allow the Browser Rendering tier. Default true. */\n\trender?: boolean | undefined\n\t/**\n\t * ISO country of the market to read the page in. A Shopify store redirects\n\t * by the visitor's IP - our worker reads a Danish shop as its EU market in\n\t * euros and English (Good Habit audit §3.2) - and `?country=DK` pins it.\n\t * Applied to the request only: the page keeps the URL it was asked for.\n\t */\n\tcountry?: string | null | undefined\n\t/** The last read's `htmlHash`; identical HTML comes back `unchanged`. */\n\thtmlHash?: string | null | undefined\n\tsignal?: AbortSignal | undefined\n}\n\nexport type PdfParser = (input: {\n\tbytes: ArrayBuffer\n\tfilename: string\n\tmimeType: string\n\t/** Pages to read at most; a parse that reaches it is marked incomplete. */\n\tmaxPages: number\n\tsignal?: AbortSignal | undefined\n}) => Promise<{ markdown: string; numPages: number | null }>\n\nexport type WebsiteToMarkdownOptions = {\n\t/** Workers AI and Browser Rendering; read on first use, so getters work. */\n\tcloudflare: CloudflareCredentials\n\t/**\n\t * Builds every pool: the two Cloudflare ones and one per origin host.\n\t * Defaults to an in-memory limiter; pass a shared one (Redis) when several\n\t * processes share the Cloudflare quota.\n\t */\n\tcreateLimit?: CreateConcurrencyLimit\n\t/** Workers AI client errors other than 429s, for an error tracker. */\n\tonApiError?: (error: ApiError) => void\n\tlogger?: Logger\n\t/** The OCR tier for a PDF Workers AI returned nothing for. */\n\tparsePdf?: PdfParser\n\t/**\n\t * SSRF guard run before every origin request. Defaults to\n\t * {@link isHostAllowed}: public addresses only, localhost outside production.\n\t */\n\tisHostAllowed?: (hostname: string) => Promise<boolean>\n}\n\n/**\n * One converter: its Cloudflare client, pools and per-host politeness limits.\n * Create it once per process; every call shares the pools.\n */\nexport function createWebsiteToMarkdown(options: WebsiteToMarkdownOptions) {\n\tconst logger = options.logger ?? silentLogger\n\tconst createLimit = options.createLimit ?? createMemoryConcurrencyLimit\n\tconst client = createCloudflareClient({\n\t\tcredentials: options.cloudflare,\n\t\tcreateLimit,\n\t\tonApiError: options.onApiError,\n\t\tlogger\n\t})\n\tconst hostLimits = createHostLimits(createLimit)\n\tconst allowHost =\n\t\toptions.isHostAllowed ??\n\t\t((hostname: string) => isHostAllowed(hostname, logger))\n\n\tasync function requestOrigin(\n\t\turl: URL,\n\t\theaders: Record<string, string>,\n\t\tsignal?: AbortSignal | undefined\n\t): Promise<OriginAttempt> {\n\t\tif (signal?.aborted) return { kind: 'failed', retryable: false }\n\t\tif (!(await allowHost(url.hostname))) {\n\t\t\treturn { kind: 'refused' }\n\t\t}\n\t\ttry {\n\t\t\t// The politeness slot spans the connection *and* the body: freed at the\n\t\t\t// response headers it would bound connections only, and the whole 80-wide\n\t\t\t// page pool could still be streaming from one small WordPress host at\n\t\t\t// once - the burst the cap exists to prevent, at any cap we would pick.\n\t\t\t// The Cloudflare tiers stay outside it - they are\n\t\t\t// our upstreams with their own pools, and holding a customer's slot\n\t\t\t// across a 60s render would stall the crawl for no politeness gain.\n\t\t\tconst read = await hostLimits.withHostLimit(url.hostname, async () => {\n\t\t\t\tconst response = await fetch(url, {\n\t\t\t\t\tredirect: 'follow',\n\t\t\t\t\tsignal: timeoutSignal(FETCH_TIMEOUT_MS, signal),\n\t\t\t\t\theaders\n\t\t\t\t})\n\t\t\t\tconst finalUrl = parseFinalUrl(response.url, url)\n\t\t\t\treturn {\n\t\t\t\t\tresponse,\n\t\t\t\t\tfinalUrl,\n\t\t\t\t\tbody: await readOriginBody(response, finalUrl, logger)\n\t\t\t\t}\n\t\t\t})\n\t\t\treturn { kind: 'read', read }\n\t\t} catch (error) {\n\t\t\tlogger.warn('url-to-markdown.origin_failed', {\n\t\t\t\turl: url.href,\n\t\t\t\t...errorLogAttributes(error)\n\t\t\t})\n\t\t\t// A body that dies mid-stream is the same class of failure as a refused\n\t\t\t// connection, and the same www retry is worth a try.\n\t\t\treturn { kind: 'failed', retryable: !isAbortError(error) }\n\t\t}\n\t}\n\n\t/**\n\t * Bytes of a document to markdown: Workers AI `toMarkdown` first, the\n\t * `parsePdf` tier (e.g. Firecrawl: text-first, OCR fallback) for a PDF Workers\n\t * AI returned nothing for. Shared by the crawler (a linked terms PDF) and file\n\t * uploads (the same PDF dropped in by hand), so both read a file the same\n\t * way. `workersAiMaxBytes` is the caller's budget: the crawler keeps it at\n\t * {@link MAX_DOCUMENT_BYTES}; an upload of a 50 MB scanned catalogue may go\n\t * straight to the OCR tier.\n\t */\n\tasync function convertDocumentBytes(input: {\n\t\tbytes: ArrayBuffer\n\t\tfilename: string\n\t\tmimeType: string\n\t\textension: string\n\t\tworkersAiMaxBytes?: number | undefined\n\t\tsignal?: AbortSignal | undefined\n\t}): Promise<PageBody> {\n\t\tconst { bytes } = input\n\t\tif (!bytes.byteLength) return { outcome: 'unsupported', source: null }\n\t\tif (input.signal?.aborted) return { outcome: 'transient', source: null }\n\t\tlet workersAiRateLimited = false\n\t\tlet markdown: string | null = null\n\t\tif (bytes.byteLength <= (input.workersAiMaxBytes ?? MAX_DOCUMENT_BYTES)) {\n\t\t\ttry {\n\t\t\t\tmarkdown = await client.convertFileWithWorkersAi({\n\t\t\t\t\tbody: input.extension === 'xlsx' ? preserveXlsxZeros(bytes) : bytes,\n\t\t\t\t\tfilename: input.filename,\n\t\t\t\t\tmimeType: input.mimeType,\n\t\t\t\t\tsignal: input.signal\n\t\t\t\t})\n\t\t\t} catch (error) {\n\t\t\t\tif (!(error instanceof CloudflareRateLimitedError)) throw error\n\t\t\t\tworkersAiRateLimited = true\n\t\t\t}\n\t\t}\n\t\tconst body = stripDocumentPreamble(markdown ?? '')\n\t\tif (!isBlankMarkdown(body))\n\t\t\treturn { outcome: 'ok', source: 'workers-ai', markdown: body }\n\t\tif (input.extension !== 'pdf') {\n\t\t\treturn {\n\t\t\t\toutcome: 'transient',\n\t\t\t\tsource: null,\n\t\t\t\tcaptureIssue: workersAiRateLimited ? 'workers_ai_rate_limited' : null\n\t\t\t}\n\t\t}\n\t\tif (input.signal?.aborted) return { outcome: 'transient', source: null }\n\t\ttry {\n\t\t\tif (!options.parsePdf) throw new Error('No PDF parser configured')\n\t\t\tconst parsed = await options.parsePdf({\n\t\t\t\tbytes,\n\t\t\t\tfilename: input.filename,\n\t\t\t\tmimeType: input.mimeType,\n\t\t\t\tmaxPages: MAX_PDF_PAGES,\n\t\t\t\tsignal: input.signal\n\t\t\t})\n\t\t\treturn {\n\t\t\t\toutcome: 'ok',\n\t\t\t\tsource: 'firecrawl-parse',\n\t\t\t\tmarkdown: parsed.markdown,\n\t\t\t\t// The parser may report parsed pages rather than total pages; reaching the\n\t\t\t\t// limit is therefore explicitly incomplete even when the count equals it.\n\t\t\t\tcaptureIssue:\n\t\t\t\t\tparsed.numPages === null || parsed.numPages >= MAX_PDF_PAGES\n\t\t\t\t\t\t? 'pdf_page_limit_unverified'\n\t\t\t\t\t\t: null\n\t\t\t}\n\t\t} catch (error) {\n\t\t\tlogger.warn('url-to-markdown.pdf_parse_failed', {\n\t\t\t\tfilename: input.filename,\n\t\t\t\t...errorLogAttributes(error)\n\t\t\t})\n\t\t\treturn {\n\t\t\t\toutcome: 'transient',\n\t\t\t\tsource: null,\n\t\t\t\tcaptureIssue: workersAiRateLimited\n\t\t\t\t\t? 'workers_ai_rate_limited'\n\t\t\t\t\t: 'pdf_extraction_failed'\n\t\t\t}\n\t\t}\n\t}\n\n\tasync function convertDocument(input: {\n\t\tbody: OriginBody\n\t\tcontentType: string | null\n\t\tfinalUrl: URL\n\t\tsignal?: AbortSignal | undefined\n\t}): Promise<PageBody> {\n\t\tconst type = convertibleDocumentType(input.contentType)\n\t\tif (!type || input.body.kind !== 'bytes') {\n\t\t\treturn {\n\t\t\t\toutcome: 'unsupported',\n\t\t\t\tsource: null,\n\t\t\t\tcaptureIssue:\n\t\t\t\t\tinput.body.kind === 'none' ? input.body.captureIssue : null\n\t\t\t}\n\t\t}\n\t\treturn convertDocumentBytes({\n\t\t\tbytes: input.body.bytes,\n\t\t\tfilename: documentFilename(input.finalUrl.href, type.extension),\n\t\t\tmimeType: type.mimeType,\n\t\t\textension: type.extension,\n\t\t\tsignal: input.signal\n\t\t})\n\t}\n\n\tasync function convertTextual(input: {\n\t\ttext: string\n\t\tkind: ContentKind\n\t\tfinalUrl: URL\n\t\trender: boolean\n\t\t/** The hash of the HTML last read; see `FetchedPage.htmlHash`. */\n\t\thtmlHash?: string | null | undefined\n\t\tsignal?: AbortSignal | undefined\n\t}): Promise<PageBody> {\n\t\tconst body = input.text\n\t\t// The `Content-Type` header is authoritative - sniffing for a leading\n\t\t// `<html` misses every page that opens with a comment or a CDN banner - but\n\t\t// a `text/plain` body that is plainly HTML still needs the HTML tier.\n\t\tconst isHtml =\n\t\t\tinput.kind === 'html' ||\n\t\t\t(input.kind !== 'markdown' && looksLikeHtml(body))\n\n\t\tif (!isHtml && !isBlankMarkdown(body)) {\n\t\t\treturn { outcome: 'ok', source: 'content-negotiation', markdown: body }\n\t\t}\n\n\t\tconst html = isHtml ? body : null\n\t\t// Most origins send no validators, so without this every page of a quiet\n\t\t// site went through Workers AI again every night to learn nothing moved.\n\t\tif (html && input.htmlHash && htmlHash(html) === input.htmlHash) {\n\t\t\treturn { outcome: 'unchanged', source: 'not-modified' }\n\t\t}\n\t\tif (html && Buffer.byteLength(html) > MAX_HTML_BYTES) {\n\t\t\t// A 6 MB HTML upload came back as 6 MB of markdown through Browser\n\t\t\t// Rendering, and Workers AI would bill the same bytes. It is not a page,\n\t\t\t// and it will be just as oversized next run - so not `transient`.\n\t\t\tlogger.info('url-to-markdown.html_oversized', {\n\t\t\t\turl: input.finalUrl.href,\n\t\t\t\tbytes: Buffer.byteLength(html)\n\t\t\t})\n\t\t\treturn { outcome: 'unsupported', source: null, html }\n\t\t}\n\n\t\tlet shell = false\n\t\tlet workersAiRateLimited = false\n\t\tif (html) {\n\t\t\tconst visible = withoutHiddenUi(html)\n\t\t\tlet converted: string | null = null\n\t\t\ttry {\n\t\t\t\tconverted = await client.convertHtmlWithWorkersAi({\n\t\t\t\t\thtml: visible,\n\t\t\t\t\t// The post-redirect host: Cloudflare resolves relative links against it.\n\t\t\t\t\thostname: input.finalUrl.hostname,\n\t\t\t\t\tsignal: input.signal\n\t\t\t\t})\n\t\t\t} catch (error) {\n\t\t\t\tif (!(error instanceof CloudflareRateLimitedError)) throw error\n\t\t\t\tworkersAiRateLimited = true\n\t\t\t}\n\t\t\tif (converted && !isBlankMarkdown(converted)) {\n\t\t\t\tshell = isApplicationShell(visible, converted)\n\t\t\t\tif (!shell)\n\t\t\t\t\treturn {\n\t\t\t\t\t\toutcome: 'ok',\n\t\t\t\t\t\tsource: 'workers-ai',\n\t\t\t\t\t\tmarkdown: converted,\n\t\t\t\t\t\thtml\n\t\t\t\t\t}\n\t\t\t}\n\t\t}\n\n\t\t// Browser Rendering is the exception path: slow, rate limited, and the only\n\t\t// tier that ever timed out in the probe.\n\t\tif (input.render && !input.signal?.aborted) {\n\t\t\tlet rendered: string | null = null\n\t\t\ttry {\n\t\t\t\trendered = await client.renderUrlToMarkdown(\n\t\t\t\t\tinput.finalUrl.href,\n\t\t\t\t\tinput.signal\n\t\t\t\t)\n\t\t\t} catch (error) {\n\t\t\t\tif (!(error instanceof CloudflareRateLimitedError)) throw error\n\t\t\t\tworkersAiRateLimited = true\n\t\t\t}\n\t\t\tif (\n\t\t\t\trendered &&\n\t\t\t\t!isBlankMarkdown(rendered) &&\n\t\t\t\t(!html || !isApplicationShell(html, rendered))\n\t\t\t) {\n\t\t\t\treturn {\n\t\t\t\t\toutcome: 'ok',\n\t\t\t\t\tsource: 'browser-rendering',\n\t\t\t\t\tmarkdown: rendered,\n\t\t\t\t\thtml\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\t\treturn {\n\t\t\toutcome: 'transient',\n\t\t\tsource: null,\n\t\t\thtml,\n\t\t\tcaptureIssue: shell\n\t\t\t\t? 'render_incomplete'\n\t\t\t\t: workersAiRateLimited\n\t\t\t\t\t? 'workers_ai_rate_limited'\n\t\t\t\t\t: null\n\t\t}\n\t}\n\n\t/** Runs after the origin's slot is released: every tier it reaches is ours. */\n\tasync function readResponse(\n\t\tinput: OriginRead & {\n\t\t\trender: boolean\n\t\t\thtmlHash?: string | null | undefined\n\t\t\tsignal?: AbortSignal | undefined\n\t\t}\n\t): Promise<PageBody> {\n\t\tconst { response, body } = input\n\t\tconst status = response.status\n\t\tconst contentType = response.headers.get('content-type')\n\n\t\tif (status === 304) return { outcome: 'unchanged', source: 'not-modified' }\n\t\tif (status === 404 || status === 410) {\n\t\t\treturn { outcome: 'gone', source: null }\n\t\t}\n\t\t// Still refused after the host's pauses: rendering would only be refused\n\t\t// too (see `MAX_RATE_LIMIT_RETRIES`).\n\t\tif (status >= 500 || status === 429)\n\t\t\treturn { outcome: 'transient', source: null }\n\t\tif (status >= 400) {\n\t\t\tif (!input.render || !BOT_WALL_STATUSES.has(status)) {\n\t\t\t\treturn { outcome: 'blocked', source: null }\n\t\t\t}\n\t\t\tconst rendered = await client.renderUrlToMarkdown(\n\t\t\t\tinput.finalUrl.href,\n\t\t\t\tinput.signal\n\t\t\t)\n\t\t\treturn rendered\n\t\t\t\t? { outcome: 'ok', source: 'browser-rendering', markdown: rendered }\n\t\t\t\t: { outcome: 'blocked', source: null }\n\t\t}\n\t\tif (status < 200 || status >= 300) {\n\t\t\treturn { outcome: 'transient', source: null }\n\t\t}\n\n\t\tconst kind = classifyContentType(contentType)\n\t\tif (kind === 'asset') return { outcome: 'unsupported', source: null }\n\t\tif (kind === 'document') {\n\t\t\treturn convertDocument({\n\t\t\t\tbody,\n\t\t\t\tcontentType,\n\t\t\t\tfinalUrl: input.finalUrl,\n\t\t\t\tsignal: input.signal\n\t\t\t})\n\t\t}\n\t\tif (body.kind !== 'text') return { outcome: 'unsupported', source: null }\n\t\treturn convertTextual({\n\t\t\ttext: body.text,\n\t\t\tkind,\n\t\t\tfinalUrl: input.finalUrl,\n\t\t\trender: input.render,\n\t\t\thtmlHash: input.htmlHash,\n\t\t\tsignal: input.signal\n\t\t})\n\t}\n\n\t/**\n\t * Read one URL as markdown through Cloudflare's three tiers - content\n\t * negotiation, Workers AI `toMarkdown`, Browser Rendering - and report\n\t * everything the import pipeline needs about the page: real status, redirect\n\t * target, validators, HTML metadata and absolute links.\n\t *\n\t * Never throws: a page that cannot be read is an outcome, not an exception,\n\t * because the caller is crawling hundreds of URLs at a time. A knowledge bench\n\t * run reads through its crawl snapshot (`crawl-snapshot.ts`).\n\t */\n\tasync function fetchPageAsMarkdown(\n\t\tinput: FetchPageInput\n\t): Promise<FetchedPage> {\n\t\tif (!currentCrawlSnapshot()) return fetchPageLive(input)\n\n\t\t// The snapshot holds the whole page, read without validators; whether it is\n\t\t// \"unchanged\" for this run is decided here, the way the origin would.\n\t\tconst page = await throughCrawlSnapshot(\n\t\t\t`page ${input.url} ${input.country ?? ''} ${input.render ?? true}`,\n\t\t\t() =>\n\t\t\t\tfetchPageLive({\n\t\t\t\t\turl: input.url,\n\t\t\t\t\tcountry: input.country,\n\t\t\t\t\trender: input.render,\n\t\t\t\t\tsignal: input.signal\n\t\t\t\t})\n\t\t)\n\t\tconst unchanged =\n\t\t\tpage.outcome === 'ok' &&\n\t\t\t((input.htmlHash && page.htmlHash === input.htmlHash) ||\n\t\t\t\t(input.etag && page.etag === input.etag))\n\t\tif (!unchanged) return page\n\n\t\treturn {\n\t\t\t...page,\n\t\t\toutcome: 'unchanged',\n\t\t\tsource: 'not-modified',\n\t\t\tmarkdown: '',\n\t\t\ttitle: null,\n\t\t\tlang: null,\n\t\t\tcanonical: null,\n\t\t\talternates: [],\n\t\t\talternateLanguages: {},\n\t\t\tmetaDescription: null,\n\t\t\tjsonLd: [],\n\t\t\tlinks: [],\n\t\t\timages: [],\n\t\t\tcaptureIssue: null,\n\t\t\thtmlHash: null\n\t\t}\n\t}\n\n\tasync function fetchPageLive(input: FetchPageInput): Promise<FetchedPage> {\n\t\tconst signal = input.signal ?? currentOperationSignal()\n\t\tconst startedAt = performance.now()\n\t\tconst render = input.render ?? true\n\n\t\tconst prepared = prepareRequestUrl(input.url)\n\t\tif (!prepared) {\n\t\t\treturn buildPage(\n\t\t\t\t{\n\t\t\t\t\turl: input.url,\n\t\t\t\t\tfinalUrl: input.url,\n\t\t\t\t\tstatus: 0,\n\t\t\t\t\tcontentType: null,\n\t\t\t\t\tetag: null,\n\t\t\t\t\tlastModified: null,\n\t\t\t\t\tstartedAt\n\t\t\t\t},\n\t\t\t\t{ outcome: 'unsupported', source: null },\n\t\t\t\tlogger\n\t\t\t)\n\t\t}\n\n\t\tconst headers: Record<string, string> = {\n\t\t\tAccept: 'text/markdown, text/html;q=0.9, */*;q=0.8',\n\t\t\t'Accept-Language': '*',\n\t\t\t'User-Agent': BROWSER_USER_AGENT\n\t\t}\n\t\tif (input.etag) headers['If-None-Match'] = input.etag\n\t\tif (input.lastModified) headers['If-Modified-Since'] = input.lastModified\n\n\t\tconst request = (url: URL) =>\n\t\t\trequestOrigin(inMarket(url, input.country), headers, signal)\n\n\t\tlet requestUrl = prepared\n\t\tlet attempt: OriginAttempt = await request(requestUrl)\n\n\t\tif (attempt.kind === 'failed' && attempt.retryable) {\n\t\t\tconst fallback = wwwFallbackUrl(requestUrl)\n\t\t\tif (fallback) {\n\t\t\t\tconst retry = await request(fallback)\n\t\t\t\tif (retry.kind === 'read') {\n\t\t\t\t\trequestUrl = fallback\n\t\t\t\t\tattempt = retry\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\n\t\tfor (\n\t\t\tlet retry = 1;\n\t\t\tretry <= MAX_RATE_LIMIT_RETRIES &&\n\t\t\tattempt.kind === 'read' &&\n\t\t\tattempt.read.response.status === 429;\n\t\t\tretry++\n\t\t) {\n\t\t\tconst waitMs = rateLimitWaitMs(attempt.read.response, retry)\n\t\t\tif (await hostLimits.coolDownHost(requestUrl.hostname, waitMs)) {\n\t\t\t\tlogger.warn('url-to-markdown.host_rate_limited', {\n\t\t\t\t\thost: requestUrl.hostname,\n\t\t\t\t\tretry_after: attempt.read.response.headers.get('retry-after') ?? '',\n\t\t\t\t\twait_ms: waitMs\n\t\t\t\t})\n\t\t\t}\n\t\t\tattempt = await request(requestUrl)\n\t\t}\n\n\t\tif (attempt.kind !== 'read') {\n\t\t\treturn buildPage(\n\t\t\t\t{\n\t\t\t\t\turl: requestUrl.href,\n\t\t\t\t\tfinalUrl: requestUrl.href,\n\t\t\t\t\tstatus: 0,\n\t\t\t\t\tcontentType: null,\n\t\t\t\t\tetag: null,\n\t\t\t\t\tlastModified: null,\n\t\t\t\t\tstartedAt\n\t\t\t\t},\n\t\t\t\t// A refused host is a policy decision, not a retryable network blip.\n\t\t\t\t{\n\t\t\t\t\toutcome: attempt.kind === 'refused' ? 'blocked' : 'transient',\n\t\t\t\t\tsource: null\n\t\t\t\t},\n\t\t\t\tlogger\n\t\t\t)\n\t\t}\n\n\t\tconst { response, finalUrl } = attempt.read\n\t\tconst notModified = (stored: string | null | undefined) =>\n\t\t\tresponse.status === 304 ? (stored ?? null) : null\n\t\tconst body = await readResponse({\n\t\t\t...attempt.read,\n\t\t\trender,\n\t\t\thtmlHash: input.htmlHash,\n\t\t\tsignal\n\t\t})\n\n\t\tconst page = buildPage(\n\t\t\t{\n\t\t\t\turl: requestUrl.href,\n\t\t\t\tfinalUrl: finalUrl.href,\n\t\t\t\tstatus: response.status,\n\t\t\t\tcontentType: response.headers.get('content-type'),\n\t\t\t\t// A 304 need not repeat the validators; the stored ones stay valid. On\n\t\t\t\t// any other status only this response's headers count - carrying an old\n\t\t\t\t// ETag onto changed content would freeze the page forever.\n\t\t\t\tetag: response.headers.get('etag') ?? notModified(input.etag),\n\t\t\t\tlastModified:\n\t\t\t\t\tresponse.headers.get('last-modified') ??\n\t\t\t\t\tnotModified(input.lastModified),\n\t\t\t\tstartedAt\n\t\t\t},\n\t\t\tbody,\n\t\t\tlogger\n\t\t)\n\n\t\tlogger.info('url-to-markdown.completed', {\n\t\t\turl: page.finalUrl,\n\t\t\tstatus: page.status,\n\t\t\toutcome: page.outcome,\n\t\t\tsource: page.source ?? '',\n\t\t\tchars: page.markdown.length,\n\t\t\tduration_ms: page.durationMs\n\t\t})\n\t\treturn page\n\t}\n\n\t/** CLI/one-shot shape: markdown or an error. */\n\tasync function urlToMarkdown(urlString: string) {\n\t\tconst page = await fetchPageAsMarkdown({ url: urlString })\n\t\tconst { markdown, source } = page\n\t\tif (page.outcome !== 'ok' || !markdown || !source) {\n\t\t\tthrow new Error(\n\t\t\t\t`Failed to convert URL to markdown (${page.outcome}, status ${page.status}): ${urlString}`\n\t\t\t)\n\t\t}\n\t\treturn { markdown, source }\n\t}\n\n\treturn {\n\t\tfetchPageAsMarkdown,\n\t\tconvertDocumentBytes,\n\t\turlToMarkdown,\n\t\tconvertFileWithWorkersAi: client.convertFileWithWorkersAi,\n\t\trenderUrlToMarkdown: client.renderUrlToMarkdown\n\t}\n}\n\nexport type WebsiteToMarkdown = ReturnType<typeof createWebsiteToMarkdown>\n"]}
+{"version":3,"file":"fetch-page.js","sourceRoot":"","sources":["../src/fetch-page.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,eAAe,EAAE,MAAM,QAAQ,CAAA;AACxC,OAAO,EAEN,0BAA0B,EAC1B,sBAAsB,EACtB,MAAM,aAAa,CAAA;AACpB,OAAO,EACN,mBAAmB,EAEnB,uBAAuB,EACvB,MAAM,mBAAmB,CAAA;AAC1B,OAAO,EAGN,oBAAoB,EACpB,oBAAoB,EACpB,iBAAiB,EACjB,MAAM,qBAAqB,CAAA;AAC5B,OAAO,EACN,kBAAkB,EAClB,eAAe,EACf,aAAa,EACb,MAAM,sBAAsB,CAAA;AAC7B,OAAO,EAAE,SAAS,EAAE,MAAM,WAAW,CAAA;AACrC,OAAO,EAAE,mBAAmB,EAAE,MAAM,oBAAoB,CAAA;AACxD,OAAO,EAA+B,gBAAgB,EAAE,MAAM,YAAY,CAAA;AAC1E,OAAO,EAAE,kBAAkB,EAAe,YAAY,EAAE,MAAM,aAAa,CAAA;AAC3E,OAAO,EACN,UAAU,EACV,SAAS,EACT,YAAY,EACZ,qBAAqB,EACrB,gBAAgB,EAChB,iBAAiB,EACjB,MAAM,oBAAoB,CAAA;AAC3B,OAAO,EAAE,4BAA4B,EAAE,MAAM,mBAAmB,CAAA;AAChE,OAAO,EAAE,aAAa,EAAE,mBAAmB,EAAE,MAAM,gBAAgB,CAAA;AACnE,OAAO,EAAE,sBAAsB,EAAE,MAAM,uBAAuB,CAAA;AAC9D,OAAO,EAAE,kBAAkB,EAAE,MAAM,iBAAiB,CAAA;AACpD,OAAO,EAAE,iBAAiB,EAAE,MAAM,iBAAiB,CAAA;AAEnD,MAAM,gBAAgB,GAAG,MAAM,CAAA;AAC/B,+EAA+E;AAC/E,MAAM,cAAc,GAAG,CAAC,GAAG,IAAI,GAAG,IAAI,CAAA;AACtC,MAAM,kBAAkB,GAAG,CAAC,GAAG,IAAI,GAAG,IAAI,CAAA;AAC1C,MAAM,aAAa,GAAG,EAAE,GAAG,IAAI,GAAG,IAAI,CAAA;AACtC,MAAM,aAAa,GAAG,EAAE,CAAA;AAExB,0FAA0F;AAC1F,MAAM,CAAC,MAAM,kBAAkB,GAAG,OAAO,CAAA;AAiFzC;;;;GAIG;AACH,SAAS,SAAS,CACjB,IAAc,EACd,OAAiB,EACjB,MAAc;IAEd,0EAA0E;IAC1E,4EAA4E;IAC5E,2EAA2E;IAC3E,yEAAyE;IACzE,2EAA2E;IAC3E,0BAA0B;IAC1B,MAAM,KAAK,GAAG,CAAC,IAA+B,EAAE,EAAE,CACjD,IAAI,EAAE,UAAU,CAAC,QAAQ,EAAE,EAAE,CAAC,CAAC,SAAS,CAAC,KAAK,CAAC,CAAA;IAChD,MAAM,IAAI,GAAG;QACZ,GAAG,OAAO;QACV,QAAQ,EAAE,KAAK,CAAC,OAAO,CAAC,QAAQ,CAAC;QACjC,IAAI,EAAE,KAAK,CAAC,OAAO,CAAC,IAAI,CAAC;KACzB,CAAA;IACD,MAAM,WAAW,GAAG,gBAAgB,CAAC,IAAI,CAAC,QAAQ,IAAI,EAAE,CAAC,CAAA;IACzD,IAAI,QAAQ,GAAG,WAAW,CAAC,QAAQ,CAAA;IACnC,IAAI,QAAQ,CAAC,MAAM,GAAG,kBAAkB,EAAE,CAAC;QAC1C,MAAM,CAAC,IAAI,CAAC,2BAA2B,EAAE;YACxC,GAAG,EAAE,IAAI,CAAC,QAAQ;YAClB,KAAK,EAAE,QAAQ,CAAC,MAAM;SACtB,CAAC,CAAA;QACF,QAAQ,GAAG,QAAQ,CAAC,KAAK,CAAC,CAAC,EAAE,kBAAkB,CAAC,CAAA;IACjD,CAAC;IAED,MAAM,QAAQ,GAAG,IAAI,CAAC,IAAI;QACzB,CAAC,CAAC,mBAAmB,CAAC,IAAI,CAAC,IAAI,EAAE,IAAI,CAAC,QAAQ,CAAC;QAC/C,CAAC,CAAC,IAAI,CAAA;IACP,MAAM,IAAI,GAAG,YAAY,CAAC,QAAQ,EAAE,IAAI,CAAC,QAAQ,CAAC,CAAA;IAClD,sEAAsE;IACtE,wEAAwE;IACxE,0DAA0D;IAC1D,MAAM,KAAK,GAAG,IAAI,CAAC,IAAI;QACtB,CAAC,CAAC,CAAC,GAAG,IAAI,GAAG,CAAC,CAAC,GAAG,IAAI,CAAC,KAAK,EAAE,GAAG,SAAS,CAAC,IAAI,CAAC,IAAI,EAAE,IAAI,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC;QACvE,CAAC,CAAC,IAAI,CAAC,KAAK,CAAA;IAEb,OAAO;QACN,GAAG,EAAE,IAAI,CAAC,GAAG;QACb,QAAQ,EAAE,IAAI,CAAC,QAAQ;QACvB,MAAM,EAAE,IAAI,CAAC,MAAM;QACnB,OAAO,EAAE,IAAI,CAAC,OAAO;QACrB,MAAM,EAAE,IAAI,CAAC,MAAM;QACnB,WAAW,EAAE,IAAI,CAAC,WAAW;QAC7B,IAAI,EAAE,IAAI,CAAC,IAAI;QACf,YAAY,EAAE,IAAI,CAAC,YAAY;QAC/B,QAAQ;QACR,KAAK,EAAE,QAAQ,EAAE,KAAK,IAAI,WAAW,CAAC,KAAK,IAAI,iBAAiB,CAAC,QAAQ,CAAC;QAC1E,IAAI,EAAE,QAAQ,EAAE,IAAI,IAAI,IAAI;QAC5B,SAAS,EAAE,QAAQ,EAAE,SAAS,IAAI,IAAI;QACtC,UAAU,EAAE,QAAQ,EAAE,UAAU,IAAI,EAAE;QACtC,kBAAkB,EAAE,QAAQ,EAAE,kBAAkB,IAAI,EAAE;QACtD,eAAe,EACd,QAAQ,EAAE,eAAe,IAAI,WAAW,CAAC,WAAW,IAAI,IAAI;QAC7D,MAAM,EAAE,QAAQ,EAAE,MAAM,IAAI,EAAE;QAC9B,KAAK;QACL,MAAM,EAAE;YACP,GAAG,IAAI,GAAG,CAAC;gBACV,GAAG,IAAI,CAAC,MAAM;gBACd,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,UAAU,CAAC,IAAI,CAAC,IAAI,EAAE,IAAI,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;aAC1D,CAAC;SACF;QACD,YAAY,EACX,WAAW,CAAC,QAAQ,CAAC,MAAM,GAAG,kBAAkB;YAC/C,CAAC,CAAC,gBAAgB;YAClB,CAAC,CAAC,CAAC,IAAI,CAAC,YAAY,IAAI,IAAI,CAAC;QAC/B,UAAU,EAAE,IAAI,CAAC,KAAK,CAAC,WAAW,CAAC,GAAG,EAAE,GAAG,IAAI,CAAC,SAAS,CAAC;QAC1D,QAAQ,EAAE,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,IAAI;KAChD,CAAA;AACF,CAAC;AAED;;;;GAIG;AACH,SAAS,iBAAiB,CAAC,GAAW;IACrC,IAAI,MAAW,CAAA;IACf,IAAI,CAAC;QACJ,MAAM,GAAG,IAAI,GAAG,CAAC,GAAG,CAAC,CAAA;IACtB,CAAC;IAAC,MAAM,CAAC;QACR,OAAO,IAAI,CAAA;IACZ,CAAC;IACD,IAAI,MAAM,CAAC,QAAQ,KAAK,QAAQ;QAAE,OAAO,MAAM,CAAA;IAC/C,IAAI,MAAM,CAAC,QAAQ,KAAK,OAAO;QAAE,OAAO,IAAI,CAAA;IAC5C,IAAI,mBAAmB,CAAC,MAAM,CAAC,QAAQ,CAAC,EAAE,CAAC;QAC1C,OAAO,OAAO,CAAC,GAAG,CAAC,QAAQ,KAAK,YAAY,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,MAAM,CAAA;IAC7D,CAAC;IACD,MAAM,CAAC,QAAQ,GAAG,QAAQ,CAAA;IAC1B,IAAI,MAAM,CAAC,IAAI,KAAK,IAAI;QAAE,MAAM,CAAC,IAAI,GAAG,EAAE,CAAA;IAC1C,OAAO,MAAM,CAAA;AACd,CAAC;AAED;;;GAGG;AACH,SAAS,cAAc,CAAC,GAAQ;IAC/B,MAAM,IAAI,GAAG,GAAG,CAAC,QAAQ,CAAA;IACzB,IAAI,IAAI,CAAC,UAAU,CAAC,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC;QAAE,OAAO,IAAI,CAAA;IAC/D,IAAI,mBAAmB,CAAC,IAAI,CAAC,IAAI,UAAU,CAAC,IAAI,CAAC,IAAI,CAAC;QAAE,OAAO,IAAI,CAAA;IACnE,MAAM,IAAI,GAAG,IAAI,GAAG,CAAC,GAAG,CAAC,IAAI,CAAC,CAAA;IAC9B,IAAI,CAAC,QAAQ,GAAG,OAAO,IAAI,EAAE,CAAA;IAC7B,OAAO,IAAI,CAAA;AACZ,CAAC;AAED;;;;;;;GAOG;AACH,MAAM,sBAAsB,GAAG,CAAC,CAAA;AAChC,MAAM,sBAAsB,GAAG,MAAM,CAAA;AAErC,uEAAuE;AACvE,SAAS,eAAe,CAAC,QAAkB,EAAE,KAAa;IACzD,MAAM,OAAO,GAAG,eAAe,CAAC,QAAQ,CAAC,OAAO,CAAC,GAAG,CAAC,aAAa,CAAC,CAAC,CAAA;IACpE,OAAO,IAAI,CAAC,GAAG,CAAC,sBAAsB,EAAE,OAAO,IAAI,KAAK,GAAG,CAAC,IAAI,CAAC,KAAK,GAAG,CAAC,CAAC,CAAC,CAAA;AAC7E,CAAC;AAED;;;;;GAKG;AACH,MAAM,kBAAkB,GAAG,CAAC,CAAA;AAE5B,SAAS,QAAQ,CAAC,IAAY;IAC7B,OAAO,SAAS,CAAC,GAAG,kBAAkB,KAAK,IAAI,EAAE,CAAC,CAAA;AACnD,CAAC;AAED,iFAAiF;AACjF,SAAS,QAAQ,CAAC,GAAQ,EAAE,OAAkC;IAC7D,IAAI,CAAC,OAAO,IAAI,GAAG,CAAC,YAAY,CAAC,GAAG,CAAC,SAAS,CAAC;QAAE,OAAO,GAAG,CAAA;IAC3D,MAAM,MAAM,GAAG,IAAI,GAAG,CAAC,GAAG,CAAC,IAAI,CAAC,CAAA;IAChC,MAAM,CAAC,YAAY,CAAC,GAAG,CAAC,SAAS,EAAE,OAAO,CAAC,CAAA;IAC3C,OAAO,MAAM,CAAA;AACd,CAAC;AAED,gFAAgF;AAChF,SAAS,aAAa,CAAC,WAAmB,EAAE,UAAe;IAC1D,IAAI,CAAC,WAAW;QAAE,OAAO,UAAU,CAAA;IACnC,IAAI,CAAC;QACJ,OAAO,IAAI,GAAG,CAAC,WAAW,CAAC,CAAA;IAC5B,CAAC;IAAC,MAAM,CAAC;QACR,OAAO,UAAU,CAAA;IAClB,CAAC;AACF,CAAC;AAED,SAAS,YAAY,CAAC,KAAc;IACnC,MAAM,IAAI,GACT,OAAO,KAAK,KAAK,QAAQ,IAAI,KAAK,IAAI,MAAM,IAAI,KAAK;QACpD,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC;QACpB,CAAC,CAAC,EAAE,CAAA;IACN,OAAO,IAAI,KAAK,YAAY,IAAI,IAAI,KAAK,cAAc,CAAA;AACxD,CAAC;AAeD,mFAAmF;AACnF,SAAS,WAAW,CAAC,QAAkB;IACtC,KAAK,QAAQ,CAAC,IAAI,EAAE,MAAM,EAAE,CAAC,KAAK,CAAC,GAAG,EAAE,GAAE,CAAC,CAAC,CAAA;AAC7C,CAAC;AAED;;;;GAIG;AACH,KAAK,UAAU,cAAc,CAC5B,QAAkB,EAClB,QAAa,EACb,MAAc;IAEd,MAAM,MAAM,GAAG,QAAQ,CAAC,MAAM,CAAA;IAC9B,IAAI,MAAM,GAAG,GAAG,IAAI,MAAM,IAAI,GAAG,EAAE,CAAC;QACnC,WAAW,CAAC,QAAQ,CAAC,CAAA;QACrB,OAAO,EAAE,IAAI,EAAE,MAAM,EAAE,CAAA;IACxB,CAAC;IAED,MAAM,WAAW,GAAG,QAAQ,CAAC,OAAO,CAAC,GAAG,CAAC,cAAc,CAAC,CAAA;IACxD,MAAM,IAAI,GAAG,mBAAmB,CAAC,WAAW,CAAC,CAAA;IAC7C,IAAI,IAAI,KAAK,OAAO,EAAE,CAAC;QACtB,WAAW,CAAC,QAAQ,CAAC,CAAA;QACrB,OAAO,EAAE,IAAI,EAAE,MAAM,EAAE,CAAA;IACxB,CAAC;IACD,IAAI,IAAI,KAAK,UAAU,EAAE,CAAC;QACzB,MAAM,IAAI,GAAG,uBAAuB,CAAC,WAAW,CAAC,CAAA;QACjD,IAAI,CAAC,IAAI,EAAE,CAAC;YACX,WAAW,CAAC,QAAQ,CAAC,CAAA;YACrB,OAAO,EAAE,IAAI,EAAE,MAAM,EAAE,CAAA;QACxB,CAAC;QACD,MAAM,KAAK,GAAG,IAAI,CAAC,SAAS,KAAK,KAAK,CAAC,CAAC,CAAC,aAAa,CAAC,CAAC,CAAC,kBAAkB,CAAA;QAC3E,MAAM,QAAQ,GAAG,MAAM,CAAC,QAAQ,CAAC,OAAO,CAAC,GAAG,CAAC,gBAAgB,CAAC,CAAC,CAAA;QAC/D,IAAI,MAAM,CAAC,QAAQ,CAAC,QAAQ,CAAC,IAAI,QAAQ,GAAG,KAAK,EAAE,CAAC;YACnD,WAAW,CAAC,QAAQ,CAAC,CAAA;YACrB,MAAM,CAAC,IAAI,CAAC,oCAAoC,EAAE;gBACjD,GAAG,EAAE,QAAQ,CAAC,IAAI;gBAClB,KAAK,EAAE,QAAQ;aACf,CAAC,CAAA;YACF,OAAO,EAAE,IAAI,EAAE,MAAM,EAAE,YAAY,EAAE,qBAAqB,EAAE,CAAA;QAC7D,CAAC;QACD,MAAM,KAAK,GAAG,MAAM,eAAe,CAAC,QAAQ,EAAE,KAAK,CAAC,CAAA;QACpD,OAAO,KAAK;YACX,CAAC,CAAC,EAAE,IAAI,EAAE,OAAO,EAAE,KAAK,EAAE;YAC1B,CAAC,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,YAAY,EAAE,qBAAqB,EAAE,CAAA;IACzD,CAAC;IACD,OAAO,EAAE,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,MAAM,QAAQ,CAAC,IAAI,EAAE,EAAE,CAAA;AACrD,CAAC;AAED,SAAS,gBAAgB,CAAC,QAAgB,EAAE,SAAiB;IAC5D,MAAM,OAAO,GAAG,QAAQ,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,EAAE,KAAK,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,IAAI,UAAU,CAAA;IACzE,MAAM,IAAI,GAAG,OAAO,CAAC,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC,CAAA;IACjC,OAAO,IAAI,CAAC,WAAW,EAAE,CAAC,QAAQ,CAAC,IAAI,SAAS,EAAE,CAAC;QAClD,CAAC,CAAC,IAAI;QACN,CAAC,CAAC,GAAG,IAAI,IAAI,SAAS,EAAE,CAAA;AAC1B,CAAC;AAED,gFAAgF;AAChF,KAAK,UAAU,eAAe,CAC7B,QAAkB,EAClB,QAAgB;IAEhB,MAAM,MAAM,GAAG,QAAQ,CAAC,IAAI,EAAE,SAAS,EAAE,CAAA;IACzC,IAAI,CAAC,MAAM;QAAE,OAAO,IAAI,WAAW,CAAC,CAAC,CAAC,CAAA;IACtC,MAAM,MAAM,GAAiB,EAAE,CAAA;IAC/B,IAAI,IAAI,GAAG,CAAC,CAAA;IACZ,OAAO,IAAI,EAAE,CAAC;QACb,MAAM,IAAI,GAAG,MAAM,MAAM,CAAC,IAAI,EAAE,CAAA;QAChC,IAAI,IAAI,CAAC,IAAI;YAAE,MAAK;QACpB,IAAI,IAAI,IAAI,CAAC,KAAK,CAAC,UAAU,CAAA;QAC7B,IAAI,IAAI,GAAG,QAAQ,EAAE,CAAC;YACrB,MAAM,MAAM,CAAC,MAAM,EAAE,CAAA;YACrB,OAAO,IAAI,CAAA;QACZ,CAAC;QACD,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,CAAA;IACxB,CAAC;IACD,MAAM,KAAK,GAAG,IAAI,UAAU,CAAC,IAAI,CAAC,CAAA;IAClC,IAAI,MAAM,GAAG,CAAC,CAAA;IACd,KAAK,MAAM,KAAK,IAAI,MAAM,EAAE,CAAC;QAC5B,KAAK,CAAC,GAAG,CAAC,KAAK,EAAE,MAAM,CAAC,CAAA;QACxB,MAAM,IAAI,KAAK,CAAC,UAAU,CAAA;IAC3B,CAAC;IACD,OAAO,KAAK,CAAC,MAAM,CAAA;AACpB,CAAC;AAED;;;;;;;;;;;;;;;GAeG;AACH,MAAM,kBAAkB,GAAG;IAC1B,UAAU;IACV,oBAAoB;IACpB,0BAA0B;IAC1B,+BAA+B;IAC/B,gCAAgC;CAChC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;AAEZ,SAAS,eAAe,CAAC,IAAY;IACpC,OAAO,IAAI,YAAY,EAAE;SACvB,EAAE,CAAC,kBAAkB,EAAE;QACvB,OAAO,CAAC,OAAO;YACd,OAAO,CAAC,MAAM,EAAE,CAAA;QACjB,CAAC;KACD,CAAC;SACD,SAAS,CAAC,IAAI,CAAC,CAAA;AAClB,CAAC;AAED,+EAA+E;AAC/E,MAAM,iBAAiB,GAAG,IAAI,GAAG,CAAC,CAAC,GAAG,EAAE,GAAG,CAAC,CAAC,CAAA;AAE7C,4EAA4E;AAC5E,SAAS,aAAa,CAAC,EAAU,EAAE,MAA+B;IACjE,MAAM,OAAO,GAAG,WAAW,CAAC,OAAO,CAAC,EAAE,CAAC,CAAA;IACvC,OAAO,MAAM,CAAC,CAAC,CAAC,WAAW,CAAC,GAAG,CAAC,CAAC,OAAO,EAAE,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC,OAAO,CAAA;AAC7D,CAAC;AA8DD;;;GAGG;AACH,MAAM,UAAU,uBAAuB,CAAC,OAAiC;IACxE,MAAM,MAAM,GAAG,OAAO,CAAC,MAAM,IAAI,YAAY,CAAA;IAC7C,MAAM,WAAW,GAAG,OAAO,CAAC,WAAW,IAAI,4BAA4B,CAAA;IACvE,MAAM,MAAM,GAAG,sBAAsB,CAAC;QACrC,WAAW,EAAE,OAAO,CAAC,UAAU;QAC/B,WAAW;QACX,UAAU,EAAE,OAAO,CAAC,UAAU;QAC9B,cAAc,EAAE,OAAO,CAAC,cAAc;QACtC,MAAM;KACN,CAAC,CAAA;IACF,MAAM,UAAU,GAAG,gBAAgB,CAAC,WAAW,CAAC,CAAA;IAChD,MAAM,SAAS,GACd,OAAO,CAAC,aAAa;QACrB,CAAC,CAAC,QAAgB,EAAE,EAAE,CAAC,aAAa,CAAC,QAAQ,EAAE,MAAM,CAAC,CAAC,CAAA;IAExD,KAAK,UAAU,aAAa,CAC3B,GAAQ,EACR,OAA+B,EAC/B,MAAgC;QAEhC,IAAI,MAAM,EAAE,OAAO;YAAE,OAAO,EAAE,IAAI,EAAE,QAAQ,EAAE,SAAS,EAAE,KAAK,EAAE,CAAA;QAChE,IAAI,CAAC,CAAC,MAAM,SAAS,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC,EAAE,CAAC;YACtC,OAAO,EAAE,IAAI,EAAE,SAAS,EAAE,CAAA;QAC3B,CAAC;QACD,IAAI,CAAC;YACJ,wEAAwE;YACxE,0EAA0E;YAC1E,sEAAsE;YACtE,wEAAwE;YACxE,kDAAkD;YAClD,oEAAoE;YACpE,oEAAoE;YACpE,MAAM,IAAI,GAAG,MAAM,UAAU,CAAC,aAAa,CAAC,GAAG,CAAC,QAAQ,EAAE,KAAK,IAAI,EAAE;gBACpE,MAAM,QAAQ,GAAG,MAAM,KAAK,CAAC,GAAG,EAAE;oBACjC,QAAQ,EAAE,QAAQ;oBAClB,MAAM,EAAE,aAAa,CAAC,gBAAgB,EAAE,MAAM,CAAC;oBAC/C,OAAO;iBACP,CAAC,CAAA;gBACF,MAAM,QAAQ,GAAG,aAAa,CAAC,QAAQ,CAAC,GAAG,EAAE,GAAG,CAAC,CAAA;gBACjD,OAAO;oBACN,QAAQ;oBACR,QAAQ;oBACR,IAAI,EAAE,MAAM,cAAc,CAAC,QAAQ,EAAE,QAAQ,EAAE,MAAM,CAAC;iBACtD,CAAA;YACF,CAAC,CAAC,CAAA;YACF,OAAO,EAAE,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,CAAA;QAC9B,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YAChB,MAAM,CAAC,IAAI,CAAC,+BAA+B,EAAE;gBAC5C,GAAG,EAAE,GAAG,CAAC,IAAI;gBACb,GAAG,kBAAkB,CAAC,KAAK,CAAC;aAC5B,CAAC,CAAA;YACF,wEAAwE;YACxE,qDAAqD;YACrD,OAAO,EAAE,IAAI,EAAE,QAAQ,EAAE,SAAS,EAAE,CAAC,YAAY,CAAC,KAAK,CAAC,EAAE,CAAA;QAC3D,CAAC;IACF,CAAC;IAED;;;;;;;;OAQG;IACH,KAAK,UAAU,oBAAoB,CAAC,KAOnC;QACA,MAAM,EAAE,KAAK,EAAE,GAAG,KAAK,CAAA;QACvB,IAAI,CAAC,KAAK,CAAC,UAAU;YAAE,OAAO,EAAE,OAAO,EAAE,aAAa,EAAE,MAAM,EAAE,IAAI,EAAE,CAAA;QACtE,IAAI,KAAK,CAAC,MAAM,EAAE,OAAO;YAAE,OAAO,EAAE,OAAO,EAAE,WAAW,EAAE,MAAM,EAAE,IAAI,EAAE,CAAA;QACxE,IAAI,oBAAoB,GAAG,KAAK,CAAA;QAChC,IAAI,QAAQ,GAAkB,IAAI,CAAA;QAClC,IAAI,KAAK,CAAC,UAAU,IAAI,CAAC,KAAK,CAAC,iBAAiB,IAAI,kBAAkB,CAAC,EAAE,CAAC;YACzE,IAAI,CAAC;gBACJ,QAAQ,GAAG,MAAM,MAAM,CAAC,wBAAwB,CAAC;oBAChD,IAAI,EAAE,KAAK,CAAC,SAAS,KAAK,MAAM,CAAC,CAAC,CAAC,iBAAiB,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,KAAK;oBACnE,QAAQ,EAAE,KAAK,CAAC,QAAQ;oBACxB,QAAQ,EAAE,KAAK,CAAC,QAAQ;oBACxB,MAAM,EAAE,KAAK,CAAC,MAAM;iBACpB,CAAC,CAAA;YACH,CAAC;YAAC,OAAO,KAAK,EAAE,CAAC;gBAChB,IAAI,CAAC,CAAC,KAAK,YAAY,0BAA0B,CAAC;oBAAE,MAAM,KAAK,CAAA;gBAC/D,oBAAoB,GAAG,IAAI,CAAA;YAC5B,CAAC;QACF,CAAC;QACD,MAAM,IAAI,GAAG,qBAAqB,CAAC,QAAQ,IAAI,EAAE,CAAC,CAAA;QAClD,IAAI,CAAC,eAAe,CAAC,IAAI,CAAC;YACzB,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE,MAAM,EAAE,YAAY,EAAE,QAAQ,EAAE,IAAI,EAAE,CAAA;QAC/D,IAAI,KAAK,CAAC,SAAS,KAAK,KAAK,EAAE,CAAC;YAC/B,OAAO;gBACN,OAAO,EAAE,WAAW;gBACpB,MAAM,EAAE,IAAI;gBACZ,YAAY,EAAE,oBAAoB,CAAC,CAAC,CAAC,yBAAyB,CAAC,CAAC,CAAC,IAAI;aACrE,CAAA;QACF,CAAC;QACD,IAAI,KAAK,CAAC,MAAM,EAAE,OAAO;YAAE,OAAO,EAAE,OAAO,EAAE,WAAW,EAAE,MAAM,EAAE,IAAI,EAAE,CAAA;QACxE,IAAI,CAAC;YACJ,IAAI,CAAC,OAAO,CAAC,QAAQ;gBAAE,MAAM,IAAI,KAAK,CAAC,0BAA0B,CAAC,CAAA;YAClE,MAAM,MAAM,GAAG,MAAM,OAAO,CAAC,QAAQ,CAAC;gBACrC,KAAK;gBACL,QAAQ,EAAE,KAAK,CAAC,QAAQ;gBACxB,QAAQ,EAAE,KAAK,CAAC,QAAQ;gBACxB,QAAQ,EAAE,aAAa;gBACvB,MAAM,EAAE,KAAK,CAAC,MAAM;aACpB,CAAC,CAAA;YACF,OAAO;gBACN,OAAO,EAAE,IAAI;gBACb,MAAM,EAAE,iBAAiB;gBACzB,QAAQ,EAAE,MAAM,CAAC,QAAQ;gBACzB,2EAA2E;gBAC3E,0EAA0E;gBAC1E,YAAY,EACX,MAAM,CAAC,QAAQ,KAAK,IAAI,IAAI,MAAM,CAAC,QAAQ,IAAI,aAAa;oBAC3D,CAAC,CAAC,2BAA2B;oBAC7B,CAAC,CAAC,IAAI;aACR,CAAA;QACF,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YAChB,MAAM,CAAC,IAAI,CAAC,kCAAkC,EAAE;gBAC/C,QAAQ,EAAE,KAAK,CAAC,QAAQ;gBACxB,GAAG,kBAAkB,CAAC,KAAK,CAAC;aAC5B,CAAC,CAAA;YACF,OAAO;gBACN,OAAO,EAAE,WAAW;gBACpB,MAAM,EAAE,IAAI;gBACZ,YAAY,EAAE,oBAAoB;oBACjC,CAAC,CAAC,yBAAyB;oBAC3B,CAAC,CAAC,uBAAuB;aAC1B,CAAA;QACF,CAAC;IACF,CAAC;IAED,KAAK,UAAU,eAAe,CAAC,KAK9B;QACA,MAAM,IAAI,GAAG,uBAAuB,CAAC,KAAK,CAAC,WAAW,CAAC,CAAA;QACvD,IAAI,CAAC,IAAI,IAAI,KAAK,CAAC,IAAI,CAAC,IAAI,KAAK,OAAO,EAAE,CAAC;YAC1C,OAAO;gBACN,OAAO,EAAE,aAAa;gBACtB,MAAM,EAAE,IAAI;gBACZ,YAAY,EACX,KAAK,CAAC,IAAI,CAAC,IAAI,KAAK,MAAM,CAAC,CAAC,CAAC,KAAK,CAAC,IAAI,CAAC,YAAY,CAAC,CAAC,CAAC,IAAI;aAC5D,CAAA;QACF,CAAC;QACD,OAAO,oBAAoB,CAAC;YAC3B,KAAK,EAAE,KAAK,CAAC,IAAI,CAAC,KAAK;YACvB,QAAQ,EAAE,gBAAgB,CAAC,KAAK,CAAC,QAAQ,CAAC,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC;YAC/D,QAAQ,EAAE,IAAI,CAAC,QAAQ;YACvB,SAAS,EAAE,IAAI,CAAC,SAAS;YACzB,MAAM,EAAE,KAAK,CAAC,MAAM;SACpB,CAAC,CAAA;IACH,CAAC;IAED,KAAK,UAAU,cAAc,CAAC,KAQ7B;QACA,MAAM,IAAI,GAAG,KAAK,CAAC,IAAI,CAAA;QACvB,sEAAsE;QACtE,4EAA4E;QAC5E,sEAAsE;QACtE,MAAM,MAAM,GACX,KAAK,CAAC,IAAI,KAAK,MAAM;YACrB,CAAC,KAAK,CAAC,IAAI,KAAK,UAAU,IAAI,aAAa,CAAC,IAAI,CAAC,CAAC,CAAA;QAEnD,IAAI,CAAC,MAAM,IAAI,CAAC,eAAe,CAAC,IAAI,CAAC,EAAE,CAAC;YACvC,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE,MAAM,EAAE,qBAAqB,EAAE,QAAQ,EAAE,IAAI,EAAE,CAAA;QACxE,CAAC;QAED,MAAM,IAAI,GAAG,MAAM,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,CAAA;QACjC,yEAAyE;QACzE,yEAAyE;QACzE,IAAI,IAAI,IAAI,KAAK,CAAC,QAAQ,IAAI,QAAQ,CAAC,IAAI,CAAC,KAAK,KAAK,CAAC,QAAQ,EAAE,CAAC;YACjE,OAAO,EAAE,OAAO,EAAE,WAAW,EAAE,MAAM,EAAE,cAAc,EAAE,CAAA;QACxD,CAAC;QACD,IAAI,IAAI,IAAI,MAAM,CAAC,UAAU,CAAC,IAAI,CAAC,GAAG,cAAc,EAAE,CAAC;YACtD,mEAAmE;YACnE,yEAAyE;YACzE,kEAAkE;YAClE,MAAM,CAAC,IAAI,CAAC,gCAAgC,EAAE;gBAC7C,GAAG,EAAE,KAAK,CAAC,QAAQ,CAAC,IAAI;gBACxB,KAAK,EAAE,MAAM,CAAC,UAAU,CAAC,IAAI,CAAC;aAC9B,CAAC,CAAA;YACF,OAAO,EAAE,OAAO,EAAE,aAAa,EAAE,MAAM,EAAE,IAAI,EAAE,IAAI,EAAE,CAAA;QACtD,CAAC;QAED,IAAI,KAAK,GAAG,KAAK,CAAA;QACjB,IAAI,oBAAoB,GAAG,KAAK,CAAA;QAChC,IAAI,IAAI,EAAE,CAAC;YACV,MAAM,OAAO,GAAG,eAAe,CAAC,IAAI,CAAC,CAAA;YACrC,IAAI,SAAS,GAAkB,IAAI,CAAA;YACnC,IAAI,CAAC;gBACJ,SAAS,GAAG,MAAM,MAAM,CAAC,wBAAwB,CAAC;oBACjD,IAAI,EAAE,OAAO;oBACb,yEAAyE;oBACzE,QAAQ,EAAE,KAAK,CAAC,QAAQ,CAAC,QAAQ;oBACjC,MAAM,EAAE,KAAK,CAAC,MAAM;iBACpB,CAAC,CAAA;YACH,CAAC;YAAC,OAAO,KAAK,EAAE,CAAC;gBAChB,IAAI,CAAC,CAAC,KAAK,YAAY,0BAA0B,CAAC;oBAAE,MAAM,KAAK,CAAA;gBAC/D,oBAAoB,GAAG,IAAI,CAAA;YAC5B,CAAC;YACD,IAAI,SAAS,IAAI,CAAC,eAAe,CAAC,SAAS,CAAC,EAAE,CAAC;gBAC9C,KAAK,GAAG,kBAAkB,CAAC,OAAO,EAAE,SAAS,CAAC,CAAA;gBAC9C,IAAI,CAAC,KAAK;oBACT,OAAO;wBACN,OAAO,EAAE,IAAI;wBACb,MAAM,EAAE,YAAY;wBACpB,QAAQ,EAAE,SAAS;wBACnB,IAAI;qBACJ,CAAA;YACH,CAAC;QACF,CAAC;QAED,4EAA4E;QAC5E,yCAAyC;QACzC,IAAI,KAAK,CAAC,MAAM,IAAI,CAAC,KAAK,CAAC,MAAM,EAAE,OAAO,EAAE,CAAC;YAC5C,IAAI,QAAQ,GAAkB,IAAI,CAAA;YAClC,IAAI,CAAC;gBACJ,QAAQ,GAAG,MAAM,MAAM,CAAC,mBAAmB,CAC1C,KAAK,CAAC,QAAQ,CAAC,IAAI,EACnB,KAAK,CAAC,MAAM,CACZ,CAAA;YACF,CAAC;YAAC,OAAO,KAAK,EAAE,CAAC;gBAChB,IAAI,CAAC,CAAC,KAAK,YAAY,0BAA0B,CAAC;oBAAE,MAAM,KAAK,CAAA;gBAC/D,oBAAoB,GAAG,IAAI,CAAA;YAC5B,CAAC;YACD,IACC,QAAQ;gBACR,CAAC,eAAe,CAAC,QAAQ,CAAC;gBAC1B,CAAC,CAAC,IAAI,IAAI,CAAC,kBAAkB,CAAC,IAAI,EAAE,QAAQ,CAAC,CAAC,EAC7C,CAAC;gBACF,OAAO;oBACN,OAAO,EAAE,IAAI;oBACb,MAAM,EAAE,mBAAmB;oBAC3B,QAAQ,EAAE,QAAQ;oBAClB,IAAI;iBACJ,CAAA;YACF,CAAC;QACF,CAAC;QACD,OAAO;YACN,OAAO,EAAE,WAAW;YACpB,MAAM,EAAE,IAAI;YACZ,IAAI;YACJ,YAAY,EAAE,KAAK;gBAClB,CAAC,CAAC,mBAAmB;gBACrB,CAAC,CAAC,oBAAoB;oBACrB,CAAC,CAAC,yBAAyB;oBAC3B,CAAC,CAAC,IAAI;SACR,CAAA;IACF,CAAC;IAED,+EAA+E;IAC/E,KAAK,UAAU,YAAY,CAC1B,KAIC;QAED,MAAM,EAAE,QAAQ,EAAE,IAAI,EAAE,GAAG,KAAK,CAAA;QAChC,MAAM,MAAM,GAAG,QAAQ,CAAC,MAAM,CAAA;QAC9B,MAAM,WAAW,GAAG,QAAQ,CAAC,OAAO,CAAC,GAAG,CAAC,cAAc,CAAC,CAAA;QAExD,IAAI,MAAM,KAAK,GAAG;YAAE,OAAO,EAAE,OAAO,EAAE,WAAW,EAAE,MAAM,EAAE,cAAc,EAAE,CAAA;QAC3E,IAAI,MAAM,KAAK,GAAG,IAAI,MAAM,KAAK,GAAG,EAAE,CAAC;YACtC,OAAO,EAAE,OAAO,EAAE,MAAM,EAAE,MAAM,EAAE,IAAI,EAAE,CAAA;QACzC,CAAC;QACD,yEAAyE;QACzE,sCAAsC;QACtC,IAAI,MAAM,IAAI,GAAG,IAAI,MAAM,KAAK,GAAG;YAClC,OAAO,EAAE,OAAO,EAAE,WAAW,EAAE,MAAM,EAAE,IAAI,EAAE,CAAA;QAC9C,IAAI,MAAM,IAAI,GAAG,EAAE,CAAC;YACnB,IAAI,CAAC,KAAK,CAAC,MAAM,IAAI,CAAC,iBAAiB,CAAC,GAAG,CAAC,MAAM,CAAC,EAAE,CAAC;gBACrD,OAAO,EAAE,OAAO,EAAE,SAAS,EAAE,MAAM,EAAE,IAAI,EAAE,CAAA;YAC5C,CAAC;YACD,MAAM,QAAQ,GAAG,MAAM,MAAM,CAAC,mBAAmB,CAChD,KAAK,CAAC,QAAQ,CAAC,IAAI,EACnB,KAAK,CAAC,MAAM,CACZ,CAAA;YACD,OAAO,QAAQ;gBACd,CAAC,CAAC,EAAE,OAAO,EAAE,IAAI,EAAE,MAAM,EAAE,mBAAmB,EAAE,QAAQ,EAAE,QAAQ,EAAE;gBACpE,CAAC,CAAC,EAAE,OAAO,EAAE,SAAS,EAAE,MAAM,EAAE,IAAI,EAAE,CAAA;QACxC,CAAC;QACD,IAAI,MAAM,GAAG,GAAG,IAAI,MAAM,IAAI,GAAG,EAAE,CAAC;YACnC,OAAO,EAAE,OAAO,EAAE,WAAW,EAAE,MAAM,EAAE,IAAI,EAAE,CAAA;QAC9C,CAAC;QAED,MAAM,IAAI,GAAG,mBAAmB,CAAC,WAAW,CAAC,CAAA;QAC7C,IAAI,IAAI,KAAK,OAAO;YAAE,OAAO,EAAE,OAAO,EAAE,aAAa,EAAE,MAAM,EAAE,IAAI,EAAE,CAAA;QACrE,IAAI,IAAI,KAAK,UAAU,EAAE,CAAC;YACzB,OAAO,eAAe,CAAC;gBACtB,IAAI;gBACJ,WAAW;gBACX,QAAQ,EAAE,KAAK,CAAC,QAAQ;gBACxB,MAAM,EAAE,KAAK,CAAC,MAAM;aACpB,CAAC,CAAA;QACH,CAAC;QACD,IAAI,IAAI,CAAC,IAAI,KAAK,MAAM;YAAE,OAAO,EAAE,OAAO,EAAE,aAAa,EAAE,MAAM,EAAE,IAAI,EAAE,CAAA;QACzE,OAAO,cAAc,CAAC;YACrB,IAAI,EAAE,IAAI,CAAC,IAAI;YACf,IAAI;YACJ,QAAQ,EAAE,KAAK,CAAC,QAAQ;YACxB,MAAM,EAAE,KAAK,CAAC,MAAM;YACpB,QAAQ,EAAE,KAAK,CAAC,QAAQ;YACxB,MAAM,EAAE,KAAK,CAAC,MAAM;SACpB,CAAC,CAAA;IACH,CAAC;IAED;;;;;;;;;OASG;IACH,KAAK,UAAU,mBAAmB,CACjC,KAAqB;QAErB,IAAI,CAAC,oBAAoB,EAAE;YAAE,OAAO,aAAa,CAAC,KAAK,CAAC,CAAA;QAExD,4EAA4E;QAC5E,sEAAsE;QACtE,MAAM,IAAI,GAAG,MAAM,oBAAoB,CACtC,QAAQ,KAAK,CAAC,GAAG,IAAI,KAAK,CAAC,OAAO,IAAI,EAAE,IAAI,KAAK,CAAC,MAAM,IAAI,IAAI,EAAE,EAClE,GAAG,EAAE,CACJ,aAAa,CAAC;YACb,GAAG,EAAE,KAAK,CAAC,GAAG;YACd,OAAO,EAAE,KAAK,CAAC,OAAO;YACtB,MAAM,EAAE,KAAK,CAAC,MAAM;YACpB,MAAM,EAAE,KAAK,CAAC,MAAM;SACpB,CAAC,CACH,CAAA;QACD,MAAM,SAAS,GACd,IAAI,CAAC,OAAO,KAAK,IAAI;YACrB,CAAC,CAAC,KAAK,CAAC,QAAQ,IAAI,IAAI,CAAC,QAAQ,KAAK,KAAK,CAAC,QAAQ,CAAC;gBACpD,CAAC,KAAK,CAAC,IAAI,IAAI,IAAI,CAAC,IAAI,KAAK,KAAK,CAAC,IAAI,CAAC,CAAC,CAAA;QAC3C,IAAI,CAAC,SAAS;YAAE,OAAO,IAAI,CAAA;QAE3B,OAAO;YACN,GAAG,IAAI;YACP,OAAO,EAAE,WAAW;YACpB,MAAM,EAAE,cAAc;YACtB,QAAQ,EAAE,EAAE;YACZ,KAAK,EAAE,IAAI;YACX,IAAI,EAAE,IAAI;YACV,SAAS,EAAE,IAAI;YACf,UAAU,EAAE,EAAE;YACd,kBAAkB,EAAE,EAAE;YACtB,eAAe,EAAE,IAAI;YACrB,MAAM,EAAE,EAAE;YACV,KAAK,EAAE,EAAE;YACT,MAAM,EAAE,EAAE;YACV,YAAY,EAAE,IAAI;YAClB,QAAQ,EAAE,IAAI;SACd,CAAA;IACF,CAAC;IAED,KAAK,UAAU,aAAa,CAAC,KAAqB;QACjD,MAAM,MAAM,GAAG,KAAK,CAAC,MAAM,IAAI,sBAAsB,EAAE,CAAA;QACvD,MAAM,SAAS,GAAG,WAAW,CAAC,GAAG,EAAE,CAAA;QACnC,MAAM,MAAM,GAAG,KAAK,CAAC,MAAM,IAAI,IAAI,CAAA;QAEnC,MAAM,QAAQ,GAAG,iBAAiB,CAAC,KAAK,CAAC,GAAG,CAAC,CAAA;QAC7C,IAAI,CAAC,QAAQ,EAAE,CAAC;YACf,OAAO,SAAS,CACf;gBACC,GAAG,EAAE,KAAK,CAAC,GAAG;gBACd,QAAQ,EAAE,KAAK,CAAC,GAAG;gBACnB,MAAM,EAAE,CAAC;gBACT,WAAW,EAAE,IAAI;gBACjB,IAAI,EAAE,IAAI;gBACV,YAAY,EAAE,IAAI;gBAClB,SAAS;aACT,EACD,EAAE,OAAO,EAAE,aAAa,EAAE,MAAM,EAAE,IAAI,EAAE,EACxC,MAAM,CACN,CAAA;QACF,CAAC;QAED,MAAM,OAAO,GAA2B;YACvC,MAAM,EAAE,2CAA2C;YACnD,iBAAiB,EAAE,GAAG;YACtB,YAAY,EAAE,kBAAkB;SAChC,CAAA;QACD,IAAI,KAAK,CAAC,IAAI;YAAE,OAAO,CAAC,eAAe,CAAC,GAAG,KAAK,CAAC,IAAI,CAAA;QACrD,IAAI,KAAK,CAAC,YAAY;YAAE,OAAO,CAAC,mBAAmB,CAAC,GAAG,KAAK,CAAC,YAAY,CAAA;QAEzE,MAAM,OAAO,GAAG,CAAC,GAAQ,EAAE,EAAE,CAC5B,aAAa,CAAC,QAAQ,CAAC,GAAG,EAAE,KAAK,CAAC,OAAO,CAAC,EAAE,OAAO,EAAE,MAAM,CAAC,CAAA;QAE7D,IAAI,UAAU,GAAG,QAAQ,CAAA;QACzB,IAAI,OAAO,GAAkB,MAAM,OAAO,CAAC,UAAU,CAAC,CAAA;QAEtD,IAAI,OAAO,CAAC,IAAI,KAAK,QAAQ,IAAI,OAAO,CAAC,SAAS,EAAE,CAAC;YACpD,MAAM,QAAQ,GAAG,cAAc,CAAC,UAAU,CAAC,CAAA;YAC3C,IAAI,QAAQ,EAAE,CAAC;gBACd,MAAM,KAAK,GAAG,MAAM,OAAO,CAAC,QAAQ,CAAC,CAAA;gBACrC,IAAI,KAAK,CAAC,IAAI,KAAK,MAAM,EAAE,CAAC;oBAC3B,UAAU,GAAG,QAAQ,CAAA;oBACrB,OAAO,GAAG,KAAK,CAAA;gBAChB,CAAC;YACF,CAAC;QACF,CAAC;QAED,KACC,IAAI,KAAK,GAAG,CAAC,EACb,KAAK,IAAI,sBAAsB;YAC/B,OAAO,CAAC,IAAI,KAAK,MAAM;YACvB,OAAO,CAAC,IAAI,CAAC,QAAQ,CAAC,MAAM,KAAK,GAAG,EACpC,KAAK,EAAE,EACN,CAAC;YACF,MAAM,MAAM,GAAG,eAAe,CAAC,OAAO,CAAC,IAAI,CAAC,QAAQ,EAAE,KAAK,CAAC,CAAA;YAC5D,IAAI,MAAM,UAAU,CAAC,YAAY,CAAC,UAAU,CAAC,QAAQ,EAAE,MAAM,CAAC,EAAE,CAAC;gBAChE,MAAM,CAAC,IAAI,CAAC,mCAAmC,EAAE;oBAChD,IAAI,EAAE,UAAU,CAAC,QAAQ;oBACzB,WAAW,EAAE,OAAO,CAAC,IAAI,CAAC,QAAQ,CAAC,OAAO,CAAC,GAAG,CAAC,aAAa,CAAC,IAAI,EAAE;oBACnE,OAAO,EAAE,MAAM;iBACf,CAAC,CAAA;YACH,CAAC;YACD,OAAO,GAAG,MAAM,OAAO,CAAC,UAAU,CAAC,CAAA;QACpC,CAAC;QAED,IAAI,OAAO,CAAC,IAAI,KAAK,MAAM,EAAE,CAAC;YAC7B,OAAO,SAAS,CACf;gBACC,GAAG,EAAE,UAAU,CAAC,IAAI;gBACpB,QAAQ,EAAE,UAAU,CAAC,IAAI;gBACzB,MAAM,EAAE,CAAC;gBACT,WAAW,EAAE,IAAI;gBACjB,IAAI,EAAE,IAAI;gBACV,YAAY,EAAE,IAAI;gBAClB,SAAS;aACT;YACD,qEAAqE;YACrE;gBACC,OAAO,EAAE,OAAO,CAAC,IAAI,KAAK,SAAS,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,WAAW;gBAC7D,MAAM,EAAE,IAAI;aACZ,EACD,MAAM,CACN,CAAA;QACF,CAAC;QAED,MAAM,EAAE,QAAQ,EAAE,QAAQ,EAAE,GAAG,OAAO,CAAC,IAAI,CAAA;QAC3C,MAAM,WAAW,GAAG,CAAC,MAAiC,EAAE,EAAE,CACzD,QAAQ,CAAC,MAAM,KAAK,GAAG,CAAC,CAAC,CAAC,CAAC,MAAM,IAAI,IAAI,CAAC,CAAC,CAAC,CAAC,IAAI,CAAA;QAClD,MAAM,IAAI,GAAG,MAAM,YAAY,CAAC;YAC/B,GAAG,OAAO,CAAC,IAAI;YACf,MAAM;YACN,QAAQ,EAAE,KAAK,CAAC,QAAQ;YACxB,MAAM;SACN,CAAC,CAAA;QAEF,MAAM,IAAI,GAAG,SAAS,CACrB;YACC,GAAG,EAAE,UAAU,CAAC,IAAI;YACpB,QAAQ,EAAE,QAAQ,CAAC,IAAI;YACvB,MAAM,EAAE,QAAQ,CAAC,MAAM;YACvB,WAAW,EAAE,QAAQ,CAAC,OAAO,CAAC,GAAG,CAAC,cAAc,CAAC;YACjD,uEAAuE;YACvE,wEAAwE;YACxE,2DAA2D;YAC3D,IAAI,EAAE,QAAQ,CAAC,OAAO,CAAC,GAAG,CAAC,MAAM,CAAC,IAAI,WAAW,CAAC,KAAK,CAAC,IAAI,CAAC;YAC7D,YAAY,EACX,QAAQ,CAAC,OAAO,CAAC,GAAG,CAAC,eAAe,CAAC;gBACrC,WAAW,CAAC,KAAK,CAAC,YAAY,CAAC;YAChC,SAAS;SACT,EACD,IAAI,EACJ,MAAM,CACN,CAAA;QAED,MAAM,CAAC,IAAI,CAAC,2BAA2B,EAAE;YACxC,GAAG,EAAE,IAAI,CAAC,QAAQ;YAClB,MAAM,EAAE,IAAI,CAAC,MAAM;YACnB,OAAO,EAAE,IAAI,CAAC,OAAO;YACrB,MAAM,EAAE,IAAI,CAAC,MAAM,IAAI,EAAE;YACzB,KAAK,EAAE,IAAI,CAAC,QAAQ,CAAC,MAAM;YAC3B,WAAW,EAAE,IAAI,CAAC,UAAU;SAC5B,CAAC,CAAA;QACF,OAAO,IAAI,CAAA;IACZ,CAAC;IAED,gDAAgD;IAChD,KAAK,UAAU,aAAa,CAAC,SAAiB;QAC7C,MAAM,IAAI,GAAG,MAAM,mBAAmB,CAAC,EAAE,GAAG,EAAE,SAAS,EAAE,CAAC,CAAA;QAC1D,MAAM,EAAE,QAAQ,EAAE,MAAM,EAAE,GAAG,IAAI,CAAA;QACjC,IAAI,IAAI,CAAC,OAAO,KAAK,IAAI,IAAI,CAAC,QAAQ,IAAI,CAAC,MAAM,EAAE,CAAC;YACnD,MAAM,IAAI,KAAK,CACd,sCAAsC,IAAI,CAAC,OAAO,YAAY,IAAI,CAAC,MAAM,MAAM,SAAS,EAAE,CAC1F,CAAA;QACF,CAAC;QACD,OAAO,EAAE,QAAQ,EAAE,MAAM,EAAE,CAAA;IAC5B,CAAC;IAED,OAAO;QACN,mBAAmB;QACnB,oBAAoB;QACpB,aAAa;QACb,wBAAwB,EAAE,MAAM,CAAC,wBAAwB;QACzD,mBAAmB,EAAE,MAAM,CAAC,mBAAmB;QAC/C,gEAAgE;QAChE,aAAa,EAAE,SAAS;QACxB,+DAA+D;QAC/D,iBAAiB,CAChB,KAA4C,EAC5C,GAAqB;YAErB,IAAI,CAAC,KAAK;gBAAE,OAAO,GAAG,EAAE,CAAA;YACxB,IAAI,CAAC,OAAO,CAAC,aAAa,EAAE,CAAC;gBAC5B,MAAM,IAAI,KAAK,CAAC,+CAA+C,CAAC,CAAA;YACjE,CAAC;YACD,OAAO,iBAAiB,CAAC,KAAK,EAAE,GAAG,EAAE;gBACpC,IAAI,EAAE,OAAO,CAAC,aAAa,CAAC,IAAI;gBAChC,MAAM,EAAE,OAAO,CAAC,aAAa,CAAC,MAAM,IAAI,MAAM;aAC9C,CAAC,CAAA;QACH,CAAC;KACD,CAAA;AACF,CAAC","sourcesContent":["import type { ApiError } from 'hanzio/api-wrapper'\nimport { parseRetryAfter } from 'hanzio'\nimport {\n\ttype CloudflareCredentials,\n\tCloudflareRateLimitedError,\n\tcreateCloudflareClient\n} from './client.js'\nimport {\n\tclassifyContentType,\n\ttype ContentKind,\n\tconvertibleDocumentType\n} from './content-type.js'\nimport {\n\ttype CrawlSnapshotInput,\n\ttype CrawlSnapshotOptions,\n\tcurrentCrawlSnapshot,\n\tthroughCrawlSnapshot,\n\twithCrawlSnapshot\n} from './crawl-snapshot.js'\nimport {\n\tisApplicationShell,\n\tisBlankMarkdown,\n\tlooksLikeHtml\n} from './detect-markdown.js'\nimport { sha256Hex } from './hash.js'\nimport { extractHtmlMetadata } from './html-metadata.js'\nimport { type CreateConcurrencyLimit, createHostLimits } from './limit.js'\nimport { errorLogAttributes, type Logger, silentLogger } from './logger.js'\nimport {\n\thtmlImages,\n\thtmlLinks,\n\tmarkdownRefs,\n\tstripDocumentPreamble,\n\tstripFrontMatter,\n\ttitleFromMarkdown\n} from './markdown-refs.js'\nimport { createMemoryConcurrencyLimit } from './memory-limit.js'\nimport { isHostAllowed, isLocalhostHostname } from './net-guard.js'\nimport { currentOperationSignal } from './operation-signal.js'\nimport { BROWSER_USER_AGENT } from './user-agent.js'\nimport { preserveXlsxZeros } from './xlsx-zeros.js'\n\nconst FETCH_TIMEOUT_MS = 30_000\n/** Browser Rendering once returned a 6 MB markdown page from a 6 MB upload. */\nconst MAX_HTML_BYTES = 2 * 1024 * 1024\nconst MAX_DOCUMENT_BYTES = 2 * 1024 * 1024\nconst MAX_PDF_BYTES = 12 * 1024 * 1024\nconst MAX_PDF_PAGES = 60\n\n/** Stored markdown ceiling; a page over this is a listing or an upload, not knowledge. */\nexport const MAX_MARKDOWN_CHARS = 300_000\n\nexport type FetchOutcome =\n\t| 'ok'\n\t| 'unchanged'\n\t| 'gone'\n\t| 'blocked'\n\t| 'transient'\n\t| 'unsupported'\nexport type FetchSource =\n\t| 'content-negotiation'\n\t| 'workers-ai'\n\t| 'browser-rendering'\n\t| 'not-modified'\n\t| 'firecrawl-parse'\n\nexport type FetchedPage = {\n\t/** The URL we requested, after http->https upgrade and www fallback. */\n\turl: string\n\t/** After redirects; may differ in host or path. */\n\tfinalUrl: string\n\t/** HTTP status of the origin response, or 0 when the request never completed. */\n\tstatus: number\n\toutcome: FetchOutcome\n\tsource: FetchSource | null\n\tcontentType: string | null\n\tetag: string | null\n\tlastModified: string | null\n\t/** Empty string unless outcome === 'ok'. Front-matter stripped, capped. */\n\tmarkdown: string\n\t/** <title> when an HTML tier read the page, else the first markdown H1. */\n\ttitle: string | null\n\t/** ISO 639-1 from <html lang>, lowercased, else null. */\n\tlang: string | null\n\t/** <link rel=\"canonical\">, absolute. */\n\tcanonical: string | null\n\t/** `<link rel=\"alternate\" hreflang>` targets, absolute, `x-default` excluded. Optional: many callers build a page from parts. */\n\talternates?: string[]\n\t/** `alternates` by language: URL -> ISO 639-1 (`da` of `da-DK`). */\n\talternateLanguages?: Record<string, string>\n\t/** <meta name=\"description\"> or the og: equivalent. */\n\tmetaDescription: string | null\n\t/** Parsed application/ld+json blocks (any shape; the profile layer reads LocalBusiness/OpeningHoursSpecification). */\n\tjsonLd: unknown[]\n\t/**\n\t * Absolute, deduped link targets: the markdown's, unioned with every\n\t * `<a href>` when an HTML tier read the page. Conversion loses navigation,\n\t * and a link lost here is a page discovery can never reach.\n\t */\n\tlinks: string[]\n\t/** Absolute, deduped image URLs found in the markdown. */\n\timages: string[]\n\tdurationMs: number\n\tcaptureIssue?: string | null | undefined\n\t/**\n\t * Hash of the HTML the origin served, when an origin tier read HTML. Given\n\t * back on the next read (`fetchPageAsMarkdown({ htmlHash })`), identical HTML\n\t * is answered as `unchanged` without converting it again.\n\t */\n\thtmlHash?: string | null | undefined\n}\n\ntype PageBase = {\n\turl: string\n\tfinalUrl: string\n\tstatus: number\n\tcontentType: string | null\n\tetag: string | null\n\tlastModified: string | null\n\tstartedAt: number\n}\n\nexport type PageBody = {\n\tcaptureIssue?: string | null | undefined\n\toutcome: FetchOutcome\n\tsource: FetchSource | null\n\tmarkdown?: string | null | undefined\n\t/** The HTML an origin tier read, when one did - the only source of metadata. */\n\thtml?: string | null | undefined\n}\n\n/**\n * The one place markdown, front-matter and HTML metadata are reconciled: the\n * HTML `<title>` wins, the Workers AI front-matter is the fallback for pages we\n * only ever saw as markdown, and links/images are resolved against `finalUrl`.\n */\nfunction buildPage(\n\tbase: PageBase,\n\trawBody: PageBody,\n\tlogger: Logger\n): FetchedPage {\n\t// A NUL byte is never text, and Postgres refuses a string that holds one:\n\t// a single page with one (a Wix site's conversion) failed the whole import.\n\t// Composed Unicode, because every comparison downstream assumes it: a site\n\t// that wrote \"Gård\" as \"a\" plus a combining ring gave the fact guard the\n\t// name \"Broges Ga\", which no draft could contain, and the document shipped\n\t// as its verbatim source.\n\tconst clean = (text: string | null | undefined) =>\n\t\ttext?.replaceAll('\\u0000', '').normalize('NFC')\n\tconst body = {\n\t\t...rawBody,\n\t\tmarkdown: clean(rawBody.markdown),\n\t\thtml: clean(rawBody.html)\n\t}\n\tconst frontMatter = stripFrontMatter(body.markdown ?? '')\n\tlet markdown = frontMatter.markdown\n\tif (markdown.length > MAX_MARKDOWN_CHARS) {\n\t\tlogger.warn('url-to-markdown.truncated', {\n\t\t\turl: base.finalUrl,\n\t\t\tchars: markdown.length\n\t\t})\n\t\tmarkdown = markdown.slice(0, MAX_MARKDOWN_CHARS)\n\t}\n\n\tconst metadata = body.html\n\t\t? extractHtmlMetadata(body.html, base.finalUrl)\n\t\t: null\n\tconst refs = markdownRefs(markdown, base.finalUrl)\n\t// The markdown is the document; the HTML is the map. Conversion drops\n\t// navigation (image-wrapped anchors, folded menus), so when we hold the\n\t// HTML its anchors decide what discovery can still reach.\n\tconst links = body.html\n\t\t? [...new Set([...refs.links, ...htmlLinks(body.html, base.finalUrl)])]\n\t\t: refs.links\n\n\treturn {\n\t\turl: base.url,\n\t\tfinalUrl: base.finalUrl,\n\t\tstatus: base.status,\n\t\toutcome: body.outcome,\n\t\tsource: body.source,\n\t\tcontentType: base.contentType,\n\t\tetag: base.etag,\n\t\tlastModified: base.lastModified,\n\t\tmarkdown,\n\t\ttitle: metadata?.title ?? frontMatter.title ?? titleFromMarkdown(markdown),\n\t\tlang: metadata?.lang ?? null,\n\t\tcanonical: metadata?.canonical ?? null,\n\t\talternates: metadata?.alternates ?? [],\n\t\talternateLanguages: metadata?.alternateLanguages ?? {},\n\t\tmetaDescription:\n\t\t\tmetadata?.metaDescription ?? frontMatter.description ?? null,\n\t\tjsonLd: metadata?.jsonLd ?? [],\n\t\tlinks,\n\t\timages: [\n\t\t\t...new Set([\n\t\t\t\t...refs.images,\n\t\t\t\t...(body.html ? htmlImages(body.html, base.finalUrl) : [])\n\t\t\t])\n\t\t],\n\t\tcaptureIssue:\n\t\t\tfrontMatter.markdown.length > MAX_MARKDOWN_CHARS\n\t\t\t\t? 'markdown_limit'\n\t\t\t\t: (body.captureIssue ?? null),\n\t\tdurationMs: Math.round(performance.now() - base.startedAt),\n\t\thtmlHash: body.html ? htmlHash(body.html) : null\n\t}\n}\n\n/**\n * `http://` is upgraded rather than rejected: two carpenter sites in the probe\n * link internally with `http://` and the https-only guard threw away most of\n * their pages. Localhost http stays reachable outside production.\n */\nfunction prepareRequestUrl(raw: string) {\n\tlet parsed: URL\n\ttry {\n\t\tparsed = new URL(raw)\n\t} catch {\n\t\treturn null\n\t}\n\tif (parsed.protocol === 'https:') return parsed\n\tif (parsed.protocol !== 'http:') return null\n\tif (isLocalhostHostname(parsed.hostname)) {\n\t\treturn process.env.NODE_ENV === 'production' ? null : parsed\n\t}\n\tparsed.protocol = 'https:'\n\tif (parsed.port === '80') parsed.port = ''\n\treturn parsed\n}\n\n/**\n * `apoteket.dk` answers no TLS handshake at the apex while `www.apoteket.dk`\n * serves the whole site, so a connection failure gets one www retry.\n */\nfunction wwwFallbackUrl(url: URL) {\n\tconst host = url.hostname\n\tif (host.startsWith('www.') || !host.includes('.')) return null\n\tif (isLocalhostHostname(host) || /^[\\d.]+$/.test(host)) return null\n\tconst next = new URL(url.href)\n\tnext.hostname = `www.${host}`\n\treturn next\n}\n\n/**\n * A 429 is an origin asking us to slow down, not a bot wall. Shopify answered\n * 117 of Good Habit's 244 reads with it; sent on to Browser Rendering they\n * tripped Cloudflare's own limit (79 times, 25-30 s each) and 7 came back.\n * So the host is narrowed and paused (`coolDownHost`), the URL is asked again,\n * and a page still refused after that is `transient`: its stored copy stands\n * and the next run reads it.\n */\nconst MAX_RATE_LIMIT_RETRIES = 3\nconst MAX_RATE_LIMIT_WAIT_MS = 30_000\n\n/** The origin's `Retry-After` when it sent one, else 1 s, 2 s, 4 s. */\nfunction rateLimitWaitMs(response: Response, retry: number) {\n\tconst askedMs = parseRetryAfter(response.headers.get('retry-after'))\n\treturn Math.min(MAX_RATE_LIMIT_WAIT_MS, askedMs ?? 1_000 * 2 ** (retry - 1))\n}\n\n/**\n * Bump whenever conversion changes what the same HTML becomes (`withoutHiddenUi`,\n * the Workers AI call, the fallback tiers): the stored hash then no longer\n * matches, and every page is converted again once instead of keeping the\n * markdown the old conversion made.\n */\nconst CONVERSION_VERSION = 1\n\nfunction htmlHash(html: string) {\n\treturn sha256Hex(`${CONVERSION_VERSION}\\n${html}`)\n}\n\n/** The request URL with the market pinned; the URL itself when there is none. */\nfunction inMarket(url: URL, country: string | null | undefined) {\n\tif (!country || url.searchParams.has('country')) return url\n\tconst pinned = new URL(url.href)\n\tpinned.searchParams.set('country', country)\n\treturn pinned\n}\n\n/** `Response.url` is empty on a synthesised response and after some proxies. */\nfunction parseFinalUrl(responseUrl: string, requestUrl: URL) {\n\tif (!responseUrl) return requestUrl\n\ttry {\n\t\treturn new URL(responseUrl)\n\t} catch {\n\t\treturn requestUrl\n\t}\n}\n\nfunction isAbortError(error: unknown) {\n\tconst name =\n\t\ttypeof error === 'object' && error && 'name' in error\n\t\t\t? String(error.name)\n\t\t\t: ''\n\treturn name === 'AbortError' || name === 'TimeoutError'\n}\n\n/** What the origin gave us, already downloaded under the host's politeness slot. */\ntype OriginBody =\n\t| { kind: 'none'; captureIssue?: string }\n\t| { kind: 'text'; text: string }\n\t| { kind: 'bytes'; bytes: ArrayBuffer }\n\ntype OriginRead = { response: Response; finalUrl: URL; body: OriginBody }\n\ntype OriginAttempt =\n\t| { kind: 'read'; read: OriginRead }\n\t| { kind: 'refused' }\n\t| { kind: 'failed'; retryable: boolean }\n\n/** Reading the body is optional; an unsupported type is dropped at the headers. */\nfunction discardBody(response: Response) {\n\tvoid response.body?.cancel().catch(() => {})\n}\n\n/**\n * Decides from the headers alone whether the body is worth downloading, and\n * downloads it when it is. Everything downstream works from bytes already in\n * hand, so nothing past this point needs the origin's slot.\n */\nasync function readOriginBody(\n\tresponse: Response,\n\tfinalUrl: URL,\n\tlogger: Logger\n): Promise<OriginBody> {\n\tconst status = response.status\n\tif (status < 200 || status >= 300) {\n\t\tdiscardBody(response)\n\t\treturn { kind: 'none' }\n\t}\n\n\tconst contentType = response.headers.get('content-type')\n\tconst kind = classifyContentType(contentType)\n\tif (kind === 'asset') {\n\t\tdiscardBody(response)\n\t\treturn { kind: 'none' }\n\t}\n\tif (kind === 'document') {\n\t\tconst type = convertibleDocumentType(contentType)\n\t\tif (!type) {\n\t\t\tdiscardBody(response)\n\t\t\treturn { kind: 'none' }\n\t\t}\n\t\tconst limit = type.extension === 'pdf' ? MAX_PDF_BYTES : MAX_DOCUMENT_BYTES\n\t\tconst declared = Number(response.headers.get('content-length'))\n\t\tif (Number.isFinite(declared) && declared > limit) {\n\t\t\tdiscardBody(response)\n\t\t\tlogger.info('url-to-markdown.document_oversized', {\n\t\t\t\turl: finalUrl.href,\n\t\t\t\tbytes: declared\n\t\t\t})\n\t\t\treturn { kind: 'none', captureIssue: 'document_byte_limit' }\n\t\t}\n\t\tconst bytes = await readBoundedBody(response, limit)\n\t\treturn bytes\n\t\t\t? { kind: 'bytes', bytes }\n\t\t\t: { kind: 'none', captureIssue: 'document_byte_limit' }\n\t}\n\treturn { kind: 'text', text: await response.text() }\n}\n\nfunction documentFilename(finalUrl: string, extension: string) {\n\tconst segment = finalUrl.split(/[?#]/)[0]?.split('/').pop() || 'document'\n\tconst name = segment.slice(0, 80)\n\treturn name.toLowerCase().endsWith(`.${extension}`)\n\t\t? name\n\t\t: `${name}.${extension}`\n}\n\n/** Streaming limits bound chunked bodies as well as declared Content-Length. */\nasync function readBoundedBody(\n\tresponse: Response,\n\tmaxBytes: number\n): Promise<ArrayBuffer | null> {\n\tconst reader = response.body?.getReader()\n\tif (!reader) return new ArrayBuffer(0)\n\tconst chunks: Uint8Array[] = []\n\tlet size = 0\n\twhile (true) {\n\t\tconst next = await reader.read()\n\t\tif (next.done) break\n\t\tsize += next.value.byteLength\n\t\tif (size > maxBytes) {\n\t\t\tawait reader.cancel()\n\t\t\treturn null\n\t\t}\n\t\tchunks.push(next.value)\n\t}\n\tconst bytes = new Uint8Array(size)\n\tlet offset = 0\n\tfor (const chunk of chunks) {\n\t\tbytes.set(chunk, offset)\n\t\toffset += chunk.byteLength\n\t}\n\treturn bytes.buffer\n}\n\n/**\n * Markup a visitor never sees until they open it: `<template>` content and\n * closed `<dialog>`s. A Shopify theme ships its cart drawer (\"Your cart is\n * empty\") and search overlay (a product list with prices) in every page this\n * way; converted, they reached 70 of one store's 89 pages and a price table\n * in seven of its documents (Good Habit audit §3.4). An open dialog is shown\n * and stays. A contact or newsletter form (Shopify posts both to `/contact`)\n * is labels and a button, and \"Telefon Kommentar Send us a note\" read as the\n * name `Kommentar Send` the fact guard then demanded of the document.\n *\n * Plus the form fields a visitor cannot see: an anti-spam honeypot inside a\n * `display:none` block renamed its label (\"Navn\" / \"Url\") on every request,\n * so a carpenter's seven pages hashed as changed and five documents were\n * rewritten every night. Only labels go - a collapsed FAQ answer is hidden the\n * same way and must stay.\n */\nconst HIDDEN_UI_SELECTOR = [\n\t'template',\n\t'dialog:not([open])',\n\t'form[action*=\"/contact\"]',\n\t'[style*=\"display:none\"] label',\n\t'[style*=\"display: none\"] label'\n].join(', ')\n\nfunction withoutHiddenUi(html: string) {\n\treturn new HTMLRewriter()\n\t\t.on(HIDDEN_UI_SELECTOR, {\n\t\t\telement(element) {\n\t\t\t\telement.remove()\n\t\t\t}\n\t\t})\n\t\t.transform(html)\n}\n\n/** A real browser passes these; every other 4xx has nothing for it to find. */\nconst BOT_WALL_STATUSES = new Set([401, 403])\n\n/** `AbortSignal.timeout` and the caller's signal, whichever fires first. */\nfunction timeoutSignal(ms: number, signal: AbortSignal | undefined) {\n\tconst timeout = AbortSignal.timeout(ms)\n\treturn signal ? AbortSignal.any([timeout, signal]) : timeout\n}\n\nexport type FetchPageInput = {\n\turl: string\n\t/** Stored validators from the last run; a 304 returns outcome 'unchanged'. */\n\tetag?: string | null | undefined\n\tlastModified?: string | null | undefined\n\t/** Allow the Browser Rendering tier. Default true. */\n\trender?: boolean | undefined\n\t/**\n\t * ISO country of the market to read the page in. A Shopify store redirects\n\t * by the visitor's IP - our worker reads a Danish shop as its EU market in\n\t * euros and English (Good Habit audit §3.2) - and `?country=DK` pins it.\n\t * Applied to the request only: the page keeps the URL it was asked for.\n\t */\n\tcountry?: string | null | undefined\n\t/** The last read's `htmlHash`; identical HTML comes back `unchanged`. */\n\thtmlHash?: string | null | undefined\n\tsignal?: AbortSignal | undefined\n}\n\nexport type PdfParser = (input: {\n\tbytes: ArrayBuffer\n\tfilename: string\n\tmimeType: string\n\t/** Pages to read at most; a parse that reaches it is marked incomplete. */\n\tmaxPages: number\n\tsignal?: AbortSignal | undefined\n}) => Promise<{ markdown: string; numPages: number | null }>\n\nexport type WebsiteToMarkdownOptions = {\n\t/** Workers AI and Browser Rendering; read on first use, so getters work. */\n\tcloudflare: CloudflareCredentials\n\t/**\n\t * Builds every pool: the two Cloudflare ones and one per origin host.\n\t * Defaults to an in-memory limiter; pass a shared one (Redis) when several\n\t * processes share the Cloudflare quota.\n\t */\n\tcreateLimit?: CreateConcurrencyLimit\n\t/** Workers AI client errors other than 429s, for an error tracker. */\n\tonApiError?: (error: ApiError) => void\n\t/**\n\t * Pass Workers AI timeouts to `onApiError`. A timed-out page falls through\n\t * to the next tier, so `false` keeps import bursts out of error tracking.\n\t * Default true.\n\t */\n\treportTimeouts?: boolean\n\tlogger?: Logger\n\t/**\n\t * Where the returned `withCrawlSnapshot` keeps snapshots. Its logger\n\t * defaults to `logger`.\n\t */\n\tcrawlSnapshot?: CrawlSnapshotOptions\n\t/** The OCR tier for a PDF Workers AI returned nothing for. */\n\tparsePdf?: PdfParser\n\t/**\n\t * SSRF guard run before every origin request. Defaults to\n\t * {@link isHostAllowed}: public addresses only, localhost outside production.\n\t */\n\tisHostAllowed?: (hostname: string) => Promise<boolean>\n}\n\n/**\n * One converter: its Cloudflare client, pools and per-host politeness limits.\n * Create it once per process; every call shares the pools.\n */\nexport function createWebsiteToMarkdown(options: WebsiteToMarkdownOptions) {\n\tconst logger = options.logger ?? silentLogger\n\tconst createLimit = options.createLimit ?? createMemoryConcurrencyLimit\n\tconst client = createCloudflareClient({\n\t\tcredentials: options.cloudflare,\n\t\tcreateLimit,\n\t\tonApiError: options.onApiError,\n\t\treportTimeouts: options.reportTimeouts,\n\t\tlogger\n\t})\n\tconst hostLimits = createHostLimits(createLimit)\n\tconst allowHost =\n\t\toptions.isHostAllowed ??\n\t\t((hostname: string) => isHostAllowed(hostname, logger))\n\n\tasync function requestOrigin(\n\t\turl: URL,\n\t\theaders: Record<string, string>,\n\t\tsignal?: AbortSignal | undefined\n\t): Promise<OriginAttempt> {\n\t\tif (signal?.aborted) return { kind: 'failed', retryable: false }\n\t\tif (!(await allowHost(url.hostname))) {\n\t\t\treturn { kind: 'refused' }\n\t\t}\n\t\ttry {\n\t\t\t// The politeness slot spans the connection *and* the body: freed at the\n\t\t\t// response headers it would bound connections only, and the whole 80-wide\n\t\t\t// page pool could still be streaming from one small WordPress host at\n\t\t\t// once - the burst the cap exists to prevent, at any cap we would pick.\n\t\t\t// The Cloudflare tiers stay outside it - they are\n\t\t\t// our upstreams with their own pools, and holding a customer's slot\n\t\t\t// across a 60s render would stall the crawl for no politeness gain.\n\t\t\tconst read = await hostLimits.withHostLimit(url.hostname, async () => {\n\t\t\t\tconst response = await fetch(url, {\n\t\t\t\t\tredirect: 'follow',\n\t\t\t\t\tsignal: timeoutSignal(FETCH_TIMEOUT_MS, signal),\n\t\t\t\t\theaders\n\t\t\t\t})\n\t\t\t\tconst finalUrl = parseFinalUrl(response.url, url)\n\t\t\t\treturn {\n\t\t\t\t\tresponse,\n\t\t\t\t\tfinalUrl,\n\t\t\t\t\tbody: await readOriginBody(response, finalUrl, logger)\n\t\t\t\t}\n\t\t\t})\n\t\t\treturn { kind: 'read', read }\n\t\t} catch (error) {\n\t\t\tlogger.warn('url-to-markdown.origin_failed', {\n\t\t\t\turl: url.href,\n\t\t\t\t...errorLogAttributes(error)\n\t\t\t})\n\t\t\t// A body that dies mid-stream is the same class of failure as a refused\n\t\t\t// connection, and the same www retry is worth a try.\n\t\t\treturn { kind: 'failed', retryable: !isAbortError(error) }\n\t\t}\n\t}\n\n\t/**\n\t * Bytes of a document to markdown: Workers AI `toMarkdown` first, the\n\t * `parsePdf` tier (e.g. Firecrawl: text-first, OCR fallback) for a PDF Workers\n\t * AI returned nothing for. Shared by the crawler (a linked terms PDF) and file\n\t * uploads (the same PDF dropped in by hand), so both read a file the same\n\t * way. `workersAiMaxBytes` is the caller's budget: the crawler keeps it at\n\t * {@link MAX_DOCUMENT_BYTES}; an upload of a 50 MB scanned catalogue may go\n\t * straight to the OCR tier.\n\t */\n\tasync function convertDocumentBytes(input: {\n\t\tbytes: ArrayBuffer\n\t\tfilename: string\n\t\tmimeType: string\n\t\textension: string\n\t\tworkersAiMaxBytes?: number | undefined\n\t\tsignal?: AbortSignal | undefined\n\t}): Promise<PageBody> {\n\t\tconst { bytes } = input\n\t\tif (!bytes.byteLength) return { outcome: 'unsupported', source: null }\n\t\tif (input.signal?.aborted) return { outcome: 'transient', source: null }\n\t\tlet workersAiRateLimited = false\n\t\tlet markdown: string | null = null\n\t\tif (bytes.byteLength <= (input.workersAiMaxBytes ?? MAX_DOCUMENT_BYTES)) {\n\t\t\ttry {\n\t\t\t\tmarkdown = await client.convertFileWithWorkersAi({\n\t\t\t\t\tbody: input.extension === 'xlsx' ? preserveXlsxZeros(bytes) : bytes,\n\t\t\t\t\tfilename: input.filename,\n\t\t\t\t\tmimeType: input.mimeType,\n\t\t\t\t\tsignal: input.signal\n\t\t\t\t})\n\t\t\t} catch (error) {\n\t\t\t\tif (!(error instanceof CloudflareRateLimitedError)) throw error\n\t\t\t\tworkersAiRateLimited = true\n\t\t\t}\n\t\t}\n\t\tconst body = stripDocumentPreamble(markdown ?? '')\n\t\tif (!isBlankMarkdown(body))\n\t\t\treturn { outcome: 'ok', source: 'workers-ai', markdown: body }\n\t\tif (input.extension !== 'pdf') {\n\t\t\treturn {\n\t\t\t\toutcome: 'transient',\n\t\t\t\tsource: null,\n\t\t\t\tcaptureIssue: workersAiRateLimited ? 'workers_ai_rate_limited' : null\n\t\t\t}\n\t\t}\n\t\tif (input.signal?.aborted) return { outcome: 'transient', source: null }\n\t\ttry {\n\t\t\tif (!options.parsePdf) throw new Error('No PDF parser configured')\n\t\t\tconst parsed = await options.parsePdf({\n\t\t\t\tbytes,\n\t\t\t\tfilename: input.filename,\n\t\t\t\tmimeType: input.mimeType,\n\t\t\t\tmaxPages: MAX_PDF_PAGES,\n\t\t\t\tsignal: input.signal\n\t\t\t})\n\t\t\treturn {\n\t\t\t\toutcome: 'ok',\n\t\t\t\tsource: 'firecrawl-parse',\n\t\t\t\tmarkdown: parsed.markdown,\n\t\t\t\t// The parser may report parsed pages rather than total pages; reaching the\n\t\t\t\t// limit is therefore explicitly incomplete even when the count equals it.\n\t\t\t\tcaptureIssue:\n\t\t\t\t\tparsed.numPages === null || parsed.numPages >= MAX_PDF_PAGES\n\t\t\t\t\t\t? 'pdf_page_limit_unverified'\n\t\t\t\t\t\t: null\n\t\t\t}\n\t\t} catch (error) {\n\t\t\tlogger.warn('url-to-markdown.pdf_parse_failed', {\n\t\t\t\tfilename: input.filename,\n\t\t\t\t...errorLogAttributes(error)\n\t\t\t})\n\t\t\treturn {\n\t\t\t\toutcome: 'transient',\n\t\t\t\tsource: null,\n\t\t\t\tcaptureIssue: workersAiRateLimited\n\t\t\t\t\t? 'workers_ai_rate_limited'\n\t\t\t\t\t: 'pdf_extraction_failed'\n\t\t\t}\n\t\t}\n\t}\n\n\tasync function convertDocument(input: {\n\t\tbody: OriginBody\n\t\tcontentType: string | null\n\t\tfinalUrl: URL\n\t\tsignal?: AbortSignal | undefined\n\t}): Promise<PageBody> {\n\t\tconst type = convertibleDocumentType(input.contentType)\n\t\tif (!type || input.body.kind !== 'bytes') {\n\t\t\treturn {\n\t\t\t\toutcome: 'unsupported',\n\t\t\t\tsource: null,\n\t\t\t\tcaptureIssue:\n\t\t\t\t\tinput.body.kind === 'none' ? input.body.captureIssue : null\n\t\t\t}\n\t\t}\n\t\treturn convertDocumentBytes({\n\t\t\tbytes: input.body.bytes,\n\t\t\tfilename: documentFilename(input.finalUrl.href, type.extension),\n\t\t\tmimeType: type.mimeType,\n\t\t\textension: type.extension,\n\t\t\tsignal: input.signal\n\t\t})\n\t}\n\n\tasync function convertTextual(input: {\n\t\ttext: string\n\t\tkind: ContentKind\n\t\tfinalUrl: URL\n\t\trender: boolean\n\t\t/** The hash of the HTML last read; see `FetchedPage.htmlHash`. */\n\t\thtmlHash?: string | null | undefined\n\t\tsignal?: AbortSignal | undefined\n\t}): Promise<PageBody> {\n\t\tconst body = input.text\n\t\t// The `Content-Type` header is authoritative - sniffing for a leading\n\t\t// `<html` misses every page that opens with a comment or a CDN banner - but\n\t\t// a `text/plain` body that is plainly HTML still needs the HTML tier.\n\t\tconst isHtml =\n\t\t\tinput.kind === 'html' ||\n\t\t\t(input.kind !== 'markdown' && looksLikeHtml(body))\n\n\t\tif (!isHtml && !isBlankMarkdown(body)) {\n\t\t\treturn { outcome: 'ok', source: 'content-negotiation', markdown: body }\n\t\t}\n\n\t\tconst html = isHtml ? body : null\n\t\t// Most origins send no validators, so without this every page of a quiet\n\t\t// site went through Workers AI again every night to learn nothing moved.\n\t\tif (html && input.htmlHash && htmlHash(html) === input.htmlHash) {\n\t\t\treturn { outcome: 'unchanged', source: 'not-modified' }\n\t\t}\n\t\tif (html && Buffer.byteLength(html) > MAX_HTML_BYTES) {\n\t\t\t// A 6 MB HTML upload came back as 6 MB of markdown through Browser\n\t\t\t// Rendering, and Workers AI would bill the same bytes. It is not a page,\n\t\t\t// and it will be just as oversized next run - so not `transient`.\n\t\t\tlogger.info('url-to-markdown.html_oversized', {\n\t\t\t\turl: input.finalUrl.href,\n\t\t\t\tbytes: Buffer.byteLength(html)\n\t\t\t})\n\t\t\treturn { outcome: 'unsupported', source: null, html }\n\t\t}\n\n\t\tlet shell = false\n\t\tlet workersAiRateLimited = false\n\t\tif (html) {\n\t\t\tconst visible = withoutHiddenUi(html)\n\t\t\tlet converted: string | null = null\n\t\t\ttry {\n\t\t\t\tconverted = await client.convertHtmlWithWorkersAi({\n\t\t\t\t\thtml: visible,\n\t\t\t\t\t// The post-redirect host: Cloudflare resolves relative links against it.\n\t\t\t\t\thostname: input.finalUrl.hostname,\n\t\t\t\t\tsignal: input.signal\n\t\t\t\t})\n\t\t\t} catch (error) {\n\t\t\t\tif (!(error instanceof CloudflareRateLimitedError)) throw error\n\t\t\t\tworkersAiRateLimited = true\n\t\t\t}\n\t\t\tif (converted && !isBlankMarkdown(converted)) {\n\t\t\t\tshell = isApplicationShell(visible, converted)\n\t\t\t\tif (!shell)\n\t\t\t\t\treturn {\n\t\t\t\t\t\toutcome: 'ok',\n\t\t\t\t\t\tsource: 'workers-ai',\n\t\t\t\t\t\tmarkdown: converted,\n\t\t\t\t\t\thtml\n\t\t\t\t\t}\n\t\t\t}\n\t\t}\n\n\t\t// Browser Rendering is the exception path: slow, rate limited, and the only\n\t\t// tier that ever timed out in the probe.\n\t\tif (input.render && !input.signal?.aborted) {\n\t\t\tlet rendered: string | null = null\n\t\t\ttry {\n\t\t\t\trendered = await client.renderUrlToMarkdown(\n\t\t\t\t\tinput.finalUrl.href,\n\t\t\t\t\tinput.signal\n\t\t\t\t)\n\t\t\t} catch (error) {\n\t\t\t\tif (!(error instanceof CloudflareRateLimitedError)) throw error\n\t\t\t\tworkersAiRateLimited = true\n\t\t\t}\n\t\t\tif (\n\t\t\t\trendered &&\n\t\t\t\t!isBlankMarkdown(rendered) &&\n\t\t\t\t(!html || !isApplicationShell(html, rendered))\n\t\t\t) {\n\t\t\t\treturn {\n\t\t\t\t\toutcome: 'ok',\n\t\t\t\t\tsource: 'browser-rendering',\n\t\t\t\t\tmarkdown: rendered,\n\t\t\t\t\thtml\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\t\treturn {\n\t\t\toutcome: 'transient',\n\t\t\tsource: null,\n\t\t\thtml,\n\t\t\tcaptureIssue: shell\n\t\t\t\t? 'render_incomplete'\n\t\t\t\t: workersAiRateLimited\n\t\t\t\t\t? 'workers_ai_rate_limited'\n\t\t\t\t\t: null\n\t\t}\n\t}\n\n\t/** Runs after the origin's slot is released: every tier it reaches is ours. */\n\tasync function readResponse(\n\t\tinput: OriginRead & {\n\t\t\trender: boolean\n\t\t\thtmlHash?: string | null | undefined\n\t\t\tsignal?: AbortSignal | undefined\n\t\t}\n\t): Promise<PageBody> {\n\t\tconst { response, body } = input\n\t\tconst status = response.status\n\t\tconst contentType = response.headers.get('content-type')\n\n\t\tif (status === 304) return { outcome: 'unchanged', source: 'not-modified' }\n\t\tif (status === 404 || status === 410) {\n\t\t\treturn { outcome: 'gone', source: null }\n\t\t}\n\t\t// Still refused after the host's pauses: rendering would only be refused\n\t\t// too (see `MAX_RATE_LIMIT_RETRIES`).\n\t\tif (status >= 500 || status === 429)\n\t\t\treturn { outcome: 'transient', source: null }\n\t\tif (status >= 400) {\n\t\t\tif (!input.render || !BOT_WALL_STATUSES.has(status)) {\n\t\t\t\treturn { outcome: 'blocked', source: null }\n\t\t\t}\n\t\t\tconst rendered = await client.renderUrlToMarkdown(\n\t\t\t\tinput.finalUrl.href,\n\t\t\t\tinput.signal\n\t\t\t)\n\t\t\treturn rendered\n\t\t\t\t? { outcome: 'ok', source: 'browser-rendering', markdown: rendered }\n\t\t\t\t: { outcome: 'blocked', source: null }\n\t\t}\n\t\tif (status < 200 || status >= 300) {\n\t\t\treturn { outcome: 'transient', source: null }\n\t\t}\n\n\t\tconst kind = classifyContentType(contentType)\n\t\tif (kind === 'asset') return { outcome: 'unsupported', source: null }\n\t\tif (kind === 'document') {\n\t\t\treturn convertDocument({\n\t\t\t\tbody,\n\t\t\t\tcontentType,\n\t\t\t\tfinalUrl: input.finalUrl,\n\t\t\t\tsignal: input.signal\n\t\t\t})\n\t\t}\n\t\tif (body.kind !== 'text') return { outcome: 'unsupported', source: null }\n\t\treturn convertTextual({\n\t\t\ttext: body.text,\n\t\t\tkind,\n\t\t\tfinalUrl: input.finalUrl,\n\t\t\trender: input.render,\n\t\t\thtmlHash: input.htmlHash,\n\t\t\tsignal: input.signal\n\t\t})\n\t}\n\n\t/**\n\t * Read one URL as markdown through Cloudflare's three tiers - content\n\t * negotiation, Workers AI `toMarkdown`, Browser Rendering - and report\n\t * everything the import pipeline needs about the page: real status, redirect\n\t * target, validators, HTML metadata and absolute links.\n\t *\n\t * Never throws: a page that cannot be read is an outcome, not an exception,\n\t * because the caller is crawling hundreds of URLs at a time. A knowledge bench\n\t * run reads through its crawl snapshot (`crawl-snapshot.ts`).\n\t */\n\tasync function fetchPageAsMarkdown(\n\t\tinput: FetchPageInput\n\t): Promise<FetchedPage> {\n\t\tif (!currentCrawlSnapshot()) return fetchPageLive(input)\n\n\t\t// The snapshot holds the whole page, read without validators; whether it is\n\t\t// \"unchanged\" for this run is decided here, the way the origin would.\n\t\tconst page = await throughCrawlSnapshot(\n\t\t\t`page ${input.url} ${input.country ?? ''} ${input.render ?? true}`,\n\t\t\t() =>\n\t\t\t\tfetchPageLive({\n\t\t\t\t\turl: input.url,\n\t\t\t\t\tcountry: input.country,\n\t\t\t\t\trender: input.render,\n\t\t\t\t\tsignal: input.signal\n\t\t\t\t})\n\t\t)\n\t\tconst unchanged =\n\t\t\tpage.outcome === 'ok' &&\n\t\t\t((input.htmlHash && page.htmlHash === input.htmlHash) ||\n\t\t\t\t(input.etag && page.etag === input.etag))\n\t\tif (!unchanged) return page\n\n\t\treturn {\n\t\t\t...page,\n\t\t\toutcome: 'unchanged',\n\t\t\tsource: 'not-modified',\n\t\t\tmarkdown: '',\n\t\t\ttitle: null,\n\t\t\tlang: null,\n\t\t\tcanonical: null,\n\t\t\talternates: [],\n\t\t\talternateLanguages: {},\n\t\t\tmetaDescription: null,\n\t\t\tjsonLd: [],\n\t\t\tlinks: [],\n\t\t\timages: [],\n\t\t\tcaptureIssue: null,\n\t\t\thtmlHash: null\n\t\t}\n\t}\n\n\tasync function fetchPageLive(input: FetchPageInput): Promise<FetchedPage> {\n\t\tconst signal = input.signal ?? currentOperationSignal()\n\t\tconst startedAt = performance.now()\n\t\tconst render = input.render ?? true\n\n\t\tconst prepared = prepareRequestUrl(input.url)\n\t\tif (!prepared) {\n\t\t\treturn buildPage(\n\t\t\t\t{\n\t\t\t\t\turl: input.url,\n\t\t\t\t\tfinalUrl: input.url,\n\t\t\t\t\tstatus: 0,\n\t\t\t\t\tcontentType: null,\n\t\t\t\t\tetag: null,\n\t\t\t\t\tlastModified: null,\n\t\t\t\t\tstartedAt\n\t\t\t\t},\n\t\t\t\t{ outcome: 'unsupported', source: null },\n\t\t\t\tlogger\n\t\t\t)\n\t\t}\n\n\t\tconst headers: Record<string, string> = {\n\t\t\tAccept: 'text/markdown, text/html;q=0.9, */*;q=0.8',\n\t\t\t'Accept-Language': '*',\n\t\t\t'User-Agent': BROWSER_USER_AGENT\n\t\t}\n\t\tif (input.etag) headers['If-None-Match'] = input.etag\n\t\tif (input.lastModified) headers['If-Modified-Since'] = input.lastModified\n\n\t\tconst request = (url: URL) =>\n\t\t\trequestOrigin(inMarket(url, input.country), headers, signal)\n\n\t\tlet requestUrl = prepared\n\t\tlet attempt: OriginAttempt = await request(requestUrl)\n\n\t\tif (attempt.kind === 'failed' && attempt.retryable) {\n\t\t\tconst fallback = wwwFallbackUrl(requestUrl)\n\t\t\tif (fallback) {\n\t\t\t\tconst retry = await request(fallback)\n\t\t\t\tif (retry.kind === 'read') {\n\t\t\t\t\trequestUrl = fallback\n\t\t\t\t\tattempt = retry\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\n\t\tfor (\n\t\t\tlet retry = 1;\n\t\t\tretry <= MAX_RATE_LIMIT_RETRIES &&\n\t\t\tattempt.kind === 'read' &&\n\t\t\tattempt.read.response.status === 429;\n\t\t\tretry++\n\t\t) {\n\t\t\tconst waitMs = rateLimitWaitMs(attempt.read.response, retry)\n\t\t\tif (await hostLimits.coolDownHost(requestUrl.hostname, waitMs)) {\n\t\t\t\tlogger.warn('url-to-markdown.host_rate_limited', {\n\t\t\t\t\thost: requestUrl.hostname,\n\t\t\t\t\tretry_after: attempt.read.response.headers.get('retry-after') ?? '',\n\t\t\t\t\twait_ms: waitMs\n\t\t\t\t})\n\t\t\t}\n\t\t\tattempt = await request(requestUrl)\n\t\t}\n\n\t\tif (attempt.kind !== 'read') {\n\t\t\treturn buildPage(\n\t\t\t\t{\n\t\t\t\t\turl: requestUrl.href,\n\t\t\t\t\tfinalUrl: requestUrl.href,\n\t\t\t\t\tstatus: 0,\n\t\t\t\t\tcontentType: null,\n\t\t\t\t\tetag: null,\n\t\t\t\t\tlastModified: null,\n\t\t\t\t\tstartedAt\n\t\t\t\t},\n\t\t\t\t// A refused host is a policy decision, not a retryable network blip.\n\t\t\t\t{\n\t\t\t\t\toutcome: attempt.kind === 'refused' ? 'blocked' : 'transient',\n\t\t\t\t\tsource: null\n\t\t\t\t},\n\t\t\t\tlogger\n\t\t\t)\n\t\t}\n\n\t\tconst { response, finalUrl } = attempt.read\n\t\tconst notModified = (stored: string | null | undefined) =>\n\t\t\tresponse.status === 304 ? (stored ?? null) : null\n\t\tconst body = await readResponse({\n\t\t\t...attempt.read,\n\t\t\trender,\n\t\t\thtmlHash: input.htmlHash,\n\t\t\tsignal\n\t\t})\n\n\t\tconst page = buildPage(\n\t\t\t{\n\t\t\t\turl: requestUrl.href,\n\t\t\t\tfinalUrl: finalUrl.href,\n\t\t\t\tstatus: response.status,\n\t\t\t\tcontentType: response.headers.get('content-type'),\n\t\t\t\t// A 304 need not repeat the validators; the stored ones stay valid. On\n\t\t\t\t// any other status only this response's headers count - carrying an old\n\t\t\t\t// ETag onto changed content would freeze the page forever.\n\t\t\t\tetag: response.headers.get('etag') ?? notModified(input.etag),\n\t\t\t\tlastModified:\n\t\t\t\t\tresponse.headers.get('last-modified') ??\n\t\t\t\t\tnotModified(input.lastModified),\n\t\t\t\tstartedAt\n\t\t\t},\n\t\t\tbody,\n\t\t\tlogger\n\t\t)\n\n\t\tlogger.info('url-to-markdown.completed', {\n\t\t\turl: page.finalUrl,\n\t\t\tstatus: page.status,\n\t\t\toutcome: page.outcome,\n\t\t\tsource: page.source ?? '',\n\t\t\tchars: page.markdown.length,\n\t\t\tduration_ms: page.durationMs\n\t\t})\n\t\treturn page\n\t}\n\n\t/** CLI/one-shot shape: markdown or an error. */\n\tasync function urlToMarkdown(urlString: string) {\n\t\tconst page = await fetchPageAsMarkdown({ url: urlString })\n\t\tconst { markdown, source } = page\n\t\tif (page.outcome !== 'ok' || !markdown || !source) {\n\t\t\tthrow new Error(\n\t\t\t\t`Failed to convert URL to markdown (${page.outcome}, status ${page.status}): ${urlString}`\n\t\t\t)\n\t\t}\n\t\treturn { markdown, source }\n\t}\n\n\treturn {\n\t\tfetchPageAsMarkdown,\n\t\tconvertDocumentBytes,\n\t\turlToMarkdown,\n\t\tconvertFileWithWorkersAi: client.convertFileWithWorkersAi,\n\t\trenderUrlToMarkdown: client.renderUrlToMarkdown,\n\t\t/** The SSRF guard the converter runs, bound to its `logger`. */\n\t\tisHostAllowed: allowHost,\n\t\t/** `withCrawlSnapshot` bound to the `crawlSnapshot` option. */\n\t\twithCrawlSnapshot<T>(\n\t\t\tinput: CrawlSnapshotInput | null | undefined,\n\t\t\trun: () => Promise<T>\n\t\t) {\n\t\t\tif (!input) return run()\n\t\t\tif (!options.crawlSnapshot) {\n\t\t\t\tthrow new Error('Set crawlSnapshot.root to use crawl snapshots')\n\t\t\t}\n\t\t\treturn withCrawlSnapshot(input, run, {\n\t\t\t\troot: options.crawlSnapshot.root,\n\t\t\t\tlogger: options.crawlSnapshot.logger ?? logger\n\t\t\t})\n\t\t}\n\t}\n}\n\nexport type WebsiteToMarkdown = ReturnType<typeof createWebsiteToMarkdown>\n"]}

package/dist/index.d.ts

@@ -1,6 +1,6 @@
 export { CloudflareRateLimitedError, type CloudflareCredentials, type ConvertFileInput } from './client.js';
 export { type ContentKind, classifyContentType, convertibleDocumentType } from './content-type.js';
-export { CRAWL_SNAPSHOT_NAME_PATTERN, type CrawlSnapshotInput, currentCrawlSnapshot, throughCrawlSnapshot, withCrawlSnapshot } from './crawl-snapshot.js';
+export { CRAWL_SNAPSHOT_NAME_PATTERN, type CrawlSnapshotInput, type CrawlSnapshotOptions, currentCrawlSnapshot, throughCrawlSnapshot, withCrawlSnapshot } from './crawl-snapshot.js';
 export { createWebsiteToMarkdown, type FetchedPage, type FetchOutcome, type FetchPageInput, type FetchSource, MAX_MARKDOWN_CHARS, type PageBody, type PdfParser, type WebsiteToMarkdown, type WebsiteToMarkdownOptions } from './fetch-page.js';
 export { decodeHtmlEntities } from './html-entities.js';
 export { extractHtmlMetadata } from './html-metadata.js';

package/dist/index.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,OAAO,EACN,0BAA0B,EAC1B,KAAK,qBAAqB,EAC1B,KAAK,gBAAgB,EACrB,MAAM,aAAa,CAAA;AACpB,OAAO,EACN,KAAK,WAAW,EAChB,mBAAmB,EACnB,uBAAuB,EACvB,MAAM,mBAAmB,CAAA;AAC1B,OAAO,EACN,2BAA2B,EAC3B,KAAK,kBAAkB,EACvB,oBAAoB,EACpB,oBAAoB,EACpB,iBAAiB,EACjB,MAAM,qBAAqB,CAAA;AAC5B,OAAO,EACN,uBAAuB,EACvB,KAAK,WAAW,EAChB,KAAK,YAAY,EACjB,KAAK,cAAc,EACnB,KAAK,WAAW,EAChB,kBAAkB,EAClB,KAAK,QAAQ,EACb,KAAK,SAAS,EACd,KAAK,iBAAiB,EACtB,KAAK,wBAAwB,EAC7B,MAAM,iBAAiB,CAAA;AACxB,OAAO,EAAE,kBAAkB,EAAE,MAAM,oBAAoB,CAAA;AACvD,OAAO,EAAE,mBAAmB,EAAE,MAAM,oBAAoB,CAAA;AACxD,OAAO,EACN,KAAK,gBAAgB,EACrB,KAAK,uBAAuB,EAC5B,KAAK,sBAAsB,EAC3B,gCAAgC,EAChC,MAAM,YAAY,CAAA;AACnB,YAAY,EAAE,aAAa,EAAE,MAAM,EAAE,MAAM,aAAa,CAAA;AACxD,OAAO,EACN,UAAU,EACV,SAAS,EACT,YAAY,EACZ,qBAAqB,EACrB,gBAAgB,EAChB,iBAAiB,EACjB,MAAM,oBAAoB,CAAA;AAC3B,OAAO,EAAE,4BAA4B,EAAE,MAAM,mBAAmB,CAAA;AAChE,OAAO,EACN,gBAAgB,EAChB,aAAa,EACb,mBAAmB,EACnB,MAAM,gBAAgB,CAAA;AACvB,OAAO,EACN,sBAAsB,EACtB,oBAAoB,EACpB,mBAAmB,EACnB,MAAM,uBAAuB,CAAA;AAC9B,OAAO,EAAE,kBAAkB,EAAE,MAAM,iBAAiB,CAAA"}
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,OAAO,EACN,0BAA0B,EAC1B,KAAK,qBAAqB,EAC1B,KAAK,gBAAgB,EACrB,MAAM,aAAa,CAAA;AACpB,OAAO,EACN,KAAK,WAAW,EAChB,mBAAmB,EACnB,uBAAuB,EACvB,MAAM,mBAAmB,CAAA;AAC1B,OAAO,EACN,2BAA2B,EAC3B,KAAK,kBAAkB,EACvB,KAAK,oBAAoB,EACzB,oBAAoB,EACpB,oBAAoB,EACpB,iBAAiB,EACjB,MAAM,qBAAqB,CAAA;AAC5B,OAAO,EACN,uBAAuB,EACvB,KAAK,WAAW,EAChB,KAAK,YAAY,EACjB,KAAK,cAAc,EACnB,KAAK,WAAW,EAChB,kBAAkB,EAClB,KAAK,QAAQ,EACb,KAAK,SAAS,EACd,KAAK,iBAAiB,EACtB,KAAK,wBAAwB,EAC7B,MAAM,iBAAiB,CAAA;AACxB,OAAO,EAAE,kBAAkB,EAAE,MAAM,oBAAoB,CAAA;AACvD,OAAO,EAAE,mBAAmB,EAAE,MAAM,oBAAoB,CAAA;AACxD,OAAO,EACN,KAAK,gBAAgB,EACrB,KAAK,uBAAuB,EAC5B,KAAK,sBAAsB,EAC3B,gCAAgC,EAChC,MAAM,YAAY,CAAA;AACnB,YAAY,EAAE,aAAa,EAAE,MAAM,EAAE,MAAM,aAAa,CAAA;AACxD,OAAO,EACN,UAAU,EACV,SAAS,EACT,YAAY,EACZ,qBAAqB,EACrB,gBAAgB,EAChB,iBAAiB,EACjB,MAAM,oBAAoB,CAAA;AAC3B,OAAO,EAAE,4BAA4B,EAAE,MAAM,mBAAmB,CAAA;AAChE,OAAO,EACN,gBAAgB,EAChB,aAAa,EACb,mBAAmB,EACnB,MAAM,gBAAgB,CAAA;AACvB,OAAO,EACN,sBAAsB,EACtB,oBAAoB,EACpB,mBAAmB,EACnB,MAAM,uBAAuB,CAAA;AAC9B,OAAO,EAAE,kBAAkB,EAAE,MAAM,iBAAiB,CAAA"}

package/dist/index.js.map

@@ -1 +1 @@
-{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,OAAO,EACN,0BAA0B,EAG1B,MAAM,aAAa,CAAA;AACpB,OAAO,EAEN,mBAAmB,EACnB,uBAAuB,EACvB,MAAM,mBAAmB,CAAA;AAC1B,OAAO,EACN,2BAA2B,EAE3B,oBAAoB,EACpB,oBAAoB,EACpB,iBAAiB,EACjB,MAAM,qBAAqB,CAAA;AAC5B,OAAO,EACN,uBAAuB,EAKvB,kBAAkB,EAKlB,MAAM,iBAAiB,CAAA;AACxB,OAAO,EAAE,kBAAkB,EAAE,MAAM,oBAAoB,CAAA;AACvD,OAAO,EAAE,mBAAmB,EAAE,MAAM,oBAAoB,CAAA;AACxD,OAAO,EAIN,gCAAgC,EAChC,MAAM,YAAY,CAAA;AAEnB,OAAO,EACN,UAAU,EACV,SAAS,EACT,YAAY,EACZ,qBAAqB,EACrB,gBAAgB,EAChB,iBAAiB,EACjB,MAAM,oBAAoB,CAAA;AAC3B,OAAO,EAAE,4BAA4B,EAAE,MAAM,mBAAmB,CAAA;AAChE,OAAO,EACN,gBAAgB,EAChB,aAAa,EACb,mBAAmB,EACnB,MAAM,gBAAgB,CAAA;AACvB,OAAO,EACN,sBAAsB,EACtB,oBAAoB,EACpB,mBAAmB,EACnB,MAAM,uBAAuB,CAAA;AAC9B,OAAO,EAAE,kBAAkB,EAAE,MAAM,iBAAiB,CAAA","sourcesContent":["export {\n\tCloudflareRateLimitedError,\n\ttype CloudflareCredentials,\n\ttype ConvertFileInput\n} from './client.js'\nexport {\n\ttype ContentKind,\n\tclassifyContentType,\n\tconvertibleDocumentType\n} from './content-type.js'\nexport {\n\tCRAWL_SNAPSHOT_NAME_PATTERN,\n\ttype CrawlSnapshotInput,\n\tcurrentCrawlSnapshot,\n\tthroughCrawlSnapshot,\n\twithCrawlSnapshot\n} from './crawl-snapshot.js'\nexport {\n\tcreateWebsiteToMarkdown,\n\ttype FetchedPage,\n\ttype FetchOutcome,\n\ttype FetchPageInput,\n\ttype FetchSource,\n\tMAX_MARKDOWN_CHARS,\n\ttype PageBody,\n\ttype PdfParser,\n\ttype WebsiteToMarkdown,\n\ttype WebsiteToMarkdownOptions\n} from './fetch-page.js'\nexport { decodeHtmlEntities } from './html-entities.js'\nexport { extractHtmlMetadata } from './html-metadata.js'\nexport {\n\ttype ConcurrencyLimit,\n\ttype ConcurrencyLimitOptions,\n\ttype CreateConcurrencyLimit,\n\tURL_TO_MARKDOWN_PAGE_CONCURRENCY\n} from './limit.js'\nexport type { LogAttributes, Logger } from './logger.js'\nexport {\n\thtmlImages,\n\thtmlLinks,\n\tmarkdownRefs,\n\tstripDocumentPreamble,\n\tstripFrontMatter,\n\ttitleFromMarkdown\n} from './markdown-refs.js'\nexport { createMemoryConcurrencyLimit } from './memory-limit.js'\nexport {\n\tisBlockedAddress,\n\tisHostAllowed,\n\tisLocalhostHostname\n} from './net-guard.js'\nexport {\n\tcurrentOperationSignal,\n\tfailCurrentOperation,\n\twithOperationSignal\n} from './operation-signal.js'\nexport { BROWSER_USER_AGENT } from './user-agent.js'\n"]}
+{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,OAAO,EACN,0BAA0B,EAG1B,MAAM,aAAa,CAAA;AACpB,OAAO,EAEN,mBAAmB,EACnB,uBAAuB,EACvB,MAAM,mBAAmB,CAAA;AAC1B,OAAO,EACN,2BAA2B,EAG3B,oBAAoB,EACpB,oBAAoB,EACpB,iBAAiB,EACjB,MAAM,qBAAqB,CAAA;AAC5B,OAAO,EACN,uBAAuB,EAKvB,kBAAkB,EAKlB,MAAM,iBAAiB,CAAA;AACxB,OAAO,EAAE,kBAAkB,EAAE,MAAM,oBAAoB,CAAA;AACvD,OAAO,EAAE,mBAAmB,EAAE,MAAM,oBAAoB,CAAA;AACxD,OAAO,EAIN,gCAAgC,EAChC,MAAM,YAAY,CAAA;AAEnB,OAAO,EACN,UAAU,EACV,SAAS,EACT,YAAY,EACZ,qBAAqB,EACrB,gBAAgB,EAChB,iBAAiB,EACjB,MAAM,oBAAoB,CAAA;AAC3B,OAAO,EAAE,4BAA4B,EAAE,MAAM,mBAAmB,CAAA;AAChE,OAAO,EACN,gBAAgB,EAChB,aAAa,EACb,mBAAmB,EACnB,MAAM,gBAAgB,CAAA;AACvB,OAAO,EACN,sBAAsB,EACtB,oBAAoB,EACpB,mBAAmB,EACnB,MAAM,uBAAuB,CAAA;AAC9B,OAAO,EAAE,kBAAkB,EAAE,MAAM,iBAAiB,CAAA","sourcesContent":["export {\n\tCloudflareRateLimitedError,\n\ttype CloudflareCredentials,\n\ttype ConvertFileInput\n} from './client.js'\nexport {\n\ttype ContentKind,\n\tclassifyContentType,\n\tconvertibleDocumentType\n} from './content-type.js'\nexport {\n\tCRAWL_SNAPSHOT_NAME_PATTERN,\n\ttype CrawlSnapshotInput,\n\ttype CrawlSnapshotOptions,\n\tcurrentCrawlSnapshot,\n\tthroughCrawlSnapshot,\n\twithCrawlSnapshot\n} from './crawl-snapshot.js'\nexport {\n\tcreateWebsiteToMarkdown,\n\ttype FetchedPage,\n\ttype FetchOutcome,\n\ttype FetchPageInput,\n\ttype FetchSource,\n\tMAX_MARKDOWN_CHARS,\n\ttype PageBody,\n\ttype PdfParser,\n\ttype WebsiteToMarkdown,\n\ttype WebsiteToMarkdownOptions\n} from './fetch-page.js'\nexport { decodeHtmlEntities } from './html-entities.js'\nexport { extractHtmlMetadata } from './html-metadata.js'\nexport {\n\ttype ConcurrencyLimit,\n\ttype ConcurrencyLimitOptions,\n\ttype CreateConcurrencyLimit,\n\tURL_TO_MARKDOWN_PAGE_CONCURRENCY\n} from './limit.js'\nexport type { LogAttributes, Logger } from './logger.js'\nexport {\n\thtmlImages,\n\thtmlLinks,\n\tmarkdownRefs,\n\tstripDocumentPreamble,\n\tstripFrontMatter,\n\ttitleFromMarkdown\n} from './markdown-refs.js'\nexport { createMemoryConcurrencyLimit } from './memory-limit.js'\nexport {\n\tisBlockedAddress,\n\tisHostAllowed,\n\tisLocalhostHostname\n} from './net-guard.js'\nexport {\n\tcurrentOperationSignal,\n\tfailCurrentOperation,\n\twithOperationSignal\n} from './operation-signal.js'\nexport { BROWSER_USER_AGENT } from './user-agent.js'\n"]}

package/dist/limit.d.ts

@@ -1,6 +1,7 @@
 /**
- * A semaphore: at most `max` callbacks in flight. hellu's Redis Lua limiter
- * (`createRedisConcurrencyLimit`) fits this shape as is; the default is
+ * A semaphore: at most `max` callbacks in flight. `hanzio/redis`'s
+ * `createRedisConcurrencyLimit` returns this shape (plus `queueDepth`, which is
+ * not needed here) once `send` is bound; the default is
  * {@link createMemoryConcurrencyLimit}.
  */
 export type ConcurrencyLimit = {

package/dist/limit.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"limit.d.ts","sourceRoot":"","sources":["../src/limit.ts"],"names":[],"mappings":"AAEA;;;;GAIG;AACH,MAAM,MAAM,gBAAgB,GAAG;IAC9B,CAAC,CAAC,EAAE,GAAG,EAAE,MAAM,OAAO,CAAC,CAAC,CAAC,EAAE,MAAM,CAAC,EAAE,WAAW,GAAG,OAAO,CAAC,CAAC,CAAC,CAAA;IAC5D,2EAA2E;IAC3E,YAAY,EAAE,CAAC,MAAM,EAAE,MAAM,KAAK,OAAO,CAAC,IAAI,CAAC,CAAA;IAC/C,2EAA2E;IAC3E,OAAO,EAAE,MAAM,IAAI,CAAA;CACnB,CAAA;AAED,MAAM,MAAM,uBAAuB,GAAG;IACrC,qEAAqE;IACrE,GAAG,EAAE,MAAM,CAAA;IACX,GAAG,EAAE,MAAM,CAAA;IACX,6DAA6D;IAC7D,SAAS,EAAE,MAAM,CAAA;IACjB,4EAA4E;IAC5E,kBAAkB,CAAC,EAAE,MAAM,CAAA;IAC3B,6EAA6E;IAC7E,WAAW,CAAC,EAAE,MAAM,CAAA;CACpB,CAAA;AAED,MAAM,MAAM,sBAAsB,GAAG,CACpC,OAAO,EAAE,uBAAuB,KAC5B,gBAAgB,CAAA;AAErB;;;;GAIG;AACH,eAAO,MAAM,gCAAgC,KAAK,CAAA;AAElD;;;;GAIG;AACH,eAAO,MAAM,oCAAoC,KAAK,CAAA;AACtD,eAAO,MAAM,4CAA4C,KAAK,CAAA;AAE9D;;;;GAIG;AACH,eAAO,MAAM,6BAA6B,KAAK,CAAA;AAC/C,eAAO,MAAM,qCAAqC,KAAK,CAAA;AAEvD,yEAAyE;AACzE,wBAAgB,sBAAsB,CAAC,WAAW,EAAE,sBAAsB;;;EAgBzE;AAED;;;;;;;;;;;;;GAaG;AACH,eAAO,MAAM,oCAAoC,KAAK,CAAA;AAEtD;;;;;GAKG;AACH,eAAO,MAAM,8CAA8C,IAAI,CAAA;AAa/D,+EAA+E;AAC/E,eAAO,MAAM,oCAAoC,MAAM,CAAA;AAUvD,qFAAqF;AACrF,wBAAgB,gBAAgB,CAC/B,WAAW,GAAE,sBAAqD;kBA+C7C,CAAC,YAAY,MAAM,OAAO,MAAM,OAAO,CAAC,CAAC,CAAC;IAiB9D;;;;;OAKG;2BAC0B,MAAM,UAAU,MAAM;EAepD"}
+{"version":3,"file":"limit.d.ts","sourceRoot":"","sources":["../src/limit.ts"],"names":[],"mappings":"AAEA;;;;;GAKG;AACH,MAAM,MAAM,gBAAgB,GAAG;IAC9B,CAAC,CAAC,EAAE,GAAG,EAAE,MAAM,OAAO,CAAC,CAAC,CAAC,EAAE,MAAM,CAAC,EAAE,WAAW,GAAG,OAAO,CAAC,CAAC,CAAC,CAAA;IAC5D,2EAA2E;IAC3E,YAAY,EAAE,CAAC,MAAM,EAAE,MAAM,KAAK,OAAO,CAAC,IAAI,CAAC,CAAA;IAC/C,2EAA2E;IAC3E,OAAO,EAAE,MAAM,IAAI,CAAA;CACnB,CAAA;AAED,MAAM,MAAM,uBAAuB,GAAG;IACrC,qEAAqE;IACrE,GAAG,EAAE,MAAM,CAAA;IACX,GAAG,EAAE,MAAM,CAAA;IACX,6DAA6D;IAC7D,SAAS,EAAE,MAAM,CAAA;IACjB,4EAA4E;IAC5E,kBAAkB,CAAC,EAAE,MAAM,CAAA;IAC3B,6EAA6E;IAC7E,WAAW,CAAC,EAAE,MAAM,CAAA;CACpB,CAAA;AAED,MAAM,MAAM,sBAAsB,GAAG,CACpC,OAAO,EAAE,uBAAuB,KAC5B,gBAAgB,CAAA;AAErB;;;;GAIG;AACH,eAAO,MAAM,gCAAgC,KAAK,CAAA;AAElD;;;;GAIG;AACH,eAAO,MAAM,oCAAoC,KAAK,CAAA;AACtD,eAAO,MAAM,4CAA4C,KAAK,CAAA;AAE9D;;;;GAIG;AACH,eAAO,MAAM,6BAA6B,KAAK,CAAA;AAC/C,eAAO,MAAM,qCAAqC,KAAK,CAAA;AAEvD,yEAAyE;AACzE,wBAAgB,sBAAsB,CAAC,WAAW,EAAE,sBAAsB;;;EAgBzE;AAED;;;;;;;;;;;;;GAaG;AACH,eAAO,MAAM,oCAAoC,KAAK,CAAA;AAEtD;;;;;GAKG;AACH,eAAO,MAAM,8CAA8C,IAAI,CAAA;AAa/D,+EAA+E;AAC/E,eAAO,MAAM,oCAAoC,MAAM,CAAA;AAUvD,qFAAqF;AACrF,wBAAgB,gBAAgB,CAC/B,WAAW,GAAE,sBAAqD;kBA+C7C,CAAC,YAAY,MAAM,OAAO,MAAM,OAAO,CAAC,CAAC,CAAC;IAiB9D;;;;;OAKG;2BAC0B,MAAM,UAAU,MAAM;EAepD"}

package/dist/limit.js.map

@@ -1 +1 @@
-{"version":3,"file":"limit.js","sourceRoot":"","sources":["../src/limit.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,4BAA4B,EAAE,MAAM,mBAAmB,CAAA;AA+BhE;;;;GAIG;AACH,MAAM,CAAC,MAAM,gCAAgC,GAAG,EAAE,CAAA;AAElD;;;;GAIG;AACH,MAAM,CAAC,MAAM,oCAAoC,GAAG,EAAE,CAAA;AACtD,MAAM,CAAC,MAAM,4CAA4C,GAAG,EAAE,CAAA;AAE9D;;;;GAIG;AACH,MAAM,CAAC,MAAM,6BAA6B,GAAG,EAAE,CAAA;AAC/C,MAAM,CAAC,MAAM,qCAAqC,GAAG,EAAE,CAAA;AAEvD,yEAAyE;AACzE,MAAM,UAAU,sBAAsB,CAAC,WAAmC;IACzE,OAAO;QACN,gBAAgB,EAAE,WAAW,CAAC;YAC7B,GAAG,EAAE,8BAA8B;YACnC,GAAG,EAAE,4CAA4C;YACjD,kBAAkB,EAAE,oCAAoC;YACxD,6EAA6E;YAC7E,SAAS,EAAE,MAAM;SACjB,CAAC;QACF,SAAS,EAAE,WAAW,CAAC;YACtB,GAAG,EAAE,uBAAuB;YAC5B,GAAG,EAAE,qCAAqC;YAC1C,kBAAkB,EAAE,6BAA6B;YACjD,SAAS,EAAE,MAAM;SACjB,CAAC;KACF,CAAA;AACF,CAAC;AAED;;;;;;;;;;;;;GAaG;AACH,MAAM,CAAC,MAAM,oCAAoC,GAAG,EAAE,CAAA;AAEtD;;;;;GAKG;AACH,MAAM,CAAC,MAAM,8CAA8C,GAAG,CAAC,CAAA;AAC/D,MAAM,gCAAgC,GAAG,CAAC,CAAA;AAE1C,oFAAoF;AACpF,MAAM,gBAAgB,GAAG,EAAE,GAAG,EAAE,GAAG,IAAI,CAAA;AAEvC;;;;GAIG;AACH,MAAM,iBAAiB,GAAG,sBAAsB,CAAA;AAEhD,+EAA+E;AAC/E,MAAM,CAAC,MAAM,oCAAoC,GAAG,GAAG,CAAA;AAUvD,qFAAqF;AACrF,MAAM,UAAU,gBAAgB,CAC/B,cAAsC,4BAA4B;IAElE,uEAAuE;IACvE,uEAAuE;IACvE,MAAM,OAAO,GAAG,IAAI,GAAG,EAAqB,CAAA;IAE5C,SAAS,SAAS;QACjB,IAAI,OAAO,CAAC,IAAI,GAAG,oCAAoC;YAAE,OAAM;QAC/D,KAAK,MAAM,CAAC,IAAI,EAAE,KAAK,CAAC,IAAI,OAAO,EAAE,CAAC;YACrC,IAAI,KAAK,CAAC,QAAQ,GAAG,CAAC;gBAAE,SAAQ;YAChC,mEAAmE;YACnE,wEAAwE;YACxE,KAAK,CAAC,KAAK,CAAC,OAAO,EAAE,CAAA;YACrB,KAAK,CAAC,QAAQ,EAAE,OAAO,EAAE,CAAA;YACzB,OAAO,CAAC,MAAM,CAAC,IAAI,CAAC,CAAA;YACpB,OAAM;QACP,CAAC;QACD,oEAAoE;QACpE,kEAAkE;IACnE,CAAC;IAED,SAAS,QAAQ,CAAC,QAAgB;QACjC,MAAM,GAAG,GAAG,QAAQ,CAAC,WAAW,EAAE,CAAA;QAClC,MAAM,MAAM,GAAG,OAAO,CAAC,GAAG,CAAC,GAAG,CAAC,CAAA;QAC/B,IAAI,MAAM,EAAE,CAAC;YACZ,OAAO,CAAC,MAAM,CAAC,GAAG,CAAC,CAAA;YACnB,OAAO,CAAC,GAAG,CAAC,GAAG,EAAE,MAAM,CAAC,CAAA;YACxB,OAAO,MAAM,CAAA;QACd,CAAC;QACD,SAAS,EAAE,CAAA;QACX,MAAM,KAAK,GAAc;YACxB,KAAK,EAAE,WAAW,CAAC;gBAClB,GAAG,EAAE,wBAAwB,GAAG,EAAE;gBAClC,WAAW,EAAE,iBAAiB;gBAC9B,GAAG,EAAE,oCAAoC;gBACzC,oEAAoE;gBACpE,SAAS,EAAE,MAAM;aACjB,CAAC;YACF,QAAQ,EAAE,CAAC;YACX,QAAQ,EAAE,IAAI;YACd,cAAc,EAAE,CAAC;SACjB,CAAA;QACD,OAAO,CAAC,GAAG,CAAC,GAAG,EAAE,KAAK,CAAC,CAAA;QACvB,OAAO,KAAK,CAAA;IACb,CAAC;IAED,OAAO;QACN,KAAK,CAAC,aAAa,CAAI,QAAgB,EAAE,GAAqB;YAC7D,MAAM,KAAK,GAAG,QAAQ,CAAC,QAAQ,CAAC,CAAA;YAChC,wEAAwE;YACxE,6DAA6D;YAC7D,KAAK,CAAC,QAAQ,EAAE,CAAA;YAChB,IAAI,CAAC;gBACJ,OAAO,MAAM,KAAK,CAAC,KAAK,CAAC,GAAG,EAAE;oBAC7B,2EAA2E;oBAC3E,MAAM,QAAQ,GACb,IAAI,CAAC,GAAG,EAAE,GAAG,KAAK,CAAC,cAAc,CAAC,CAAC,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAC,CAAC,IAAI,CAAA;oBAC1D,OAAO,QAAQ,CAAC,CAAC,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,EAAE,CAAA;gBACxC,CAAC,CAAC,CAAA;YACH,CAAC;oBAAS,CAAC;gBACV,KAAK,CAAC,QAAQ,EAAE,CAAA;YACjB,CAAC;QACF,CAAC;QAED;;;;;WAKG;QACH,KAAK,CAAC,YAAY,CAAC,QAAgB,EAAE,MAAc;YAClD,MAAM,KAAK,GAAG,QAAQ,CAAC,QAAQ,CAAC,CAAA;YAChC,MAAM,QAAQ,GAAG,IAAI,CAAC,GAAG,EAAE,IAAI,KAAK,CAAC,cAAc,CAAA;YACnD,KAAK,CAAC,QAAQ,KAAK,WAAW,CAAC;gBAC9B,GAAG,EAAE,wBAAwB,QAAQ,CAAC,WAAW,EAAE,YAAY;gBAC/D,WAAW,EAAE,iBAAiB;gBAC9B,GAAG,EAAE,8CAA8C;gBACnD,kBAAkB,EAAE,gCAAgC;gBACpD,SAAS,EAAE,MAAM;aACjB,CAAC,CAAA;YACF,KAAK,CAAC,cAAc,GAAG,IAAI,CAAC,GAAG,EAAE,GAAG,gBAAgB,CAAA;YACpD,MAAM,KAAK,CAAC,QAAQ,CAAC,YAAY,CAAC,MAAM,CAAC,CAAA;YACzC,OAAO,QAAQ,CAAA;QAChB,CAAC;KACD,CAAA;AACF,CAAC","sourcesContent":["import { createMemoryConcurrencyLimit } from './memory-limit.js'\n\n/**\n * A semaphore: at most `max` callbacks in flight. hellu's Redis Lua limiter\n * (`createRedisConcurrencyLimit`) fits this shape as is; the default is\n * {@link createMemoryConcurrencyLimit}.\n */\nexport type ConcurrencyLimit = {\n\t<T>(run: () => Promise<T>, signal?: AbortSignal): Promise<T>\n\t/** Pause every start for `waitMs`, e.g. from an upstream `Retry-After`. */\n\ttripCooldown: (waitMs: number) => Promise<void>\n\t/** Drop the limiter's timers and metric registrations; it is discarded. */\n\tdispose: () => void\n}\n\nexport type ConcurrencyLimitOptions = {\n\t/** Shared limiters key their state on this; one per pool or host. */\n\tkey: string\n\tmax: number\n\t/** How long a slot held by a crashed process stays taken. */\n\tslotTtlMs: number\n\t/** A separate sliding one-second window of starts (provider rps quotas). */\n\tmaxStartsPerSecond?: number\n\t/** The pool a per-host limiter reports metrics under, instead of its key. */\n\tmetricLabel?: string\n}\n\nexport type CreateConcurrencyLimit = (\n\toptions: ConcurrencyLimitOptions\n) => ConcurrencyLimit\n\n/**\n * Page-level fetch parallelism. Content negotiation is a plain GET and is not\n * a Cloudflare API call, so this can sit above the Workers Paid start rates.\n * Workers AI and Browser Rendering still queue on their own pools.\n */\nexport const URL_TO_MARKDOWN_PAGE_CONCURRENCY = 80\n\n/**\n * Workers Paid Quick Actions are 30 rps, not 30 in-flight. Leave a little\n * headroom so two processes cannot land a burst on the same clock second.\n * Concurrency is higher so a 10-20s render does not leave the rps budget idle.\n */\nexport const CLOUDFLARE_BROWSER_RENDERING_MAX_RPS = 28\nexport const CLOUDFLARE_BROWSER_RENDERING_MAX_CONCURRENCY = 80\n\n/**\n * toMarkdown has no published rps cap. Stay below the Client API 200/s IP\n * burst and let `tripCooldown` pull the fleet back if the 1200/5min token\n * window applies.\n */\nexport const CLOUDFLARE_WORKERS_AI_MAX_RPS = 12\nexport const CLOUDFLARE_WORKERS_AI_MAX_CONCURRENCY = 24\n\n/** The two Cloudflare pools; a page can wait on both and on its host. */\nexport function createCloudflareLimits(createLimit: CreateConcurrencyLimit) {\n\treturn {\n\t\tbrowserRendering: createLimit({\n\t\t\tkey: 'cloudflare-browser-rendering',\n\t\t\tmax: CLOUDFLARE_BROWSER_RENDERING_MAX_CONCURRENCY,\n\t\t\tmaxStartsPerSecond: CLOUDFLARE_BROWSER_RENDERING_MAX_RPS,\n\t\t\t// Browser Rendering can run up to 60s; TTL is crash detection, not a budget.\n\t\t\tslotTtlMs: 90_000\n\t\t}),\n\t\tworkersAi: createLimit({\n\t\t\tkey: 'cloudflare-workers-ai',\n\t\t\tmax: CLOUDFLARE_WORKERS_AI_MAX_CONCURRENCY,\n\t\t\tmaxStartsPerSecond: CLOUDFLARE_WORKERS_AI_MAX_RPS,\n\t\t\tslotTtlMs: 45_000\n\t\t})\n\t}\n}\n\n/**\n * Politeness cap per customer host. This is the only thing standing\n * between an 80-wide page pool and one small WordPress box, and it applies to a\n * SOLO import as much as a concurrent one: two orgs are two domains and never\n * contend, so the burst this prevents is a single site's own crawl.\n *\n * Deliberately generous rather than cautious - a healthy origin should be read\n * at speed, and 500 pages at 16 in flight is well inside the run timeout. It\n * stays a cap because the failure it prevents is silent and un-retryable: a WAF\n * that decides we are an attack blocks the whole import, and the customer-site\n * probe already found an origin that 403s on User-Agent alone.\n *\n * Separate from the Cloudflare pools - a page can wait on both.\n */\nexport const URL_TO_MARKDOWN_HOST_MAX_CONCURRENCY = 16\n\n/**\n * A host that answered 429 is read this narrowly from then on. At 16 in\n * flight Shopify refused 117 of Good Habit's 244 reads even after each had\n * waited out its pause: the pause only delayed the next burst. Calibration\n * knobs - set them from the crawl log's `rate_limited`, not by theory.\n */\nexport const URL_TO_MARKDOWN_THROTTLED_HOST_MAX_CONCURRENCY = 2\nconst THROTTLED_HOST_STARTS_PER_SECOND = 2\n\n/** A throttled host is read at the full cap again after this long without a 429. */\nconst HOST_THROTTLE_MS = 15 * 60 * 1000\n\n/**\n * Every host limiter meters under this one label. Per-host series would grow\n * with the customer base and never shrink, and the question the metric answers\n * - is the crawler backed up behind origins - is a pool-wide question anyway.\n */\nconst HOST_METRIC_LABEL = 'url-to-markdown:host'\n\n/** Ceiling on live host limiters, and so on the closures the gauge retains. */\nexport const URL_TO_MARKDOWN_HOST_LIMIT_CACHE_MAX = 512\n\ntype HostEntry = {\n\tlimit: ConcurrencyLimit\n\tinFlight: number\n\t/** Set by the host's first 429 (`coolDownHost`); every read passes it too. */\n\tthrottle: ConcurrencyLimit | null\n\tthrottledUntil: number\n}\n\n/** Limiters are per host and created on demand; one per request would leak pumps. */\nexport function createHostLimits(\n\tcreateLimit: CreateConcurrencyLimit = createMemoryConcurrencyLimit\n) {\n\t// Insertion order is maintained as LRU order: a hit re-inserts, so the\n\t// eviction scan starts at the host we have not touched in the longest.\n\tconst entries = new Map<string, HostEntry>()\n\n\tfunction evictIdle() {\n\t\tif (entries.size < URL_TO_MARKDOWN_HOST_LIMIT_CACHE_MAX) return\n\t\tfor (const [host, entry] of entries) {\n\t\t\tif (entry.inFlight > 0) continue\n\t\t\t// Disposing drops the metric registration, and with it the closure\n\t\t\t// that would otherwise retain this limiter for the life of the process.\n\t\t\tentry.limit.dispose()\n\t\t\tentry.throttle?.dispose()\n\t\t\tentries.delete(host)\n\t\t\treturn\n\t\t}\n\t\t// Every cached host is busy. Running over the cap beats disposing a\n\t\t// limiter whose slots are still held and still being heartbeated.\n\t}\n\n\tfunction getEntry(hostname: string) {\n\t\tconst key = hostname.toLowerCase()\n\t\tconst cached = entries.get(key)\n\t\tif (cached) {\n\t\t\tentries.delete(key)\n\t\t\tentries.set(key, cached)\n\t\t\treturn cached\n\t\t}\n\t\tevictIdle()\n\t\tconst entry: HostEntry = {\n\t\t\tlimit: createLimit({\n\t\t\t\tkey: `url-to-markdown:host:${key}`,\n\t\t\t\tmetricLabel: HOST_METRIC_LABEL,\n\t\t\t\tmax: URL_TO_MARKDOWN_HOST_MAX_CONCURRENCY,\n\t\t\t\t// Origin fetches time out at 30s; the TTL only has to outlive that.\n\t\t\t\tslotTtlMs: 45_000\n\t\t\t}),\n\t\t\tinFlight: 0,\n\t\t\tthrottle: null,\n\t\t\tthrottledUntil: 0\n\t\t}\n\t\tentries.set(key, entry)\n\t\treturn entry\n\t}\n\n\treturn {\n\t\tasync withHostLimit<T>(hostname: string, run: () => Promise<T>) {\n\t\t\tconst entry = getEntry(hostname)\n\t\t\t// Counted around the whole wait+run so eviction can tell a limiter that\n\t\t\t// is merely cached from one that is holding slots right now.\n\t\t\tentry.inFlight++\n\t\t\ttry {\n\t\t\t\treturn await entry.limit(() => {\n\t\t\t\t\t// A lapsed throttle is kept for the host's next 429; eviction disposes it.\n\t\t\t\t\tconst throttle =\n\t\t\t\t\t\tDate.now() < entry.throttledUntil ? entry.throttle : null\n\t\t\t\t\treturn throttle ? throttle(run) : run()\n\t\t\t\t})\n\t\t\t} finally {\n\t\t\t\tentry.inFlight--\n\t\t\t}\n\t\t},\n\n\t\t/**\n\t\t * The host answered 429: from now on it is read at the throttled cap, and\n\t\t * nothing is sent to it for `waitMs`. A shared limiter shares the pause, so\n\t\t * every worker reading the host waits it out. True when this 429 is what\n\t\t * narrowed the host.\n\t\t */\n\t\tasync coolDownHost(hostname: string, waitMs: number) {\n\t\t\tconst entry = getEntry(hostname)\n\t\t\tconst narrowed = Date.now() >= entry.throttledUntil\n\t\t\tentry.throttle ??= createLimit({\n\t\t\t\tkey: `url-to-markdown:host:${hostname.toLowerCase()}:throttled`,\n\t\t\t\tmetricLabel: HOST_METRIC_LABEL,\n\t\t\t\tmax: URL_TO_MARKDOWN_THROTTLED_HOST_MAX_CONCURRENCY,\n\t\t\t\tmaxStartsPerSecond: THROTTLED_HOST_STARTS_PER_SECOND,\n\t\t\t\tslotTtlMs: 45_000\n\t\t\t})\n\t\t\tentry.throttledUntil = Date.now() + HOST_THROTTLE_MS\n\t\t\tawait entry.throttle.tripCooldown(waitMs)\n\t\t\treturn narrowed\n\t\t}\n\t}\n}\n"]}
+{"version":3,"file":"limit.js","sourceRoot":"","sources":["../src/limit.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,4BAA4B,EAAE,MAAM,mBAAmB,CAAA;AAgChE;;;;GAIG;AACH,MAAM,CAAC,MAAM,gCAAgC,GAAG,EAAE,CAAA;AAElD;;;;GAIG;AACH,MAAM,CAAC,MAAM,oCAAoC,GAAG,EAAE,CAAA;AACtD,MAAM,CAAC,MAAM,4CAA4C,GAAG,EAAE,CAAA;AAE9D;;;;GAIG;AACH,MAAM,CAAC,MAAM,6BAA6B,GAAG,EAAE,CAAA;AAC/C,MAAM,CAAC,MAAM,qCAAqC,GAAG,EAAE,CAAA;AAEvD,yEAAyE;AACzE,MAAM,UAAU,sBAAsB,CAAC,WAAmC;IACzE,OAAO;QACN,gBAAgB,EAAE,WAAW,CAAC;YAC7B,GAAG,EAAE,8BAA8B;YACnC,GAAG,EAAE,4CAA4C;YACjD,kBAAkB,EAAE,oCAAoC;YACxD,6EAA6E;YAC7E,SAAS,EAAE,MAAM;SACjB,CAAC;QACF,SAAS,EAAE,WAAW,CAAC;YACtB,GAAG,EAAE,uBAAuB;YAC5B,GAAG,EAAE,qCAAqC;YAC1C,kBAAkB,EAAE,6BAA6B;YACjD,SAAS,EAAE,MAAM;SACjB,CAAC;KACF,CAAA;AACF,CAAC;AAED;;;;;;;;;;;;;GAaG;AACH,MAAM,CAAC,MAAM,oCAAoC,GAAG,EAAE,CAAA;AAEtD;;;;;GAKG;AACH,MAAM,CAAC,MAAM,8CAA8C,GAAG,CAAC,CAAA;AAC/D,MAAM,gCAAgC,GAAG,CAAC,CAAA;AAE1C,oFAAoF;AACpF,MAAM,gBAAgB,GAAG,EAAE,GAAG,EAAE,GAAG,IAAI,CAAA;AAEvC;;;;GAIG;AACH,MAAM,iBAAiB,GAAG,sBAAsB,CAAA;AAEhD,+EAA+E;AAC/E,MAAM,CAAC,MAAM,oCAAoC,GAAG,GAAG,CAAA;AAUvD,qFAAqF;AACrF,MAAM,UAAU,gBAAgB,CAC/B,cAAsC,4BAA4B;IAElE,uEAAuE;IACvE,uEAAuE;IACvE,MAAM,OAAO,GAAG,IAAI,GAAG,EAAqB,CAAA;IAE5C,SAAS,SAAS;QACjB,IAAI,OAAO,CAAC,IAAI,GAAG,oCAAoC;YAAE,OAAM;QAC/D,KAAK,MAAM,CAAC,IAAI,EAAE,KAAK,CAAC,IAAI,OAAO,EAAE,CAAC;YACrC,IAAI,KAAK,CAAC,QAAQ,GAAG,CAAC;gBAAE,SAAQ;YAChC,mEAAmE;YACnE,wEAAwE;YACxE,KAAK,CAAC,KAAK,CAAC,OAAO,EAAE,CAAA;YACrB,KAAK,CAAC,QAAQ,EAAE,OAAO,EAAE,CAAA;YACzB,OAAO,CAAC,MAAM,CAAC,IAAI,CAAC,CAAA;YACpB,OAAM;QACP,CAAC;QACD,oEAAoE;QACpE,kEAAkE;IACnE,CAAC;IAED,SAAS,QAAQ,CAAC,QAAgB;QACjC,MAAM,GAAG,GAAG,QAAQ,CAAC,WAAW,EAAE,CAAA;QAClC,MAAM,MAAM,GAAG,OAAO,CAAC,GAAG,CAAC,GAAG,CAAC,CAAA;QAC/B,IAAI,MAAM,EAAE,CAAC;YACZ,OAAO,CAAC,MAAM,CAAC,GAAG,CAAC,CAAA;YACnB,OAAO,CAAC,GAAG,CAAC,GAAG,EAAE,MAAM,CAAC,CAAA;YACxB,OAAO,MAAM,CAAA;QACd,CAAC;QACD,SAAS,EAAE,CAAA;QACX,MAAM,KAAK,GAAc;YACxB,KAAK,EAAE,WAAW,CAAC;gBAClB,GAAG,EAAE,wBAAwB,GAAG,EAAE;gBAClC,WAAW,EAAE,iBAAiB;gBAC9B,GAAG,EAAE,oCAAoC;gBACzC,oEAAoE;gBACpE,SAAS,EAAE,MAAM;aACjB,CAAC;YACF,QAAQ,EAAE,CAAC;YACX,QAAQ,EAAE,IAAI;YACd,cAAc,EAAE,CAAC;SACjB,CAAA;QACD,OAAO,CAAC,GAAG,CAAC,GAAG,EAAE,KAAK,CAAC,CAAA;QACvB,OAAO,KAAK,CAAA;IACb,CAAC;IAED,OAAO;QACN,KAAK,CAAC,aAAa,CAAI,QAAgB,EAAE,GAAqB;YAC7D,MAAM,KAAK,GAAG,QAAQ,CAAC,QAAQ,CAAC,CAAA;YAChC,wEAAwE;YACxE,6DAA6D;YAC7D,KAAK,CAAC,QAAQ,EAAE,CAAA;YAChB,IAAI,CAAC;gBACJ,OAAO,MAAM,KAAK,CAAC,KAAK,CAAC,GAAG,EAAE;oBAC7B,2EAA2E;oBAC3E,MAAM,QAAQ,GACb,IAAI,CAAC,GAAG,EAAE,GAAG,KAAK,CAAC,cAAc,CAAC,CAAC,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAC,CAAC,IAAI,CAAA;oBAC1D,OAAO,QAAQ,CAAC,CAAC,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,EAAE,CAAA;gBACxC,CAAC,CAAC,CAAA;YACH,CAAC;oBAAS,CAAC;gBACV,KAAK,CAAC,QAAQ,EAAE,CAAA;YACjB,CAAC;QACF,CAAC;QAED;;;;;WAKG;QACH,KAAK,CAAC,YAAY,CAAC,QAAgB,EAAE,MAAc;YAClD,MAAM,KAAK,GAAG,QAAQ,CAAC,QAAQ,CAAC,CAAA;YAChC,MAAM,QAAQ,GAAG,IAAI,CAAC,GAAG,EAAE,IAAI,KAAK,CAAC,cAAc,CAAA;YACnD,KAAK,CAAC,QAAQ,KAAK,WAAW,CAAC;gBAC9B,GAAG,EAAE,wBAAwB,QAAQ,CAAC,WAAW,EAAE,YAAY;gBAC/D,WAAW,EAAE,iBAAiB;gBAC9B,GAAG,EAAE,8CAA8C;gBACnD,kBAAkB,EAAE,gCAAgC;gBACpD,SAAS,EAAE,MAAM;aACjB,CAAC,CAAA;YACF,KAAK,CAAC,cAAc,GAAG,IAAI,CAAC,GAAG,EAAE,GAAG,gBAAgB,CAAA;YACpD,MAAM,KAAK,CAAC,QAAQ,CAAC,YAAY,CAAC,MAAM,CAAC,CAAA;YACzC,OAAO,QAAQ,CAAA;QAChB,CAAC;KACD,CAAA;AACF,CAAC","sourcesContent":["import { createMemoryConcurrencyLimit } from './memory-limit.js'\n\n/**\n * A semaphore: at most `max` callbacks in flight. `hanzio/redis`'s\n * `createRedisConcurrencyLimit` returns this shape (plus `queueDepth`, which is\n * not needed here) once `send` is bound; the default is\n * {@link createMemoryConcurrencyLimit}.\n */\nexport type ConcurrencyLimit = {\n\t<T>(run: () => Promise<T>, signal?: AbortSignal): Promise<T>\n\t/** Pause every start for `waitMs`, e.g. from an upstream `Retry-After`. */\n\ttripCooldown: (waitMs: number) => Promise<void>\n\t/** Drop the limiter's timers and metric registrations; it is discarded. */\n\tdispose: () => void\n}\n\nexport type ConcurrencyLimitOptions = {\n\t/** Shared limiters key their state on this; one per pool or host. */\n\tkey: string\n\tmax: number\n\t/** How long a slot held by a crashed process stays taken. */\n\tslotTtlMs: number\n\t/** A separate sliding one-second window of starts (provider rps quotas). */\n\tmaxStartsPerSecond?: number\n\t/** The pool a per-host limiter reports metrics under, instead of its key. */\n\tmetricLabel?: string\n}\n\nexport type CreateConcurrencyLimit = (\n\toptions: ConcurrencyLimitOptions\n) => ConcurrencyLimit\n\n/**\n * Page-level fetch parallelism. Content negotiation is a plain GET and is not\n * a Cloudflare API call, so this can sit above the Workers Paid start rates.\n * Workers AI and Browser Rendering still queue on their own pools.\n */\nexport const URL_TO_MARKDOWN_PAGE_CONCURRENCY = 80\n\n/**\n * Workers Paid Quick Actions are 30 rps, not 30 in-flight. Leave a little\n * headroom so two processes cannot land a burst on the same clock second.\n * Concurrency is higher so a 10-20s render does not leave the rps budget idle.\n */\nexport const CLOUDFLARE_BROWSER_RENDERING_MAX_RPS = 28\nexport const CLOUDFLARE_BROWSER_RENDERING_MAX_CONCURRENCY = 80\n\n/**\n * toMarkdown has no published rps cap. Stay below the Client API 200/s IP\n * burst and let `tripCooldown` pull the fleet back if the 1200/5min token\n * window applies.\n */\nexport const CLOUDFLARE_WORKERS_AI_MAX_RPS = 12\nexport const CLOUDFLARE_WORKERS_AI_MAX_CONCURRENCY = 24\n\n/** The two Cloudflare pools; a page can wait on both and on its host. */\nexport function createCloudflareLimits(createLimit: CreateConcurrencyLimit) {\n\treturn {\n\t\tbrowserRendering: createLimit({\n\t\t\tkey: 'cloudflare-browser-rendering',\n\t\t\tmax: CLOUDFLARE_BROWSER_RENDERING_MAX_CONCURRENCY,\n\t\t\tmaxStartsPerSecond: CLOUDFLARE_BROWSER_RENDERING_MAX_RPS,\n\t\t\t// Browser Rendering can run up to 60s; TTL is crash detection, not a budget.\n\t\t\tslotTtlMs: 90_000\n\t\t}),\n\t\tworkersAi: createLimit({\n\t\t\tkey: 'cloudflare-workers-ai',\n\t\t\tmax: CLOUDFLARE_WORKERS_AI_MAX_CONCURRENCY,\n\t\t\tmaxStartsPerSecond: CLOUDFLARE_WORKERS_AI_MAX_RPS,\n\t\t\tslotTtlMs: 45_000\n\t\t})\n\t}\n}\n\n/**\n * Politeness cap per customer host. This is the only thing standing\n * between an 80-wide page pool and one small WordPress box, and it applies to a\n * SOLO import as much as a concurrent one: two orgs are two domains and never\n * contend, so the burst this prevents is a single site's own crawl.\n *\n * Deliberately generous rather than cautious - a healthy origin should be read\n * at speed, and 500 pages at 16 in flight is well inside the run timeout. It\n * stays a cap because the failure it prevents is silent and un-retryable: a WAF\n * that decides we are an attack blocks the whole import, and the customer-site\n * probe already found an origin that 403s on User-Agent alone.\n *\n * Separate from the Cloudflare pools - a page can wait on both.\n */\nexport const URL_TO_MARKDOWN_HOST_MAX_CONCURRENCY = 16\n\n/**\n * A host that answered 429 is read this narrowly from then on. At 16 in\n * flight Shopify refused 117 of Good Habit's 244 reads even after each had\n * waited out its pause: the pause only delayed the next burst. Calibration\n * knobs - set them from the crawl log's `rate_limited`, not by theory.\n */\nexport const URL_TO_MARKDOWN_THROTTLED_HOST_MAX_CONCURRENCY = 2\nconst THROTTLED_HOST_STARTS_PER_SECOND = 2\n\n/** A throttled host is read at the full cap again after this long without a 429. */\nconst HOST_THROTTLE_MS = 15 * 60 * 1000\n\n/**\n * Every host limiter meters under this one label. Per-host series would grow\n * with the customer base and never shrink, and the question the metric answers\n * - is the crawler backed up behind origins - is a pool-wide question anyway.\n */\nconst HOST_METRIC_LABEL = 'url-to-markdown:host'\n\n/** Ceiling on live host limiters, and so on the closures the gauge retains. */\nexport const URL_TO_MARKDOWN_HOST_LIMIT_CACHE_MAX = 512\n\ntype HostEntry = {\n\tlimit: ConcurrencyLimit\n\tinFlight: number\n\t/** Set by the host's first 429 (`coolDownHost`); every read passes it too. */\n\tthrottle: ConcurrencyLimit | null\n\tthrottledUntil: number\n}\n\n/** Limiters are per host and created on demand; one per request would leak pumps. */\nexport function createHostLimits(\n\tcreateLimit: CreateConcurrencyLimit = createMemoryConcurrencyLimit\n) {\n\t// Insertion order is maintained as LRU order: a hit re-inserts, so the\n\t// eviction scan starts at the host we have not touched in the longest.\n\tconst entries = new Map<string, HostEntry>()\n\n\tfunction evictIdle() {\n\t\tif (entries.size < URL_TO_MARKDOWN_HOST_LIMIT_CACHE_MAX) return\n\t\tfor (const [host, entry] of entries) {\n\t\t\tif (entry.inFlight > 0) continue\n\t\t\t// Disposing drops the metric registration, and with it the closure\n\t\t\t// that would otherwise retain this limiter for the life of the process.\n\t\t\tentry.limit.dispose()\n\t\t\tentry.throttle?.dispose()\n\t\t\tentries.delete(host)\n\t\t\treturn\n\t\t}\n\t\t// Every cached host is busy. Running over the cap beats disposing a\n\t\t// limiter whose slots are still held and still being heartbeated.\n\t}\n\n\tfunction getEntry(hostname: string) {\n\t\tconst key = hostname.toLowerCase()\n\t\tconst cached = entries.get(key)\n\t\tif (cached) {\n\t\t\tentries.delete(key)\n\t\t\tentries.set(key, cached)\n\t\t\treturn cached\n\t\t}\n\t\tevictIdle()\n\t\tconst entry: HostEntry = {\n\t\t\tlimit: createLimit({\n\t\t\t\tkey: `url-to-markdown:host:${key}`,\n\t\t\t\tmetricLabel: HOST_METRIC_LABEL,\n\t\t\t\tmax: URL_TO_MARKDOWN_HOST_MAX_CONCURRENCY,\n\t\t\t\t// Origin fetches time out at 30s; the TTL only has to outlive that.\n\t\t\t\tslotTtlMs: 45_000\n\t\t\t}),\n\t\t\tinFlight: 0,\n\t\t\tthrottle: null,\n\t\t\tthrottledUntil: 0\n\t\t}\n\t\tentries.set(key, entry)\n\t\treturn entry\n\t}\n\n\treturn {\n\t\tasync withHostLimit<T>(hostname: string, run: () => Promise<T>) {\n\t\t\tconst entry = getEntry(hostname)\n\t\t\t// Counted around the whole wait+run so eviction can tell a limiter that\n\t\t\t// is merely cached from one that is holding slots right now.\n\t\t\tentry.inFlight++\n\t\t\ttry {\n\t\t\t\treturn await entry.limit(() => {\n\t\t\t\t\t// A lapsed throttle is kept for the host's next 429; eviction disposes it.\n\t\t\t\t\tconst throttle =\n\t\t\t\t\t\tDate.now() < entry.throttledUntil ? entry.throttle : null\n\t\t\t\t\treturn throttle ? throttle(run) : run()\n\t\t\t\t})\n\t\t\t} finally {\n\t\t\t\tentry.inFlight--\n\t\t\t}\n\t\t},\n\n\t\t/**\n\t\t * The host answered 429: from now on it is read at the throttled cap, and\n\t\t * nothing is sent to it for `waitMs`. A shared limiter shares the pause, so\n\t\t * every worker reading the host waits it out. True when this 429 is what\n\t\t * narrowed the host.\n\t\t */\n\t\tasync coolDownHost(hostname: string, waitMs: number) {\n\t\t\tconst entry = getEntry(hostname)\n\t\t\tconst narrowed = Date.now() >= entry.throttledUntil\n\t\t\tentry.throttle ??= createLimit({\n\t\t\t\tkey: `url-to-markdown:host:${hostname.toLowerCase()}:throttled`,\n\t\t\t\tmetricLabel: HOST_METRIC_LABEL,\n\t\t\t\tmax: URL_TO_MARKDOWN_THROTTLED_HOST_MAX_CONCURRENCY,\n\t\t\t\tmaxStartsPerSecond: THROTTLED_HOST_STARTS_PER_SECOND,\n\t\t\t\tslotTtlMs: 45_000\n\t\t\t})\n\t\t\tentry.throttledUntil = Date.now() + HOST_THROTTLE_MS\n\t\t\tawait entry.throttle.tripCooldown(waitMs)\n\t\t\treturn narrowed\n\t\t}\n\t}\n}\n"]}

package/dist/report-api-error.d.ts

@@ -0,0 +1,9 @@
+import { type ApiError } from 'hanzio/api-wrapper';
+/**
+ * Whether `onApiError` hears about a Workers AI client error. A 429 never: it
+ * is retried and trips the shared cooldown, and reporting every one floods the
+ * error tracker. A timeout only with `reportTimeouts`, since the page falls
+ * through to the next tier anyway.
+ */
+export declare function isReportableApiError(error: ApiError, reportTimeouts?: boolean): boolean;
+//# sourceMappingURL=report-api-error.d.ts.map

package/dist/report-api-error.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"report-api-error.d.ts","sourceRoot":"","sources":["../src/report-api-error.ts"],"names":[],"mappings":"AAAA,OAAO,EACN,KAAK,QAAQ,EAGb,MAAM,oBAAoB,CAAA;AAE3B;;;;;GAKG;AACH,wBAAgB,oBAAoB,CAAC,KAAK,EAAE,QAAQ,EAAE,cAAc,UAAO,WAG1E"}

package/dist/report-api-error.js

@@ -0,0 +1,13 @@
+import { HttpResponseError, RequestTimeoutError } from 'hanzio/api-wrapper';
+/**
+ * Whether `onApiError` hears about a Workers AI client error. A 429 never: it
+ * is retried and trips the shared cooldown, and reporting every one floods the
+ * error tracker. A timeout only with `reportTimeouts`, since the page falls
+ * through to the next tier anyway.
+ */
+export function isReportableApiError(error, reportTimeouts = true) {
+    if (error instanceof HttpResponseError && error.status === 429)
+        return false;
+    return reportTimeouts || !(error instanceof RequestTimeoutError);
+}
+//# sourceMappingURL=report-api-error.js.map

package/dist/report-api-error.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"report-api-error.js","sourceRoot":"","sources":["../src/report-api-error.ts"],"names":[],"mappings":"AAAA,OAAO,EAEN,iBAAiB,EACjB,mBAAmB,EACnB,MAAM,oBAAoB,CAAA;AAE3B;;;;;GAKG;AACH,MAAM,UAAU,oBAAoB,CAAC,KAAe,EAAE,cAAc,GAAG,IAAI;IAC1E,IAAI,KAAK,YAAY,iBAAiB,IAAI,KAAK,CAAC,MAAM,KAAK,GAAG;QAAE,OAAO,KAAK,CAAA;IAC5E,OAAO,cAAc,IAAI,CAAC,CAAC,KAAK,YAAY,mBAAmB,CAAC,CAAA;AACjE,CAAC","sourcesContent":["import {\n\ttype ApiError,\n\tHttpResponseError,\n\tRequestTimeoutError\n} from 'hanzio/api-wrapper'\n\n/**\n * Whether `onApiError` hears about a Workers AI client error. A 429 never: it\n * is retried and trips the shared cooldown, and reporting every one floods the\n * error tracker. A timeout only with `reportTimeouts`, since the page falls\n * through to the next tier anyway.\n */\nexport function isReportableApiError(error: ApiError, reportTimeouts = true) {\n\tif (error instanceof HttpResponseError && error.status === 429) return false\n\treturn reportTimeouts || !(error instanceof RequestTimeoutError)\n}\n"]}

package/dist/schema/index.d.ts

@@ -0,0 +1,12 @@
+/**
+ * Values a shared schema can import without the runtime: no Node built-ins,
+ * no peer libraries.
+ */
+/** A crawl snapshot name, `<snapshot>/<host>`; it becomes a path, so it may not climb out of the root. */
+export declare const CRAWL_SNAPSHOT_NAME_PATTERN: RegExp;
+export type CrawlSnapshotInput = {
+    mode: 'record' | 'replay';
+    /** `<snapshot>/<host>`: the bench's snapshot label, then the site. */
+    name: string;
+};
+//# sourceMappingURL=index.d.ts.map

package/dist/schema/index.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/schema/index.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAEH,0GAA0G;AAC1G,eAAO,MAAM,2BAA2B,QACE,CAAA;AAE1C,MAAM,MAAM,kBAAkB,GAAG;IAChC,IAAI,EAAE,QAAQ,GAAG,QAAQ,CAAA;IACzB,sEAAsE;IACtE,IAAI,EAAE,MAAM,CAAA;CACZ,CAAA"}

package/dist/schema/index.js

@@ -0,0 +1,7 @@
+/**
+ * Values a shared schema can import without the runtime: no Node built-ins,
+ * no peer libraries.
+ */
+/** A crawl snapshot name, `<snapshot>/<host>`; it becomes a path, so it may not climb out of the root. */
+export const CRAWL_SNAPSHOT_NAME_PATTERN = /^[a-z0-9._-]{1,80}\/[a-z0-9._-]{1,120}$/;
+//# sourceMappingURL=index.js.map

package/dist/schema/index.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.js","sourceRoot":"","sources":["../../src/schema/index.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAEH,0GAA0G;AAC1G,MAAM,CAAC,MAAM,2BAA2B,GACvC,yCAAyC,CAAA","sourcesContent":["/**\n * Values a shared schema can import without the runtime: no Node built-ins,\n * no peer libraries.\n */\n\n/** A crawl snapshot name, `<snapshot>/<host>`; it becomes a path, so it may not climb out of the root. */\nexport const CRAWL_SNAPSHOT_NAME_PATTERN =\n\t/^[a-z0-9._-]{1,80}\\/[a-z0-9._-]{1,120}$/\n\nexport type CrawlSnapshotInput = {\n\tmode: 'record' | 'replay'\n\t/** `<snapshot>/<host>`: the bench's snapshot label, then the site. */\n\tname: string\n}\n"]}

package/package.json

@@ -1,6 +1,6 @@
 {
 	"name": "website-to-markdown",
-	"version": "1.0.0",
+	"version": "1.1.0",
 	"description": "Turn URLs, HTML and files into markdown, with sitemap discovery and per-host limits.",
 	"type": "module",
 	"license": "MIT",
@@ -31,6 +31,10 @@
 		"./sitemap": {
 			"types": "./dist/sitemap/index.d.ts",
 			"import": "./dist/sitemap/index.js"
+		},
+		"./schema": {
+			"types": "./dist/schema/index.d.ts",
+			"import": "./dist/schema/index.js"
 		}
 	},
 	"bin": {
@@ -62,16 +66,17 @@
 		"fflate": "^0.8.2"
 	},
 	"peerDependencies": {
-		"hanzio": "^3.0.0",
+		"hanzio": "^3.1.0",
 		"zod": "^4.0.0"
 	},
 	"devDependencies": {
+		"@arethetypeswrong/cli": "^0.18.2",
 		"@biomejs/biome": "^2.2.4",
 		"@types/bun": "^1.2.0",
-		"typescript": "^5.9.2",
-		"@arethetypeswrong/cli": "^0.18.2",
-		"hanzio": "^3.0.0",
+		"@types/node": "^24",
+		"hanzio": "^3.1.0",
 		"publint": "^0.3.12",
+		"typescript": "^5.9.2",
 		"zod": "^4.4.3"
 	},
 	"bugs": {