website-to-markdown 1.1.0 → 2.0.0

package/dist/fetch-page.js.map

@@ -1 +1 @@
-{"version":3,"file":"fetch-page.js","sourceRoot":"","sources":["../src/fetch-page.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,eAAe,EAAE,MAAM,QAAQ,CAAA;AACxC,OAAO,EAEN,0BAA0B,EAC1B,sBAAsB,EACtB,MAAM,aAAa,CAAA;AACpB,OAAO,EACN,mBAAmB,EAEnB,uBAAuB,EACvB,MAAM,mBAAmB,CAAA;AAC1B,OAAO,EAGN,oBAAoB,EACpB,oBAAoB,EACpB,iBAAiB,EACjB,MAAM,qBAAqB,CAAA;AAC5B,OAAO,EACN,kBAAkB,EAClB,eAAe,EACf,aAAa,EACb,MAAM,sBAAsB,CAAA;AAC7B,OAAO,EAAE,SAAS,EAAE,MAAM,WAAW,CAAA;AACrC,OAAO,EAAE,mBAAmB,EAAE,MAAM,oBAAoB,CAAA;AACxD,OAAO,EAA+B,gBAAgB,EAAE,MAAM,YAAY,CAAA;AAC1E,OAAO,EAAE,kBAAkB,EAAe,YAAY,EAAE,MAAM,aAAa,CAAA;AAC3E,OAAO,EACN,UAAU,EACV,SAAS,EACT,YAAY,EACZ,qBAAqB,EACrB,gBAAgB,EAChB,iBAAiB,EACjB,MAAM,oBAAoB,CAAA;AAC3B,OAAO,EAAE,4BAA4B,EAAE,MAAM,mBAAmB,CAAA;AAChE,OAAO,EAAE,aAAa,EAAE,mBAAmB,EAAE,MAAM,gBAAgB,CAAA;AACnE,OAAO,EAAE,sBAAsB,EAAE,MAAM,uBAAuB,CAAA;AAC9D,OAAO,EAAE,kBAAkB,EAAE,MAAM,iBAAiB,CAAA;AACpD,OAAO,EAAE,iBAAiB,EAAE,MAAM,iBAAiB,CAAA;AAEnD,MAAM,gBAAgB,GAAG,MAAM,CAAA;AAC/B,+EAA+E;AAC/E,MAAM,cAAc,GAAG,CAAC,GAAG,IAAI,GAAG,IAAI,CAAA;AACtC,MAAM,kBAAkB,GAAG,CAAC,GAAG,IAAI,GAAG,IAAI,CAAA;AAC1C,MAAM,aAAa,GAAG,EAAE,GAAG,IAAI,GAAG,IAAI,CAAA;AACtC,MAAM,aAAa,GAAG,EAAE,CAAA;AAExB,0FAA0F;AAC1F,MAAM,CAAC,MAAM,kBAAkB,GAAG,OAAO,CAAA;AAiFzC;;;;GAIG;AACH,SAAS,SAAS,CACjB,IAAc,EACd,OAAiB,EACjB,MAAc;IAEd,0EAA0E;IAC1E,4EAA4E;IAC5E,2EAA2E;IAC3E,yEAAyE;IACzE,2EAA2E;IAC3E,0BAA0B;IAC1B,MAAM,KAAK,GAAG,CAAC,IAA+B,EAAE,EAAE,CACjD,IAAI,EAAE,UAAU,CAAC,QAAQ,EAAE,EAAE,CAAC,CAAC,SAAS,CAAC,KAAK,CAAC,CAAA;IAChD,MAAM,IAAI,GAAG;QACZ,GAAG,OAAO;QACV,QAAQ,EAAE,KAAK,CAAC,OAAO,CAAC,QAAQ,CAAC;QACjC,IAAI,EAAE,KAAK,CAAC,OAAO,CAAC,IAAI,CAAC;KACzB,CAAA;IACD,MAAM,WAAW,GAAG,gBAAgB,CAAC,IAAI,CAAC,QAAQ,IAAI,EAAE,CAAC,CAAA;IACzD,IAAI,QAAQ,GAAG,WAAW,CAAC,QAAQ,CAAA;IACnC,IAAI,QAAQ,CAAC,MAAM,GAAG,kBAAkB,EAAE,CAAC;QAC1C,MAAM,CAAC,IAAI,CAAC,2BAA2B,EAAE;YACxC,GAAG,EAAE,IAAI,CAAC,QAAQ;YAClB,KAAK,EAAE,QAAQ,CAAC,MAAM;SACtB,CAAC,CAAA;QACF,QAAQ,GAAG,QAAQ,CAAC,KAAK,CAAC,CAAC,EAAE,kBAAkB,CAAC,CAAA;IACjD,CAAC;IAED,MAAM,QAAQ,GAAG,IAAI,CAAC,IAAI;QACzB,CAAC,CAAC,mBAAmB,CAAC,IAAI,CAAC,IAAI,EAAE,IAAI,CAAC,QAAQ,CAAC;QAC/C,CAAC,CAAC,IAAI,CAAA;IACP,MAAM,IAAI,GAAG,YAAY,CAAC,QAAQ,EAAE,IAAI,CAAC,QAAQ,CAAC,CAAA;IAClD,sEAAsE;IACtE,wEAAwE;IACxE,0DAA0D;IAC1D,MAAM,KAAK,GAAG,IAAI,CAAC,IAAI;QACtB,CAAC,CAAC,CAAC,GAAG,IAAI,GAAG,CAAC,CAAC,GAAG,IAAI,CAAC,KAAK,EAAE,GAAG,SAAS,CAAC,IAAI,CAAC,IAAI,EAAE,IAAI,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC;QACvE,CAAC,CAAC,IAAI,CAAC,KAAK,CAAA;IAEb,OAAO;QACN,GAAG,EAAE,IAAI,CAAC,GAAG;QACb,QAAQ,EAAE,IAAI,CAAC,QAAQ;QACvB,MAAM,EAAE,IAAI,CAAC,MAAM;QACnB,OAAO,EAAE,IAAI,CAAC,OAAO;QACrB,MAAM,EAAE,IAAI,CAAC,MAAM;QACnB,WAAW,EAAE,IAAI,CAAC,WAAW;QAC7B,IAAI,EAAE,IAAI,CAAC,IAAI;QACf,YAAY,EAAE,IAAI,CAAC,YAAY;QAC/B,QAAQ;QACR,KAAK,EAAE,QAAQ,EAAE,KAAK,IAAI,WAAW,CAAC,KAAK,IAAI,iBAAiB,CAAC,QAAQ,CAAC;QAC1E,IAAI,EAAE,QAAQ,EAAE,IAAI,IAAI,IAAI;QAC5B,SAAS,EAAE,QAAQ,EAAE,SAAS,IAAI,IAAI;QACtC,UAAU,EAAE,QAAQ,EAAE,UAAU,IAAI,EAAE;QACtC,kBAAkB,EAAE,QAAQ,EAAE,kBAAkB,IAAI,EAAE;QACtD,eAAe,EACd,QAAQ,EAAE,eAAe,IAAI,WAAW,CAAC,WAAW,IAAI,IAAI;QAC7D,MAAM,EAAE,QAAQ,EAAE,MAAM,IAAI,EAAE;QAC9B,KAAK;QACL,MAAM,EAAE;YACP,GAAG,IAAI,GAAG,CAAC;gBACV,GAAG,IAAI,CAAC,MAAM;gBACd,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,UAAU,CAAC,IAAI,CAAC,IAAI,EAAE,IAAI,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;aAC1D,CAAC;SACF;QACD,YAAY,EACX,WAAW,CAAC,QAAQ,CAAC,MAAM,GAAG,kBAAkB;YAC/C,CAAC,CAAC,gBAAgB;YAClB,CAAC,CAAC,CAAC,IAAI,CAAC,YAAY,IAAI,IAAI,CAAC;QAC/B,UAAU,EAAE,IAAI,CAAC,KAAK,CAAC,WAAW,CAAC,GAAG,EAAE,GAAG,IAAI,CAAC,SAAS,CAAC;QAC1D,QAAQ,EAAE,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,IAAI;KAChD,CAAA;AACF,CAAC;AAED;;;;GAIG;AACH,SAAS,iBAAiB,CAAC,GAAW;IACrC,IAAI,MAAW,CAAA;IACf,IAAI,CAAC;QACJ,MAAM,GAAG,IAAI,GAAG,CAAC,GAAG,CAAC,CAAA;IACtB,CAAC;IAAC,MAAM,CAAC;QACR,OAAO,IAAI,CAAA;IACZ,CAAC;IACD,IAAI,MAAM,CAAC,QAAQ,KAAK,QAAQ;QAAE,OAAO,MAAM,CAAA;IAC/C,IAAI,MAAM,CAAC,QAAQ,KAAK,OAAO;QAAE,OAAO,IAAI,CAAA;IAC5C,IAAI,mBAAmB,CAAC,MAAM,CAAC,QAAQ,CAAC,EAAE,CAAC;QAC1C,OAAO,OAAO,CAAC,GAAG,CAAC,QAAQ,KAAK,YAAY,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,MAAM,CAAA;IAC7D,CAAC;IACD,MAAM,CAAC,QAAQ,GAAG,QAAQ,CAAA;IAC1B,IAAI,MAAM,CAAC,IAAI,KAAK,IAAI;QAAE,MAAM,CAAC,IAAI,GAAG,EAAE,CAAA;IAC1C,OAAO,MAAM,CAAA;AACd,CAAC;AAED;;;GAGG;AACH,SAAS,cAAc,CAAC,GAAQ;IAC/B,MAAM,IAAI,GAAG,GAAG,CAAC,QAAQ,CAAA;IACzB,IAAI,IAAI,CAAC,UAAU,CAAC,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC;QAAE,OAAO,IAAI,CAAA;IAC/D,IAAI,mBAAmB,CAAC,IAAI,CAAC,IAAI,UAAU,CAAC,IAAI,CAAC,IAAI,CAAC;QAAE,OAAO,IAAI,CAAA;IACnE,MAAM,IAAI,GAAG,IAAI,GAAG,CAAC,GAAG,CAAC,IAAI,CAAC,CAAA;IAC9B,IAAI,CAAC,QAAQ,GAAG,OAAO,IAAI,EAAE,CAAA;IAC7B,OAAO,IAAI,CAAA;AACZ,CAAC;AAED;;;;;;;GAOG;AACH,MAAM,sBAAsB,GAAG,CAAC,CAAA;AAChC,MAAM,sBAAsB,GAAG,MAAM,CAAA;AAErC,uEAAuE;AACvE,SAAS,eAAe,CAAC,QAAkB,EAAE,KAAa;IACzD,MAAM,OAAO,GAAG,eAAe,CAAC,QAAQ,CAAC,OAAO,CAAC,GAAG,CAAC,aAAa,CAAC,CAAC,CAAA;IACpE,OAAO,IAAI,CAAC,GAAG,CAAC,sBAAsB,EAAE,OAAO,IAAI,KAAK,GAAG,CAAC,IAAI,CAAC,KAAK,GAAG,CAAC,CAAC,CAAC,CAAA;AAC7E,CAAC;AAED;;;;;GAKG;AACH,MAAM,kBAAkB,GAAG,CAAC,CAAA;AAE5B,SAAS,QAAQ,CAAC,IAAY;IAC7B,OAAO,SAAS,CAAC,GAAG,kBAAkB,KAAK,IAAI,EAAE,CAAC,CAAA;AACnD,CAAC;AAED,iFAAiF;AACjF,SAAS,QAAQ,CAAC,GAAQ,EAAE,OAAkC;IAC7D,IAAI,CAAC,OAAO,IAAI,GAAG,CAAC,YAAY,CAAC,GAAG,CAAC,SAAS,CAAC;QAAE,OAAO,GAAG,CAAA;IAC3D,MAAM,MAAM,GAAG,IAAI,GAAG,CAAC,GAAG,CAAC,IAAI,CAAC,CAAA;IAChC,MAAM,CAAC,YAAY,CAAC,GAAG,CAAC,SAAS,EAAE,OAAO,CAAC,CAAA;IAC3C,OAAO,MAAM,CAAA;AACd,CAAC;AAED,gFAAgF;AAChF,SAAS,aAAa,CAAC,WAAmB,EAAE,UAAe;IAC1D,IAAI,CAAC,WAAW;QAAE,OAAO,UAAU,CAAA;IACnC,IAAI,CAAC;QACJ,OAAO,IAAI,GAAG,CAAC,WAAW,CAAC,CAAA;IAC5B,CAAC;IAAC,MAAM,CAAC;QACR,OAAO,UAAU,CAAA;IAClB,CAAC;AACF,CAAC;AAED,SAAS,YAAY,CAAC,KAAc;IACnC,MAAM,IAAI,GACT,OAAO,KAAK,KAAK,QAAQ,IAAI,KAAK,IAAI,MAAM,IAAI,KAAK;QACpD,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC;QACpB,CAAC,CAAC,EAAE,CAAA;IACN,OAAO,IAAI,KAAK,YAAY,IAAI,IAAI,KAAK,cAAc,CAAA;AACxD,CAAC;AAeD,mFAAmF;AACnF,SAAS,WAAW,CAAC,QAAkB;IACtC,KAAK,QAAQ,CAAC,IAAI,EAAE,MAAM,EAAE,CAAC,KAAK,CAAC,GAAG,EAAE,GAAE,CAAC,CAAC,CAAA;AAC7C,CAAC;AAED;;;;GAIG;AACH,KAAK,UAAU,cAAc,CAC5B,QAAkB,EAClB,QAAa,EACb,MAAc;IAEd,MAAM,MAAM,GAAG,QAAQ,CAAC,MAAM,CAAA;IAC9B,IAAI,MAAM,GAAG,GAAG,IAAI,MAAM,IAAI,GAAG,EAAE,CAAC;QACnC,WAAW,CAAC,QAAQ,CAAC,CAAA;QACrB,OAAO,EAAE,IAAI,EAAE,MAAM,EAAE,CAAA;IACxB,CAAC;IAED,MAAM,WAAW,GAAG,QAAQ,CAAC,OAAO,CAAC,GAAG,CAAC,cAAc,CAAC,CAAA;IACxD,MAAM,IAAI,GAAG,mBAAmB,CAAC,WAAW,CAAC,CAAA;IAC7C,IAAI,IAAI,KAAK,OAAO,EAAE,CAAC;QACtB,WAAW,CAAC,QAAQ,CAAC,CAAA;QACrB,OAAO,EAAE,IAAI,EAAE,MAAM,EAAE,CAAA;IACxB,CAAC;IACD,IAAI,IAAI,KAAK,UAAU,EAAE,CAAC;QACzB,MAAM,IAAI,GAAG,uBAAuB,CAAC,WAAW,CAAC,CAAA;QACjD,IAAI,CAAC,IAAI,EAAE,CAAC;YACX,WAAW,CAAC,QAAQ,CAAC,CAAA;YACrB,OAAO,EAAE,IAAI,EAAE,MAAM,EAAE,CAAA;QACxB,CAAC;QACD,MAAM,KAAK,GAAG,IAAI,CAAC,SAAS,KAAK,KAAK,CAAC,CAAC,CAAC,aAAa,CAAC,CAAC,CAAC,kBAAkB,CAAA;QAC3E,MAAM,QAAQ,GAAG,MAAM,CAAC,QAAQ,CAAC,OAAO,CAAC,GAAG,CAAC,gBAAgB,CAAC,CAAC,CAAA;QAC/D,IAAI,MAAM,CAAC,QAAQ,CAAC,QAAQ,CAAC,IAAI,QAAQ,GAAG,KAAK,EAAE,CAAC;YACnD,WAAW,CAAC,QAAQ,CAAC,CAAA;YACrB,MAAM,CAAC,IAAI,CAAC,oCAAoC,EAAE;gBACjD,GAAG,EAAE,QAAQ,CAAC,IAAI;gBAClB,KAAK,EAAE,QAAQ;aACf,CAAC,CAAA;YACF,OAAO,EAAE,IAAI,EAAE,MAAM,EAAE,YAAY,EAAE,qBAAqB,EAAE,CAAA;QAC7D,CAAC;QACD,MAAM,KAAK,GAAG,MAAM,eAAe,CAAC,QAAQ,EAAE,KAAK,CAAC,CAAA;QACpD,OAAO,KAAK;YACX,CAAC,CAAC,EAAE,IAAI,EAAE,OAAO,EAAE,KAAK,EAAE;YAC1B,CAAC,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,YAAY,EAAE,qBAAqB,EAAE,CAAA;IACzD,CAAC;IACD,OAAO,EAAE,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,MAAM,QAAQ,CAAC,IAAI,EAAE,EAAE,CAAA;AACrD,CAAC;AAED,SAAS,gBAAgB,CAAC,QAAgB,EAAE,SAAiB;IAC5D,MAAM,OAAO,GAAG,QAAQ,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,EAAE,KAAK,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,IAAI,UAAU,CAAA;IACzE,MAAM,IAAI,GAAG,OAAO,CAAC,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC,CAAA;IACjC,OAAO,IAAI,CAAC,WAAW,EAAE,CAAC,QAAQ,CAAC,IAAI,SAAS,EAAE,CAAC;QAClD,CAAC,CAAC,IAAI;QACN,CAAC,CAAC,GAAG,IAAI,IAAI,SAAS,EAAE,CAAA;AAC1B,CAAC;AAED,gFAAgF;AAChF,KAAK,UAAU,eAAe,CAC7B,QAAkB,EAClB,QAAgB;IAEhB,MAAM,MAAM,GAAG,QAAQ,CAAC,IAAI,EAAE,SAAS,EAAE,CAAA;IACzC,IAAI,CAAC,MAAM;QAAE,OAAO,IAAI,WAAW,CAAC,CAAC,CAAC,CAAA;IACtC,MAAM,MAAM,GAAiB,EAAE,CAAA;IAC/B,IAAI,IAAI,GAAG,CAAC,CAAA;IACZ,OAAO,IAAI,EAAE,CAAC;QACb,MAAM,IAAI,GAAG,MAAM,MAAM,CAAC,IAAI,EAAE,CAAA;QAChC,IAAI,IAAI,CAAC,IAAI;YAAE,MAAK;QACpB,IAAI,IAAI,IAAI,CAAC,KAAK,CAAC,UAAU,CAAA;QAC7B,IAAI,IAAI,GAAG,QAAQ,EAAE,CAAC;YACrB,MAAM,MAAM,CAAC,MAAM,EAAE,CAAA;YACrB,OAAO,IAAI,CAAA;QACZ,CAAC;QACD,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,CAAA;IACxB,CAAC;IACD,MAAM,KAAK,GAAG,IAAI,UAAU,CAAC,IAAI,CAAC,CAAA;IAClC,IAAI,MAAM,GAAG,CAAC,CAAA;IACd,KAAK,MAAM,KAAK,IAAI,MAAM,EAAE,CAAC;QAC5B,KAAK,CAAC,GAAG,CAAC,KAAK,EAAE,MAAM,CAAC,CAAA;QACxB,MAAM,IAAI,KAAK,CAAC,UAAU,CAAA;IAC3B,CAAC;IACD,OAAO,KAAK,CAAC,MAAM,CAAA;AACpB,CAAC;AAED;;;;;;;;;;;;;;;GAeG;AACH,MAAM,kBAAkB,GAAG;IAC1B,UAAU;IACV,oBAAoB;IACpB,0BAA0B;IAC1B,+BAA+B;IAC/B,gCAAgC;CAChC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;AAEZ,SAAS,eAAe,CAAC,IAAY;IACpC,OAAO,IAAI,YAAY,EAAE;SACvB,EAAE,CAAC,kBAAkB,EAAE;QACvB,OAAO,CAAC,OAAO;YACd,OAAO,CAAC,MAAM,EAAE,CAAA;QACjB,CAAC;KACD,CAAC;SACD,SAAS,CAAC,IAAI,CAAC,CAAA;AAClB,CAAC;AAED,+EAA+E;AAC/E,MAAM,iBAAiB,GAAG,IAAI,GAAG,CAAC,CAAC,GAAG,EAAE,GAAG,CAAC,CAAC,CAAA;AAE7C,4EAA4E;AAC5E,SAAS,aAAa,CAAC,EAAU,EAAE,MAA+B;IACjE,MAAM,OAAO,GAAG,WAAW,CAAC,OAAO,CAAC,EAAE,CAAC,CAAA;IACvC,OAAO,MAAM,CAAC,CAAC,CAAC,WAAW,CAAC,GAAG,CAAC,CAAC,OAAO,EAAE,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC,OAAO,CAAA;AAC7D,CAAC;AA8DD;;;GAGG;AACH,MAAM,UAAU,uBAAuB,CAAC,OAAiC;IACxE,MAAM,MAAM,GAAG,OAAO,CAAC,MAAM,IAAI,YAAY,CAAA;IAC7C,MAAM,WAAW,GAAG,OAAO,CAAC,WAAW,IAAI,4BAA4B,CAAA;IACvE,MAAM,MAAM,GAAG,sBAAsB,CAAC;QACrC,WAAW,EAAE,OAAO,CAAC,UAAU;QAC/B,WAAW;QACX,UAAU,EAAE,OAAO,CAAC,UAAU;QAC9B,cAAc,EAAE,OAAO,CAAC,cAAc;QACtC,MAAM;KACN,CAAC,CAAA;IACF,MAAM,UAAU,GAAG,gBAAgB,CAAC,WAAW,CAAC,CAAA;IAChD,MAAM,SAAS,GACd,OAAO,CAAC,aAAa;QACrB,CAAC,CAAC,QAAgB,EAAE,EAAE,CAAC,aAAa,CAAC,QAAQ,EAAE,MAAM,CAAC,CAAC,CAAA;IAExD,KAAK,UAAU,aAAa,CAC3B,GAAQ,EACR,OAA+B,EAC/B,MAAgC;QAEhC,IAAI,MAAM,EAAE,OAAO;YAAE,OAAO,EAAE,IAAI,EAAE,QAAQ,EAAE,SAAS,EAAE,KAAK,EAAE,CAAA;QAChE,IAAI,CAAC,CAAC,MAAM,SAAS,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC,EAAE,CAAC;YACtC,OAAO,EAAE,IAAI,EAAE,SAAS,EAAE,CAAA;QAC3B,CAAC;QACD,IAAI,CAAC;YACJ,wEAAwE;YACxE,0EAA0E;YAC1E,sEAAsE;YACtE,wEAAwE;YACxE,kDAAkD;YAClD,oEAAoE;YACpE,oEAAoE;YACpE,MAAM,IAAI,GAAG,MAAM,UAAU,CAAC,aAAa,CAAC,GAAG,CAAC,QAAQ,EAAE,KAAK,IAAI,EAAE;gBACpE,MAAM,QAAQ,GAAG,MAAM,KAAK,CAAC,GAAG,EAAE;oBACjC,QAAQ,EAAE,QAAQ;oBAClB,MAAM,EAAE,aAAa,CAAC,gBAAgB,EAAE,MAAM,CAAC;oBAC/C,OAAO;iBACP,CAAC,CAAA;gBACF,MAAM,QAAQ,GAAG,aAAa,CAAC,QAAQ,CAAC,GAAG,EAAE,GAAG,CAAC,CAAA;gBACjD,OAAO;oBACN,QAAQ;oBACR,QAAQ;oBACR,IAAI,EAAE,MAAM,cAAc,CAAC,QAAQ,EAAE,QAAQ,EAAE,MAAM,CAAC;iBACtD,CAAA;YACF,CAAC,CAAC,CAAA;YACF,OAAO,EAAE,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,CAAA;QAC9B,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YAChB,MAAM,CAAC,IAAI,CAAC,+BAA+B,EAAE;gBAC5C,GAAG,EAAE,GAAG,CAAC,IAAI;gBACb,GAAG,kBAAkB,CAAC,KAAK,CAAC;aAC5B,CAAC,CAAA;YACF,wEAAwE;YACxE,qDAAqD;YACrD,OAAO,EAAE,IAAI,EAAE,QAAQ,EAAE,SAAS,EAAE,CAAC,YAAY,CAAC,KAAK,CAAC,EAAE,CAAA;QAC3D,CAAC;IACF,CAAC;IAED;;;;;;;;OAQG;IACH,KAAK,UAAU,oBAAoB,CAAC,KAOnC;QACA,MAAM,EAAE,KAAK,EAAE,GAAG,KAAK,CAAA;QACvB,IAAI,CAAC,KAAK,CAAC,UAAU;YAAE,OAAO,EAAE,OAAO,EAAE,aAAa,EAAE,MAAM,EAAE,IAAI,EAAE,CAAA;QACtE,IAAI,KAAK,CAAC,MAAM,EAAE,OAAO;YAAE,OAAO,EAAE,OAAO,EAAE,WAAW,EAAE,MAAM,EAAE,IAAI,EAAE,CAAA;QACxE,IAAI,oBAAoB,GAAG,KAAK,CAAA;QAChC,IAAI,QAAQ,GAAkB,IAAI,CAAA;QAClC,IAAI,KAAK,CAAC,UAAU,IAAI,CAAC,KAAK,CAAC,iBAAiB,IAAI,kBAAkB,CAAC,EAAE,CAAC;YACzE,IAAI,CAAC;gBACJ,QAAQ,GAAG,MAAM,MAAM,CAAC,wBAAwB,CAAC;oBAChD,IAAI,EAAE,KAAK,CAAC,SAAS,KAAK,MAAM,CAAC,CAAC,CAAC,iBAAiB,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,KAAK;oBACnE,QAAQ,EAAE,KAAK,CAAC,QAAQ;oBACxB,QAAQ,EAAE,KAAK,CAAC,QAAQ;oBACxB,MAAM,EAAE,KAAK,CAAC,MAAM;iBACpB,CAAC,CAAA;YACH,CAAC;YAAC,OAAO,KAAK,EAAE,CAAC;gBAChB,IAAI,CAAC,CAAC,KAAK,YAAY,0BAA0B,CAAC;oBAAE,MAAM,KAAK,CAAA;gBAC/D,oBAAoB,GAAG,IAAI,CAAA;YAC5B,CAAC;QACF,CAAC;QACD,MAAM,IAAI,GAAG,qBAAqB,CAAC,QAAQ,IAAI,EAAE,CAAC,CAAA;QAClD,IAAI,CAAC,eAAe,CAAC,IAAI,CAAC;YACzB,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE,MAAM,EAAE,YAAY,EAAE,QAAQ,EAAE,IAAI,EAAE,CAAA;QAC/D,IAAI,KAAK,CAAC,SAAS,KAAK,KAAK,EAAE,CAAC;YAC/B,OAAO;gBACN,OAAO,EAAE,WAAW;gBACpB,MAAM,EAAE,IAAI;gBACZ,YAAY,EAAE,oBAAoB,CAAC,CAAC,CAAC,yBAAyB,CAAC,CAAC,CAAC,IAAI;aACrE,CAAA;QACF,CAAC;QACD,IAAI,KAAK,CAAC,MAAM,EAAE,OAAO;YAAE,OAAO,EAAE,OAAO,EAAE,WAAW,EAAE,MAAM,EAAE,IAAI,EAAE,CAAA;QACxE,IAAI,CAAC;YACJ,IAAI,CAAC,OAAO,CAAC,QAAQ;gBAAE,MAAM,IAAI,KAAK,CAAC,0BAA0B,CAAC,CAAA;YAClE,MAAM,MAAM,GAAG,MAAM,OAAO,CAAC,QAAQ,CAAC;gBACrC,KAAK;gBACL,QAAQ,EAAE,KAAK,CAAC,QAAQ;gBACxB,QAAQ,EAAE,KAAK,CAAC,QAAQ;gBACxB,QAAQ,EAAE,aAAa;gBACvB,MAAM,EAAE,KAAK,CAAC,MAAM;aACpB,CAAC,CAAA;YACF,OAAO;gBACN,OAAO,EAAE,IAAI;gBACb,MAAM,EAAE,iBAAiB;gBACzB,QAAQ,EAAE,MAAM,CAAC,QAAQ;gBACzB,2EAA2E;gBAC3E,0EAA0E;gBAC1E,YAAY,EACX,MAAM,CAAC,QAAQ,KAAK,IAAI,IAAI,MAAM,CAAC,QAAQ,IAAI,aAAa;oBAC3D,CAAC,CAAC,2BAA2B;oBAC7B,CAAC,CAAC,IAAI;aACR,CAAA;QACF,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YAChB,MAAM,CAAC,IAAI,CAAC,kCAAkC,EAAE;gBAC/C,QAAQ,EAAE,KAAK,CAAC,QAAQ;gBACxB,GAAG,kBAAkB,CAAC,KAAK,CAAC;aAC5B,CAAC,CAAA;YACF,OAAO;gBACN,OAAO,EAAE,WAAW;gBACpB,MAAM,EAAE,IAAI;gBACZ,YAAY,EAAE,oBAAoB;oBACjC,CAAC,CAAC,yBAAyB;oBAC3B,CAAC,CAAC,uBAAuB;aAC1B,CAAA;QACF,CAAC;IACF,CAAC;IAED,KAAK,UAAU,eAAe,CAAC,KAK9B;QACA,MAAM,IAAI,GAAG,uBAAuB,CAAC,KAAK,CAAC,WAAW,CAAC,CAAA;QACvD,IAAI,CAAC,IAAI,IAAI,KAAK,CAAC,IAAI,CAAC,IAAI,KAAK,OAAO,EAAE,CAAC;YAC1C,OAAO;gBACN,OAAO,EAAE,aAAa;gBACtB,MAAM,EAAE,IAAI;gBACZ,YAAY,EACX,KAAK,CAAC,IAAI,CAAC,IAAI,KAAK,MAAM,CAAC,CAAC,CAAC,KAAK,CAAC,IAAI,CAAC,YAAY,CAAC,CAAC,CAAC,IAAI;aAC5D,CAAA;QACF,CAAC;QACD,OAAO,oBAAoB,CAAC;YAC3B,KAAK,EAAE,KAAK,CAAC,IAAI,CAAC,KAAK;YACvB,QAAQ,EAAE,gBAAgB,CAAC,KAAK,CAAC,QAAQ,CAAC,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC;YAC/D,QAAQ,EAAE,IAAI,CAAC,QAAQ;YACvB,SAAS,EAAE,IAAI,CAAC,SAAS;YACzB,MAAM,EAAE,KAAK,CAAC,MAAM;SACpB,CAAC,CAAA;IACH,CAAC;IAED,KAAK,UAAU,cAAc,CAAC,KAQ7B;QACA,MAAM,IAAI,GAAG,KAAK,CAAC,IAAI,CAAA;QACvB,sEAAsE;QACtE,4EAA4E;QAC5E,sEAAsE;QACtE,MAAM,MAAM,GACX,KAAK,CAAC,IAAI,KAAK,MAAM;YACrB,CAAC,KAAK,CAAC,IAAI,KAAK,UAAU,IAAI,aAAa,CAAC,IAAI,CAAC,CAAC,CAAA;QAEnD,IAAI,CAAC,MAAM,IAAI,CAAC,eAAe,CAAC,IAAI,CAAC,EAAE,CAAC;YACvC,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE,MAAM,EAAE,qBAAqB,EAAE,QAAQ,EAAE,IAAI,EAAE,CAAA;QACxE,CAAC;QAED,MAAM,IAAI,GAAG,MAAM,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,CAAA;QACjC,yEAAyE;QACzE,yEAAyE;QACzE,IAAI,IAAI,IAAI,KAAK,CAAC,QAAQ,IAAI,QAAQ,CAAC,IAAI,CAAC,KAAK,KAAK,CAAC,QAAQ,EAAE,CAAC;YACjE,OAAO,EAAE,OAAO,EAAE,WAAW,EAAE,MAAM,EAAE,cAAc,EAAE,CAAA;QACxD,CAAC;QACD,IAAI,IAAI,IAAI,MAAM,CAAC,UAAU,CAAC,IAAI,CAAC,GAAG,cAAc,EAAE,CAAC;YACtD,mEAAmE;YACnE,yEAAyE;YACzE,kEAAkE;YAClE,MAAM,CAAC,IAAI,CAAC,gCAAgC,EAAE;gBAC7C,GAAG,EAAE,KAAK,CAAC,QAAQ,CAAC,IAAI;gBACxB,KAAK,EAAE,MAAM,CAAC,UAAU,CAAC,IAAI,CAAC;aAC9B,CAAC,CAAA;YACF,OAAO,EAAE,OAAO,EAAE,aAAa,EAAE,MAAM,EAAE,IAAI,EAAE,IAAI,EAAE,CAAA;QACtD,CAAC;QAED,IAAI,KAAK,GAAG,KAAK,CAAA;QACjB,IAAI,oBAAoB,GAAG,KAAK,CAAA;QAChC,IAAI,IAAI,EAAE,CAAC;YACV,MAAM,OAAO,GAAG,eAAe,CAAC,IAAI,CAAC,CAAA;YACrC,IAAI,SAAS,GAAkB,IAAI,CAAA;YACnC,IAAI,CAAC;gBACJ,SAAS,GAAG,MAAM,MAAM,CAAC,wBAAwB,CAAC;oBACjD,IAAI,EAAE,OAAO;oBACb,yEAAyE;oBACzE,QAAQ,EAAE,KAAK,CAAC,QAAQ,CAAC,QAAQ;oBACjC,MAAM,EAAE,KAAK,CAAC,MAAM;iBACpB,CAAC,CAAA;YACH,CAAC;YAAC,OAAO,KAAK,EAAE,CAAC;gBAChB,IAAI,CAAC,CAAC,KAAK,YAAY,0BAA0B,CAAC;oBAAE,MAAM,KAAK,CAAA;gBAC/D,oBAAoB,GAAG,IAAI,CAAA;YAC5B,CAAC;YACD,IAAI,SAAS,IAAI,CAAC,eAAe,CAAC,SAAS,CAAC,EAAE,CAAC;gBAC9C,KAAK,GAAG,kBAAkB,CAAC,OAAO,EAAE,SAAS,CAAC,CAAA;gBAC9C,IAAI,CAAC,KAAK;oBACT,OAAO;wBACN,OAAO,EAAE,IAAI;wBACb,MAAM,EAAE,YAAY;wBACpB,QAAQ,EAAE,SAAS;wBACnB,IAAI;qBACJ,CAAA;YACH,CAAC;QACF,CAAC;QAED,4EAA4E;QAC5E,yCAAyC;QACzC,IAAI,KAAK,CAAC,MAAM,IAAI,CAAC,KAAK,CAAC,MAAM,EAAE,OAAO,EAAE,CAAC;YAC5C,IAAI,QAAQ,GAAkB,IAAI,CAAA;YAClC,IAAI,CAAC;gBACJ,QAAQ,GAAG,MAAM,MAAM,CAAC,mBAAmB,CAC1C,KAAK,CAAC,QAAQ,CAAC,IAAI,EACnB,KAAK,CAAC,MAAM,CACZ,CAAA;YACF,CAAC;YAAC,OAAO,KAAK,EAAE,CAAC;gBAChB,IAAI,CAAC,CAAC,KAAK,YAAY,0BAA0B,CAAC;oBAAE,MAAM,KAAK,CAAA;gBAC/D,oBAAoB,GAAG,IAAI,CAAA;YAC5B,CAAC;YACD,IACC,QAAQ;gBACR,CAAC,eAAe,CAAC,QAAQ,CAAC;gBAC1B,CAAC,CAAC,IAAI,IAAI,CAAC,kBAAkB,CAAC,IAAI,EAAE,QAAQ,CAAC,CAAC,EAC7C,CAAC;gBACF,OAAO;oBACN,OAAO,EAAE,IAAI;oBACb,MAAM,EAAE,mBAAmB;oBAC3B,QAAQ,EAAE,QAAQ;oBAClB,IAAI;iBACJ,CAAA;YACF,CAAC;QACF,CAAC;QACD,OAAO;YACN,OAAO,EAAE,WAAW;YACpB,MAAM,EAAE,IAAI;YACZ,IAAI;YACJ,YAAY,EAAE,KAAK;gBAClB,CAAC,CAAC,mBAAmB;gBACrB,CAAC,CAAC,oBAAoB;oBACrB,CAAC,CAAC,yBAAyB;oBAC3B,CAAC,CAAC,IAAI;SACR,CAAA;IACF,CAAC;IAED,+EAA+E;IAC/E,KAAK,UAAU,YAAY,CAC1B,KAIC;QAED,MAAM,EAAE,QAAQ,EAAE,IAAI,EAAE,GAAG,KAAK,CAAA;QAChC,MAAM,MAAM,GAAG,QAAQ,CAAC,MAAM,CAAA;QAC9B,MAAM,WAAW,GAAG,QAAQ,CAAC,OAAO,CAAC,GAAG,CAAC,cAAc,CAAC,CAAA;QAExD,IAAI,MAAM,KAAK,GAAG;YAAE,OAAO,EAAE,OAAO,EAAE,WAAW,EAAE,MAAM,EAAE,cAAc,EAAE,CAAA;QAC3E,IAAI,MAAM,KAAK,GAAG,IAAI,MAAM,KAAK,GAAG,EAAE,CAAC;YACtC,OAAO,EAAE,OAAO,EAAE,MAAM,EAAE,MAAM,EAAE,IAAI,EAAE,CAAA;QACzC,CAAC;QACD,yEAAyE;QACzE,sCAAsC;QACtC,IAAI,MAAM,IAAI,GAAG,IAAI,MAAM,KAAK,GAAG;YAClC,OAAO,EAAE,OAAO,EAAE,WAAW,EAAE,MAAM,EAAE,IAAI,EAAE,CAAA;QAC9C,IAAI,MAAM,IAAI,GAAG,EAAE,CAAC;YACnB,IAAI,CAAC,KAAK,CAAC,MAAM,IAAI,CAAC,iBAAiB,CAAC,GAAG,CAAC,MAAM,CAAC,EAAE,CAAC;gBACrD,OAAO,EAAE,OAAO,EAAE,SAAS,EAAE,MAAM,EAAE,IAAI,EAAE,CAAA;YAC5C,CAAC;YACD,MAAM,QAAQ,GAAG,MAAM,MAAM,CAAC,mBAAmB,CAChD,KAAK,CAAC,QAAQ,CAAC,IAAI,EACnB,KAAK,CAAC,MAAM,CACZ,CAAA;YACD,OAAO,QAAQ;gBACd,CAAC,CAAC,EAAE,OAAO,EAAE,IAAI,EAAE,MAAM,EAAE,mBAAmB,EAAE,QAAQ,EAAE,QAAQ,EAAE;gBACpE,CAAC,CAAC,EAAE,OAAO,EAAE,SAAS,EAAE,MAAM,EAAE,IAAI,EAAE,CAAA;QACxC,CAAC;QACD,IAAI,MAAM,GAAG,GAAG,IAAI,MAAM,IAAI,GAAG,EAAE,CAAC;YACnC,OAAO,EAAE,OAAO,EAAE,WAAW,EAAE,MAAM,EAAE,IAAI,EAAE,CAAA;QAC9C,CAAC;QAED,MAAM,IAAI,GAAG,mBAAmB,CAAC,WAAW,CAAC,CAAA;QAC7C,IAAI,IAAI,KAAK,OAAO;YAAE,OAAO,EAAE,OAAO,EAAE,aAAa,EAAE,MAAM,EAAE,IAAI,EAAE,CAAA;QACrE,IAAI,IAAI,KAAK,UAAU,EAAE,CAAC;YACzB,OAAO,eAAe,CAAC;gBACtB,IAAI;gBACJ,WAAW;gBACX,QAAQ,EAAE,KAAK,CAAC,QAAQ;gBACxB,MAAM,EAAE,KAAK,CAAC,MAAM;aACpB,CAAC,CAAA;QACH,CAAC;QACD,IAAI,IAAI,CAAC,IAAI,KAAK,MAAM;YAAE,OAAO,EAAE,OAAO,EAAE,aAAa,EAAE,MAAM,EAAE,IAAI,EAAE,CAAA;QACzE,OAAO,cAAc,CAAC;YACrB,IAAI,EAAE,IAAI,CAAC,IAAI;YACf,IAAI;YACJ,QAAQ,EAAE,KAAK,CAAC,QAAQ;YACxB,MAAM,EAAE,KAAK,CAAC,MAAM;YACpB,QAAQ,EAAE,KAAK,CAAC,QAAQ;YACxB,MAAM,EAAE,KAAK,CAAC,MAAM;SACpB,CAAC,CAAA;IACH,CAAC;IAED;;;;;;;;;OASG;IACH,KAAK,UAAU,mBAAmB,CACjC,KAAqB;QAErB,IAAI,CAAC,oBAAoB,EAAE;YAAE,OAAO,aAAa,CAAC,KAAK,CAAC,CAAA;QAExD,4EAA4E;QAC5E,sEAAsE;QACtE,MAAM,IAAI,GAAG,MAAM,oBAAoB,CACtC,QAAQ,KAAK,CAAC,GAAG,IAAI,KAAK,CAAC,OAAO,IAAI,EAAE,IAAI,KAAK,CAAC,MAAM,IAAI,IAAI,EAAE,EAClE,GAAG,EAAE,CACJ,aAAa,CAAC;YACb,GAAG,EAAE,KAAK,CAAC,GAAG;YACd,OAAO,EAAE,KAAK,CAAC,OAAO;YACtB,MAAM,EAAE,KAAK,CAAC,MAAM;YACpB,MAAM,EAAE,KAAK,CAAC,MAAM;SACpB,CAAC,CACH,CAAA;QACD,MAAM,SAAS,GACd,IAAI,CAAC,OAAO,KAAK,IAAI;YACrB,CAAC,CAAC,KAAK,CAAC,QAAQ,IAAI,IAAI,CAAC,QAAQ,KAAK,KAAK,CAAC,QAAQ,CAAC;gBACpD,CAAC,KAAK,CAAC,IAAI,IAAI,IAAI,CAAC,IAAI,KAAK,KAAK,CAAC,IAAI,CAAC,CAAC,CAAA;QAC3C,IAAI,CAAC,SAAS;YAAE,OAAO,IAAI,CAAA;QAE3B,OAAO;YACN,GAAG,IAAI;YACP,OAAO,EAAE,WAAW;YACpB,MAAM,EAAE,cAAc;YACtB,QAAQ,EAAE,EAAE;YACZ,KAAK,EAAE,IAAI;YACX,IAAI,EAAE,IAAI;YACV,SAAS,EAAE,IAAI;YACf,UAAU,EAAE,EAAE;YACd,kBAAkB,EAAE,EAAE;YACtB,eAAe,EAAE,IAAI;YACrB,MAAM,EAAE,EAAE;YACV,KAAK,EAAE,EAAE;YACT,MAAM,EAAE,EAAE;YACV,YAAY,EAAE,IAAI;YAClB,QAAQ,EAAE,IAAI;SACd,CAAA;IACF,CAAC;IAED,KAAK,UAAU,aAAa,CAAC,KAAqB;QACjD,MAAM,MAAM,GAAG,KAAK,CAAC,MAAM,IAAI,sBAAsB,EAAE,CAAA;QACvD,MAAM,SAAS,GAAG,WAAW,CAAC,GAAG,EAAE,CAAA;QACnC,MAAM,MAAM,GAAG,KAAK,CAAC,MAAM,IAAI,IAAI,CAAA;QAEnC,MAAM,QAAQ,GAAG,iBAAiB,CAAC,KAAK,CAAC,GAAG,CAAC,CAAA;QAC7C,IAAI,CAAC,QAAQ,EAAE,CAAC;YACf,OAAO,SAAS,CACf;gBACC,GAAG,EAAE,KAAK,CAAC,GAAG;gBACd,QAAQ,EAAE,KAAK,CAAC,GAAG;gBACnB,MAAM,EAAE,CAAC;gBACT,WAAW,EAAE,IAAI;gBACjB,IAAI,EAAE,IAAI;gBACV,YAAY,EAAE,IAAI;gBAClB,SAAS;aACT,EACD,EAAE,OAAO,EAAE,aAAa,EAAE,MAAM,EAAE,IAAI,EAAE,EACxC,MAAM,CACN,CAAA;QACF,CAAC;QAED,MAAM,OAAO,GAA2B;YACvC,MAAM,EAAE,2CAA2C;YACnD,iBAAiB,EAAE,GAAG;YACtB,YAAY,EAAE,kBAAkB;SAChC,CAAA;QACD,IAAI,KAAK,CAAC,IAAI;YAAE,OAAO,CAAC,eAAe,CAAC,GAAG,KAAK,CAAC,IAAI,CAAA;QACrD,IAAI,KAAK,CAAC,YAAY;YAAE,OAAO,CAAC,mBAAmB,CAAC,GAAG,KAAK,CAAC,YAAY,CAAA;QAEzE,MAAM,OAAO,GAAG,CAAC,GAAQ,EAAE,EAAE,CAC5B,aAAa,CAAC,QAAQ,CAAC,GAAG,EAAE,KAAK,CAAC,OAAO,CAAC,EAAE,OAAO,EAAE,MAAM,CAAC,CAAA;QAE7D,IAAI,UAAU,GAAG,QAAQ,CAAA;QACzB,IAAI,OAAO,GAAkB,MAAM,OAAO,CAAC,UAAU,CAAC,CAAA;QAEtD,IAAI,OAAO,CAAC,IAAI,KAAK,QAAQ,IAAI,OAAO,CAAC,SAAS,EAAE,CAAC;YACpD,MAAM,QAAQ,GAAG,cAAc,CAAC,UAAU,CAAC,CAAA;YAC3C,IAAI,QAAQ,EAAE,CAAC;gBACd,MAAM,KAAK,GAAG,MAAM,OAAO,CAAC,QAAQ,CAAC,CAAA;gBACrC,IAAI,KAAK,CAAC,IAAI,KAAK,MAAM,EAAE,CAAC;oBAC3B,UAAU,GAAG,QAAQ,CAAA;oBACrB,OAAO,GAAG,KAAK,CAAA;gBAChB,CAAC;YACF,CAAC;QACF,CAAC;QAED,KACC,IAAI,KAAK,GAAG,CAAC,EACb,KAAK,IAAI,sBAAsB;YAC/B,OAAO,CAAC,IAAI,KAAK,MAAM;YACvB,OAAO,CAAC,IAAI,CAAC,QAAQ,CAAC,MAAM,KAAK,GAAG,EACpC,KAAK,EAAE,EACN,CAAC;YACF,MAAM,MAAM,GAAG,eAAe,CAAC,OAAO,CAAC,IAAI,CAAC,QAAQ,EAAE,KAAK,CAAC,CAAA;YAC5D,IAAI,MAAM,UAAU,CAAC,YAAY,CAAC,UAAU,CAAC,QAAQ,EAAE,MAAM,CAAC,EAAE,CAAC;gBAChE,MAAM,CAAC,IAAI,CAAC,mCAAmC,EAAE;oBAChD,IAAI,EAAE,UAAU,CAAC,QAAQ;oBACzB,WAAW,EAAE,OAAO,CAAC,IAAI,CAAC,QAAQ,CAAC,OAAO,CAAC,GAAG,CAAC,aAAa,CAAC,IAAI,EAAE;oBACnE,OAAO,EAAE,MAAM;iBACf,CAAC,CAAA;YACH,CAAC;YACD,OAAO,GAAG,MAAM,OAAO,CAAC,UAAU,CAAC,CAAA;QACpC,CAAC;QAED,IAAI,OAAO,CAAC,IAAI,KAAK,MAAM,EAAE,CAAC;YAC7B,OAAO,SAAS,CACf;gBACC,GAAG,EAAE,UAAU,CAAC,IAAI;gBACpB,QAAQ,EAAE,UAAU,CAAC,IAAI;gBACzB,MAAM,EAAE,CAAC;gBACT,WAAW,EAAE,IAAI;gBACjB,IAAI,EAAE,IAAI;gBACV,YAAY,EAAE,IAAI;gBAClB,SAAS;aACT;YACD,qEAAqE;YACrE;gBACC,OAAO,EAAE,OAAO,CAAC,IAAI,KAAK,SAAS,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,WAAW;gBAC7D,MAAM,EAAE,IAAI;aACZ,EACD,MAAM,CACN,CAAA;QACF,CAAC;QAED,MAAM,EAAE,QAAQ,EAAE,QAAQ,EAAE,GAAG,OAAO,CAAC,IAAI,CAAA;QAC3C,MAAM,WAAW,GAAG,CAAC,MAAiC,EAAE,EAAE,CACzD,QAAQ,CAAC,MAAM,KAAK,GAAG,CAAC,CAAC,CAAC,CAAC,MAAM,IAAI,IAAI,CAAC,CAAC,CAAC,CAAC,IAAI,CAAA;QAClD,MAAM,IAAI,GAAG,MAAM,YAAY,CAAC;YAC/B,GAAG,OAAO,CAAC,IAAI;YACf,MAAM;YACN,QAAQ,EAAE,KAAK,CAAC,QAAQ;YACxB,MAAM;SACN,CAAC,CAAA;QAEF,MAAM,IAAI,GAAG,SAAS,CACrB;YACC,GAAG,EAAE,UAAU,CAAC,IAAI;YACpB,QAAQ,EAAE,QAAQ,CAAC,IAAI;YACvB,MAAM,EAAE,QAAQ,CAAC,MAAM;YACvB,WAAW,EAAE,QAAQ,CAAC,OAAO,CAAC,GAAG,CAAC,cAAc,CAAC;YACjD,uEAAuE;YACvE,wEAAwE;YACxE,2DAA2D;YAC3D,IAAI,EAAE,QAAQ,CAAC,OAAO,CAAC,GAAG,CAAC,MAAM,CAAC,IAAI,WAAW,CAAC,KAAK,CAAC,IAAI,CAAC;YAC7D,YAAY,EACX,QAAQ,CAAC,OAAO,CAAC,GAAG,CAAC,eAAe,CAAC;gBACrC,WAAW,CAAC,KAAK,CAAC,YAAY,CAAC;YAChC,SAAS;SACT,EACD,IAAI,EACJ,MAAM,CACN,CAAA;QAED,MAAM,CAAC,IAAI,CAAC,2BAA2B,EAAE;YACxC,GAAG,EAAE,IAAI,CAAC,QAAQ;YAClB,MAAM,EAAE,IAAI,CAAC,MAAM;YACnB,OAAO,EAAE,IAAI,CAAC,OAAO;YACrB,MAAM,EAAE,IAAI,CAAC,MAAM,IAAI,EAAE;YACzB,KAAK,EAAE,IAAI,CAAC,QAAQ,CAAC,MAAM;YAC3B,WAAW,EAAE,IAAI,CAAC,UAAU;SAC5B,CAAC,CAAA;QACF,OAAO,IAAI,CAAA;IACZ,CAAC;IAED,gDAAgD;IAChD,KAAK,UAAU,aAAa,CAAC,SAAiB;QAC7C,MAAM,IAAI,GAAG,MAAM,mBAAmB,CAAC,EAAE,GAAG,EAAE,SAAS,EAAE,CAAC,CAAA;QAC1D,MAAM,EAAE,QAAQ,EAAE,MAAM,EAAE,GAAG,IAAI,CAAA;QACjC,IAAI,IAAI,CAAC,OAAO,KAAK,IAAI,IAAI,CAAC,QAAQ,IAAI,CAAC,MAAM,EAAE,CAAC;YACnD,MAAM,IAAI,KAAK,CACd,sCAAsC,IAAI,CAAC,OAAO,YAAY,IAAI,CAAC,MAAM,MAAM,SAAS,EAAE,CAC1F,CAAA;QACF,CAAC;QACD,OAAO,EAAE,QAAQ,EAAE,MAAM,EAAE,CAAA;IAC5B,CAAC;IAED,OAAO;QACN,mBAAmB;QACnB,oBAAoB;QACpB,aAAa;QACb,wBAAwB,EAAE,MAAM,CAAC,wBAAwB;QACzD,mBAAmB,EAAE,MAAM,CAAC,mBAAmB;QAC/C,gEAAgE;QAChE,aAAa,EAAE,SAAS;QACxB,+DAA+D;QAC/D,iBAAiB,CAChB,KAA4C,EAC5C,GAAqB;YAErB,IAAI,CAAC,KAAK;gBAAE,OAAO,GAAG,EAAE,CAAA;YACxB,IAAI,CAAC,OAAO,CAAC,aAAa,EAAE,CAAC;gBAC5B,MAAM,IAAI,KAAK,CAAC,+CAA+C,CAAC,CAAA;YACjE,CAAC;YACD,OAAO,iBAAiB,CAAC,KAAK,EAAE,GAAG,EAAE;gBACpC,IAAI,EAAE,OAAO,CAAC,aAAa,CAAC,IAAI;gBAChC,MAAM,EAAE,OAAO,CAAC,aAAa,CAAC,MAAM,IAAI,MAAM;aAC9C,CAAC,CAAA;QACH,CAAC;KACD,CAAA;AACF,CAAC","sourcesContent":["import type { ApiError } from 'hanzio/api-wrapper'\nimport { parseRetryAfter } from 'hanzio'\nimport {\n\ttype CloudflareCredentials,\n\tCloudflareRateLimitedError,\n\tcreateCloudflareClient\n} from './client.js'\nimport {\n\tclassifyContentType,\n\ttype ContentKind,\n\tconvertibleDocumentType\n} from './content-type.js'\nimport {\n\ttype CrawlSnapshotInput,\n\ttype CrawlSnapshotOptions,\n\tcurrentCrawlSnapshot,\n\tthroughCrawlSnapshot,\n\twithCrawlSnapshot\n} from './crawl-snapshot.js'\nimport {\n\tisApplicationShell,\n\tisBlankMarkdown,\n\tlooksLikeHtml\n} from './detect-markdown.js'\nimport { sha256Hex } from './hash.js'\nimport { extractHtmlMetadata } from './html-metadata.js'\nimport { type CreateConcurrencyLimit, createHostLimits } from './limit.js'\nimport { errorLogAttributes, type Logger, silentLogger } from './logger.js'\nimport {\n\thtmlImages,\n\thtmlLinks,\n\tmarkdownRefs,\n\tstripDocumentPreamble,\n\tstripFrontMatter,\n\ttitleFromMarkdown\n} from './markdown-refs.js'\nimport { createMemoryConcurrencyLimit } from './memory-limit.js'\nimport { isHostAllowed, isLocalhostHostname } from './net-guard.js'\nimport { currentOperationSignal } from './operation-signal.js'\nimport { BROWSER_USER_AGENT } from './user-agent.js'\nimport { preserveXlsxZeros } from './xlsx-zeros.js'\n\nconst FETCH_TIMEOUT_MS = 30_000\n/** Browser Rendering once returned a 6 MB markdown page from a 6 MB upload. */\nconst MAX_HTML_BYTES = 2 * 1024 * 1024\nconst MAX_DOCUMENT_BYTES = 2 * 1024 * 1024\nconst MAX_PDF_BYTES = 12 * 1024 * 1024\nconst MAX_PDF_PAGES = 60\n\n/** Stored markdown ceiling; a page over this is a listing or an upload, not knowledge. */\nexport const MAX_MARKDOWN_CHARS = 300_000\n\nexport type FetchOutcome =\n\t| 'ok'\n\t| 'unchanged'\n\t| 'gone'\n\t| 'blocked'\n\t| 'transient'\n\t| 'unsupported'\nexport type FetchSource =\n\t| 'content-negotiation'\n\t| 'workers-ai'\n\t| 'browser-rendering'\n\t| 'not-modified'\n\t| 'firecrawl-parse'\n\nexport type FetchedPage = {\n\t/** The URL we requested, after http->https upgrade and www fallback. */\n\turl: string\n\t/** After redirects; may differ in host or path. */\n\tfinalUrl: string\n\t/** HTTP status of the origin response, or 0 when the request never completed. */\n\tstatus: number\n\toutcome: FetchOutcome\n\tsource: FetchSource | null\n\tcontentType: string | null\n\tetag: string | null\n\tlastModified: string | null\n\t/** Empty string unless outcome === 'ok'. Front-matter stripped, capped. */\n\tmarkdown: string\n\t/** <title> when an HTML tier read the page, else the first markdown H1. */\n\ttitle: string | null\n\t/** ISO 639-1 from <html lang>, lowercased, else null. */\n\tlang: string | null\n\t/** <link rel=\"canonical\">, absolute. */\n\tcanonical: string | null\n\t/** `<link rel=\"alternate\" hreflang>` targets, absolute, `x-default` excluded. Optional: many callers build a page from parts. */\n\talternates?: string[]\n\t/** `alternates` by language: URL -> ISO 639-1 (`da` of `da-DK`). */\n\talternateLanguages?: Record<string, string>\n\t/** <meta name=\"description\"> or the og: equivalent. */\n\tmetaDescription: string | null\n\t/** Parsed application/ld+json blocks (any shape; the profile layer reads LocalBusiness/OpeningHoursSpecification). */\n\tjsonLd: unknown[]\n\t/**\n\t * Absolute, deduped link targets: the markdown's, unioned with every\n\t * `<a href>` when an HTML tier read the page. Conversion loses navigation,\n\t * and a link lost here is a page discovery can never reach.\n\t */\n\tlinks: string[]\n\t/** Absolute, deduped image URLs found in the markdown. */\n\timages: string[]\n\tdurationMs: number\n\tcaptureIssue?: string | null | undefined\n\t/**\n\t * Hash of the HTML the origin served, when an origin tier read HTML. Given\n\t * back on the next read (`fetchPageAsMarkdown({ htmlHash })`), identical HTML\n\t * is answered as `unchanged` without converting it again.\n\t */\n\thtmlHash?: string | null | undefined\n}\n\ntype PageBase = {\n\turl: string\n\tfinalUrl: string\n\tstatus: number\n\tcontentType: string | null\n\tetag: string | null\n\tlastModified: string | null\n\tstartedAt: number\n}\n\nexport type PageBody = {\n\tcaptureIssue?: string | null | undefined\n\toutcome: FetchOutcome\n\tsource: FetchSource | null\n\tmarkdown?: string | null | undefined\n\t/** The HTML an origin tier read, when one did - the only source of metadata. */\n\thtml?: string | null | undefined\n}\n\n/**\n * The one place markdown, front-matter and HTML metadata are reconciled: the\n * HTML `<title>` wins, the Workers AI front-matter is the fallback for pages we\n * only ever saw as markdown, and links/images are resolved against `finalUrl`.\n */\nfunction buildPage(\n\tbase: PageBase,\n\trawBody: PageBody,\n\tlogger: Logger\n): FetchedPage {\n\t// A NUL byte is never text, and Postgres refuses a string that holds one:\n\t// a single page with one (a Wix site's conversion) failed the whole import.\n\t// Composed Unicode, because every comparison downstream assumes it: a site\n\t// that wrote \"Gård\" as \"a\" plus a combining ring gave the fact guard the\n\t// name \"Broges Ga\", which no draft could contain, and the document shipped\n\t// as its verbatim source.\n\tconst clean = (text: string | null | undefined) =>\n\t\ttext?.replaceAll('\\u0000', '').normalize('NFC')\n\tconst body = {\n\t\t...rawBody,\n\t\tmarkdown: clean(rawBody.markdown),\n\t\thtml: clean(rawBody.html)\n\t}\n\tconst frontMatter = stripFrontMatter(body.markdown ?? '')\n\tlet markdown = frontMatter.markdown\n\tif (markdown.length > MAX_MARKDOWN_CHARS) {\n\t\tlogger.warn('url-to-markdown.truncated', {\n\t\t\turl: base.finalUrl,\n\t\t\tchars: markdown.length\n\t\t})\n\t\tmarkdown = markdown.slice(0, MAX_MARKDOWN_CHARS)\n\t}\n\n\tconst metadata = body.html\n\t\t? extractHtmlMetadata(body.html, base.finalUrl)\n\t\t: null\n\tconst refs = markdownRefs(markdown, base.finalUrl)\n\t// The markdown is the document; the HTML is the map. Conversion drops\n\t// navigation (image-wrapped anchors, folded menus), so when we hold the\n\t// HTML its anchors decide what discovery can still reach.\n\tconst links = body.html\n\t\t? [...new Set([...refs.links, ...htmlLinks(body.html, base.finalUrl)])]\n\t\t: refs.links\n\n\treturn {\n\t\turl: base.url,\n\t\tfinalUrl: base.finalUrl,\n\t\tstatus: base.status,\n\t\toutcome: body.outcome,\n\t\tsource: body.source,\n\t\tcontentType: base.contentType,\n\t\tetag: base.etag,\n\t\tlastModified: base.lastModified,\n\t\tmarkdown,\n\t\ttitle: metadata?.title ?? frontMatter.title ?? titleFromMarkdown(markdown),\n\t\tlang: metadata?.lang ?? null,\n\t\tcanonical: metadata?.canonical ?? null,\n\t\talternates: metadata?.alternates ?? [],\n\t\talternateLanguages: metadata?.alternateLanguages ?? {},\n\t\tmetaDescription:\n\t\t\tmetadata?.metaDescription ?? frontMatter.description ?? null,\n\t\tjsonLd: metadata?.jsonLd ?? [],\n\t\tlinks,\n\t\timages: [\n\t\t\t...new Set([\n\t\t\t\t...refs.images,\n\t\t\t\t...(body.html ? htmlImages(body.html, base.finalUrl) : [])\n\t\t\t])\n\t\t],\n\t\tcaptureIssue:\n\t\t\tfrontMatter.markdown.length > MAX_MARKDOWN_CHARS\n\t\t\t\t? 'markdown_limit'\n\t\t\t\t: (body.captureIssue ?? null),\n\t\tdurationMs: Math.round(performance.now() - base.startedAt),\n\t\thtmlHash: body.html ? htmlHash(body.html) : null\n\t}\n}\n\n/**\n * `http://` is upgraded rather than rejected: two carpenter sites in the probe\n * link internally with `http://` and the https-only guard threw away most of\n * their pages. Localhost http stays reachable outside production.\n */\nfunction prepareRequestUrl(raw: string) {\n\tlet parsed: URL\n\ttry {\n\t\tparsed = new URL(raw)\n\t} catch {\n\t\treturn null\n\t}\n\tif (parsed.protocol === 'https:') return parsed\n\tif (parsed.protocol !== 'http:') return null\n\tif (isLocalhostHostname(parsed.hostname)) {\n\t\treturn process.env.NODE_ENV === 'production' ? null : parsed\n\t}\n\tparsed.protocol = 'https:'\n\tif (parsed.port === '80') parsed.port = ''\n\treturn parsed\n}\n\n/**\n * `apoteket.dk` answers no TLS handshake at the apex while `www.apoteket.dk`\n * serves the whole site, so a connection failure gets one www retry.\n */\nfunction wwwFallbackUrl(url: URL) {\n\tconst host = url.hostname\n\tif (host.startsWith('www.') || !host.includes('.')) return null\n\tif (isLocalhostHostname(host) || /^[\\d.]+$/.test(host)) return null\n\tconst next = new URL(url.href)\n\tnext.hostname = `www.${host}`\n\treturn next\n}\n\n/**\n * A 429 is an origin asking us to slow down, not a bot wall. Shopify answered\n * 117 of Good Habit's 244 reads with it; sent on to Browser Rendering they\n * tripped Cloudflare's own limit (79 times, 25-30 s each) and 7 came back.\n * So the host is narrowed and paused (`coolDownHost`), the URL is asked again,\n * and a page still refused after that is `transient`: its stored copy stands\n * and the next run reads it.\n */\nconst MAX_RATE_LIMIT_RETRIES = 3\nconst MAX_RATE_LIMIT_WAIT_MS = 30_000\n\n/** The origin's `Retry-After` when it sent one, else 1 s, 2 s, 4 s. */\nfunction rateLimitWaitMs(response: Response, retry: number) {\n\tconst askedMs = parseRetryAfter(response.headers.get('retry-after'))\n\treturn Math.min(MAX_RATE_LIMIT_WAIT_MS, askedMs ?? 1_000 * 2 ** (retry - 1))\n}\n\n/**\n * Bump whenever conversion changes what the same HTML becomes (`withoutHiddenUi`,\n * the Workers AI call, the fallback tiers): the stored hash then no longer\n * matches, and every page is converted again once instead of keeping the\n * markdown the old conversion made.\n */\nconst CONVERSION_VERSION = 1\n\nfunction htmlHash(html: string) {\n\treturn sha256Hex(`${CONVERSION_VERSION}\\n${html}`)\n}\n\n/** The request URL with the market pinned; the URL itself when there is none. */\nfunction inMarket(url: URL, country: string | null | undefined) {\n\tif (!country || url.searchParams.has('country')) return url\n\tconst pinned = new URL(url.href)\n\tpinned.searchParams.set('country', country)\n\treturn pinned\n}\n\n/** `Response.url` is empty on a synthesised response and after some proxies. */\nfunction parseFinalUrl(responseUrl: string, requestUrl: URL) {\n\tif (!responseUrl) return requestUrl\n\ttry {\n\t\treturn new URL(responseUrl)\n\t} catch {\n\t\treturn requestUrl\n\t}\n}\n\nfunction isAbortError(error: unknown) {\n\tconst name =\n\t\ttypeof error === 'object' && error && 'name' in error\n\t\t\t? String(error.name)\n\t\t\t: ''\n\treturn name === 'AbortError' || name === 'TimeoutError'\n}\n\n/** What the origin gave us, already downloaded under the host's politeness slot. */\ntype OriginBody =\n\t| { kind: 'none'; captureIssue?: string }\n\t| { kind: 'text'; text: string }\n\t| { kind: 'bytes'; bytes: ArrayBuffer }\n\ntype OriginRead = { response: Response; finalUrl: URL; body: OriginBody }\n\ntype OriginAttempt =\n\t| { kind: 'read'; read: OriginRead }\n\t| { kind: 'refused' }\n\t| { kind: 'failed'; retryable: boolean }\n\n/** Reading the body is optional; an unsupported type is dropped at the headers. */\nfunction discardBody(response: Response) {\n\tvoid response.body?.cancel().catch(() => {})\n}\n\n/**\n * Decides from the headers alone whether the body is worth downloading, and\n * downloads it when it is. Everything downstream works from bytes already in\n * hand, so nothing past this point needs the origin's slot.\n */\nasync function readOriginBody(\n\tresponse: Response,\n\tfinalUrl: URL,\n\tlogger: Logger\n): Promise<OriginBody> {\n\tconst status = response.status\n\tif (status < 200 || status >= 300) {\n\t\tdiscardBody(response)\n\t\treturn { kind: 'none' }\n\t}\n\n\tconst contentType = response.headers.get('content-type')\n\tconst kind = classifyContentType(contentType)\n\tif (kind === 'asset') {\n\t\tdiscardBody(response)\n\t\treturn { kind: 'none' }\n\t}\n\tif (kind === 'document') {\n\t\tconst type = convertibleDocumentType(contentType)\n\t\tif (!type) {\n\t\t\tdiscardBody(response)\n\t\t\treturn { kind: 'none' }\n\t\t}\n\t\tconst limit = type.extension === 'pdf' ? MAX_PDF_BYTES : MAX_DOCUMENT_BYTES\n\t\tconst declared = Number(response.headers.get('content-length'))\n\t\tif (Number.isFinite(declared) && declared > limit) {\n\t\t\tdiscardBody(response)\n\t\t\tlogger.info('url-to-markdown.document_oversized', {\n\t\t\t\turl: finalUrl.href,\n\t\t\t\tbytes: declared\n\t\t\t})\n\t\t\treturn { kind: 'none', captureIssue: 'document_byte_limit' }\n\t\t}\n\t\tconst bytes = await readBoundedBody(response, limit)\n\t\treturn bytes\n\t\t\t? { kind: 'bytes', bytes }\n\t\t\t: { kind: 'none', captureIssue: 'document_byte_limit' }\n\t}\n\treturn { kind: 'text', text: await response.text() }\n}\n\nfunction documentFilename(finalUrl: string, extension: string) {\n\tconst segment = finalUrl.split(/[?#]/)[0]?.split('/').pop() || 'document'\n\tconst name = segment.slice(0, 80)\n\treturn name.toLowerCase().endsWith(`.${extension}`)\n\t\t? name\n\t\t: `${name}.${extension}`\n}\n\n/** Streaming limits bound chunked bodies as well as declared Content-Length. */\nasync function readBoundedBody(\n\tresponse: Response,\n\tmaxBytes: number\n): Promise<ArrayBuffer | null> {\n\tconst reader = response.body?.getReader()\n\tif (!reader) return new ArrayBuffer(0)\n\tconst chunks: Uint8Array[] = []\n\tlet size = 0\n\twhile (true) {\n\t\tconst next = await reader.read()\n\t\tif (next.done) break\n\t\tsize += next.value.byteLength\n\t\tif (size > maxBytes) {\n\t\t\tawait reader.cancel()\n\t\t\treturn null\n\t\t}\n\t\tchunks.push(next.value)\n\t}\n\tconst bytes = new Uint8Array(size)\n\tlet offset = 0\n\tfor (const chunk of chunks) {\n\t\tbytes.set(chunk, offset)\n\t\toffset += chunk.byteLength\n\t}\n\treturn bytes.buffer\n}\n\n/**\n * Markup a visitor never sees until they open it: `<template>` content and\n * closed `<dialog>`s. A Shopify theme ships its cart drawer (\"Your cart is\n * empty\") and search overlay (a product list with prices) in every page this\n * way; converted, they reached 70 of one store's 89 pages and a price table\n * in seven of its documents (Good Habit audit §3.4). An open dialog is shown\n * and stays. A contact or newsletter form (Shopify posts both to `/contact`)\n * is labels and a button, and \"Telefon Kommentar Send us a note\" read as the\n * name `Kommentar Send` the fact guard then demanded of the document.\n *\n * Plus the form fields a visitor cannot see: an anti-spam honeypot inside a\n * `display:none` block renamed its label (\"Navn\" / \"Url\") on every request,\n * so a carpenter's seven pages hashed as changed and five documents were\n * rewritten every night. Only labels go - a collapsed FAQ answer is hidden the\n * same way and must stay.\n */\nconst HIDDEN_UI_SELECTOR = [\n\t'template',\n\t'dialog:not([open])',\n\t'form[action*=\"/contact\"]',\n\t'[style*=\"display:none\"] label',\n\t'[style*=\"display: none\"] label'\n].join(', ')\n\nfunction withoutHiddenUi(html: string) {\n\treturn new HTMLRewriter()\n\t\t.on(HIDDEN_UI_SELECTOR, {\n\t\t\telement(element) {\n\t\t\t\telement.remove()\n\t\t\t}\n\t\t})\n\t\t.transform(html)\n}\n\n/** A real browser passes these; every other 4xx has nothing for it to find. */\nconst BOT_WALL_STATUSES = new Set([401, 403])\n\n/** `AbortSignal.timeout` and the caller's signal, whichever fires first. */\nfunction timeoutSignal(ms: number, signal: AbortSignal | undefined) {\n\tconst timeout = AbortSignal.timeout(ms)\n\treturn signal ? AbortSignal.any([timeout, signal]) : timeout\n}\n\nexport type FetchPageInput = {\n\turl: string\n\t/** Stored validators from the last run; a 304 returns outcome 'unchanged'. */\n\tetag?: string | null | undefined\n\tlastModified?: string | null | undefined\n\t/** Allow the Browser Rendering tier. Default true. */\n\trender?: boolean | undefined\n\t/**\n\t * ISO country of the market to read the page in. A Shopify store redirects\n\t * by the visitor's IP - our worker reads a Danish shop as its EU market in\n\t * euros and English (Good Habit audit §3.2) - and `?country=DK` pins it.\n\t * Applied to the request only: the page keeps the URL it was asked for.\n\t */\n\tcountry?: string | null | undefined\n\t/** The last read's `htmlHash`; identical HTML comes back `unchanged`. */\n\thtmlHash?: string | null | undefined\n\tsignal?: AbortSignal | undefined\n}\n\nexport type PdfParser = (input: {\n\tbytes: ArrayBuffer\n\tfilename: string\n\tmimeType: string\n\t/** Pages to read at most; a parse that reaches it is marked incomplete. */\n\tmaxPages: number\n\tsignal?: AbortSignal | undefined\n}) => Promise<{ markdown: string; numPages: number | null }>\n\nexport type WebsiteToMarkdownOptions = {\n\t/** Workers AI and Browser Rendering; read on first use, so getters work. */\n\tcloudflare: CloudflareCredentials\n\t/**\n\t * Builds every pool: the two Cloudflare ones and one per origin host.\n\t * Defaults to an in-memory limiter; pass a shared one (Redis) when several\n\t * processes share the Cloudflare quota.\n\t */\n\tcreateLimit?: CreateConcurrencyLimit\n\t/** Workers AI client errors other than 429s, for an error tracker. */\n\tonApiError?: (error: ApiError) => void\n\t/**\n\t * Pass Workers AI timeouts to `onApiError`. A timed-out page falls through\n\t * to the next tier, so `false` keeps import bursts out of error tracking.\n\t * Default true.\n\t */\n\treportTimeouts?: boolean\n\tlogger?: Logger\n\t/**\n\t * Where the returned `withCrawlSnapshot` keeps snapshots. Its logger\n\t * defaults to `logger`.\n\t */\n\tcrawlSnapshot?: CrawlSnapshotOptions\n\t/** The OCR tier for a PDF Workers AI returned nothing for. */\n\tparsePdf?: PdfParser\n\t/**\n\t * SSRF guard run before every origin request. Defaults to\n\t * {@link isHostAllowed}: public addresses only, localhost outside production.\n\t */\n\tisHostAllowed?: (hostname: string) => Promise<boolean>\n}\n\n/**\n * One converter: its Cloudflare client, pools and per-host politeness limits.\n * Create it once per process; every call shares the pools.\n */\nexport function createWebsiteToMarkdown(options: WebsiteToMarkdownOptions) {\n\tconst logger = options.logger ?? silentLogger\n\tconst createLimit = options.createLimit ?? createMemoryConcurrencyLimit\n\tconst client = createCloudflareClient({\n\t\tcredentials: options.cloudflare,\n\t\tcreateLimit,\n\t\tonApiError: options.onApiError,\n\t\treportTimeouts: options.reportTimeouts,\n\t\tlogger\n\t})\n\tconst hostLimits = createHostLimits(createLimit)\n\tconst allowHost =\n\t\toptions.isHostAllowed ??\n\t\t((hostname: string) => isHostAllowed(hostname, logger))\n\n\tasync function requestOrigin(\n\t\turl: URL,\n\t\theaders: Record<string, string>,\n\t\tsignal?: AbortSignal | undefined\n\t): Promise<OriginAttempt> {\n\t\tif (signal?.aborted) return { kind: 'failed', retryable: false }\n\t\tif (!(await allowHost(url.hostname))) {\n\t\t\treturn { kind: 'refused' }\n\t\t}\n\t\ttry {\n\t\t\t// The politeness slot spans the connection *and* the body: freed at the\n\t\t\t// response headers it would bound connections only, and the whole 80-wide\n\t\t\t// page pool could still be streaming from one small WordPress host at\n\t\t\t// once - the burst the cap exists to prevent, at any cap we would pick.\n\t\t\t// The Cloudflare tiers stay outside it - they are\n\t\t\t// our upstreams with their own pools, and holding a customer's slot\n\t\t\t// across a 60s render would stall the crawl for no politeness gain.\n\t\t\tconst read = await hostLimits.withHostLimit(url.hostname, async () => {\n\t\t\t\tconst response = await fetch(url, {\n\t\t\t\t\tredirect: 'follow',\n\t\t\t\t\tsignal: timeoutSignal(FETCH_TIMEOUT_MS, signal),\n\t\t\t\t\theaders\n\t\t\t\t})\n\t\t\t\tconst finalUrl = parseFinalUrl(response.url, url)\n\t\t\t\treturn {\n\t\t\t\t\tresponse,\n\t\t\t\t\tfinalUrl,\n\t\t\t\t\tbody: await readOriginBody(response, finalUrl, logger)\n\t\t\t\t}\n\t\t\t})\n\t\t\treturn { kind: 'read', read }\n\t\t} catch (error) {\n\t\t\tlogger.warn('url-to-markdown.origin_failed', {\n\t\t\t\turl: url.href,\n\t\t\t\t...errorLogAttributes(error)\n\t\t\t})\n\t\t\t// A body that dies mid-stream is the same class of failure as a refused\n\t\t\t// connection, and the same www retry is worth a try.\n\t\t\treturn { kind: 'failed', retryable: !isAbortError(error) }\n\t\t}\n\t}\n\n\t/**\n\t * Bytes of a document to markdown: Workers AI `toMarkdown` first, the\n\t * `parsePdf` tier (e.g. Firecrawl: text-first, OCR fallback) for a PDF Workers\n\t * AI returned nothing for. Shared by the crawler (a linked terms PDF) and file\n\t * uploads (the same PDF dropped in by hand), so both read a file the same\n\t * way. `workersAiMaxBytes` is the caller's budget: the crawler keeps it at\n\t * {@link MAX_DOCUMENT_BYTES}; an upload of a 50 MB scanned catalogue may go\n\t * straight to the OCR tier.\n\t */\n\tasync function convertDocumentBytes(input: {\n\t\tbytes: ArrayBuffer\n\t\tfilename: string\n\t\tmimeType: string\n\t\textension: string\n\t\tworkersAiMaxBytes?: number | undefined\n\t\tsignal?: AbortSignal | undefined\n\t}): Promise<PageBody> {\n\t\tconst { bytes } = input\n\t\tif (!bytes.byteLength) return { outcome: 'unsupported', source: null }\n\t\tif (input.signal?.aborted) return { outcome: 'transient', source: null }\n\t\tlet workersAiRateLimited = false\n\t\tlet markdown: string | null = null\n\t\tif (bytes.byteLength <= (input.workersAiMaxBytes ?? MAX_DOCUMENT_BYTES)) {\n\t\t\ttry {\n\t\t\t\tmarkdown = await client.convertFileWithWorkersAi({\n\t\t\t\t\tbody: input.extension === 'xlsx' ? preserveXlsxZeros(bytes) : bytes,\n\t\t\t\t\tfilename: input.filename,\n\t\t\t\t\tmimeType: input.mimeType,\n\t\t\t\t\tsignal: input.signal\n\t\t\t\t})\n\t\t\t} catch (error) {\n\t\t\t\tif (!(error instanceof CloudflareRateLimitedError)) throw error\n\t\t\t\tworkersAiRateLimited = true\n\t\t\t}\n\t\t}\n\t\tconst body = stripDocumentPreamble(markdown ?? '')\n\t\tif (!isBlankMarkdown(body))\n\t\t\treturn { outcome: 'ok', source: 'workers-ai', markdown: body }\n\t\tif (input.extension !== 'pdf') {\n\t\t\treturn {\n\t\t\t\toutcome: 'transient',\n\t\t\t\tsource: null,\n\t\t\t\tcaptureIssue: workersAiRateLimited ? 'workers_ai_rate_limited' : null\n\t\t\t}\n\t\t}\n\t\tif (input.signal?.aborted) return { outcome: 'transient', source: null }\n\t\ttry {\n\t\t\tif (!options.parsePdf) throw new Error('No PDF parser configured')\n\t\t\tconst parsed = await options.parsePdf({\n\t\t\t\tbytes,\n\t\t\t\tfilename: input.filename,\n\t\t\t\tmimeType: input.mimeType,\n\t\t\t\tmaxPages: MAX_PDF_PAGES,\n\t\t\t\tsignal: input.signal\n\t\t\t})\n\t\t\treturn {\n\t\t\t\toutcome: 'ok',\n\t\t\t\tsource: 'firecrawl-parse',\n\t\t\t\tmarkdown: parsed.markdown,\n\t\t\t\t// The parser may report parsed pages rather than total pages; reaching the\n\t\t\t\t// limit is therefore explicitly incomplete even when the count equals it.\n\t\t\t\tcaptureIssue:\n\t\t\t\t\tparsed.numPages === null || parsed.numPages >= MAX_PDF_PAGES\n\t\t\t\t\t\t? 'pdf_page_limit_unverified'\n\t\t\t\t\t\t: null\n\t\t\t}\n\t\t} catch (error) {\n\t\t\tlogger.warn('url-to-markdown.pdf_parse_failed', {\n\t\t\t\tfilename: input.filename,\n\t\t\t\t...errorLogAttributes(error)\n\t\t\t})\n\t\t\treturn {\n\t\t\t\toutcome: 'transient',\n\t\t\t\tsource: null,\n\t\t\t\tcaptureIssue: workersAiRateLimited\n\t\t\t\t\t? 'workers_ai_rate_limited'\n\t\t\t\t\t: 'pdf_extraction_failed'\n\t\t\t}\n\t\t}\n\t}\n\n\tasync function convertDocument(input: {\n\t\tbody: OriginBody\n\t\tcontentType: string | null\n\t\tfinalUrl: URL\n\t\tsignal?: AbortSignal | undefined\n\t}): Promise<PageBody> {\n\t\tconst type = convertibleDocumentType(input.contentType)\n\t\tif (!type || input.body.kind !== 'bytes') {\n\t\t\treturn {\n\t\t\t\toutcome: 'unsupported',\n\t\t\t\tsource: null,\n\t\t\t\tcaptureIssue:\n\t\t\t\t\tinput.body.kind === 'none' ? input.body.captureIssue : null\n\t\t\t}\n\t\t}\n\t\treturn convertDocumentBytes({\n\t\t\tbytes: input.body.bytes,\n\t\t\tfilename: documentFilename(input.finalUrl.href, type.extension),\n\t\t\tmimeType: type.mimeType,\n\t\t\textension: type.extension,\n\t\t\tsignal: input.signal\n\t\t})\n\t}\n\n\tasync function convertTextual(input: {\n\t\ttext: string\n\t\tkind: ContentKind\n\t\tfinalUrl: URL\n\t\trender: boolean\n\t\t/** The hash of the HTML last read; see `FetchedPage.htmlHash`. */\n\t\thtmlHash?: string | null | undefined\n\t\tsignal?: AbortSignal | undefined\n\t}): Promise<PageBody> {\n\t\tconst body = input.text\n\t\t// The `Content-Type` header is authoritative - sniffing for a leading\n\t\t// `<html` misses every page that opens with a comment or a CDN banner - but\n\t\t// a `text/plain` body that is plainly HTML still needs the HTML tier.\n\t\tconst isHtml =\n\t\t\tinput.kind === 'html' ||\n\t\t\t(input.kind !== 'markdown' && looksLikeHtml(body))\n\n\t\tif (!isHtml && !isBlankMarkdown(body)) {\n\t\t\treturn { outcome: 'ok', source: 'content-negotiation', markdown: body }\n\t\t}\n\n\t\tconst html = isHtml ? body : null\n\t\t// Most origins send no validators, so without this every page of a quiet\n\t\t// site went through Workers AI again every night to learn nothing moved.\n\t\tif (html && input.htmlHash && htmlHash(html) === input.htmlHash) {\n\t\t\treturn { outcome: 'unchanged', source: 'not-modified' }\n\t\t}\n\t\tif (html && Buffer.byteLength(html) > MAX_HTML_BYTES) {\n\t\t\t// A 6 MB HTML upload came back as 6 MB of markdown through Browser\n\t\t\t// Rendering, and Workers AI would bill the same bytes. It is not a page,\n\t\t\t// and it will be just as oversized next run - so not `transient`.\n\t\t\tlogger.info('url-to-markdown.html_oversized', {\n\t\t\t\turl: input.finalUrl.href,\n\t\t\t\tbytes: Buffer.byteLength(html)\n\t\t\t})\n\t\t\treturn { outcome: 'unsupported', source: null, html }\n\t\t}\n\n\t\tlet shell = false\n\t\tlet workersAiRateLimited = false\n\t\tif (html) {\n\t\t\tconst visible = withoutHiddenUi(html)\n\t\t\tlet converted: string | null = null\n\t\t\ttry {\n\t\t\t\tconverted = await client.convertHtmlWithWorkersAi({\n\t\t\t\t\thtml: visible,\n\t\t\t\t\t// The post-redirect host: Cloudflare resolves relative links against it.\n\t\t\t\t\thostname: input.finalUrl.hostname,\n\t\t\t\t\tsignal: input.signal\n\t\t\t\t})\n\t\t\t} catch (error) {\n\t\t\t\tif (!(error instanceof CloudflareRateLimitedError)) throw error\n\t\t\t\tworkersAiRateLimited = true\n\t\t\t}\n\t\t\tif (converted && !isBlankMarkdown(converted)) {\n\t\t\t\tshell = isApplicationShell(visible, converted)\n\t\t\t\tif (!shell)\n\t\t\t\t\treturn {\n\t\t\t\t\t\toutcome: 'ok',\n\t\t\t\t\t\tsource: 'workers-ai',\n\t\t\t\t\t\tmarkdown: converted,\n\t\t\t\t\t\thtml\n\t\t\t\t\t}\n\t\t\t}\n\t\t}\n\n\t\t// Browser Rendering is the exception path: slow, rate limited, and the only\n\t\t// tier that ever timed out in the probe.\n\t\tif (input.render && !input.signal?.aborted) {\n\t\t\tlet rendered: string | null = null\n\t\t\ttry {\n\t\t\t\trendered = await client.renderUrlToMarkdown(\n\t\t\t\t\tinput.finalUrl.href,\n\t\t\t\t\tinput.signal\n\t\t\t\t)\n\t\t\t} catch (error) {\n\t\t\t\tif (!(error instanceof CloudflareRateLimitedError)) throw error\n\t\t\t\tworkersAiRateLimited = true\n\t\t\t}\n\t\t\tif (\n\t\t\t\trendered &&\n\t\t\t\t!isBlankMarkdown(rendered) &&\n\t\t\t\t(!html || !isApplicationShell(html, rendered))\n\t\t\t) {\n\t\t\t\treturn {\n\t\t\t\t\toutcome: 'ok',\n\t\t\t\t\tsource: 'browser-rendering',\n\t\t\t\t\tmarkdown: rendered,\n\t\t\t\t\thtml\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\t\treturn {\n\t\t\toutcome: 'transient',\n\t\t\tsource: null,\n\t\t\thtml,\n\t\t\tcaptureIssue: shell\n\t\t\t\t? 'render_incomplete'\n\t\t\t\t: workersAiRateLimited\n\t\t\t\t\t? 'workers_ai_rate_limited'\n\t\t\t\t\t: null\n\t\t}\n\t}\n\n\t/** Runs after the origin's slot is released: every tier it reaches is ours. */\n\tasync function readResponse(\n\t\tinput: OriginRead & {\n\t\t\trender: boolean\n\t\t\thtmlHash?: string | null | undefined\n\t\t\tsignal?: AbortSignal | undefined\n\t\t}\n\t): Promise<PageBody> {\n\t\tconst { response, body } = input\n\t\tconst status = response.status\n\t\tconst contentType = response.headers.get('content-type')\n\n\t\tif (status === 304) return { outcome: 'unchanged', source: 'not-modified' }\n\t\tif (status === 404 || status === 410) {\n\t\t\treturn { outcome: 'gone', source: null }\n\t\t}\n\t\t// Still refused after the host's pauses: rendering would only be refused\n\t\t// too (see `MAX_RATE_LIMIT_RETRIES`).\n\t\tif (status >= 500 || status === 429)\n\t\t\treturn { outcome: 'transient', source: null }\n\t\tif (status >= 400) {\n\t\t\tif (!input.render || !BOT_WALL_STATUSES.has(status)) {\n\t\t\t\treturn { outcome: 'blocked', source: null }\n\t\t\t}\n\t\t\tconst rendered = await client.renderUrlToMarkdown(\n\t\t\t\tinput.finalUrl.href,\n\t\t\t\tinput.signal\n\t\t\t)\n\t\t\treturn rendered\n\t\t\t\t? { outcome: 'ok', source: 'browser-rendering', markdown: rendered }\n\t\t\t\t: { outcome: 'blocked', source: null }\n\t\t}\n\t\tif (status < 200 || status >= 300) {\n\t\t\treturn { outcome: 'transient', source: null }\n\t\t}\n\n\t\tconst kind = classifyContentType(contentType)\n\t\tif (kind === 'asset') return { outcome: 'unsupported', source: null }\n\t\tif (kind === 'document') {\n\t\t\treturn convertDocument({\n\t\t\t\tbody,\n\t\t\t\tcontentType,\n\t\t\t\tfinalUrl: input.finalUrl,\n\t\t\t\tsignal: input.signal\n\t\t\t})\n\t\t}\n\t\tif (body.kind !== 'text') return { outcome: 'unsupported', source: null }\n\t\treturn convertTextual({\n\t\t\ttext: body.text,\n\t\t\tkind,\n\t\t\tfinalUrl: input.finalUrl,\n\t\t\trender: input.render,\n\t\t\thtmlHash: input.htmlHash,\n\t\t\tsignal: input.signal\n\t\t})\n\t}\n\n\t/**\n\t * Read one URL as markdown through Cloudflare's three tiers - content\n\t * negotiation, Workers AI `toMarkdown`, Browser Rendering - and report\n\t * everything the import pipeline needs about the page: real status, redirect\n\t * target, validators, HTML metadata and absolute links.\n\t *\n\t * Never throws: a page that cannot be read is an outcome, not an exception,\n\t * because the caller is crawling hundreds of URLs at a time. A knowledge bench\n\t * run reads through its crawl snapshot (`crawl-snapshot.ts`).\n\t */\n\tasync function fetchPageAsMarkdown(\n\t\tinput: FetchPageInput\n\t): Promise<FetchedPage> {\n\t\tif (!currentCrawlSnapshot()) return fetchPageLive(input)\n\n\t\t// The snapshot holds the whole page, read without validators; whether it is\n\t\t// \"unchanged\" for this run is decided here, the way the origin would.\n\t\tconst page = await throughCrawlSnapshot(\n\t\t\t`page ${input.url} ${input.country ?? ''} ${input.render ?? true}`,\n\t\t\t() =>\n\t\t\t\tfetchPageLive({\n\t\t\t\t\turl: input.url,\n\t\t\t\t\tcountry: input.country,\n\t\t\t\t\trender: input.render,\n\t\t\t\t\tsignal: input.signal\n\t\t\t\t})\n\t\t)\n\t\tconst unchanged =\n\t\t\tpage.outcome === 'ok' &&\n\t\t\t((input.htmlHash && page.htmlHash === input.htmlHash) ||\n\t\t\t\t(input.etag && page.etag === input.etag))\n\t\tif (!unchanged) return page\n\n\t\treturn {\n\t\t\t...page,\n\t\t\toutcome: 'unchanged',\n\t\t\tsource: 'not-modified',\n\t\t\tmarkdown: '',\n\t\t\ttitle: null,\n\t\t\tlang: null,\n\t\t\tcanonical: null,\n\t\t\talternates: [],\n\t\t\talternateLanguages: {},\n\t\t\tmetaDescription: null,\n\t\t\tjsonLd: [],\n\t\t\tlinks: [],\n\t\t\timages: [],\n\t\t\tcaptureIssue: null,\n\t\t\thtmlHash: null\n\t\t}\n\t}\n\n\tasync function fetchPageLive(input: FetchPageInput): Promise<FetchedPage> {\n\t\tconst signal = input.signal ?? currentOperationSignal()\n\t\tconst startedAt = performance.now()\n\t\tconst render = input.render ?? true\n\n\t\tconst prepared = prepareRequestUrl(input.url)\n\t\tif (!prepared) {\n\t\t\treturn buildPage(\n\t\t\t\t{\n\t\t\t\t\turl: input.url,\n\t\t\t\t\tfinalUrl: input.url,\n\t\t\t\t\tstatus: 0,\n\t\t\t\t\tcontentType: null,\n\t\t\t\t\tetag: null,\n\t\t\t\t\tlastModified: null,\n\t\t\t\t\tstartedAt\n\t\t\t\t},\n\t\t\t\t{ outcome: 'unsupported', source: null },\n\t\t\t\tlogger\n\t\t\t)\n\t\t}\n\n\t\tconst headers: Record<string, string> = {\n\t\t\tAccept: 'text/markdown, text/html;q=0.9, */*;q=0.8',\n\t\t\t'Accept-Language': '*',\n\t\t\t'User-Agent': BROWSER_USER_AGENT\n\t\t}\n\t\tif (input.etag) headers['If-None-Match'] = input.etag\n\t\tif (input.lastModified) headers['If-Modified-Since'] = input.lastModified\n\n\t\tconst request = (url: URL) =>\n\t\t\trequestOrigin(inMarket(url, input.country), headers, signal)\n\n\t\tlet requestUrl = prepared\n\t\tlet attempt: OriginAttempt = await request(requestUrl)\n\n\t\tif (attempt.kind === 'failed' && attempt.retryable) {\n\t\t\tconst fallback = wwwFallbackUrl(requestUrl)\n\t\t\tif (fallback) {\n\t\t\t\tconst retry = await request(fallback)\n\t\t\t\tif (retry.kind === 'read') {\n\t\t\t\t\trequestUrl = fallback\n\t\t\t\t\tattempt = retry\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\n\t\tfor (\n\t\t\tlet retry = 1;\n\t\t\tretry <= MAX_RATE_LIMIT_RETRIES &&\n\t\t\tattempt.kind === 'read' &&\n\t\t\tattempt.read.response.status === 429;\n\t\t\tretry++\n\t\t) {\n\t\t\tconst waitMs = rateLimitWaitMs(attempt.read.response, retry)\n\t\t\tif (await hostLimits.coolDownHost(requestUrl.hostname, waitMs)) {\n\t\t\t\tlogger.warn('url-to-markdown.host_rate_limited', {\n\t\t\t\t\thost: requestUrl.hostname,\n\t\t\t\t\tretry_after: attempt.read.response.headers.get('retry-after') ?? '',\n\t\t\t\t\twait_ms: waitMs\n\t\t\t\t})\n\t\t\t}\n\t\t\tattempt = await request(requestUrl)\n\t\t}\n\n\t\tif (attempt.kind !== 'read') {\n\t\t\treturn buildPage(\n\t\t\t\t{\n\t\t\t\t\turl: requestUrl.href,\n\t\t\t\t\tfinalUrl: requestUrl.href,\n\t\t\t\t\tstatus: 0,\n\t\t\t\t\tcontentType: null,\n\t\t\t\t\tetag: null,\n\t\t\t\t\tlastModified: null,\n\t\t\t\t\tstartedAt\n\t\t\t\t},\n\t\t\t\t// A refused host is a policy decision, not a retryable network blip.\n\t\t\t\t{\n\t\t\t\t\toutcome: attempt.kind === 'refused' ? 'blocked' : 'transient',\n\t\t\t\t\tsource: null\n\t\t\t\t},\n\t\t\t\tlogger\n\t\t\t)\n\t\t}\n\n\t\tconst { response, finalUrl } = attempt.read\n\t\tconst notModified = (stored: string | null | undefined) =>\n\t\t\tresponse.status === 304 ? (stored ?? null) : null\n\t\tconst body = await readResponse({\n\t\t\t...attempt.read,\n\t\t\trender,\n\t\t\thtmlHash: input.htmlHash,\n\t\t\tsignal\n\t\t})\n\n\t\tconst page = buildPage(\n\t\t\t{\n\t\t\t\turl: requestUrl.href,\n\t\t\t\tfinalUrl: finalUrl.href,\n\t\t\t\tstatus: response.status,\n\t\t\t\tcontentType: response.headers.get('content-type'),\n\t\t\t\t// A 304 need not repeat the validators; the stored ones stay valid. On\n\t\t\t\t// any other status only this response's headers count - carrying an old\n\t\t\t\t// ETag onto changed content would freeze the page forever.\n\t\t\t\tetag: response.headers.get('etag') ?? notModified(input.etag),\n\t\t\t\tlastModified:\n\t\t\t\t\tresponse.headers.get('last-modified') ??\n\t\t\t\t\tnotModified(input.lastModified),\n\t\t\t\tstartedAt\n\t\t\t},\n\t\t\tbody,\n\t\t\tlogger\n\t\t)\n\n\t\tlogger.info('url-to-markdown.completed', {\n\t\t\turl: page.finalUrl,\n\t\t\tstatus: page.status,\n\t\t\toutcome: page.outcome,\n\t\t\tsource: page.source ?? '',\n\t\t\tchars: page.markdown.length,\n\t\t\tduration_ms: page.durationMs\n\t\t})\n\t\treturn page\n\t}\n\n\t/** CLI/one-shot shape: markdown or an error. */\n\tasync function urlToMarkdown(urlString: string) {\n\t\tconst page = await fetchPageAsMarkdown({ url: urlString })\n\t\tconst { markdown, source } = page\n\t\tif (page.outcome !== 'ok' || !markdown || !source) {\n\t\t\tthrow new Error(\n\t\t\t\t`Failed to convert URL to markdown (${page.outcome}, status ${page.status}): ${urlString}`\n\t\t\t)\n\t\t}\n\t\treturn { markdown, source }\n\t}\n\n\treturn {\n\t\tfetchPageAsMarkdown,\n\t\tconvertDocumentBytes,\n\t\turlToMarkdown,\n\t\tconvertFileWithWorkersAi: client.convertFileWithWorkersAi,\n\t\trenderUrlToMarkdown: client.renderUrlToMarkdown,\n\t\t/** The SSRF guard the converter runs, bound to its `logger`. */\n\t\tisHostAllowed: allowHost,\n\t\t/** `withCrawlSnapshot` bound to the `crawlSnapshot` option. */\n\t\twithCrawlSnapshot<T>(\n\t\t\tinput: CrawlSnapshotInput | null | undefined,\n\t\t\trun: () => Promise<T>\n\t\t) {\n\t\t\tif (!input) return run()\n\t\t\tif (!options.crawlSnapshot) {\n\t\t\t\tthrow new Error('Set crawlSnapshot.root to use crawl snapshots')\n\t\t\t}\n\t\t\treturn withCrawlSnapshot(input, run, {\n\t\t\t\troot: options.crawlSnapshot.root,\n\t\t\t\tlogger: options.crawlSnapshot.logger ?? logger\n\t\t\t})\n\t\t}\n\t}\n}\n\nexport type WebsiteToMarkdown = ReturnType<typeof createWebsiteToMarkdown>\n"]}
+{"version":3,"file":"fetch-page.js","sourceRoot":"","sources":["../src/fetch-page.ts"],"names":[],"mappings":"AAAA,OAAO,EAEN,kBAAkB,EAClB,aAAa,EACb,eAAe,EACf,SAAS,EACT,OAAO,EACP,QAAQ,EACR,IAAI,EAEJ,eAAe,EACf,UAAU,EACV,aAAa,EACb,WAAW,EACX,gBAAgB,EAChB,MAAM,QAAQ,CAAA;AAEf,OAAO,EAEN,0BAA0B,EAC1B,sBAAsB,EACtB,MAAM,aAAa,CAAA;AACpB,OAAO,EAAE,mBAAmB,EAAE,uBAAuB,EAAE,MAAM,mBAAmB,CAAA;AAChF,OAAO,EAGN,oBAAoB,EACpB,oBAAoB,EACpB,iBAAiB,EACjB,MAAM,qBAAqB,CAAA;AAC5B,OAAO,EACN,kBAAkB,EAClB,eAAe,EACf,aAAa,EACb,MAAM,sBAAsB,CAAA;AAC7B,OAAO,EAAE,SAAS,EAAE,MAAM,WAAW,CAAA;AACrC,OAAO,EAAE,mBAAmB,EAAE,MAAM,oBAAoB,CAAA;AACxD,OAAO,EAA+B,gBAAgB,EAAE,MAAM,YAAY,CAAA;AAC1E,OAAO,EAAE,kBAAkB,EAAe,YAAY,EAAE,MAAM,aAAa,CAAA;AAC3E,OAAO,EACN,sBAAsB,EACtB,UAAU,EACV,SAAS,EACT,YAAY,EACZ,qBAAqB,EACrB,gBAAgB,EAChB,iBAAiB,EACjB,MAAM,oBAAoB,CAAA;AAC3B,OAAO,EAAE,4BAA4B,EAAE,MAAM,mBAAmB,CAAA;AAChE,OAAO,EAEN,eAAe,EAEf,YAAY,EACZ,mBAAmB,EACnB,aAAa,EACb,mBAAmB,EACnB,MAAM,gBAAgB,CAAA;AACvB,OAAO,EAAE,sBAAsB,EAAE,MAAM,uBAAuB,CAAA;AAC9D,OAAO,EAAE,UAAU,EAAE,QAAQ,EAAE,MAAM,gBAAgB,CAAA;AAErD,OAAO,EAAE,WAAW,EAAE,cAAc,EAAE,MAAM,qBAAqB,CAAA;AACjE,OAAO,EAAE,kBAAkB,EAAE,MAAM,iBAAiB,CAAA;AACpD,OAAO,EAAE,iBAAiB,EAAE,MAAM,iBAAiB,CAAA;AAInD,MAAM,gBAAgB,GAAG,MAAM,CAAA;AAC/B,+EAA+E;AAC/E,MAAM,cAAc,GAAG,CAAC,GAAG,IAAI,GAAG,IAAI,CAAA;AACtC;;;GAGG;AACH,MAAM,mBAAmB,GAAG,CAAC,GAAG,cAAc,CAAA;AAC9C,MAAM,kBAAkB,GAAG,CAAC,GAAG,IAAI,GAAG,IAAI,CAAA;AAC1C,MAAM,aAAa,GAAG,EAAE,GAAG,IAAI,GAAG,IAAI,CAAA;AACtC,MAAM,aAAa,GAAG,EAAE,CAAA;AAExB,0FAA0F;AAC1F,MAAM,CAAC,MAAM,kBAAkB,GAAG,OAAO,CAAA;AAoEzC;;;;;;GAMG;AACH,SAAS,SAAS,CAAC,IAAY;IAC9B,OAAO,IAAI,CAAC,UAAU,CAAC,QAAQ,EAAE,EAAE,CAAC,CAAC,SAAS,CAAC,KAAK,CAAC,CAAA;AACtD,CAAC;AAED;;;;GAIG;AACH,SAAS,SAAS,CACjB,IAAc,EACd,OAAiB,EACjB,MAAc;IAEd,MAAM,IAAI,GAAG;QACZ,GAAG,OAAO;QACV,QAAQ,EAAE,OAAO,CAAC,QAAQ,CAAC,CAAC,CAAC,SAAS,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,IAAI;QAC/D,IAAI,EAAE,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,SAAS,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,IAAI;KACnD,CAAA;IACD,MAAM,WAAW,GAAG,gBAAgB,CAAC,IAAI,CAAC,QAAQ,IAAI,EAAE,CAAC,CAAA;IACzD,IAAI,QAAQ,GAAG,sBAAsB,CAAC,WAAW,CAAC,QAAQ,CAAC,CAAA;IAC3D,IAAI,QAAQ,CAAC,MAAM,GAAG,kBAAkB,EAAE,CAAC;QAC1C,MAAM,CAAC,IAAI,CAAC,2BAA2B,EAAE;YACxC,GAAG,EAAE,IAAI,CAAC,QAAQ;YAClB,KAAK,EAAE,QAAQ,CAAC,MAAM;SACtB,CAAC,CAAA;QACF,QAAQ,GAAG,QAAQ,CAAC,KAAK,CAAC,CAAC,EAAE,kBAAkB,CAAC,CAAA;IACjD,CAAC;IAED,MAAM,QAAQ,GAAG,IAAI,CAAC,IAAI;QACzB,CAAC,CAAC,mBAAmB,CAAC,IAAI,CAAC,IAAI,EAAE,IAAI,CAAC,QAAQ,CAAC;QAC/C,CAAC,CAAC,IAAI,CAAA;IACP,MAAM,IAAI,GAAG,YAAY,CAAC,QAAQ,EAAE,IAAI,CAAC,QAAQ,CAAC,CAAA;IAClD,sEAAsE;IACtE,wEAAwE;IACxE,0DAA0D;IAC1D,MAAM,KAAK,GAAG,IAAI,CAAC,IAAI;QACtB,CAAC,CAAC,eAAe,CAAC,CAAC,GAAG,IAAI,CAAC,KAAK,EAAE,GAAG,SAAS,CAAC,IAAI,CAAC,IAAI,EAAE,IAAI,CAAC,QAAQ,CAAC,CAAC,CAAC;QAC1E,CAAC,CAAC,IAAI,CAAC,KAAK,CAAA;IAEb,OAAO;QACN,GAAG,EAAE,IAAI,CAAC,GAAG;QACb,QAAQ,EAAE,IAAI,CAAC,QAAQ;QACvB,MAAM,EAAE,IAAI,CAAC,MAAM;QACnB,OAAO,EAAE,IAAI,CAAC,OAAO;QACrB,MAAM,EAAE,IAAI,CAAC,MAAM;QACnB,WAAW,EAAE,IAAI,CAAC,WAAW;QAC7B,IAAI,EAAE,IAAI,CAAC,IAAI;QACf,YAAY,EAAE,IAAI,CAAC,YAAY;QAC/B,QAAQ;QACR,KAAK,EAAE,QAAQ,EAAE,KAAK,IAAI,WAAW,CAAC,KAAK,IAAI,iBAAiB,CAAC,QAAQ,CAAC;QAC1E,IAAI,EAAE,QAAQ,EAAE,IAAI,IAAI,IAAI;QAC5B,SAAS,EAAE,QAAQ,EAAE,SAAS,IAAI,IAAI;QACtC,UAAU,EAAE,QAAQ,EAAE,UAAU,IAAI,EAAE;QACtC,kBAAkB,EAAE,QAAQ,EAAE,kBAAkB,IAAI,EAAE;QACtD,eAAe,EACd,QAAQ,EAAE,eAAe,IAAI,WAAW,CAAC,WAAW,IAAI,IAAI;QAC7D,MAAM,EAAE,QAAQ,EAAE,MAAM,IAAI,EAAE;QAC9B,KAAK;QACL,MAAM,EAAE,eAAe,CAAC;YACvB,GAAG,IAAI,CAAC,MAAM;YACd,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,UAAU,CAAC,IAAI,CAAC,IAAI,EAAE,IAAI,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;SAC1D,CAAC;QACF,YAAY,EACX,WAAW,CAAC,QAAQ,CAAC,MAAM,GAAG,kBAAkB;YAC/C,CAAC,CAAC,gBAAgB;YAClB,CAAC,CAAC,CAAC,IAAI,CAAC,YAAY,IAAI,IAAI,CAAC;QAC/B,UAAU,EAAE,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,SAAS,EAAE,CAAC;QACxC,0DAA0D;QAC1D,QAAQ,EAAE,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,IAAI;KAChD,CAAA;AACF,CAAC;AAED;;;;;GAKG;AACH,SAAS,iBAAiB,CAAC,GAAW,EAAE,cAAuB;IAC9D,IAAI,MAAW,CAAA;IACf,IAAI,CAAC;QACJ,MAAM,GAAG,IAAI,GAAG,CAAC,GAAG,CAAC,CAAA;IACtB,CAAC;IAAC,MAAM,CAAC;QACR,OAAO,IAAI,CAAA;IACZ,CAAC;IACD,IAAI,MAAM,CAAC,QAAQ,KAAK,QAAQ;QAAE,OAAO,MAAM,CAAA;IAC/C,IAAI,MAAM,CAAC,QAAQ,KAAK,OAAO;QAAE,OAAO,IAAI,CAAA;IAC5C,IAAI,mBAAmB,CAAC,MAAM,CAAC,QAAQ,CAAC,EAAE,CAAC;QAC1C,OAAO,cAAc,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,IAAI,CAAA;IACtC,CAAC;IACD,MAAM,CAAC,QAAQ,GAAG,QAAQ,CAAA;IAC1B,IAAI,MAAM,CAAC,IAAI,KAAK,IAAI;QAAE,MAAM,CAAC,IAAI,GAAG,EAAE,CAAA;IAC1C,OAAO,MAAM,CAAA;AACd,CAAC;AAED;;;GAGG;AACH,SAAS,cAAc,CAAC,GAAQ;IAC/B,MAAM,IAAI,GAAG,GAAG,CAAC,QAAQ,CAAA;IACzB,IAAI,IAAI,CAAC,UAAU,CAAC,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC;QAAE,OAAO,IAAI,CAAA;IAC/D,IAAI,mBAAmB,CAAC,IAAI,CAAC,IAAI,UAAU,CAAC,IAAI,CAAC,IAAI,CAAC;QAAE,OAAO,IAAI,CAAA;IACnE,MAAM,IAAI,GAAG,IAAI,GAAG,CAAC,GAAG,CAAC,IAAI,CAAC,CAAA;IAC9B,IAAI,CAAC,QAAQ,GAAG,OAAO,IAAI,EAAE,CAAA;IAC7B,OAAO,IAAI,CAAA;AACZ,CAAC;AAED;;;;;;;GAOG;AACH,MAAM,sBAAsB,GAAG,CAAC,CAAA;AAChC,MAAM,sBAAsB,GAAG,MAAM,CAAA;AAErC,uEAAuE;AACvE,SAAS,eAAe,CAAC,QAAkB,EAAE,KAAa;IACzD,MAAM,OAAO,GAAG,eAAe,CAAC,QAAQ,CAAC,OAAO,CAAC,GAAG,CAAC,aAAa,CAAC,CAAC,CAAA;IACpE,OAAO,IAAI,CAAC,GAAG,CACd,sBAAsB,EACtB,OAAO;QACN,kBAAkB,CAAC,KAAK,EAAE;YACzB,MAAM,EAAE,KAAK;YACb,KAAK,EAAE,sBAAsB;YAC7B,MAAM,EAAE,KAAK;SACb,CAAC,CACH,CAAA;AACF,CAAC;AAED;;;;;GAKG;AACH,MAAM,kBAAkB,GAAG,CAAC,CAAA;AAE5B,SAAS,QAAQ,CAAC,IAAY;IAC7B,OAAO,SAAS,CAAC,GAAG,kBAAkB,KAAK,IAAI,EAAE,CAAC,CAAA;AACnD,CAAC;AAED,iFAAiF;AACjF,SAAS,QAAQ,CAAC,GAAQ,EAAE,OAAyB;IACpD,IAAI,CAAC,OAAO,IAAI,GAAG,CAAC,YAAY,CAAC,GAAG,CAAC,SAAS,CAAC;QAAE,OAAO,GAAG,CAAA;IAC3D,MAAM,MAAM,GAAG,IAAI,GAAG,CAAC,GAAG,CAAC,IAAI,CAAC,CAAA;IAChC,MAAM,CAAC,YAAY,CAAC,GAAG,CAAC,SAAS,EAAE,OAAO,CAAC,CAAA;IAC3C,OAAO,MAAM,CAAA;AACd,CAAC;AAED,gFAAgF;AAChF,SAAS,aAAa,CAAC,WAAmB,EAAE,UAAe;IAC1D,IAAI,CAAC,WAAW;QAAE,OAAO,UAAU,CAAA;IACnC,IAAI,CAAC;QACJ,OAAO,IAAI,GAAG,CAAC,WAAW,CAAC,CAAA;IAC5B,CAAC;IAAC,MAAM,CAAC;QACR,OAAO,UAAU,CAAA;IAClB,CAAC;AACF,CAAC;AAED,SAAS,YAAY,CAAC,KAAc;IACnC,OAAO,QAAQ,CAAC,KAAK,CAAC,IAAI,OAAO,CAAC,CAAC,YAAY,EAAE,cAAc,CAAC,EAAE,KAAK,CAAC,IAAI,CAAC,CAAA;AAC9E,CAAC;AAED,sFAAsF;AACtF,MAAM,aAAa,GAAG,CAAC,UAAU,EAAE,MAAM,EAAE,MAAM,CAAU,CAAA;AAgB3D,mFAAmF;AACnF,SAAS,WAAW,CAAC,QAAkB;IACtC,KAAK,QAAQ,CAAC,IAAI,EAAE,MAAM,EAAE,CAAC,KAAK,CAAC,GAAG,EAAE,GAAE,CAAC,CAAC,CAAA;AAC7C,CAAC;AAED;;;;GAIG;AACH,KAAK,UAAU,cAAc,CAC5B,QAAkB,EAClB,QAAa,EACb,MAAc;IAEd,MAAM,MAAM,GAAG,QAAQ,CAAC,MAAM,CAAA;IAC9B,IAAI,MAAM,GAAG,GAAG,IAAI,MAAM,IAAI,GAAG,EAAE,CAAC;QACnC,WAAW,CAAC,QAAQ,CAAC,CAAA;QACrB,OAAO,EAAE,IAAI,EAAE,MAAM,EAAE,CAAA;IACxB,CAAC;IAED,MAAM,WAAW,GAAG,QAAQ,CAAC,OAAO,CAAC,GAAG,CAAC,cAAc,CAAC,CAAA;IACxD,MAAM,IAAI,GAAG,mBAAmB,CAAC,WAAW,CAAC,CAAA;IAC7C,OAAO,WAAW,CAAC,IAAI,EAAE;QACxB,KAAK,EAAE,GAAG,EAAE;YACX,WAAW,CAAC,QAAQ,CAAC,CAAA;YACrB,OAAO,EAAE,IAAI,EAAE,MAAM,EAAE,CAAA;QACxB,CAAC;QACD,QAAQ,EAAE,KAAK,IAAI,EAAE;YACpB,MAAM,IAAI,GAAG,uBAAuB,CAAC,WAAW,CAAC,CAAA;YACjD,IAAI,CAAC,IAAI,EAAE,CAAC;gBACX,WAAW,CAAC,QAAQ,CAAC,CAAA;gBACrB,OAAO,EAAE,IAAI,EAAE,MAAM,EAAE,CAAA;YACxB,CAAC;YACD,MAAM,KAAK,GACV,IAAI,CAAC,SAAS,KAAK,KAAK,CAAC,CAAC,CAAC,aAAa,CAAC,CAAC,CAAC,kBAAkB,CAAA;YAC9D,MAAM,QAAQ,GAAG,MAAM,CAAC,QAAQ,CAAC,OAAO,CAAC,GAAG,CAAC,gBAAgB,CAAC,CAAC,CAAA;YAC/D,IAAI,MAAM,CAAC,QAAQ,CAAC,QAAQ,CAAC,IAAI,QAAQ,GAAG,KAAK,EAAE,CAAC;gBACnD,WAAW,CAAC,QAAQ,CAAC,CAAA;gBACrB,MAAM,CAAC,IAAI,CAAC,oCAAoC,EAAE;oBACjD,GAAG,EAAE,QAAQ,CAAC,IAAI;oBAClB,KAAK,EAAE,QAAQ;iBACf,CAAC,CAAA;gBACF,OAAO,EAAE,IAAI,EAAE,MAAM,EAAE,YAAY,EAAE,qBAAqB,EAAE,CAAA;YAC7D,CAAC;YACD,MAAM,KAAK,GAAG,MAAM,eAAe,CAAC,QAAQ,EAAE,KAAK,CAAC,CAAA;YACpD,OAAO,KAAK;gBACX,CAAC,CAAC,EAAE,IAAI,EAAE,OAAO,EAAE,KAAK,EAAE;gBAC1B,CAAC,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,YAAY,EAAE,qBAAqB,EAAE,CAAA;QACzD,CAAC;QACD,GAAG,gBAAgB,CAAC,IAAI,EAAE,aAAa,EAAE,KAAK,IAAyB,EAAE;YACxE,MAAM,QAAQ,GAAG,MAAM,CAAC,QAAQ,CAAC,OAAO,CAAC,GAAG,CAAC,gBAAgB,CAAC,CAAC,CAAA;YAC/D,MAAM,IAAI,GACT,MAAM,CAAC,QAAQ,CAAC,QAAQ,CAAC,IAAI,QAAQ,GAAG,mBAAmB;gBAC1D,CAAC,CAAC,IAAI;gBACN,CAAC,CAAC,MAAM,QAAQ,CAAC,QAAQ,EAAE,mBAAmB,CAAC,CAAA;YACjD,IAAI,CAAC,IAAI,IAAI,IAAI,CAAC,SAAS,EAAE,CAAC;gBAC7B,IAAI,CAAC,IAAI;oBAAE,WAAW,CAAC,QAAQ,CAAC,CAAA;gBAChC,MAAM,CAAC,IAAI,CAAC,gCAAgC,EAAE;oBAC7C,GAAG,EAAE,QAAQ,CAAC,IAAI;oBAClB,KAAK,EAAE,IAAI,CAAC,CAAC,CAAC,mBAAmB,CAAC,CAAC,CAAC,QAAQ;iBAC5C,CAAC,CAAA;gBACF,OAAO,EAAE,IAAI,EAAE,MAAM,EAAE,YAAY,EAAE,iBAAiB,EAAE,CAAA;YACzD,CAAC;YACD,OAAO,EAAE,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,UAAU,CAAC,IAAI,CAAC,KAAK,CAAC,EAAE,CAAA;QACtD,CAAC,CAAC;KACF,CAAC,CAAA;AACH,CAAC;AAED,SAAS,gBAAgB,CAAC,QAAgB,EAAE,SAAiB;IAC5D,MAAM,OAAO,GAAG,QAAQ,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,EAAE,KAAK,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,IAAI,UAAU,CAAA;IACzE,MAAM,IAAI,GAAG,OAAO,CAAC,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC,CAAA;IACjC,OAAO,aAAa,CAAC,IAAI,CAAC,KAAK,SAAS,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,GAAG,IAAI,IAAI,SAAS,EAAE,CAAA;AACzE,CAAC;AAED,KAAK,UAAU,eAAe,CAAC,QAAkB,EAAE,QAAgB;IAClE,MAAM,IAAI,GAAG,MAAM,QAAQ,CAAC,QAAQ,EAAE,QAAQ,CAAC,CAAA;IAC/C,OAAO,IAAI,CAAC,SAAS,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,MAAM,CAAA;AACjD,CAAC;AAED;;;;;;;;;;;;;;GAcG;AACH,MAAM,kBAAkB,GAAG;IAC1B,UAAU;IACV,oBAAoB;IACpB,0BAA0B;IAC1B,+BAA+B;IAC/B,gCAAgC;CAChC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;AAEZ,SAAS,eAAe,CAAC,IAAY;IACpC,OAAO,IAAI,YAAY,EAAE;SACvB,EAAE,CAAC,kBAAkB,EAAE;QACvB,OAAO,CAAC,OAAO;YACd,OAAO,CAAC,MAAM,EAAE,CAAA;QACjB,CAAC;KACD,CAAC;SACD,SAAS,CAAC,IAAI,CAAC,CAAA;AAClB,CAAC;AAED,+EAA+E;AAC/E,MAAM,iBAAiB,GAAG,CAAC,GAAG,EAAE,GAAG,CAAU,CAAA;AAqE7C;;;GAGG;AACH,MAAM,UAAU,uBAAuB,CAAC,OAAiC;IACxE,MAAM,MAAM,GAAG,OAAO,CAAC,MAAM,IAAI,YAAY,CAAA;IAC7C,MAAM,WAAW,GAAG,OAAO,CAAC,WAAW,IAAI,4BAA4B,CAAA;IACvE,MAAM,MAAM,GAAG,sBAAsB,CAAC;QACrC,WAAW,EAAE,OAAO,CAAC,UAAU;QAC/B,WAAW;QACX,UAAU,EAAE,OAAO,CAAC,UAAU;QAC9B,cAAc,EAAE,OAAO,CAAC,cAAc;QACtC,MAAM;KACN,CAAC,CAAA;IACF,MAAM,UAAU,GAAG,gBAAgB,CAAC,WAAW,CAAC,CAAA;IAChD,MAAM,cAAc,GAAG,OAAO,CAAC,cAAc,KAAK,IAAI,CAAA;IACtD,MAAM,SAAS,GACd,OAAO,CAAC,aAAa;QACrB,CAAC,CAAC,QAAgB,EAAE,EAAE,CAAC,aAAa,CAAC,QAAQ,EAAE,EAAE,MAAM,EAAE,cAAc,EAAE,CAAC,CAAC,CAAA;IAC5E,MAAM,KAAK,GAAG,EAAE,aAAa,EAAE,SAAS,EAAE,KAAK,EAAE,OAAO,CAAC,KAAK,EAAE,CAAA;IAEhE,KAAK,UAAU,aAAa,CAC3B,GAAQ,EACR,OAA+B,EAC/B,MAAgC;QAEhC,IAAI,MAAM,EAAE,OAAO;YAAE,OAAO,EAAE,IAAI,EAAE,QAAQ,EAAE,SAAS,EAAE,KAAK,EAAE,CAAA;QAChE,IAAI,CAAC;YACJ,wEAAwE;YACxE,0EAA0E;YAC1E,sEAAsE;YACtE,wEAAwE;YACxE,kDAAkD;YAClD,oEAAoE;YACpE,oEAAoE;YACpE,MAAM,IAAI,GAAG,MAAM,UAAU,CAAC,aAAa,CAAC,GAAG,CAAC,QAAQ,EAAE,KAAK,IAAI,EAAE;gBACpE,MAAM,GAAG,GAAG,MAAM,eAAe,CAChC,GAAG,EACH,EAAE,MAAM,EAAE,aAAa,CAAC,gBAAgB,EAAE,MAAM,CAAC,EAAE,OAAO,EAAE,EAC5D,KAAK,CACL,CAAA;gBACD,MAAM,QAAQ,GAAG,GAAG,CAAC,QAAQ,CAAA;gBAC7B,MAAM,QAAQ,GAAG,aAAa,CAAC,QAAQ,CAAC,GAAG,EAAE,GAAG,CAAC,GAAG,CAAC,CAAA;gBACrD,OAAO;oBACN,QAAQ;oBACR,QAAQ;oBACR,IAAI,EAAE,MAAM,cAAc,CAAC,QAAQ,EAAE,QAAQ,EAAE,MAAM,CAAC;iBACtD,CAAA;YACF,CAAC,CAAC,CAAA;YACF,OAAO,EAAE,IAAI,EAAE,MAAM,EAAE,IAAI,EAAE,CAAA;QAC9B,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YAChB,uEAAuE;YACvE,IAAI,KAAK,YAAY,mBAAmB;gBAAE,OAAO,EAAE,IAAI,EAAE,SAAS,EAAE,CAAA;YACpE,MAAM,CAAC,IAAI,CAAC,+BAA+B,EAAE;gBAC5C,GAAG,EAAE,GAAG,CAAC,IAAI;gBACb,GAAG,kBAAkB,CAAC,KAAK,CAAC;aAC5B,CAAC,CAAA;YACF,wEAAwE;YACxE,qDAAqD;YACrD,OAAO,EAAE,IAAI,EAAE,QAAQ,EAAE,SAAS,EAAE,CAAC,YAAY,CAAC,KAAK,CAAC,EAAE,CAAA;QAC3D,CAAC;IACF,CAAC;IAED;;;;;;;;OAQG;IACH,KAAK,UAAU,oBAAoB,CAAC,KAOnC;QACA,MAAM,EAAE,KAAK,EAAE,GAAG,KAAK,CAAA;QACvB,IAAI,CAAC,KAAK,CAAC,UAAU;YAAE,OAAO,EAAE,OAAO,EAAE,aAAa,EAAE,MAAM,EAAE,IAAI,EAAE,CAAA;QACtE,IAAI,KAAK,CAAC,MAAM,EAAE,OAAO;YAAE,OAAO,EAAE,OAAO,EAAE,WAAW,EAAE,MAAM,EAAE,IAAI,EAAE,CAAA;QACxE,IAAI,oBAAoB,GAAG,KAAK,CAAA;QAChC,IAAI,QAAQ,GAAkB,IAAI,CAAA;QAClC,IAAI,KAAK,CAAC,UAAU,IAAI,CAAC,KAAK,CAAC,iBAAiB,IAAI,kBAAkB,CAAC,EAAE,CAAC;YACzE,IAAI,CAAC;gBACJ,QAAQ,GAAG,MAAM,MAAM,CAAC,wBAAwB,CAAC;oBAChD,IAAI,EAAE,KAAK,CAAC,SAAS,KAAK,MAAM,CAAC,CAAC,CAAC,iBAAiB,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,KAAK;oBACnE,QAAQ,EAAE,KAAK,CAAC,QAAQ;oBACxB,QAAQ,EAAE,KAAK,CAAC,QAAQ;oBACxB,MAAM,EAAE,KAAK,CAAC,MAAM;iBACpB,CAAC,CAAA;YACH,CAAC;YAAC,OAAO,KAAK,EAAE,CAAC;gBAChB,IAAI,CAAC,CAAC,KAAK,YAAY,0BAA0B,CAAC;oBAAE,MAAM,KAAK,CAAA;gBAC/D,oBAAoB,GAAG,IAAI,CAAA;YAC5B,CAAC;QACF,CAAC;QACD,MAAM,IAAI,GAAG,qBAAqB,CAAC,QAAQ,IAAI,EAAE,CAAC,CAAA;QAClD,IAAI,CAAC,eAAe,CAAC,IAAI,CAAC;YACzB,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE,MAAM,EAAE,YAAY,EAAE,QAAQ,EAAE,IAAI,EAAE,CAAA;QAC/D,IAAI,KAAK,CAAC,SAAS,KAAK,KAAK,EAAE,CAAC;YAC/B,OAAO;gBACN,OAAO,EAAE,WAAW;gBACpB,MAAM,EAAE,IAAI;gBACZ,YAAY,EAAE,oBAAoB,CAAC,CAAC,CAAC,yBAAyB,CAAC,CAAC,CAAC,IAAI;aACrE,CAAA;QACF,CAAC;QACD,IAAI,KAAK,CAAC,MAAM,EAAE,OAAO;YAAE,OAAO,EAAE,OAAO,EAAE,WAAW,EAAE,MAAM,EAAE,IAAI,EAAE,CAAA;QACxE,IAAI,CAAC;YACJ,MAAM,QAAQ,GAAG,IAAI,CAAC,OAAO,CAAC,QAAQ,EAAE,0BAA0B,CAAC,CAAA;YACnE,MAAM,MAAM,GAAG,MAAM,QAAQ,CAAC;gBAC7B,KAAK;gBACL,QAAQ,EAAE,KAAK,CAAC,QAAQ;gBACxB,QAAQ,EAAE,KAAK,CAAC,QAAQ;gBACxB,QAAQ,EAAE,aAAa;gBACvB,MAAM,EAAE,KAAK,CAAC,MAAM;aACpB,CAAC,CAAA;YACF,OAAO;gBACN,OAAO,EAAE,IAAI;gBACb,MAAM,EAAE,iBAAiB;gBACzB,QAAQ,EAAE,MAAM,CAAC,QAAQ;gBACzB,2EAA2E;gBAC3E,0EAA0E;gBAC1E,YAAY,EACX,MAAM,CAAC,QAAQ,KAAK,IAAI,IAAI,MAAM,CAAC,QAAQ,IAAI,aAAa;oBAC3D,CAAC,CAAC,2BAA2B;oBAC7B,CAAC,CAAC,IAAI;aACR,CAAA;QACF,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YAChB,MAAM,CAAC,IAAI,CAAC,kCAAkC,EAAE;gBAC/C,QAAQ,EAAE,KAAK,CAAC,QAAQ;gBACxB,GAAG,kBAAkB,CAAC,KAAK,CAAC;aAC5B,CAAC,CAAA;YACF,OAAO;gBACN,OAAO,EAAE,WAAW;gBACpB,MAAM,EAAE,IAAI;gBACZ,YAAY,EAAE,oBAAoB;oBACjC,CAAC,CAAC,yBAAyB;oBAC3B,CAAC,CAAC,uBAAuB;aAC1B,CAAA;QACF,CAAC;IACF,CAAC;IAED,KAAK,UAAU,eAAe,CAAC,KAK9B;QACA,MAAM,IAAI,GAAG,uBAAuB,CAAC,KAAK,CAAC,WAAW,CAAC,CAAA;QACvD,IAAI,CAAC,IAAI,IAAI,KAAK,CAAC,IAAI,CAAC,IAAI,KAAK,OAAO,EAAE,CAAC;YAC1C,OAAO;gBACN,OAAO,EAAE,aAAa;gBACtB,MAAM,EAAE,IAAI;gBACZ,YAAY,EAAE,WAAW,CAAC,KAAK,CAAC,IAAI,EAAE,MAAM,EAAE;oBAC7C,IAAI,EAAE,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,YAAY;oBACjC,IAAI,EAAE,GAAG,EAAE,CAAC,IAAI;oBAChB,KAAK,EAAE,GAAG,EAAE,CAAC,IAAI;iBACjB,CAAC;aACF,CAAA;QACF,CAAC;QACD,OAAO,oBAAoB,CAAC;YAC3B,KAAK,EAAE,KAAK,CAAC,IAAI,CAAC,KAAK;YACvB,QAAQ,EAAE,gBAAgB,CAAC,KAAK,CAAC,QAAQ,CAAC,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC;YAC/D,QAAQ,EAAE,IAAI,CAAC,QAAQ;YACvB,SAAS,EAAE,IAAI,CAAC,SAAS;YACzB,MAAM,EAAE,KAAK,CAAC,MAAM;SACpB,CAAC,CAAA;IACH,CAAC;IAED,KAAK,UAAU,cAAc,CAAC,KAQ7B;QACA,MAAM,IAAI,GAAG,KAAK,CAAC,IAAI,CAAA;QACvB,sEAAsE;QACtE,4EAA4E;QAC5E,sEAAsE;QACtE,MAAM,MAAM,GAAG,WAAW,CAAC,KAAK,CAAC,IAAI,EAAE;YACtC,IAAI,EAAE,GAAG,EAAE,CAAC,IAAI;YAChB,QAAQ,EAAE,GAAG,EAAE,CAAC,KAAK;YACrB,IAAI,EAAE,GAAG,EAAE,CAAC,aAAa,CAAC,IAAI,CAAC;SAC/B,CAAC,CAAA;QAEF,IAAI,CAAC,MAAM,IAAI,CAAC,eAAe,CAAC,IAAI,CAAC,EAAE,CAAC;YACvC,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE,MAAM,EAAE,qBAAqB,EAAE,QAAQ,EAAE,IAAI,EAAE,CAAA;QACxE,CAAC;QAED,MAAM,IAAI,GAAG,MAAM,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,CAAA;QACjC,yEAAyE;QACzE,yEAAyE;QACzE,IACC,IAAI;YACJ,KAAK,CAAC,QAAQ;YACd,QAAQ,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC,KAAK,KAAK,CAAC,QAAQ,EAC3C,CAAC;YACF,OAAO,EAAE,OAAO,EAAE,WAAW,EAAE,MAAM,EAAE,cAAc,EAAE,CAAA;QACxD,CAAC;QACD,IAAI,IAAI,IAAI,MAAM,CAAC,UAAU,CAAC,IAAI,CAAC,GAAG,cAAc,EAAE,CAAC;YACtD,mEAAmE;YACnE,yEAAyE;YACzE,kEAAkE;YAClE,MAAM,CAAC,IAAI,CAAC,gCAAgC,EAAE;gBAC7C,GAAG,EAAE,KAAK,CAAC,QAAQ,CAAC,IAAI;gBACxB,KAAK,EAAE,MAAM,CAAC,UAAU,CAAC,IAAI,CAAC;aAC9B,CAAC,CAAA;YACF,OAAO;gBACN,OAAO,EAAE,aAAa;gBACtB,MAAM,EAAE,IAAI;gBACZ,IAAI;gBACJ,YAAY,EAAE,iBAAiB;aAC/B,CAAA;QACF,CAAC;QAED,IAAI,KAAK,GAAG,KAAK,CAAA;QACjB,IAAI,oBAAoB,GAAG,KAAK,CAAA;QAChC,IAAI,iBAAiB,GAAG,KAAK,CAAA;QAC7B,IAAI,IAAI,EAAE,CAAC;YACV,MAAM,OAAO,GAAG,eAAe,CAAC,IAAI,CAAC,CAAA;YACrC,IAAI,SAAS,GAAkB,IAAI,CAAA;YACnC,IAAI,CAAC;gBACJ,SAAS,GAAG,MAAM,MAAM,CAAC,wBAAwB,CAAC;oBACjD,IAAI,EAAE,OAAO;oBACb,yEAAyE;oBACzE,QAAQ,EAAE,KAAK,CAAC,QAAQ,CAAC,QAAQ;oBACjC,MAAM,EAAE,KAAK,CAAC,MAAM;iBACpB,CAAC,CAAA;YACH,CAAC;YAAC,OAAO,KAAK,EAAE,CAAC;gBAChB,IAAI,CAAC,CAAC,KAAK,YAAY,0BAA0B,CAAC;oBAAE,MAAM,KAAK,CAAA;gBAC/D,oBAAoB,GAAG,IAAI,CAAA;YAC5B,CAAC;YACD,IAAI,SAAS,IAAI,CAAC,eAAe,CAAC,SAAS,CAAC,EAAE,CAAC;gBAC9C,KAAK,GAAG,kBAAkB,CAAC,OAAO,EAAE,SAAS,CAAC,CAAA;gBAC9C,IAAI,CAAC,KAAK;oBACT,OAAO;wBACN,OAAO,EAAE,IAAI;wBACb,MAAM,EAAE,YAAY;wBACpB,QAAQ,EAAE,SAAS;wBACnB,IAAI;qBACJ,CAAA;YACH,CAAC;QACF,CAAC;QAED,4EAA4E;QAC5E,yCAAyC;QACzC,IAAI,KAAK,CAAC,MAAM,IAAI,CAAC,KAAK,CAAC,MAAM,EAAE,OAAO,EAAE,CAAC;YAC5C,IAAI,QAAQ,GAAkB,IAAI,CAAA;YAClC,IAAI,CAAC;gBACJ,QAAQ,GAAG,MAAM,MAAM,CAAC,mBAAmB,CAC1C,KAAK,CAAC,QAAQ,CAAC,IAAI,EACnB,KAAK,CAAC,MAAM,CACZ,CAAA;YACF,CAAC;YAAC,OAAO,KAAK,EAAE,CAAC;gBAChB,IAAI,CAAC,CAAC,KAAK,YAAY,0BAA0B,CAAC;oBAAE,MAAM,KAAK,CAAA;gBAC/D,iBAAiB,GAAG,IAAI,CAAA;YACzB,CAAC;YACD,IACC,QAAQ;gBACR,CAAC,eAAe,CAAC,QAAQ,CAAC;gBAC1B,CAAC,CAAC,IAAI,IAAI,CAAC,kBAAkB,CAAC,IAAI,EAAE,QAAQ,CAAC,CAAC,EAC7C,CAAC;gBACF,OAAO;oBACN,OAAO,EAAE,IAAI;oBACb,MAAM,EAAE,mBAAmB;oBAC3B,QAAQ,EAAE,QAAQ;oBAClB,IAAI;iBACJ,CAAA;YACF,CAAC;QACF,CAAC;QACD,OAAO;YACN,OAAO,EAAE,WAAW;YACpB,MAAM,EAAE,IAAI;YACZ,IAAI;YACJ,YAAY,EAAE,KAAK;gBAClB,CAAC,CAAC,mBAAmB;gBACrB,CAAC,CAAC,oBAAoB;oBACrB,CAAC,CAAC,yBAAyB;oBAC3B,CAAC,CAAC,iBAAiB;wBAClB,CAAC,CAAC,gCAAgC;wBAClC,CAAC,CAAC,IAAI;SACT,CAAA;IACF,CAAC;IAED,+EAA+E;IAC/E,KAAK,UAAU,YAAY,CAC1B,KAIC;QAED,MAAM,EAAE,QAAQ,EAAE,IAAI,EAAE,GAAG,KAAK,CAAA;QAChC,MAAM,MAAM,GAAG,QAAQ,CAAC,MAAM,CAAA;QAC9B,MAAM,WAAW,GAAG,QAAQ,CAAC,OAAO,CAAC,GAAG,CAAC,cAAc,CAAC,CAAA;QAExD,IAAI,MAAM,KAAK,GAAG;YAAE,OAAO,EAAE,OAAO,EAAE,WAAW,EAAE,MAAM,EAAE,cAAc,EAAE,CAAA;QAC3E,IAAI,OAAO,CAAC,CAAC,GAAG,EAAE,GAAG,CAAC,EAAE,MAAM,CAAC,EAAE,CAAC;YACjC,OAAO,EAAE,OAAO,EAAE,MAAM,EAAE,MAAM,EAAE,IAAI,EAAE,CAAA;QACzC,CAAC;QACD,yEAAyE;QACzE,sCAAsC;QACtC,IAAI,MAAM,IAAI,GAAG,IAAI,MAAM,KAAK,GAAG;YAClC,OAAO,EAAE,OAAO,EAAE,WAAW,EAAE,MAAM,EAAE,IAAI,EAAE,CAAA;QAC9C,IAAI,MAAM,IAAI,GAAG,EAAE,CAAC;YACnB,IAAI,CAAC,KAAK,CAAC,MAAM,IAAI,CAAC,OAAO,CAAC,iBAAiB,EAAE,MAAM,CAAC,EAAE,CAAC;gBAC1D,OAAO,EAAE,OAAO,EAAE,SAAS,EAAE,MAAM,EAAE,IAAI,EAAE,CAAA;YAC5C,CAAC;YACD,IAAI,QAAuB,CAAA;YAC3B,IAAI,CAAC;gBACJ,QAAQ,GAAG,MAAM,MAAM,CAAC,mBAAmB,CAC1C,KAAK,CAAC,QAAQ,CAAC,IAAI,EACnB,KAAK,CAAC,MAAM,CACZ,CAAA;YACF,CAAC;YAAC,OAAO,KAAK,EAAE,CAAC;gBAChB,IAAI,CAAC,CAAC,KAAK,YAAY,0BAA0B,CAAC;oBAAE,MAAM,KAAK,CAAA;gBAC/D,2DAA2D;gBAC3D,OAAO;oBACN,OAAO,EAAE,WAAW;oBACpB,MAAM,EAAE,IAAI;oBACZ,YAAY,EAAE,gCAAgC;iBAC9C,CAAA;YACF,CAAC;YACD,OAAO,QAAQ;gBACd,CAAC,CAAC,EAAE,OAAO,EAAE,IAAI,EAAE,MAAM,EAAE,mBAAmB,EAAE,QAAQ,EAAE,QAAQ,EAAE;gBACpE,CAAC,CAAC,EAAE,OAAO,EAAE,SAAS,EAAE,MAAM,EAAE,IAAI,EAAE,CAAA;QACxC,CAAC;QACD,IAAI,MAAM,GAAG,GAAG,IAAI,MAAM,IAAI,GAAG,EAAE,CAAC;YACnC,OAAO,EAAE,OAAO,EAAE,WAAW,EAAE,MAAM,EAAE,IAAI,EAAE,CAAA;QAC9C,CAAC;QAED,MAAM,IAAI,GAAG,mBAAmB,CAAC,WAAW,CAAC,CAAA;QAC7C,OAAO,WAAW,CAAC,IAAI,EAAE;YACxB,KAAK,EAAE,GAAG,EAAE,CAAC,CAAC,EAAE,OAAO,EAAE,aAAa,EAAE,MAAM,EAAE,IAAI,EAAE,CAAC;YACvD,QAAQ,EAAE,GAAG,EAAE,CACd,eAAe,CAAC;gBACf,IAAI;gBACJ,WAAW;gBACX,QAAQ,EAAE,KAAK,CAAC,QAAQ;gBACxB,MAAM,EAAE,KAAK,CAAC,MAAM;aACpB,CAAC;YACH,GAAG,gBAAgB,CAClB,IAAI,EACJ,aAAa,EACb,KAAK,EAAE,WAAW,EAAqB,EAAE,CACxC,IAAI,CAAC,IAAI,KAAK,MAAM;gBACnB,CAAC,CAAC,cAAc,CAAC;oBACf,IAAI,EAAE,IAAI,CAAC,IAAI;oBACf,IAAI,EAAE,WAAW;oBACjB,QAAQ,EAAE,KAAK,CAAC,QAAQ;oBACxB,MAAM,EAAE,KAAK,CAAC,MAAM;oBACpB,QAAQ,EAAE,KAAK,CAAC,QAAQ;oBACxB,MAAM,EAAE,KAAK,CAAC,MAAM;iBACpB,CAAC;gBACH,CAAC,CAAC;oBACA,OAAO,EAAE,aAAa;oBACtB,MAAM,EAAE,IAAI;oBACZ,YAAY,EAAE,IAAI,CAAC,IAAI,KAAK,MAAM,CAAC,CAAC,CAAC,IAAI,CAAC,YAAY,CAAC,CAAC,CAAC,IAAI;iBAC7D,CACJ;SACD,CAAC,CAAA;IACH,CAAC;IAED;;;;;;;;;;;OAWG;IACH,KAAK,UAAU,mBAAmB,CACjC,KAAqB;QAErB,MAAM,SAAS,GAAG,UAAU,EAAE,CAAA;QAC9B,IAAI,CAAC;YACJ,OAAO,MAAM,QAAQ,CAAC,KAAK,CAAC,CAAA;QAC7B,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YAChB,MAAM,CAAC,IAAI,CAAC,wBAAwB,EAAE;gBACrC,GAAG,EAAE,KAAK,CAAC,GAAG;gBACd,GAAG,kBAAkB,CAAC,KAAK,CAAC;aAC5B,CAAC,CAAA;YACF,OAAO,SAAS,CACf;gBACC,GAAG,EAAE,KAAK,CAAC,GAAG;gBACd,QAAQ,EAAE,KAAK,CAAC,GAAG;gBACnB,MAAM,EAAE,CAAC;gBACT,WAAW,EAAE,IAAI;gBACjB,IAAI,EAAE,IAAI;gBACV,YAAY,EAAE,IAAI;gBAClB,SAAS;aACT,EACD,EAAE,OAAO,EAAE,WAAW,EAAE,MAAM,EAAE,IAAI,EAAE,EACtC,MAAM,CACN,CAAA;QACF,CAAC;IACF,CAAC;IAED,KAAK,UAAU,QAAQ,CAAC,KAAqB;QAC5C,IAAI,CAAC,oBAAoB,EAAE;YAAE,OAAO,aAAa,CAAC,KAAK,CAAC,CAAA;QAExD,4EAA4E;QAC5E,sEAAsE;QACtE,MAAM,IAAI,GAAG,MAAM,oBAAoB,CACtC,QAAQ,KAAK,CAAC,GAAG,IAAI,KAAK,CAAC,OAAO,IAAI,EAAE,IAAI,KAAK,CAAC,MAAM,IAAI,IAAI,EAAE,EAClE,GAAG,EAAE,CACJ,aAAa,CAAC;YACb,GAAG,EAAE,KAAK,CAAC,GAAG;YACd,OAAO,EAAE,KAAK,CAAC,OAAO;YACtB,MAAM,EAAE,KAAK,CAAC,MAAM;YACpB,MAAM,EAAE,KAAK,CAAC,MAAM;SACpB,CAAC,CACH,CAAA;QACD,OAAO,cAAc,CAAC,IAAI,EAAE,KAAK,CAAC,CAAC,CAAC,CAAC,WAAW,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,IAAI,CAAA;IAC9D,CAAC;IAED,KAAK,UAAU,aAAa,CAAC,KAAqB;QACjD,MAAM,MAAM,GAAG,KAAK,CAAC,MAAM,IAAI,sBAAsB,EAAE,CAAA;QACvD,MAAM,SAAS,GAAG,UAAU,EAAE,CAAA;QAC9B,MAAM,MAAM,GAAG,KAAK,CAAC,MAAM,IAAI,IAAI,CAAA;QAEnC,MAAM,QAAQ,GAAG,iBAAiB,CAAC,KAAK,CAAC,GAAG,EAAE,cAAc,CAAC,CAAA;QAC7D,IAAI,CAAC,QAAQ,EAAE,CAAC;YACf,OAAO,SAAS,CACf;gBACC,GAAG,EAAE,KAAK,CAAC,GAAG;gBACd,QAAQ,EAAE,KAAK,CAAC,GAAG;gBACnB,MAAM,EAAE,CAAC;gBACT,WAAW,EAAE,IAAI;gBACjB,IAAI,EAAE,IAAI;gBACV,YAAY,EAAE,IAAI;gBAClB,SAAS;aACT,EACD,EAAE,OAAO,EAAE,aAAa,EAAE,MAAM,EAAE,IAAI,EAAE,EACxC,MAAM,CACN,CAAA;QACF,CAAC;QAED,MAAM,OAAO,GAA2B;YACvC,MAAM,EAAE,2CAA2C;YACnD,iBAAiB,EAAE,GAAG;YACtB,YAAY,EAAE,kBAAkB;SAChC,CAAA;QACD,IAAI,KAAK,CAAC,IAAI;YAAE,OAAO,CAAC,eAAe,CAAC,GAAG,KAAK,CAAC,IAAI,CAAA;QACrD,IAAI,KAAK,CAAC,YAAY;YAAE,OAAO,CAAC,mBAAmB,CAAC,GAAG,KAAK,CAAC,YAAY,CAAA;QAEzE,MAAM,OAAO,GAAG,CAAC,GAAQ,EAAE,EAAE,CAC5B,aAAa,CAAC,QAAQ,CAAC,GAAG,EAAE,KAAK,CAAC,OAAO,CAAC,EAAE,OAAO,EAAE,MAAM,CAAC,CAAA;QAE7D,IAAI,UAAU,GAAG,QAAQ,CAAA;QACzB,IAAI,OAAO,GAAkB,MAAM,OAAO,CAAC,UAAU,CAAC,CAAA;QAEtD,IAAI,OAAO,CAAC,IAAI,KAAK,QAAQ,IAAI,OAAO,CAAC,SAAS,EAAE,CAAC;YACpD,MAAM,QAAQ,GAAG,cAAc,CAAC,UAAU,CAAC,CAAA;YAC3C,IAAI,QAAQ,EAAE,CAAC;gBACd,MAAM,KAAK,GAAG,MAAM,OAAO,CAAC,QAAQ,CAAC,CAAA;gBACrC,IAAI,KAAK,CAAC,IAAI,KAAK,MAAM,EAAE,CAAC;oBAC3B,UAAU,GAAG,QAAQ,CAAA;oBACrB,OAAO,GAAG,KAAK,CAAA;gBAChB,CAAC;YACF,CAAC;QACF,CAAC;QAED,KACC,IAAI,KAAK,GAAG,CAAC,EACb,KAAK,IAAI,sBAAsB;YAC/B,OAAO,CAAC,IAAI,KAAK,MAAM;YACvB,OAAO,CAAC,IAAI,CAAC,QAAQ,CAAC,MAAM,KAAK,GAAG,EACpC,KAAK,EAAE,EACN,CAAC;YACF,MAAM,MAAM,GAAG,eAAe,CAAC,OAAO,CAAC,IAAI,CAAC,QAAQ,EAAE,KAAK,CAAC,CAAA;YAC5D,IAAI,MAAM,UAAU,CAAC,YAAY,CAAC,UAAU,CAAC,QAAQ,EAAE,MAAM,CAAC,EAAE,CAAC;gBAChE,MAAM,CAAC,IAAI,CAAC,mCAAmC,EAAE;oBAChD,IAAI,EAAE,UAAU,CAAC,QAAQ;oBACzB,WAAW,EAAE,OAAO,CAAC,IAAI,CAAC,QAAQ,CAAC,OAAO,CAAC,GAAG,CAAC,aAAa,CAAC,IAAI,EAAE;oBACnE,OAAO,EAAE,MAAM;iBACf,CAAC,CAAA;YACH,CAAC;YACD,OAAO,GAAG,MAAM,OAAO,CAAC,UAAU,CAAC,CAAA;QACpC,CAAC;QAED,IAAI,OAAO,CAAC,IAAI,KAAK,MAAM,EAAE,CAAC;YAC7B,OAAO,SAAS,CACf;gBACC,GAAG,EAAE,UAAU,CAAC,IAAI;gBACpB,QAAQ,EAAE,UAAU,CAAC,IAAI;gBACzB,MAAM,EAAE,CAAC;gBACT,WAAW,EAAE,IAAI;gBACjB,IAAI,EAAE,IAAI;gBACV,YAAY,EAAE,IAAI;gBAClB,SAAS;aACT,EACD;gBACC,OAAO,EAAE,WAAW,CAAC,OAAO,EAAE,MAAM,EAAE;oBACrC,qEAAqE;oBACrE,OAAO,EAAE,GAAG,EAAE,CAAC,SAAS;oBACxB,MAAM,EAAE,GAAG,EAAE,CAAC,WAAW;iBACzB,CAAC;gBACF,MAAM,EAAE,IAAI;aACZ,EACD,MAAM,CACN,CAAA;QACF,CAAC;QAED,MAAM,EAAE,QAAQ,EAAE,QAAQ,EAAE,GAAG,OAAO,CAAC,IAAI,CAAA;QAC3C,MAAM,WAAW,GAAG,CAAC,MAAwB,EAAE,EAAE,CAChD,QAAQ,CAAC,MAAM,KAAK,GAAG,CAAC,CAAC,CAAC,CAAC,MAAM,IAAI,IAAI,CAAC,CAAC,CAAC,CAAC,IAAI,CAAA;QAClD,MAAM,IAAI,GAAG,MAAM,YAAY,CAAC;YAC/B,GAAG,OAAO,CAAC,IAAI;YACf,MAAM;YACN,QAAQ,EAAE,KAAK,CAAC,QAAQ;YACxB,MAAM;SACN,CAAC,CAAA;QAEF,MAAM,IAAI,GAAG,SAAS,CACrB;YACC,GAAG,EAAE,UAAU,CAAC,IAAI;YACpB,QAAQ,EAAE,QAAQ,CAAC,IAAI;YACvB,MAAM,EAAE,QAAQ,CAAC,MAAM;YACvB,WAAW,EAAE,QAAQ,CAAC,OAAO,CAAC,GAAG,CAAC,cAAc,CAAC;YACjD,uEAAuE;YACvE,wEAAwE;YACxE,2DAA2D;YAC3D,IAAI,EAAE,QAAQ,CAAC,OAAO,CAAC,GAAG,CAAC,MAAM,CAAC,IAAI,WAAW,CAAC,KAAK,CAAC,IAAI,CAAC;YAC7D,YAAY,EACX,QAAQ,CAAC,OAAO,CAAC,GAAG,CAAC,eAAe,CAAC;gBACrC,WAAW,CAAC,KAAK,CAAC,YAAY,CAAC;YAChC,SAAS;SACT,EACD,IAAI,EACJ,MAAM,CACN,CAAA;QAED,MAAM,CAAC,IAAI,CAAC,2BAA2B,EAAE;YACxC,GAAG,EAAE,IAAI,CAAC,QAAQ;YAClB,MAAM,EAAE,IAAI,CAAC,MAAM;YACnB,OAAO,EAAE,IAAI,CAAC,OAAO;YACrB,MAAM,EAAE,IAAI,CAAC,MAAM,IAAI,EAAE;YACzB,KAAK,EAAE,IAAI,CAAC,QAAQ,CAAC,MAAM;YAC3B,WAAW,EAAE,IAAI,CAAC,UAAU;SAC5B,CAAC,CAAA;QACF,OAAO,IAAI,CAAA;IACZ,CAAC;IAED,gDAAgD;IAChD,KAAK,UAAU,aAAa,CAAC,SAAiB;QAC7C,MAAM,IAAI,GAAG,MAAM,mBAAmB,CAAC,EAAE,GAAG,EAAE,SAAS,EAAE,CAAC,CAAA;QAC1D,MAAM,EAAE,QAAQ,EAAE,MAAM,EAAE,GAAG,IAAI,CAAA;QACjC,IAAI,IAAI,CAAC,OAAO,KAAK,IAAI,IAAI,CAAC,QAAQ,IAAI,CAAC,MAAM,EAAE,CAAC;YACnD,MAAM,IAAI,KAAK,CACd,sCAAsC,IAAI,CAAC,OAAO,YAAY,IAAI,CAAC,MAAM,MAAM,SAAS,EAAE,CAC1F,CAAA;QACF,CAAC;QACD,OAAO,EAAE,QAAQ,EAAE,MAAM,EAAE,CAAA;IAC5B,CAAC;IAED,OAAO;QACN,mBAAmB;QACnB,oBAAoB;QACpB,aAAa;QACb,wBAAwB,EAAE,MAAM,CAAC,wBAAwB;QACzD,mBAAmB,EAAE,MAAM,CAAC,mBAAmB;QAC/C,gEAAgE;QAChE,aAAa,EAAE,SAAS;QACxB;;;;WAIG;QACH,YAAY,CACX,KAAmB,EACnB,IAAkB,EAClB,YAAwD;YAExD,OAAO,YAAY,CAAC,KAAK,EAAE,IAAI,EAAE,EAAE,GAAG,KAAK,EAAE,GAAG,YAAY,EAAE,CAAC,CAAA;QAChE,CAAC;QACD,+DAA+D;QAC/D,iBAAiB,CAChB,KAAmC,EACnC,GAAqB;YAErB,IAAI,CAAC,KAAK;gBAAE,OAAO,GAAG,EAAE,CAAA;YACxB,SAAS,CACR,OAAO,CAAC,aAAa,EACrB,+CAA+C,CAC/C,CAAA;YACD,OAAO,iBAAiB,CAAC,KAAK,EAAE,GAAG,EAAE;gBACpC,IAAI,EAAE,OAAO,CAAC,aAAa,CAAC,IAAI;gBAChC,MAAM,EAAE,OAAO,CAAC,aAAa,CAAC,MAAM,IAAI,MAAM;aAC9C,CAAC,CAAA;QACH,CAAC;KACD,CAAA;AACF,CAAC","sourcesContent":["import {\n\ttype ElementOf,\n\texponentialBackoff,\n\tfileExtension,\n\tgetUniqueValues,\n\tinvariant,\n\tisOneOf,\n\tisRecord,\n\tmust,\n\ttype Nullable,\n\tparseRetryAfter,\n\tstartTimer,\n\ttimeoutSignal,\n\ttypedSwitch,\n\ttypedSwitchGroup\n} from 'hanzio'\nimport type { ApiError } from 'hanzio/api-wrapper'\nimport {\n\ttype CloudflareCredentials,\n\tCloudflareRateLimitedError,\n\tcreateCloudflareClient\n} from './client.js'\nimport { classifyContentType, convertibleDocumentType } from './content-type.js'\nimport {\n\ttype CrawlSnapshotInput,\n\ttype CrawlSnapshotOptions,\n\tcurrentCrawlSnapshot,\n\tthroughCrawlSnapshot,\n\twithCrawlSnapshot\n} from './crawl-snapshot.js'\nimport {\n\tisApplicationShell,\n\tisBlankMarkdown,\n\tlooksLikeHtml\n} from './detect-markdown.js'\nimport { sha256Hex } from './hash.js'\nimport { extractHtmlMetadata } from './html-metadata.js'\nimport { type CreateConcurrencyLimit, createHostLimits } from './limit.js'\nimport { errorLogAttributes, type Logger, silentLogger } from './logger.js'\nimport {\n\tcleanConvertedMarkdown,\n\thtmlImages,\n\thtmlLinks,\n\tmarkdownRefs,\n\tstripDocumentPreamble,\n\tstripFrontMatter,\n\ttitleFromMarkdown\n} from './markdown-refs.js'\nimport { createMemoryConcurrencyLimit } from './memory-limit.js'\nimport {\n\ttype Fetch,\n\tfollowRedirects,\n\ttype GuardedFetchOptions,\n\tguardedFetch,\n\tHostNotAllowedError,\n\tisHostAllowed,\n\tisLocalhostHostname\n} from './net-guard.js'\nimport { currentOperationSignal } from './operation-signal.js'\nimport { decodeText, readBody } from './read-body.js'\nimport type { CaptureIssue, FetchOutcome, FetchSource } from './schema/index.js'\nimport { asUnchanged, isUnchangedFor } from './unchanged-page.js'\nimport { BROWSER_USER_AGENT } from './user-agent.js'\nimport { preserveXlsxZeros } from './xlsx-zeros.js'\n\nexport type { CaptureIssue, FetchOutcome, FetchSource }\n\nconst FETCH_TIMEOUT_MS = 30_000\n/** Browser Rendering once returned a 6 MB markdown page from a 6 MB upload. */\nconst MAX_HTML_BYTES = 2 * 1024 * 1024\n/**\n * Read cap for a textual body. Above `MAX_HTML_BYTES` a page is not converted\n * but still yields its metadata and links; above this it is not downloaded.\n */\nconst MAX_TEXT_READ_BYTES = 2 * MAX_HTML_BYTES\nconst MAX_DOCUMENT_BYTES = 2 * 1024 * 1024\nconst MAX_PDF_BYTES = 12 * 1024 * 1024\nconst MAX_PDF_PAGES = 60\n\n/** Stored markdown ceiling; a page over this is a listing or an upload, not knowledge. */\nexport const MAX_MARKDOWN_CHARS = 300_000\n\nexport type FetchedPage = {\n\t/** The URL we requested, after http->https upgrade and www fallback. */\n\turl: string\n\t/** After redirects; may differ in host or path. */\n\tfinalUrl: string\n\t/** HTTP status of the origin response, or 0 when the request never completed. */\n\tstatus: number\n\toutcome: FetchOutcome\n\tsource: FetchSource | null\n\tcontentType: string | null\n\tetag: string | null\n\tlastModified: string | null\n\t/** Empty string unless outcome === 'ok'. Front-matter stripped, capped. */\n\tmarkdown: string\n\t/** <title> when an HTML tier read the page, else the first markdown H1. */\n\ttitle: string | null\n\t/** ISO 639-1 from <html lang>, lowercased, else null. */\n\tlang: string | null\n\t/** <link rel=\"canonical\">, absolute. */\n\tcanonical: string | null\n\t/** `<link rel=\"alternate\" hreflang>` targets, absolute, `x-default` excluded. */\n\talternates: string[]\n\t/** `alternates` by language: URL -> ISO 639-1 (`da` of `da-DK`). */\n\talternateLanguages: Record<string, string>\n\t/** <meta name=\"description\"> or the og: equivalent. */\n\tmetaDescription: string | null\n\t/** Parsed application/ld+json blocks, any shape. */\n\tjsonLd: unknown[]\n\t/**\n\t * Absolute, deduped link targets: the markdown's, unioned with every\n\t * `<a href>` when an HTML tier read the page. Conversion loses navigation,\n\t * and a link lost here is a page discovery can never reach.\n\t */\n\tlinks: string[]\n\t/** Absolute, deduped image URLs found in the markdown. */\n\timages: string[]\n\tdurationMs: number\n\t/** Why the page is incomplete or missing, when the library knows. */\n\tcaptureIssue: CaptureIssue | null\n\t/**\n\t * Hash of the HTML the origin served, when an origin tier read HTML. Given\n\t * back on the next read (`fetchPageAsMarkdown({ htmlHash })`), identical HTML\n\t * is answered as `unchanged` without converting it again.\n\t */\n\thtmlHash: string | null\n}\n\ntype PageBase = {\n\turl: string\n\tfinalUrl: string\n\tstatus: number\n\tcontentType: string | null\n\tetag: string | null\n\tlastModified: string | null\n\telapsedMs: () => number\n}\n\nexport type PageBody = {\n\tcaptureIssue?: Nullable<CaptureIssue>\n\toutcome: FetchOutcome\n\tsource: FetchSource | null\n\tmarkdown?: Nullable<string>\n\t/** The HTML an origin tier read, when one did - the only source of metadata. */\n\thtml?: Nullable<string>\n}\n\n/**\n * A NUL byte is never text, and Postgres refuses a string that holds one: a\n * single page with one (a Wix site's conversion) failed a whole import.\n * Composed Unicode, because string comparisons downstream assume it: a site\n * that wrote \"Gård\" as \"a\" plus a combining ring read as \"Ga\" to anything\n * matching names in the text.\n */\nfunction cleanText(text: string) {\n\treturn text.replaceAll('\\u0000', '').normalize('NFC')\n}\n\n/**\n * The one place markdown, front-matter and HTML metadata are reconciled: the\n * HTML `<title>` wins, the Workers AI front-matter is the fallback for pages we\n * only ever saw as markdown, and links/images are resolved against `finalUrl`.\n */\nfunction buildPage(\n\tbase: PageBase,\n\trawBody: PageBody,\n\tlogger: Logger\n): FetchedPage {\n\tconst body = {\n\t\t...rawBody,\n\t\tmarkdown: rawBody.markdown ? cleanText(rawBody.markdown) : null,\n\t\thtml: rawBody.html ? cleanText(rawBody.html) : null\n\t}\n\tconst frontMatter = stripFrontMatter(body.markdown ?? '')\n\tlet markdown = cleanConvertedMarkdown(frontMatter.markdown)\n\tif (markdown.length > MAX_MARKDOWN_CHARS) {\n\t\tlogger.warn('url-to-markdown.truncated', {\n\t\t\turl: base.finalUrl,\n\t\t\tchars: markdown.length\n\t\t})\n\t\tmarkdown = markdown.slice(0, MAX_MARKDOWN_CHARS)\n\t}\n\n\tconst metadata = body.html\n\t\t? extractHtmlMetadata(body.html, base.finalUrl)\n\t\t: null\n\tconst refs = markdownRefs(markdown, base.finalUrl)\n\t// The markdown is the document; the HTML is the map. Conversion drops\n\t// navigation (image-wrapped anchors, folded menus), so when we hold the\n\t// HTML its anchors decide what discovery can still reach.\n\tconst links = body.html\n\t\t? getUniqueValues([...refs.links, ...htmlLinks(body.html, base.finalUrl)])\n\t\t: refs.links\n\n\treturn {\n\t\turl: base.url,\n\t\tfinalUrl: base.finalUrl,\n\t\tstatus: base.status,\n\t\toutcome: body.outcome,\n\t\tsource: body.source,\n\t\tcontentType: base.contentType,\n\t\tetag: base.etag,\n\t\tlastModified: base.lastModified,\n\t\tmarkdown,\n\t\ttitle: metadata?.title ?? frontMatter.title ?? titleFromMarkdown(markdown),\n\t\tlang: metadata?.lang ?? null,\n\t\tcanonical: metadata?.canonical ?? null,\n\t\talternates: metadata?.alternates ?? [],\n\t\talternateLanguages: metadata?.alternateLanguages ?? {},\n\t\tmetaDescription:\n\t\t\tmetadata?.metaDescription ?? frontMatter.description ?? null,\n\t\tjsonLd: metadata?.jsonLd ?? [],\n\t\tlinks,\n\t\timages: getUniqueValues([\n\t\t\t...refs.images,\n\t\t\t...(body.html ? htmlImages(body.html, base.finalUrl) : [])\n\t\t]),\n\t\tcaptureIssue:\n\t\t\tfrontMatter.markdown.length > MAX_MARKDOWN_CHARS\n\t\t\t\t? 'markdown_limit'\n\t\t\t\t: (body.captureIssue ?? null),\n\t\tdurationMs: Math.round(base.elapsedMs()),\n\t\t// Over the cleaned HTML, as `convertTextual` compares it.\n\t\thtmlHash: body.html ? htmlHash(body.html) : null\n\t}\n}\n\n/**\n * `http://` is upgraded rather than rejected: two carpenter sites in the probe\n * link internally with `http://` and the https-only guard threw away most of\n * their pages. Localhost http stays http, and is reachable only with\n * `allowLocalhost`.\n */\nfunction prepareRequestUrl(raw: string, allowLocalhost: boolean) {\n\tlet parsed: URL\n\ttry {\n\t\tparsed = new URL(raw)\n\t} catch {\n\t\treturn null\n\t}\n\tif (parsed.protocol === 'https:') return parsed\n\tif (parsed.protocol !== 'http:') return null\n\tif (isLocalhostHostname(parsed.hostname)) {\n\t\treturn allowLocalhost ? parsed : null\n\t}\n\tparsed.protocol = 'https:'\n\tif (parsed.port === '80') parsed.port = ''\n\treturn parsed\n}\n\n/**\n * `apoteket.dk` answers no TLS handshake at the apex while `www.apoteket.dk`\n * serves the whole site, so a connection failure gets one www retry.\n */\nfunction wwwFallbackUrl(url: URL) {\n\tconst host = url.hostname\n\tif (host.startsWith('www.') || !host.includes('.')) return null\n\tif (isLocalhostHostname(host) || /^[\\d.]+$/.test(host)) return null\n\tconst next = new URL(url.href)\n\tnext.hostname = `www.${host}`\n\treturn next\n}\n\n/**\n * A 429 is an origin asking us to slow down, not a bot wall. Shopify answered\n * 117 of one store's 244 reads with it; sent on to Browser Rendering they\n * tripped Cloudflare's own limit (79 times, 25-30 s each) and 7 came back.\n * So the host is narrowed and paused (`coolDownHost`), the URL is asked again,\n * and a page still refused after that is `transient`: its stored copy stands\n * and the next run reads it.\n */\nconst MAX_RATE_LIMIT_RETRIES = 3\nconst MAX_RATE_LIMIT_WAIT_MS = 30_000\n\n/** The origin's `Retry-After` when it sent one, else 1 s, 2 s, 4 s. */\nfunction rateLimitWaitMs(response: Response, retry: number) {\n\tconst askedMs = parseRetryAfter(response.headers.get('retry-after'))\n\treturn Math.min(\n\t\tMAX_RATE_LIMIT_WAIT_MS,\n\t\taskedMs ??\n\t\t\texponentialBackoff(retry, {\n\t\t\t\tbaseMs: 1_000,\n\t\t\t\tmaxMs: MAX_RATE_LIMIT_WAIT_MS,\n\t\t\t\tjitter: false\n\t\t\t})\n\t)\n}\n\n/**\n * Bump whenever conversion changes what the same HTML becomes (`withoutHiddenUi`,\n * the Workers AI call, the fallback tiers): the stored hash then no longer\n * matches, and every page is converted again once instead of keeping the\n * markdown the old conversion made.\n */\nconst CONVERSION_VERSION = 2\n\nfunction htmlHash(html: string) {\n\treturn sha256Hex(`${CONVERSION_VERSION}\\n${html}`)\n}\n\n/** The request URL with the market pinned; the URL itself when there is none. */\nfunction inMarket(url: URL, country: Nullable<string>) {\n\tif (!country || url.searchParams.has('country')) return url\n\tconst pinned = new URL(url.href)\n\tpinned.searchParams.set('country', country)\n\treturn pinned\n}\n\n/** `Response.url` is empty on a synthesised response and after some proxies. */\nfunction parseFinalUrl(responseUrl: string, requestUrl: URL) {\n\tif (!responseUrl) return requestUrl\n\ttry {\n\t\treturn new URL(responseUrl)\n\t} catch {\n\t\treturn requestUrl\n\t}\n}\n\nfunction isAbortError(error: unknown) {\n\treturn isRecord(error) && isOneOf(['AbortError', 'TimeoutError'], error.name)\n}\n\n/** The content kinds whose body is read as text; the rest are documents or assets. */\nconst TEXTUAL_KINDS = ['markdown', 'html', 'text'] as const\ntype TextualKind = ElementOf<typeof TEXTUAL_KINDS>\n\n/** What the origin gave us, already downloaded under the host's politeness slot. */\ntype OriginBody =\n\t| { kind: 'none'; captureIssue?: CaptureIssue }\n\t| { kind: 'text'; text: string }\n\t| { kind: 'bytes'; bytes: ArrayBuffer }\n\ntype OriginRead = { response: Response; finalUrl: URL; body: OriginBody }\n\ntype OriginAttempt =\n\t| { kind: 'read'; read: OriginRead }\n\t| { kind: 'refused' }\n\t| { kind: 'failed'; retryable: boolean }\n\n/** Reading the body is optional; an unsupported type is dropped at the headers. */\nfunction discardBody(response: Response) {\n\tvoid response.body?.cancel().catch(() => {})\n}\n\n/**\n * Decides from the headers alone whether the body is worth downloading, and\n * downloads it when it is. Everything downstream works from bytes already in\n * hand, so nothing past this point needs the origin's slot.\n */\nasync function readOriginBody(\n\tresponse: Response,\n\tfinalUrl: URL,\n\tlogger: Logger\n): Promise<OriginBody> {\n\tconst status = response.status\n\tif (status < 200 || status >= 300) {\n\t\tdiscardBody(response)\n\t\treturn { kind: 'none' }\n\t}\n\n\tconst contentType = response.headers.get('content-type')\n\tconst kind = classifyContentType(contentType)\n\treturn typedSwitch(kind, {\n\t\tasset: () => {\n\t\t\tdiscardBody(response)\n\t\t\treturn { kind: 'none' }\n\t\t},\n\t\tdocument: async () => {\n\t\t\tconst type = convertibleDocumentType(contentType)\n\t\t\tif (!type) {\n\t\t\t\tdiscardBody(response)\n\t\t\t\treturn { kind: 'none' }\n\t\t\t}\n\t\t\tconst limit =\n\t\t\t\ttype.extension === 'pdf' ? MAX_PDF_BYTES : MAX_DOCUMENT_BYTES\n\t\t\tconst declared = Number(response.headers.get('content-length'))\n\t\t\tif (Number.isFinite(declared) && declared > limit) {\n\t\t\t\tdiscardBody(response)\n\t\t\t\tlogger.info('url-to-markdown.document_oversized', {\n\t\t\t\t\turl: finalUrl.href,\n\t\t\t\t\tbytes: declared\n\t\t\t\t})\n\t\t\t\treturn { kind: 'none', captureIssue: 'document_byte_limit' }\n\t\t\t}\n\t\t\tconst bytes = await readBoundedBody(response, limit)\n\t\t\treturn bytes\n\t\t\t\t? { kind: 'bytes', bytes }\n\t\t\t\t: { kind: 'none', captureIssue: 'document_byte_limit' }\n\t\t},\n\t\t...typedSwitchGroup(kind, TEXTUAL_KINDS, async (): Promise<OriginBody> => {\n\t\t\tconst declared = Number(response.headers.get('content-length'))\n\t\t\tconst read =\n\t\t\t\tNumber.isFinite(declared) && declared > MAX_TEXT_READ_BYTES\n\t\t\t\t\t? null\n\t\t\t\t\t: await readBody(response, MAX_TEXT_READ_BYTES)\n\t\t\tif (!read || read.truncated) {\n\t\t\t\tif (!read) discardBody(response)\n\t\t\t\tlogger.info('url-to-markdown.html_oversized', {\n\t\t\t\t\turl: finalUrl.href,\n\t\t\t\t\tbytes: read ? MAX_TEXT_READ_BYTES : declared\n\t\t\t\t})\n\t\t\t\treturn { kind: 'none', captureIssue: 'html_byte_limit' }\n\t\t\t}\n\t\t\treturn { kind: 'text', text: decodeText(read.bytes) }\n\t\t})\n\t})\n}\n\nfunction documentFilename(finalUrl: string, extension: string) {\n\tconst segment = finalUrl.split(/[?#]/)[0]?.split('/').pop() || 'document'\n\tconst name = segment.slice(0, 80)\n\treturn fileExtension(name) === extension ? name : `${name}.${extension}`\n}\n\nasync function readBoundedBody(response: Response, maxBytes: number) {\n\tconst read = await readBody(response, maxBytes)\n\treturn read.truncated ? null : read.bytes.buffer\n}\n\n/**\n * Markup a visitor never sees until they open it: `<template>` content and\n * closed `<dialog>`s. A Shopify theme ships its cart drawer (\"Your cart is\n * empty\") and search overlay (a product list with prices) in every page this\n * way; converted, they reached 70 of one store's 89 pages. An open dialog is\n * shown and stays. A contact or newsletter form (Shopify posts both to\n * `/contact`) is labels and a button, and \"Telefon Kommentar Send us a note\"\n * read as if `Kommentar Send` were a name on the page.\n *\n * Plus the form fields a visitor cannot see: an anti-spam honeypot inside a\n * `display:none` block renamed its label (\"Navn\" / \"Url\") on every request,\n * so a carpenter's seven pages hashed as changed and five documents were\n * rewritten every night. Only labels go - a collapsed FAQ answer is hidden the\n * same way and must stay.\n */\nconst HIDDEN_UI_SELECTOR = [\n\t'template',\n\t'dialog:not([open])',\n\t'form[action*=\"/contact\"]',\n\t'[style*=\"display:none\"] label',\n\t'[style*=\"display: none\"] label'\n].join(', ')\n\nfunction withoutHiddenUi(html: string) {\n\treturn new HTMLRewriter()\n\t\t.on(HIDDEN_UI_SELECTOR, {\n\t\t\telement(element) {\n\t\t\t\telement.remove()\n\t\t\t}\n\t\t})\n\t\t.transform(html)\n}\n\n/** A real browser passes these; every other 4xx has nothing for it to find. */\nconst BOT_WALL_STATUSES = [401, 403] as const\n\nexport type FetchPageInput = {\n\turl: string\n\t/** Stored validators from the last run; a 304 returns outcome 'unchanged'. */\n\tetag?: Nullable<string>\n\tlastModified?: Nullable<string>\n\t/** Allow the Browser Rendering tier. Default true. */\n\trender?: boolean | undefined\n\t/**\n\t * ISO country of the market to read the page in. A Shopify store redirects\n\t * by the visitor's IP - a worker abroad reads a Danish shop as its EU market\n\t * in euros and English - and `?country=DK` pins it. Applied to the request\n\t * only: the page keeps the URL it was asked for.\n\t */\n\tcountry?: Nullable<string>\n\t/** The last read's `htmlHash`; identical HTML comes back `unchanged`. */\n\thtmlHash?: Nullable<string>\n\tsignal?: AbortSignal | undefined\n}\n\nexport type PdfParser = (input: {\n\tbytes: ArrayBuffer\n\tfilename: string\n\tmimeType: string\n\t/** Pages to read at most; a parse that reaches it is marked incomplete. */\n\tmaxPages: number\n\tsignal?: AbortSignal | undefined\n}) => Promise<{ markdown: string; numPages: number | null }>\n\nexport type WebsiteToMarkdownOptions = {\n\t/** Workers AI and Browser Rendering; read on first use, so getters work. */\n\tcloudflare: CloudflareCredentials\n\t/**\n\t * Builds every pool: the two Cloudflare ones and one per origin host.\n\t * Defaults to an in-memory limiter; pass a shared one (Redis) when several\n\t * processes share the Cloudflare quota.\n\t */\n\tcreateLimit?: CreateConcurrencyLimit\n\t/** Workers AI client errors other than 429s, for an error tracker. */\n\tonApiError?: (error: ApiError) => void\n\t/**\n\t * Pass Workers AI timeouts to `onApiError`. A timed-out page falls through\n\t * to the next tier, so `false` keeps import bursts out of error tracking.\n\t * Default true.\n\t */\n\treportTimeouts?: boolean\n\tlogger?: Logger\n\t/**\n\t * Where the returned `withCrawlSnapshot` keeps snapshots. Its logger\n\t * defaults to `logger`.\n\t */\n\tcrawlSnapshot?: CrawlSnapshotOptions\n\t/** The OCR tier for a PDF Workers AI returned nothing for. */\n\tparsePdf?: PdfParser\n\t/**\n\t * SSRF guard run before every origin request and every redirect it\n\t * follows. Defaults to {@link isHostAllowed}: public addresses only.\n\t */\n\tisHostAllowed?: (hostname: string) => Promise<boolean>\n\t/**\n\t * Let the default guard and `http://` through to `localhost`, `127.0.0.1`\n\t * and `::1`, e.g. outside production for a dev server. Default false.\n\t */\n\tallowLocalhost?: boolean\n\t/** The `fetch` origin requests go through, behind the guard. Default the global. */\n\tfetch?: Fetch\n}\n\n/**\n * One converter: its Cloudflare client, pools and per-host politeness limits.\n * Create it once per process; every call shares the pools.\n */\nexport function createWebsiteToMarkdown(options: WebsiteToMarkdownOptions) {\n\tconst logger = options.logger ?? silentLogger\n\tconst createLimit = options.createLimit ?? createMemoryConcurrencyLimit\n\tconst client = createCloudflareClient({\n\t\tcredentials: options.cloudflare,\n\t\tcreateLimit,\n\t\tonApiError: options.onApiError,\n\t\treportTimeouts: options.reportTimeouts,\n\t\tlogger\n\t})\n\tconst hostLimits = createHostLimits(createLimit)\n\tconst allowLocalhost = options.allowLocalhost === true\n\tconst allowHost =\n\t\toptions.isHostAllowed ??\n\t\t((hostname: string) => isHostAllowed(hostname, { logger, allowLocalhost }))\n\tconst guard = { isHostAllowed: allowHost, fetch: options.fetch }\n\n\tasync function requestOrigin(\n\t\turl: URL,\n\t\theaders: Record<string, string>,\n\t\tsignal?: AbortSignal | undefined\n\t): Promise<OriginAttempt> {\n\t\tif (signal?.aborted) return { kind: 'failed', retryable: false }\n\t\ttry {\n\t\t\t// The politeness slot spans the connection *and* the body: freed at the\n\t\t\t// response headers it would bound connections only, and the whole 80-wide\n\t\t\t// page pool could still be streaming from one small WordPress host at\n\t\t\t// once - the burst the cap exists to prevent, at any cap we would pick.\n\t\t\t// The Cloudflare tiers stay outside it - they are\n\t\t\t// our upstreams with their own pools, and holding a customer's slot\n\t\t\t// across a 60s render would stall the crawl for no politeness gain.\n\t\t\tconst read = await hostLimits.withHostLimit(url.hostname, async () => {\n\t\t\t\tconst hop = await followRedirects(\n\t\t\t\t\turl,\n\t\t\t\t\t{ signal: timeoutSignal(FETCH_TIMEOUT_MS, signal), headers },\n\t\t\t\t\tguard\n\t\t\t\t)\n\t\t\t\tconst response = hop.response\n\t\t\t\tconst finalUrl = parseFinalUrl(response.url, hop.url)\n\t\t\t\treturn {\n\t\t\t\t\tresponse,\n\t\t\t\t\tfinalUrl,\n\t\t\t\t\tbody: await readOriginBody(response, finalUrl, logger)\n\t\t\t\t}\n\t\t\t})\n\t\t\treturn { kind: 'read', read }\n\t\t} catch (error) {\n\t\t\t// The guard refused the URL or a redirect: policy, not a network blip.\n\t\t\tif (error instanceof HostNotAllowedError) return { kind: 'refused' }\n\t\t\tlogger.warn('url-to-markdown.origin_failed', {\n\t\t\t\turl: url.href,\n\t\t\t\t...errorLogAttributes(error)\n\t\t\t})\n\t\t\t// A body that dies mid-stream is the same class of failure as a refused\n\t\t\t// connection, and the same www retry is worth a try.\n\t\t\treturn { kind: 'failed', retryable: !isAbortError(error) }\n\t\t}\n\t}\n\n\t/**\n\t * Bytes of a document to markdown: Workers AI `toMarkdown` first, the\n\t * `parsePdf` tier (e.g. Firecrawl: text-first, OCR fallback) for a PDF Workers\n\t * AI returned nothing for. Shared by the crawler (a linked terms PDF) and file\n\t * uploads (the same PDF dropped in by hand), so both read a file the same\n\t * way. `workersAiMaxBytes` is the caller's budget: the crawler keeps it at\n\t * {@link MAX_DOCUMENT_BYTES}; an upload of a 50 MB scanned catalogue may go\n\t * straight to the OCR tier.\n\t */\n\tasync function convertDocumentBytes(input: {\n\t\tbytes: ArrayBuffer\n\t\tfilename: string\n\t\tmimeType: string\n\t\textension: string\n\t\tworkersAiMaxBytes?: number | undefined\n\t\tsignal?: AbortSignal | undefined\n\t}): Promise<PageBody> {\n\t\tconst { bytes } = input\n\t\tif (!bytes.byteLength) return { outcome: 'unsupported', source: null }\n\t\tif (input.signal?.aborted) return { outcome: 'transient', source: null }\n\t\tlet workersAiRateLimited = false\n\t\tlet markdown: string | null = null\n\t\tif (bytes.byteLength <= (input.workersAiMaxBytes ?? MAX_DOCUMENT_BYTES)) {\n\t\t\ttry {\n\t\t\t\tmarkdown = await client.convertFileWithWorkersAi({\n\t\t\t\t\tbody: input.extension === 'xlsx' ? preserveXlsxZeros(bytes) : bytes,\n\t\t\t\t\tfilename: input.filename,\n\t\t\t\t\tmimeType: input.mimeType,\n\t\t\t\t\tsignal: input.signal\n\t\t\t\t})\n\t\t\t} catch (error) {\n\t\t\t\tif (!(error instanceof CloudflareRateLimitedError)) throw error\n\t\t\t\tworkersAiRateLimited = true\n\t\t\t}\n\t\t}\n\t\tconst body = stripDocumentPreamble(markdown ?? '')\n\t\tif (!isBlankMarkdown(body))\n\t\t\treturn { outcome: 'ok', source: 'workers-ai', markdown: body }\n\t\tif (input.extension !== 'pdf') {\n\t\t\treturn {\n\t\t\t\toutcome: 'transient',\n\t\t\t\tsource: null,\n\t\t\t\tcaptureIssue: workersAiRateLimited ? 'workers_ai_rate_limited' : null\n\t\t\t}\n\t\t}\n\t\tif (input.signal?.aborted) return { outcome: 'transient', source: null }\n\t\ttry {\n\t\t\tconst parsePdf = must(options.parsePdf, 'No PDF parser configured')\n\t\t\tconst parsed = await parsePdf({\n\t\t\t\tbytes,\n\t\t\t\tfilename: input.filename,\n\t\t\t\tmimeType: input.mimeType,\n\t\t\t\tmaxPages: MAX_PDF_PAGES,\n\t\t\t\tsignal: input.signal\n\t\t\t})\n\t\t\treturn {\n\t\t\t\toutcome: 'ok',\n\t\t\t\tsource: 'firecrawl-parse',\n\t\t\t\tmarkdown: parsed.markdown,\n\t\t\t\t// The parser may report parsed pages rather than total pages; reaching the\n\t\t\t\t// limit is therefore explicitly incomplete even when the count equals it.\n\t\t\t\tcaptureIssue:\n\t\t\t\t\tparsed.numPages === null || parsed.numPages >= MAX_PDF_PAGES\n\t\t\t\t\t\t? 'pdf_page_limit_unverified'\n\t\t\t\t\t\t: null\n\t\t\t}\n\t\t} catch (error) {\n\t\t\tlogger.warn('url-to-markdown.pdf_parse_failed', {\n\t\t\t\tfilename: input.filename,\n\t\t\t\t...errorLogAttributes(error)\n\t\t\t})\n\t\t\treturn {\n\t\t\t\toutcome: 'transient',\n\t\t\t\tsource: null,\n\t\t\t\tcaptureIssue: workersAiRateLimited\n\t\t\t\t\t? 'workers_ai_rate_limited'\n\t\t\t\t\t: 'pdf_extraction_failed'\n\t\t\t}\n\t\t}\n\t}\n\n\tasync function convertDocument(input: {\n\t\tbody: OriginBody\n\t\tcontentType: string | null\n\t\tfinalUrl: URL\n\t\tsignal?: AbortSignal | undefined\n\t}): Promise<PageBody> {\n\t\tconst type = convertibleDocumentType(input.contentType)\n\t\tif (!type || input.body.kind !== 'bytes') {\n\t\t\treturn {\n\t\t\t\toutcome: 'unsupported',\n\t\t\t\tsource: null,\n\t\t\t\tcaptureIssue: typedSwitch(input.body, 'kind', {\n\t\t\t\t\tnone: (body) => body.captureIssue,\n\t\t\t\t\ttext: () => null,\n\t\t\t\t\tbytes: () => null\n\t\t\t\t})\n\t\t\t}\n\t\t}\n\t\treturn convertDocumentBytes({\n\t\t\tbytes: input.body.bytes,\n\t\t\tfilename: documentFilename(input.finalUrl.href, type.extension),\n\t\t\tmimeType: type.mimeType,\n\t\t\textension: type.extension,\n\t\t\tsignal: input.signal\n\t\t})\n\t}\n\n\tasync function convertTextual(input: {\n\t\ttext: string\n\t\tkind: TextualKind\n\t\tfinalUrl: URL\n\t\trender: boolean\n\t\t/** The hash of the HTML last read; see `FetchedPage.htmlHash`. */\n\t\thtmlHash?: Nullable<string>\n\t\tsignal?: AbortSignal | undefined\n\t}): Promise<PageBody> {\n\t\tconst body = input.text\n\t\t// The `Content-Type` header is authoritative - sniffing for a leading\n\t\t// `<html` misses every page that opens with a comment or a CDN banner - but\n\t\t// a `text/plain` body that is plainly HTML still needs the HTML tier.\n\t\tconst isHtml = typedSwitch(input.kind, {\n\t\t\thtml: () => true,\n\t\t\tmarkdown: () => false,\n\t\t\ttext: () => looksLikeHtml(body)\n\t\t})\n\n\t\tif (!isHtml && !isBlankMarkdown(body)) {\n\t\t\treturn { outcome: 'ok', source: 'content-negotiation', markdown: body }\n\t\t}\n\n\t\tconst html = isHtml ? body : null\n\t\t// Most origins send no validators, so without this every page of a quiet\n\t\t// site went through Workers AI again every night to learn nothing moved.\n\t\tif (\n\t\t\thtml &&\n\t\t\tinput.htmlHash &&\n\t\t\thtmlHash(cleanText(html)) === input.htmlHash\n\t\t) {\n\t\t\treturn { outcome: 'unchanged', source: 'not-modified' }\n\t\t}\n\t\tif (html && Buffer.byteLength(html) > MAX_HTML_BYTES) {\n\t\t\t// A 6 MB HTML upload came back as 6 MB of markdown through Browser\n\t\t\t// Rendering, and Workers AI would bill the same bytes. It is not a page,\n\t\t\t// and it will be just as oversized next run - so not `transient`.\n\t\t\tlogger.info('url-to-markdown.html_oversized', {\n\t\t\t\turl: input.finalUrl.href,\n\t\t\t\tbytes: Buffer.byteLength(html)\n\t\t\t})\n\t\t\treturn {\n\t\t\t\toutcome: 'unsupported',\n\t\t\t\tsource: null,\n\t\t\t\thtml,\n\t\t\t\tcaptureIssue: 'html_byte_limit'\n\t\t\t}\n\t\t}\n\n\t\tlet shell = false\n\t\tlet workersAiRateLimited = false\n\t\tlet renderRateLimited = false\n\t\tif (html) {\n\t\t\tconst visible = withoutHiddenUi(html)\n\t\t\tlet converted: string | null = null\n\t\t\ttry {\n\t\t\t\tconverted = await client.convertHtmlWithWorkersAi({\n\t\t\t\t\thtml: visible,\n\t\t\t\t\t// The post-redirect host: Cloudflare resolves relative links against it.\n\t\t\t\t\thostname: input.finalUrl.hostname,\n\t\t\t\t\tsignal: input.signal\n\t\t\t\t})\n\t\t\t} catch (error) {\n\t\t\t\tif (!(error instanceof CloudflareRateLimitedError)) throw error\n\t\t\t\tworkersAiRateLimited = true\n\t\t\t}\n\t\t\tif (converted && !isBlankMarkdown(converted)) {\n\t\t\t\tshell = isApplicationShell(visible, converted)\n\t\t\t\tif (!shell)\n\t\t\t\t\treturn {\n\t\t\t\t\t\toutcome: 'ok',\n\t\t\t\t\t\tsource: 'workers-ai',\n\t\t\t\t\t\tmarkdown: converted,\n\t\t\t\t\t\thtml\n\t\t\t\t\t}\n\t\t\t}\n\t\t}\n\n\t\t// Browser Rendering is the exception path: slow, rate limited, and the only\n\t\t// tier that ever timed out in the probe.\n\t\tif (input.render && !input.signal?.aborted) {\n\t\t\tlet rendered: string | null = null\n\t\t\ttry {\n\t\t\t\trendered = await client.renderUrlToMarkdown(\n\t\t\t\t\tinput.finalUrl.href,\n\t\t\t\t\tinput.signal\n\t\t\t\t)\n\t\t\t} catch (error) {\n\t\t\t\tif (!(error instanceof CloudflareRateLimitedError)) throw error\n\t\t\t\trenderRateLimited = true\n\t\t\t}\n\t\t\tif (\n\t\t\t\trendered &&\n\t\t\t\t!isBlankMarkdown(rendered) &&\n\t\t\t\t(!html || !isApplicationShell(html, rendered))\n\t\t\t) {\n\t\t\t\treturn {\n\t\t\t\t\toutcome: 'ok',\n\t\t\t\t\tsource: 'browser-rendering',\n\t\t\t\t\tmarkdown: rendered,\n\t\t\t\t\thtml\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\t\treturn {\n\t\t\toutcome: 'transient',\n\t\t\tsource: null,\n\t\t\thtml,\n\t\t\tcaptureIssue: shell\n\t\t\t\t? 'render_incomplete'\n\t\t\t\t: workersAiRateLimited\n\t\t\t\t\t? 'workers_ai_rate_limited'\n\t\t\t\t\t: renderRateLimited\n\t\t\t\t\t\t? 'browser_rendering_rate_limited'\n\t\t\t\t\t\t: null\n\t\t}\n\t}\n\n\t/** Runs after the origin's slot is released: every tier it reaches is ours. */\n\tasync function readResponse(\n\t\tinput: OriginRead & {\n\t\t\trender: boolean\n\t\t\thtmlHash?: Nullable<string>\n\t\t\tsignal?: AbortSignal | undefined\n\t\t}\n\t): Promise<PageBody> {\n\t\tconst { response, body } = input\n\t\tconst status = response.status\n\t\tconst contentType = response.headers.get('content-type')\n\n\t\tif (status === 304) return { outcome: 'unchanged', source: 'not-modified' }\n\t\tif (isOneOf([404, 410], status)) {\n\t\t\treturn { outcome: 'gone', source: null }\n\t\t}\n\t\t// Still refused after the host's pauses: rendering would only be refused\n\t\t// too (see `MAX_RATE_LIMIT_RETRIES`).\n\t\tif (status >= 500 || status === 429)\n\t\t\treturn { outcome: 'transient', source: null }\n\t\tif (status >= 400) {\n\t\t\tif (!input.render || !isOneOf(BOT_WALL_STATUSES, status)) {\n\t\t\t\treturn { outcome: 'blocked', source: null }\n\t\t\t}\n\t\t\tlet rendered: string | null\n\t\t\ttry {\n\t\t\t\trendered = await client.renderUrlToMarkdown(\n\t\t\t\t\tinput.finalUrl.href,\n\t\t\t\t\tinput.signal\n\t\t\t\t)\n\t\t\t} catch (error) {\n\t\t\t\tif (!(error instanceof CloudflareRateLimitedError)) throw error\n\t\t\t\t// The wall may well let a browser through on the next run.\n\t\t\t\treturn {\n\t\t\t\t\toutcome: 'transient',\n\t\t\t\t\tsource: null,\n\t\t\t\t\tcaptureIssue: 'browser_rendering_rate_limited'\n\t\t\t\t}\n\t\t\t}\n\t\t\treturn rendered\n\t\t\t\t? { outcome: 'ok', source: 'browser-rendering', markdown: rendered }\n\t\t\t\t: { outcome: 'blocked', source: null }\n\t\t}\n\t\tif (status < 200 || status >= 300) {\n\t\t\treturn { outcome: 'transient', source: null }\n\t\t}\n\n\t\tconst kind = classifyContentType(contentType)\n\t\treturn typedSwitch(kind, {\n\t\t\tasset: () => ({ outcome: 'unsupported', source: null }),\n\t\t\tdocument: () =>\n\t\t\t\tconvertDocument({\n\t\t\t\t\tbody,\n\t\t\t\t\tcontentType,\n\t\t\t\t\tfinalUrl: input.finalUrl,\n\t\t\t\t\tsignal: input.signal\n\t\t\t\t}),\n\t\t\t...typedSwitchGroup(\n\t\t\t\tkind,\n\t\t\t\tTEXTUAL_KINDS,\n\t\t\t\tasync (textualKind): Promise<PageBody> =>\n\t\t\t\t\tbody.kind === 'text'\n\t\t\t\t\t\t? convertTextual({\n\t\t\t\t\t\t\t\ttext: body.text,\n\t\t\t\t\t\t\t\tkind: textualKind,\n\t\t\t\t\t\t\t\tfinalUrl: input.finalUrl,\n\t\t\t\t\t\t\t\trender: input.render,\n\t\t\t\t\t\t\t\thtmlHash: input.htmlHash,\n\t\t\t\t\t\t\t\tsignal: input.signal\n\t\t\t\t\t\t\t})\n\t\t\t\t\t\t: {\n\t\t\t\t\t\t\t\toutcome: 'unsupported',\n\t\t\t\t\t\t\t\tsource: null,\n\t\t\t\t\t\t\t\tcaptureIssue: body.kind === 'none' ? body.captureIssue : null\n\t\t\t\t\t\t\t}\n\t\t\t)\n\t\t})\n\t}\n\n\t/**\n\t * Read one URL as markdown through Cloudflare's three tiers - content\n\t * negotiation, Workers AI `toMarkdown`, Browser Rendering - and report\n\t * everything the import pipeline needs about the page: real status, redirect\n\t * target, validators, HTML metadata and absolute links.\n\t *\n\t * Never throws: a page that cannot be read is an outcome, not an exception,\n\t * because the caller is crawling hundreds of URLs at a time. Anything\n\t * unexpected (a failing shared limiter, a corrupt snapshot) is logged and\n\t * comes back `transient`. Inside `withCrawlSnapshot` it reads through the\n\t * snapshot.\n\t */\n\tasync function fetchPageAsMarkdown(\n\t\tinput: FetchPageInput\n\t): Promise<FetchedPage> {\n\t\tconst elapsedMs = startTimer()\n\t\ttry {\n\t\t\treturn await readPage(input)\n\t\t} catch (error) {\n\t\t\tlogger.warn('url-to-markdown.failed', {\n\t\t\t\turl: input.url,\n\t\t\t\t...errorLogAttributes(error)\n\t\t\t})\n\t\t\treturn buildPage(\n\t\t\t\t{\n\t\t\t\t\turl: input.url,\n\t\t\t\t\tfinalUrl: input.url,\n\t\t\t\t\tstatus: 0,\n\t\t\t\t\tcontentType: null,\n\t\t\t\t\tetag: null,\n\t\t\t\t\tlastModified: null,\n\t\t\t\t\telapsedMs\n\t\t\t\t},\n\t\t\t\t{ outcome: 'transient', source: null },\n\t\t\t\tlogger\n\t\t\t)\n\t\t}\n\t}\n\n\tasync function readPage(input: FetchPageInput): Promise<FetchedPage> {\n\t\tif (!currentCrawlSnapshot()) return fetchPageLive(input)\n\n\t\t// The snapshot holds the whole page, read without validators; whether it is\n\t\t// \"unchanged\" for this run is decided here, the way the origin would.\n\t\tconst page = await throughCrawlSnapshot(\n\t\t\t`page ${input.url} ${input.country ?? ''} ${input.render ?? true}`,\n\t\t\t() =>\n\t\t\t\tfetchPageLive({\n\t\t\t\t\turl: input.url,\n\t\t\t\t\tcountry: input.country,\n\t\t\t\t\trender: input.render,\n\t\t\t\t\tsignal: input.signal\n\t\t\t\t})\n\t\t)\n\t\treturn isUnchangedFor(page, input) ? asUnchanged(page) : page\n\t}\n\n\tasync function fetchPageLive(input: FetchPageInput): Promise<FetchedPage> {\n\t\tconst signal = input.signal ?? currentOperationSignal()\n\t\tconst elapsedMs = startTimer()\n\t\tconst render = input.render ?? true\n\n\t\tconst prepared = prepareRequestUrl(input.url, allowLocalhost)\n\t\tif (!prepared) {\n\t\t\treturn buildPage(\n\t\t\t\t{\n\t\t\t\t\turl: input.url,\n\t\t\t\t\tfinalUrl: input.url,\n\t\t\t\t\tstatus: 0,\n\t\t\t\t\tcontentType: null,\n\t\t\t\t\tetag: null,\n\t\t\t\t\tlastModified: null,\n\t\t\t\t\telapsedMs\n\t\t\t\t},\n\t\t\t\t{ outcome: 'unsupported', source: null },\n\t\t\t\tlogger\n\t\t\t)\n\t\t}\n\n\t\tconst headers: Record<string, string> = {\n\t\t\tAccept: 'text/markdown, text/html;q=0.9, */*;q=0.8',\n\t\t\t'Accept-Language': '*',\n\t\t\t'User-Agent': BROWSER_USER_AGENT\n\t\t}\n\t\tif (input.etag) headers['If-None-Match'] = input.etag\n\t\tif (input.lastModified) headers['If-Modified-Since'] = input.lastModified\n\n\t\tconst request = (url: URL) =>\n\t\t\trequestOrigin(inMarket(url, input.country), headers, signal)\n\n\t\tlet requestUrl = prepared\n\t\tlet attempt: OriginAttempt = await request(requestUrl)\n\n\t\tif (attempt.kind === 'failed' && attempt.retryable) {\n\t\t\tconst fallback = wwwFallbackUrl(requestUrl)\n\t\t\tif (fallback) {\n\t\t\t\tconst retry = await request(fallback)\n\t\t\t\tif (retry.kind === 'read') {\n\t\t\t\t\trequestUrl = fallback\n\t\t\t\t\tattempt = retry\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\n\t\tfor (\n\t\t\tlet retry = 1;\n\t\t\tretry <= MAX_RATE_LIMIT_RETRIES &&\n\t\t\tattempt.kind === 'read' &&\n\t\t\tattempt.read.response.status === 429;\n\t\t\tretry++\n\t\t) {\n\t\t\tconst waitMs = rateLimitWaitMs(attempt.read.response, retry)\n\t\t\tif (await hostLimits.coolDownHost(requestUrl.hostname, waitMs)) {\n\t\t\t\tlogger.warn('url-to-markdown.host_rate_limited', {\n\t\t\t\t\thost: requestUrl.hostname,\n\t\t\t\t\tretry_after: attempt.read.response.headers.get('retry-after') ?? '',\n\t\t\t\t\twait_ms: waitMs\n\t\t\t\t})\n\t\t\t}\n\t\t\tattempt = await request(requestUrl)\n\t\t}\n\n\t\tif (attempt.kind !== 'read') {\n\t\t\treturn buildPage(\n\t\t\t\t{\n\t\t\t\t\turl: requestUrl.href,\n\t\t\t\t\tfinalUrl: requestUrl.href,\n\t\t\t\t\tstatus: 0,\n\t\t\t\t\tcontentType: null,\n\t\t\t\t\tetag: null,\n\t\t\t\t\tlastModified: null,\n\t\t\t\t\telapsedMs\n\t\t\t\t},\n\t\t\t\t{\n\t\t\t\t\toutcome: typedSwitch(attempt, 'kind', {\n\t\t\t\t\t\t// A refused host is a policy decision, not a retryable network blip.\n\t\t\t\t\t\trefused: () => 'blocked',\n\t\t\t\t\t\tfailed: () => 'transient'\n\t\t\t\t\t}),\n\t\t\t\t\tsource: null\n\t\t\t\t},\n\t\t\t\tlogger\n\t\t\t)\n\t\t}\n\n\t\tconst { response, finalUrl } = attempt.read\n\t\tconst notModified = (stored: Nullable<string>) =>\n\t\t\tresponse.status === 304 ? (stored ?? null) : null\n\t\tconst body = await readResponse({\n\t\t\t...attempt.read,\n\t\t\trender,\n\t\t\thtmlHash: input.htmlHash,\n\t\t\tsignal\n\t\t})\n\n\t\tconst page = buildPage(\n\t\t\t{\n\t\t\t\turl: requestUrl.href,\n\t\t\t\tfinalUrl: finalUrl.href,\n\t\t\t\tstatus: response.status,\n\t\t\t\tcontentType: response.headers.get('content-type'),\n\t\t\t\t// A 304 need not repeat the validators; the stored ones stay valid. On\n\t\t\t\t// any other status only this response's headers count - carrying an old\n\t\t\t\t// ETag onto changed content would freeze the page forever.\n\t\t\t\tetag: response.headers.get('etag') ?? notModified(input.etag),\n\t\t\t\tlastModified:\n\t\t\t\t\tresponse.headers.get('last-modified') ??\n\t\t\t\t\tnotModified(input.lastModified),\n\t\t\t\telapsedMs\n\t\t\t},\n\t\t\tbody,\n\t\t\tlogger\n\t\t)\n\n\t\tlogger.info('url-to-markdown.completed', {\n\t\t\turl: page.finalUrl,\n\t\t\tstatus: page.status,\n\t\t\toutcome: page.outcome,\n\t\t\tsource: page.source ?? '',\n\t\t\tchars: page.markdown.length,\n\t\t\tduration_ms: page.durationMs\n\t\t})\n\t\treturn page\n\t}\n\n\t/** CLI/one-shot shape: markdown or an error. */\n\tasync function urlToMarkdown(urlString: string) {\n\t\tconst page = await fetchPageAsMarkdown({ url: urlString })\n\t\tconst { markdown, source } = page\n\t\tif (page.outcome !== 'ok' || !markdown || !source) {\n\t\t\tthrow new Error(\n\t\t\t\t`Failed to convert URL to markdown (${page.outcome}, status ${page.status}): ${urlString}`\n\t\t\t)\n\t\t}\n\t\treturn { markdown, source }\n\t}\n\n\treturn {\n\t\tfetchPageAsMarkdown,\n\t\tconvertDocumentBytes,\n\t\turlToMarkdown,\n\t\tconvertFileWithWorkersAi: client.convertFileWithWorkersAi,\n\t\trenderUrlToMarkdown: client.renderUrlToMarkdown,\n\t\t/** The SSRF guard the converter runs, bound to its `logger`. */\n\t\tisHostAllowed: allowHost,\n\t\t/**\n\t\t * `fetch` behind the converter's guard, run on the URL and on every\n\t\t * redirect. Throws `HostNotAllowedError` when one is refused. Use it for\n\t\t * any other request to a customer-supplied URL.\n\t\t */\n\t\tguardedFetch(\n\t\t\tinput: string | URL,\n\t\t\tinit?: RequestInit,\n\t\t\tfetchOptions?: Pick<GuardedFetchOptions, 'maxRedirects'>\n\t\t) {\n\t\t\treturn guardedFetch(input, init, { ...guard, ...fetchOptions })\n\t\t},\n\t\t/** `withCrawlSnapshot` bound to the `crawlSnapshot` option. */\n\t\twithCrawlSnapshot<T>(\n\t\t\tinput: Nullable<CrawlSnapshotInput>,\n\t\t\trun: () => Promise<T>\n\t\t) {\n\t\t\tif (!input) return run()\n\t\t\tinvariant(\n\t\t\t\toptions.crawlSnapshot,\n\t\t\t\t'Set crawlSnapshot.root to use crawl snapshots'\n\t\t\t)\n\t\t\treturn withCrawlSnapshot(input, run, {\n\t\t\t\troot: options.crawlSnapshot.root,\n\t\t\t\tlogger: options.crawlSnapshot.logger ?? logger\n\t\t\t})\n\t\t}\n\t}\n}\n\nexport type WebsiteToMarkdown = ReturnType<typeof createWebsiteToMarkdown>\n"]}

package/dist/hash.d.ts

@@ -1,11 +1,7 @@
 /**
- * Hex SHA-256 of a UTF-8 string or of raw bytes - the repo's one content-hash
- * function.
- *
- * Used as a change detector (has this text moved since we last derived from
- * it?), as an API-secret digest, and to fingerprint fetched binaries. The value
- * is a stored identity in several tables, so it must stay byte-for-byte what it
- * has always been.
+ * Hex SHA-256 of a UTF-8 string or of raw bytes. `htmlHash` and snapshot file
+ * names are built on it, and callers store them, so it must stay byte-for-byte
+ * what it has always been.
  */
 export declare function sha256Hex(data: string | Uint8Array): string;
 //# sourceMappingURL=hash.d.ts.map

package/dist/hash.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"hash.d.ts","sourceRoot":"","sources":["../src/hash.ts"],"names":[],"mappings":"AAEA;;;;;;;;GAQG;AACH,wBAAgB,SAAS,CAAC,IAAI,EAAE,MAAM,GAAG,UAAU,GAAG,MAAM,CAI3D"}
+{"version":3,"file":"hash.d.ts","sourceRoot":"","sources":["../src/hash.ts"],"names":[],"mappings":"AAEA;;;;GAIG;AACH,wBAAgB,SAAS,CAAC,IAAI,EAAE,MAAM,GAAG,UAAU,GAAG,MAAM,CAI3D"}

package/dist/hash.js

@@ -1,12 +1,8 @@
 import { createHash } from 'node:crypto';
 /**
- * Hex SHA-256 of a UTF-8 string or of raw bytes - the repo's one content-hash
- * function.
- *
- * Used as a change detector (has this text moved since we last derived from
- * it?), as an API-secret digest, and to fingerprint fetched binaries. The value
- * is a stored identity in several tables, so it must stay byte-for-byte what it
- * has always been.
+ * Hex SHA-256 of a UTF-8 string or of raw bytes. `htmlHash` and snapshot file
+ * names are built on it, and callers store them, so it must stay byte-for-byte
+ * what it has always been.
  */
 export function sha256Hex(data) {
     return typeof data === 'string'

package/dist/hash.js.map

@@ -1 +1 @@
-{"version":3,"file":"hash.js","sourceRoot":"","sources":["../src/hash.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,UAAU,EAAE,MAAM,aAAa,CAAA;AAExC;;;;;;;;GAQG;AACH,MAAM,UAAU,SAAS,CAAC,IAAyB;IAClD,OAAO,OAAO,IAAI,KAAK,QAAQ;QAC9B,CAAC,CAAC,UAAU,CAAC,QAAQ,CAAC,CAAC,MAAM,CAAC,IAAI,EAAE,MAAM,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC;QACzD,CAAC,CAAC,UAAU,CAAC,QAAQ,CAAC,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAA;AACnD,CAAC","sourcesContent":["import { createHash } from 'node:crypto'\n\n/**\n * Hex SHA-256 of a UTF-8 string or of raw bytes - the repo's one content-hash\n * function.\n *\n * Used as a change detector (has this text moved since we last derived from\n * it?), as an API-secret digest, and to fingerprint fetched binaries. The value\n * is a stored identity in several tables, so it must stay byte-for-byte what it\n * has always been.\n */\nexport function sha256Hex(data: string | Uint8Array): string {\n\treturn typeof data === 'string'\n\t\t? createHash('sha256').update(data, 'utf8').digest('hex')\n\t\t: createHash('sha256').update(data).digest('hex')\n}\n"]}
+{"version":3,"file":"hash.js","sourceRoot":"","sources":["../src/hash.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,UAAU,EAAE,MAAM,aAAa,CAAA;AAExC;;;;GAIG;AACH,MAAM,UAAU,SAAS,CAAC,IAAyB;IAClD,OAAO,OAAO,IAAI,KAAK,QAAQ;QAC9B,CAAC,CAAC,UAAU,CAAC,QAAQ,CAAC,CAAC,MAAM,CAAC,IAAI,EAAE,MAAM,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC;QACzD,CAAC,CAAC,UAAU,CAAC,QAAQ,CAAC,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAA;AACnD,CAAC","sourcesContent":["import { createHash } from 'node:crypto'\n\n/**\n * Hex SHA-256 of a UTF-8 string or of raw bytes. `htmlHash` and snapshot file\n * names are built on it, and callers store them, so it must stay byte-for-byte\n * what it has always been.\n */\nexport function sha256Hex(data: string | Uint8Array): string {\n\treturn typeof data === 'string'\n\t\t? createHash('sha256').update(data, 'utf8').digest('hex')\n\t\t: createHash('sha256').update(data).digest('hex')\n}\n"]}

package/dist/html-metadata.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"html-metadata.d.ts","sourceRoot":"","sources":["../src/html-metadata.ts"],"names":[],"mappings":"AAkEA;;;;GAIG;AACH,wBAAgB,mBAAmB,CAAC,IAAI,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM;;;;;;;;EAsDhE"}
+{"version":3,"file":"html-metadata.d.ts","sourceRoot":"","sources":["../src/html-metadata.ts"],"names":[],"mappings":"AAoEA;;;;GAIG;AACH,wBAAgB,mBAAmB,CAAC,IAAI,EAAE,MAAM,EAAE,OAAO,EAAE,MAAM;;;;;;;;EAsDhE"}

package/dist/html-metadata.js

@@ -1,3 +1,4 @@
+import { getUniqueValues, isOneOf, normalizeWhitespace, safeJsonParse } from 'hanzio';
 import { decodeHtmlEntities } from './html-entities.js';
 const TITLE_RE = /<title[^>]*>([\s\S]*?)<\/title>/i;
 const HTML_LANG_RE = /<html\b[^>]*?\blang\s*=\s*("[^"]*"|'[^']*'|[^\s>]+)/i;
@@ -25,16 +26,13 @@ function attributesOf(tag) {
     return attributes;
 }
 function cleanText(value) {
-    return decodeHtmlEntities(value.replace(HTML_TAG_RE, ' '))
-        .replace(/\s+/g, ' ')
-        .trim();
+    return normalizeWhitespace(decodeHtmlEntities(value.replace(HTML_TAG_RE, ' ')));
 }
 function absolute(url, baseUrl) {
     try {
         const resolved = new URL(url, baseUrl);
-        if (resolved.protocol !== 'https:' && resolved.protocol !== 'http:') {
+        if (!isOneOf(['https:', 'http:'], resolved.protocol))
             return null;
-        }
         return resolved.href;
     }
     catch {
@@ -47,18 +45,16 @@ function parseJsonLd(html) {
         const body = match[1]?.trim();
         if (!body)
             continue;
-        try {
-            const parsed = JSON.parse(body);
-            // A single script may hold an array of nodes; flatten so the profile
-            // layer can look for `@type` without walking two shapes.
-            if (Array.isArray(parsed))
-                blocks.push(...parsed);
-            else
-                blocks.push(parsed);
-        }
-        catch {
-            // Hand-written JSON-LD with a trailing comma is common; skip the block.
-        }
+        const parsed = safeJsonParse(body);
+        // Hand-written JSON-LD with a trailing comma is common; skip the block.
+        if (!parsed.ok)
+            continue;
+        // A single script may hold an array of nodes; flatten so the profile
+        // layer can look for `@type` without walking two shapes.
+        if (Array.isArray(parsed.data))
+            blocks.push(...parsed.data);
+        else
+            blocks.push(parsed.data);
     }
     return blocks;
 }
@@ -77,7 +73,7 @@ export function extractHtmlMetadata(html, baseUrl) {
     // `<link rel="alternate" hreflang>`: the site's own declaration of which
     // URLs are translations of this page. `x-default` is a language picker,
     // not a translation.
-    const alternates = new Set();
+    const alternates = [];
     /** Each alternate's language (`da` of `da-DK`); what lets a crawl skip a mirror. */
     const alternateLanguages = {};
     for (const match of html.matchAll(LINK_TAG_RE)) {
@@ -94,7 +90,7 @@ export function extractHtmlMetadata(html, baseUrl) {
             const href = absolute(attributes.href, baseUrl);
             const language = hreflang.split('-')[0] ?? '';
             if (href)
-                alternates.add(href);
+                alternates.push(href);
             if (href && /^[a-z]{2}$/.test(language))
                 alternateLanguages[href] = language;
         }
@@ -116,7 +112,7 @@ export function extractHtmlMetadata(html, baseUrl) {
         title,
         lang,
         canonical,
-        alternates: [...alternates],
+        alternates: getUniqueValues(alternates),
         alternateLanguages,
         metaDescription: metaDescription || ogDescription || null,
         jsonLd: parseJsonLd(html)

package/dist/html-metadata.js.map

@@ -1 +1 @@
-{"version":3,"file":"html-metadata.js","sourceRoot":"","sources":["../src/html-metadata.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,kBAAkB,EAAE,MAAM,oBAAoB,CAAA;AAEvD,MAAM,QAAQ,GAAG,kCAAkC,CAAA;AACnD,MAAM,YAAY,GAAG,sDAAsD,CAAA;AAC3E,MAAM,WAAW,GAAG,iBAAiB,CAAA;AACrC,MAAM,WAAW,GAAG,iBAAiB,CAAA;AACrC,MAAM,UAAU,GACf,oFAAoF,CAAA;AACrF,MAAM,YAAY,GAAG,0DAA0D,CAAA;AAC/E,MAAM,WAAW,GAAG,UAAU,CAAA;AAE9B,SAAS,OAAO,CAAC,KAAa;IAC7B,MAAM,KAAK,GAAG,KAAK,CAAC,CAAC,CAAC,CAAA;IACtB,IAAI,CAAC,KAAK,KAAK,GAAG,IAAI,KAAK,KAAK,GAAG,CAAC,IAAI,KAAK,CAAC,QAAQ,CAAC,KAAK,CAAC,EAAE,CAAC;QAC/D,OAAO,KAAK,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAA;IAC1B,CAAC;IACD,OAAO,KAAK,CAAA;AACb,CAAC;AAED,SAAS,YAAY,CAAC,GAAW;IAChC,MAAM,UAAU,GAA2B,EAAE,CAAA;IAC7C,KAAK,MAAM,KAAK,IAAI,GAAG,CAAC,QAAQ,CAAC,YAAY,CAAC,EAAE,CAAC;QAChD,MAAM,IAAI,GAAG,KAAK,CAAC,CAAC,CAAC,EAAE,WAAW,EAAE,CAAA;QACpC,MAAM,KAAK,GAAG,KAAK,CAAC,CAAC,CAAC,CAAA;QACtB,IAAI,CAAC,IAAI,IAAI,KAAK,KAAK,SAAS;YAAE,SAAQ;QAC1C,UAAU,CAAC,IAAI,CAAC,GAAG,kBAAkB,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,CAAA;IACtD,CAAC;IACD,OAAO,UAAU,CAAA;AAClB,CAAC;AAED,SAAS,SAAS,CAAC,KAAa;IAC/B,OAAO,kBAAkB,CAAC,KAAK,CAAC,OAAO,CAAC,WAAW,EAAE,GAAG,CAAC,CAAC;SACxD,OAAO,CAAC,MAAM,EAAE,GAAG,CAAC;SACpB,IAAI,EAAE,CAAA;AACT,CAAC;AAED,SAAS,QAAQ,CAAC,GAAW,EAAE,OAAe;IAC7C,IAAI,CAAC;QACJ,MAAM,QAAQ,GAAG,IAAI,GAAG,CAAC,GAAG,EAAE,OAAO,CAAC,CAAA;QACtC,IAAI,QAAQ,CAAC,QAAQ,KAAK,QAAQ,IAAI,QAAQ,CAAC,QAAQ,KAAK,OAAO,EAAE,CAAC;YACrE,OAAO,IAAI,CAAA;QACZ,CAAC;QACD,OAAO,QAAQ,CAAC,IAAI,CAAA;IACrB,CAAC;IAAC,MAAM,CAAC;QACR,OAAO,IAAI,CAAA;IACZ,CAAC;AACF,CAAC;AAED,SAAS,WAAW,CAAC,IAAY;IAChC,MAAM,MAAM,GAAc,EAAE,CAAA;IAC5B,KAAK,MAAM,KAAK,IAAI,IAAI,CAAC,QAAQ,CAAC,UAAU,CAAC,EAAE,CAAC;QAC/C,MAAM,IAAI,GAAG,KAAK,CAAC,CAAC,CAAC,EAAE,IAAI,EAAE,CAAA;QAC7B,IAAI,CAAC,IAAI;YAAE,SAAQ;QACnB,IAAI,CAAC;YACJ,MAAM,MAAM,GAAY,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,CAAA;YACxC,qEAAqE;YACrE,yDAAyD;YACzD,IAAI,KAAK,CAAC,OAAO,CAAC,MAAM,CAAC;gBAAE,MAAM,CAAC,IAAI,CAAC,GAAG,MAAM,CAAC,CAAA;;gBAC5C,MAAM,CAAC,IAAI,CAAC,MAAM,CAAC,CAAA;QACzB,CAAC;QAAC,MAAM,CAAC;YACR,wEAAwE;QACzE,CAAC;IACF,CAAC;IACD,OAAO,MAAM,CAAA;AACd,CAAC;AAED;;;;GAIG;AACH,MAAM,UAAU,mBAAmB,CAAC,IAAY,EAAE,OAAe;IAChE,MAAM,KAAK,GAAG,SAAS,CAAC,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,IAAI,IAAI,CAAA;IAE/D,MAAM,OAAO,GAAG,YAAY,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC,CAAC,CAAA;IAC5C,MAAM,QAAQ,GAAG,OAAO,CAAC,CAAC,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC,IAAI,EAAE,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAA;IAC1E,2EAA2E;IAC3E,MAAM,IAAI,GACT,QAAQ,IAAI,aAAa,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,WAAW,EAAE,CAAC,CAAC,CAAC,IAAI,CAAA;IAEzE,IAAI,SAAS,GAAkB,IAAI,CAAA;IACnC,yEAAyE;IACzE,wEAAwE;IACxE,qBAAqB;IACrB,MAAM,UAAU,GAAG,IAAI,GAAG,EAAU,CAAA;IACpC,oFAAoF;IACpF,MAAM,kBAAkB,GAA2B,EAAE,CAAA;IACrD,KAAK,MAAM,KAAK,IAAI,IAAI,CAAC,QAAQ,CAAC,WAAW,CAAC,EAAE,CAAC;QAChD,MAAM,UAAU,GAAG,YAAY,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAA;QACzC,sEAAsE;QACtE,MAAM,GAAG,GAAG,UAAU,CAAC,GAAG,EAAE,WAAW,EAAE,CAAC,KAAK,CAAC,KAAK,CAAC,IAAI,EAAE,CAAA;QAC5D,IAAI,CAAC,UAAU,CAAC,IAAI;YAAE,SAAQ;QAC9B,IAAI,GAAG,CAAC,QAAQ,CAAC,WAAW,CAAC,IAAI,CAAC,SAAS,EAAE,CAAC;YAC7C,SAAS,GAAG,QAAQ,CAAC,UAAU,CAAC,IAAI,EAAE,OAAO,CAAC,CAAA;QAC/C,CAAC;QACD,MAAM,QAAQ,GAAG,UAAU,CAAC,QAAQ,EAAE,IAAI,EAAE,CAAC,WAAW,EAAE,CAAA;QAC1D,IAAI,GAAG,CAAC,QAAQ,CAAC,WAAW,CAAC,IAAI,QAAQ,IAAI,QAAQ,KAAK,WAAW,EAAE,CAAC;YACvE,MAAM,IAAI,GAAG,QAAQ,CAAC,UAAU,CAAC,IAAI,EAAE,OAAO,CAAC,CAAA;YAC/C,MAAM,QAAQ,GAAG,QAAQ,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,CAAA;YAC7C,IAAI,IAAI;gBAAE,UAAU,CAAC,GAAG,CAAC,IAAI,CAAC,CAAA;YAC9B,IAAI,IAAI,IAAI,YAAY,CAAC,IAAI,CAAC,QAAQ,CAAC;gBACtC,kBAAkB,CAAC,IAAI,CAAC,GAAG,QAAQ,CAAA;QACrC,CAAC;IACF,CAAC;IAED,IAAI,eAAe,GAAkB,IAAI,CAAA;IACzC,IAAI,aAAa,GAAkB,IAAI,CAAA;IACvC,KAAK,MAAM,KAAK,IAAI,IAAI,CAAC,QAAQ,CAAC,WAAW,CAAC,EAAE,CAAC;QAChD,MAAM,UAAU,GAAG,YAAY,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAA;QACzC,MAAM,GAAG,GAAG,CAAC,UAAU,CAAC,IAAI,IAAI,UAAU,CAAC,QAAQ,IAAI,EAAE,CAAC,CAAC,WAAW,EAAE,CAAA;QACxE,MAAM,OAAO,GAAG,UAAU,CAAC,OAAO,EAAE,IAAI,EAAE,CAAA;QAC1C,IAAI,CAAC,OAAO;YAAE,SAAQ;QACtB,IAAI,GAAG,KAAK,aAAa;YAAE,eAAe,KAAK,SAAS,CAAC,OAAO,CAAC,CAAA;QACjE,IAAI,GAAG,KAAK,gBAAgB;YAAE,aAAa,KAAK,SAAS,CAAC,OAAO,CAAC,CAAA;IACnE,CAAC;IAED,OAAO;QACN,KAAK;QACL,IAAI;QACJ,SAAS;QACT,UAAU,EAAE,CAAC,GAAG,UAAU,CAAC;QAC3B,kBAAkB;QAClB,eAAe,EAAE,eAAe,IAAI,aAAa,IAAI,IAAI;QACzD,MAAM,EAAE,WAAW,CAAC,IAAI,CAAC;KACzB,CAAA;AACF,CAAC","sourcesContent":["import { decodeHtmlEntities } from './html-entities.js'\n\nconst TITLE_RE = /<title[^>]*>([\\s\\S]*?)<\\/title>/i\nconst HTML_LANG_RE = /<html\\b[^>]*?\\blang\\s*=\\s*(\"[^\"]*\"|'[^']*'|[^\\s>]+)/i\nconst LINK_TAG_RE = /<link\\b[^>]*>/gi\nconst META_TAG_RE = /<meta\\b[^>]*>/gi\nconst JSON_LD_RE =\n\t/<script\\b[^>]*type\\s*=\\s*[\"']application\\/ld\\+json[\"'][^>]*>([\\s\\S]*?)<\\/script>/gi\nconst ATTRIBUTE_RE = /([a-z_:][-a-z0-9_:.]*)\\s*=\\s*(\"[^\"]*\"|'[^']*'|[^\\s>]+)/gi\nconst HTML_TAG_RE = /<[^>]*>/g\n\nfunction unquote(value: string) {\n\tconst first = value[0]\n\tif ((first === '\"' || first === \"'\") && value.endsWith(first)) {\n\t\treturn value.slice(1, -1)\n\t}\n\treturn value\n}\n\nfunction attributesOf(tag: string) {\n\tconst attributes: Record<string, string> = {}\n\tfor (const match of tag.matchAll(ATTRIBUTE_RE)) {\n\t\tconst name = match[1]?.toLowerCase()\n\t\tconst value = match[2]\n\t\tif (!name || value === undefined) continue\n\t\tattributes[name] = decodeHtmlEntities(unquote(value))\n\t}\n\treturn attributes\n}\n\nfunction cleanText(value: string) {\n\treturn decodeHtmlEntities(value.replace(HTML_TAG_RE, ' '))\n\t\t.replace(/\\s+/g, ' ')\n\t\t.trim()\n}\n\nfunction absolute(url: string, baseUrl: string) {\n\ttry {\n\t\tconst resolved = new URL(url, baseUrl)\n\t\tif (resolved.protocol !== 'https:' && resolved.protocol !== 'http:') {\n\t\t\treturn null\n\t\t}\n\t\treturn resolved.href\n\t} catch {\n\t\treturn null\n\t}\n}\n\nfunction parseJsonLd(html: string) {\n\tconst blocks: unknown[] = []\n\tfor (const match of html.matchAll(JSON_LD_RE)) {\n\t\tconst body = match[1]?.trim()\n\t\tif (!body) continue\n\t\ttry {\n\t\t\tconst parsed: unknown = JSON.parse(body)\n\t\t\t// A single script may hold an array of nodes; flatten so the profile\n\t\t\t// layer can look for `@type` without walking two shapes.\n\t\t\tif (Array.isArray(parsed)) blocks.push(...parsed)\n\t\t\telse blocks.push(parsed)\n\t\t} catch {\n\t\t\t// Hand-written JSON-LD with a trailing comma is common; skip the block.\n\t\t}\n\t}\n\treturn blocks\n}\n\n/**\n * Everything the pipeline needs from the HTML that markdown cannot carry.\n * Regex-level on purpose: we read five fields plus JSON-LD out of pages that\n * are often invalid HTML, and a parser would cost more than it buys.\n */\nexport function extractHtmlMetadata(html: string, baseUrl: string) {\n\tconst title = cleanText(TITLE_RE.exec(html)?.[1] ?? '') || null\n\n\tconst langRaw = HTML_LANG_RE.exec(html)?.[1]\n\tconst langCode = langRaw ? unquote(langRaw).trim().split(/[-_]/)[0] : null\n\t// ISO 639-1 only: `da-DK` → `da`, and a bogus `lang=\"Dansk\"` becomes null.\n\tconst lang =\n\t\tlangCode && /^[a-z]{2}$/i.test(langCode) ? langCode.toLowerCase() : null\n\n\tlet canonical: string | null = null\n\t// `<link rel=\"alternate\" hreflang>`: the site's own declaration of which\n\t// URLs are translations of this page. `x-default` is a language picker,\n\t// not a translation.\n\tconst alternates = new Set<string>()\n\t/** Each alternate's language (`da` of `da-DK`); what lets a crawl skip a mirror. */\n\tconst alternateLanguages: Record<string, string> = {}\n\tfor (const match of html.matchAll(LINK_TAG_RE)) {\n\t\tconst attributes = attributesOf(match[0])\n\t\t// `rel` is a token list: WordPress emits `rel=\"shortlink canonical\"`.\n\t\tconst rel = attributes.rel?.toLowerCase().split(/\\s+/) ?? []\n\t\tif (!attributes.href) continue\n\t\tif (rel.includes('canonical') && !canonical) {\n\t\t\tcanonical = absolute(attributes.href, baseUrl)\n\t\t}\n\t\tconst hreflang = attributes.hreflang?.trim().toLowerCase()\n\t\tif (rel.includes('alternate') && hreflang && hreflang !== 'x-default') {\n\t\t\tconst href = absolute(attributes.href, baseUrl)\n\t\t\tconst language = hreflang.split('-')[0] ?? ''\n\t\t\tif (href) alternates.add(href)\n\t\t\tif (href && /^[a-z]{2}$/.test(language))\n\t\t\t\talternateLanguages[href] = language\n\t\t}\n\t}\n\n\tlet metaDescription: string | null = null\n\tlet ogDescription: string | null = null\n\tfor (const match of html.matchAll(META_TAG_RE)) {\n\t\tconst attributes = attributesOf(match[0])\n\t\tconst key = (attributes.name ?? attributes.property ?? '').toLowerCase()\n\t\tconst content = attributes.content?.trim()\n\t\tif (!content) continue\n\t\tif (key === 'description') metaDescription ??= cleanText(content)\n\t\tif (key === 'og:description') ogDescription ??= cleanText(content)\n\t}\n\n\treturn {\n\t\ttitle,\n\t\tlang,\n\t\tcanonical,\n\t\talternates: [...alternates],\n\t\talternateLanguages,\n\t\tmetaDescription: metaDescription || ogDescription || null,\n\t\tjsonLd: parseJsonLd(html)\n\t}\n}\n"]}
+{"version":3,"file":"html-metadata.js","sourceRoot":"","sources":["../src/html-metadata.ts"],"names":[],"mappings":"AAAA,OAAO,EACN,eAAe,EACf,OAAO,EACP,mBAAmB,EACnB,aAAa,EACb,MAAM,QAAQ,CAAA;AACf,OAAO,EAAE,kBAAkB,EAAE,MAAM,oBAAoB,CAAA;AAEvD,MAAM,QAAQ,GAAG,kCAAkC,CAAA;AACnD,MAAM,YAAY,GAAG,sDAAsD,CAAA;AAC3E,MAAM,WAAW,GAAG,iBAAiB,CAAA;AACrC,MAAM,WAAW,GAAG,iBAAiB,CAAA;AACrC,MAAM,UAAU,GACf,oFAAoF,CAAA;AACrF,MAAM,YAAY,GAAG,0DAA0D,CAAA;AAC/E,MAAM,WAAW,GAAG,UAAU,CAAA;AAE9B,SAAS,OAAO,CAAC,KAAa;IAC7B,MAAM,KAAK,GAAG,KAAK,CAAC,CAAC,CAAC,CAAA;IACtB,IAAI,CAAC,KAAK,KAAK,GAAG,IAAI,KAAK,KAAK,GAAG,CAAC,IAAI,KAAK,CAAC,QAAQ,CAAC,KAAK,CAAC,EAAE,CAAC;QAC/D,OAAO,KAAK,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,CAAA;IAC1B,CAAC;IACD,OAAO,KAAK,CAAA;AACb,CAAC;AAED,SAAS,YAAY,CAAC,GAAW;IAChC,MAAM,UAAU,GAA2B,EAAE,CAAA;IAC7C,KAAK,MAAM,KAAK,IAAI,GAAG,CAAC,QAAQ,CAAC,YAAY,CAAC,EAAE,CAAC;QAChD,MAAM,IAAI,GAAG,KAAK,CAAC,CAAC,CAAC,EAAE,WAAW,EAAE,CAAA;QACpC,MAAM,KAAK,GAAG,KAAK,CAAC,CAAC,CAAC,CAAA;QACtB,IAAI,CAAC,IAAI,IAAI,KAAK,KAAK,SAAS;YAAE,SAAQ;QAC1C,UAAU,CAAC,IAAI,CAAC,GAAG,kBAAkB,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,CAAA;IACtD,CAAC;IACD,OAAO,UAAU,CAAA;AAClB,CAAC;AAED,SAAS,SAAS,CAAC,KAAa;IAC/B,OAAO,mBAAmB,CACzB,kBAAkB,CAAC,KAAK,CAAC,OAAO,CAAC,WAAW,EAAE,GAAG,CAAC,CAAC,CACnD,CAAA;AACF,CAAC;AAED,SAAS,QAAQ,CAAC,GAAW,EAAE,OAAe;IAC7C,IAAI,CAAC;QACJ,MAAM,QAAQ,GAAG,IAAI,GAAG,CAAC,GAAG,EAAE,OAAO,CAAC,CAAA;QACtC,IAAI,CAAC,OAAO,CAAC,CAAC,QAAQ,EAAE,OAAO,CAAC,EAAE,QAAQ,CAAC,QAAQ,CAAC;YAAE,OAAO,IAAI,CAAA;QACjE,OAAO,QAAQ,CAAC,IAAI,CAAA;IACrB,CAAC;IAAC,MAAM,CAAC;QACR,OAAO,IAAI,CAAA;IACZ,CAAC;AACF,CAAC;AAED,SAAS,WAAW,CAAC,IAAY;IAChC,MAAM,MAAM,GAAc,EAAE,CAAA;IAC5B,KAAK,MAAM,KAAK,IAAI,IAAI,CAAC,QAAQ,CAAC,UAAU,CAAC,EAAE,CAAC;QAC/C,MAAM,IAAI,GAAG,KAAK,CAAC,CAAC,CAAC,EAAE,IAAI,EAAE,CAAA;QAC7B,IAAI,CAAC,IAAI;YAAE,SAAQ;QACnB,MAAM,MAAM,GAAG,aAAa,CAAC,IAAI,CAAC,CAAA;QAClC,wEAAwE;QACxE,IAAI,CAAC,MAAM,CAAC,EAAE;YAAE,SAAQ;QACxB,qEAAqE;QACrE,yDAAyD;QACzD,IAAI,KAAK,CAAC,OAAO,CAAC,MAAM,CAAC,IAAI,CAAC;YAAE,MAAM,CAAC,IAAI,CAAC,GAAG,MAAM,CAAC,IAAI,CAAC,CAAA;;YACtD,MAAM,CAAC,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,CAAA;IAC9B,CAAC;IACD,OAAO,MAAM,CAAA;AACd,CAAC;AAED;;;;GAIG;AACH,MAAM,UAAU,mBAAmB,CAAC,IAAY,EAAE,OAAe;IAChE,MAAM,KAAK,GAAG,SAAS,CAAC,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,IAAI,IAAI,CAAA;IAE/D,MAAM,OAAO,GAAG,YAAY,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC,CAAC,CAAA;IAC5C,MAAM,QAAQ,GAAG,OAAO,CAAC,CAAC,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC,IAAI,EAAE,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAA;IAC1E,2EAA2E;IAC3E,MAAM,IAAI,GACT,QAAQ,IAAI,aAAa,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,WAAW,EAAE,CAAC,CAAC,CAAC,IAAI,CAAA;IAEzE,IAAI,SAAS,GAAkB,IAAI,CAAA;IACnC,yEAAyE;IACzE,wEAAwE;IACxE,qBAAqB;IACrB,MAAM,UAAU,GAAa,EAAE,CAAA;IAC/B,oFAAoF;IACpF,MAAM,kBAAkB,GAA2B,EAAE,CAAA;IACrD,KAAK,MAAM,KAAK,IAAI,IAAI,CAAC,QAAQ,CAAC,WAAW,CAAC,EAAE,CAAC;QAChD,MAAM,UAAU,GAAG,YAAY,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAA;QACzC,sEAAsE;QACtE,MAAM,GAAG,GAAG,UAAU,CAAC,GAAG,EAAE,WAAW,EAAE,CAAC,KAAK,CAAC,KAAK,CAAC,IAAI,EAAE,CAAA;QAC5D,IAAI,CAAC,UAAU,CAAC,IAAI;YAAE,SAAQ;QAC9B,IAAI,GAAG,CAAC,QAAQ,CAAC,WAAW,CAAC,IAAI,CAAC,SAAS,EAAE,CAAC;YAC7C,SAAS,GAAG,QAAQ,CAAC,UAAU,CAAC,IAAI,EAAE,OAAO,CAAC,CAAA;QAC/C,CAAC;QACD,MAAM,QAAQ,GAAG,UAAU,CAAC,QAAQ,EAAE,IAAI,EAAE,CAAC,WAAW,EAAE,CAAA;QAC1D,IAAI,GAAG,CAAC,QAAQ,CAAC,WAAW,CAAC,IAAI,QAAQ,IAAI,QAAQ,KAAK,WAAW,EAAE,CAAC;YACvE,MAAM,IAAI,GAAG,QAAQ,CAAC,UAAU,CAAC,IAAI,EAAE,OAAO,CAAC,CAAA;YAC/C,MAAM,QAAQ,GAAG,QAAQ,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,CAAA;YAC7C,IAAI,IAAI;gBAAE,UAAU,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;YAC/B,IAAI,IAAI,IAAI,YAAY,CAAC,IAAI,CAAC,QAAQ,CAAC;gBACtC,kBAAkB,CAAC,IAAI,CAAC,GAAG,QAAQ,CAAA;QACrC,CAAC;IACF,CAAC;IAED,IAAI,eAAe,GAAkB,IAAI,CAAA;IACzC,IAAI,aAAa,GAAkB,IAAI,CAAA;IACvC,KAAK,MAAM,KAAK,IAAI,IAAI,CAAC,QAAQ,CAAC,WAAW,CAAC,EAAE,CAAC;QAChD,MAAM,UAAU,GAAG,YAAY,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAA;QACzC,MAAM,GAAG,GAAG,CAAC,UAAU,CAAC,IAAI,IAAI,UAAU,CAAC,QAAQ,IAAI,EAAE,CAAC,CAAC,WAAW,EAAE,CAAA;QACxE,MAAM,OAAO,GAAG,UAAU,CAAC,OAAO,EAAE,IAAI,EAAE,CAAA;QAC1C,IAAI,CAAC,OAAO;YAAE,SAAQ;QACtB,IAAI,GAAG,KAAK,aAAa;YAAE,eAAe,KAAK,SAAS,CAAC,OAAO,CAAC,CAAA;QACjE,IAAI,GAAG,KAAK,gBAAgB;YAAE,aAAa,KAAK,SAAS,CAAC,OAAO,CAAC,CAAA;IACnE,CAAC;IAED,OAAO;QACN,KAAK;QACL,IAAI;QACJ,SAAS;QACT,UAAU,EAAE,eAAe,CAAC,UAAU,CAAC;QACvC,kBAAkB;QAClB,eAAe,EAAE,eAAe,IAAI,aAAa,IAAI,IAAI;QACzD,MAAM,EAAE,WAAW,CAAC,IAAI,CAAC;KACzB,CAAA;AACF,CAAC","sourcesContent":["import {\n\tgetUniqueValues,\n\tisOneOf,\n\tnormalizeWhitespace,\n\tsafeJsonParse\n} from 'hanzio'\nimport { decodeHtmlEntities } from './html-entities.js'\n\nconst TITLE_RE = /<title[^>]*>([\\s\\S]*?)<\\/title>/i\nconst HTML_LANG_RE = /<html\\b[^>]*?\\blang\\s*=\\s*(\"[^\"]*\"|'[^']*'|[^\\s>]+)/i\nconst LINK_TAG_RE = /<link\\b[^>]*>/gi\nconst META_TAG_RE = /<meta\\b[^>]*>/gi\nconst JSON_LD_RE =\n\t/<script\\b[^>]*type\\s*=\\s*[\"']application\\/ld\\+json[\"'][^>]*>([\\s\\S]*?)<\\/script>/gi\nconst ATTRIBUTE_RE = /([a-z_:][-a-z0-9_:.]*)\\s*=\\s*(\"[^\"]*\"|'[^']*'|[^\\s>]+)/gi\nconst HTML_TAG_RE = /<[^>]*>/g\n\nfunction unquote(value: string) {\n\tconst first = value[0]\n\tif ((first === '\"' || first === \"'\") && value.endsWith(first)) {\n\t\treturn value.slice(1, -1)\n\t}\n\treturn value\n}\n\nfunction attributesOf(tag: string) {\n\tconst attributes: Record<string, string> = {}\n\tfor (const match of tag.matchAll(ATTRIBUTE_RE)) {\n\t\tconst name = match[1]?.toLowerCase()\n\t\tconst value = match[2]\n\t\tif (!name || value === undefined) continue\n\t\tattributes[name] = decodeHtmlEntities(unquote(value))\n\t}\n\treturn attributes\n}\n\nfunction cleanText(value: string) {\n\treturn normalizeWhitespace(\n\t\tdecodeHtmlEntities(value.replace(HTML_TAG_RE, ' '))\n\t)\n}\n\nfunction absolute(url: string, baseUrl: string) {\n\ttry {\n\t\tconst resolved = new URL(url, baseUrl)\n\t\tif (!isOneOf(['https:', 'http:'], resolved.protocol)) return null\n\t\treturn resolved.href\n\t} catch {\n\t\treturn null\n\t}\n}\n\nfunction parseJsonLd(html: string) {\n\tconst blocks: unknown[] = []\n\tfor (const match of html.matchAll(JSON_LD_RE)) {\n\t\tconst body = match[1]?.trim()\n\t\tif (!body) continue\n\t\tconst parsed = safeJsonParse(body)\n\t\t// Hand-written JSON-LD with a trailing comma is common; skip the block.\n\t\tif (!parsed.ok) continue\n\t\t// A single script may hold an array of nodes; flatten so the profile\n\t\t// layer can look for `@type` without walking two shapes.\n\t\tif (Array.isArray(parsed.data)) blocks.push(...parsed.data)\n\t\telse blocks.push(parsed.data)\n\t}\n\treturn blocks\n}\n\n/**\n * Everything the pipeline needs from the HTML that markdown cannot carry.\n * Regex-level on purpose: we read five fields plus JSON-LD out of pages that\n * are often invalid HTML, and a parser would cost more than it buys.\n */\nexport function extractHtmlMetadata(html: string, baseUrl: string) {\n\tconst title = cleanText(TITLE_RE.exec(html)?.[1] ?? '') || null\n\n\tconst langRaw = HTML_LANG_RE.exec(html)?.[1]\n\tconst langCode = langRaw ? unquote(langRaw).trim().split(/[-_]/)[0] : null\n\t// ISO 639-1 only: `da-DK` → `da`, and a bogus `lang=\"Dansk\"` becomes null.\n\tconst lang =\n\t\tlangCode && /^[a-z]{2}$/i.test(langCode) ? langCode.toLowerCase() : null\n\n\tlet canonical: string | null = null\n\t// `<link rel=\"alternate\" hreflang>`: the site's own declaration of which\n\t// URLs are translations of this page. `x-default` is a language picker,\n\t// not a translation.\n\tconst alternates: string[] = []\n\t/** Each alternate's language (`da` of `da-DK`); what lets a crawl skip a mirror. */\n\tconst alternateLanguages: Record<string, string> = {}\n\tfor (const match of html.matchAll(LINK_TAG_RE)) {\n\t\tconst attributes = attributesOf(match[0])\n\t\t// `rel` is a token list: WordPress emits `rel=\"shortlink canonical\"`.\n\t\tconst rel = attributes.rel?.toLowerCase().split(/\\s+/) ?? []\n\t\tif (!attributes.href) continue\n\t\tif (rel.includes('canonical') && !canonical) {\n\t\t\tcanonical = absolute(attributes.href, baseUrl)\n\t\t}\n\t\tconst hreflang = attributes.hreflang?.trim().toLowerCase()\n\t\tif (rel.includes('alternate') && hreflang && hreflang !== 'x-default') {\n\t\t\tconst href = absolute(attributes.href, baseUrl)\n\t\t\tconst language = hreflang.split('-')[0] ?? ''\n\t\t\tif (href) alternates.push(href)\n\t\t\tif (href && /^[a-z]{2}$/.test(language))\n\t\t\t\talternateLanguages[href] = language\n\t\t}\n\t}\n\n\tlet metaDescription: string | null = null\n\tlet ogDescription: string | null = null\n\tfor (const match of html.matchAll(META_TAG_RE)) {\n\t\tconst attributes = attributesOf(match[0])\n\t\tconst key = (attributes.name ?? attributes.property ?? '').toLowerCase()\n\t\tconst content = attributes.content?.trim()\n\t\tif (!content) continue\n\t\tif (key === 'description') metaDescription ??= cleanText(content)\n\t\tif (key === 'og:description') ogDescription ??= cleanText(content)\n\t}\n\n\treturn {\n\t\ttitle,\n\t\tlang,\n\t\tcanonical,\n\t\talternates: getUniqueValues(alternates),\n\t\talternateLanguages,\n\t\tmetaDescription: metaDescription || ogDescription || null,\n\t\tjsonLd: parseJsonLd(html)\n\t}\n}\n"]}

package/dist/index.d.ts

@@ -1,14 +1,14 @@
 export { CloudflareRateLimitedError, type CloudflareCredentials, type ConvertFileInput } from './client.js';
 export { type ContentKind, classifyContentType, convertibleDocumentType } from './content-type.js';
 export { CRAWL_SNAPSHOT_NAME_PATTERN, type CrawlSnapshotInput, type CrawlSnapshotOptions, currentCrawlSnapshot, throughCrawlSnapshot, withCrawlSnapshot } from './crawl-snapshot.js';
-export { createWebsiteToMarkdown, type FetchedPage, type FetchOutcome, type FetchPageInput, type FetchSource, MAX_MARKDOWN_CHARS, type PageBody, type PdfParser, type WebsiteToMarkdown, type WebsiteToMarkdownOptions } from './fetch-page.js';
+export { type CaptureIssue, createWebsiteToMarkdown, type FetchedPage, type FetchOutcome, type FetchPageInput, type FetchSource, MAX_MARKDOWN_CHARS, type PageBody, type PdfParser, type WebsiteToMarkdown, type WebsiteToMarkdownOptions } from './fetch-page.js';
 export { decodeHtmlEntities } from './html-entities.js';
 export { extractHtmlMetadata } from './html-metadata.js';
 export { type ConcurrencyLimit, type ConcurrencyLimitOptions, type CreateConcurrencyLimit, URL_TO_MARKDOWN_PAGE_CONCURRENCY } from './limit.js';
 export type { LogAttributes, Logger } from './logger.js';
-export { htmlImages, htmlLinks, markdownRefs, stripDocumentPreamble, stripFrontMatter, titleFromMarkdown } from './markdown-refs.js';
+export { cleanConvertedMarkdown, htmlImages, htmlLinks, markdownRefs, stripDocumentPreamble, stripFrontMatter, titleFromMarkdown } from './markdown-refs.js';
 export { createMemoryConcurrencyLimit } from './memory-limit.js';
-export { isBlockedAddress, isHostAllowed, isLocalhostHostname } from './net-guard.js';
+export { type Fetch, type GuardedFetchOptions, guardedFetch, type HostGuardOptions, HostNotAllowedError, isBlockedAddress, isHostAllowed, isLocalhostHostname } from './net-guard.js';
 export { currentOperationSignal, failCurrentOperation, withOperationSignal } from './operation-signal.js';
 export { BROWSER_USER_AGENT } from './user-agent.js';
 //# sourceMappingURL=index.d.ts.map

package/dist/index.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,OAAO,EACN,0BAA0B,EAC1B,KAAK,qBAAqB,EAC1B,KAAK,gBAAgB,EACrB,MAAM,aAAa,CAAA;AACpB,OAAO,EACN,KAAK,WAAW,EAChB,mBAAmB,EACnB,uBAAuB,EACvB,MAAM,mBAAmB,CAAA;AAC1B,OAAO,EACN,2BAA2B,EAC3B,KAAK,kBAAkB,EACvB,KAAK,oBAAoB,EACzB,oBAAoB,EACpB,oBAAoB,EACpB,iBAAiB,EACjB,MAAM,qBAAqB,CAAA;AAC5B,OAAO,EACN,uBAAuB,EACvB,KAAK,WAAW,EAChB,KAAK,YAAY,EACjB,KAAK,cAAc,EACnB,KAAK,WAAW,EAChB,kBAAkB,EAClB,KAAK,QAAQ,EACb,KAAK,SAAS,EACd,KAAK,iBAAiB,EACtB,KAAK,wBAAwB,EAC7B,MAAM,iBAAiB,CAAA;AACxB,OAAO,EAAE,kBAAkB,EAAE,MAAM,oBAAoB,CAAA;AACvD,OAAO,EAAE,mBAAmB,EAAE,MAAM,oBAAoB,CAAA;AACxD,OAAO,EACN,KAAK,gBAAgB,EACrB,KAAK,uBAAuB,EAC5B,KAAK,sBAAsB,EAC3B,gCAAgC,EAChC,MAAM,YAAY,CAAA;AACnB,YAAY,EAAE,aAAa,EAAE,MAAM,EAAE,MAAM,aAAa,CAAA;AACxD,OAAO,EACN,UAAU,EACV,SAAS,EACT,YAAY,EACZ,qBAAqB,EACrB,gBAAgB,EAChB,iBAAiB,EACjB,MAAM,oBAAoB,CAAA;AAC3B,OAAO,EAAE,4BAA4B,EAAE,MAAM,mBAAmB,CAAA;AAChE,OAAO,EACN,gBAAgB,EAChB,aAAa,EACb,mBAAmB,EACnB,MAAM,gBAAgB,CAAA;AACvB,OAAO,EACN,sBAAsB,EACtB,oBAAoB,EACpB,mBAAmB,EACnB,MAAM,uBAAuB,CAAA;AAC9B,OAAO,EAAE,kBAAkB,EAAE,MAAM,iBAAiB,CAAA"}
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,OAAO,EACN,0BAA0B,EAC1B,KAAK,qBAAqB,EAC1B,KAAK,gBAAgB,EACrB,MAAM,aAAa,CAAA;AACpB,OAAO,EACN,KAAK,WAAW,EAChB,mBAAmB,EACnB,uBAAuB,EACvB,MAAM,mBAAmB,CAAA;AAC1B,OAAO,EACN,2BAA2B,EAC3B,KAAK,kBAAkB,EACvB,KAAK,oBAAoB,EACzB,oBAAoB,EACpB,oBAAoB,EACpB,iBAAiB,EACjB,MAAM,qBAAqB,CAAA;AAC5B,OAAO,EACN,KAAK,YAAY,EACjB,uBAAuB,EACvB,KAAK,WAAW,EAChB,KAAK,YAAY,EACjB,KAAK,cAAc,EACnB,KAAK,WAAW,EAChB,kBAAkB,EAClB,KAAK,QAAQ,EACb,KAAK,SAAS,EACd,KAAK,iBAAiB,EACtB,KAAK,wBAAwB,EAC7B,MAAM,iBAAiB,CAAA;AACxB,OAAO,EAAE,kBAAkB,EAAE,MAAM,oBAAoB,CAAA;AACvD,OAAO,EAAE,mBAAmB,EAAE,MAAM,oBAAoB,CAAA;AACxD,OAAO,EACN,KAAK,gBAAgB,EACrB,KAAK,uBAAuB,EAC5B,KAAK,sBAAsB,EAC3B,gCAAgC,EAChC,MAAM,YAAY,CAAA;AACnB,YAAY,EAAE,aAAa,EAAE,MAAM,EAAE,MAAM,aAAa,CAAA;AACxD,OAAO,EACN,sBAAsB,EACtB,UAAU,EACV,SAAS,EACT,YAAY,EACZ,qBAAqB,EACrB,gBAAgB,EAChB,iBAAiB,EACjB,MAAM,oBAAoB,CAAA;AAC3B,OAAO,EAAE,4BAA4B,EAAE,MAAM,mBAAmB,CAAA;AAChE,OAAO,EACN,KAAK,KAAK,EACV,KAAK,mBAAmB,EACxB,YAAY,EACZ,KAAK,gBAAgB,EACrB,mBAAmB,EACnB,gBAAgB,EAChB,aAAa,EACb,mBAAmB,EACnB,MAAM,gBAAgB,CAAA;AACvB,OAAO,EACN,sBAAsB,EACtB,oBAAoB,EACpB,mBAAmB,EACnB,MAAM,uBAAuB,CAAA;AAC9B,OAAO,EAAE,kBAAkB,EAAE,MAAM,iBAAiB,CAAA"}

package/dist/index.js

@@ -5,9 +5,9 @@ export { createWebsiteToMarkdown, MAX_MARKDOWN_CHARS } from './fetch-page.js';
 export { decodeHtmlEntities } from './html-entities.js';
 export { extractHtmlMetadata } from './html-metadata.js';
 export { URL_TO_MARKDOWN_PAGE_CONCURRENCY } from './limit.js';
-export { htmlImages, htmlLinks, markdownRefs, stripDocumentPreamble, stripFrontMatter, titleFromMarkdown } from './markdown-refs.js';
+export { cleanConvertedMarkdown, htmlImages, htmlLinks, markdownRefs, stripDocumentPreamble, stripFrontMatter, titleFromMarkdown } from './markdown-refs.js';
 export { createMemoryConcurrencyLimit } from './memory-limit.js';
-export { isBlockedAddress, isHostAllowed, isLocalhostHostname } from './net-guard.js';
+export { guardedFetch, HostNotAllowedError, isBlockedAddress, isHostAllowed, isLocalhostHostname } from './net-guard.js';
 export { currentOperationSignal, failCurrentOperation, withOperationSignal } from './operation-signal.js';
 export { BROWSER_USER_AGENT } from './user-agent.js';
 //# sourceMappingURL=index.js.map

package/dist/index.js.map

@@ -1 +1 @@
-{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,OAAO,EACN,0BAA0B,EAG1B,MAAM,aAAa,CAAA;AACpB,OAAO,EAEN,mBAAmB,EACnB,uBAAuB,EACvB,MAAM,mBAAmB,CAAA;AAC1B,OAAO,EACN,2BAA2B,EAG3B,oBAAoB,EACpB,oBAAoB,EACpB,iBAAiB,EACjB,MAAM,qBAAqB,CAAA;AAC5B,OAAO,EACN,uBAAuB,EAKvB,kBAAkB,EAKlB,MAAM,iBAAiB,CAAA;AACxB,OAAO,EAAE,kBAAkB,EAAE,MAAM,oBAAoB,CAAA;AACvD,OAAO,EAAE,mBAAmB,EAAE,MAAM,oBAAoB,CAAA;AACxD,OAAO,EAIN,gCAAgC,EAChC,MAAM,YAAY,CAAA;AAEnB,OAAO,EACN,UAAU,EACV,SAAS,EACT,YAAY,EACZ,qBAAqB,EACrB,gBAAgB,EAChB,iBAAiB,EACjB,MAAM,oBAAoB,CAAA;AAC3B,OAAO,EAAE,4BAA4B,EAAE,MAAM,mBAAmB,CAAA;AAChE,OAAO,EACN,gBAAgB,EAChB,aAAa,EACb,mBAAmB,EACnB,MAAM,gBAAgB,CAAA;AACvB,OAAO,EACN,sBAAsB,EACtB,oBAAoB,EACpB,mBAAmB,EACnB,MAAM,uBAAuB,CAAA;AAC9B,OAAO,EAAE,kBAAkB,EAAE,MAAM,iBAAiB,CAAA","sourcesContent":["export {\n\tCloudflareRateLimitedError,\n\ttype CloudflareCredentials,\n\ttype ConvertFileInput\n} from './client.js'\nexport {\n\ttype ContentKind,\n\tclassifyContentType,\n\tconvertibleDocumentType\n} from './content-type.js'\nexport {\n\tCRAWL_SNAPSHOT_NAME_PATTERN,\n\ttype CrawlSnapshotInput,\n\ttype CrawlSnapshotOptions,\n\tcurrentCrawlSnapshot,\n\tthroughCrawlSnapshot,\n\twithCrawlSnapshot\n} from './crawl-snapshot.js'\nexport {\n\tcreateWebsiteToMarkdown,\n\ttype FetchedPage,\n\ttype FetchOutcome,\n\ttype FetchPageInput,\n\ttype FetchSource,\n\tMAX_MARKDOWN_CHARS,\n\ttype PageBody,\n\ttype PdfParser,\n\ttype WebsiteToMarkdown,\n\ttype WebsiteToMarkdownOptions\n} from './fetch-page.js'\nexport { decodeHtmlEntities } from './html-entities.js'\nexport { extractHtmlMetadata } from './html-metadata.js'\nexport {\n\ttype ConcurrencyLimit,\n\ttype ConcurrencyLimitOptions,\n\ttype CreateConcurrencyLimit,\n\tURL_TO_MARKDOWN_PAGE_CONCURRENCY\n} from './limit.js'\nexport type { LogAttributes, Logger } from './logger.js'\nexport {\n\thtmlImages,\n\thtmlLinks,\n\tmarkdownRefs,\n\tstripDocumentPreamble,\n\tstripFrontMatter,\n\ttitleFromMarkdown\n} from './markdown-refs.js'\nexport { createMemoryConcurrencyLimit } from './memory-limit.js'\nexport {\n\tisBlockedAddress,\n\tisHostAllowed,\n\tisLocalhostHostname\n} from './net-guard.js'\nexport {\n\tcurrentOperationSignal,\n\tfailCurrentOperation,\n\twithOperationSignal\n} from './operation-signal.js'\nexport { BROWSER_USER_AGENT } from './user-agent.js'\n"]}
+{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,OAAO,EACN,0BAA0B,EAG1B,MAAM,aAAa,CAAA;AACpB,OAAO,EAEN,mBAAmB,EACnB,uBAAuB,EACvB,MAAM,mBAAmB,CAAA;AAC1B,OAAO,EACN,2BAA2B,EAG3B,oBAAoB,EACpB,oBAAoB,EACpB,iBAAiB,EACjB,MAAM,qBAAqB,CAAA;AAC5B,OAAO,EAEN,uBAAuB,EAKvB,kBAAkB,EAKlB,MAAM,iBAAiB,CAAA;AACxB,OAAO,EAAE,kBAAkB,EAAE,MAAM,oBAAoB,CAAA;AACvD,OAAO,EAAE,mBAAmB,EAAE,MAAM,oBAAoB,CAAA;AACxD,OAAO,EAIN,gCAAgC,EAChC,MAAM,YAAY,CAAA;AAEnB,OAAO,EACN,sBAAsB,EACtB,UAAU,EACV,SAAS,EACT,YAAY,EACZ,qBAAqB,EACrB,gBAAgB,EAChB,iBAAiB,EACjB,MAAM,oBAAoB,CAAA;AAC3B,OAAO,EAAE,4BAA4B,EAAE,MAAM,mBAAmB,CAAA;AAChE,OAAO,EAGN,YAAY,EAEZ,mBAAmB,EACnB,gBAAgB,EAChB,aAAa,EACb,mBAAmB,EACnB,MAAM,gBAAgB,CAAA;AACvB,OAAO,EACN,sBAAsB,EACtB,oBAAoB,EACpB,mBAAmB,EACnB,MAAM,uBAAuB,CAAA;AAC9B,OAAO,EAAE,kBAAkB,EAAE,MAAM,iBAAiB,CAAA","sourcesContent":["export {\n\tCloudflareRateLimitedError,\n\ttype CloudflareCredentials,\n\ttype ConvertFileInput\n} from './client.js'\nexport {\n\ttype ContentKind,\n\tclassifyContentType,\n\tconvertibleDocumentType\n} from './content-type.js'\nexport {\n\tCRAWL_SNAPSHOT_NAME_PATTERN,\n\ttype CrawlSnapshotInput,\n\ttype CrawlSnapshotOptions,\n\tcurrentCrawlSnapshot,\n\tthroughCrawlSnapshot,\n\twithCrawlSnapshot\n} from './crawl-snapshot.js'\nexport {\n\ttype CaptureIssue,\n\tcreateWebsiteToMarkdown,\n\ttype FetchedPage,\n\ttype FetchOutcome,\n\ttype FetchPageInput,\n\ttype FetchSource,\n\tMAX_MARKDOWN_CHARS,\n\ttype PageBody,\n\ttype PdfParser,\n\ttype WebsiteToMarkdown,\n\ttype WebsiteToMarkdownOptions\n} from './fetch-page.js'\nexport { decodeHtmlEntities } from './html-entities.js'\nexport { extractHtmlMetadata } from './html-metadata.js'\nexport {\n\ttype ConcurrencyLimit,\n\ttype ConcurrencyLimitOptions,\n\ttype CreateConcurrencyLimit,\n\tURL_TO_MARKDOWN_PAGE_CONCURRENCY\n} from './limit.js'\nexport type { LogAttributes, Logger } from './logger.js'\nexport {\n\tcleanConvertedMarkdown,\n\thtmlImages,\n\thtmlLinks,\n\tmarkdownRefs,\n\tstripDocumentPreamble,\n\tstripFrontMatter,\n\ttitleFromMarkdown\n} from './markdown-refs.js'\nexport { createMemoryConcurrencyLimit } from './memory-limit.js'\nexport {\n\ttype Fetch,\n\ttype GuardedFetchOptions,\n\tguardedFetch,\n\ttype HostGuardOptions,\n\tHostNotAllowedError,\n\tisBlockedAddress,\n\tisHostAllowed,\n\tisLocalhostHostname\n} from './net-guard.js'\nexport {\n\tcurrentOperationSignal,\n\tfailCurrentOperation,\n\twithOperationSignal\n} from './operation-signal.js'\nexport { BROWSER_USER_AGENT } from './user-agent.js'\n"]}

package/dist/limit.d.ts

@@ -65,7 +65,7 @@ export declare function createCloudflareLimits(createLimit: CreateConcurrencyLim
 export declare const URL_TO_MARKDOWN_HOST_MAX_CONCURRENCY = 16;
 /**
  * A host that answered 429 is read this narrowly from then on. At 16 in
- * flight Shopify refused 117 of Good Habit's 244 reads even after each had
+ * flight Shopify refused 117 of one store's 244 reads even after each had
  * waited out its pause: the pause only delayed the next burst. Calibration
  * knobs - set them from the crawl log's `rate_limited`, not by theory.
  */

package/dist/limit.js

@@ -54,7 +54,7 @@ export function createCloudflareLimits(createLimit) {
 export const URL_TO_MARKDOWN_HOST_MAX_CONCURRENCY = 16;
 /**
  * A host that answered 429 is read this narrowly from then on. At 16 in
- * flight Shopify refused 117 of Good Habit's 244 reads even after each had
+ * flight Shopify refused 117 of one store's 244 reads even after each had
  * waited out its pause: the pause only delayed the next burst. Calibration
  * knobs - set them from the crawl log's `rate_limited`, not by theory.
  */